sensordatastore.cpp 10.1 KB
Newer Older
1
2
3
4
5
6
7
/*
 * sensordatastore.cpp
 *
 *  Created on: Jul 24, 2013
 *      Author: Axel Auweter
 */

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
/**
 * @mainpage
 * The DCDBLib library is a dynamic runtime library providing
 * functions to initialize and access the DCDB data store. It
 * is being used by the CollectAgent to handle insertion of
 * data and can be used by tools responsible for data analysis.
 *
 * Its main class is the SensorDataStore class which provides
 * functions to connect to the data store, initialize an empty
 * data base and to (TODO) retrieve data.
 *
 * For its internal handling, SensorDataStore relies on the
 * SensorDataStoreImpl class (which hides all private member
 * functions belonging to the SensorDataStore class from the
 * header that is used by programmers who link against this
 * library). Raw database functionality is abstracted into the
 * CassandraBackend class (to easy switching to other
 * key-value style databases in the future).
 *
27
 * To use the library in your client application, simply
28
29
30
31
 * include the sensordatastore.h header file and initialize
 * an object of the SensorDataStore class.
 */

Axel Auweter's avatar
Axel Auweter committed
32
33
#include <string>
#include <iostream>
34
35
#include <cstdint>
#include <cinttypes>
Axel Auweter's avatar
Axel Auweter committed
36

37
#include "cassandra.h"
Axel Auweter's avatar
Axel Auweter committed
38

39
#include "sensordatastore.h"
40
#include "sensordatastore_internal.h"
41
42
43
44
#include "connection.h"
#include "dcdbglobals.h"

#include "dcdbendian.h"
45

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
/**
 * @details
 * Since we want high-performance inserts, we prepare the
 * insert CQL query in advance and only bind it on the actual
 * insert.
 */
void SensorDataStoreImpl::prepareInsert(uint64_t ttl)
{
  CassError rc = CASS_OK;
  CassFuture* future = NULL;
  const char* query;

  /*
   * Free the old prepared if necessary.
   */
  if (preparedInsert) {
      cass_prepared_free(preparedInsert);
  }

  char *queryBuf = NULL;
  if (ttl == 0) {
      query = "INSERT INTO dcdb.sensordata (sid, ts, value) VALUES (?, ?, ?);";
  }
  else {
      queryBuf = (char*)malloc(256);
      snprintf(queryBuf, 256, "INSERT INTO dcdb.sensordata (sid, ts, value) VALUES (?, ?, ?) USING TTL %" PRIu64 " ;", ttl);
      query = queryBuf;
  }

  future = cass_session_prepare(session, query);
  cass_future_wait(future);

  rc = cass_future_error_code(future);
  if (rc != CASS_OK) {
      connection->printError(future);
  } else {
      preparedInsert = cass_future_get_prepared(future);
  }

  cass_future_free(future);
  if (queryBuf) {
      free(queryBuf);
  }
}

91
92
93
94
95
96
97
98
99
100
101
102
103
104
/**
 * @details
 * To insert a sensor reading, the Rsvd field of the SensorId must
 * be filled with a time component that ensures that the maximum
 * number of 2^32 columns per key is not exceeded while still
 * allowing relatively easy retrieval of data.
 *
 * We achieve this by using a "week-stamp" (i.e. number of weeks
 * since Unix epoch) within the Rsvd field of the SensorId before
 * calling the Cassandra Backend to do the raw insert.
 *
 * Applications should not call this function directly, but
 * use the insert function provided by the SensorDataStore class.
 */
105
106
void SensorDataStoreImpl::insert(SensorId* sid, uint64_t ts, uint64_t value)
{
107
#if 0
108
109
110
  std::cout << "Inserting@SensorDataStoreImpl (" << sid->raw[0] << " " << sid->raw[1] << ", " << ts << ", " << value << ")" << std::endl;
#endif

111
  /* Calculate and insert week number */
112
  uint16_t week = ts / 604800000000000;
Axel Auweter's avatar
Axel Auweter committed
113
  sid->setRsvd(week);
114
115

  /* Insert into Cassandra */
Axel Auweter's avatar
Axel Auweter committed
116
  std::string key = sid->serialize();
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

  CassError rc = CASS_OK;
  CassStatement* statement = NULL;
  CassFuture *future = NULL;

  statement = cass_prepared_bind(preparedInsert);

  cass_statement_bind_bytes_by_name(statement, "sid", (cass_byte_t*)(key.c_str()), 16);
  cass_statement_bind_int64_by_name(statement, "ts", ts);
  cass_statement_bind_int64_by_name(statement, "value", value);

  future = cass_session_execute(session, statement);
  cass_future_wait(future);

  rc = cass_future_error_code(future);
  if (rc != CASS_OK) {
      connection->printError(future);
  }

  cass_future_free(future);
  cass_statement_free(statement);
138
139
}

140
141
/**
 * @details
142
143
 * This function updates the prepared statement for inserts
 * with the new TTL value.
144
 */
145
void SensorDataStoreImpl::setTTL(uint64_t ttl)
146
{
147
  prepareInsert(ttl);
148
149
}

Axel Auweter's avatar
Axel Auweter committed
150
151
152
153
/**
 * @details
 * This function issues a regular query to the data store
 * and creates a SensorDataStoreReading object for each
154
 * entry which is stored in the result list.
Axel Auweter's avatar
Axel Auweter committed
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 */
void SensorDataStoreImpl::query(std::list<SensorDataStoreReading>& result, SensorId sid, DCDBTimeStamp start, DCDBTimeStamp end)
{
  CassError rc = CASS_OK;
  CassStatement* statement = NULL;
  CassFuture *future = NULL;
  const CassPrepared* prepared = nullptr;
  const char* query = "SELECT * FROM " KEYSPACE_NAME "." CF_SENSORDATA " WHERE sid = ? AND ts >= ? AND ts <= ? ;";

  future = cass_session_prepare(session, query);
  cass_future_wait(future);

  rc = cass_future_error_code(future);
  if (rc != CASS_OK) {
    connection->printError(future);
    cass_future_free(future);
    return;
  }

  prepared = cass_future_get_prepared(future);
  cass_future_free(future);

  std::string key = sid.serialize();

179
180
  std::cout << "Query: " << query << std::endl << "sid: " << key << " ts1: " << start.getRaw() << " ts2: " << end.getRaw() << std::endl;

Axel Auweter's avatar
Axel Auweter committed
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
  statement = cass_prepared_bind(prepared);
  cass_statement_bind_bytes(statement, 0, (const cass_byte_t*)(key.c_str()), 16);
  cass_statement_bind_int64(statement, 1, start.getRaw());
  cass_statement_bind_int64(statement, 2, end.getRaw());

  future = cass_session_execute(session, statement);
  cass_future_wait(future);

  if (cass_future_error_code(future) == CASS_OK) {
      const CassResult* cresult = cass_future_get_result(future);
      CassIterator* rows = cass_iterator_from_result(cresult);

      while (cass_iterator_next(rows)) {
          const CassRow* row = cass_iterator_get_row(rows);

          SensorDataStoreReading entry;

          cass_int64_t ts, value;
          cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
          cass_value_get_int64(cass_row_get_column_by_name(row, "value"), &value);

          entry.sensorId = sid;
          entry.timeStamp = DCDBTimeStamp((uint64_t)ts);
          entry.value = (uint64_t)value;

          result.push_back(entry);
#if 0
          if (localtime) {
              t.convertToLocal();
          }
          if (raw) {
              std::cout << sensorName << "," << std::dec << t.getRaw() << "," << std::dec << value << std::endl;
          }
          else {
              std::cout << sensorName << "," << t.getString() << "," << std::dec << value << std::endl;
          }
#endif
      }
      cass_iterator_free(rows);
      cass_result_free(cresult);
  }

  cass_statement_free(statement);
  cass_future_free(future);
  cass_prepared_free(prepared);
}

228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
/**
 * @details
 * This function generates an integrated value of the time series
 * by first querying for the result set list using query() and then
 * summing up the result.
 */
void SensorDataStoreImpl::querySum(int64_t& result, SensorId sid, DCDBTimeStamp start, DCDBTimeStamp end)
{
  std::list<SensorDataStoreReading> queryResult;

  /* Issue a standard query */
  query(queryResult, sid, start, end);

  /* Integrate the result */
  result = 0;

  SensorDataStoreReading prev;
  for (std::list<SensorDataStoreReading>::iterator it = queryResult.begin(); it != queryResult.end(); it++) {
      if (!(it == queryResult.begin())) {
          SensorDataStoreReading cur = *it;

          /* Calculate average between two readings */
          int64_t avg = (cur.value + prev.value) / 2;

          /* Calculate time difference */
          uint64_t dt = cur.timeStamp.getRaw() - prev.timeStamp.getRaw();

          /* Sum up (with lousy attempt to keep it numerically stable - should probably use double instead) */
          if (dt > 10000000000) {
              /* dt > 10s => convert dt to s first */
              dt /= 1000000000;
              result += avg * dt;
          }
          else {
              /* dt < 10s => multiply first */
              avg *= dt;
              result += avg / 1000000000;
          }
      }
      prev = *it;
  }
}

271
272
/**
 * @details
273
274
275
 * This constructor sets the internal connection variable to
 * the externally provided DCDBConnection object and also
 * retrieves the CassSession pointer of the connection.
276
 */
277
SensorDataStoreImpl::SensorDataStoreImpl(DCDBConnection* conn)
278
{
279
280
  connection = conn;
  session = connection->getSessionHandle();
Axel Auweter's avatar
Axel Auweter committed
281
282
283

  preparedInsert = nullptr;
  prepareInsert(0);
284
285
}

286
287
/**
 * @details
288
 * Due to the simplicity of the class, the destructor is left empty.
289
 */
290
SensorDataStoreImpl::~SensorDataStoreImpl()
291
{
292
  connection = nullptr;
Axel Auweter's avatar
Axel Auweter committed
293
294
295
296
  session = nullptr;
  if (preparedInsert) {
      cass_prepared_free(preparedInsert);
  }
297
298
}

299
300
301
302
303
304
/**
 * @details
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
 */
305
306
307
308
309
void SensorDataStore::insert(SensorId* sid, uint64_t ts, uint64_t value)
{
    impl->insert(sid, ts, value);
}

310
311
/**
 * @details
312
313
314
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
315
 */
316
void SensorDataStore::setTTL(uint64_t ttl)
317
{
318
    impl->setTTL(ttl);
319
320
}

Axel Auweter's avatar
Axel Auweter committed
321
322
323
324
325
326
327
328
329
330
331
/**
 * @details
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
 */
void SensorDataStore::query(std::list<SensorDataStoreReading>& result, SensorId sid, DCDBTimeStamp start, DCDBTimeStamp end)
{
    impl->query(result, sid, start, end);
}

332
333
334
335
336
337
338
339
340
341
342
/**
 * @details
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
 */
void SensorDataStore::querySum(int64_t& result, SensorId sid, DCDBTimeStamp start, DCDBTimeStamp end)
{
    impl->querySum(result, sid, start, end);
}

343
344
/**
 * @details
345
346
 * This constructor allocates the implementation class which
 * holds the actual implementation of the class functionality.
347
 */
348
SensorDataStore::SensorDataStore(DCDBConnection* conn)
349
{
350
    impl = new SensorDataStoreImpl(conn);
351
352
}

353
354
355
356
357
/**
 * @details
 * The SensorDataStore desctructor deallocates the
 * SensorDataStoreImpl and CassandraBackend objects.
 */
358
359
SensorDataStore::~SensorDataStore()
{
360
361
362
  /* Clean up... */
  if (impl)
    delete impl;
363
}