sensordatastore.cpp 13.3 KB
Newer Older
1
2
3
4
5
6
7
/*
 * sensordatastore.cpp
 *
 *  Created on: Jul 24, 2013
 *      Author: Axel Auweter
 */

8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
/**
 * @mainpage
 * The DCDBLib library is a dynamic runtime library providing
 * functions to initialize and access the DCDB data store. It
 * is being used by the CollectAgent to handle insertion of
 * data and can be used by tools responsible for data analysis.
 *
 * Its main class is the SensorDataStore class which provides
 * functions to connect to the data store, initialize an empty
 * data base and to (TODO) retrieve data.
 *
 * For its internal handling, SensorDataStore relies on the
 * SensorDataStoreImpl class (which hides all private member
 * functions belonging to the SensorDataStore class from the
 * header that is used by programmers who link against this
 * library). Raw database functionality is abstracted into the
 * CassandraBackend class (to easy switching to other
 * key-value style databases in the future).
 *
27
 * To use the library in your client application, simply
28
29
30
31
 * include the sensordatastore.h header file and initialize
 * an object of the SensorDataStore class.
 */

Axel Auweter's avatar
Axel Auweter committed
32
33
#include <string>
#include <iostream>
34
35
#include <cstdint>
#include <cinttypes>
Axel Auweter's avatar
Axel Auweter committed
36

37
#include "cassandra.h"
Axel Auweter's avatar
Axel Auweter committed
38

39
#include "sensordatastore.h"
40
#include "sensordatastore_internal.h"
41
42
43
44
#include "connection.h"
#include "dcdbglobals.h"

#include "dcdbendian.h"
45

46
47
using namespace DCDB;

48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
/**
 * @details
 * Since we want high-performance inserts, we prepare the
 * insert CQL query in advance and only bind it on the actual
 * insert.
 */
void SensorDataStoreImpl::prepareInsert(uint64_t ttl)
{
  CassError rc = CASS_OK;
  CassFuture* future = NULL;
  const char* query;

  /*
   * Free the old prepared if necessary.
   */
  if (preparedInsert) {
      cass_prepared_free(preparedInsert);
  }

  char *queryBuf = NULL;
  if (ttl == 0) {
      query = "INSERT INTO dcdb.sensordata (sid, ts, value) VALUES (?, ?, ?);";
  }
  else {
      queryBuf = (char*)malloc(256);
      snprintf(queryBuf, 256, "INSERT INTO dcdb.sensordata (sid, ts, value) VALUES (?, ?, ?) USING TTL %" PRIu64 " ;", ttl);
      query = queryBuf;
  }

  future = cass_session_prepare(session, query);
  cass_future_wait(future);

  rc = cass_future_error_code(future);
  if (rc != CASS_OK) {
      connection->printError(future);
  } else {
      preparedInsert = cass_future_get_prepared(future);
  }

  cass_future_free(future);
  if (queryBuf) {
      free(queryBuf);
  }
}

93
94
95
96
97
98
99
100
101
102
103
104
105
106
/**
 * @details
 * To insert a sensor reading, the Rsvd field of the SensorId must
 * be filled with a time component that ensures that the maximum
 * number of 2^32 columns per key is not exceeded while still
 * allowing relatively easy retrieval of data.
 *
 * We achieve this by using a "week-stamp" (i.e. number of weeks
 * since Unix epoch) within the Rsvd field of the SensorId before
 * calling the Cassandra Backend to do the raw insert.
 *
 * Applications should not call this function directly, but
 * use the insert function provided by the SensorDataStore class.
 */
107
void SensorDataStoreImpl::insert(SensorId* sid, uint64_t ts, int64_t value)
108
{
109
#if 0
110
111
112
  std::cout << "Inserting@SensorDataStoreImpl (" << sid->raw[0] << " " << sid->raw[1] << ", " << ts << ", " << value << ")" << std::endl;
#endif

113
  /* Calculate and insert week number */
114
  uint16_t week = ts / 604800000000000;
Axel Auweter's avatar
Axel Auweter committed
115
  sid->setRsvd(week);
116
117

  /* Insert into Cassandra */
Axel Auweter's avatar
Axel Auweter committed
118
  std::string key = sid->serialize();
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

  CassError rc = CASS_OK;
  CassStatement* statement = NULL;
  CassFuture *future = NULL;

  statement = cass_prepared_bind(preparedInsert);

  cass_statement_bind_bytes_by_name(statement, "sid", (cass_byte_t*)(key.c_str()), 16);
  cass_statement_bind_int64_by_name(statement, "ts", ts);
  cass_statement_bind_int64_by_name(statement, "value", value);

  future = cass_session_execute(session, statement);
  cass_future_wait(future);

  rc = cass_future_error_code(future);
  if (rc != CASS_OK) {
      connection->printError(future);
  }

  cass_future_free(future);
  cass_statement_free(statement);
140
141
}

142
143
/**
 * @details
144
145
 * This function updates the prepared statement for inserts
 * with the new TTL value.
146
 */
147
void SensorDataStoreImpl::setTTL(uint64_t ttl)
148
{
149
  prepareInsert(ttl);
150
151
}

Axel Auweter's avatar
Axel Auweter committed
152
153
154
155
/**
 * @details
 * This function issues a regular query to the data store
 * and creates a SensorDataStoreReading object for each
156
 * entry which is stored in the result list.
Axel Auweter's avatar
Axel Auweter committed
157
 */
158
void SensorDataStoreImpl::query(std::list<SensorDataStoreReading>& result, SensorId& sid, TimeStamp& start, TimeStamp& end)
Axel Auweter's avatar
Axel Auweter committed
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
{
  CassError rc = CASS_OK;
  CassStatement* statement = NULL;
  CassFuture *future = NULL;
  const CassPrepared* prepared = nullptr;
  const char* query = "SELECT * FROM " KEYSPACE_NAME "." CF_SENSORDATA " WHERE sid = ? AND ts >= ? AND ts <= ? ;";

  future = cass_session_prepare(session, query);
  cass_future_wait(future);

  rc = cass_future_error_code(future);
  if (rc != CASS_OK) {
    connection->printError(future);
    cass_future_free(future);
    return;
  }

  prepared = cass_future_get_prepared(future);
  cass_future_free(future);

  std::string key = sid.serialize();

181
#if 0
182
  std::cout << "Query: " << query << std::endl << "sid: " << key << " ts1: " << start.getRaw() << " ts2: " << end.getRaw() << std::endl;
183
#endif
184

Axel Auweter's avatar
Axel Auweter committed
185
186
187
188
189
190
191
192
193
194
195
196
  statement = cass_prepared_bind(prepared);
  cass_statement_bind_bytes(statement, 0, (const cass_byte_t*)(key.c_str()), 16);
  cass_statement_bind_int64(statement, 1, start.getRaw());
  cass_statement_bind_int64(statement, 2, end.getRaw());

  future = cass_session_execute(session, statement);
  cass_future_wait(future);

  if (cass_future_error_code(future) == CASS_OK) {
      const CassResult* cresult = cass_future_get_result(future);
      CassIterator* rows = cass_iterator_from_result(cresult);

197
198
      SensorDataStoreReading entry;

Axel Auweter's avatar
Axel Auweter committed
199
200
201
202
203
204
205
206
      while (cass_iterator_next(rows)) {
          const CassRow* row = cass_iterator_get_row(rows);

          cass_int64_t ts, value;
          cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
          cass_value_get_int64(cass_row_get_column_by_name(row, "value"), &value);

          entry.sensorId = sid;
207
          entry.timeStamp = (uint64_t)ts;
208
          entry.value = (int64_t)value;
Axel Auweter's avatar
Axel Auweter committed
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231

          result.push_back(entry);
#if 0
          if (localtime) {
              t.convertToLocal();
          }
          if (raw) {
              std::cout << sensorName << "," << std::dec << t.getRaw() << "," << std::dec << value << std::endl;
          }
          else {
              std::cout << sensorName << "," << t.getString() << "," << std::dec << value << std::endl;
          }
#endif
      }
      cass_iterator_free(rows);
      cass_result_free(cresult);
  }

  cass_statement_free(statement);
  cass_future_free(future);
  cass_prepared_free(prepared);
}

232
233
234
235
236
237
/**
 * @details
 * This function generates an integrated value of the time series
 * by first querying for the result set list using query() and then
 * summing up the result.
 */
238
SDSQueryResult SensorDataStoreImpl::querySum(int64_t& result, SensorId& sid, TimeStamp& start, TimeStamp& end)
239
240
241
242
243
244
{
  std::list<SensorDataStoreReading> queryResult;

  /* Issue a standard query */
  query(queryResult, sid, start, end);

Axel Auweter's avatar
Axel Auweter committed
245
246
247
248
  /* Check if at least 2 readings in result */
  if (queryResult.size() < 2)
    return SDS_EMPTYSET;

249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
  /* Integrate the result */
  result = 0;

  SensorDataStoreReading prev;
  for (std::list<SensorDataStoreReading>::iterator it = queryResult.begin(); it != queryResult.end(); it++) {
      if (!(it == queryResult.begin())) {
          SensorDataStoreReading cur = *it;

          /* Calculate average between two readings */
          int64_t avg = (cur.value + prev.value) / 2;

          /* Calculate time difference */
          uint64_t dt = cur.timeStamp.getRaw() - prev.timeStamp.getRaw();

          /* Sum up (with lousy attempt to keep it numerically stable - should probably use double instead) */
          if (dt > 10000000000) {
              /* dt > 10s => convert dt to s first */
              dt /= 1000000000;
              result += avg * dt;
          }
          else {
              /* dt < 10s => multiply first */
              avg *= dt;
              result += avg / 1000000000;
          }
      }
      prev = *it;
  }
Axel Auweter's avatar
Axel Auweter committed
277
  return SDS_OK;
278
279
}

280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
/**
 * @details
 * This function deletes all data from the sensordata store
 * that is older than weekStamp-1 weeks.
 */
void SensorDataStoreImpl::truncBeforeWeek(uint16_t weekStamp)
{
  /* List of rows that should be deleted */
  std::list<SensorId> deleteList;

  /* Query the database to collect all rows */
  CassError rc = CASS_OK;
  CassStatement* statement = nullptr;
  CassFuture* future = nullptr;
  const char* query = "SELECT DISTINCT sid FROM " KEYSPACE_NAME "." CF_SENSORDATA ";";

  statement = cass_statement_new(query, 0);
  future = cass_session_execute(session, statement);
  cass_future_wait(future);

  rc = cass_future_error_code(future);
  if (rc != CASS_OK) {
      connection->printError(future);
      return;
  }

  const CassResult* result = cass_future_get_result(future);
  cass_future_free(future);

  CassIterator* iterator = cass_iterator_from_result(result);

  /* Iterate over all rows and filter out those, that are too old */
  while (cass_iterator_next(iterator)) {
      const CassRow* row = cass_iterator_get_row(iterator);
      const cass_byte_t* res;
      size_t       res_len;
      cass_value_get_bytes(cass_row_get_column_by_name(row, "sid"), &res, &res_len);

      uint64_t raw[2];
      raw[0] = Endian::BEToHost(((uint64_t*)res)[0]);
      raw[1] = Endian::BEToHost(((uint64_t*)res)[1]);

      SensorId sensor;
      sensor.setRaw(raw);

      /* Check if the sensorId's rsvd field is smaller than the weekStamp */
      if (sensor.getRsvd() < weekStamp) {
          deleteList.push_back(sensor);
      }
  }
  cass_result_free(result);
  cass_iterator_free(iterator);
  cass_statement_free(statement);

  /* Now iterate over all entries in the deleteList and delete them */
  for (std::list<SensorId>::iterator it = deleteList.begin(); it != deleteList.end(); it++) {
      deleteRow(*it);
  }
}

/**
 * @details
 * Deleting entire rows is rather efficient compared to deleting individual columns.
 */
void SensorDataStoreImpl::deleteRow(SensorId& sid)
{
  CassError rc = CASS_OK;
  CassStatement* statement = NULL;
  CassFuture *future = NULL;
  const CassPrepared* prepared = nullptr;
  const char* query = "DELETE FROM " KEYSPACE_NAME "." CF_SENSORDATA " WHERE sid = ?;";

  future = cass_session_prepare(session, query);
  cass_future_wait(future);

  rc = cass_future_error_code(future);
  if (rc != CASS_OK) {
    connection->printError(future);
    cass_future_free(future);
    return;
  }

  prepared = cass_future_get_prepared(future);
  cass_future_free(future);

  std::string key = sid.serialize();

  statement = cass_prepared_bind(prepared);
  cass_statement_bind_bytes(statement, 0, (const cass_byte_t*)(key.c_str()), 16);

  future = cass_session_execute(session, statement);
  cass_future_wait(future);

  cass_statement_free(statement);
  cass_future_free(future);
  cass_prepared_free(prepared);
}

378
379
/**
 * @details
380
 * This constructor sets the internal connection variable to
381
 * the externally provided Connection object and also
382
 * retrieves the CassSession pointer of the connection.
383
 */
384
SensorDataStoreImpl::SensorDataStoreImpl(Connection* conn)
385
{
386
387
  connection = conn;
  session = connection->getSessionHandle();
Axel Auweter's avatar
Axel Auweter committed
388
389
390

  preparedInsert = nullptr;
  prepareInsert(0);
391
392
}

393
394
/**
 * @details
395
 * Due to the simplicity of the class, the destructor is left empty.
396
 */
397
SensorDataStoreImpl::~SensorDataStoreImpl()
398
{
399
  connection = nullptr;
Axel Auweter's avatar
Axel Auweter committed
400
401
402
403
  session = nullptr;
  if (preparedInsert) {
      cass_prepared_free(preparedInsert);
  }
404
405
}

406
407
408
409
410
411
/**
 * @details
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
 */
412
void SensorDataStore::insert(SensorId* sid, uint64_t ts, int64_t value)
413
414
415
416
{
    impl->insert(sid, ts, value);
}

417
418
/**
 * @details
419
420
421
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
422
 */
423
void SensorDataStore::setTTL(uint64_t ttl)
424
{
425
    impl->setTTL(ttl);
426
427
}

Axel Auweter's avatar
Axel Auweter committed
428
429
430
431
432
433
/**
 * @details
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
 */
434
void SensorDataStore::query(std::list<SensorDataStoreReading>& result, SensorId& sid, TimeStamp& start, TimeStamp& end)
Axel Auweter's avatar
Axel Auweter committed
435
436
437
438
{
    impl->query(result, sid, start, end);
}

439
440
441
442
443
444
/**
 * @details
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
 */
445
SDSQueryResult SensorDataStore::querySum(int64_t& result, SensorId& sid, TimeStamp& start, TimeStamp& end)
446
{
Axel Auweter's avatar
Axel Auweter committed
447
    return impl->querySum(result, sid, start, end);
448
449
}

450
451
452
453
454
455
456
457
458
459
460
/**
 * @details
 * Instead of doing the actual work, this function simply
 * forwards to the insert function of the SensorDataStoreImpl
 * class.
 */
void SensorDataStore::truncBeforeWeek(uint16_t weekStamp)
{
    return impl->truncBeforeWeek(weekStamp);
}

461
462
/**
 * @details
463
464
 * This constructor allocates the implementation class which
 * holds the actual implementation of the class functionality.
465
 */
466
SensorDataStore::SensorDataStore(Connection* conn)
467
{
468
    impl = new SensorDataStoreImpl(conn);
469
470
}

471
472
473
474
475
/**
 * @details
 * The SensorDataStore desctructor deallocates the
 * SensorDataStoreImpl and CassandraBackend objects.
 */
476
477
SensorDataStore::~SensorDataStore()
{
478
479
480
  /* Clean up... */
  if (impl)
    delete impl;
481
}