Commit 70e07df2 authored by Alessio Netti's avatar Alessio Netti

libDCDB: paging for SensorDataStore and SensorConfig queries

parent 39db19f7
...@@ -44,6 +44,7 @@ ...@@ -44,6 +44,7 @@
#define CF_SENSORDATA "sensordata" #define CF_SENSORDATA "sensordata"
#define SENSORDATA_GC_GRACE_SECONDS "600" #define SENSORDATA_GC_GRACE_SECONDS "600"
#define SENSORDATA_COMPACTION "{'class' : 'TimeWindowCompactionStrategy', 'compaction_window_unit' : 'DAYS', 'compaction_window_size' : 1 }" #define SENSORDATA_COMPACTION "{'class' : 'TimeWindowCompactionStrategy', 'compaction_window_unit' : 'DAYS', 'compaction_window_size' : 1 }"
#define PAGING_SIZE 50000
#define CONFIG_KEYSPACE_NAME KEYSPACE_NAME "_config" #define CONFIG_KEYSPACE_NAME KEYSPACE_NAME "_config"
#define CF_PUBLISHEDSENSORS "publishedsensors" #define CF_PUBLISHEDSENSORS "publishedsensors"
......
...@@ -372,8 +372,7 @@ bool ConnectionImpl::connect() { ...@@ -372,8 +372,7 @@ bool ConnectionImpl::connect() {
cass_cluster_set_num_threads_io(cluster, numThreadsIo); cass_cluster_set_num_threads_io(cluster, numThreadsIo);
cass_cluster_set_queue_size_io(cluster, queueSizeIo); cass_cluster_set_queue_size_io(cluster, queueSizeIo);
cass_cluster_set_core_connections_per_host(cluster, coreConnPerHost); cass_cluster_set_core_connections_per_host(cluster, coreConnPerHost);
//TODO: avoid this and use actual paging in queries cass_cluster_set_request_timeout(cluster, 60000);
cass_cluster_set_request_timeout(cluster, 300000);
/* Force protcol version to 1 */ /* Force protcol version to 1 */
cass_cluster_set_protocol_version(cluster, 1); cass_cluster_set_protocol_version(cluster, 1);
......
This diff is collapsed.
...@@ -262,50 +262,62 @@ void SensorDataStoreImpl::query(std::list<SensorDataStoreReading>& result, Senso ...@@ -262,50 +262,62 @@ void SensorDataStoreImpl::query(std::list<SensorDataStoreReading>& result, Senso
#endif #endif
statement = cass_prepared_bind(prepared); statement = cass_prepared_bind(prepared);
cass_statement_set_paging_size(statement, PAGING_SIZE);
cass_statement_bind_string(statement, 0, sid.getId().c_str()); cass_statement_bind_string(statement, 0, sid.getId().c_str());
cass_statement_bind_int16(statement, 1, sid.getRsvd()); cass_statement_bind_int16(statement, 1, sid.getRsvd());
cass_statement_bind_int64(statement, 2, start.getRaw()); cass_statement_bind_int64(statement, 2, start.getRaw());
cass_statement_bind_int64(statement, 3, end.getRaw()); cass_statement_bind_int64(statement, 3, end.getRaw());
future = cass_session_execute(session, statement); bool morePages = false;
cass_future_wait(future); do {
future = cass_session_execute(session, statement);
cass_future_wait(future);
if (cass_future_error_code(future) == CASS_OK) { if (cass_future_error_code(future) == CASS_OK) {
const CassResult* cresult = cass_future_get_result(future); const CassResult *cresult = cass_future_get_result(future);
CassIterator* rows = cass_iterator_from_result(cresult); CassIterator *rows = cass_iterator_from_result(cresult);
SensorDataStoreReading entry; SensorDataStoreReading entry;
while (cass_iterator_next(rows)) { while (cass_iterator_next(rows)) {
const CassRow* row = cass_iterator_get_row(rows); const CassRow *row = cass_iterator_get_row(rows);
cass_int64_t ts, value; cass_int64_t ts, value;
cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts); cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
cass_value_get_int64(cass_row_get_column_by_name(row, "value"), &value); cass_value_get_int64(cass_row_get_column_by_name(row, "value"), &value);
entry.sensorId = sid; entry.sensorId = sid;
entry.timeStamp = (uint64_t)ts; entry.timeStamp = (uint64_t) ts;
entry.value = (int64_t)value; entry.value = (int64_t) value;
result.push_back(entry); result.push_back(entry);
#if 0 #if 0
if (localtime) { if (localtime) {
t.convertToLocal(); t.convertToLocal();
} }
if (raw) { if (raw) {
std::cout << sensorName << "," << std::dec << t.getRaw() << "," << std::dec << value << std::endl; std::cout << sensorName << "," << std::dec << t.getRaw() << "," << std::dec << value << std::endl;
} }
else { else {
std::cout << sensorName << "," << t.getString() << "," << std::dec << value << std::endl; std::cout << sensorName << "," << t.getString() << "," << std::dec << value << std::endl;
} }
#endif #endif
}
if((morePages = cass_result_has_more_pages(cresult)))
cass_statement_set_paging_state(statement, cresult);
cass_iterator_free(rows);
cass_result_free(cresult);
} else {
morePages = false;
} }
cass_iterator_free(rows);
cass_result_free(cresult); cass_future_free(future);
} }
while(morePages);
cass_statement_free(statement); cass_statement_free(statement);
cass_future_free(future);
cass_prepared_free(prepared); cass_prepared_free(prepared);
} }
...@@ -360,55 +372,67 @@ void SensorDataStoreImpl::query(std::list<SensorDataStoreReading>& result, std:: ...@@ -360,55 +372,67 @@ void SensorDataStoreImpl::query(std::list<SensorDataStoreReading>& result, std::
} }
statement = cass_prepared_bind(prepared); statement = cass_prepared_bind(prepared);
cass_statement_set_paging_size(statement, -1); cass_statement_set_paging_size(statement, PAGING_SIZE);
cass_statement_bind_collection(statement, 0, cassList); cass_statement_bind_collection(statement, 0, cassList);
cass_statement_bind_int16(statement, 1, sids.front().getRsvd()); cass_statement_bind_int16(statement, 1, sids.front().getRsvd());
cass_statement_bind_int64(statement, 2, start.getRaw()); cass_statement_bind_int64(statement, 2, start.getRaw());
cass_statement_bind_int64(statement, 3, end.getRaw()); cass_statement_bind_int64(statement, 3, end.getRaw());
future = cass_session_execute(session, statement); bool morePages = false;
cass_future_wait(future); do {
future = cass_session_execute(session, statement);
if (cass_future_error_code(future) == CASS_OK) { cass_future_wait(future);
const CassResult *cresult = cass_future_get_result(future);
CassIterator *rows = cass_iterator_from_result(cresult); if (cass_future_error_code(future) == CASS_OK) {
SensorDataStoreReading entry; const CassResult *cresult = cass_future_get_result(future);
cass_int64_t ts, value; CassIterator *rows = cass_iterator_from_result(cresult);
const char *name; SensorDataStoreReading entry;
size_t name_len; cass_int64_t ts, value;
const char *name;
while (cass_iterator_next(rows)) { size_t name_len;
const CassRow *row = cass_iterator_get_row(rows);
while (cass_iterator_next(rows)) {
cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts); const CassRow *row = cass_iterator_get_row(rows);
cass_value_get_int64(cass_row_get_column_by_name(row, "value"), &value);
entry.timeStamp = (uint64_t) ts; cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
entry.value = (int64_t) value; cass_value_get_int64(cass_row_get_column_by_name(row, "value"), &value);
entry.timeStamp = (uint64_t) ts;
if(storeSids) { entry.value = (int64_t) value;
cass_value_get_string(cass_row_get_column_by_name(row, "sid"), &name, &name_len);
entry.sensorId = SensorId(std::string(name, name_len)); if(storeSids) {
} cass_value_get_string(cass_row_get_column_by_name(row, "sid"), &name, &name_len);
entry.sensorId = SensorId(std::string(name, name_len));
result.push_back(entry); }
result.push_back(entry);
#if 0 #if 0
if (localtime) { if (localtime) {
t.convertToLocal(); t.convertToLocal();
} }
if (raw) { if (raw) {
std::cout << sensorName << "," << std::dec << t.getRaw() << "," << std::dec << value << std::endl; std::cout << sensorName << "," << std::dec << t.getRaw() << "," << std::dec << value << std::endl;
} }
else { else {
std::cout << sensorName << "," << t.getString() << "," << std::dec << value << std::endl; std::cout << sensorName << "," << t.getString() << "," << std::dec << value << std::endl;
} }
#endif #endif
}
if((morePages = cass_result_has_more_pages(cresult)))
cass_statement_set_paging_state(statement, cresult);
cass_iterator_free(rows);
cass_result_free(cresult);
} else {
morePages = false;
} }
cass_iterator_free(rows);
cass_result_free(cresult); cass_future_free(future);
} }
while(morePages);
cass_statement_free(statement); cass_statement_free(statement);
cass_future_free(future);
cass_collection_free(cassList); cass_collection_free(cassList);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment