libDCDB: paging for JobDataStore and CaliEvtDataStore queries

lib/include_internal/dcdbglobals.h

@@ -44,7 +44,7 @@
 #define CF_SENSORDATA        "sensordata"
 #define SENSORDATA_GC_GRACE_SECONDS "600"
 #define SENSORDATA_COMPACTION "{'class' : 'TimeWindowCompactionStrategy', 'compaction_window_unit' : 'DAYS', 'compaction_window_size' : 1 }"
-#define PAGING_SIZE 50000
+#define PAGING_SIZE 10000
 
 #define CONFIG_KEYSPACE_NAME KEYSPACE_NAME "_config"
 #define CF_PUBLISHEDSENSORS  "publishedsensors"

lib/src/calievtdatastore.cpp

@@ -222,41 +222,54 @@ void CaliEvtDataStoreImpl::query(std::list<CaliEvtData>& result, SensorId& sid,
     cass_future_free(future);
 
     statement = cass_prepared_bind(prepared);
+    cass_statement_set_paging_size(statement, PAGING_SIZE);
     cass_statement_bind_string(statement, 0, sid.getId().c_str());
     cass_statement_bind_int16(statement, 1, sid.getRsvd());
     cass_statement_bind_int64(statement, 2, start.getRaw());
     cass_statement_bind_int64(statement, 3, end.getRaw());
 
-    future = cass_session_execute(session, statement);
-    cass_future_wait(future);
-
-    if (cass_future_error_code(future) == CASS_OK) {
-        const CassResult* cresult = cass_future_get_result(future);
-        CassIterator* rows = cass_iterator_from_result(cresult);
-
-        CaliEvtData entry;
-
-        while (cass_iterator_next(rows)) {
-            const CassRow* row = cass_iterator_get_row(rows);
+    bool morePages = false;
+    do {
+        future = cass_session_execute(session, statement);
+        cass_future_wait(future);
+
+        if (cass_future_error_code(future) == CASS_OK) {
+            const CassResult* cresult = cass_future_get_result(future);
+            CassIterator* rows = cass_iterator_from_result(cresult);
+    
+            CaliEvtData entry;
+    
+            while (cass_iterator_next(rows)) {
+                const CassRow* row = cass_iterator_get_row(rows);
+    
+                cass_int64_t ts;
+                const char* eventStr;
+                size_t eventStr_len;
+                cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
+                cass_value_get_string(cass_row_get_column_by_name(row, "value"), &eventStr, &eventStr_len);
+    
+                entry.eventId = sid;
+                entry.timeStamp = (uint64_t)ts;
+                entry.event = std::string(eventStr, eventStr_len);
+    
+                result.push_back(entry);
+            }
 
-            cass_int64_t ts;
-            const char* eventStr;
-            size_t eventStr_len;
-            cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
-            cass_value_get_string(cass_row_get_column_by_name(row, "value"), &eventStr, &eventStr_len);
+            if((morePages = cass_result_has_more_pages(cresult)))
+                cass_statement_set_paging_state(statement, cresult);
+            
+            cass_iterator_free(rows);
+            cass_result_free(cresult);
+        } else {
+            morePages = false;
+        }
 
-            entry.eventId = sid;
-            entry.timeStamp = (uint64_t)ts;
-            entry.event = std::string(eventStr, eventStr_len);
+        cass_future_free(future);
 
-            result.push_back(entry);
-        }
-        cass_iterator_free(rows);
-        cass_result_free(cresult);
     }
+    while(morePages);
 
     cass_statement_free(statement);
-    cass_future_free(future);
     cass_prepared_free(prepared);
 }
 
@@ -417,41 +430,54 @@ void CaliEvtDataStoreImpl::queryCB(CaliEvtDataStore::QueryCECbFunc cbFunc, void*
     cass_future_free(future);
 
     statement = cass_prepared_bind(prepared);
+    cass_statement_set_paging_size(statement, PAGING_SIZE);
     cass_statement_bind_string(statement, 0, sid.getId().c_str());
     cass_statement_bind_int16(statement, 1, sid.getRsvd());
     cass_statement_bind_int64(statement, 2, start.getRaw());
     cass_statement_bind_int64(statement, 3, end.getRaw());
 
-    future = cass_session_execute(session, statement);
-    cass_future_wait(future);
-
-    if (cass_future_error_code(future) == CASS_OK) {
-        const CassResult* cresult = cass_future_get_result(future);
-        CassIterator* rows = cass_iterator_from_result(cresult);
-
-        CaliEvtData entry;
-
-        while (cass_iterator_next(rows)) {
-            const CassRow* row = cass_iterator_get_row(rows);
+    bool morePages = false;
+    do {
+        future = cass_session_execute(session, statement);
+        cass_future_wait(future);
+    
+        if (cass_future_error_code(future) == CASS_OK) {
+            const CassResult* cresult = cass_future_get_result(future);
+            CassIterator* rows = cass_iterator_from_result(cresult);
+    
+            CaliEvtData entry;
+    
+            while (cass_iterator_next(rows)) {
+                const CassRow* row = cass_iterator_get_row(rows);
+    
+                cass_int64_t ts;
+                const char* eventStr;
+                size_t eventStr_len;
+                cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
+                cass_value_get_string(cass_row_get_column_by_name(row, "value"), &eventStr, &eventStr_len);
+    
+                entry.eventId = sid;
+                entry.timeStamp = (uint64_t)ts;
+                entry.event = std::string(eventStr, eventStr_len);
+    
+                cbFunc(entry, userData);
+            }
 
-            cass_int64_t ts;
-            const char* eventStr;
-            size_t eventStr_len;
-            cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
-            cass_value_get_string(cass_row_get_column_by_name(row, "value"), &eventStr, &eventStr_len);
+            if((morePages = cass_result_has_more_pages(cresult)))
+                cass_statement_set_paging_state(statement, cresult);
+            
+            cass_iterator_free(rows);
+            cass_result_free(cresult);
+        } else {
+            morePages = false;
+        }
 
-            entry.eventId = sid;
-            entry.timeStamp = (uint64_t)ts;
-            entry.event = std::string(eventStr, eventStr_len);
+        cass_future_free(future);
 
-            cbFunc(entry, userData);
-        }
-        cass_iterator_free(rows);
-        cass_result_free(cresult);
     }
+    while(morePages);
 
     cass_statement_free(statement);
-    cass_future_free(future);
     cass_prepared_free(prepared);
 }
 

lib/src/sensordatastore.cpp

@@ -628,39 +628,52 @@ void SensorDataStoreImpl::queryCB(SensorDataStore::QueryCbFunc cbFunc, void* use
   cass_future_free(future);
   
   statement = cass_prepared_bind(prepared);
+  cass_statement_set_paging_size(statement, PAGING_SIZE);
   cass_statement_bind_string(statement, 0, sid.getId().c_str());
   cass_statement_bind_int16(statement, 1, sid.getRsvd());
   cass_statement_bind_int64(statement, 2, start.getRaw());
   cass_statement_bind_int64(statement, 3, end.getRaw());
 
-  future = cass_session_execute(session, statement);
-  cass_future_wait(future);
-
-  if (cass_future_error_code(future) == CASS_OK) {
-      const CassResult* cresult = cass_future_get_result(future);
-      CassIterator* rows = cass_iterator_from_result(cresult);
-
-      SensorDataStoreReading entry;
-
-      while (cass_iterator_next(rows)) {
-          const CassRow* row = cass_iterator_get_row(rows);
+  bool morePages = false;
+  do {
+      future = cass_session_execute(session, statement);
+      cass_future_wait(future);
+    
+      if (cass_future_error_code(future) == CASS_OK) {
+          const CassResult* cresult = cass_future_get_result(future);
+          CassIterator* rows = cass_iterator_from_result(cresult);
+    
+          SensorDataStoreReading entry;
+    
+          while (cass_iterator_next(rows)) {
+              const CassRow* row = cass_iterator_get_row(rows);
+    
+              cass_int64_t ts, value;
+              cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
+              cass_value_get_int64(cass_row_get_column_by_name(row, "value"), &value);
+    
+              entry.sensorId = sid;
+              entry.timeStamp = (uint64_t)ts;
+              entry.value = (int64_t)value;
+    
+              cbFunc(entry, userData);
+          }
 
-          cass_int64_t ts, value;
-          cass_value_get_int64(cass_row_get_column_by_name(row, "ts"), &ts);
-          cass_value_get_int64(cass_row_get_column_by_name(row, "value"), &value);
+          if((morePages = cass_result_has_more_pages(cresult)))
+              cass_statement_set_paging_state(statement, cresult);
+          
+          cass_iterator_free(rows);
+          cass_result_free(cresult);
+      } else {
+          morePages = false;
+      }
 
-          entry.sensorId = sid;
-          entry.timeStamp = (uint64_t)ts;
-          entry.value = (int64_t)value;
+      cass_future_free(future);
 
-          cbFunc(entry, userData);
-      }
-      cass_iterator_free(rows);
-      cass_result_free(cresult);
   }
+  while(morePages);
 
   cass_statement_free(statement);
-  cass_future_free(future);
   cass_prepared_free(prepared);
 }