Grafana: performance improvements

- Caching metadata information
- Using a single persistent SensorDataStore instance
- Now properly querying multiple weekstamps if necessary

collectagent/analyticscontroller.cpp

@@ -221,7 +221,7 @@ bool AnalyticsController::rebuildSensorNavigator() {
         LOG(error) << "Failed to retrieve public sensors. Sensor Navigator will be empty.";
     for (const auto &s : publicSensors)
         if (!s.is_virtual) {
-            sBuf = Configuration::publicSensorToMetadata(s);
+            sBuf = DCDB::PublicSensor::publicSensorToMetadata(s);
             if(sBuf.isValid())
                 topics.push_back(*sBuf.getPattern());
         }

collectagent/collectagent.cpp

@@ -239,7 +239,7 @@ bool metadataQueryCallback(const string& name, SensorMetadata& buffer) {
                 --queryEngine.access;
                 return false;
             }
-            buffer = Configuration::publicSensorToMetadata(publicSensor);
+            buffer = DCDB::PublicSensor::publicSensorToMetadata(publicSensor);
         }
         catch (const std::exception &e) {
             --queryEngine.access;
@@ -737,7 +737,7 @@ int main(int argc, char* const argv[]) {
       SensorMetadata sBuf;
       for (const auto &s : publicSensors)
           if (!s.is_virtual) {
-              sBuf = Configuration::publicSensorToMetadata(s);
+              sBuf = DCDB::PublicSensor::publicSensorToMetadata(s);
               if(sBuf.isValid())
                   metadataStore->store(*sBuf.getPattern(), sBuf);
           }

collectagent/configuration.cpp

@@ -73,50 +73,3 @@ void Configuration::readAdditionalBlocks(boost::property_tree::iptree& cfg) {
         }
     }
 }
-
-DCDB::PublicSensor Configuration::metadataToPublicSensor(const SensorMetadata& sm) {
-    DCDB::PublicSensor ps;
-    if(sm.getPublicName())
-        ps.name = *sm.getPublicName();
-    if(sm.getIsVirtual())
-        ps.is_virtual = *sm.getIsVirtual();
-    if(sm.getPattern())
-        ps.pattern = *sm.getPattern();
-    if(sm.getUnit())
-        ps.unit = *sm.getUnit();
-    if(sm.getScale())
-        ps.scaling_factor = *sm.getScale();
-    if(sm.getTTL())
-        ps.ttl = *sm.getTTL();
-    if(sm.getInterval())
-        ps.interval = *sm.getInterval();
-    if(sm.getOperations())
-        ps.operations = *sm.getOperations();
-    uint64_t sensorMask = 0;
-    if(sm.getIntegrable() && *sm.getIntegrable())
-        sensorMask = sensorMask | INTEGRABLE;
-    if(sm.getMonotonic() && *sm.getMonotonic())
-        sensorMask = sensorMask | MONOTONIC;
-    ps.sensor_mask = sensorMask;
-    return ps;
-}
-
-SensorMetadata Configuration::publicSensorToMetadata(const DCDB::PublicSensor& ps) {
-    SensorMetadata sm;
-    sm.setPublicName(ps.name);
-    sm.setIsVirtual(ps.is_virtual);
-    
-    // Stripping whitespace from the sensor pattern in the SID
-    string stripPattern = ps.pattern;
-    boost::algorithm::trim(stripPattern);
-    sm.setPattern(stripPattern);
-    
-    sm.setUnit(ps.unit);
-    sm.setScale(ps.scaling_factor);
-    sm.setTTL(ps.ttl);
-    sm.setInterval(ps.interval);
-    sm.setOperations(ps.operations);
-    sm.setIntegrable(ps.sensor_mask & INTEGRABLE);
-    sm.setMonotonic(ps.sensor_mask & MONOTONIC);
-    return sm;
-}

collectagent/configuration.h

@@ -86,22 +86,6 @@ public:
     
     virtual ~Configuration() {}
     
-    /**
-     * @brief               Converts a SensorMetadata object to a DCDB::PublicSensor one.
-     * 
-     * @param s             SensorMetadata object to be converted
-     * @return              Output PublicSensor object
-     */
-    static DCDB::PublicSensor metadataToPublicSensor(const SensorMetadata& s);
-
-    /**
-     * @brief               Converts a DCDB::PublicSensor object to its SensorMetadata representation.
-     * 
-     * @param ps            The PublicSensor object to be converted
-     * @return              Output SensorMetadata object
-     */
-    static SensorMetadata publicSensorToMetadata(const DCDB::PublicSensor& ps);
-    
     // Additional configuration parameters to be parsed within the global block
     std::string mqttListenHost = string(DEFAULT_LISTENHOST);
     std::string mqttListenPort = string(DEFAULT_LISTENPORT);

grafana/RestAPI.cpp

@@ -50,8 +50,7 @@ RestAPI::RestAPI(serverSettings_t settings,
     addEndpoint("/query",       {http::verb::post, stdBind(POST_query)});
 }
 
-//Initializes the internal sensor navigator
-bool RestAPI::initTree() {
+void RestAPI::clear() {
     if(_sensorConfig) {
         delete _sensorConfig;
         _sensorConfig = NULL;
@@ -60,19 +59,32 @@ bool RestAPI::initTree() {
         delete _navigator;
         _navigator = NULL;
     }
+    if(_sensorDataStore) {
+        delete _sensorDataStore;
+        _sensorDataStore = NULL;
+    }
+    _metadataStore.clear();
+}
+
+//Initializes the internal sensor navigator
+bool RestAPI::initTree() {
+    clear();
     
     //Get the list of all public sensors and topics.
     std::vector<std::string> topics;
     std::list<DCDB::PublicSensor> publicSensors;
 
     _sensorConfig = new DCDB::SensorConfig(_connection);
+    _sensorDataStore = new DCDB::SensorDataStore(_connection);
     if(_sensorConfig->getPublicSensorsVerbose(publicSensors)!=DCDB::SC_OK) {
         LOG(error) << "Unable to fetch list of public sensors!";
+        clear();
         return false;
     }
-
+    
     for(auto& s : publicSensors) {
         topics.push_back(s.pattern);
+        _metadataStore.store(s.pattern, DCDB::PublicSensor::publicSensorToMetadata(s));
     }
 
     //Build the tree navigator
@@ -213,32 +225,34 @@ void RestAPI::POST_query(endpointArgs) {
     
     //Prepare query
     DCDB::TimeStamp start, end;
-    DCDB::PublicSensor ps;
+    SensorMetadata sm;
     start = DCDB::TimeStamp(startTime, false);
     end   = DCDB::TimeStamp(endTime, false);
-
-    for(auto& sensorName : sensors) {
+    std::list<DCDB::SensorDataStoreReading> results;
     
-        std::list<DCDB::SensorDataStoreReading> results;
-        DCDB::SensorDataStore sensorDataStore(_connection);
-        _sensorConfig->getPublicSensorByName(ps,sensorName.c_str());
-
-        if(ps.interval && !ps.operations.empty()) {
-
+    for(auto& sensorName : sensors) {
+        results.clear();
+        try {
+            sm = _metadataStore.get(sensorName);
+        } catch(const std::invalid_argument &e) {}
+        
+        uint64_t sInterval = sm.getInterval() ? *sm.getInterval() : 0;
+        double sScale = sm.getScale() ? *sm.getScale() : 1.0;
+        
+        if(sInterval && sm.getOperations() && !sm.getOperations()->empty()) {
+            
             //Estimate the number of requested datapoints, for all sensors' data plotted in the panel. The number is calculated assuming all sensors plotted on the same panel have the same sampling period. If we exceed the maximum, then apply smoothing.
             uint64_t grafanaInterval = end.getRaw() - start.getRaw();
-            if((int)((grafanaInterval/ps.interval) * sensors.size()) > MAX_DATAPOINTS) {
+            if((int)((grafanaInterval / sInterval) * sensors.size()) > MAX_DATAPOINTS) {
 
                 //Choose a smoother
-                if(grafanaInterval/10000000000 < MAX_DATAPOINTS &&
-                   ps.operations.find("-avg10") != ps.operations.end())
+                if(grafanaInterval/10000000000 < MAX_DATAPOINTS && sm.getOperations()->find("-avg10") != sm.getOperations()->end())
                     sensorName += "-avg10";
 
-                else if(grafanaInterval/300000000000 < MAX_DATAPOINTS &&
-                    ps.operations.find("-avg300") != ps.operations.end())
+                else if(grafanaInterval/300000000000 < MAX_DATAPOINTS && sm.getOperations()->find("-avg300") != sm.getOperations()->end())
                     sensorName += "-avg300";
                 
-                else if(ps.operations.find("-avg3600") != ps.operations.end())
+                else if(sm.getOperations()->find("-avg3600") != sm.getOperations()->end())
                     //Hourly averages should be fine for intervals of years.
                     sensorName += "-avg3600";
             }
@@ -246,15 +260,18 @@ void RestAPI::POST_query(endpointArgs) {
 
         //We need the Sensor ID, since smoothed sensors are not published.
         DCDB::SensorId sid(sensorName);
-        sid.setRsvd(start.getWeekstamp());
+        uint16_t startWs=start.getWeekstamp(), endWs=end.getWeekstamp();
         
         //Shoot the query for this sensor.
-        sensorDataStore.query(results, sid, start, end, DCDB::AGGREGATE_NONE);
+        for(uint16_t currWs=startWs; currWs<=endWs; currWs++) {
+            sid.setRsvd(currWs);
+            _sensorDataStore->query(results, sid, start, end, DCDB::AGGREGATE_NONE);
+        }
         
         //Format the output for the response to Grafana.
         std::string datapoints = "[";
         for (auto& r : results)
-            datapoints += "[" + std::to_string(r.value * ps.scaling_factor) + ","
+            datapoints += "[" + std::to_string(r.value * sScale) + ","
             + std::to_string(r.timeStamp.getRaw()/1000000) + "],";
         if(datapoints.back() == ',')
             datapoints.pop_back();

grafana/RestAPI.h

@@ -33,6 +33,7 @@
 #include "sensornavigator.h"
 #include "cassandra.h"
 #include "mqttchecker.h"
+#include "metadatastore.h"
 
 #include "dcdb/connection.h"
 #include "dcdb/timestamp.h"
@@ -57,16 +58,7 @@ public:
             hierarchySettings_t hierarchySettings,
             DCDB::Connection* cassandraConnection);
     
-    virtual ~RestAPI() {
-        if(_navigator) {
-            delete _navigator;
-            _navigator = NULL;
-        }
-        if(_sensorConfig) {
-            delete _sensorConfig;
-            _sensorConfig = NULL;
-        }
-    }
+    virtual ~RestAPI() { clear(); }
 
     bool initTree();
 
@@ -133,9 +125,14 @@ private:
 
 /******************************************************************************/
 
+    //Clears all internal storage
+    void clear();
+
     SensorNavigator*    _navigator;
     DCDB::Connection*   _connection;
     DCDB::SensorConfig* _sensorConfig;
+    DCDB::SensorDataStore* _sensorDataStore;
+    MetadataStore _metadataStore;
     std::string _separator;
     hierarchySettings_t _hierarchySettings;
     

lib/include/dcdb/sensorconfig.h

@@ -80,6 +80,22 @@ public:
   PublicSensor(const PublicSensor &copy);
     
   inline bool operator <  (const PublicSensor& rhs) const {return name <  rhs.name;}
+
+  /**
+  * @brief               Converts a SensorMetadata object to a DCDB::PublicSensor one.
+  * 
+  * @param s             SensorMetadata object to be converted
+  * @return              Output PublicSensor object
+  */
+  static PublicSensor metadataToPublicSensor(const SensorMetadata& s);
+
+  /**
+  * @brief               Converts a DCDB::PublicSensor object to its SensorMetadata representation.
+  * 
+  * @param ps            The PublicSensor object to be converted
+  * @return              Output SensorMetadata object
+  */
+  static SensorMetadata publicSensorToMetadata(const PublicSensor& ps);
 };
 
 /**

lib/src/sensorconfig.cpp

@@ -77,6 +77,53 @@ PublicSensor::PublicSensor (const PublicSensor &copy)
     ttl = copy.ttl;
 }
 
+PublicSensor PublicSensor::metadataToPublicSensor(const SensorMetadata& sm) {
+    PublicSensor ps;
+    if(sm.getPublicName())
+        ps.name = *sm.getPublicName();
+    if(sm.getIsVirtual())
+        ps.is_virtual = *sm.getIsVirtual();
+    if(sm.getPattern())
+        ps.pattern = *sm.getPattern();
+    if(sm.getUnit())
+        ps.unit = *sm.getUnit();
+    if(sm.getScale())
+        ps.scaling_factor = *sm.getScale();
+    if(sm.getTTL())
+        ps.ttl = *sm.getTTL();
+    if(sm.getInterval())
+        ps.interval = *sm.getInterval();
+    if(sm.getOperations())
+        ps.operations = *sm.getOperations();
+    uint64_t sensorMask = 0;
+    if(sm.getIntegrable() && *sm.getIntegrable())
+        sensorMask = sensorMask | INTEGRABLE;
+    if(sm.getMonotonic() && *sm.getMonotonic())
+        sensorMask = sensorMask | MONOTONIC;
+    ps.sensor_mask = sensorMask;
+    return ps;
+}
+
+SensorMetadata PublicSensor::publicSensorToMetadata(const PublicSensor& ps) {
+    SensorMetadata sm;
+    sm.setPublicName(ps.name);
+    sm.setIsVirtual(ps.is_virtual);
+
+    // Stripping whitespace from the sensor pattern in the SID
+    string stripPattern = ps.pattern;
+    boost::algorithm::trim(stripPattern);
+    sm.setPattern(stripPattern);
+
+    sm.setUnit(ps.unit);
+    sm.setScale(ps.scaling_factor);
+    sm.setTTL(ps.ttl);
+    sm.setInterval(ps.interval);
+    sm.setOperations(ps.operations);
+    sm.setIntegrable(ps.sensor_mask & INTEGRABLE);
+    sm.setMonotonic(ps.sensor_mask & MONOTONIC);
+    return sm;
+}
+
 
 /*
  * SensorConfig functions