Commit 433a196c authored by Alessio Netti's avatar Alessio Netti
Browse files

Grafana: performance improvements

- Caching metadata information
- Using a single persistent SensorDataStore instance
- Now properly querying multiple weekstamps if necessary
parent 0178119d
...@@ -221,7 +221,7 @@ bool AnalyticsController::rebuildSensorNavigator() { ...@@ -221,7 +221,7 @@ bool AnalyticsController::rebuildSensorNavigator() {
LOG(error) << "Failed to retrieve public sensors. Sensor Navigator will be empty."; LOG(error) << "Failed to retrieve public sensors. Sensor Navigator will be empty.";
for (const auto &s : publicSensors) for (const auto &s : publicSensors)
if (!s.is_virtual) { if (!s.is_virtual) {
sBuf = Configuration::publicSensorToMetadata(s); sBuf = DCDB::PublicSensor::publicSensorToMetadata(s);
if(sBuf.isValid()) if(sBuf.isValid())
topics.push_back(*sBuf.getPattern()); topics.push_back(*sBuf.getPattern());
} }
......
...@@ -239,7 +239,7 @@ bool metadataQueryCallback(const string& name, SensorMetadata& buffer) { ...@@ -239,7 +239,7 @@ bool metadataQueryCallback(const string& name, SensorMetadata& buffer) {
--queryEngine.access; --queryEngine.access;
return false; return false;
} }
buffer = Configuration::publicSensorToMetadata(publicSensor); buffer = DCDB::PublicSensor::publicSensorToMetadata(publicSensor);
} }
catch (const std::exception &e) { catch (const std::exception &e) {
--queryEngine.access; --queryEngine.access;
...@@ -737,7 +737,7 @@ int main(int argc, char* const argv[]) { ...@@ -737,7 +737,7 @@ int main(int argc, char* const argv[]) {
SensorMetadata sBuf; SensorMetadata sBuf;
for (const auto &s : publicSensors) for (const auto &s : publicSensors)
if (!s.is_virtual) { if (!s.is_virtual) {
sBuf = Configuration::publicSensorToMetadata(s); sBuf = DCDB::PublicSensor::publicSensorToMetadata(s);
if(sBuf.isValid()) if(sBuf.isValid())
metadataStore->store(*sBuf.getPattern(), sBuf); metadataStore->store(*sBuf.getPattern(), sBuf);
} }
......
...@@ -73,50 +73,3 @@ void Configuration::readAdditionalBlocks(boost::property_tree::iptree& cfg) { ...@@ -73,50 +73,3 @@ void Configuration::readAdditionalBlocks(boost::property_tree::iptree& cfg) {
} }
} }
} }
DCDB::PublicSensor Configuration::metadataToPublicSensor(const SensorMetadata& sm) {
DCDB::PublicSensor ps;
if(sm.getPublicName())
ps.name = *sm.getPublicName();
if(sm.getIsVirtual())
ps.is_virtual = *sm.getIsVirtual();
if(sm.getPattern())
ps.pattern = *sm.getPattern();
if(sm.getUnit())
ps.unit = *sm.getUnit();
if(sm.getScale())
ps.scaling_factor = *sm.getScale();
if(sm.getTTL())
ps.ttl = *sm.getTTL();
if(sm.getInterval())
ps.interval = *sm.getInterval();
if(sm.getOperations())
ps.operations = *sm.getOperations();
uint64_t sensorMask = 0;
if(sm.getIntegrable() && *sm.getIntegrable())
sensorMask = sensorMask | INTEGRABLE;
if(sm.getMonotonic() && *sm.getMonotonic())
sensorMask = sensorMask | MONOTONIC;
ps.sensor_mask = sensorMask;
return ps;
}
SensorMetadata Configuration::publicSensorToMetadata(const DCDB::PublicSensor& ps) {
SensorMetadata sm;
sm.setPublicName(ps.name);
sm.setIsVirtual(ps.is_virtual);
// Stripping whitespace from the sensor pattern in the SID
string stripPattern = ps.pattern;
boost::algorithm::trim(stripPattern);
sm.setPattern(stripPattern);
sm.setUnit(ps.unit);
sm.setScale(ps.scaling_factor);
sm.setTTL(ps.ttl);
sm.setInterval(ps.interval);
sm.setOperations(ps.operations);
sm.setIntegrable(ps.sensor_mask & INTEGRABLE);
sm.setMonotonic(ps.sensor_mask & MONOTONIC);
return sm;
}
...@@ -86,22 +86,6 @@ public: ...@@ -86,22 +86,6 @@ public:
virtual ~Configuration() {} virtual ~Configuration() {}
/**
* @brief Converts a SensorMetadata object to a DCDB::PublicSensor one.
*
* @param s SensorMetadata object to be converted
* @return Output PublicSensor object
*/
static DCDB::PublicSensor metadataToPublicSensor(const SensorMetadata& s);
/**
* @brief Converts a DCDB::PublicSensor object to its SensorMetadata representation.
*
* @param ps The PublicSensor object to be converted
* @return Output SensorMetadata object
*/
static SensorMetadata publicSensorToMetadata(const DCDB::PublicSensor& ps);
// Additional configuration parameters to be parsed within the global block // Additional configuration parameters to be parsed within the global block
std::string mqttListenHost = string(DEFAULT_LISTENHOST); std::string mqttListenHost = string(DEFAULT_LISTENHOST);
std::string mqttListenPort = string(DEFAULT_LISTENPORT); std::string mqttListenPort = string(DEFAULT_LISTENPORT);
......
...@@ -50,8 +50,7 @@ RestAPI::RestAPI(serverSettings_t settings, ...@@ -50,8 +50,7 @@ RestAPI::RestAPI(serverSettings_t settings,
addEndpoint("/query", {http::verb::post, stdBind(POST_query)}); addEndpoint("/query", {http::verb::post, stdBind(POST_query)});
} }
//Initializes the internal sensor navigator void RestAPI::clear() {
bool RestAPI::initTree() {
if(_sensorConfig) { if(_sensorConfig) {
delete _sensorConfig; delete _sensorConfig;
_sensorConfig = NULL; _sensorConfig = NULL;
...@@ -60,19 +59,32 @@ bool RestAPI::initTree() { ...@@ -60,19 +59,32 @@ bool RestAPI::initTree() {
delete _navigator; delete _navigator;
_navigator = NULL; _navigator = NULL;
} }
if(_sensorDataStore) {
delete _sensorDataStore;
_sensorDataStore = NULL;
}
_metadataStore.clear();
}
//Initializes the internal sensor navigator
bool RestAPI::initTree() {
clear();
//Get the list of all public sensors and topics. //Get the list of all public sensors and topics.
std::vector<std::string> topics; std::vector<std::string> topics;
std::list<DCDB::PublicSensor> publicSensors; std::list<DCDB::PublicSensor> publicSensors;
_sensorConfig = new DCDB::SensorConfig(_connection); _sensorConfig = new DCDB::SensorConfig(_connection);
_sensorDataStore = new DCDB::SensorDataStore(_connection);
if(_sensorConfig->getPublicSensorsVerbose(publicSensors)!=DCDB::SC_OK) { if(_sensorConfig->getPublicSensorsVerbose(publicSensors)!=DCDB::SC_OK) {
LOG(error) << "Unable to fetch list of public sensors!"; LOG(error) << "Unable to fetch list of public sensors!";
clear();
return false; return false;
} }
for(auto& s : publicSensors) { for(auto& s : publicSensors) {
topics.push_back(s.pattern); topics.push_back(s.pattern);
_metadataStore.store(s.pattern, DCDB::PublicSensor::publicSensorToMetadata(s));
} }
//Build the tree navigator //Build the tree navigator
...@@ -213,32 +225,34 @@ void RestAPI::POST_query(endpointArgs) { ...@@ -213,32 +225,34 @@ void RestAPI::POST_query(endpointArgs) {
//Prepare query //Prepare query
DCDB::TimeStamp start, end; DCDB::TimeStamp start, end;
DCDB::PublicSensor ps; SensorMetadata sm;
start = DCDB::TimeStamp(startTime, false); start = DCDB::TimeStamp(startTime, false);
end = DCDB::TimeStamp(endTime, false); end = DCDB::TimeStamp(endTime, false);
std::list<DCDB::SensorDataStoreReading> results;
for(auto& sensorName : sensors) {
std::list<DCDB::SensorDataStoreReading> results; for(auto& sensorName : sensors) {
DCDB::SensorDataStore sensorDataStore(_connection); results.clear();
_sensorConfig->getPublicSensorByName(ps,sensorName.c_str()); try {
sm = _metadataStore.get(sensorName);
if(ps.interval && !ps.operations.empty()) { } catch(const std::invalid_argument &e) {}
uint64_t sInterval = sm.getInterval() ? *sm.getInterval() : 0;
double sScale = sm.getScale() ? *sm.getScale() : 1.0;
if(sInterval && sm.getOperations() && !sm.getOperations()->empty()) {
//Estimate the number of requested datapoints, for all sensors' data plotted in the panel. The number is calculated assuming all sensors plotted on the same panel have the same sampling period. If we exceed the maximum, then apply smoothing. //Estimate the number of requested datapoints, for all sensors' data plotted in the panel. The number is calculated assuming all sensors plotted on the same panel have the same sampling period. If we exceed the maximum, then apply smoothing.
uint64_t grafanaInterval = end.getRaw() - start.getRaw(); uint64_t grafanaInterval = end.getRaw() - start.getRaw();
if((int)((grafanaInterval/ps.interval) * sensors.size()) > MAX_DATAPOINTS) { if((int)((grafanaInterval / sInterval) * sensors.size()) > MAX_DATAPOINTS) {
//Choose a smoother //Choose a smoother
if(grafanaInterval/10000000000 < MAX_DATAPOINTS && if(grafanaInterval/10000000000 < MAX_DATAPOINTS && sm.getOperations()->find("-avg10") != sm.getOperations()->end())
ps.operations.find("-avg10") != ps.operations.end())
sensorName += "-avg10"; sensorName += "-avg10";
else if(grafanaInterval/300000000000 < MAX_DATAPOINTS && else if(grafanaInterval/300000000000 < MAX_DATAPOINTS && sm.getOperations()->find("-avg300") != sm.getOperations()->end())
ps.operations.find("-avg300") != ps.operations.end())
sensorName += "-avg300"; sensorName += "-avg300";
else if(ps.operations.find("-avg3600") != ps.operations.end()) else if(sm.getOperations()->find("-avg3600") != sm.getOperations()->end())
//Hourly averages should be fine for intervals of years. //Hourly averages should be fine for intervals of years.
sensorName += "-avg3600"; sensorName += "-avg3600";
} }
...@@ -246,15 +260,18 @@ void RestAPI::POST_query(endpointArgs) { ...@@ -246,15 +260,18 @@ void RestAPI::POST_query(endpointArgs) {
//We need the Sensor ID, since smoothed sensors are not published. //We need the Sensor ID, since smoothed sensors are not published.
DCDB::SensorId sid(sensorName); DCDB::SensorId sid(sensorName);
sid.setRsvd(start.getWeekstamp()); uint16_t startWs=start.getWeekstamp(), endWs=end.getWeekstamp();
//Shoot the query for this sensor. //Shoot the query for this sensor.
sensorDataStore.query(results, sid, start, end, DCDB::AGGREGATE_NONE); for(uint16_t currWs=startWs; currWs<=endWs; currWs++) {
sid.setRsvd(currWs);
_sensorDataStore->query(results, sid, start, end, DCDB::AGGREGATE_NONE);
}
//Format the output for the response to Grafana. //Format the output for the response to Grafana.
std::string datapoints = "["; std::string datapoints = "[";
for (auto& r : results) for (auto& r : results)
datapoints += "[" + std::to_string(r.value * ps.scaling_factor) + "," datapoints += "[" + std::to_string(r.value * sScale) + ","
+ std::to_string(r.timeStamp.getRaw()/1000000) + "],"; + std::to_string(r.timeStamp.getRaw()/1000000) + "],";
if(datapoints.back() == ',') if(datapoints.back() == ',')
datapoints.pop_back(); datapoints.pop_back();
......
...@@ -33,6 +33,7 @@ ...@@ -33,6 +33,7 @@
#include "sensornavigator.h" #include "sensornavigator.h"
#include "cassandra.h" #include "cassandra.h"
#include "mqttchecker.h" #include "mqttchecker.h"
#include "metadatastore.h"
#include "dcdb/connection.h" #include "dcdb/connection.h"
#include "dcdb/timestamp.h" #include "dcdb/timestamp.h"
...@@ -57,16 +58,7 @@ public: ...@@ -57,16 +58,7 @@ public:
hierarchySettings_t hierarchySettings, hierarchySettings_t hierarchySettings,
DCDB::Connection* cassandraConnection); DCDB::Connection* cassandraConnection);
virtual ~RestAPI() { virtual ~RestAPI() { clear(); }
if(_navigator) {
delete _navigator;
_navigator = NULL;
}
if(_sensorConfig) {
delete _sensorConfig;
_sensorConfig = NULL;
}
}
bool initTree(); bool initTree();
...@@ -133,9 +125,14 @@ private: ...@@ -133,9 +125,14 @@ private:
/******************************************************************************/ /******************************************************************************/
//Clears all internal storage
void clear();
SensorNavigator* _navigator; SensorNavigator* _navigator;
DCDB::Connection* _connection; DCDB::Connection* _connection;
DCDB::SensorConfig* _sensorConfig; DCDB::SensorConfig* _sensorConfig;
DCDB::SensorDataStore* _sensorDataStore;
MetadataStore _metadataStore;
std::string _separator; std::string _separator;
hierarchySettings_t _hierarchySettings; hierarchySettings_t _hierarchySettings;
......
...@@ -80,6 +80,22 @@ public: ...@@ -80,6 +80,22 @@ public:
PublicSensor(const PublicSensor &copy); PublicSensor(const PublicSensor &copy);
inline bool operator < (const PublicSensor& rhs) const {return name < rhs.name;} inline bool operator < (const PublicSensor& rhs) const {return name < rhs.name;}
/**
* @brief Converts a SensorMetadata object to a DCDB::PublicSensor one.
*
* @param s SensorMetadata object to be converted
* @return Output PublicSensor object
*/
static PublicSensor metadataToPublicSensor(const SensorMetadata& s);
/**
* @brief Converts a DCDB::PublicSensor object to its SensorMetadata representation.
*
* @param ps The PublicSensor object to be converted
* @return Output SensorMetadata object
*/
static SensorMetadata publicSensorToMetadata(const PublicSensor& ps);
}; };
/** /**
......
...@@ -77,6 +77,53 @@ PublicSensor::PublicSensor (const PublicSensor &copy) ...@@ -77,6 +77,53 @@ PublicSensor::PublicSensor (const PublicSensor &copy)
ttl = copy.ttl; ttl = copy.ttl;
} }
PublicSensor PublicSensor::metadataToPublicSensor(const SensorMetadata& sm) {
PublicSensor ps;
if(sm.getPublicName())
ps.name = *sm.getPublicName();
if(sm.getIsVirtual())
ps.is_virtual = *sm.getIsVirtual();
if(sm.getPattern())
ps.pattern = *sm.getPattern();
if(sm.getUnit())
ps.unit = *sm.getUnit();
if(sm.getScale())
ps.scaling_factor = *sm.getScale();
if(sm.getTTL())
ps.ttl = *sm.getTTL();
if(sm.getInterval())
ps.interval = *sm.getInterval();
if(sm.getOperations())
ps.operations = *sm.getOperations();
uint64_t sensorMask = 0;
if(sm.getIntegrable() && *sm.getIntegrable())
sensorMask = sensorMask | INTEGRABLE;
if(sm.getMonotonic() && *sm.getMonotonic())
sensorMask = sensorMask | MONOTONIC;
ps.sensor_mask = sensorMask;
return ps;
}
SensorMetadata PublicSensor::publicSensorToMetadata(const PublicSensor& ps) {
SensorMetadata sm;
sm.setPublicName(ps.name);
sm.setIsVirtual(ps.is_virtual);
// Stripping whitespace from the sensor pattern in the SID
string stripPattern = ps.pattern;
boost::algorithm::trim(stripPattern);
sm.setPattern(stripPattern);
sm.setUnit(ps.unit);
sm.setScale(ps.scaling_factor);
sm.setTTL(ps.ttl);
sm.setInterval(ps.interval);
sm.setOperations(ps.operations);
sm.setIntegrable(ps.sensor_mask & INTEGRABLE);
sm.setMonotonic(ps.sensor_mask & MONOTONIC);
return sm;
}
/* /*
* SensorConfig functions * SensorConfig functions
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment