Commit 433a196c authored by Alessio Netti's avatar Alessio Netti
Browse files

Grafana: performance improvements

- Caching metadata information
- Using a single persistent SensorDataStore instance
- Now properly querying multiple weekstamps if necessary
parent 0178119d
......@@ -221,7 +221,7 @@ bool AnalyticsController::rebuildSensorNavigator() {
LOG(error) << "Failed to retrieve public sensors. Sensor Navigator will be empty.";
for (const auto &s : publicSensors)
if (!s.is_virtual) {
sBuf = Configuration::publicSensorToMetadata(s);
sBuf = DCDB::PublicSensor::publicSensorToMetadata(s);
if(sBuf.isValid())
topics.push_back(*sBuf.getPattern());
}
......
......@@ -239,7 +239,7 @@ bool metadataQueryCallback(const string& name, SensorMetadata& buffer) {
--queryEngine.access;
return false;
}
buffer = Configuration::publicSensorToMetadata(publicSensor);
buffer = DCDB::PublicSensor::publicSensorToMetadata(publicSensor);
}
catch (const std::exception &e) {
--queryEngine.access;
......@@ -737,7 +737,7 @@ int main(int argc, char* const argv[]) {
SensorMetadata sBuf;
for (const auto &s : publicSensors)
if (!s.is_virtual) {
sBuf = Configuration::publicSensorToMetadata(s);
sBuf = DCDB::PublicSensor::publicSensorToMetadata(s);
if(sBuf.isValid())
metadataStore->store(*sBuf.getPattern(), sBuf);
}
......
......@@ -73,50 +73,3 @@ void Configuration::readAdditionalBlocks(boost::property_tree::iptree& cfg) {
}
}
}
DCDB::PublicSensor Configuration::metadataToPublicSensor(const SensorMetadata& sm) {
DCDB::PublicSensor ps;
if(sm.getPublicName())
ps.name = *sm.getPublicName();
if(sm.getIsVirtual())
ps.is_virtual = *sm.getIsVirtual();
if(sm.getPattern())
ps.pattern = *sm.getPattern();
if(sm.getUnit())
ps.unit = *sm.getUnit();
if(sm.getScale())
ps.scaling_factor = *sm.getScale();
if(sm.getTTL())
ps.ttl = *sm.getTTL();
if(sm.getInterval())
ps.interval = *sm.getInterval();
if(sm.getOperations())
ps.operations = *sm.getOperations();
uint64_t sensorMask = 0;
if(sm.getIntegrable() && *sm.getIntegrable())
sensorMask = sensorMask | INTEGRABLE;
if(sm.getMonotonic() && *sm.getMonotonic())
sensorMask = sensorMask | MONOTONIC;
ps.sensor_mask = sensorMask;
return ps;
}
SensorMetadata Configuration::publicSensorToMetadata(const DCDB::PublicSensor& ps) {
SensorMetadata sm;
sm.setPublicName(ps.name);
sm.setIsVirtual(ps.is_virtual);
// Stripping whitespace from the sensor pattern in the SID
string stripPattern = ps.pattern;
boost::algorithm::trim(stripPattern);
sm.setPattern(stripPattern);
sm.setUnit(ps.unit);
sm.setScale(ps.scaling_factor);
sm.setTTL(ps.ttl);
sm.setInterval(ps.interval);
sm.setOperations(ps.operations);
sm.setIntegrable(ps.sensor_mask & INTEGRABLE);
sm.setMonotonic(ps.sensor_mask & MONOTONIC);
return sm;
}
......@@ -86,22 +86,6 @@ public:
virtual ~Configuration() {}
/**
* @brief Converts a SensorMetadata object to a DCDB::PublicSensor one.
*
* @param s SensorMetadata object to be converted
* @return Output PublicSensor object
*/
static DCDB::PublicSensor metadataToPublicSensor(const SensorMetadata& s);
/**
* @brief Converts a DCDB::PublicSensor object to its SensorMetadata representation.
*
* @param ps The PublicSensor object to be converted
* @return Output SensorMetadata object
*/
static SensorMetadata publicSensorToMetadata(const DCDB::PublicSensor& ps);
// Additional configuration parameters to be parsed within the global block
std::string mqttListenHost = string(DEFAULT_LISTENHOST);
std::string mqttListenPort = string(DEFAULT_LISTENPORT);
......
......@@ -50,8 +50,7 @@ RestAPI::RestAPI(serverSettings_t settings,
addEndpoint("/query", {http::verb::post, stdBind(POST_query)});
}
//Initializes the internal sensor navigator
bool RestAPI::initTree() {
void RestAPI::clear() {
if(_sensorConfig) {
delete _sensorConfig;
_sensorConfig = NULL;
......@@ -60,19 +59,32 @@ bool RestAPI::initTree() {
delete _navigator;
_navigator = NULL;
}
if(_sensorDataStore) {
delete _sensorDataStore;
_sensorDataStore = NULL;
}
_metadataStore.clear();
}
//Initializes the internal sensor navigator
bool RestAPI::initTree() {
clear();
//Get the list of all public sensors and topics.
std::vector<std::string> topics;
std::list<DCDB::PublicSensor> publicSensors;
_sensorConfig = new DCDB::SensorConfig(_connection);
_sensorDataStore = new DCDB::SensorDataStore(_connection);
if(_sensorConfig->getPublicSensorsVerbose(publicSensors)!=DCDB::SC_OK) {
LOG(error) << "Unable to fetch list of public sensors!";
clear();
return false;
}
for(auto& s : publicSensors) {
topics.push_back(s.pattern);
_metadataStore.store(s.pattern, DCDB::PublicSensor::publicSensorToMetadata(s));
}
//Build the tree navigator
......@@ -213,32 +225,34 @@ void RestAPI::POST_query(endpointArgs) {
//Prepare query
DCDB::TimeStamp start, end;
DCDB::PublicSensor ps;
SensorMetadata sm;
start = DCDB::TimeStamp(startTime, false);
end = DCDB::TimeStamp(endTime, false);
std::list<DCDB::SensorDataStoreReading> results;
for(auto& sensorName : sensors) {
results.clear();
try {
sm = _metadataStore.get(sensorName);
} catch(const std::invalid_argument &e) {}
std::list<DCDB::SensorDataStoreReading> results;
DCDB::SensorDataStore sensorDataStore(_connection);
_sensorConfig->getPublicSensorByName(ps,sensorName.c_str());
uint64_t sInterval = sm.getInterval() ? *sm.getInterval() : 0;
double sScale = sm.getScale() ? *sm.getScale() : 1.0;
if(ps.interval && !ps.operations.empty()) {
if(sInterval && sm.getOperations() && !sm.getOperations()->empty()) {
//Estimate the number of requested datapoints, for all sensors' data plotted in the panel. The number is calculated assuming all sensors plotted on the same panel have the same sampling period. If we exceed the maximum, then apply smoothing.
uint64_t grafanaInterval = end.getRaw() - start.getRaw();
if((int)((grafanaInterval/ps.interval) * sensors.size()) > MAX_DATAPOINTS) {
if((int)((grafanaInterval / sInterval) * sensors.size()) > MAX_DATAPOINTS) {
//Choose a smoother
if(grafanaInterval/10000000000 < MAX_DATAPOINTS &&
ps.operations.find("-avg10") != ps.operations.end())
if(grafanaInterval/10000000000 < MAX_DATAPOINTS && sm.getOperations()->find("-avg10") != sm.getOperations()->end())
sensorName += "-avg10";
else if(grafanaInterval/300000000000 < MAX_DATAPOINTS &&
ps.operations.find("-avg300") != ps.operations.end())
else if(grafanaInterval/300000000000 < MAX_DATAPOINTS && sm.getOperations()->find("-avg300") != sm.getOperations()->end())
sensorName += "-avg300";
else if(ps.operations.find("-avg3600") != ps.operations.end())
else if(sm.getOperations()->find("-avg3600") != sm.getOperations()->end())
//Hourly averages should be fine for intervals of years.
sensorName += "-avg3600";
}
......@@ -246,15 +260,18 @@ void RestAPI::POST_query(endpointArgs) {
//We need the Sensor ID, since smoothed sensors are not published.
DCDB::SensorId sid(sensorName);
sid.setRsvd(start.getWeekstamp());
uint16_t startWs=start.getWeekstamp(), endWs=end.getWeekstamp();
//Shoot the query for this sensor.
sensorDataStore.query(results, sid, start, end, DCDB::AGGREGATE_NONE);
for(uint16_t currWs=startWs; currWs<=endWs; currWs++) {
sid.setRsvd(currWs);
_sensorDataStore->query(results, sid, start, end, DCDB::AGGREGATE_NONE);
}
//Format the output for the response to Grafana.
std::string datapoints = "[";
for (auto& r : results)
datapoints += "[" + std::to_string(r.value * ps.scaling_factor) + ","
datapoints += "[" + std::to_string(r.value * sScale) + ","
+ std::to_string(r.timeStamp.getRaw()/1000000) + "],";
if(datapoints.back() == ',')
datapoints.pop_back();
......
......@@ -33,6 +33,7 @@
#include "sensornavigator.h"
#include "cassandra.h"
#include "mqttchecker.h"
#include "metadatastore.h"
#include "dcdb/connection.h"
#include "dcdb/timestamp.h"
......@@ -57,16 +58,7 @@ public:
hierarchySettings_t hierarchySettings,
DCDB::Connection* cassandraConnection);
virtual ~RestAPI() {
if(_navigator) {
delete _navigator;
_navigator = NULL;
}
if(_sensorConfig) {
delete _sensorConfig;
_sensorConfig = NULL;
}
}
virtual ~RestAPI() { clear(); }
bool initTree();
......@@ -133,9 +125,14 @@ private:
/******************************************************************************/
//Clears all internal storage
void clear();
SensorNavigator* _navigator;
DCDB::Connection* _connection;
DCDB::SensorConfig* _sensorConfig;
DCDB::SensorDataStore* _sensorDataStore;
MetadataStore _metadataStore;
std::string _separator;
hierarchySettings_t _hierarchySettings;
......
......@@ -80,6 +80,22 @@ public:
PublicSensor(const PublicSensor &copy);
inline bool operator < (const PublicSensor& rhs) const {return name < rhs.name;}
/**
* @brief Converts a SensorMetadata object to a DCDB::PublicSensor one.
*
* @param s SensorMetadata object to be converted
* @return Output PublicSensor object
*/
static PublicSensor metadataToPublicSensor(const SensorMetadata& s);
/**
* @brief Converts a DCDB::PublicSensor object to its SensorMetadata representation.
*
* @param ps The PublicSensor object to be converted
* @return Output SensorMetadata object
*/
static SensorMetadata publicSensorToMetadata(const PublicSensor& ps);
};
/**
......
......@@ -77,6 +77,53 @@ PublicSensor::PublicSensor (const PublicSensor &copy)
ttl = copy.ttl;
}
PublicSensor PublicSensor::metadataToPublicSensor(const SensorMetadata& sm) {
PublicSensor ps;
if(sm.getPublicName())
ps.name = *sm.getPublicName();
if(sm.getIsVirtual())
ps.is_virtual = *sm.getIsVirtual();
if(sm.getPattern())
ps.pattern = *sm.getPattern();
if(sm.getUnit())
ps.unit = *sm.getUnit();
if(sm.getScale())
ps.scaling_factor = *sm.getScale();
if(sm.getTTL())
ps.ttl = *sm.getTTL();
if(sm.getInterval())
ps.interval = *sm.getInterval();
if(sm.getOperations())
ps.operations = *sm.getOperations();
uint64_t sensorMask = 0;
if(sm.getIntegrable() && *sm.getIntegrable())
sensorMask = sensorMask | INTEGRABLE;
if(sm.getMonotonic() && *sm.getMonotonic())
sensorMask = sensorMask | MONOTONIC;
ps.sensor_mask = sensorMask;
return ps;
}
SensorMetadata PublicSensor::publicSensorToMetadata(const PublicSensor& ps) {
SensorMetadata sm;
sm.setPublicName(ps.name);
sm.setIsVirtual(ps.is_virtual);
// Stripping whitespace from the sensor pattern in the SID
string stripPattern = ps.pattern;
boost::algorithm::trim(stripPattern);
sm.setPattern(stripPattern);
sm.setUnit(ps.unit);
sm.setScale(ps.scaling_factor);
sm.setTTL(ps.ttl);
sm.setInterval(ps.interval);
sm.setOperations(ps.operations);
sm.setIntegrable(ps.sensor_mask & INTEGRABLE);
sm.setMonotonic(ps.sensor_mask & MONOTONIC);
return sm;
}
/*
* SensorConfig functions
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment