Commit a0874ea2 authored by Alessio Netti's avatar Alessio Netti
Browse files

Merge remote-tracking branch 'remotes/origin/mqttTopics'

# Conflicts:
#	dcdbpusher/Makefile
parents 561c39ac 3e683acf
...@@ -262,12 +262,14 @@ void AnalyticsManager::removeTopics(an_dl_t p) { ...@@ -262,12 +262,14 @@ void AnalyticsManager::removeTopics(an_dl_t p) {
MQTTChecker& mqttCheck = MQTTChecker::getInstance(); MQTTChecker& mqttCheck = MQTTChecker::getInstance();
for(const auto& a : p.configurator->getAnalyzers()) { for(const auto& a : p.configurator->getAnalyzers()) {
mqttCheck.removeGroup(a->getName()); mqttCheck.removeGroup(a->getName());
if (a->getStreaming()) if (a->getStreaming()) {
for (const auto &u : a->getUnits()) for (const auto &u : a->getUnits())
for (const auto &o: u->getBaseOutputs()) { for (const auto &o: u->getBaseOutputs()) {
mqttCheck.removeTopic(o->getMqtt()); mqttCheck.removeTopic(o->getMqtt());
mqttCheck.removeName(o->getName()); mqttCheck.removeName(o->getName());
} }
a->releaseUnits();
}
} }
} }
...@@ -277,11 +279,13 @@ bool AnalyticsManager::checkTopics(an_dl_t p) { ...@@ -277,11 +279,13 @@ bool AnalyticsManager::checkTopics(an_dl_t p) {
for(const auto& a : p.configurator->getAnalyzers()) { for(const auto& a : p.configurator->getAnalyzers()) {
if (!mqttCheck.checkGroup(a->getName())) if (!mqttCheck.checkGroup(a->getName()))
validTopics = false; validTopics = false;
if (a->getStreaming()) if (a->getStreaming()) {
for (const auto &u : a->getUnits()) for (const auto &u : a->getUnits())
for (const auto &o: u->getBaseOutputs()) for (const auto &o: u->getBaseOutputs())
if (!mqttCheck.checkTopic(o->getMqtt()) || !mqttCheck.checkName(o->getName())) if (!mqttCheck.checkTopic(o->getMqtt()) || !mqttCheck.checkName(o->getName()))
validTopics = false; validTopics = false;
a->releaseUnits();
}
} }
return validTopics; return validTopics;
} }
...@@ -404,9 +408,10 @@ void AnalyticsManager::GET_analytics_sensors(endpointArgs) { ...@@ -404,9 +408,10 @@ void AnalyticsManager::GET_analytics_sensors(endpointArgs) {
for (const auto& s : u->getBaseOutputs()) { for (const auto& s : u->getBaseOutputs()) {
// Explicitly adding nodes to the ptree as to prevent BOOST from performing // Explicitly adding nodes to the ptree as to prevent BOOST from performing
// parsing on the node names // parsing on the node names
group.push_back(boost::property_tree::ptree::value_type(s->getName(), boost::property_tree::ptree(s->getMqtt()))); group.push_back(boost::property_tree::ptree::value_type("", boost::property_tree::ptree(s->getMqtt())));
} }
} }
a->releaseUnits();
sensors.add_child(a->getName(), group); sensors.add_child(a->getName(), group);
} }
} }
...@@ -418,9 +423,10 @@ void AnalyticsManager::GET_analytics_sensors(endpointArgs) { ...@@ -418,9 +423,10 @@ void AnalyticsManager::GET_analytics_sensors(endpointArgs) {
found = true; found = true;
for (const auto& u : a->getUnits()) { for (const auto& u : a->getUnits()) {
for (const auto& s : u->getBaseOutputs()) { for (const auto& s : u->getBaseOutputs()) {
data << a->getName() << "." << s->getName() << " " << s->getMqtt() << "\n"; data << a->getName() << "::" << s->getMqtt() << "\n";
} }
} }
a->releaseUnits();
} }
} }
} }
...@@ -461,8 +467,9 @@ void AnalyticsManager::GET_analytics_units(endpointArgs) { ...@@ -461,8 +467,9 @@ void AnalyticsManager::GET_analytics_units(endpointArgs) {
found = true; found = true;
boost::property_tree::ptree group; boost::property_tree::ptree group;
for (const auto& u : a->getUnits()) { for (const auto& u : a->getUnits()) {
group.push_back(boost::property_tree::ptree::value_type(u->getName(), boost::property_tree::ptree())); group.push_back(boost::property_tree::ptree::value_type("", boost::property_tree::ptree(u->getName())));
} }
a->releaseUnits();
units.add_child(a->getName(), group); units.add_child(a->getName(), group);
} }
root.add_child(p.id, units); root.add_child(p.id, units);
...@@ -472,8 +479,9 @@ void AnalyticsManager::GET_analytics_units(endpointArgs) { ...@@ -472,8 +479,9 @@ void AnalyticsManager::GET_analytics_units(endpointArgs) {
if (a->getStreaming() && (analyzer == "" || analyzer == a->getName())) { if (a->getStreaming() && (analyzer == "" || analyzer == a->getName())) {
found = true; found = true;
for (const auto& u : a->getUnits()) { for (const auto& u : a->getUnits()) {
data << a->getName() << "." << u->getName() << "\n"; data << a->getName() << "::" << u->getName() << "\n";
} }
a->releaseUnits();
} }
} }
} }
......
...@@ -158,7 +158,9 @@ file. The following is instead a list of configuration parameters that are avail ...@@ -158,7 +158,9 @@ file. The following is instead a list of configuration parameters that are avail
|:----- |:----------- | |:----- |:----------- |
| default | Name of the template that must be used to configure this analyzer. | default | Name of the template that must be used to configure this analyzer.
| interval | Specifies how often the analyzer will be invoked to perform computations, and thus the sampling interval of its output sensors. Only used for analyzers in _streaming_ mode. | interval | Specifies how often the analyzer will be invoked to perform computations, and thus the sampling interval of its output sensors. Only used for analyzers in _streaming_ mode.
| relaxed | If set to _true_, the units of this analyzer will be instantiated even if some of the respective input sensors do not exist.
| delay | Delay in milliseconds to be applied to the start of the analyzer. This parameter only applies to streaming analyzers. It can be used to allow for input sensor caches to be populated before the analyzer is started. | delay | Delay in milliseconds to be applied to the start of the analyzer. This parameter only applies to streaming analyzers. It can be used to allow for input sensor caches to be populated before the analyzer is started.
| unitCacheLimit | Defines the maximum size of the unit cache that is used in the on-demand and job modes. Default is 1000.
| minValues | Minimum number of readings that need to be stored in output sensors before these are pushed as MQTT messages. Only used for analyzers in _streaming_ mode. | minValues | Minimum number of readings that need to be stored in output sensors before these are pushed as MQTT messages. Only used for analyzers in _streaming_ mode.
| mqttPart | Part of the MQTT topic associated to this analyzer. Only used when the Unit system is not employed (see this [section](#mqttTopics)). | mqttPart | Part of the MQTT topic associated to this analyzer. Only used when the Unit system is not employed (see this [section](#mqttTopics)).
| sync | If set to _true_, computation will be performed at time intervals synchronized with sensor readings. | sync | If set to _true_, computation will be performed at time intervals synchronized with sensor readings.
...@@ -182,7 +184,7 @@ streaming true ...@@ -182,7 +184,7 @@ streaming true
average avg1 { average avg1 {
default def1 default def1
mqttPart FF0 mqttPart /avg1
input { input {
sensor col_user sensor col_user
...@@ -191,15 +193,15 @@ mqttPart FF0 ...@@ -191,15 +193,15 @@ mqttPart FF0
output { output {
sensor sum { sensor sum {
mqttsuffix 76 mqttsuffix /sum
} }
sensor max { sensor max {
mqttsuffix 77 mqttsuffix /max
} }
sensor avg { sensor avg {
mqttsuffix 78 mqttsuffix /avg
} }
} }
} }
...@@ -224,7 +226,7 @@ streaming true ...@@ -224,7 +226,7 @@ streaming true
average avg1 { average avg1 {
default def1 default def1
mqttPart FF0 mqttPart /avg1
input { input {
sensor "<bottomup>col_user" sensor "<bottomup>col_user"
...@@ -233,15 +235,15 @@ mqttPart FF0 ...@@ -233,15 +235,15 @@ mqttPart FF0
output { output {
sensor "<bottomup, filter cpu01>sum" { sensor "<bottomup, filter cpu01>sum" {
mqttsuffix 76 mqttsuffix /sum
} }
sensor "<bottomup, filter cpu01>max" { sensor "<bottomup, filter cpu01>max" {
mqttsuffix 77 mqttsuffix /max
} }
sensor "<bottomup, filter cpu01>avg" { sensor "<bottomup, filter cpu01>avg" {
mqttsuffix 78 mqttsuffix /avg
} }
} }
} }
...@@ -315,6 +317,14 @@ the MQTT prefix, analyzer part and sensor suffix that are defined; ...@@ -315,6 +317,14 @@ the MQTT prefix, analyzer part and sensor suffix that are defined;
by concatenating the MQTT prefix associated to the unit (which is defined as _the portion of the MQTT topic shared by all sensors by concatenating the MQTT prefix associated to the unit (which is defined as _the portion of the MQTT topic shared by all sensors
belonging to such unit_) and the sensor suffix. The middle part of the topic is padded accordingly to ensure a fixed length. belonging to such unit_) and the sensor suffix. The middle part of the topic is padded accordingly to ensure a fixed length.
#### Pipelining Analyzers <a name="pipelining"></a>
The inputs and outputs of streaming analyzers can be chained so as to form a processing pipeline. To enable this, users
need to configure analyzers by enabling the _relaxed_ configuration parameter, and by selecting as input the output sensors
of other analyzers. This is necessary as the analyzers are instantiated sequentially at startup, and
the framework cannot infer the correct order of initialization so as to resolve all dependencies transparently.
> NOTE &ensp;&ensp;&ensp;&ensp;&ensp; This feature is not supported when using analyzers in _on demand_ mode.
## Rest API <a name="restApi"></a> ## Rest API <a name="restApi"></a>
DCDBAnalytics provides a REST API that can be used to perform various management operations on the framework. The DCDBAnalytics provides a REST API that can be used to perform various management operations on the framework. The
......
...@@ -41,29 +41,29 @@ void AggregatorAnalyzer::printConfig(LOG_LEVEL ll) { ...@@ -41,29 +41,29 @@ void AggregatorAnalyzer::printConfig(LOG_LEVEL ll) {
AnalyzerTemplate<SensorBase>::printConfig(ll); AnalyzerTemplate<SensorBase>::printConfig(ll);
} }
void AggregatorAnalyzer::compute(int unitID) { void AggregatorAnalyzer::compute(U_Ptr unit) {
switch(_op) { switch(_op) {
case SUM: case SUM:
computeSum(unitID); computeSum(unit);
break; break;
case AVG: case AVG:
computeAvg(unitID); computeAvg(unit);
break; break;
case MIN: case MIN:
computeMin(unitID); computeMin(unit);
break; break;
case MAX: case MAX:
computeMax(unitID); computeMax(unit);
break; break;
default: default:
break; break;
} }
} }
void AggregatorAnalyzer::computeSum(int unitID) { void AggregatorAnalyzer::computeSum(U_Ptr unit) {
int64_t acc=0; int64_t acc=0;
for(const auto& in : _units[unitID]->getInputs()) { for(const auto& in : unit->getInputs()) {
// Getting the most recent values as specified in _window // Getting the most recent values as specified in _window
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer); _buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer);
if(!_buffer || _buffer->empty()) { if(!_buffer || _buffer->empty()) {
...@@ -78,13 +78,13 @@ void AggregatorAnalyzer::computeSum(int unitID) { ...@@ -78,13 +78,13 @@ void AggregatorAnalyzer::computeSum(int unitID) {
reading_t out; reading_t out;
out.timestamp = getTimestamp(); out.timestamp = getTimestamp();
out.value = acc; out.value = acc;
_units[unitID]->getOutputs()[0]->storeReading(out); unit->getOutputs()[0]->storeReading(out);
} }
void AggregatorAnalyzer::computeAvg(int unitID) { void AggregatorAnalyzer::computeAvg(U_Ptr unit) {
int64_t acc=0, ctr=0; int64_t acc=0, ctr=0;
for(const auto& in : _units[unitID]->getInputs()) { for(const auto& in : unit->getInputs()) {
// Getting the most recent values as specified in _window // Getting the most recent values as specified in _window
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer); _buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer);
if(!_buffer || _buffer->empty()) { if(!_buffer || _buffer->empty()) {
...@@ -103,13 +103,13 @@ void AggregatorAnalyzer::computeAvg(int unitID) { ...@@ -103,13 +103,13 @@ void AggregatorAnalyzer::computeAvg(int unitID) {
reading_t out; reading_t out;
out.timestamp = getTimestamp(); out.timestamp = getTimestamp();
out.value = acc; out.value = acc;
_units[unitID]->getOutputs()[0]->storeReading(out); unit->getOutputs()[0]->storeReading(out);
} }
void AggregatorAnalyzer::computeMax(int unitID) { void AggregatorAnalyzer::computeMax(U_Ptr unit) {
int64_t acc=0; int64_t acc=0;
for(const auto& in : _units[unitID]->getInputs()) { for(const auto& in : unit->getInputs()) {
// Getting the most recent values as specified in _window // Getting the most recent values as specified in _window
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer); _buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer);
if(!_buffer || _buffer->empty()) { if(!_buffer || _buffer->empty()) {
...@@ -125,14 +125,14 @@ void AggregatorAnalyzer::computeMax(int unitID) { ...@@ -125,14 +125,14 @@ void AggregatorAnalyzer::computeMax(int unitID) {
reading_t out; reading_t out;
out.timestamp = getTimestamp(); out.timestamp = getTimestamp();
out.value = acc; out.value = acc;
_units[unitID]->getOutputs()[0]->storeReading(out); unit->getOutputs()[0]->storeReading(out);
} }
void AggregatorAnalyzer::computeMin(int unitID) { void AggregatorAnalyzer::computeMin(U_Ptr unit) {
int64_t acc=0; int64_t acc=0;
bool minInit=false; bool minInit=false;
for(const auto& in : _units[unitID]->getInputs()) { for(const auto& in : unit->getInputs()) {
// Getting the most recent values as specified in _window // Getting the most recent values as specified in _window
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer); _buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer);
if(!_buffer || _buffer->empty()) { if(!_buffer || _buffer->empty()) {
...@@ -150,6 +150,6 @@ void AggregatorAnalyzer::computeMin(int unitID) { ...@@ -150,6 +150,6 @@ void AggregatorAnalyzer::computeMin(int unitID) {
reading_t out; reading_t out;
out.timestamp = getTimestamp(); out.timestamp = getTimestamp();
out.value = acc; out.value = acc;
_units[unitID]->getOutputs()[0]->storeReading(out); unit->getOutputs()[0]->storeReading(out);
} }
...@@ -47,13 +47,13 @@ public: ...@@ -47,13 +47,13 @@ public:
private: private:
void compute(int unitID) override; void compute(U_Ptr unit) override;
// A separate method for each operation implies code redundancy, but also better efficiency and less useless // A separate method for each operation implies code redundancy, but also better efficiency and less useless
// variables used by specific operations lying around // variables used by specific operations lying around
void computeSum(int unitID); void computeSum(U_Ptr unit);
void computeAvg(int unitID); void computeAvg(U_Ptr unit);
void computeMax(int unitID); void computeMax(U_Ptr unit);
void computeMin(int unitID); void computeMin(U_Ptr unit);
vector<reading_t> *_buffer = NULL; vector<reading_t> *_buffer = NULL;
unsigned long long _window; unsigned long long _window;
......
global { global {
mqttPrefix /FF112233445566778899AAB mqttPrefix /test
} }
template_aggregator def1 { template_aggregator def1 {
interval 1000 interval 1000
minValues 3 minValues 3
mqttPart FF0
duplicate false duplicate false
streaming true streaming true
} }
aggregator avg1 { aggregator avg1 {
default def1 default def1
mqttPart FF0
window 2000 window 2000
operation sum operation sum
...@@ -27,7 +25,7 @@ operation sum ...@@ -27,7 +25,7 @@ operation sum
output { output {
sensor "<bottomup, filter cpu250>sum" { sensor "<bottomup, filter cpu250>sum" {
mqttsuffix 76 mqttsuffix /sum
} }
} }
...@@ -37,7 +35,6 @@ operation sum ...@@ -37,7 +35,6 @@ operation sum
aggregator avg2 { aggregator avg2 {
default def1 default def1
interval 1500 interval 1500
mqttPart FF1
operation average operation average
input { input {
...@@ -51,7 +48,7 @@ operation average ...@@ -51,7 +48,7 @@ operation average
output { output {
sensor "<bottomup 1>avg" { sensor "<bottomup 1>avg" {
mqttsuffix 78 mqttsuffix /avg
} }
} }
...@@ -61,7 +58,7 @@ operation average ...@@ -61,7 +58,7 @@ operation average
aggregator avg3 { aggregator avg3 {
default def1 default def1
interval 1500 interval 1500
mqttPart FF2 mqttPart /mypart
operation maximum operation maximum
input { input {
...@@ -73,7 +70,7 @@ operation maximum ...@@ -73,7 +70,7 @@ operation maximum
output { output {
sensor "<bottomup 1>maxall" { sensor "<bottomup 1>maxall" {
mqttsuffix 81 mqttsuffix /maxall
} }
} }
......
...@@ -73,10 +73,7 @@ protected: ...@@ -73,10 +73,7 @@ protected:
const string OUTPUT_BLOCK = "output"; const string OUTPUT_BLOCK = "output";
const string ALL_CLAUSE = "all"; const string ALL_CLAUSE = "all";
const string ALL_REC_CLAUSE = "all-recursive"; const string ALL_REC_CLAUSE = "all-recursive";
const std::string SENSOR_PATTERN = "(?i)<sensor>";
const std::string GROUP_PATTERN = "(?i)<group>";
public: public:
/** /**
...@@ -88,7 +85,6 @@ public: ...@@ -88,7 +85,6 @@ public:
_baseName("INVALID"), _baseName("INVALID"),
_cfgPath(""), _cfgPath(""),
_mqttPrefix(""), _mqttPrefix(""),
_sensorPattern(""),
_cacheInterval(900000) {} _cacheInterval(900000) {}
/** /**
...@@ -127,7 +123,6 @@ public: ...@@ -127,7 +123,6 @@ public:
*/ */
virtual void setGlobalSettings(const pluginSettings_t& pluginSettings) final { virtual void setGlobalSettings(const pluginSettings_t& pluginSettings) final {
_mqttPrefix = pluginSettings.mqttPrefix; _mqttPrefix = pluginSettings.mqttPrefix;
_sensorPattern = pluginSettings.sensorPattern;
_cacheInterval = pluginSettings.cacheInterval; _cacheInterval = pluginSettings.cacheInterval;
} }
...@@ -139,8 +134,7 @@ public: ...@@ -139,8 +134,7 @@ public:
void printConfig(LOG_LEVEL ll) final { void printConfig(LOG_LEVEL ll) final {
LOG_VAR(ll) << " General: "; LOG_VAR(ll) << " General: ";
LOG_VAR(ll) << " MQTT-Prefix: " << (_mqttPrefix != "" ? _mqttPrefix : std::string("DEFAULT")); LOG_VAR(ll) << " MQTT-Prefix: " << (_mqttPrefix != "" ? _mqttPrefix : std::string("DEFAULT"));
LOG_VAR(ll) << " Sensor Pattern: " << (_sensorPattern != "" ? _sensorPattern : std::string("DEFAULT")); LOG_VAR(ll) << " CacheInterval: " << _cacheInterval/1000 << " [s]";
LOG_VAR(ll) << " Cache interval: " << _cacheInterval << " ms";
//prints plugin specific configurator attributes and entities if present //prints plugin specific configurator attributes and entities if present
printConfiguratorConfig(ll); printConfiguratorConfig(ll);
...@@ -204,6 +198,7 @@ public: ...@@ -204,6 +198,7 @@ public:
// If the analyzer must be duplicated for each compute unit, we copy-construct identical // If the analyzer must be duplicated for each compute unit, we copy-construct identical
// instances that have different unit IDs // instances that have different unit IDs
unsigned numUnits = an->getUnits().size(); unsigned numUnits = an->getUnits().size();
an->releaseUnits();
if(an->getDuplicate() && numUnits>1) { if(an->getDuplicate() && numUnits>1) {
for(unsigned int i=0; i < numUnits; i++) { for(unsigned int i=0; i < numUnits; i++) {
A_Ptr anCopy = std::make_shared<Analyzer>(*an); A_Ptr anCopy = std::make_shared<Analyzer>(*an);
...@@ -390,6 +385,10 @@ protected: ...@@ -390,6 +385,10 @@ protected:
an.setDelayInterval(stoull(val.second.data()) / 1000); an.setDelayInterval(stoull(val.second.data()) / 1000);
} else if (boost::iequals(val.first, "duplicate")) { } else if (boost::iequals(val.first, "duplicate")) {
an.setDuplicate(to_bool(val.second.data())); an.setDuplicate(to_bool(val.second.data()));
} else if (boost::iequals(val.first, "relaxed")) {
an.setRelaxed(to_bool(val.second.data()));
} else if (boost::iequals(val.first, "unitCacheLimit")) {
an.setUnitCacheLimit(stoull(val.second.data()));
} else if (boost::iequals(val.first, "streaming")) { } else if (boost::iequals(val.first, "streaming")) {
an.setStreaming(to_bool(val.second.data())); an.setStreaming(to_bool(val.second.data()));
} else if (boost::iequals(val.first, INPUT_BLOCK) || boost::iequals(val.first, OUTPUT_BLOCK)) { } else if (boost::iequals(val.first, INPUT_BLOCK) || boost::iequals(val.first, OUTPUT_BLOCK)) {
...@@ -416,51 +415,12 @@ protected: ...@@ -416,51 +415,12 @@ protected:
} }
} }
} }
// Reading all derived attributes, if any // Reading all derived attributes, if any
analyzer(an, config); analyzer(an, config);
// Instantiating units
// Instantiating units and returning the result
if(!an.getTemplate()) { if(!an.getTemplate()) {
vector <shared_ptr<UnitTemplate<SBase>>> *units = NULL; return readUnits(an, protoInputs, protoOutputs, inputMode);
try {
units = _unitGen.generateUnits(protoInputs, protoOutputs, inputMode, _mqttPrefix + an.getMqttPart(), !an.getStreaming());
}
catch (const std::exception &e) {
LOG(error) << _analyzerName << " " << an.getName() << ": Error when creating units: " << e.what();
delete units;
return false;
}