Commit f8e58718 authored by Alessio Netti's avatar Alessio Netti

Analytics: tester analyzer plugin

- Test plugin that can be used to evaluate the performance and functionality
of the query engine and unit system
parent 800a0157
......@@ -4,7 +4,7 @@ include ../config.mk
CXXFLAGS += -DBOOST_NETWORK_ENABLE_HTTPS -I../common/include -I$(DCDBDEPLOYPATH)/include -I$(DCDBDEPLOYPATH)/include/opencv4
LIBS = -L../lib -L$(DCDBDEPLOYPATH)/lib/ -ldl -lboost_system -lboost_thread -lboost_log_setup -lboost_log -lboost_regex -lpthread -rdynamic
ANALYZERS = aggregator regressor job_aggregator
ANALYZERS = aggregator regressor job_aggregator testeranalyzer
ifeq ($(OS),Darwin)
BACNET_PORT = bsd
......@@ -51,3 +51,6 @@ libdcdbanalyzer_regressor.$(LIBEXT): analyzers/regressor/RegressorAnalyzer.o ana
libdcdbanalyzer_job_aggregator.$(LIBEXT): analyzers/aggregator/AggregatorAnalyzer.o analyzers/aggregator/JobAggregatorAnalyzer.o analyzers/aggregator/JobAggregatorConfigurator.o ../common/src/sensornavigator.o
$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system -lboost_regex
libdcdbanalyzer_testeranalyzer.$(LIBEXT): analyzers/testeranalyzer/TesterAnalyzer.o analyzers/testeranalyzer/TesterAnalyzerConfigurator.o ../common/src/sensornavigator.o
$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system -lboost_regex
......@@ -21,7 +21,8 @@
1. [Aggregator Plugin](#averagePlugin)
2. [Job Aggregator Plugin](#jobaveragePlugin)
3. [Regressor Plugin](#regressorPlugin)
4. [Writing Plugins](#writingPlugins)
4. [Tester Plugin](#testerPlugin)
5. [Writing Plugins](#writingPlugins)
# Introduction <a name="introduction"></a>
In this Readme we describe the DCDB Data Analytics framework, and all data abstractions that are associated with it.
......@@ -665,6 +666,14 @@ Finally, the Regressor plugin supports the following additional REST API action:
|:----- |:----------- |
| train | Triggers a new training phase for the random forest model. Feature vectors are temporarily collected in-memory until _trainingSamples_ vectors are obtained. Until this moment, the old random forest model is still used to perform prediction.
## Tester Plugin <a name="testerPlugin"></a>
The _Tester_ plugin can be used to test the functionality and performance of the query engine, as well as of the unit system. It will perform a specified number of queries over the set of input sensors for each unit, and then output as a sensor the total number of retrieved readings. The following are the configuration parameters for analyzers in the _Tester_ plugin:
| Value | Explanation |
|:----- |:----------- |
| window | Length in milliseconds of the time window that is used to retrieve recent readings for the input sensors, starting from the latest one.
| queries | Number of queries to be performed at each computation interval. If more than the number of input sensors per unit, these will be looped over multiple times.
| relative | If true, the _relative_ query mode will be used. Otherwise the _absolute_ mode is used.
## Writing DCDB Analytics Plugins <a name="writingPlugins"></a>
Generating a DCDBAnalytics plugin requires implementing a _Analyzer_ and _Configurator_ class which contain all logic
......
//================================================================================
// Name : TesterAnalyzer.cpp
// Author : Alessio Netti
// Copyright : Leibniz Supercomputing Centre
// Description :
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2019-2019 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include "TesterAnalyzer.h"
TesterAnalyzer::TesterAnalyzer(const std::string& name) : AnalyzerTemplate(name) {
_window = 0;
_numQueries = 1;
_buffer = nullptr;
}
TesterAnalyzer::TesterAnalyzer(const TesterAnalyzer& other) : AnalyzerTemplate(other) {
_numQueries = other._numQueries;
_window = other._window;
_buffer = nullptr;
}
TesterAnalyzer::~TesterAnalyzer() {
if(_buffer)
delete _buffer;
}
void TesterAnalyzer::printConfig(LOG_LEVEL ll) {
LOG_VAR(ll) << " Window: " << _window;
LOG_VAR(ll) << " Queries: " << _numQueries;
LOG_VAR(ll) << " Relative mode: " << (_relative ? "enabled" : "disabled");;
AnalyzerTemplate<SensorBase>::printConfig(ll);
}
void TesterAnalyzer::compute(U_Ptr unit) {
uint64_t elCtr=0, queryCtr=0;
reading_t outR;
outR.timestamp = getTimestamp();
// Looping to the desired number of queries
while(queryCtr < _numQueries) {
for (const auto &in : unit->getInputs()) {
// Clearing the buffer, if already allocated
if (_buffer)
_buffer->clear();
if(_relative)
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer, true);
else
_buffer = _queryEngine.querySensor(in->getName(), outR.timestamp-_window-TESTERAN_OFFSET, outR.timestamp-TESTERAN_OFFSET, _buffer, false);
if (!_buffer || _buffer->empty())
throw std::runtime_error("Analyzer " + _name + ": cannot read from sensor " + in->getName() + "!");
elCtr += _buffer->size();
if(++queryCtr >= _numQueries)
break;
}
}
outR.value = (int64_t)elCtr;
unit->getOutputs()[0]->storeReading(outR);
}
//================================================================================
// Name : TesterAnalyzer.h
// Author : Alessio Netti
// Copyright : Leibniz Supercomputing Centre
// Description :
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2019-2019 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#ifndef PROJECT_TESTERANALYZER_H
#define PROJECT_TESTERANALYZER_H
#include "../../includes/AnalyzerTemplate.h"
#include "sensorbase.h"
#define TESTERAN_OFFSET 1000000000
/**
* @brief Tester analyzer plugin.
*
* @ingroup testeranalyzer
*/
class TesterAnalyzer : virtual public AnalyzerTemplate<SensorBase> {
public:
TesterAnalyzer(const std::string& name);
TesterAnalyzer(const TesterAnalyzer& other);
virtual ~TesterAnalyzer();
void setWindow(unsigned long long w) { _window = w; }
void setNumQueries(unsigned long long q) { _numQueries = q;}
void setRelative(bool r) { _relative = r; }
unsigned long long getWindow() { return _window; }
unsigned long long getNumQueries() { return _numQueries; }
bool getRelative() { return _relative; }
void printConfig(LOG_LEVEL ll) override;
protected:
virtual void compute(U_Ptr unit) override;
vector<reading_t> *_buffer;
unsigned long long _window;
unsigned long long _numQueries;
bool _relative;
};
#endif //PROJECT_AGGREGATORANALYZER_H
//================================================================================
// Name : TesterAnalyzerConfigurator.cpp
// Author : Alessio Netti
// Copyright : Leibniz Supercomputing Centre
// Description :
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2019-2019 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include "TesterAnalyzerConfigurator.h"
TesterAnalyzerConfigurator::TesterAnalyzerConfigurator() : AnalyzerConfiguratorTemplate() {
_analyzerName = "analyzer";
_baseName = "sensor";
}
TesterAnalyzerConfigurator::~TesterAnalyzerConfigurator() {}
void TesterAnalyzerConfigurator::sensorBase(SensorBase& s, CFG_VAL config) {}
void TesterAnalyzerConfigurator::analyzer(TesterAnalyzer& a, CFG_VAL config) {
BOOST_FOREACH(boost::property_tree::iptree::value_type &val, config)
{
if (boost::iequals(val.first, "window"))
a.setWindow(stoull(val.second.data()) * 1000000);
else if (boost::iequals(val.first, "queries"))
a.setNumQueries(stoull(val.second.data()));
else if (boost::iequals(val.first, "relative"))
a.setRelative(to_bool(val.second.data()));
}
}
bool TesterAnalyzerConfigurator::unit(UnitTemplate<SensorBase>& u) {
if(u.getOutputs().size()!=1) {
LOG(error) << _analyzerName << ": Only one output sensor per unit is allowed!";
return false;
}
else
return true;
}
//================================================================================
// Name : TesterAnalyzerConfigurator.h
// Author : Alessio Netti
// Copyright : Leibniz Supercomputing Centre
// Description :
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2019-2019 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#ifndef PROJECT_TESTERANALYZERCONFIGURATOR_H
#define PROJECT_TESTERANALYZERCONFIGURATOR_H
#include "../../includes/AnalyzerConfiguratorTemplate.h"
#include "TesterAnalyzer.h"
/**
* @brief Configurator for the tester analyzer plugin.
*
* @ingroup testeranalyzer
*/
class TesterAnalyzerConfigurator : virtual public AnalyzerConfiguratorTemplate<TesterAnalyzer> {
public:
TesterAnalyzerConfigurator();
~TesterAnalyzerConfigurator();
private:
void sensorBase(SensorBase& s, CFG_VAL config) override;
void analyzer(TesterAnalyzer& a, CFG_VAL config) override;
bool unit(UnitTemplate<SensorBase>& u) override;
};
extern "C" AnalyzerConfiguratorInterface* create() {
return new TesterAnalyzerConfigurator;
}
extern "C" void destroy(AnalyzerConfiguratorInterface* c) {
delete c;
}
#endif //PROJECT_TESTERANALYZERCONFIGURATOR_H
global {
mqttPrefix /test
}
template_analyzer def1 {
interval 1000
minValues 3
duplicate false
streaming true
}
analyzer tes1 {
default def1
window 2000
relative false
input {
sensor "<bottomup>col_user"
sensor "<topdown>MemFree"
}
output {
sensor "<bottomup>queries" {
mqttsuffix /queries
}
}
}
analyzer tes2 {
default def1
interval 1500
relative true
input {
sensor "<topdown 1>col_user"
sensor "<bottomup 1>MemFree"
}
output {
sensor "<bottomup 1>queries" {
mqttsuffix /queries
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment