16.12.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit 7142e0f0 authored by Alessio Netti's avatar Alessio Netti
Browse files

Merge remote-tracking branch 'remotes/origin/development'

parents dda07f02 9b82bd9f
...@@ -33,7 +33,7 @@ $ make depsinstall install ...@@ -33,7 +33,7 @@ $ make depsinstall install
$ make depsinstall install doc $ make depsinstall install doc
``` ```
To run DCDB locally with the included default configuration files: If you would like to enable DCDB's advanced caching features, add the -DUSE_SENSOR_CACHE parameter to the CXXFLAGS string in the config.mk file. To run DCDB locally with the included default configuration files:
```bash ```bash
//Change to the DCDB directory you created during installation //Change to the DCDB directory you created during installation
......
...@@ -75,3 +75,8 @@ libdcdboperator_smucngperf.$(LIBEXT): operators/smucngperf/SMUCNGPerfOperator.o ...@@ -75,3 +75,8 @@ libdcdboperator_smucngperf.$(LIBEXT): operators/smucngperf/SMUCNGPerfOperator.o
libdcdboperator_persystsql.$(LIBEXT): operators/persystsql/PerSystSqlOperator.o operators/persystsql/PerSystSqlConfigurator.o operators/persystsql/MariaDB.o ../common/src/sensornavigator.o libdcdboperator_persystsql.$(LIBEXT): operators/persystsql/PerSystSqlOperator.o operators/persystsql/PerSystSqlConfigurator.o operators/persystsql/MariaDB.o ../common/src/sensornavigator.o
$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -L$(DCDBDEPLOYPATH)/lib/mariadb/ -lboost_log -lboost_system -lboost_regex -lmariadb $(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -L$(DCDBDEPLOYPATH)/lib/mariadb/ -lboost_log -lboost_system -lboost_regex -lmariadb
libdcdboperator_coolingcontrol.$(LIBEXT): operators/coolingcontrol/CoolingControlOperator.o operators/coolingcontrol/CoolingControlConfigurator.o operators/coolingcontrol/SNMPController.o ../dcdbpusher/sensors/snmp/SNMPConnection.o ../common/src/sensornavigator.o
$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system -lboost_regex -lnetsnmp -lnetsnmpagent
libdcdboperator_healthchecker.$(LIBEXT): operators/healthchecker/HealthCheckerOperator.o operators/healthchecker/HealthCheckerConfigurator.o ../common/src/sensornavigator.o
$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system -lboost_regex
\ No newline at end of file
...@@ -111,7 +111,7 @@ bool OperatorManager::load(const string& path, const string& globalFile, const p ...@@ -111,7 +111,7 @@ bool OperatorManager::load(const string& path, const string& globalFile, const p
} }
bool OperatorManager::loadPlugin(const string& name, const string& pluginPath, const string& config) { bool OperatorManager::loadPlugin(const string& name, const string& pluginPath, const string& config) {
LOG(info) << "Loading operator plugin \"" << name << "\"..."; LOG(info) << "Loading operator plugin " << name << "...";
std::string pluginConfig; //path to config file for plugin std::string pluginConfig; //path to config file for plugin
std::string pluginLib = "libdcdboperator_" + name; std::string pluginLib = "libdcdboperator_" + name;
#if __APPLE__ #if __APPLE__
...@@ -171,20 +171,20 @@ bool OperatorManager::loadPlugin(const string& name, const string& pluginPath, c ...@@ -171,20 +171,20 @@ bool OperatorManager::loadPlugin(const string& name, const string& pluginPath, c
dynLib.configurator->setGlobalSettings(_pluginSettings); dynLib.configurator->setGlobalSettings(_pluginSettings);
// Read the operator plugin configuration // Read the operator plugin configuration
if (!(dynLib.configurator->readConfig(pluginConfig))) { if (!(dynLib.configurator->readConfig(pluginConfig))) {
LOG(error) << "Plugin \"" << dynLib.id << "\" could not read configuration!"; LOG(error) << "Plugin " << dynLib.id << " could not read configuration!";
return false; return false;
} }
// Returning an empty vector may indicate problems with the config file // Returning an empty vector may indicate problems with the config file
if(dynLib.configurator->getOperators().size() == 0) { if(dynLib.configurator->getOperators().size() == 0) {
LOG(warning) << "Plugin \"" << dynLib.id << "\" created no operators!"; LOG(warning) << "Plugin " << dynLib.id << " created no operators!";
} else if(!checkTopics(dynLib)) { } else if(!checkTopics(dynLib)) {
LOG(error) << "Problematic MQTT topics or sensor names, please check your config files!"; LOG(error) << "Problematic MQTT topics or sensor names, please check your config files!";
return false; return false;
} }
//save dl-struct //save dl-struct
_plugins.push_back(dynLib); _plugins.push_back(dynLib);
LOG(info) << "Plugin \"" << dynLib.id << "\" loaded!"; LOG(info) << "Plugin " << dynLib.id << " " << dynLib.configurator->getVersion() << " loaded!";
} else { } else {
LOG(info) << pluginConfig << " not found. Omitting"; LOG(info) << pluginConfig << " not found. Omitting";
return false; return false;
...@@ -230,7 +230,7 @@ bool OperatorManager::init(boost::asio::io_service& io, const string& plugin) { ...@@ -230,7 +230,7 @@ bool OperatorManager::init(boost::asio::io_service& io, const string& plugin) {
//Actions always affect either one or all plugins, and always all operators within said plugin //Actions always affect either one or all plugins, and always all operators within said plugin
if(plugin=="" || plugin==p.id) { if(plugin=="" || plugin==p.id) {
out = true; out = true;
LOG(info) << "Init \"" << p.id << "\" operator plugin"; LOG(info) << "Init " << p.id << " operator plugin";
for (const auto &op : p.configurator->getOperators()) for (const auto &op : p.configurator->getOperators())
op->init(io); op->init(io);
} }
...@@ -245,7 +245,7 @@ bool OperatorManager::reload(boost::asio::io_service& io, const string& plugin) ...@@ -245,7 +245,7 @@ bool OperatorManager::reload(boost::asio::io_service& io, const string& plugin)
bool out=false; bool out=false;
for (const auto &p : _plugins) for (const auto &p : _plugins)
if(plugin=="" || plugin==p.id) { if(plugin=="" || plugin==p.id) {
LOG(info) << "Reload \"" << p.id << "\" operator plugin"; LOG(info) << "Reload " << p.id << " operator plugin";
out = true; out = true;
//Removing obsolete MQTT topics //Removing obsolete MQTT topics
removeTopics(p); removeTopics(p);
...@@ -304,7 +304,7 @@ bool OperatorManager::start(const string& plugin, const string& operatorN) { ...@@ -304,7 +304,7 @@ bool OperatorManager::start(const string& plugin, const string& operatorN) {
bool out=false; bool out=false;
for (const auto &p : _plugins) for (const auto &p : _plugins)
if(plugin=="" || plugin==p.id) { if(plugin=="" || plugin==p.id) {
LOG(info) << "Start \"" << p.id << "\" operator plugin"; LOG(info) << "Start " << p.id << " operator plugin";
for (const auto &op : p.configurator->getOperators()) for (const auto &op : p.configurator->getOperators())
// Only streaming operators can be started // Only streaming operators can be started
if(op->getStreaming() && (operatorN=="" || operatorN==op->getName())) { if(op->getStreaming() && (operatorN=="" || operatorN==op->getName())) {
...@@ -323,7 +323,7 @@ bool OperatorManager::stop(const string& plugin, const string& operatorN) { ...@@ -323,7 +323,7 @@ bool OperatorManager::stop(const string& plugin, const string& operatorN) {
bool out=false; bool out=false;
for (const auto &p : _plugins) for (const auto &p : _plugins)
if(plugin=="" || plugin==p.id) { if(plugin=="" || plugin==p.id) {
LOG(info) << "Stop \"" << p.id << "\" operator plugin"; LOG(info) << "Stop " << p.id << " operator plugin";
for (const auto &op : p.configurator->getOperators()) for (const auto &op : p.configurator->getOperators())
// Only streaming operators can be stopped // Only streaming operators can be stopped
if(op->getStreaming() && (operatorN=="" || operatorN==op->getName())) { if(op->getStreaming() && (operatorN=="" || operatorN==op->getName())) {
......
...@@ -29,6 +29,8 @@ ...@@ -29,6 +29,8 @@
7. [Tester Plugin](#testerPlugin) 7. [Tester Plugin](#testerPlugin)
4. [Sink Plugins](#sinkplugins) 4. [Sink Plugins](#sinkplugins)
1. [File Sink Plugin](#filesinkPlugin) 1. [File Sink Plugin](#filesinkPlugin)
2. [Cooling Control Plugin](#coolingcontrolPlugin)
3. [Health Checker Plugin](#healthcheckerPlugin)
5. [Writing Plugins](#writingPlugins) 5. [Writing Plugins](#writingPlugins)
### Additional Resources ### Additional Resources
...@@ -804,6 +806,7 @@ The configuration parameters specific to the _Aggregator_ plugin are the followi ...@@ -804,6 +806,7 @@ The configuration parameters specific to the _Aggregator_ plugin are the followi
| Value | Explanation | | Value | Explanation |
|:----- |:----------- | |:----- |:----------- |
| window | Length in milliseconds of the time window that is used to retrieve recent readings for the input sensors, starting from the latest one. | window | Length in milliseconds of the time window that is used to retrieve recent readings for the input sensors, starting from the latest one.
| goBack | Value in milliseconds that can be used to shift back the aggregation window. Useful when very recent sensor data is not always available.
Additionally, output sensors in operators of the _Aggregator_ plugin accept the following parameters: Additionally, output sensors in operators of the _Aggregator_ plugin accept the following parameters:
...@@ -844,7 +847,7 @@ Additionally, output sensors in operators of the _Smoothing_ plugin accept the f ...@@ -844,7 +847,7 @@ Additionally, output sensors in operators of the _Smoothing_ plugin accept the f
## Regressor Plugin <a name="regressorPlugin"></a> ## Regressor Plugin <a name="regressorPlugin"></a>
The _Regressor_ plugin is able to perform regression of sensors, by using _random forest_ machine learning predictors. The algorithmic backend is provided by the OpenCV library. The _Regressor_ plugin is able to perform regression of sensors, by using _random forest_ machine learning predictors. The algorithmic backend is provided by the OpenCV library.
For each input sensor in a certain unit, statistical features from recent values of its time series are computed. These are the average, standard deviation, sum of differences, 25th quantile and 75th quantile. These statistical features are then combined together in a single _feature vector_. For each input sensor in a certain unit, statistical features from recent values of its time series are computed. These are, in this specific order, the average, standard deviation, sum of differences, 25th quantile, 75th quantile and latest value. These statistical features are then combined together in a single _feature vector_.
In order to operate correctly, the models used by the regressor plugin need to be trained: this procedure is performed automatically online when using the _streaming_ mode, and can be triggered arbitrarily over the REST API. In _on demand_ mode, automatic training cannot be performed, and as such a pre-trained model must be loaded from a file. In order to operate correctly, the models used by the regressor plugin need to be trained: this procedure is performed automatically online when using the _streaming_ mode, and can be triggered arbitrarily over the REST API. In _on demand_ mode, automatic training cannot be performed, and as such a pre-trained model must be loaded from a file.
The following are the configuration parameters available for the _Regressor_ plugin: The following are the configuration parameters available for the _Regressor_ plugin:
...@@ -854,6 +857,7 @@ The following are the configuration parameters available for the _Regressor_ plu ...@@ -854,6 +857,7 @@ The following are the configuration parameters available for the _Regressor_ plu
| window | Length in milliseconds of the time window that is used to retrieve recent readings for the input sensors, starting from the latest one. | window | Length in milliseconds of the time window that is used to retrieve recent readings for the input sensors, starting from the latest one.
| trainingSamples | Number of samples necessary to perform training of the current model. | trainingSamples | Number of samples necessary to perform training of the current model.
| targetDistance | Temporal distance (in terms of lags) of the sample that is to be predicted. | targetDistance | Temporal distance (in terms of lags) of the sample that is to be predicted.
| smoothResponses | If false, the regressor will attempt to predict one single sensor reading as specified by the _targetDistance_ parameter. If true, it will instead predict the _average_ of the upcoming _targetDistance_ sensor readings.
| inputPath | Path of a file from which a pre-trained random forest model must be loaded. | inputPath | Path of a file from which a pre-trained random forest model must be loaded.
| outputPath | Path of a file to which the random forest model trained at runtime must be saved. | outputPath | Path of a file to which the random forest model trained at runtime must be saved.
| getImportances | If true, the random forest will also compute feature importance values when trained, which are printed. | getImportances | If true, the random forest will also compute feature importance values when trained, which are printed.
...@@ -884,6 +888,21 @@ The _Classifier_ plugin, as the name implies, performs machine learning classifi ...@@ -884,6 +888,21 @@ The _Classifier_ plugin, as the name implies, performs machine learning classifi
Regressor plugin, the target sensor is always excluded from the feature vectors. Regressor plugin, the target sensor is always excluded from the feature vectors.
* The _targetDistance_ parameter has a default value of 0. It can be set to higher values to perform predictive classification. * The _targetDistance_ parameter has a default value of 0. It can be set to higher values to perform predictive classification.
### Managing Models with the Classifier and Regressor Plugins <a name="classifierPluginModels"></a>
The _Classifier_ and _Regressor_ Wintermute plugins support saving models generated via online training, as well as loading pre-trained models from files.
The models to be loaded to not need to necessarily come from the Wintermute plugins themselves: the OpenCV pipeline is fully supported, and users can load _RTrees_ models
that have been generated and trained offline using the Python front-end. However, users need to make sure that the loaded models are _compatible_ with the specification in the corresponding Wintermute configuration.
The following aspects should be considered:
* The trained model must have the same number of inputs as the input specification in the Wintermute configuration file. Hence, the two models must use the same number of sensors and the same number of features (see _rawMode_ parameter).
* Currently, only models with a single output are supported.
* Both the scale and numerical format of the inputs and outputs should be taken into consideration. DCDB and Wintermute only support integers, hence this should be taken into account when training the model.
* Wintermute respects the order of sensors specified in the configuration files to build feature vectors. These are divided in consecutive blocks, each containing all features for a given sensor (again, see _rawMode_ parameter). In the case of classifier models, the _target_ sensor is never included in the feature vectors and is always skipped.
* If using a custom data processing step with different features from those available in the Wintermute plugins, it is advisable to set the _rawMode_ parameter to true and perform the necessary data processing in a separate plugin, constructing a pipeline.
* When loading a model and thus online training is not required, there is no need to specify a _target_ sensor.
## Clustering Plugin <a name="clusteringPlugin"></a> ## Clustering Plugin <a name="clusteringPlugin"></a>
The _Clustering_ plugin implements a gaussian mixture model for performance variation analysis and outlier detection. The plugin is based on the OpenCV library, similarly to the _Regressor_ plugin. The _Clustering_ plugin implements a gaussian mixture model for performance variation analysis and outlier detection. The plugin is based on the OpenCV library, similarly to the _Regressor_ plugin.
...@@ -964,6 +983,70 @@ Additionally, input sensors in sinks accept the following parameters: ...@@ -964,6 +983,70 @@ Additionally, input sensors in sinks accept the following parameters:
|:----- |:----------- | |:----- |:----------- |
| path | The path to which the sensors's readings should be written. It is interpreted as described above for the _autoName_ attribute. | path | The path to which the sensors's readings should be written. It is interpreted as described above for the _autoName_ attribute.
## Cooling Control Plugin <a name="coolingcontrolPlugin"></a>
The _Cooling Control_ plugin enables proactive control of the inlet temperature in a warm-water cooling system using the SNMP protocol. Its logic is simple: at each computation interval, operators fetch as input temperature readings for a set of sensors, as specified in the configuration.
Then, for each sensor, an operator establishes if the associated temperature is above a certain _hotThreshold_ value, in which case the sensor is flagged as _hot_. If the percentage of hot sensors is higher than a _hotPercentage_ value, the operator proceeds to decrease the inlet temperature. Otherwise, it increases it.
The plugin provides two different possible cooling strategies:
* _continuous_: at each computation interval the operator sets a new inlet temperature value. The more sensors are flagged as hot in comparison to _hotPercentage_, the steeper the decrease in temperature - conversely, the increase in temperature will be stronger if just a minimal amount of nodes is flagged as hot.
* _stepped_: this strategy functions similarly to _continuous_, with a more conservative approach. The main difference is that the space of possible inlet temperature settings is not continuous, but rather separated in evenly-spaced bins: the new inlet temperature setting is applied via SNMP only upon crossing boundaries between bins, and otherwise it is simply stored locally.
Operators in this plugin cannot have any output sensors. The operators themselves provide the following configuration parameters:
| Value | Explanation |
|:----- |:----------- |
| window | Length in milliseconds of the time window that is used to query temperature sensors. In order for a sensor to be flagged as _hot_, all of the readings in the time window must be above _hotThreshold_. Defaults to 0.
| hotPercentage | Percentage of sensors (from 0 to 100) to be flagged as _hot_ in order to trigger a decrease in the inlet temperature. Defaults to 20.
| minTemperature | Minimum allowed value to be set as inlet temperature. The scale must be appropriate for the corresponding SNMP sensor. Defaults to 350.
| maxTemperature | Maximum allowed value to be set as inlet temperature. Defaults to 450.
| bins | Number of bins in which the (_minTemperature_, _maxTemperature_)range is divided when using the _stepped_ strategy. Defaults to 4.
| strategy | Cooling strategy to be used. Can either be _continuous_ or _stepped_. Defaults to _stepped_.
| OIDPrefix | OID prefix of the SNMP sensor that can be used to control the inlet temperature.
| OIDSuffix | OID suffix of the SNMP sensor that can be used to control the inlet temperature.
Cooling Control operators additionally support all of the configuration parameters available in the _dcdbpusher_ SNMP plugin in order to set up connections to SNMP hosts. In particular, these are _Host_, _Community_, _Version_, _Username_, _SecLevel_, _AuthProto_, _PrivProto_, _AuthKey_ and _PrivKey_.
Sensors in the Cooling Control plugin support the following parameters:
| Value | Explanation |
|:----- |:----------- |
| hotThreshold | Threshold value for the sensor to be considered _hot_. It must have the same scale as the readings of the sensor itself. Defaults to 70.
| critThreshold | Additional threshold value higher than _hotThreshold_. If a component reaches this threshold it is considered to be in a critical state (e.g., subject to thermal throttling) and hence a decrease in inlet temperature is triggered regardless of the current number of _hot_ nodes. Disabled by default.
Finally, the plugin supports the following REST API actions:
| Action | Explanation |
|:----- |:----------- |
| status | Displays the temperature settings currently in use, as well as the cooling strategy.
## Health Checker Plugin <a name="healthcheckerPlugin"></a>
The _Health Checker_ plugin allows to monitor sensors and raise alarms if anomalous conditions are detected - these are threshold-based, and can be defined on a per-sensor basis.
Whenever an alarm is raised, the plugin can execute arbitrary scripts and programs to respond accordingly: for example, this can be used to send automatic emails to system administrators. The following is an example of alarm produced by the Health Checker plugin:
```
[11:45:55] <warning>: The following alarm conditions were detected by the DCDB Health Checker plugin:
- Sensor /system/node1/power is not providing any data.
- Sensor /system/node1/temp has a reading greater than threshold 95000.
```
Operators in this plugin cannot have any output sensors. The operators themselves provide the following configuration parameters:
| Value | Explanation |
|:----- |:----------- |
| window | Length in milliseconds of the time window that is used to query sensors. Defaults to 0.
| command | Command to be executed when an alarm is raised. This must contain the _%s_ marker, which is replaced at runtime with a descriptive text of the current alarm. The command is executed in a shell environment (similarly to _popen_) and thus can contain most typical shell constructs (e.g., pipes or re-directs). Default is none.
| log | Boolean. If true, whenever an alarm is raised the event is written to the standard DCDB log on top of being transmitted to the external command. Default is true.
| cooldown | Length in milliseconds of a _cooldown_ time window, in which a given alarm cannot be raised again a second time. Defaults to 0.
Sensors in the Health Checker plugin support the following parameters:
| Value | Explanation |
|:----- |:----------- |
| condition | Condition at which an alarm is raised for a given sensor, with respect to a certain threshold. This can be _above_ (readings are greater than the threshold), _below_ (readings are smaller than the threshold), _equals_ (readings are equal to the threshold) or _exists_ (sensor data must be available at all times). Only one condition can be defined per sensor.
| threshold | Numerical value that is used to verify the condition explained by the _condition_ parameter.
# Writing Wintermute Plugins <a name="writingPlugins"></a> # Writing Wintermute Plugins <a name="writingPlugins"></a>
Generating a DCDB Wintermute plugin requires implementing a _Operator_ and _Configurator_ class which contain all logic Generating a DCDB Wintermute plugin requires implementing a _Operator_ and _Configurator_ class which contain all logic
tied to the specific plugin. Such classes should be derived from _OperatorTemplate_ and _OperatorConfiguratorTemplate_ tied to the specific plugin. Such classes should be derived from _OperatorTemplate_ and _OperatorConfiguratorTemplate_
......
template_controller def1 {
interval 1000
streaming true
Version 2c
Host 127.0.0.1
Community Private
OIDPrefix 1.3.6.1.4.1.1000.15
}
controller c1 {
default def1
OIDSuffix .2.1.0
strategy stepped
maxTemperature 450
minTemperature 350
bins 5
window 0
hotPercentage 20
input {
sensor "<bottomup 1, filter esb>temp" {
hotThreshold 700
}
}
}
template_healthchecker def1 {
shell /bin/sh
log true
cooldown 3600000
}
healthchecker h1 {
default def1
command "/usr/dir/myscript.sh %s"
window 60000
input {
sensor "<bottomup, filter socket>temp" {
condition above
threshold 95000
}
sensor "<bottomup 1>power" {
condition exists
}
}
}
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "version.h"
#include "QueryEngine.h" #include "QueryEngine.h"
#include "OperatorInterface.h" #include "OperatorInterface.h"
...@@ -115,6 +116,15 @@ public: ...@@ -115,6 +116,15 @@ public:
*/ */
virtual void printConfig(LOG_LEVEL ll) = 0; virtual void printConfig(LOG_LEVEL ll) = 0;
/**
* @brief Get current DCDB version.
*
* @return Version number as string.
*/
std::string getVersion() {
return std::string(VERSION);
}
protected: protected:
// Logger object // Logger object
......
...@@ -66,6 +66,8 @@ void AggregatorConfigurator::operatorAttributes(AggregatorOperator& op, CFG_VAL ...@@ -66,6 +66,8 @@ void AggregatorConfigurator::operatorAttributes(AggregatorOperator& op, CFG_VAL
{ {
if (boost::iequals(val.first, "window")) if (boost::iequals(val.first, "window"))
op.setWindow(stoull(val.second.data()) * 1000000); op.setWindow(stoull(val.second.data()) * 1000000);
else if (boost::iequals(val.first, "goBack"))
op.setGoBack(stoull(val.second.data()) * 1000000);
else if (boost::iequals(val.first, "relative")) else if (boost::iequals(val.first, "relative"))
op.setRelative(to_bool(val.second.data())); op.setRelative(to_bool(val.second.data()));
} }
......
...@@ -30,11 +30,13 @@ ...@@ -30,11 +30,13 @@
AggregatorOperator::AggregatorOperator(const std::string& name) : OperatorTemplate(name) { AggregatorOperator::AggregatorOperator(const std::string& name) : OperatorTemplate(name) {
_window = 0; _window = 0;
_goBack = 0;
_relative = true; _relative = true;
} }
AggregatorOperator::AggregatorOperator(const AggregatorOperator& other) : OperatorTemplate(other) { AggregatorOperator::AggregatorOperator(const AggregatorOperator& other) : OperatorTemplate(other) {
_window = other._window; _window = other._window;
_goBack = other._goBack;
_relative = other._relative; _relative = other._relative;
} }
...@@ -42,14 +44,15 @@ AggregatorOperator::~AggregatorOperator() {} ...@@ -42,14 +44,15 @@ AggregatorOperator::~AggregatorOperator() {}
void AggregatorOperator::printConfig(LOG_LEVEL ll) { void AggregatorOperator::printConfig(LOG_LEVEL ll) {
LOG_VAR(ll) << " Window: " << _window; LOG_VAR(ll) << " Window: " << _window;
LOG_VAR(ll) << " Go Back: " << _goBack;
LOG_VAR(ll) << " Relative mode: " << (_relative ? "enabled" : "disabled"); LOG_VAR(ll) << " Relative mode: " << (_relative ? "enabled" : "disabled");
OperatorTemplate<AggregatorSensorBase>::printConfig(ll); OperatorTemplate<AggregatorSensorBase>::printConfig(ll);
} }
void AggregatorOperator::compute(U_Ptr unit) { void AggregatorOperator::compute(U_Ptr unit) {
uint64_t startTs=0, endTs=0, now=getTimestamp(); uint64_t startTs=0, endTs=0, now=getTimestamp();
startTs = _relative ? _window : now - _window; startTs = _relative ? (_window + _goBack) : (now - _window - _goBack);
endTs = _relative ? 0 : now; endTs = _relative ? (_goBack) : (now - _goBack);
// Clearing the buffer // Clearing the buffer
_buffer.clear(); _buffer.clear();
std::vector<std::string> sensorNames; std::vector<std::string> sensorNames;
...@@ -68,7 +71,7 @@ void AggregatorOperator::compute_internal(U_Ptr unit, vector<reading_t>& buffer) ...@@ -68,7 +71,7 @@ void AggregatorOperator::compute_internal(U_Ptr unit, vector<reading_t>& buffer)
_percentiles.clear(); _percentiles.clear();
reading_t reading; reading_t reading;
AggregatorSensorBase::aggregationOps_t op; AggregatorSensorBase::aggregationOps_t op;
reading.timestamp = getTimestamp(); reading.timestamp = getTimestamp() - _goBack;
// Performing the actual aggregation operation // Performing the actual aggregation operation
for(const auto& out : unit->getOutputs()) { for(const auto& out : unit->getOutputs()) {
op = out->getOperation(); op = out->getOperation();
......
...@@ -48,9 +48,11 @@ public: ...@@ -48,9 +48,11 @@ public:
virtual ~AggregatorOperator(); virtual ~AggregatorOperator();
void setWindow(unsigned long long w) { _window = w; } void setWindow(unsigned long long w) { _window = w; }
void setGoBack(unsigned long long g) { _goBack = g; }
void setRelative(bool r) { _relative = r; } void setRelative(bool r) { _relative = r; }
unsigned long long getWindow() { return _window; } unsigned long long getWindow() { return _window; }
unsigned long long getGoBack() { return _goBack; }
bool getRelative() { return _relative; } bool getRelative() { return _relative; }
void printConfig(LOG_LEVEL ll) override; void printConfig(LOG_LEVEL ll) override;
...@@ -66,6 +68,7 @@ protected: ...@@ -66,6 +68,7 @@ protected:
vector<size_t> _percentiles; vector<size_t> _percentiles;
vector<int64_t> _percentileResult; vector<int64_t> _percentileResult;
unsigned long long _window; unsigned long long _window;
unsigned long long _goBack;
bool _relative; bool _relative;
}; };
......
...@@ -66,6 +66,8 @@ void JobAggregatorConfigurator::operatorAttributes(JobAggregatorOperator& op, CF ...@@ -66,6 +66,8 @@ void JobAggregatorConfigurator::operatorAttributes(JobAggregatorOperator& op, CF
{ {
if (boost::iequals(val.first, "window")) if (boost::iequals(val.first, "window"))
op.setWindow(stoull(val.second.data()) * 1000000); op.setWindow(stoull(val.second.data()) * 1000000);
else if (boost::iequals(val.first, "goBack"))
op.setGoBack(stoull(val.second.data()) * 1000000);
} }
} }
......
...@@ -37,7 +37,11 @@ JobAggregatorOperator::JobAggregatorOperator(const JobAggregatorOperator& other) ...@@ -37,7 +37,11 @@ JobAggregatorOperator::JobAggregatorOperator(const JobAggregatorOperator& other)
JobAggregatorOperator::~JobAggregatorOperator() {} JobAggregatorOperator::~JobAggregatorOperator() {}
void JobAggregatorOperator::compute(U_Ptr unit, qeJobData& jobData) { void JobAggregatorOperator::compute(U_Ptr unit, qeJobData& jobData) {
uint64_t now = getTimestamp(); uint64_t now = getTimestamp() - _goBack;
// Too early to fetch job data
if(now < jobData.startTime) {
return;
}
// Making sure that the aggregation boundaries do not go past the job start/end time // Making sure that the aggregation boundaries do not go past the job start/end time
uint64_t jobEnd = jobData.endTime!=0 && now > jobData.endTime ? jobData.endTime : now; uint64_t jobEnd = jobData.endTime!=0 && now > jobData.endTime ? jobData.endTime : now;
uint64_t jobStart = jobEnd-_window < jobData.startTime ? jobData.startTime : jobEnd-_window; uint64_t jobStart = jobEnd-_window < jobData.startTime ? jobData.startTime : jobEnd-_window;
......
//================================================================================
// Name : CoolingControlConfigurator.cpp
// Author : Alessio Netti
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description :
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2019-2019 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include "CoolingControlConfigurator.h"
CoolingControlConfigurator::CoolingControlConfigurator() : OperatorConfiguratorTemplate() {
_operatorName = "controller";
_baseName = "sensor";
}
CoolingControlConfigurator::~CoolingControlConfigurator() {}
void CoolingControlConfigurator::sensorBase(CoolingControlSensorBase& s, CFG_VAL config) {
BOOST_FOREACH(boost::property_tree::iptree::value_type &val, config)
{
if (boost::iequals(val.first, "hotThreshold")) {
s.setHotThreshold(std::stoull(val.second.data()));
} else if (boost::iequals(val.first, "critThreshold")) {
s.setCriticalThreshold(std::stoull(val.second.data()));
}
}
}
void CoolingControlConfigurator::operatorAttributes(CoolingControlOperator& op, CFG_VAL config) {
BOOST_FOREACH(boost::property_tree::iptree::value_type &val, config)
{
if (boost::iequals(val.first, "OIDSuffix"))
op.getController().setOIDSuffix(val.second.data());
else if (boost::iequals(val.first, "OIDPrefix"))
op.getController().setOIDPrefix(val.second.data());
else if (boost::iequals(val.first, "Community"))
op.getController().setSNMPCommunity(val.second.data());
else if (boost::iequals(val.first, "Version"))
op.getController().setVersion(val.second.data());
else if (boost::iequals(val.first, "Host"))
op.getController().setHost(val.second.data());
else if (boost::iequals(val.first, "Username"))
op.getController().setUsername(val.second.data());
else if (boost::iequals(val.first, "SecLevel"))
op.getController().setSecurityLevel(val.second.data());
else if (boost::iequals(val.first, "AuthProto"))
op.getController().setAuthProto(val.second.data());
else if (boost::iequals(val.first, "PrivProto"))
op.getController().setPrivProto(val.second.data());
else if (boost::iequals(val.first, "AuthKey"))
op.getController().setAuthKey(val.second.data());
else if (boost::iequals(val.first, "PrivKey"))
op.getController().setPrivKey(val.second.data());
else if (boost::iequals(val.first, "maxTemperature"))
op.setMaxTemp(std::stoull(val.second.data()));
else if