The expiration time for new job artifacts in CI/CD pipelines is now 30 days (GitLab default). Previously generated artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 0e018db4 authored by Daniele Tafani's avatar Daniele Tafani
Browse files

version working with sensor navigator and new dcdb grafana plugin

parent 8e056d72
......@@ -7,6 +7,7 @@
//
#include "Configuration.h"
#include "SensorNavigator.h"
#include <string>
#include <unistd.h>
#include <dlfcn.h>
......@@ -14,6 +15,9 @@
#include <boost/foreach.hpp>
#include <boost/property_tree/info_parser.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/regex.h>
using namespace std;
......@@ -27,6 +31,7 @@ _cfgFilePath(cfgFilePath) {
//set default values for global variables
_global.daemonize = 0;
_global.threads = 1;
//log levels will get inverted...
_global.logLevelFile = boost::log::trivial::fatal;
_global.logLevelCmd = boost::log::trivial::warning;
......@@ -36,6 +41,7 @@ _cfgFilePath(cfgFilePath) {
_global.restAPISettings.restHost = "";
_global.restAPISettings.restPort = "8081";
}
Configuration::~Configuration() {
......@@ -65,6 +71,7 @@ bool Configuration::readGlobal() {
}
} else if (boost::iequals(global.first, "verbosity")) {
_global.logLevelFile = static_cast<boost::log::trivial::severity_level>(stoi(global.second.data()));
} else {
LOG(warning) << " Value \"" << global.first << "\" not recognized. Omitting";
}
......@@ -75,11 +82,21 @@ bool Configuration::readGlobal() {
if (boost::iequals(global.first, "hostname")) {
_global.cassandraSettings.cassandraHost = global.second.data();
}
else if (boost::iequals(global.first, "port")) {
} else if (boost::iequals(global.first, "port")) {
_global.cassandraSettings.cassandraPort = stoi(global.second.data());
} else {
LOG(warning) << " Value \"" << global.first << "\" not recognized. Omitting";
}
else {
}
//read hierarchy struct
BOOST_FOREACH(boost::property_tree::iptree::value_type &global, cfg.get_child("hierarchy")) {
if (boost::iequals(global.first, "separator")) {
_global.hierarchySettings.separator = global.second.data();
} else if (boost::iequals(global.first, "regex")) {
_global.hierarchySettings.regex = global.second.data();
} else {
LOG(warning) << " Value \"" << global.first << "\" not recognized. Omitting";
}
}
......
......@@ -28,6 +28,7 @@ typedef struct {
boost::log::trivial::severity_level logLevelCmd;
restAPISettings_t restAPISettings;
cassandraSettings_t cassandraSettings;
hierarchySettings_t hierarchySettings;
} global_t;
......@@ -74,6 +75,7 @@ private:
global_t _global;
boost::log::sources::severity_logger<boost::log::trivial::severity_level> lg;
boost::property_tree::iptree dataCenterCfg;
};
......
......@@ -13,6 +13,7 @@
#include <boost/archive/iterators/transform_width.hpp>
#include <boost/archive/iterators/insert_linebreaks.hpp>
#include <boost/archive/iterators/remove_whitespace.hpp>
#include <boost/algorithm/string.hpp>
using namespace std;
using namespace web;
......@@ -23,6 +24,7 @@ using namespace http::experimental::listener;
Configuration* _configuration;
DCDB::Connection* cassandraConnection;
HttpsServer* _httpsServer;
SensorNavigator* navigator;
boost::shared_ptr<boost::asio::io_service::work> keepAliveWork;
......@@ -115,7 +117,6 @@ void handler::handle_get(http_request message) {
return;
};
/*
* Handles POST requests.
* Primarily used for querying the Cassandra database (for sensor names and data).
......@@ -125,23 +126,94 @@ void handler::handle_post(http_request message) {
http_response response;
boost::log::sources::severity_logger<boost::log::trivial::severity_level> lg;
LOG(debug) << message.to_string();
LOG(info) << message.to_string();
std::string target = message.relative_uri().to_string();
//This request expects the list of sensors available as a response.
if(message.relative_uri().to_string() == "/search") {
if(target == "/annotations") {
//Get the list of public sensors.
DCDB::SensorConfig sensorConfig(cassandraConnection);
std::list<std::string> publicSensors;
sensorConfig.getPublicSensorNames(publicSensors);
std::string output;
std::cout << "in annotations";
}
//This request expects the list of sensors available as a response.
if(target.find("/search") != std::string::npos) {
//Format the Grafana response message with the list of sensors.
std::string output = "[";
for (std::list<std::string>::iterator it=publicSensors.begin(); it != publicSensors.end(); it++)
output = output + "\"" + (*it) + "\",";
std::string outputElement = "";
std::set<std::string> *treeOutput;
std::string hierarchy = "";
size_t sensorIndex = 0;
//Extract the hierarchy from the url.
if(target == "/search") {
try {
treeOutput = navigator->getNodes(0,false);
}
catch(std::domain_error &e) {
treeOutput->insert("");
}
}
// Ugly: needs to be removed or fixed.
else if(target == "/search/system/sensor") {
treeOutput->insert("");
}
else {
//Get everything after "/search/"
hierarchy = target.substr(8);
//Retrieve the list of nodes or sensors from the sensor navigator.
if(hierarchy.find("/sensor") != std::string::npos) {
//Remove "/sensor"
sensorIndex = hierarchy.find("/sensor");
hierarchy.erase(sensorIndex++, 7);
//Substitute backslashes with dots
boost::replace_all(hierarchy, "/", ".");
hierarchy.append(".");
//Get the sensors in the tree
treeOutput = navigator->getSensors(hierarchy,false);
}
else {
//Substitute backslashes with dots
boost::replace_all(hierarchy, "/", ".");
hierarchy.append(".");
//Get the nodes in the tree
treeOutput = navigator->getNodes(hierarchy,false);
}
}
//Build the output string.
for (set<string>::iterator it = treeOutput->begin(); it != treeOutput->end(); it++) {
//If we are not querying sensor names, remove the hierarchy from each output element.
if(!sensorIndex) {
boost::regex exp(hierarchy);
outputElement = boost::regex_replace((*it),exp,"");
}
else
outputElement = *it;
//Remove the alst dot returned by the sensor navigator output.
if(outputElement.back() == '.')
outputElement.pop_back();
output = output + "\"" + outputElement + "\",";
}
//remove the last comma.
output.pop_back();
output = output + "]";
//close the output string.
output += "]";
std::cout << output;
//Set the body and the status code for the response.
response.set_body(output);
......@@ -339,8 +411,7 @@ boost::log::trivial::severity_level invertLogLevel(boost::log::trivial::severity
* Contructor of the REST server. Establishes the connection with the Cassandra;
* sets certificate, private keys and dh file
*/
HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, boost::asio::io_service& io) {
HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, hierarchySettings_t hierarchySettings, boost::asio::io_service& io) {
// Connect to the Cassandra database server
cassandraConnection = new DCDB::Connection();
......@@ -351,6 +422,33 @@ HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t
LOG(fatal) << "Failed to connect to the Cassandra database.";
else {
LOG(info) << "Retrieving published sensor names and topics...";
//Get the list of public sensors and topics.
DCDB::SensorConfig sensorConfig(cassandraConnection);
std::list<DCDB::PublicSensor> publicSensors;
sensorConfig.getPublicSensorsVerbose(publicSensors);
std::vector<std::string> sensors;
std::vector<std::string> topics;
for (std::list<DCDB::PublicSensor>::iterator it=publicSensors.begin(); it != publicSensors.end(); it++) {
sensors.push_back((*it).name);
topics.push_back((*it).pattern);
}
LOG(info) << "Building the sensor navigator...";
//Build the tree navigator
navigator = new SensorNavigator();
//navigator->buildTree(hierarchySettings.regex, &sensors, &topics, hierarchySettings.separator);
navigator->buildTree("", &sensors, &topics, hierarchySettings.separator);
//Test
//std::set<std::string> *testSensors = navigator->getNodes("mpp3r01c02s01.",false);
std::set<std::string> *testSensors = navigator->getSensors("mpp3.r01.c02.s01.",false);
for (std::set<std::string>::iterator iter=testSensors->begin(); iter != testSensors->end(); iter++) {
std::cout << (*iter) << std::endl;
}
std::shared_ptr<asio::ssl::context> ctx = std::make_shared<asio::ssl::context>(asio::ssl::context::sslv23);
ctx->set_options(asio::ssl::context::default_workarounds | asio::ssl::context::no_sslv3 | asio::ssl::context::single_dh_use);
......@@ -428,6 +526,7 @@ int main(int argc, char *argv[])
global_t& globalSettings = _configuration->getGlobal();
cassandraSettings_t& cassandraSettings = globalSettings.cassandraSettings;
restAPISettings_t& restAPISettings = globalSettings.restAPISettings;
hierarchySettings_t& hierarchySettings = globalSettings.hierarchySettings;
//reset getopt()
optind = 1;
......@@ -527,7 +626,7 @@ int main(int argc, char *argv[])
}
//REST server gets its threads
_httpsServer = new HttpsServer(restAPISettings, cassandraSettings, io);
_httpsServer = new HttpsServer(restAPISettings, cassandraSettings, hierarchySettings, io);
boost::thread restThread(std::bind(&HttpsServer::run, _httpsServer));
......
......@@ -11,6 +11,7 @@
#include "Configuration.h"
#include "Logging.h"
#include "cassandra.h"
#include "SensorNavigator.h"
#ifndef HTTPServer_h
#define HTTPServer_h
......@@ -38,6 +39,7 @@
#include <boost/thread/condition.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/thread.hpp>
#include <boost/regex.hpp>
#include <asio/ssl.hpp>
//Headers from cpprestsdk
......@@ -74,6 +76,7 @@ typedef struct {
std::string certificate;
std::string privateKey;
std::string dhFile;
std::string hierarchy;
} restAPISettings_t;
/**
......@@ -85,6 +88,16 @@ typedef struct {
uint16_t cassandraPort;
} cassandraSettings_t;
/**
* Struct defining the hierarchy Settings.
* It holds the regular expressions defining the different elements of the datacenter.
*/
typedef struct {
std::string separator;
std::string regex;;
} hierarchySettings_t;
/**
* This class implements the behaviour of a request handler of the REST server.
*/
......@@ -168,7 +181,7 @@ class HttpsServer {
public:
HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, boost::asio::io_service& io);
HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, hierarchySettings_t hierarchySettings, boost::asio::io_service& io);
virtual ~HttpsServer();
/*
......
......@@ -3,9 +3,10 @@ include ../config.mk
CXXFLAGS = -O2 -g --std=c++11 -Wall -Wno-unused-local-typedefs -Wno-deprecated-declarations -Wno-unknown-warning-option -fmessage-length=0 -I../include/ -I../lib/include -I$(DCDBDEPLOYPATH)/include -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_LOG_DYN_LINK -I$(DCDBDEPSPATH)/cpp-netlib-0.12.0-final/deps/asio/asio/include -DBOOST_TEST_DYN_LINK
OBJS = HTTPServer.o \
Configuration.o
Configuration.o \
SensorNavigator.o
LIBS = -L$(DCDBDEPLOYPATH)/lib/ -L../lib -ldcdb -lcassandra -lboost_system -lboost_chrono -lboost_thread -lboost_log -lboost_log_setup -lpthread -lcrypto -lssl -lcpprest
LIBS = -L$(DCDBDEPLOYPATH)/lib/ -L../lib -ldcdb -lcassandra -lboost_system -lboost_chrono -lboost_thread -lboost_log -lboost_regex -lboost_log_setup -lpthread -lcrypto -lssl -lcpprest
TARGET = httpserver
.PHONY : clean install
......
//
// Created by Netti, Alessio on 11.12.18.
//
#include "SensorNavigator.h"
const string SensorNavigator::rootKey = "__root__";
const string SensorNavigator::templateKey = "__template__";
const char SensorNavigator::pathSeparator = '.';
void SensorNavigator::clearTree() {
if(_sensorTree) {
_sensorTree->clear();
delete _sensorTree;
_sensorTree = NULL;
}
if(_hierarchy) {
_hierarchy->clear();
delete _hierarchy;
_hierarchy = NULL;
}
_treeDepth = -1;
_usingTopics = false;
}
bool SensorNavigator::nodeExists(const string& node) {
return _sensorTree && _sensorTree->count(node) && !isSensorNode(node);
}
bool SensorNavigator::sensorExists(const string& node) {
return _sensorTree && _sensorTree->count(node) && isSensorNode(node);
}
string SensorNavigator::buildTopicForNode(const string& node, const string& suffix, int len) {
if(!_sensorTree || !_sensorTree->count(node) || isSensorNode(node))
throw domain_error("SensorNavigator: node not found in tree!");
string nodePrefix = getNodeTopic(node);
int cnt = (int)std::count(nodePrefix.begin(), nodePrefix.end(), '/');
if( (int)nodePrefix.length() + (int)suffix.length() - cnt > len)
throw invalid_argument("SensorNavigator: cannot build topic, too many characters!");
return nodePrefix + string(len - nodePrefix.length() - suffix.length() + cnt, 'F') + suffix;
}
bool SensorNavigator::isSensorNode(const string& node) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
return _isSensorNode(_sensorTree->at(node));
}
int SensorNavigator::getNodeDepth(const string& node) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
return _sensorTree->at(node).depth;
}
string SensorNavigator::getNodeTopic(const string& node) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
return _usingTopics ? _sensorTree->at(node).topic : node;
}
void SensorNavigator::buildTree(const string& hierarchy, const vector<string> *sensors, const vector<string> *topics, const string& delimiter) {
//if( hierarchy == "" )
// throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
vector<string> hierarchyVec;
string hBuf = hierarchy;
size_t pos;
if(!hierarchy.empty())
while( !hBuf.empty() ) {
pos = hBuf.find(delimiter);
hierarchyVec.push_back(hBuf.substr(0, pos));
if(pos == std::string::npos)
hBuf.clear();
else
hBuf.erase(0, pos + delimiter.length());
}
buildTree(&hierarchyVec, sensors, topics);
}
void SensorNavigator::buildTree(const vector<string> *hierarchy, const vector<string> *sensors, const vector<string> *topics) {
if(sensors && sensors->size() > 0)
clearTree();
else
throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
bool autoMode = false;
if(!hierarchy || hierarchy->empty()) {
_hierarchy = NULL;
autoMode = true;
}
else {
_hierarchy = new vector<boost::regex>();
string s = "";
//Each level in the hierarchy includes the regular expressions at the upper levels
for (const auto &l: *hierarchy) {
s += l;
_hierarchy->push_back(boost::regex(s));
}
autoMode = false;
}
_usingTopics = topics != NULL;
_sensorTree = new map<string, Node>();
//We initialize the sensor tree by pushing the root supernode
Node rootNode = {-1, set<string>(), set<string>(), "", ""};
_sensorTree->insert(make_pair(rootKey, rootNode));
//We iterate over all sensor names that were supplied and build up the tree
for(int i=0; i < (int)sensors->size(); i++)
if(autoMode)
_addAutoSensor(sensors->at(i), topics ? topics->at(i) : "");
else
_addSensor(sensors->at(i), topics ? topics->at(i) : "");
}
void SensorNavigator::buildCassandraTree(const map<string, vector<string> > *table, const string& root, const string& ignore) {
if(table->size() > 0 && table->count(root))
clearTree();
else
throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
_hierarchy = NULL;
_usingTopics = false;
_sensorTree = new map<string, Node>();
boost::regex ignoreReg(ignore);
//We initialize the sensor tree by pushing the root supernode
Node rootNode = {-1, set<string>(), set<string>(), "", ""};
_sensorTree->insert(make_pair(rootKey, rootNode));
//We manually add the root's children to the tree, so as to map the Cassandra root key to the one we use
for(const auto& s : table->at(root)) {
if(!boost::regex_search(s.c_str(), _match, ignoreReg)) {
Node newNode = {0, set<string>(), set<string>(), rootKey, ""};
_sensorTree->insert(make_pair(s, newNode));
if (table->count(s)) {
_sensorTree->at(rootKey).children.insert(s);
_addCassandraSensor(s, table, 1, ignoreReg);
} else {
//If the entry is a sensor it will share the same depth as its father, hence the -1 decrease
_sensorTree->at(s).depth -= 1;
_sensorTree->at(rootKey).sensors.insert(s);
}
}
}
}
map<string, SensorNavigator::Node> *SensorNavigator::getSubTree(const string& node, int depth) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
if( depth < -1 )
throw out_of_range("SensorNavigator: depth not valid for subtree query!");
//We start by picking the start node, then we call the recursive routine
map<string, SensorNavigator::Node> *m = new map<string, Node>();
m->insert(make_pair(node,_sensorTree->at(node)));
_getSubTree(node, m, depth);
return m;
}
set<string> *SensorNavigator::getNodes(int depth, bool recursive) {
if( depth < -1 || depth > _treeDepth )
throw out_of_range("SensorNavigator: depth not valid for node query!");
if(!_sensorTree)
throw runtime_error("SensorNavigator: sensor tree not initialized!");
//We pick all nodes in the tree whose depth is consistent with the input
//Iterating over the node map is sufficient for this purpose
set<string> *vec = new set<string>();
for(const auto& n : *_sensorTree)
if(!_isSensorNode(n.second) && (n.second.depth == depth || (recursive && n.second.depth >= depth)))
vec->insert(n.first);
return vec;
}
set<string> *SensorNavigator::getNodes(const string& node, bool recursive) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
if(isSensorNode(node))
throw invalid_argument("SensorNavigator: input must be a node, not a sensor!");
//We start by picking all children of the start node, then we proceed in the subtree (if recursive)
set<string> *vec = new set<string>();
_getNodes(node, vec, recursive ? 0 : 1);
return vec;
}
set<string> *SensorNavigator::getSensors(int depth, bool recursive) {
if( depth < -1 || depth > _treeDepth )
throw out_of_range("SensorNavigator: depth not valid for sensor query!");
if(!_sensorTree)
throw runtime_error("SensorNavigator: sensor tree not initialized!");
//Like in getNodes, we iterate over the node map and pick all sensors that are at least at the given input depth
set<string> *vec = new set<string>();
for(const auto& n : *_sensorTree)
if(!_isSensorNode(n.second) && (n.second.depth == depth || (recursive && n.second.depth >= depth)))
vec->insert(n.second.sensors.begin(), n.second.sensors.end());
return vec;
}
set<string> *SensorNavigator::getSensors(const string& node, bool recursive) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
if(isSensorNode(node))
throw invalid_argument("SensorNavigator: input must be a node, not a sensor!");
//We start by picking all sensors of the start node, then we call the recursive routine
set<string> *vec = new set<string>();
_getSensors(node, vec, recursive ? 0 : 1);
return vec;
}
set<string> *SensorNavigator::navigate(const string& node, int direction) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
set<string> *vec = new set<string>();
if(direction == 0)
vec->insert(node);
//if direction is negative, we go up in the tree
else if(direction < 0) {
int ctr = -direction;
string currNode = node;
//We go up the sensor tree in iterative fashion
while (ctr-- > 1 && currNode != rootKey) {
currNode = _sensorTree->at(currNode).parent;
}
vec->insert(_sensorTree->at(currNode).parent);
}
//Else, we go down to the children
else {
_getNodes(node, vec, direction);
}
return vec;
}
void SensorNavigator::_getNodes(const string& node, set<string> *vec, int depth) {
if(!_sensorTree || !_sensorTree->count(node))
return;
if(depth<=1)
vec->insert(_sensorTree->at(node).children.begin(), _sensorTree->at(node).children.end());
//We iterate over all children of the input node, and call the recursive routine over them
//depth=1 implies that we are at the last level of the search; in this case, we do not continue
//an input value of depth<1 implies that the search must proceed for the entire subtree
if(depth!=1)
for(const auto& n : _sensorTree->at(node).children)
_getNodes(n, vec, depth-1);
}
void SensorNavigator::_getSensors(const string& node, set<string> *vec, int depth) {
if(!_sensorTree || !_sensorTree->count(node))
return;
if(depth<=1)
vec->insert(_sensorTree->at(node).sensors.begin(), _sensorTree->at(node).sensors.end());
//We iterate over all children of the input node, and call the recursive routine over them
//Works the same as _getNodes; here, depth does not really refer to the depth level in the tree (as sensors share
//the depth level of the node they belong to) but rather to the search steps performed. A depth of 1 means that
//only the sensors of the original node will be added, 2 means that the sensors of its children will be added, etc.
if(depth!=