Commit 0e018db4 authored by Daniele Tafani's avatar Daniele Tafani
Browse files

version working with sensor navigator and new dcdb grafana plugin

parent 8e056d72
......@@ -7,6 +7,7 @@
//
#include "Configuration.h"
#include "SensorNavigator.h"
#include <string>
#include <unistd.h>
#include <dlfcn.h>
......@@ -14,6 +15,9 @@
#include <boost/foreach.hpp>
#include <boost/property_tree/info_parser.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/regex.h>
using namespace std;
......@@ -27,6 +31,7 @@ _cfgFilePath(cfgFilePath) {
//set default values for global variables
_global.daemonize = 0;
_global.threads = 1;
//log levels will get inverted...
_global.logLevelFile = boost::log::trivial::fatal;
_global.logLevelCmd = boost::log::trivial::warning;
......@@ -36,6 +41,7 @@ _cfgFilePath(cfgFilePath) {
_global.restAPISettings.restHost = "";
_global.restAPISettings.restPort = "8081";
}
Configuration::~Configuration() {
......@@ -65,6 +71,7 @@ bool Configuration::readGlobal() {
}
} else if (boost::iequals(global.first, "verbosity")) {
_global.logLevelFile = static_cast<boost::log::trivial::severity_level>(stoi(global.second.data()));
} else {
LOG(warning) << " Value \"" << global.first << "\" not recognized. Omitting";
}
......@@ -75,11 +82,21 @@ bool Configuration::readGlobal() {
if (boost::iequals(global.first, "hostname")) {
_global.cassandraSettings.cassandraHost = global.second.data();
}
else if (boost::iequals(global.first, "port")) {
} else if (boost::iequals(global.first, "port")) {
_global.cassandraSettings.cassandraPort = stoi(global.second.data());
} else {
LOG(warning) << " Value \"" << global.first << "\" not recognized. Omitting";
}
else {
}
//read hierarchy struct
BOOST_FOREACH(boost::property_tree::iptree::value_type &global, cfg.get_child("hierarchy")) {
if (boost::iequals(global.first, "separator")) {
_global.hierarchySettings.separator = global.second.data();
} else if (boost::iequals(global.first, "regex")) {
_global.hierarchySettings.regex = global.second.data();
} else {
LOG(warning) << " Value \"" << global.first << "\" not recognized. Omitting";
}
}
......
......@@ -28,6 +28,7 @@ typedef struct {
boost::log::trivial::severity_level logLevelCmd;
restAPISettings_t restAPISettings;
cassandraSettings_t cassandraSettings;
hierarchySettings_t hierarchySettings;
} global_t;
......@@ -74,6 +75,7 @@ private:
global_t _global;
boost::log::sources::severity_logger<boost::log::trivial::severity_level> lg;
boost::property_tree::iptree dataCenterCfg;
};
......
......@@ -13,6 +13,7 @@
#include <boost/archive/iterators/transform_width.hpp>
#include <boost/archive/iterators/insert_linebreaks.hpp>
#include <boost/archive/iterators/remove_whitespace.hpp>
#include <boost/algorithm/string.hpp>
using namespace std;
using namespace web;
......@@ -23,6 +24,7 @@ using namespace http::experimental::listener;
Configuration* _configuration;
DCDB::Connection* cassandraConnection;
HttpsServer* _httpsServer;
SensorNavigator* navigator;
boost::shared_ptr<boost::asio::io_service::work> keepAliveWork;
......@@ -115,7 +117,6 @@ void handler::handle_get(http_request message) {
return;
};
/*
* Handles POST requests.
* Primarily used for querying the Cassandra database (for sensor names and data).
......@@ -125,23 +126,94 @@ void handler::handle_post(http_request message) {
http_response response;
boost::log::sources::severity_logger<boost::log::trivial::severity_level> lg;
LOG(debug) << message.to_string();
LOG(info) << message.to_string();
std::string target = message.relative_uri().to_string();
//This request expects the list of sensors available as a response.
if(message.relative_uri().to_string() == "/search") {
if(target == "/annotations") {
//Get the list of public sensors.
DCDB::SensorConfig sensorConfig(cassandraConnection);
std::list<std::string> publicSensors;
sensorConfig.getPublicSensorNames(publicSensors);
std::string output;
std::cout << "in annotations";
}
//This request expects the list of sensors available as a response.
if(target.find("/search") != std::string::npos) {
//Format the Grafana response message with the list of sensors.
std::string output = "[";
for (std::list<std::string>::iterator it=publicSensors.begin(); it != publicSensors.end(); it++)
output = output + "\"" + (*it) + "\",";
std::string outputElement = "";
std::set<std::string> *treeOutput;
std::string hierarchy = "";
size_t sensorIndex = 0;
//Extract the hierarchy from the url.
if(target == "/search") {
try {
treeOutput = navigator->getNodes(0,false);
}
catch(std::domain_error &e) {
treeOutput->insert("");
}
}
// Ugly: needs to be removed or fixed.
else if(target == "/search/system/sensor") {
treeOutput->insert("");
}
else {
//Get everything after "/search/"
hierarchy = target.substr(8);
//Retrieve the list of nodes or sensors from the sensor navigator.
if(hierarchy.find("/sensor") != std::string::npos) {
//Remove "/sensor"
sensorIndex = hierarchy.find("/sensor");
hierarchy.erase(sensorIndex++, 7);
//Substitute backslashes with dots
boost::replace_all(hierarchy, "/", ".");
hierarchy.append(".");
//Get the sensors in the tree
treeOutput = navigator->getSensors(hierarchy,false);
}
else {
//Substitute backslashes with dots
boost::replace_all(hierarchy, "/", ".");
hierarchy.append(".");
//Get the nodes in the tree
treeOutput = navigator->getNodes(hierarchy,false);
}
}
//Build the output string.
for (set<string>::iterator it = treeOutput->begin(); it != treeOutput->end(); it++) {
//If we are not querying sensor names, remove the hierarchy from each output element.
if(!sensorIndex) {
boost::regex exp(hierarchy);
outputElement = boost::regex_replace((*it),exp,"");
}
else
outputElement = *it;
//Remove the alst dot returned by the sensor navigator output.
if(outputElement.back() == '.')
outputElement.pop_back();
output = output + "\"" + outputElement + "\",";
}
//remove the last comma.
output.pop_back();
output = output + "]";
//close the output string.
output += "]";
std::cout << output;
//Set the body and the status code for the response.
response.set_body(output);
......@@ -339,8 +411,7 @@ boost::log::trivial::severity_level invertLogLevel(boost::log::trivial::severity
* Contructor of the REST server. Establishes the connection with the Cassandra;
* sets certificate, private keys and dh file
*/
HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, boost::asio::io_service& io) {
HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, hierarchySettings_t hierarchySettings, boost::asio::io_service& io) {
// Connect to the Cassandra database server
cassandraConnection = new DCDB::Connection();
......@@ -351,6 +422,33 @@ HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t
LOG(fatal) << "Failed to connect to the Cassandra database.";
else {
LOG(info) << "Retrieving published sensor names and topics...";
//Get the list of public sensors and topics.
DCDB::SensorConfig sensorConfig(cassandraConnection);
std::list<DCDB::PublicSensor> publicSensors;
sensorConfig.getPublicSensorsVerbose(publicSensors);
std::vector<std::string> sensors;
std::vector<std::string> topics;
for (std::list<DCDB::PublicSensor>::iterator it=publicSensors.begin(); it != publicSensors.end(); it++) {
sensors.push_back((*it).name);
topics.push_back((*it).pattern);
}
LOG(info) << "Building the sensor navigator...";
//Build the tree navigator
navigator = new SensorNavigator();
//navigator->buildTree(hierarchySettings.regex, &sensors, &topics, hierarchySettings.separator);
navigator->buildTree("", &sensors, &topics, hierarchySettings.separator);
//Test
//std::set<std::string> *testSensors = navigator->getNodes("mpp3r01c02s01.",false);
std::set<std::string> *testSensors = navigator->getSensors("mpp3.r01.c02.s01.",false);
for (std::set<std::string>::iterator iter=testSensors->begin(); iter != testSensors->end(); iter++) {
std::cout << (*iter) << std::endl;
}
std::shared_ptr<asio::ssl::context> ctx = std::make_shared<asio::ssl::context>(asio::ssl::context::sslv23);
ctx->set_options(asio::ssl::context::default_workarounds | asio::ssl::context::no_sslv3 | asio::ssl::context::single_dh_use);
......@@ -428,6 +526,7 @@ int main(int argc, char *argv[])
global_t& globalSettings = _configuration->getGlobal();
cassandraSettings_t& cassandraSettings = globalSettings.cassandraSettings;
restAPISettings_t& restAPISettings = globalSettings.restAPISettings;
hierarchySettings_t& hierarchySettings = globalSettings.hierarchySettings;
//reset getopt()
optind = 1;
......@@ -527,7 +626,7 @@ int main(int argc, char *argv[])
}
//REST server gets its threads
_httpsServer = new HttpsServer(restAPISettings, cassandraSettings, io);
_httpsServer = new HttpsServer(restAPISettings, cassandraSettings, hierarchySettings, io);
boost::thread restThread(std::bind(&HttpsServer::run, _httpsServer));
......
......@@ -11,6 +11,7 @@
#include "Configuration.h"
#include "Logging.h"
#include "cassandra.h"
#include "SensorNavigator.h"
#ifndef HTTPServer_h
#define HTTPServer_h
......@@ -38,6 +39,7 @@
#include <boost/thread/condition.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/thread/thread.hpp>
#include <boost/regex.hpp>
#include <asio/ssl.hpp>
//Headers from cpprestsdk
......@@ -74,6 +76,7 @@ typedef struct {
std::string certificate;
std::string privateKey;
std::string dhFile;
std::string hierarchy;
} restAPISettings_t;
/**
......@@ -85,6 +88,16 @@ typedef struct {
uint16_t cassandraPort;
} cassandraSettings_t;
/**
* Struct defining the hierarchy Settings.
* It holds the regular expressions defining the different elements of the datacenter.
*/
typedef struct {
std::string separator;
std::string regex;;
} hierarchySettings_t;
/**
* This class implements the behaviour of a request handler of the REST server.
*/
......@@ -168,7 +181,7 @@ class HttpsServer {
public:
HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, boost::asio::io_service& io);
HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, hierarchySettings_t hierarchySettings, boost::asio::io_service& io);
virtual ~HttpsServer();
/*
......
......@@ -3,9 +3,10 @@ include ../config.mk
CXXFLAGS = -O2 -g --std=c++11 -Wall -Wno-unused-local-typedefs -Wno-deprecated-declarations -Wno-unknown-warning-option -fmessage-length=0 -I../include/ -I../lib/include -I$(DCDBDEPLOYPATH)/include -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_LOG_DYN_LINK -I$(DCDBDEPSPATH)/cpp-netlib-0.12.0-final/deps/asio/asio/include -DBOOST_TEST_DYN_LINK
OBJS = HTTPServer.o \
Configuration.o
Configuration.o \
SensorNavigator.o
LIBS = -L$(DCDBDEPLOYPATH)/lib/ -L../lib -ldcdb -lcassandra -lboost_system -lboost_chrono -lboost_thread -lboost_log -lboost_log_setup -lpthread -lcrypto -lssl -lcpprest
LIBS = -L$(DCDBDEPLOYPATH)/lib/ -L../lib -ldcdb -lcassandra -lboost_system -lboost_chrono -lboost_thread -lboost_log -lboost_regex -lboost_log_setup -lpthread -lcrypto -lssl -lcpprest
TARGET = httpserver
.PHONY : clean install
......
//
// Created by Netti, Alessio on 11.12.18.
//
#include "SensorNavigator.h"
const string SensorNavigator::rootKey = "__root__";
const string SensorNavigator::templateKey = "__template__";
const char SensorNavigator::pathSeparator = '.';
void SensorNavigator::clearTree() {
if(_sensorTree) {
_sensorTree->clear();
delete _sensorTree;
_sensorTree = NULL;
}
if(_hierarchy) {
_hierarchy->clear();
delete _hierarchy;
_hierarchy = NULL;
}
_treeDepth = -1;
_usingTopics = false;
}
bool SensorNavigator::nodeExists(const string& node) {
return _sensorTree && _sensorTree->count(node) && !isSensorNode(node);
}
bool SensorNavigator::sensorExists(const string& node) {
return _sensorTree && _sensorTree->count(node) && isSensorNode(node);
}
string SensorNavigator::buildTopicForNode(const string& node, const string& suffix, int len) {
if(!_sensorTree || !_sensorTree->count(node) || isSensorNode(node))
throw domain_error("SensorNavigator: node not found in tree!");
string nodePrefix = getNodeTopic(node);
int cnt = (int)std::count(nodePrefix.begin(), nodePrefix.end(), '/');
if( (int)nodePrefix.length() + (int)suffix.length() - cnt > len)
throw invalid_argument("SensorNavigator: cannot build topic, too many characters!");
return nodePrefix + string(len - nodePrefix.length() - suffix.length() + cnt, 'F') + suffix;
}
bool SensorNavigator::isSensorNode(const string& node) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
return _isSensorNode(_sensorTree->at(node));
}
int SensorNavigator::getNodeDepth(const string& node) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
return _sensorTree->at(node).depth;
}
string SensorNavigator::getNodeTopic(const string& node) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
return _usingTopics ? _sensorTree->at(node).topic : node;
}
void SensorNavigator::buildTree(const string& hierarchy, const vector<string> *sensors, const vector<string> *topics, const string& delimiter) {
//if( hierarchy == "" )
// throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
vector<string> hierarchyVec;
string hBuf = hierarchy;
size_t pos;
if(!hierarchy.empty())
while( !hBuf.empty() ) {
pos = hBuf.find(delimiter);
hierarchyVec.push_back(hBuf.substr(0, pos));
if(pos == std::string::npos)
hBuf.clear();
else
hBuf.erase(0, pos + delimiter.length());
}
buildTree(&hierarchyVec, sensors, topics);
}
void SensorNavigator::buildTree(const vector<string> *hierarchy, const vector<string> *sensors, const vector<string> *topics) {
if(sensors && sensors->size() > 0)
clearTree();
else
throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
bool autoMode = false;
if(!hierarchy || hierarchy->empty()) {
_hierarchy = NULL;
autoMode = true;
}
else {
_hierarchy = new vector<boost::regex>();
string s = "";
//Each level in the hierarchy includes the regular expressions at the upper levels
for (const auto &l: *hierarchy) {
s += l;
_hierarchy->push_back(boost::regex(s));
}
autoMode = false;
}
_usingTopics = topics != NULL;
_sensorTree = new map<string, Node>();
//We initialize the sensor tree by pushing the root supernode
Node rootNode = {-1, set<string>(), set<string>(), "", ""};
_sensorTree->insert(make_pair(rootKey, rootNode));
//We iterate over all sensor names that were supplied and build up the tree
for(int i=0; i < (int)sensors->size(); i++)
if(autoMode)
_addAutoSensor(sensors->at(i), topics ? topics->at(i) : "");
else
_addSensor(sensors->at(i), topics ? topics->at(i) : "");
}
void SensorNavigator::buildCassandraTree(const map<string, vector<string> > *table, const string& root, const string& ignore) {
if(table->size() > 0 && table->count(root))
clearTree();
else
throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
_hierarchy = NULL;
_usingTopics = false;
_sensorTree = new map<string, Node>();
boost::regex ignoreReg(ignore);
//We initialize the sensor tree by pushing the root supernode
Node rootNode = {-1, set<string>(), set<string>(), "", ""};
_sensorTree->insert(make_pair(rootKey, rootNode));
//We manually add the root's children to the tree, so as to map the Cassandra root key to the one we use
for(const auto& s : table->at(root)) {
if(!boost::regex_search(s.c_str(), _match, ignoreReg)) {
Node newNode = {0, set<string>(), set<string>(), rootKey, ""};
_sensorTree->insert(make_pair(s, newNode));
if (table->count(s)) {
_sensorTree->at(rootKey).children.insert(s);
_addCassandraSensor(s, table, 1, ignoreReg);
} else {
//If the entry is a sensor it will share the same depth as its father, hence the -1 decrease
_sensorTree->at(s).depth -= 1;
_sensorTree->at(rootKey).sensors.insert(s);
}
}
}
}
map<string, SensorNavigator::Node> *SensorNavigator::getSubTree(const string& node, int depth) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
if( depth < -1 )
throw out_of_range("SensorNavigator: depth not valid for subtree query!");
//We start by picking the start node, then we call the recursive routine
map<string, SensorNavigator::Node> *m = new map<string, Node>();
m->insert(make_pair(node,_sensorTree->at(node)));
_getSubTree(node, m, depth);
return m;
}
set<string> *SensorNavigator::getNodes(int depth, bool recursive) {
if( depth < -1 || depth > _treeDepth )
throw out_of_range("SensorNavigator: depth not valid for node query!");
if(!_sensorTree)
throw runtime_error("SensorNavigator: sensor tree not initialized!");
//We pick all nodes in the tree whose depth is consistent with the input
//Iterating over the node map is sufficient for this purpose
set<string> *vec = new set<string>();
for(const auto& n : *_sensorTree)
if(!_isSensorNode(n.second) && (n.second.depth == depth || (recursive && n.second.depth >= depth)))
vec->insert(n.first);
return vec;
}
set<string> *SensorNavigator::getNodes(const string& node, bool recursive) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
if(isSensorNode(node))
throw invalid_argument("SensorNavigator: input must be a node, not a sensor!");
//We start by picking all children of the start node, then we proceed in the subtree (if recursive)
set<string> *vec = new set<string>();
_getNodes(node, vec, recursive ? 0 : 1);
return vec;
}
set<string> *SensorNavigator::getSensors(int depth, bool recursive) {
if( depth < -1 || depth > _treeDepth )
throw out_of_range("SensorNavigator: depth not valid for sensor query!");
if(!_sensorTree)
throw runtime_error("SensorNavigator: sensor tree not initialized!");
//Like in getNodes, we iterate over the node map and pick all sensors that are at least at the given input depth
set<string> *vec = new set<string>();
for(const auto& n : *_sensorTree)
if(!_isSensorNode(n.second) && (n.second.depth == depth || (recursive && n.second.depth >= depth)))
vec->insert(n.second.sensors.begin(), n.second.sensors.end());
return vec;
}
set<string> *SensorNavigator::getSensors(const string& node, bool recursive) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
if(isSensorNode(node))
throw invalid_argument("SensorNavigator: input must be a node, not a sensor!");
//We start by picking all sensors of the start node, then we call the recursive routine
set<string> *vec = new set<string>();
_getSensors(node, vec, recursive ? 0 : 1);
return vec;
}
set<string> *SensorNavigator::navigate(const string& node, int direction) {
if(!_sensorTree || !_sensorTree->count(node))
throw domain_error("SensorNavigator: node not found in tree!");
set<string> *vec = new set<string>();
if(direction == 0)
vec->insert(node);
//if direction is negative, we go up in the tree
else if(direction < 0) {
int ctr = -direction;
string currNode = node;
//We go up the sensor tree in iterative fashion
while (ctr-- > 1 && currNode != rootKey) {
currNode = _sensorTree->at(currNode).parent;
}
vec->insert(_sensorTree->at(currNode).parent);
}
//Else, we go down to the children
else {
_getNodes(node, vec, direction);
}
return vec;
}
void SensorNavigator::_getNodes(const string& node, set<string> *vec, int depth) {
if(!_sensorTree || !_sensorTree->count(node))
return;
if(depth<=1)
vec->insert(_sensorTree->at(node).children.begin(), _sensorTree->at(node).children.end());
//We iterate over all children of the input node, and call the recursive routine over them
//depth=1 implies that we are at the last level of the search; in this case, we do not continue
//an input value of depth<1 implies that the search must proceed for the entire subtree
if(depth!=1)
for(const auto& n : _sensorTree->at(node).children)
_getNodes(n, vec, depth-1);
}
void SensorNavigator::_getSensors(const string& node, set<string> *vec, int depth) {
if(!_sensorTree || !_sensorTree->count(node))
return;
if(depth<=1)
vec->insert(_sensorTree->at(node).sensors.begin(), _sensorTree->at(node).sensors.end());
//We iterate over all children of the input node, and call the recursive routine over them
//Works the same as _getNodes; here, depth does not really refer to the depth level in the tree (as sensors share
//the depth level of the node they belong to) but rather to the search steps performed. A depth of 1 means that
//only the sensors of the original node will be added, 2 means that the sensors of its children will be added, etc.