version working with sensor navigator and new dcdb grafana plugin

HTTPServer/Configuration.cpp

@@ -7,6 +7,7 @@
 //
 
 #include "Configuration.h"
+#include "SensorNavigator.h"
 #include <string>
 #include <unistd.h>
 #include <dlfcn.h>
@@ -14,6 +15,9 @@
 #include <boost/foreach.hpp>
 #include <boost/property_tree/info_parser.hpp>
 #include <boost/algorithm/string.hpp>
+#include <boost/regex.h>
+
+
 
 using namespace std;
 
@@ -27,6 +31,7 @@ _cfgFilePath(cfgFilePath) {
     //set default values for global variables
     _global.daemonize = 0;
     _global.threads       = 1;
+
     //log levels will get inverted...
     _global.logLevelFile = boost::log::trivial::fatal;
     _global.logLevelCmd  = boost::log::trivial::warning;
@@ -36,6 +41,7 @@ _cfgFilePath(cfgFilePath) {
     
     _global.restAPISettings.restHost    = "";
     _global.restAPISettings.restPort    = "8081";
+    
 }
 
 Configuration::~Configuration() {
@@ -65,6 +71,7 @@ bool Configuration::readGlobal() {
             }
         } else if (boost::iequals(global.first, "verbosity")) {
             _global.logLevelFile = static_cast<boost::log::trivial::severity_level>(stoi(global.second.data()));
+        
         } else {
             LOG(warning) << "  Value \"" << global.first << "\" not recognized. Omitting";
         }
@@ -75,11 +82,21 @@ bool Configuration::readGlobal() {
         
         if (boost::iequals(global.first, "hostname")) {
             _global.cassandraSettings.cassandraHost = global.second.data();
-        }
-        else if (boost::iequals(global.first, "port")) {
+        } else if (boost::iequals(global.first, "port")) {
             _global.cassandraSettings.cassandraPort = stoi(global.second.data());
+        } else {
+            LOG(warning) << "  Value \"" << global.first << "\" not recognized. Omitting";
         }
-        else {
+    }
+    
+    //read hierarchy struct
+    BOOST_FOREACH(boost::property_tree::iptree::value_type &global, cfg.get_child("hierarchy")) {
+        
+        if (boost::iequals(global.first, "separator")) {
+            _global.hierarchySettings.separator = global.second.data();
+        } else if (boost::iequals(global.first, "regex")) {
+            _global.hierarchySettings.regex = global.second.data();
+        } else {
             LOG(warning) << "  Value \"" << global.first << "\" not recognized. Omitting";
         }
     }

HTTPServer/Configuration.h

@@ -28,6 +28,7 @@ typedef struct {
     boost::log::trivial::severity_level logLevelCmd;
     restAPISettings_t restAPISettings;
     cassandraSettings_t cassandraSettings;
+    hierarchySettings_t hierarchySettings;
 } global_t;
 
 
@@ -74,6 +75,7 @@ private:
     global_t     _global;
     
     boost::log::sources::severity_logger<boost::log::trivial::severity_level> lg;
+    boost::property_tree::iptree dataCenterCfg;
 };
 
 

HTTPServer/HTTPServer.cpp

@@ -13,6 +13,7 @@
 #include <boost/archive/iterators/transform_width.hpp>
 #include <boost/archive/iterators/insert_linebreaks.hpp>
 #include <boost/archive/iterators/remove_whitespace.hpp>
+#include <boost/algorithm/string.hpp>
 
 using namespace std;
 using namespace web;
@@ -23,6 +24,7 @@ using namespace http::experimental::listener;
 Configuration*    _configuration;
 DCDB::Connection* cassandraConnection;
 HttpsServer*    _httpsServer;
+SensorNavigator*    navigator;
 
 boost::shared_ptr<boost::asio::io_service::work> keepAliveWork;
 
@@ -115,7 +117,6 @@ void handler::handle_get(http_request message) {
     return;
 };
 
-
 /*
  * Handles POST requests.
  * Primarily used for querying the Cassandra database (for sensor names and data).
@@ -125,23 +126,94 @@ void handler::handle_post(http_request message) {
     http_response response;
     boost::log::sources::severity_logger<boost::log::trivial::severity_level> lg;
     
-    LOG(debug) <<  message.to_string();
+    LOG(info) <<  message.to_string();
+    std::string target = message.relative_uri().to_string();
     
-    //This request expects the list of sensors available as a response.
-    if(message.relative_uri().to_string() == "/search") {
+    if(target == "/annotations") {
         
-        //Get the list of public sensors.
-        DCDB::SensorConfig sensorConfig(cassandraConnection);
-        std::list<std::string> publicSensors;
-        sensorConfig.getPublicSensorNames(publicSensors);
+        std::string output;
+        std::cout << "in annotations";
+    }
+    
+    //This request expects the list of sensors available as a response.
+    if(target.find("/search") != std::string::npos) {
         
-        //Format the Grafana response message with the list of sensors.
         std::string output = "[";
-        for (std::list<std::string>::iterator it=publicSensors.begin(); it != publicSensors.end(); it++)
-            output = output + "\"" + (*it) + "\",";
-
+        std::string outputElement = "";
+        std::set<std::string> *treeOutput;
+        std::string hierarchy = "";
+        size_t sensorIndex = 0;
+        
+        //Extract the hierarchy from the url.
+        if(target == "/search") {
+            try {
+                treeOutput = navigator->getNodes(0,false);
+            }
+            catch(std::domain_error &e) {
+                treeOutput->insert("");
+            }
+        }
+        
+        // Ugly: needs to be removed or fixed.
+        else if(target == "/search/system/sensor") {
+                treeOutput->insert("");
+        }
+        else {
+            
+            //Get everything after "/search/"
+            hierarchy = target.substr(8);
+            
+            //Retrieve the list of nodes or sensors from the sensor navigator.
+            if(hierarchy.find("/sensor") != std::string::npos) {
+                
+                //Remove "/sensor"
+                sensorIndex = hierarchy.find("/sensor");
+                hierarchy.erase(sensorIndex++, 7);
+                
+                //Substitute backslashes with dots
+                boost::replace_all(hierarchy, "/", ".");
+                hierarchy.append(".");
+                
+                //Get the sensors in the tree
+                treeOutput = navigator->getSensors(hierarchy,false);
+            }
+            
+            else {
+                
+                //Substitute backslashes with dots
+                boost::replace_all(hierarchy, "/", ".");
+                hierarchy.append(".");
+                
+                //Get the nodes in the tree
+                treeOutput = navigator->getNodes(hierarchy,false);
+            }
+        }
+        
+        //Build the output string.
+        for (set<string>::iterator it = treeOutput->begin(); it != treeOutput->end(); it++) {
+            
+            //If we are not querying sensor names, remove the hierarchy from each output element.
+            if(!sensorIndex) {
+                boost::regex exp(hierarchy);
+                outputElement = boost::regex_replace((*it),exp,"");
+            }
+            else
+                outputElement = *it;
+            
+            //Remove the alst dot returned by the sensor navigator output.
+            if(outputElement.back() == '.')
+                outputElement.pop_back();
+            
+            output = output + "\"" + outputElement + "\",";
+        }
+        
+        //remove the last comma.
         output.pop_back();
-        output = output + "]";
+        
+        //close the output string.
+        output += "]";
+        
+        std::cout << output;
         
         //Set the body and the status code for the response.
         response.set_body(output);
@@ -339,8 +411,7 @@ boost::log::trivial::severity_level invertLogLevel(boost::log::trivial::severity
  * Contructor of the REST server. Establishes the connection with the Cassandra;
  * sets certificate, private keys and dh file
  */
- 
-HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, boost::asio::io_service& io) {
+HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, hierarchySettings_t hierarchySettings, boost::asio::io_service& io) {
     
     // Connect to the Cassandra database server
     cassandraConnection = new DCDB::Connection();
@@ -351,6 +422,33 @@ HttpsServer::HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t
         LOG(fatal) << "Failed to connect to the Cassandra database.";
     else {
         
+        LOG(info) << "Retrieving published sensor names and topics...";
+        //Get the list of public sensors and topics.
+        DCDB::SensorConfig sensorConfig(cassandraConnection);
+        std::list<DCDB::PublicSensor> publicSensors;
+        sensorConfig.getPublicSensorsVerbose(publicSensors);
+        
+        std::vector<std::string> sensors;
+        std::vector<std::string> topics;
+        
+        for (std::list<DCDB::PublicSensor>::iterator it=publicSensors.begin(); it != publicSensors.end(); it++) {
+            sensors.push_back((*it).name);
+            topics.push_back((*it).pattern);
+        }
+        
+        LOG(info) << "Building the sensor navigator...";
+        //Build the tree navigator
+        navigator = new SensorNavigator();
+        //navigator->buildTree(hierarchySettings.regex, &sensors, &topics, hierarchySettings.separator);
+        navigator->buildTree("", &sensors, &topics, hierarchySettings.separator);
+        
+        //Test
+        //std::set<std::string> *testSensors = navigator->getNodes("mpp3r01c02s01.",false);
+        std::set<std::string> *testSensors = navigator->getSensors("mpp3.r01.c02.s01.",false);
+        for (std::set<std::string>::iterator iter=testSensors->begin(); iter != testSensors->end(); iter++) {
+            std::cout << (*iter) << std::endl;
+        }
+
         std::shared_ptr<asio::ssl::context> ctx = std::make_shared<asio::ssl::context>(asio::ssl::context::sslv23);
         ctx->set_options(asio::ssl::context::default_workarounds | asio::ssl::context::no_sslv3 | asio::ssl::context::single_dh_use);
         
@@ -428,6 +526,7 @@ int main(int argc, char *argv[])
     global_t& globalSettings = _configuration->getGlobal();
     cassandraSettings_t& cassandraSettings = globalSettings.cassandraSettings;
     restAPISettings_t& restAPISettings = globalSettings.restAPISettings;
+    hierarchySettings_t& hierarchySettings = globalSettings.hierarchySettings;
     
     //reset getopt()
     optind = 1;
@@ -527,7 +626,7 @@ int main(int argc, char *argv[])
     }
     
     //REST server gets its threads
-    _httpsServer = new HttpsServer(restAPISettings, cassandraSettings, io);
+    _httpsServer = new HttpsServer(restAPISettings, cassandraSettings, hierarchySettings, io);
     
     boost::thread restThread(std::bind(&HttpsServer::run, _httpsServer));
     

HTTPServer/HTTPServer.h

@@ -11,6 +11,7 @@
 #include "Configuration.h"
 #include "Logging.h"
 #include "cassandra.h"
+#include "SensorNavigator.h"
 
 #ifndef HTTPServer_h
 #define HTTPServer_h
@@ -38,6 +39,7 @@
 #include <boost/thread/condition.hpp>
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/thread.hpp>
+#include <boost/regex.hpp>
 #include <asio/ssl.hpp>
 
 //Headers from cpprestsdk
@@ -74,6 +76,7 @@ typedef struct {
     std::string certificate;
     std::string privateKey;
     std::string dhFile;
+    std::string hierarchy;
 } restAPISettings_t;
 
 /**
@@ -85,6 +88,16 @@ typedef struct {
     uint16_t cassandraPort;
 } cassandraSettings_t;
 
+/**
+ * Struct defining the hierarchy Settings.
+ * It holds the regular expressions defining the different elements of the datacenter.
+ */
+typedef struct {
+    std::string separator;
+    std::string regex;;
+} hierarchySettings_t;
+
+
 /**
  * This class implements the behaviour of a request handler of the REST server.
  */
@@ -168,7 +181,7 @@ class HttpsServer {
     
 public:
     
-    HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, boost::asio::io_service& io);
+    HttpsServer(restAPISettings_t restAPISettings, cassandraSettings_t cassandraSettings, hierarchySettings_t hierarchySettings, boost::asio::io_service& io);
     virtual ~HttpsServer();
     
     /*

HTTPServer/Makefile

@@ -3,9 +3,10 @@ include ../config.mk
 CXXFLAGS = -O2 -g --std=c++11 -Wall -Wno-unused-local-typedefs -Wno-deprecated-declarations -Wno-unknown-warning-option -fmessage-length=0 -I../include/ -I../lib/include -I$(DCDBDEPLOYPATH)/include -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_LOG_DYN_LINK -I$(DCDBDEPSPATH)/cpp-netlib-0.12.0-final/deps/asio/asio/include -DBOOST_TEST_DYN_LINK
 
 OBJS =	HTTPServer.o \
-	Configuration.o
+	Configuration.o \
+    SensorNavigator.o
 
-LIBS = -L$(DCDBDEPLOYPATH)/lib/ -L../lib -ldcdb -lcassandra -lboost_system -lboost_chrono -lboost_thread -lboost_log -lboost_log_setup -lpthread -lcrypto -lssl -lcpprest
+LIBS = -L$(DCDBDEPLOYPATH)/lib/ -L../lib -ldcdb -lcassandra -lboost_system -lboost_chrono -lboost_thread -lboost_log -lboost_regex -lboost_log_setup -lpthread -lcrypto -lssl -lcpprest
 TARGET = httpserver
 
 .PHONY : clean install

HTTPServer/SensorNavigator.cpp

+//
+// Created by Netti, Alessio on 11.12.18.
+//
+
+#include "SensorNavigator.h"
+
+const string SensorNavigator::rootKey = "__root__";
+const string SensorNavigator::templateKey = "__template__";
+const char SensorNavigator::pathSeparator = '.';
+
+void SensorNavigator::clearTree() {
+    if(_sensorTree) {
+        _sensorTree->clear();
+        delete _sensorTree;
+        _sensorTree = NULL;
+    }
+    if(_hierarchy) {
+        _hierarchy->clear();
+        delete _hierarchy;
+        _hierarchy = NULL;
+    }
+    _treeDepth = -1;
+    _usingTopics = false;
+}
+
+bool SensorNavigator::nodeExists(const string& node) {
+    return _sensorTree && _sensorTree->count(node) && !isSensorNode(node);
+}
+
+bool SensorNavigator::sensorExists(const string& node) {
+    return _sensorTree && _sensorTree->count(node) && isSensorNode(node);
+}
+
+string SensorNavigator::buildTopicForNode(const string& node, const string& suffix, int len) {
+    if(!_sensorTree || !_sensorTree->count(node) || isSensorNode(node))
+        throw domain_error("SensorNavigator: node not found in tree!");
+    
+    string nodePrefix = getNodeTopic(node);
+    int cnt = (int)std::count(nodePrefix.begin(), nodePrefix.end(), '/');
+    if( (int)nodePrefix.length() + (int)suffix.length() - cnt > len)
+        throw invalid_argument("SensorNavigator: cannot build topic, too many characters!");
+    
+    return nodePrefix + string(len - nodePrefix.length() - suffix.length() + cnt, 'F') + suffix;
+}
+
+bool SensorNavigator::isSensorNode(const string& node) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        throw domain_error("SensorNavigator: node not found in tree!");
+    
+    return _isSensorNode(_sensorTree->at(node));
+}
+
+int SensorNavigator::getNodeDepth(const string& node) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        throw domain_error("SensorNavigator: node not found in tree!");
+    
+    return _sensorTree->at(node).depth;
+}
+
+string SensorNavigator::getNodeTopic(const string& node) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        throw domain_error("SensorNavigator: node not found in tree!");
+    
+    return _usingTopics ? _sensorTree->at(node).topic : node;
+}
+
+void SensorNavigator::buildTree(const string& hierarchy, const vector<string> *sensors, const vector<string> *topics, const string& delimiter) {
+    //if( hierarchy == "" )
+    //    throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
+    
+    vector<string> hierarchyVec;
+    string hBuf = hierarchy;
+    size_t pos;
+    if(!hierarchy.empty())
+        while( !hBuf.empty() ) {
+            pos = hBuf.find(delimiter);
+            hierarchyVec.push_back(hBuf.substr(0, pos));
+            if(pos == std::string::npos)
+                hBuf.clear();
+            else
+                hBuf.erase(0, pos + delimiter.length());
+        }
+    buildTree(&hierarchyVec, sensors, topics);
+}
+
+void SensorNavigator::buildTree(const vector<string> *hierarchy, const vector<string> *sensors, const vector<string> *topics) {
+    if(sensors && sensors->size() > 0)
+        clearTree();
+    else
+        throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
+    
+    bool autoMode = false;
+    if(!hierarchy || hierarchy->empty()) {
+        _hierarchy = NULL;
+        autoMode = true;
+    }
+    else {
+        _hierarchy = new vector<boost::regex>();
+        string s = "";
+        //Each level in the hierarchy includes the regular expressions at the upper levels
+        for (const auto &l: *hierarchy) {
+            s += l;
+            _hierarchy->push_back(boost::regex(s));
+        }
+        autoMode = false;
+    }
+    
+    _usingTopics = topics != NULL;
+    _sensorTree = new map<string, Node>();
+    
+    //We initialize the sensor tree by pushing the root supernode
+    Node rootNode = {-1, set<string>(), set<string>(), "", ""};
+    _sensorTree->insert(make_pair(rootKey, rootNode));
+    //We iterate over all sensor names that were supplied and build up the tree
+    for(int i=0; i < (int)sensors->size(); i++)
+        if(autoMode)
+            _addAutoSensor(sensors->at(i), topics ? topics->at(i) : "");
+        else
+            _addSensor(sensors->at(i), topics ? topics->at(i) : "");
+}
+
+void SensorNavigator::buildCassandraTree(const map<string, vector<string> > *table, const string& root, const string& ignore) {
+    if(table->size() > 0 && table->count(root))
+        clearTree();
+    else
+        throw invalid_argument("SensorNavigator: cannot build sensor hierarchy!");
+    
+    _hierarchy = NULL;
+    _usingTopics = false;
+    _sensorTree = new map<string, Node>();
+    
+    boost::regex ignoreReg(ignore);
+    
+    //We initialize the sensor tree by pushing the root supernode
+    Node rootNode = {-1, set<string>(), set<string>(), "", ""};
+    _sensorTree->insert(make_pair(rootKey, rootNode));
+    //We manually add the root's children to the tree, so as to map the Cassandra root key to the one we use
+    for(const auto& s : table->at(root)) {
+        if(!boost::regex_search(s.c_str(), _match, ignoreReg)) {
+            Node newNode = {0, set<string>(), set<string>(), rootKey, ""};
+            _sensorTree->insert(make_pair(s, newNode));
+            if (table->count(s)) {
+                _sensorTree->at(rootKey).children.insert(s);
+                _addCassandraSensor(s, table, 1, ignoreReg);
+            } else {
+                //If the entry is a sensor it will share the same depth as its father, hence the -1 decrease
+                _sensorTree->at(s).depth -= 1;
+                _sensorTree->at(rootKey).sensors.insert(s);
+            }
+        }
+    }
+}
+
+map<string, SensorNavigator::Node> *SensorNavigator::getSubTree(const string& node, int depth) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        throw domain_error("SensorNavigator: node not found in tree!");
+    
+    if( depth < -1 )
+        throw out_of_range("SensorNavigator: depth not valid for subtree query!");
+    
+    //We start by picking the start node, then we call the recursive routine
+    map<string, SensorNavigator::Node> *m = new map<string, Node>();
+    m->insert(make_pair(node,_sensorTree->at(node)));
+    _getSubTree(node, m, depth);
+    return m;
+}
+
+set<string> *SensorNavigator::getNodes(int depth, bool recursive) {
+    if( depth < -1 || depth > _treeDepth )
+        throw out_of_range("SensorNavigator: depth not valid for node query!");
+    
+    if(!_sensorTree)
+        throw runtime_error("SensorNavigator: sensor tree not initialized!");
+    
+    //We pick all nodes in the tree whose depth is consistent with the input
+    //Iterating over the node map is sufficient for this purpose
+    set<string> *vec = new set<string>();
+    for(const auto& n : *_sensorTree)
+        if(!_isSensorNode(n.second) && (n.second.depth == depth || (recursive && n.second.depth >= depth)))
+            vec->insert(n.first);
+    
+    return vec;
+}
+
+set<string> *SensorNavigator::getNodes(const string& node, bool recursive) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        throw domain_error("SensorNavigator: node not found in tree!");
+    
+    if(isSensorNode(node))
+        throw invalid_argument("SensorNavigator: input must be a node, not a sensor!");
+    
+    //We start by picking all children of the start node, then we proceed in the subtree (if recursive)
+    set<string> *vec = new set<string>();
+    _getNodes(node, vec, recursive ? 0 : 1);
+    return vec;
+}
+
+set<string> *SensorNavigator::getSensors(int depth, bool recursive) {
+    if( depth < -1 || depth > _treeDepth )
+        throw out_of_range("SensorNavigator: depth not valid for sensor query!");
+    
+    if(!_sensorTree)
+        throw runtime_error("SensorNavigator: sensor tree not initialized!");
+    
+    //Like in getNodes, we iterate over the node map and pick all sensors that are at least at the given input depth
+    set<string> *vec = new set<string>();
+    for(const auto& n : *_sensorTree)
+        if(!_isSensorNode(n.second) && (n.second.depth == depth || (recursive && n.second.depth >= depth)))
+            vec->insert(n.second.sensors.begin(), n.second.sensors.end());
+    
+    return vec;
+}
+
+set<string> *SensorNavigator::getSensors(const string& node, bool recursive) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        throw domain_error("SensorNavigator: node not found in tree!");
+    
+    if(isSensorNode(node))
+        throw invalid_argument("SensorNavigator: input must be a node, not a sensor!");
+    
+    //We start by picking all sensors of the start node, then we call the recursive routine
+    set<string> *vec = new set<string>();
+    _getSensors(node, vec, recursive ? 0 : 1);
+    return vec;
+}
+
+set<string> *SensorNavigator::navigate(const string& node, int direction) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        throw domain_error("SensorNavigator: node not found in tree!");
+    
+    set<string> *vec = new set<string>();
+    if(direction == 0)
+        vec->insert(node);
+    //if direction is negative, we go up in the tree
+    else if(direction < 0) {
+        int ctr = -direction;
+        string currNode = node;
+        //We go up the sensor tree in iterative fashion
+        while (ctr-- > 1 && currNode != rootKey) {
+            currNode = _sensorTree->at(currNode).parent;
+        }
+        vec->insert(_sensorTree->at(currNode).parent);
+    }
+    //Else, we go down to the children
+    else {
+        _getNodes(node, vec, direction);
+    }
+    return vec;
+}
+
+void SensorNavigator::_getNodes(const string& node, set<string> *vec, int depth) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        return;
+    
+    if(depth<=1)
+        vec->insert(_sensorTree->at(node).children.begin(), _sensorTree->at(node).children.end());
+    //We iterate over all children of the input node, and call the recursive routine over them
+    //depth=1 implies that we are at the last level of the search; in this case, we do not continue
+    //an input value of depth<1 implies that the search must proceed for the entire subtree
+    if(depth!=1)
+        for(const auto& n : _sensorTree->at(node).children)
+            _getNodes(n, vec, depth-1);
+}
+
+void SensorNavigator::_getSensors(const string& node, set<string> *vec, int depth) {
+    if(!_sensorTree || !_sensorTree->count(node))
+        return;
+    
+    if(depth<=1)
+        vec->insert(_sensorTree->at(node).sensors.begin(), _sensorTree->at(node).sensors.end());
+    //We iterate over all children of the input node, and call the recursive routine over them
+    //Works the same as _getNodes; here, depth does not really refer to the depth level in the tree (as sensors share
+    //the depth level of the node they belong to) but rather to the search steps performed. A depth of 1 means that
+    //only the sensors of the original node will be added, 2 means that the sensors of its children will be added, etc.
+    if(depth!=