Commit 3a254dfd authored by Michael Ott's avatar Michael Ott

Merging dcdbpusher to dcdbpusher subdirectory

parents 2cf6acc5 9798c1f6
......@@ -4,11 +4,14 @@ install/
## Ignore object files
*.o
*.so
*.dylib
## Ignore various IDE project definition files
.cproject
.project
.settings
*.xcodeproj
*.xcworkspace/
## Ignore debug symbol directories
......@@ -20,3 +23,6 @@ cscope.out
## Ignore editor temp files
*~
## Ignore log files
*.log
......@@ -4,6 +4,9 @@ stages:
- build_all
- install_all
after_script:
- make clean
build_deps:
stage: build_deps
only:
......
......@@ -39,6 +39,7 @@
#include <dcdb/sensordatastore.h>
#include <dcdb/sensorconfig.h>
#include <dcdb/version.h>
#include "version.h"
#include "configuration.h"
#include "simplemqttserver.h"
......
MAKEFILENAME := $(lastword $(MAKEFILE_LIST))
SUB_DIRS = lib CollectAgent tools scripts
SUB_DIRS = lib CollectAgent dcdbpusher tools scripts
CASSANDRA_VERSION = 2.2.10
MOSQUITTO_VERSION = 1.5.5
......@@ -10,6 +10,9 @@ CPPDRV_VERSION = 2.10.0
LIBUV_VERSION = 1.24.0
SOURCEFORGE_MROR = netcologne
CPPNET_VERSION = 0.12.0-final
BACNET-STACK_VERSION = 0.8.5
FREEIPMI_VERSION = 1.5.5
NET-SNMP_VERSION = 5.7.3
BOOST_VERSION_U = $(subst .,_,$(BOOST_VERSION))
DISTFILES = apache-cassandra-$(CASSANDRA_VERSION).tar.gz;http://archive.apache.org/dist/cassandra/$(CASSANDRA_VERSION)/apache-cassandra-$(CASSANDRA_VERSION)-bin.tar.gz \
......@@ -18,9 +21,11 @@ DISTFILES = apache-cassandra-$(CASSANDRA_VERSION).tar.gz;http://archive.apache.o
openssl-$(OPENSSL_VERSION).tar.gz;https://www.openssl.org/source/openssl-$(OPENSSL_VERSION).tar.gz \
libuv-v$(LIBUV_VERSION).tar.gz;https://dist.libuv.org/dist/v$(LIBUV_VERSION)/libuv-v$(LIBUV_VERSION).tar.gz \
cpp-driver-$(CPPDRV_VERSION).tar.gz;https://github.com/datastax/cpp-driver/archive/$(CPPDRV_VERSION).tar.gz \
cpp-netlib-$(CPPNET_VERSION).tar.gz;http://downloads.cpp-netlib.org/0.12.0/cpp-netlib-$(CPPNET_VERSION).tar.gz
DISTFILES_HASHES = apache-cassandra-2.2.10.tar.gz|4c58cb7c6753ce26f7c4d650502feece;mosquitto-1.5.5.tar.gz|a17dffc6f63b2a4ab2eb5c51139e60e9;boost_1_58_0.tar.gz|5a5d5614d9a07672e1ab2a250b5defc5;openssl-1.0.2l.tar.gz|f85123cd390e864dfbe517e7616e6566;cpp-driver-2.10.0.tar.gz|6d15dd2cccd2efd1fdc86089d26971d0;libuv-v1.24.0.tar.gz|90320330757253b07404d2a97f59c66b;cpp-netlib-0.12.0-final.tar.gz|29b87c0e8c1a9e7fbdea5afcec947d53
cpp-netlib-$(CPPNET_VERSION).tar.gz;http://downloads.cpp-netlib.org/0.12.0/cpp-netlib-$(CPPNET_VERSION).tar.gz \
bacnet-stack-$(BACNET-STACK_VERSION).tgz;https://downloads.sourceforge.net/project/bacnet/bacnet-stack/bacnet-stack-$(BACNET-STACK_VERSION)/bacnet-stack-$(BACNET-STACK_VERSION).tgz \
freeipmi-$(FREEIPMI_VERSION).tar.gz;http://ftp.gnu.org/gnu/freeipmi/freeipmi-$(FREEIPMI_VERSION).tar.gz \
net-snmp-$(NET-SNMP_VERSION).tar.gz;https://sourceforge.net/projects/net-snmp/files/net-snmp/$(NET-SNMP_VERSION)/net-snmp-$(NET-SNMP_VERSION).tar.gz/download
DISTFILES_HASHES = apache-cassandra-2.2.10.tar.gz|4c58cb7c6753ce26f7c4d650502feece;mosquitto-1.5.5.tar.gz|a17dffc6f63b2a4ab2eb5c51139e60e9;boost_1_58_0.tar.gz|5a5d5614d9a07672e1ab2a250b5defc5;openssl-1.0.2l.tar.gz|f85123cd390e864dfbe517e7616e6566;cpp-driver-2.10.0.tar.gz|6d15dd2cccd2efd1fdc86089d26971d0;libuv-v1.24.0.tar.gz|90320330757253b07404d2a97f59c66b;cpp-netlib-0.12.0-final.tar.gz|29b87c0e8c1a9e7fbdea5afcec947d53;bacnet-stack-$(BACNET-STACK_VERSION).tgz|66b69111d91432fa67a7c6c1a653434d;freeipmi-$(FREEIPMI_VERSION).tar.gz|b8abfefee0b757f351d8fab777e3c1bb;net-snmp-$(NET-SNMP_VERSION).tar.gz|d4a3459e1577d0efa8d96ca70a885e53
include common.mk
......@@ -38,9 +43,9 @@ HTTPD_PORT = 8080
# If cross-compiling for ARM, adjust the build settings
ifeq ("$(ARCH)", "arm")
OPENSSL_TARGET = "linux-generic32"
OPENSSL_TARGET = "linux-generic32"
else
OPENSSL_TARGET = $(if $(findstring $(shell uname),Darwin),"darwin64-x86_64-cc","linux-x86_64")
OPENSSL_TARGET = $(if $(findstring $(shell uname),Darwin),"darwin64-x86_64-cc","linux-x86_64")
endif
PUBHEADERS = pusherpqueue.h dcdbdaemon.h
......@@ -226,6 +231,38 @@ $(DCDBDEPSPATH)/apache-cassandra-$(CASSANDRA_VERSION)/.installed: $(DCDBDEPSPATH
@sed -i -e 's/.*tombstone_warn_threshold:.*/tombstone_warn_threshold: $(CASSANDRA_TOMBSTONE_WARN_THRESHOLD)/' $(DCDBDEPLOYPATH)/cassandra/conf/cassandra.yaml
@sed -i -e 's/.*tombstone_failure_threshold:.*/tombstone_failure_threshold: $(CASSANDRA_TOMBSTONE_FAILURE_THRESHOLD)/' $(DCDBDEPLOYPATH)/cassandra/conf/cassandra.yaml
@touch $@
$(DCDBDEPSPATH)/bacnet-stack-$(BACNET-STACK_VERSION)/.built: $(DCDBDEPSPATH)/bacnet-stack-$(BACNET-STACK_VERSION)/.patched
@echo ""
@echo "Building BACNet-Stack..."
$(eval BACNET_PORT:= $(if $(filter $(OS),Darwin),bsd,linux))
cd $(@D) && BACNET_PORT=$(BACNET_PORT) MAKE_DEFINE=-fpic make -j $(MAKETHREADS) library
touch $(@)
$(DCDBDEPSPATH)/bacnet-stack-$(BACNET-STACK_VERSION)/.installed: $(DCDBDEPSPATH)/bacnet-stack-$(BACNET-STACK_VERSION)/.built | $(DCDBDEPLOYPATH)
@echo "Installing BACNet-Stack..."
install $(DCDBDEPSPATH)/bacnet-stack-$(BACNET-STACK_VERSION)/lib/libbacnet.a /$(DCDBDEPLOYPATH)/lib/ && touch $(@)
$(DCDBDEPSPATH)/freeipmi-$(FREEIPMI_VERSION)/.built: $(DCDBDEPSPATH)/freeipmi-$(FREEIPMI_VERSION)/.patched
@echo ""
@echo "Building FreeIPMI library..."
cd $(@D) && ./configure --prefix=$(DCDBDEPLOYPATH) --without-argp
cd $(@D) && make -j $(MAKETHREADS) && touch $(@)
$(DCDBDEPSPATH)/freeipmi-$(FREEIPMI_VERSION)/.installed: $(DCDBDEPSPATH)/freeipmi-$(FREEIPMI_VERSION)/.built | $(DCDBDEPLOYPATH)
@echo "Installing FreeIPMI library..."
cd $(@D) && make install && touch $(@)
$(DCDBDEPSPATH)/net-snmp-$(NET-SNMP_VERSION)/.built: $(DCDBDEPSPATH)/net-snmp-$(NET-SNMP_VERSION)/.patched
@echo ""
@echo "Building net-SNMP library..."
cd $(@D) && ./configure --prefix=$(DCDBDEPLOYPATH) --with-default-snmp-version=3 --with-sys-contact= --with-sys-location= --with-logfile=none --with-persistent-directory=$(DCDBDEPLOYPATH)/var/net-snmp --disable-embedded-perl --disable-perl-cc-checks --without-perl-modules --disable-agent --disable-applications --disable-manuals --disable-scripts --disable-mibs
cd $(@D) && make -j $(MAKETHREADS) && touch $(@)
$(DCDBDEPSPATH)/net-snmp-$(NET-SNMP_VERSION)/.installed: $(DCDBDEPSPATH)/net-snmp-$(NET-SNMP_VERSION)/.built | $(DCDBDEPLOYPATH)
@echo ""
@echo "Installing net-SNMP library..."
cd $(@D) && make install && touch $(@)
%-build: deps
@echo "Building $*"
......
......@@ -2,4 +2,6 @@ DCDBBASEPATH ?= $(realpath $(dir $(lastword $(MAKEFILE_LIST)))/..)
DCDBDEPSPATH ?= $(DCDBBASEPATH)/deps
DCDBDEPLOYPATH ?= $(DCDBBASEPATH)/install
VERSION = $(shell git describe --tags|sed 's/-\([0-9]*\)/.\1/')
DEFAULT_VERSION = 0.2
GIT_VERSION = $(shell git describe --tags 2>/dev/null|sed 's/-\([0-9]*\)/.\1/')
VERSION := $(if $(GIT_VERSION),$(GIT_VERSION),$(DEFAULT_VERSION))
/*
* Configuration.cpp
*
* Created on: 13.12.2017
* Author: Michael Ott (original), Micha Mueller
*/
#include "Configuration.h"
#include <string>
#include <unistd.h>
#include <dlfcn.h>
#include <boost/foreach.hpp>
#include <boost/property_tree/info_parser.hpp>
#include <boost/algorithm/string.hpp>
using namespace std;
Configuration::Configuration(const std::string& cfgFilePath) :
_cfgFilePath(cfgFilePath) {
if (_cfgFilePath[_cfgFilePath.length()-1] != '/') {
_cfgFilePath.append("/");
}
//set default values for global variables
_global.validateConfig = false;
_global.qosLevel = 1;
_global.daemonize = 0;
_global.hierarchy = "";
_global.brokerHost = "";
_global.brokerPort = 1883;
_global.threads = 1;
_global.maxMsgNum = 0;
_global.maxInflightMsgNum = 20;
_global.maxQueuedMsgNum = 0;
_global.logLevelFile = boost::log::trivial::trace;
_global.logLevelCmd = boost::log::trivial::info;
_global.pluginSettings.sensorPattern = "";
_global.pluginSettings.mqttPrefix = "";
_global.pluginSettings.tempdir = "./";
_global.pluginSettings.cacheInterval = 900000;
_global.restAPISettings.restHost = "";
_global.restAPISettings.restPort = "8000";
}
Configuration::~Configuration() {
//close plugins
for (auto p : _plugins) {
if (p.configurator) {
p.destroy(p.configurator);
}
if (p.DL) {
dlclose(p.DL);
}
}
}
bool Configuration::readGlobal() {
//open file
std::string globalConfig = _cfgFilePath;
globalConfig.append("global.conf");
boost::property_tree::iptree cfg;
//parse to property_tree
try {
boost::property_tree::read_info(globalConfig, cfg);
} catch (boost::property_tree::info_parser_error& e) {
LOG(error) << "Error when reading global.conf: " << e.what();
return false;
}
//read global struct
BOOST_FOREACH(boost::property_tree::iptree::value_type &global, cfg.get_child("global")) {
if (boost::iequals(global.first, "mqttBroker")) {
_global.brokerHost = global.second.data();
size_t pos = _global.brokerHost.find(":");
if (pos != string::npos) {
_global.brokerPort = stoi(_global.brokerHost.substr(pos+1));
_global.brokerHost.erase(pos);
}
} else if (boost::iequals(global.first, "mqttprefix")) {
_global.pluginSettings.mqttPrefix = global.second.data();
if (_global.pluginSettings.mqttPrefix[_global.pluginSettings.mqttPrefix.length()-1] != '/') {
_global.pluginSettings.mqttPrefix.append("/");
}
} else if (boost::iequals(global.first, "sensorpattern")) {
_global.pluginSettings.sensorPattern = global.second.data();
} else if (boost::iequals(global.first, "hierarchy")) {
_global.hierarchy = global.second.data();
} else if (boost::iequals(global.first, "tempdir")) {
_global.pluginSettings.tempdir = global.second.data();
if (_global.pluginSettings.tempdir[_global.pluginSettings.tempdir.length()-1] != '/') {
_global.pluginSettings.tempdir.append("/");
}
} else if (boost::iequals(global.first, "qosLevel")) {
_global.qosLevel = stoi(global.second.data());
if(_global.qosLevel < 0 || _global.qosLevel > 2)
_global.qosLevel = 1;
} else if (boost::iequals(global.first, "maxInflightMsgNum")) {
_global.maxInflightMsgNum = stoull(global.second.data());
}else if (boost::iequals(global.first, "maxQueuedMsgNum")) {
_global.maxQueuedMsgNum = stoull(global.second.data());
}else if (boost::iequals(global.first, "threads")) {
_global.threads = stoi(global.second.data());
} else if (boost::iequals(global.first, "maxMsgNum")) {
_global.maxMsgNum = stoi(global.second.data());
} else if (boost::iequals(global.first, "daemonize")) {
if (global.second.data() == "true") {
_global.daemonize = 1;
}
} else if (boost::iequals(global.first, "validateConfig")) {
if (global.second.data() == "true") {
_global.validateConfig = true;
}
} else if (boost::iequals(global.first, "verbosity")) {
_global.logLevelFile = translateLogLevel(stoi(global.second.data()));
} else if (boost::iequals(global.first, "cacheInterval")) {
_global.pluginSettings.cacheInterval = stoul(global.second.data());
_global.pluginSettings.cacheInterval *= 1000;
} else {
LOG(warning) << " Value \"" << global.first << "\" not recognized. Omitting";
}
}
//read restAPI struct
BOOST_FOREACH(boost::property_tree::iptree::value_type &global, cfg.get_child("restAPI")) {
if (boost::iequals(global.first, "address")) {
_global.restAPISettings.restHost = global.second.data();
size_t pos = _global.restAPISettings.restHost.find(":");
if (pos != string::npos) {
_global.restAPISettings.restPort = _global.restAPISettings.restHost.substr(pos+1);
_global.restAPISettings.restHost.erase(pos);
}
} else if (boost::iequals(global.first, "certificate")) {
_global.restAPISettings.certificate = global.second.data();
} else if (boost::iequals(global.first, "privateKey")) {
_global.restAPISettings.privateKey = global.second.data();
} else if (boost::iequals(global.first, "dhFile")) {
_global.restAPISettings.dhFile = global.second.data();
} else if (boost::iequals(global.first, "authkey")) {
//Avoid unnecessary "Value not recognized" message
} else {
LOG(warning) << " Value \"" << global.first << "\" not recognized. Omitting";
}
}
return true;
}
bool Configuration::readAuthkeys(HttpsServer* server) {
//open file
std::string globalConfig = _cfgFilePath;
globalConfig.append("global.conf");
boost::property_tree::iptree cfg;
//parse to property_tree
try {
boost::property_tree::read_info(globalConfig, cfg);
} catch (boost::property_tree::info_parser_error& e) {
LOG(error) << "Error when reading authkeys from global.conf: " << e.what();
return false;
}
//read authkeys
BOOST_FOREACH(boost::property_tree::iptree::value_type &global, cfg.get_child("restAPI")) {
if (boost::iequals(global.first, "authkey")) {
#ifdef DEBUG
LOG(info) << "Authentication token \"" << global.second.data() << "\"";
#endif
std::bitset<NUM_PERMISSIONS> permissions;
BOOST_FOREACH(boost::property_tree::iptree::value_type &perm, global.second) {
if (boost::iequals(perm.first, "GETReq")) {
#ifdef DEBUG
LOG(info) << " Permission \"GETReq\"";
#endif
permissions[GETReq] = true;
} else if (boost::iequals(perm.first, "PUTReq")) {
#ifdef DEBUG
LOG(info) << " Permission \"PUTReq\"";
#endif
permissions[PUTReq] = true;
} else {
#ifdef DEBUG
LOG(warning) << "Permission \"" << perm.first << "\" not recognized. Omitting";
#endif
}
}
if (!server->addAuthkey(authkeyMap_t::value_type(global.second.data(), permissions))) {
#ifdef DEBUG
LOG(warning) << "Authkey already present!";
#endif
}
} else {
//
}
}
return true;
}
bool Configuration::readPlugins() {
std::string globalConfig = _cfgFilePath;
globalConfig.append("global.conf");
boost::property_tree::iptree cfg;
try {
boost::property_tree::read_info(globalConfig, cfg);
} catch (boost::property_tree::info_parser_error& e) {
LOG(error) << "Error when reading plugins from global.conf: " << e.what();
return false;
}
//read plugins
BOOST_FOREACH(boost::property_tree::iptree::value_type &plugin, cfg.get_child("plugins")) {
if (boost::iequals(plugin.first, "plugin")) {
if (!plugin.second.empty()) {
LOG(info) << "Loading plugin " << plugin.second.data() << "...";
std::string pluginConfig; //path to config file for plugin
std::string pluginLib = "libdcdbplugin_" + plugin.second.data(); //TODO add version information? //path to the plugin-lib
#if __APPLE__
pluginLib+= ".dylib";
#else
pluginLib+= ".so";
#endif
BOOST_FOREACH(boost::property_tree::iptree::value_type &val, plugin.second) {
if (boost::iequals(val.first, "path")) {
std::string path = val.second.data();
//if path not specified we will look up in the default lib-directories (usr/lib and friends)
if (path != "") {
if (path[path.length()-1] != '/') {
path.append("/");
}
pluginLib = path + pluginLib;
}
} else if (boost::iequals(val.first, "config")) {
pluginConfig = val.second.data();
//if config-path not specified we will look for pluginName.conf in the global.conf directory
if (pluginConfig == "") {
pluginConfig = _cfgFilePath + plugin.second.data() + ".conf";
}
} else {
LOG(warning) << " Value \"" << val.first << "\" not recognized. Omitting";
}
}
//open plugin
//dl-code based on http://tldp.org/HOWTO/C++-dlopen/thesolution.html
if (FILE *file = fopen(pluginConfig.c_str(), "r")) {
fclose(file);
dl_t dynLib;
dynLib.id = plugin.second.data();
dynLib.DL = NULL;
dynLib.configurator = NULL;
//plugin.conf exists --> open libdcdbplugin_pluginName.so and read config
LOG(info) << pluginConfig << " found";
dynLib.DL = dlopen(pluginLib.c_str(), RTLD_NOW);
if(!dynLib.DL) {
LOG(error) << "Cannot load " << dynLib.id << "-library: " << dlerror();
return false;
}
//reset errors
dlerror();
//set dynLib-struct
dynLib.create = (create_t*) dlsym(dynLib.DL, "create");
const char* dlsym_error = dlerror();
if (dlsym_error) {
LOG(error) << "Cannot load symbol create for " << dynLib.id << ": " << dlsym_error;
return false;
}
dynLib.destroy = (destroy_t*) dlsym(dynLib.DL, "destroy");
dlsym_error = dlerror();
if (dlsym_error) {
LOG(error) << "Cannot load symbol destroy for " << dynLib.id << ": " << dlsym_error;
return false;
}
dynLib.configurator = dynLib.create();
//set prefix to global prefix (may be overwritten)
dynLib.configurator->setGlobalSettings(_global.pluginSettings);
//read in config
if (!(dynLib.configurator->readConfig(pluginConfig))) {
LOG(error) << "Plugin " << dynLib.id << " could not read configuration!";
return false;
}
// returning an empty vector may indicate problems with the config file
if(dynLib.configurator->getSensorGroups().size() == 0) {
LOG(warning) << "Plugin " << dynLib.id << " created no sensors!";
}
//check if an MQTT-suffix was assigned twice
for(const auto& g : dynLib.configurator->getSensorGroups()) {
for(const auto& s : g->getSensors()) {
bool ok = checkMqtt(s->getMqtt());
if(!ok) {
LOG(error) << "Problematic MQTT-Topics! Please check your config files";
return false;
}
}
}
//save dl-struct
_plugins.push_back(dynLib);
LOG(info) << "Plugin " << dynLib.id << " " << dynLib.configurator->getVersion() << " loaded!";
} else {
LOG(info) << pluginConfig << " not found. Omitting";
}
}
}
}
return true;
}
boost::log::trivial::severity_level Configuration::translateLogLevel(int logLevel) {
switch (logLevel) {
case 0:
return boost::log::trivial::fatal;
break;
case 1:
return boost::log::trivial::error;
break;
case 2:
return boost::log::trivial::warning;
break;
case 3:
return boost::log::trivial::info;
break;
case 4:
return boost::log::trivial::debug;
break;
case 5:
return boost::log::trivial::trace;
break;
default:
return boost::log::trivial::info;
break;
}
}
bool Configuration::checkMqtt(const std::string& mqtt) {
//MQTT topic must have 112 bit = 14 bytes = 28 hex chars
//but can have more with some extra '/', therefore remove all '/'
std::string str(mqtt);
str.erase(std::remove(str.begin(), str.end(), '/'), str.end());
if (str.length() != 28) {
LOG(error) << "MQTT-Topic \"" << mqtt << "\" contains " << str.length() << " hex characters, not 28 as required!";
return false;
}
auto returnIt = _mqttTopics.insert(str);
if (!returnIt.second) {
LOG(error) << "MQTT-Topic \"" << mqtt << "\" used twice!";
return false;
}
return true;
}
global_t& Configuration::getGlobal() {
return _global;
}
pluginVector_t& Configuration::getPlugins() {
return _plugins;
}
/*
* Configuration.h
*
* Created on: 13.12.2017
* Author: Michael Ott (original), Micha Mueller
*/
#ifndef CONFIGURATION_H_
#define CONFIGURATION_H_
#include <set>
#include "HttpsServer.h"
#include <boost/property_tree/ptree.hpp>
#include <boost/log/trivial.hpp>
#include "includes/PluginDefinitions.h"
typedef struct {
bool validateConfig;
int daemonize;
int brokerPort;
int qosLevel;
unsigned int maxInflightMsgNum;
unsigned int maxQueuedMsgNum;
std::string brokerHost;
std::string hierarchy;
uint32_t threads;
int maxMsgNum;
boost::log::trivial::severity_level logLevelFile;
boost::log::trivial::severity_level logLevelCmd;
pluginSettings_t pluginSettings;
restAPISettings_t restAPISettings;
} global_t;
/**
* Class responsible of reading the global configuration as well as loading and invoking required dynamic libraries.
*/
class Configuration {
typedef std::set<std::string> mqttSet_t;
public:
/**
* Create new Configuration. Sets global config file to read from to cfgFile.
* @param cfgFilePath Path to where all config-files are located
*/
Configuration(const std::string& cfgFilePath);
virtual ~Configuration();
/**