Merge branch 'REST' into development

config.mk

@@ -3,7 +3,7 @@ DCDBDEPSPATH ?= $(DCDBBASEPATH)/deps
 DCDBDEPLOYPATH ?= $(DCDBBASEPATH)/install
 
 # dcdbpusher plugins to be built
-PLUGINS = sysfs ipmi pdu bacnet snmp procfs tester gpfsmon msr 
+PLUGINS = sysfs ipmi pdu bacnet snmp procfs tester gpfsmon msr rest
 
 # data analytics plugins to be built
 OPERATORS = aggregator smoothing regressor classifier clustering cssignatures job_aggregator testeroperator filesink smucngperf persystsql coolingcontrol healthchecker

dcdbpusher/Makefile

@@ -121,3 +121,7 @@ libdcdbplugin_caliper.$(LIBEXT): sensors/caliper/CaliperSensorGroup.o sensors/ca
  libdcdbplugin_nvml.$(LIBEXT): sensors/nvml/nvmlSensorGroup.o sensors/nvml/nvmlConfigurator.o
 	$(NVCC) -shared --compiler-options '-fPIC'  -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system -lnvidia-ml
 
+libdcdbplugin_rest.$(LIBEXT): sensors/rest/RESTSensorGroup.o sensors/rest/RESTUnit.o sensors/rest/RESTConfigurator.o
+	$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lcrypto -lssl -lboost_regex -lboost_log -lboost_system
+
+

dcdbpusher/sensors/rest/RESTConfigurator.cpp

+//================================================================================
+// Name        : RESTConfigurator.cpp
+// Author      : Michael Ott
+// Contact     : info@dcdb.it
+// Copyright   : Leibniz Supercomputing Centre
+// Description : Source file for REST plugin configurator class.
+//================================================================================
+
+//================================================================================
+// This file is part of DCDB (DataCenter DataBase)
+// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//================================================================================
+
+#include "RESTConfigurator.h"
+
+#include <iostream>
+#include <sstream>
+
+RESTConfigurator::RESTConfigurator() {
+	_entityName = "host";
+	_groupName = "group";
+	_baseName = "sensor";
+}
+
+RESTConfigurator::~RESTConfigurator() {}
+
+void RESTConfigurator::sensorBase(RESTSensorBase &s, CFG_VAL config) {
+	ADD {
+		ATTRIBUTE("path", setXMLPath);
+	}
+}
+
+void RESTConfigurator::sensorGroup(RESTSensorGroup &s, CFG_VAL config) {
+	ADD {
+		ATTRIBUTE("endpoint", setEndpoint);
+		ATTRIBUTE("request", setRequest);
+	}
+}
+
+void RESTConfigurator::sensorEntity(RESTUnit &s, CFG_VAL config) {
+	ADD {
+		ATTRIBUTE("baseurl", setBaseURL);
+		ATTRIBUTE("authendpoint", setAuthEndpoint);
+		ATTRIBUTE("authdata", setAuthData);
+	}
+}

dcdbpusher/sensors/rest/RESTConfigurator.h

+//================================================================================
+// Name        : RESTConfigurator.h
+// Author      : Michael Ott
+// Contact     : info@dcdb.it
+// Copyright   : Leibniz Supercomputing Centre
+// Description : Header file for REST plugin configurator class.
+//================================================================================
+
+//================================================================================
+// This file is part of DCDB (DataCenter DataBase)
+// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//================================================================================
+
+#ifndef RESTCONFIGURATOR_H_
+#define RESTCONFIGURATOR_H_
+
+#include "../../includes/ConfiguratorTemplateEntity.h"
+
+#include "RESTSensorGroup.h"
+#include "RESTUnit.h"
+
+/**
+ * @brief ConfiguratorTemplate specialization for this plugin.
+ *
+ * @ingroup rest
+ */
+class RESTConfigurator : public ConfiguratorTemplateEntity<RESTSensorBase, RESTSensorGroup, RESTUnit> {
+
+      public:
+	RESTConfigurator();
+	virtual ~RESTConfigurator();
+
+      protected:
+	/* Overwritten from ConfiguratorTemplate */
+	void sensorBase(RESTSensorBase &s, CFG_VAL config) override;
+	void sensorGroup(RESTSensorGroup &s, CFG_VAL config) override;
+	void sensorEntity(RESTUnit &s, CFG_VAL config) override;
+
+      private:
+	/**
+	 * Split the given string into multiple parts which are later required to find the sensor value in the boost property tree.
+	 * Store the individual parts into the sensors _xmlPathVector
+	 * @param sensor		Sensor where the path belongs to
+	 * @param pathString	Complete unrefined string as read from the config file
+	 */
+	void parsePathString(RESTSensorBase &sensor, const std::string &pathString);
+};
+
+extern "C" ConfiguratorInterface *create() {
+	return new RESTConfigurator;
+}
+
+extern "C" void destroy(ConfiguratorInterface *c) {
+	delete c;
+}
+
+#endif /* RESTCONFIGURATOR_H_ */

dcdbpusher/sensors/rest/RESTSensorBase.h

+//================================================================================
+// Name        : RESTSensorBase.h
+// Author      : Michael Ott
+// Contact     : info@dcdb.it
+// Copyright   : Leibniz Supercomputing Centre
+// Description : Sensor base class for REST plugin.
+//================================================================================
+
+//================================================================================
+// This file is part of DCDB (DataCenter DataBase)
+// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//================================================================================
+
+/**
+ * @defgroup rest REST plugin
+ * @ingroup  pusherplugins
+ *
+ * @brief Collect data from REST APIs
+ */
+
+#ifndef RESTSENSORBASE_H_
+#define RESTSENSORBASE_H_
+
+#include "RESTUnit.h"
+#include "sensorbase.h"
+#include <sstream>
+
+/**
+ * @brief SensorBase specialization for this plugin.
+ *
+ * @ingroup rest
+ */
+class RESTSensorBase : public SensorBase {
+      public:
+	RESTSensorBase(const std::string &name)
+	    : SensorBase(name) {}
+
+	RESTSensorBase(const RESTSensorBase &other)
+	    : SensorBase(other),
+	      _xmlPath(other._xmlPath) {}
+
+	virtual ~RESTSensorBase() {}
+
+	RESTSensorBase &operator=(const RESTSensorBase &other) {
+		SensorBase::operator=(other);
+		_xmlPath = other._xmlPath;
+
+		return *this;
+	}
+
+	const xmlPathVector_t &getXMLPath() const { return _xmlPath; }
+	std::string            getXMLPathString() const {
+                std::stringstream ss;
+                for (const auto &i : _xmlPath) {
+                        ss << "." << std::get<0>(i) << "." << std::get<1>(i);
+                        for (const auto &j : std::get<2>(i)) {
+                                ss << "(" << j.first << "=" << j.second << ")";
+                        }
+                }
+                return ss.str();
+	}
+	void setXMLPath(const std::string &path) {
+		std::vector<std::string> subStrings;
+
+		std::stringstream pathStream(path);
+		std::string       item;
+		//split into parts if a attribute (indicated by '(' ')' ) was defined
+		while (std::getline(pathStream, item, ')')) {
+			subStrings.push_back(item);
+		}
+
+		for (auto subStr : subStrings) {
+			//extract the attributes from the path-parts
+			if (subStr.find('(') != std::string::npos) { //attribute specified
+				std::stringstream pathWithAttributesSStream(subStr);
+
+				//split into path and attributes string
+				std::string subPath, attributeString;
+				std::getline(pathWithAttributesSStream, subPath, '(');
+				std::getline(pathWithAttributesSStream, attributeString);
+
+				if (subPath.front() == '.') {
+					subPath.erase(0, 1);
+				}
+
+				//now further split the attributes string as multiple attributes could be defined
+				std::vector<std::string> attributes;
+				std::stringstream        attributeStream(attributeString);
+				while (std::getline(attributeStream, item, ',')) {
+					attributes.push_back(item);
+				}
+				attributesVector_t attrs;
+				for (auto att : attributes) {
+					//part attributes into name and value
+					if (att.find('=') != std::string::npos) {
+						std::stringstream attStream(att);
+
+						std::string attName, attVal;
+						std::getline(attStream, attName, '=');
+						std::getline(attStream, attVal);
+
+						attrs.push_back(std::make_pair(attName, attVal));
+					} else { //should not happen. If it does the path was malformed
+						//LOG(error) << "  Could not parse XML-path!";
+						return;
+					}
+				}
+				//split of the last child in the path. Required to iterate over multiple nodes which only differ in the attributes with BOOST_FOREACH
+				auto index = subPath.find_last_of('.');
+				if (index != std::string::npos) {
+					std::string subPathChild(subPath.substr(++index));
+					subPath.erase(--index);
+					_xmlPath.push_back(std::make_tuple(subPath, subPathChild, attrs));
+				} else { //the path contained only one node
+					_xmlPath.push_back(std::make_tuple("", subPath, attrs));
+				}
+			} else { //no attributes specified. Last (sub)path
+				if (subStr.front() == '.') {
+					subStr.erase(0, 1);
+				}
+				_xmlPath.push_back(std::make_tuple(subStr, "", attributesVector_t()));
+				break;
+			}
+		}
+	}
+
+	void printConfig(LOG_LEVEL ll, LOGGER &lg, unsigned leadingSpaces = 16) {
+		std::string leading(leadingSpaces, ' ');
+		LOG_VAR(ll) << leading << "    XML Path: " << getXMLPathString();
+	}
+
+      protected:
+	xmlPathVector_t _xmlPath;
+};
+
+#endif /* RESTSENSORBASE_H_ */

dcdbpusher/sensors/rest/RESTSensorGroup.cpp

+//================================================================================
+// Name        : RESTSensorGroup.cpp
+// Author      : Michael Ott
+// Contact     : info@dcdb.it
+// Copyright   : Leibniz Supercomputing Centre
+// Description : Source file for REST sensor group class.
+//================================================================================
+
+//================================================================================
+// This file is part of DCDB (DataCenter DataBase)
+// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//================================================================================
+
+#include "RESTSensorGroup.h"
+
+#include <sstream>
+
+#include <boost/foreach.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+
+RESTSensorGroup::RESTSensorGroup(const std::string &name)
+    : SensorGroupTemplateEntity(name) {
+}
+
+RESTSensorGroup::RESTSensorGroup(const RESTSensorGroup &other)
+    : SensorGroupTemplateEntity(other),
+      _endpoint(other._endpoint),
+      _request(other._request) {}
+
+RESTSensorGroup::~RESTSensorGroup() {}
+
+RESTSensorGroup &RESTSensorGroup::operator=(const RESTSensorGroup &other) {
+	SensorGroupTemplate::operator=(other);
+	_endpoint = other._endpoint;
+	_request = other._request;
+
+	return *this;
+}
+
+void RESTSensorGroup::read() {
+	//send request
+	std::string response;
+	if (!_entity->sendRequest(_endpoint, _request, response)) {
+		LOG(error) << _groupName << " could not send request!";
+		return;
+	}
+
+	//parse response
+	boost::property_tree::ptree ptree;
+	std::string                 xml = response.substr(response.find("<"));
+	std::istringstream          treeStream(xml);
+	try {
+		boost::property_tree::read_xml(treeStream, ptree);
+	} catch (const std::exception &e) {
+		LOG(error) << _groupName << " got malformed XML response";
+		return;
+	}
+
+	//read values for every sensor from response
+	reading_t reading;
+	reading.timestamp = getTimestamp();
+
+	for (const auto &s : _sensors) {
+		try {
+			std::string                 readStr;
+			const xmlPathVector_t &     xmlPath = s->getXMLPath();
+			boost::property_tree::ptree node = ptree;
+			for (size_t i = 0; i < xmlPath.size(); i++) {
+				const std::string &       path = std::get<0>(xmlPath[i]);
+				const std::string &       child = std::get<1>(xmlPath[i]);
+				const attributesVector_t &attVec = std::get<2>(xmlPath[i]);
+
+				unsigned matchCount;
+				if (child != "") {
+					BOOST_FOREACH (boost::property_tree::ptree::value_type &v, node.get_child(path)) {
+						if (v.first == child) {
+							matchCount = 0;
+							for (size_t j = 0; j < attVec.size(); j++) {
+								std::string attributeVal = v.second.get_child("<xmlattr>." + attVec[j].first).data();
+
+								if (attributeVal != attVec[j].second) { //attribute values don't match
+									break;
+								} else {
+									matchCount++;
+								}
+							}
+							if (matchCount == attVec.size()) { //all attributes matched
+								readStr = v.second.data();
+								node = v.second;
+								break;
+							}
+						}
+					}
+				} else { //child == ""
+					readStr = node.get(path, "");
+					break; //last (part of the) path
+				}
+			}
+
+			if (readStr == "") {
+				throw std::runtime_error("Value not found!");
+			}
+			
+			size_t idx;
+			uint64_t ival = stoll(readStr, &idx);
+			double dval = .0;
+			bool have_float = false;
+			if (idx < readStr.size()) {
+				if (readStr[idx] == '.') {
+					dval = stod(readStr, &idx);
+					have_float = true;
+				}
+			}
+			uint64_t factor = 1;
+			if (idx < readStr.size()) {
+				switch(readStr[idx]) {
+					case 'k':
+					case 'K': factor = 1000ll;
+						break;
+					case 'm':
+					case 'M': factor = 1000000ll;
+						break;
+					default:
+						break;
+				}
+			}
+			if (have_float) {
+				reading.value = dval * factor;
+			} else {
+				reading.value = ival * factor;
+			}
+
+#ifdef DEBUG
+			LOG(debug) << _groupName << "::" << s->getName() << " raw reading: \"" << reading.value << "\"";
+#endif
+			s->storeReading(reading);
+		} catch (const std::exception &e) {
+			LOG(error) << _groupName << "::" << s->getName() << " could not read value: " << e.what();
+			continue;
+		}
+	}
+}
+
+void RESTSensorGroup::printGroupConfig(LOG_LEVEL ll, unsigned int leadingSpaces) {
+	std::string leading(leadingSpaces, ' ');
+	LOG_VAR(ll) << leading << "Endpoint: " << _endpoint;
+	LOG_VAR(ll) << leading << "Request:  " << _request;
+}

dcdbpusher/sensors/rest/RESTSensorGroup.h

+//================================================================================
+// Name        : RESTSensorGroup.h
+// Author      : Michael Ott
+// Contact     : info@dcdb.it
+// Copyright   : Leibniz Supercomputing Centre
+// Description : Header file for REST sensor group class.
+//================================================================================
+
+//================================================================================
+// This file is part of DCDB (DataCenter DataBase)
+// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//================================================================================
+
+#ifndef RESTSENSORGROUP_H_
+#define RESTSENSORGROUP_H_
+
+#include "../../includes/SensorGroupTemplateEntity.h"
+#include "RESTSensorBase.h"
+
+/**
+ * @brief SensorGroupTemplate specialization for this plugin.
+ *
+ * @ingroup rest
+ */
+class RESTSensorGroup : public SensorGroupTemplateEntity<RESTSensorBase, RESTUnit> {
+
+      public:
+	RESTSensorGroup(const std::string &name);
+	RESTSensorGroup(const RESTSensorGroup &other);
+	virtual ~RESTSensorGroup();
+	RESTSensorGroup &operator=(const RESTSensorGroup &other);
+
+	void               setEndpoint(const std::string &endpoint) { _endpoint = endpoint; }
+	const std::string &getEndpoint() { return _endpoint; }
+	void               setRequest(const std::string &request) { _request = request; }
+	const std::string &getRequest() { return _request; }
+
+	void printGroupConfig(LOG_LEVEL ll, unsigned int leadingSpaces) final override;
+
+      private:
+	void read() final override;
+
+	std::string _endpoint;
+	std::string _request;
+};
+
+#endif /* RESTSENSORGROUP_H_ */

dcdbpusher/sensors/rest/RESTUnit.cpp

+//================================================================================
+// Name        : RESTUnit.cpp
+// Author      : Michael Ott
+// Contact     : info@dcdb.it
+// Copyright   : Leibniz Supercomputing Centre
+// Description : Source file for RESTUnit class.
+//================================================================================
+
+//================================================================================
+// This file is part of DCDB (DataCenter DataBase)
+// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
+//
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//================================================================================
+
+#include "RESTUnit.h"
+#include "globalconfiguration.h"
+
+#include <iostream>
+
+#include <openssl/bio.h>
+#include <openssl/err.h>
+
+RESTUnit::RESTUnit(const std::string &name)
+    : EntityInterface(name),
+      _ctx(ssl::context::tlsv12_client),
+      _ssl(false) {
+}
+
+RESTUnit::RESTUnit(const RESTUnit &other)
+    : EntityInterface(other),
+      _ctx(ssl::context::tlsv12_client),
+      _baseURL(other._baseURL),
+      _hostname(other._hostname),
+      _port(other._port),
+      _path(other._path),
+      _authEndpoint(other._authEndpoint),
+      _authData(other._authData),
+      _ssl(false) {
+}
+
+RESTUnit::~RESTUnit() {
+}
+
+RESTUnit &RESTUnit::operator=(const RESTUnit &other) {
+	EntityInterface::operator=(other);
+	_baseURL = other._baseURL;
+	_hostname = other._hostname;
+	_port = other._port;
+	_path = other._path;
+	_authEndpoint = other._authEndpoint;
+	_authData = other._authData;
+	_ssl = other._ssl;
+
+	return *this;
+}
+
+void RESTUnit::execOnInit() {
+}
+
+bool RESTUnit::authenticate() {
+    tcp::resolver resolver(_ioc);
+    auto const hosts = resolver.resolve(_hostname, _port);
+    
+    beast::ssl_stream<beast::tcp_stream> stream(_ioc, _ctx);
+    beast::get_lowest_layer(stream).connect(hosts);
+
+    if (_ssl) {
+