The expiration time for new job artifacts in CI/CD pipelines is now 30 days (GitLab default). Previously generated artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 1afc3337 authored by Michael Ott's avatar Michael Ott
Browse files

Clone PDU plugin as REST plugin to start development of a proper HTTP compatible plugin

parent 46111551
......@@ -121,3 +121,7 @@ libdcdbplugin_caliper.$(LIBEXT): sensors/caliper/CaliperSensorGroup.o sensors/ca
libdcdbplugin_nvml.$(LIBEXT): sensors/nvml/nvmlSensorGroup.o sensors/nvml/nvmlConfigurator.o
$(NVCC) -shared --compiler-options '-fPIC' -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system -lnvidia-ml
libdcdbplugin_rest.$(LIBEXT): sensors/rest/RESTSensorGroup.o sensors/rest/RESTUnit.o sensors/rest/RESTConfigurator.o
$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lcrypto -lssl -lboost_log -lboost_system
//================================================================================
// Name : RESTConfigurator.cpp
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Source file for REST plugin configurator class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include "RESTConfigurator.h"
#include <iostream>
#include <sstream>
RESTConfigurator::RESTConfigurator() {
_entityName = "host";
_groupName = "group";
_baseName = "sensor";
}
RESTConfigurator::~RESTConfigurator() {}
void RESTConfigurator::sensorBase(RESTSensorBase &s, CFG_VAL config) {
ADD {
ATTRIBUTE("path", setXMLPath);
}
}
void RESTConfigurator::sensorGroup(RESTSensorGroup &s, CFG_VAL config) {
ADD {
ATTRIBUTE("request", setRequest);
}
}
void RESTConfigurator::sensorEntity(RESTUnit &s, CFG_VAL config) {
ADD {
ATTRIBUTE("host", setHost);
}
}
//================================================================================
// Name : RESTConfigurator.h
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Header file for REST plugin configurator class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#ifndef RESTCONFIGURATOR_H_
#define RESTCONFIGURATOR_H_
#include "../../includes/ConfiguratorTemplateEntity.h"
#include "RESTSensorGroup.h"
#include "RESTUnit.h"
/**
* @brief ConfiguratorTemplate specialization for this plugin.
*
* @ingroup rest
*/
class RESTConfigurator : public ConfiguratorTemplateEntity<RESTSensorBase, RESTSensorGroup, RESTUnit> {
public:
RESTConfigurator();
virtual ~RESTConfigurator();
protected:
/* Overwritten from ConfiguratorTemplate */
void sensorBase(RESTSensorBase &s, CFG_VAL config) override;
void sensorGroup(RESTSensorGroup &s, CFG_VAL config) override;
void sensorEntity(RESTUnit &s, CFG_VAL config) override;
private:
/**
* Split the given string into multiple parts which are later required to find the sensor value in the boost property tree.
* Store the individual parts into the sensors _xmlPathVector
* @param sensor Sensor where the path belongs to
* @param pathString Complete unrefined string as read from the config file
*/
void parsePathString(RESTSensorBase &sensor, const std::string &pathString);
};
extern "C" ConfiguratorInterface *create() {
return new RESTConfigurator;
}
extern "C" void destroy(ConfiguratorInterface *c) {
delete c;
}
#endif /* RESTCONFIGURATOR_H_ */
//================================================================================
// Name : RESTSensorBase.h
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Sensor base class for REST plugin.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
/**
* @defgroup rest REST plugin
* @ingroup pusherplugins
*
* @brief Collect data from REST APIs
*/
#ifndef RESTSENSORBASE_H_
#define RESTSENSORBASE_H_
#include "RESTUnit.h"
#include "sensorbase.h"
#include <sstream>
/**
* @brief SensorBase specialization for this plugin.
*
* @ingroup rest
*/
class RESTSensorBase : public SensorBase {
public:
RESTSensorBase(const std::string &name)
: SensorBase(name) {}
RESTSensorBase(const RESTSensorBase &other)
: SensorBase(other),
_xmlPath(other._xmlPath) {}
virtual ~RESTSensorBase() {}
RESTSensorBase &operator=(const RESTSensorBase &other) {
SensorBase::operator=(other);
_xmlPath = other._xmlPath;
return *this;
}
const xmlPathVector_t &getXMLPath() const { return _xmlPath; }
std::string getXMLPathString() const {
std::stringstream ss;
for (const auto &i : _xmlPath) {
ss << "." << std::get<0>(i) << "." << std::get<1>(i);
for (const auto &j : std::get<2>(i)) {
ss << "(" << j.first << "=" << j.second << ")";
}
}
return ss.str();
}
void setXMLPath(const std::string &path) {
std::vector<std::string> subStrings;
std::stringstream pathStream(path);
std::string item;
//split into parts if a attribute (indicated by '(' ')' ) was defined
while (std::getline(pathStream, item, ')')) {
subStrings.push_back(item);
}
for (auto subStr : subStrings) {
//extract the attributes from the path-parts
if (subStr.find('(') != std::string::npos) { //attribute specified
std::stringstream pathWithAttributesSStream(subStr);
//split into path and attributes string
std::string subPath, attributeString;
std::getline(pathWithAttributesSStream, subPath, '(');
std::getline(pathWithAttributesSStream, attributeString);
if (subPath.front() == '.') {
subPath.erase(0, 1);
}
//now further split the attributes string as multiple attributes could be defined
std::vector<std::string> attributes;
std::stringstream attributeStream(attributeString);
while (std::getline(attributeStream, item, ',')) {
attributes.push_back(item);
}
attributesVector_t attrs;
for (auto att : attributes) {
//part attributes into name and value
if (att.find('=') != std::string::npos) {
std::stringstream attStream(att);
std::string attName, attVal;
std::getline(attStream, attName, '=');
std::getline(attStream, attVal);
attrs.push_back(std::make_pair(attName, attVal));
} else { //should not happen. If it does the path was malformed
//LOG(error) << " Could not parse XML-path!";
return;
}
}
//split of the last child in the path. Required to iterate over multiple nodes which only differ in the attributes with BOOST_FOREACH
auto index = subPath.find_last_of('.');
if (index != std::string::npos) {
std::string subPathChild(subPath.substr(++index));
subPath.erase(--index);
_xmlPath.push_back(std::make_tuple(subPath, subPathChild, attrs));
} else { //the path contained only one node
_xmlPath.push_back(std::make_tuple("", subPath, attrs));
}
} else { //no attributes specified. Last (sub)path
if (subStr.front() == '.') {
subStr.erase(0, 1);
}
_xmlPath.push_back(std::make_tuple(subStr, "", attributesVector_t()));
break;
}
}
}
void printConfig(LOG_LEVEL ll, LOGGER &lg, unsigned leadingSpaces = 16) {
std::string leading(leadingSpaces, ' ');
LOG_VAR(ll) << leading << " XML Path: " << getXMLPathString();
}
protected:
xmlPathVector_t _xmlPath;
};
#endif /* RESTSENSORBASE_H_ */
//================================================================================
// Name : RESTSensorGroup.cpp
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Source file for REST sensor group class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include "RESTSensorGroup.h"
#include <sstream>
#include <boost/foreach.hpp>
#include <boost/property_tree/xml_parser.hpp>
RESTSensorGroup::RESTSensorGroup(const std::string &name)
: SensorGroupTemplateEntity(name) {
}
RESTSensorGroup::RESTSensorGroup(const RESTSensorGroup &other)
: SensorGroupTemplateEntity(other),
_request(other._request) {}
RESTSensorGroup::~RESTSensorGroup() {}
RESTSensorGroup &RESTSensorGroup::operator=(const RESTSensorGroup &other) {
SensorGroupTemplate::operator=(other);
_request = other._request;
return *this;
}
void RESTSensorGroup::read() {
//send request
std::string response;
if (!_entity->sendRequest(_request, response)) {
LOG(error) << _groupName << " could not send request!";
return;
}
//parse response
boost::property_tree::ptree ptree;
std::string xml = response.substr(response.find("<"));
std::istringstream treeStream(xml);
try {
boost::property_tree::read_xml(treeStream, ptree);
} catch (const std::exception &e) {
LOG(error) << _groupName << " got malformed XML response";
return;
}
//read values for every sensor from response
reading_t reading;
reading.timestamp = getTimestamp();
for (const auto &s : _sensors) {
try {
std::string readStr;
const xmlPathVector_t & xmlPath = s->getXMLPath();
boost::property_tree::ptree node = ptree;
for (size_t i = 0; i < xmlPath.size(); i++) {
const std::string & path = std::get<0>(xmlPath[i]);
const std::string & child = std::get<1>(xmlPath[i]);
const attributesVector_t &attVec = std::get<2>(xmlPath[i]);
unsigned matchCount;
if (child != "") {
BOOST_FOREACH (boost::property_tree::ptree::value_type &v, node.get_child(path)) {
if (v.first == child) {
matchCount = 0;
for (size_t j = 0; j < attVec.size(); j++) {
std::string attributeVal = v.second.get_child("<xmlattr>." + attVec[j].first).data();
if (attributeVal != attVec[j].second) { //attribute values don't match
break;
} else {
matchCount++;
}
}
if (matchCount == attVec.size()) { //all attributes matched
readStr = v.second.data();
node = v.second;
break;
}
}
}
} else { //child == ""
readStr = node.get(path, "");
break; //last (part of the) path
}
}
if (readStr == "") {
throw std::runtime_error("Value not found!");
}
reading.value = stoll(readStr);
#ifdef DEBUG
LOG(debug) << _groupName << "::" << s->getName() << " raw reading: \"" << reading.value << "\"";
#endif
s->storeReading(reading);
} catch (const std::exception &e) {
LOG(error) << _groupName << "::" << s->getName() << " could not read value: " << e.what();
continue;
}
}
}
void RESTSensorGroup::printGroupConfig(LOG_LEVEL ll, unsigned int leadingSpaces) {
std::string leading(leadingSpaces, ' ');
LOG_VAR(ll) << leading << "Request: " << _request;
}
//================================================================================
// Name : RESTSensorGroup.h
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Header file for REST sensor group class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#ifndef RESTSENSORGROUP_H_
#define RESTSENSORGROUP_H_
#include "../../includes/SensorGroupTemplateEntity.h"
#include "RESTSensorBase.h"
/**
* @brief SensorGroupTemplate specialization for this plugin.
*
* @ingroup rest
*/
class RESTSensorGroup : public SensorGroupTemplateEntity<RESTSensorBase, RESTUnit> {
public:
RESTSensorGroup(const std::string &name);
RESTSensorGroup(const RESTSensorGroup &other);
virtual ~RESTSensorGroup();
RESTSensorGroup &operator=(const RESTSensorGroup &other);
void setRequest(const std::string &request) { _request = request; }
const std::string &getRequest() { return _request; }
void printGroupConfig(LOG_LEVEL ll, unsigned int leadingSpaces) final override;
private:
void read() final override;
std::string _request;
};
#endif /* RESTSENSORGROUP_H_ */
//================================================================================
// Name : RESTUnit.cpp
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Source file for RESTUnit class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include "RESTUnit.h"
#include <iostream>
#include <openssl/bio.h>
#include <openssl/err.h>
RESTUnit::RESTUnit(const std::string &name)
: EntityInterface(name),
_ctx(nullptr) {
}
RESTUnit::RESTUnit(const RESTUnit &other)
: EntityInterface(other),
_ctx(nullptr) {
}
RESTUnit::~RESTUnit() {
if (_ctx) {
SSL_CTX_free(_ctx);
}
}
RESTUnit &RESTUnit::operator=(const RESTUnit &other) {
EntityInterface::operator=(other);
_ctx = nullptr;
return *this;
}
void RESTUnit::execOnInit() {
SSL_library_init();
SSL_load_error_strings();
ERR_load_BIO_strings();
OpenSSL_add_all_algorithms();
_ctx = SSL_CTX_new(SSLv23_method());
}
bool RESTUnit::sendRequest(const std::string &request, std::string &response) {
BIO *bio;
if (!_ctx) {
LOG(error) << "OpenSSL: Could not create context: " << ERR_reason_error_string(ERR_get_error());
return false;
}
/*
if (!SSL_CTX_load_verify_locations(ctx, "/home/micha/LRZ/dcdbOwnFork/deps/openssl-1.0.2l/certs/demo/ca-cert.pem", NULL)) {
LOG(error) << "OpenSSL: Could not load certificate: " << ERR_reason_error_string(ERR_get_error());
SSL_CTX_free(ctx);
return;
}
*/
bio = BIO_new_ssl_connect(_ctx);
BIO_set_conn_hostname(bio, _name.c_str());
if (BIO_do_connect(bio) <= 0) {
LOG(error) << "OpenSSL: Could not connect: " << ERR_reason_error_string(ERR_get_error());
BIO_free_all(bio);
return false;
}
size_t len = request.length();
const char *reqBuf = request.c_str(); //request
char resBuf[2048];
memset(resBuf, 0, 2048);
// do not bother too long if a write/read failed. Sensor intervals are usually small, just try again when next sensor wants to read.
// send request
if (BIO_write(bio, reqBuf, len) <= 0) {
LOG(error) << "OpenSSL: Could not send request: " << ERR_reason_error_string(ERR_get_error());
BIO_free_all(bio);
return false;
}
// read reply
response.clear();
while ((len = BIO_read(bio, resBuf, sizeof(resBuf))) > 0) {
resBuf[len] = 0;
response.append(resBuf);
}
if (len < 0) {
std::cerr << "OpenSSL: Could not read response: " << ERR_reason_error_string(ERR_get_error()) << std::endl;
BIO_free_all(bio);
return false;
}
BIO_free_all(bio);
return true;
}
//================================================================================
// Name : RESTUnit.h