Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 3f5c8649 authored by Michael Ott's avatar Michael Ott
Browse files

Leverage Boost BEAST for HTTP communication

parent 1afc3337
......@@ -3,7 +3,7 @@ DCDBDEPSPATH ?= $(DCDBBASEPATH)/deps
DCDBDEPLOYPATH ?= $(DCDBBASEPATH)/install
# dcdbpusher plugins to be built
PLUGINS = sysfs ipmi pdu bacnet snmp procfs tester gpfsmon msr
PLUGINS = sysfs ipmi pdu bacnet snmp procfs tester gpfsmon msr rest
# data analytics plugins to be built
OPERATORS = aggregator smoothing regressor classifier clustering cssignatures job_aggregator testeroperator filesink smucngperf persystsql coolingcontrol healthchecker
......
......@@ -46,12 +46,13 @@ void RESTConfigurator::sensorBase(RESTSensorBase &s, CFG_VAL config) {
void RESTConfigurator::sensorGroup(RESTSensorGroup &s, CFG_VAL config) {
ADD {
ATTRIBUTE("endpoint", setEndpoint);
ATTRIBUTE("request", setRequest);
}
}
void RESTConfigurator::sensorEntity(RESTUnit &s, CFG_VAL config) {
ADD {
ATTRIBUTE("host", setHost);
ATTRIBUTE("baseurl", setBaseURL);
}
}
......@@ -38,12 +38,14 @@ RESTSensorGroup::RESTSensorGroup(const std::string &name)
RESTSensorGroup::RESTSensorGroup(const RESTSensorGroup &other)
: SensorGroupTemplateEntity(other),
_endpoint(other._endpoint),
_request(other._request) {}
RESTSensorGroup::~RESTSensorGroup() {}
RESTSensorGroup &RESTSensorGroup::operator=(const RESTSensorGroup &other) {
SensorGroupTemplate::operator=(other);
_endpoint = other._endpoint;
_request = other._request;
return *this;
......@@ -52,7 +54,7 @@ RESTSensorGroup &RESTSensorGroup::operator=(const RESTSensorGroup &other) {
void RESTSensorGroup::read() {
//send request
std::string response;
if (!_entity->sendRequest(_request, response)) {
if (!_entity->sendRequest(_endpoint, _request, response)) {
LOG(error) << _groupName << " could not send request!";
return;
}
......@@ -112,7 +114,35 @@ void RESTSensorGroup::read() {
if (readStr == "") {
throw std::runtime_error("Value not found!");
}
reading.value = stoll(readStr);
size_t idx;
uint64_t ival = stoll(readStr, &idx);
double dval = .0;
bool have_float = false;
if (idx < readStr.size()) {
if (readStr[idx] == '.') {
dval = stod(readStr, &idx);
have_float = true;
}
}
uint64_t factor = 1;
if (idx < readStr.size()) {
switch(readStr[idx]) {
case 'k':
case 'K': factor = 1000ll;
break;
case 'm':
case 'M': factor = 1000000ll;
break;
default:
break;
}
}
if (have_float) {
reading.value = dval * factor;
} else {
reading.value = ival * factor;
}
#ifdef DEBUG
LOG(debug) << _groupName << "::" << s->getName() << " raw reading: \"" << reading.value << "\"";
......@@ -127,5 +157,6 @@ void RESTSensorGroup::read() {
void RESTSensorGroup::printGroupConfig(LOG_LEVEL ll, unsigned int leadingSpaces) {
std::string leading(leadingSpaces, ' ');
LOG_VAR(ll) << leading << "Request: " << _request;
LOG_VAR(ll) << leading << "Endpoint: " << _endpoint;
LOG_VAR(ll) << leading << "Request: " << _request;
}
......@@ -44,6 +44,8 @@ class RESTSensorGroup : public SensorGroupTemplateEntity<RESTSensorBase, RESTUni
virtual ~RESTSensorGroup();
RESTSensorGroup &operator=(const RESTSensorGroup &other);
void setEndpoint(const std::string &endpoint) { _endpoint = endpoint; }
const std::string &getEndpoint() { return _endpoint; }
void setRequest(const std::string &request) { _request = request; }
const std::string &getRequest() { return _request; }
......@@ -52,6 +54,7 @@ class RESTSensorGroup : public SensorGroupTemplateEntity<RESTSensorBase, RESTUni
private:
void read() final override;
std::string _endpoint;
std::string _request;
};
......
......@@ -26,6 +26,7 @@
//================================================================================
#include "RESTUnit.h"
#include "globalconfiguration.h"
#include <iostream>
......@@ -34,92 +35,80 @@
RESTUnit::RESTUnit(const std::string &name)
: EntityInterface(name),
_ctx(nullptr) {
_ctx(ssl::context::tlsv12_client),
_ssl(false) {
}
RESTUnit::RESTUnit(const RESTUnit &other)
: EntityInterface(other),
_ctx(nullptr) {
_ctx(ssl::context::tlsv12_client),
_ssl(false) {
}
RESTUnit::~RESTUnit() {
if (_ctx) {
SSL_CTX_free(_ctx);
}
}
RESTUnit &RESTUnit::operator=(const RESTUnit &other) {
EntityInterface::operator=(other);
_ctx = nullptr;
_ssl = other._ssl;
return *this;
}
void RESTUnit::execOnInit() {
SSL_library_init();
SSL_load_error_strings();
ERR_load_BIO_strings();
OpenSSL_add_all_algorithms();
_ctx = SSL_CTX_new(SSLv23_method());
}
bool RESTUnit::sendRequest(const std::string &request, std::string &response) {
BIO *bio;
if (!_ctx) {
LOG(error) << "OpenSSL: Could not create context: " << ERR_reason_error_string(ERR_get_error());
return false;
}
/*
if (!SSL_CTX_load_verify_locations(ctx, "/home/micha/LRZ/dcdbOwnFork/deps/openssl-1.0.2l/certs/demo/ca-cert.pem", NULL)) {
LOG(error) << "OpenSSL: Could not load certificate: " << ERR_reason_error_string(ERR_get_error());
SSL_CTX_free(ctx);
return;
bool RESTUnit::sendRequest(const std::string &endpoint, const std::string &request, std::string &response) {
// Look up the domain name
tcp::resolver resolver(_ioc);
auto const hosts = resolver.resolve(_hostname, _port);
beast::ssl_stream<beast::tcp_stream> stream(_ioc, _ctx);
// Make the connection on the IP address we get from a lookup
beast::get_lowest_layer(stream).connect(hosts);
if (_ssl) {
// Set SNI Hostname (many hosts need this to handshake successfully)
if(! SSL_set_tlsext_host_name(stream.native_handle(), _hostname.c_str()))
{
beast::error_code ec{static_cast<int>(::ERR_get_error()), net::error::get_ssl_category()};
throw beast::system_error{ec};
}
*/
bio = BIO_new_ssl_connect(_ctx);
BIO_set_conn_hostname(bio, _name.c_str());
if (BIO_do_connect(bio) <= 0) {
LOG(error) << "OpenSSL: Could not connect: " << ERR_reason_error_string(ERR_get_error());
BIO_free_all(bio);
return false;
}
size_t len = request.length();
const char *reqBuf = request.c_str(); //request
char resBuf[2048];
memset(resBuf, 0, 2048);
// do not bother too long if a write/read failed. Sensor intervals are usually small, just try again when next sensor wants to read.
// send request
if (BIO_write(bio, reqBuf, len) <= 0) {
LOG(error) << "OpenSSL: Could not send request: " << ERR_reason_error_string(ERR_get_error());
BIO_free_all(bio);
return false;
}
// read reply
response.clear();
while ((len = BIO_read(bio, resBuf, sizeof(resBuf))) > 0) {
resBuf[len] = 0;
response.append(resBuf);
}
if (len < 0) {
std::cerr << "OpenSSL: Could not read response: " << ERR_reason_error_string(ERR_get_error()) << std::endl;
BIO_free_all(bio);
return false;
}
BIO_free_all(bio);
// Perform the SSL handshake
stream.handshake(ssl::stream_base::client);
}
// Set up an HTTP GET request message
http::request<http::string_body> req{http::verb::get, _path + endpoint, 11};
req.set(http::field::host, _hostname);
req.body() = request;
req.prepare_payload();
// Send the HTTP request to the remote host
if (_ssl) {
http::write(stream, req);
} else {
http::write(beast::get_lowest_layer(stream), req);
}
// This buffer is used for reading and must be persisted
beast::flat_buffer buffer;
// Declare a container to hold the response
http::response<http::string_body> res;
// Receive the HTTP response
if (_ssl) {
http::read(stream, buffer, res);
} else {
http::read(beast::get_lowest_layer(stream), buffer, res);
}
if (res.body().size() > 0) {
response = res.body();
return true;
} else {
return false;
}
}
......@@ -35,8 +35,24 @@
#include <utility>
#include <vector>
#include <boost/beast/core.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/version.hpp>
#include <boost/asio/connect.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/ssl/error.hpp>
#include <boost/asio/ssl/stream.hpp>
#include <boost/regex.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/property_tree/ptree.hpp>
#include <openssl/ssl.h>
namespace beast = boost::beast; // from <boost/beast.hpp>
namespace http = beast::http; // from <boost/beast/http.hpp>
namespace net = boost::asio; // from <boost/asio.hpp>
namespace ssl = net::ssl; // from <boost/asio/ssl.hpp>
using tcp = net::ip::tcp; // from <boost/asio/ip/tc
typedef std::vector<std::pair<std::string, std::string>> attributesVector_t;
typedef std::vector<std::tuple<std::string, std::string, attributesVector_t>> xmlPathVector_t;
......@@ -54,16 +70,31 @@ class RESTUnit : public EntityInterface {
virtual ~RESTUnit();
RESTUnit &operator=(const RESTUnit &other);
void setHost(const std::string &host) {
size_t pos = host.find(':');
if (pos != std::string::npos) {
_name = host;
} else {
_name = host + ":443";
void setBaseURL(const std::string &baseURL) {
_baseURL = baseURL;
// URL parsing courtesy of https://stackoverflow.com/a/61214599
boost::regex ex("(http|https)://([^/ :]+):?([^/ ]*)(/?[^ #?]*)\\x3f?([^ #]*)#?([^ ]*)");
boost::cmatch what;
if(regex_match(baseURL.c_str(), what, ex))
{
std::string protocol = std::string(what[1].first, what[1].second);
_hostname = std::string(what[2].first, what[2].second);
_port = std::string(what[3].first, what[3].second);
_path = std::string(what[4].first, what[4].second);
std::string query = std::string(what[5].first, what[5].second);
if (boost::iequals(protocol, "https")) {
_ssl = true;
}
if (_port.size() == 0) {
if (_ssl) {
_port = "443";
} else {
_port = "80";
}
}
}
}
const std::string &getHost() { return _name; }
const std::string &getBaseURL() { return _baseURL; }
/**
* Send the request to the host and write the response into response.
......@@ -73,12 +104,18 @@ class RESTUnit : public EntityInterface {
*
* @return True on success, false otherwise
*/
bool sendRequest(const std::string &request, std::string &response);
bool sendRequest(const std::string &endpoint, const std::string &request, std::string &response);
void execOnInit() final override;
private:
SSL_CTX *_ctx;
net::io_context _ioc;
ssl::context _ctx;
std::string _baseURL;
std::string _hostname;
std::string _port;
std::string _path;
bool _ssl;
};
#endif /* RESTUNIT_H_ */
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment