Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 91d5be44 authored by Micha Mueller's avatar Micha Mueller
Browse files

First (incomplete) prototype of new RestAPI class for dcdbpusher

parent 3e27e299
/*
* RestAPI.cpp
*
* Created on: 22.05.2019
* Author: Micha Mueller
*/
#include "RestAPI.h"
#include <sstream>
#include <string>
#include <boost/property_tree/ptree.hpp>
RestAPI::RestAPI(serverSettings_t settings,
pluginVector_t& plugins,
MQTTPusher* mqttPusher,
AnalyticsManager* manager,
boost::asio::io_service& io) :
RESTHttpsServer(settings),
_plugins(plugins),
_mqttPusher(mqttPusher),
_manager(manager),
_io(io) {
addEndpoint("/analytics/help", {http::verb::get, stdBind(GET_analytics_help)});
addEndpoint("/analytics/plugins", {http::verb::get, stdBind(GET_analytics_plugins)});
addEndpoint("/analytics/sensors", {http::verb::get, stdBind(GET_analytics_sensors)});
addEndpoint("/analytics/units", {http::verb::get, stdBind(GET_analytics_units)});
addEndpoint("/analytics/analyzers", {http::verb::get, stdBind(GET_analytics_analyzers)});
addEndpoint("/help", {http::verb::get, stdBind(GET_help)});
addEndpoint("/plugins", {http::verb::get, stdBind(GET_plugins)});
addEndpoint("/sensors", {http::verb::get, stdBind(GET_sensors)});
addEndpoint("/average", {http::verb::get, stdBind(GET_average)});
addEndpoint("/analytics/start", {http::verb::put, stdBind(PUT_analytics_start)});
addEndpoint("/analytics/stop", {http::verb::put, stdBind(PUT_analytics_stop)});
addEndpoint("/analytics/reload", {http::verb::put, stdBind(PUT_analytics_reload)});
addEndpoint("/analytics/compute", {http::verb::put, stdBind(PUT_analytics_compute)});
addEndpoint("/analytics/analyzer", {http::verb::put, stdBind(PUT_analytics_analyzer)});
addEndpoint("/start", {http::verb::put, stdBind(PUT_start)});
addEndpoint("/stop", {http::verb::put, stdBind(PUT_stop)});
addEndpoint("/reload", {http::verb::put, stdBind(PUT_reload)});
}
void RestAPI::GET_analytics_help(endpointArgs){
if (!managerLoaded(res)) {
return;
}
res.body() = _manager->restCheatSheet;
res.result(http::status::ok);
}
void RestAPI::GET_analytics_plugins(endpointArgs) {
if (!managerLoaded(res)) {
return;
}
std::ostringstream data;
if (getQuery("json", queries) == "true") {
boost::property_tree::ptree root, plugins;
for(const auto& p : _manager->getPlugins()) {
plugins.put(p.id, "");
}
root.add_child("plugins", plugins);
boost::property_tree::write_json(data, root, true);
} else {
for(const auto& p : _plugins) {
data << p.id << "\n";
}
}
res.body() = data.str();
res.result(http::status::ok);
}
void RestAPI::GET_analytics_sensors(endpointArgs) {
if (!managerLoaded(res)) {
return;
}
const std::string plugin = getQuery("plugin", queries);
const std::string analyzer = getQuery("analyzer", queries);
if (plugin == "") {
const std::string err = "Request malformed: plugin query missing";
RESTAPILOG(error) << err;
res.body() = err;
res.result(http::status::bad_request);
return;
}
bool found = false;
std::ostringstream data;
for (const auto& p : _manager->getPlugins()) {
if (p.id == plugin) {
if (getQuery("json", queries) == "true") {
boost::property_tree::ptree root, sensors;
// In JSON mode, sensors are arranged hierarchically by plugin->analyzer->sensor
for (const auto& a : p.configurator->getAnalyzers()) {
if (a->getStreaming() && (analyzer == "" || analyzer == a->getName())) {
found = true;
boost::property_tree::ptree group;
for (const auto& u : a->getUnits()) {
for (const auto& s : u->getBaseOutputs()) {
// Explicitly adding nodes to the ptree as to prevent BOOST from performing
// parsing on the node names
group.push_back(boost::property_tree::ptree::value_type(s->getName(), boost::property_tree::ptree(s->getMqtt())));
}
}
sensors.add_child(a->getName(), group);
}
}
root.add_child(p.id, sensors);
boost::property_tree::write_json(data, root, true);
} else {
for (const auto& a : p.configurator->getAnalyzers()) {
if (a->getStreaming() && (analyzer == "" || analyzer == a->getName())) {
found = true;
for (const auto& u : a->getUnits()) {
for (const auto& s : u->getBaseOutputs()) {
data << a->getName() << "." << s->getName() << " " << s->getMqtt() << "\n";
}
}
}
}
}
res.body() = data.str();
res.result(http::status::ok);
break;
}
}
if (!found) {
res.body() = "Plugin or analyzer not found!";
res.result(http::status::not_found);
}
}
void RestAPI::GET_analytics_units(endpointArgs) {
if (!managerLoaded(res)) {
return;
}
const std::string plugin = getQuery("plugin", queries);
const std::string analyzer = getQuery("analyzer", queries);
if (plugin == "") {
const std::string err = "Request malformed: plugin query missing";
RESTAPILOG(error) << err;
res.body() = err;
res.result(http::status::bad_request);
return;
}
bool found = false;
std::ostringstream data;
for (const auto& p : _manager->getPlugins()) {
if (p.id == plugin) {
if (getQuery("json", queries) == "true") {
boost::property_tree::ptree root, units;
// In JSON mode, sensors are arranged hierarchically by plugin->analyzer->sensor
for (const auto& a : p.configurator->getAnalyzers())
if (a->getStreaming() && (analyzer == "" || analyzer == a->getName())) {
found = true;
boost::property_tree::ptree group;
for (const auto& u : a->getUnits()) {
group.push_back(boost::property_tree::ptree::value_type(u->getName(), boost::property_tree::ptree()));
}
units.add_child(a->getName(), group);
}
root.add_child(p.id, units);
boost::property_tree::write_json(data, root, true);
} else {
for (const auto& a : p.configurator->getAnalyzers()) {
if (a->getStreaming() && (analyzer == "" || analyzer == a->getName())) {
found = true;
for (const auto& u : a->getUnits()) {
data << a->getName() << "." << u->getName() << "\n";
}
}
}
}
res.body() = data.str();
res.result(http::status::ok);
break;
}
}
if (!found) {
res.body() = "Plugin or analyzer not found!";
res.result(http::status::not_found);
}
}
void RestAPI::GET_analytics_analyzers(endpointArgs) {
if (!managerLoaded(res)) {
return;
}
const std::string plugin = getQuery("plugin", queries);
if (plugin == "") {
const std::string err = "Request malformed: plugin query missing";
RESTAPILOG(error) << err;
res.body() = err;
res.result(http::status::bad_request);
return;
}
res.body() = "Plugin not found!\n";
res.result(http::status::not_found);
std::ostringstream data;
for (const auto& p : _manager->getPlugins()) {
if (p.id == plugin) {
if (getQuery("json", queries) == "true") {
boost::property_tree::ptree root, analyzers;
// For each analyzer, we output its type as well
for (const auto& a : p.configurator->getAnalyzers()) {
analyzers.push_back(boost::property_tree::ptree::value_type(a->getName(), boost::property_tree::ptree(a->getStreaming() ? "streaming" : "on-demand")));
}
root.add_child(p.id, analyzers);
boost::property_tree::write_json(data, root, true);
} else {
for (const auto& a : p.configurator->getAnalyzers()) {
data << a->getName() << " " << (a->getStreaming() ? "streaming\n" : "on-demand\n");
}
}
res.body() = data.str();
res.result(http::status::ok);
return;
}
}
}
void RestAPI::GET_help(endpointArgs) {
res.body() = restCheatSheet + _manager->restCheatSheet;
res.result(http::status::ok);
}
void RestAPI::GET_plugins(endpointArgs) {
std::ostringstream data;
if (getQuery("json", queries) == "true") {
boost::property_tree::ptree root, plugins;
for (const auto& p : _plugins) {
plugins.put(p.id, "");
}
root.add_child("plugins", plugins);
boost::property_tree::write_json(data, root, true);
} else {
for (const auto& p : _plugins) {
data << p.id << "\n";
}
}
res.body() = data.str();
res.result(http::status::ok);
}
void RestAPI::GET_sensors(endpointArgs) {
const std::string plugin = getQuery("plugin", queries);
if (plugin == "") {
const std::string err = "Request malformed: plugin query missing";
RESTAPILOG(error) << err;
res.body() = err;
res.result(http::status::bad_request);
return;
}
res.body() = "Plugin not found!\n";
res.result(http::status::not_found);
for(const auto& p : _plugins) {
if (p.id == plugin) {
std::ostringstream data;
if (getQuery("json", queries) == "true") {
boost::property_tree::ptree root, sensors;
for(const auto& g : p.configurator->getSensorGroups()) {
boost::property_tree::ptree group;
for(const auto& s : g->getSensors()) {
group.put(s->getName(), s->getMqtt());
}
sensors.add_child(g->getGroupName(), group);
}
root.add_child(p.id, sensors);
boost::property_tree::write_json(data, root, true);
} else {
for(const auto& g : p.configurator->getSensorGroups()) {
for(const auto& s : g->getSensors()) {
data << g->getGroupName() << "." << s->getName() << " " << s->getMqtt() << "\n";
}
}
}
res.body() = data.str();
res.result(http::status::ok);
return;
}
}
}
void RestAPI::GET_average(endpointArgs) {
const std::string plugin = getQuery("plugin", queries);
const std::string sensor = getQuery("sensor", queries);
const std::string interval = getQuery("interval", queries);
if (plugin == "" || sensor == "") {
const std::string err = "Request malformed: plugin or sensor query missing";
RESTAPILOG(error) << err;
res.body() = err;
res.result(http::status::bad_request);
return;
}
uint64_t time = 0;
if (interval != "") {
time = std::stoul(interval);
}
res.body() = "Plugin not found!\n";
res.result(http::status::not_found);
for(const auto& p : _plugins) {
if (p.id == plugin) {
res.body() = "Sensor not found!";
for(const auto& g : p.configurator->getSensorGroups()) {
for(const auto& s : g->getSensors()) {
if (s->getName() == sensor && s->isInit()) {
uint64_t avg = 0;
try {
avg = s->getCache()->getAverage(S_TO_NS(time));
} catch(const std::exception& e) {
res.body() = "Unable to compute average: ";
res.body() += e.what();
res.result(http::status::internal_server_error);
return;
}
res.body() = plugin + "::" + sensor + " Average of last " +
std::to_string(time) + " seconds is " + std::to_string(avg) + "\n";
res.result(http::status::ok);
return;
}
}
}
}
}
for(auto& p : _manager->getPlugins()) {
if (p.id == plugin) {
res.body() = "Sensor not found!\n";
for(const auto& a : p.configurator->getAnalyzers()) {
if(a->getStreaming()) {
for(const auto& u : a->getUnits()) {
for (const auto& s : u->getBaseOutputs()) {
if (s->getName() == sensor && s->isInit()) {
uint64_t avg = 0;
try {
avg = s->getCache()->getAverage(S_TO_NS(time));
} catch(const std::exception& e) {
res.body() = "Unable to compute average: ";
res.body() += e.what();
res.result(http::status::internal_server_error);
return;
}
res.body() = plugin + "::" + sensor + " Average of last " +
std::to_string(time) + " seconds is " + std::to_string(avg) + "\n";
res.result(http::status::ok);
return;
}
}
}
}
}
}
}
}
//TODO put methods
void RestAPI::removeTopics(dl_t p) {
MQTTChecker& mqttCheck = MQTTChecker::getInstance();
for(const auto& g : p.configurator->getSensorGroups()) {
mqttCheck.removeGroup(g->getGroupName());
for (const auto &s : g->getSensors()) {
mqttCheck.removeTopic(s->getMqtt());
mqttCheck.removeName(s->getName());
}
}
}
bool RestAPI::checkTopics(dl_t p) {
MQTTChecker& mqttCheck = MQTTChecker::getInstance();
bool validTopics=true;
for(const auto& g : p.configurator->getSensorGroups()) {
if (!mqttCheck.checkGroup(g->getGroupName()))
validTopics = false;
for (const auto &s : g->getSensors())
if (!mqttCheck.checkTopic(s->getMqtt()) || !mqttCheck.checkName(s->getName()))
validTopics = false;
}
return validTopics;
}
/*
* RestAPI.h
*
* Created on: 22.05.2019
* Author: Micha Mueller
*/
#ifndef DCDBPUSHER_RESTAPI_H_
#define DCDBPUSHER_RESTAPI_H_
#include "RESTHttpsServer.h"
#include <boost/asio.hpp>
#include "includes/PluginDefinitions.h"
#include "../analytics/AnalyticsManager.h"
#include "mqttchecker.h"
#include "MQTTPusher.h"
#define endpointArgs http::response<http::string_body>& res, queries_t& queries
#define stdBind(fun) std::bind(&RestAPI::fun, \
this, \
std::placeholders::_1, \
std::placeholders::_2)
class RestAPI : public RESTHttpsServer {
public:
RestAPI(serverSettings_t settings,
pluginVector_t& plugins,
MQTTPusher* mqttPusher,
AnalyticsManager* manager,
boost::asio::io_service& io);
virtual ~RestAPI() {}
//TODO rewrite help-section + README + endpoint docs once finished
// String used as a response for the REST GET /help command
const string restCheatSheet = "dcdbpusher RESTful API cheatsheet:\n"
" -GET: /help This help message\n"
" /analytics/help\n"
" An help message for data analytics commands\n"
" /plugins List of currently loaded plugins (Discovery)\n"
" /[plugin]/sensors\n"
" List of currently running sensors which belong\n"
" to the specified plugin (Discovery)\n"
" /[plugin]/[sensor]/avg?interval=[timeInSec]\n"
" Average of last sensor readings from the last\n"
" [interval] seconds or of all cached readings\n"
" if no interval is given\n"
" -PUT: /[plugin]/[start|stop|reload]\n"
" Start/stop the sensors of the plugin or\n"
" reload the plugin configuration\n"
"\n";
private:
/**
* GET "/analytics/help"
*
* @brief Return a cheatsheet of available REST API endpoints specific for
* the analytics manager.
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | - | - | -
* Optional | - | - | -
*/
void GET_analytics_help(endpointArgs);
/**
* GET "/analytics/plugins"
*
* @brief List all data analytic plugins.
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | - | - | -
* Optional | json | true | format response as json
*/
void GET_analytics_plugins(endpointArgs);
/**
* GET "/analytics/sensors"
*
* @brief List all running sensors in one or all analyzers of a plugin.
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | plugin | all analyzer plugin | specify the plugin
* | | names |
* Optional | analyzer| all analyzers of a | restrict sensors list to an
* | | plugin | analyzer
* | json | true | format response as json
*/
void GET_analytics_sensors(endpointArgs);
/**
* GET "/analytics/units"
*
* @brief List all units of a plugin sensors are associated with
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | plugin | all analyzer plugin | specify the plugin
* | | names |
* Optional | analyzer| all analyzers of a | restrict unit list to an
* | | plugin | analyzer
* | json | true | format response as json
*/
void GET_analytics_units(endpointArgs);
/**
* GET "/analytics/analyzers"
*
* @brief List all running analyzers of a plugin.
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | plugin | all analyzer plugin | specify the plugin
* | | names |
* Optional | json | true | format response as json
*/
void GET_analytics_analyzers(endpointArgs);
/**
* GET "/help"
*
* @brief Return a cheatsheet of possible REST API endpoints.
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | - | - | -
* Optional | - | - | -
*/
void GET_help(endpointArgs);
/**
* GET "/plugins"
*
* @brief List all loaded dcdbpusher plugins.
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | - | - | -
* Optional | json | true | format response as json
*/
void GET_plugins(endpointArgs);
/**
* GET "/sensors"
*
* @brief List all sensors of a specific plugin.
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | plugin | all plugin names | specify the plugin
* Optional | json | true | format response as json
*/
void GET_sensors(endpointArgs);
/**
* GET "/average"
*
* @brief Get the average of the last readings of a sensor.
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | plugin | all plugin names | specify the plugin
* | sensor | all sensor names of | specify the sensor within the
* | | the plugin or the | plugin
* | | analytics manager |
* Optional | interval| number of seconds | use only readings more recent
* | | | than (now - interval) for
* | | | average calculation
*/
void GET_average(endpointArgs);
/******************************************************************************/
/**
* PUT "/analytics/start"
*
* @brief
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | - | - | -
* Optional | - | - | -
*/
void PUT_analytics_start(endpointArgs);
/**
* PUT "/analytics/stop"
*
* @brief
*
* Queries | key | possible values | explanation
* -------------------------------------------------------------------------
* Required | - | - | -
* Optional | - | - | -
*/
void PUT_analytics_stop(endpointArgs);
/**
* PUT "/analytics/reload"
*
* @brief