Commit b7f193a3 authored by Micha Müller's avatar Micha Müller
Browse files

Merge branch 'master' into gcc9

parents 26e70319 06805839
# DCDB
The *DataCenter* *DataBase* (DCDB) is a modular, continuous and holistic monitoring framework targeted at HPC environments. The framework consists of the three main parts:
* _Storage_ _Backend_: Here all the collected data is stored. By default a Cassandra database is used but the framework is intended to support usage of other data storage solutions as well.
* _Collect_ _Agent_: The collect agent functions as intermediary between one storage backend and one or multiple pushers.
* _Pusher_: The main part for collecting data is the pusher framework. It allows to run arbitrary data collection plugins and pushes the data to the collect agent via MQTT messages.
Other features included by DCDB:
* libdcdb: a library to access the storage backend with external tools
* a set of tools leveraging libdcdb (e.g. a command line tool to query the storage backend)
* built in data analytics framework
* pusher plugins for a variety of typical HPC data sources
## Build, install and run
Prerequisites:
* cmake
* C++11 compiler
```bash
//Create a new directory for DCDB
$ mkdir DCDB && cd DCDB
//Download the DCDB sources
$ git clone https://gitlab.lrz.de/dcdb/dcdb.git
//Start building and installation of DCDB and all of its dependencies.
$ cd dcdb
$ make depsinstall install
//OR use this make command instead to also build the doxygen documentation
$ make depsinstall install doc
```
To run DCDB locally with the included default config files:
```bash
//Change to the DCDB directory you created during installation
$ cd DCDB
//Set required environment variables
$ source install/dcdb.bash
//Start one storage backend and one collect agent locally
$ install/etc/init.d/dcdb start
//Start a dcdbpusher instance
$ cd dcdb/dcdbpusher
$ ./dcdbpusher config/
```
## License and Copyright
Copyright (C) 2011-2019 Leibniz Supercomputing Centre
DCDB (except for the libdcdb parts) is licensed under the terms
and conditions of the GNU GENERAL PUBLIC LICENSE (Version 2 or later).
Libdcdb is licensed under the terms and conditions of the GNU LESSER
GENERAL PUBLIC LICENSE (Version 2.1 or later).
See the `LICENSE` file for more information.
<!---
TODO:
write pages for:
-how to use dcdb
-architecture of dcdb
-develop plugin for pusher/analyzer
-develop for framework itself
-dig deeper into doxygen to get a beatiful documentation. perhaps use
markdown files for pages (for tutorials, architecture explanation)
-->
\ No newline at end of file
# Table of contents
# DCDB Data Analytics Framework
### Table of contents
1. [Introduction](#introduction)
2. [DCDBAnalytics](#dcdbanalytics)
1. [Global Configuration](#globalConfiguration)
......
......@@ -67,7 +67,7 @@ void AggregatorAnalyzer::computeSum(int unitID) {
// Getting the most recent values as specified in _window
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer);
if(!_buffer || _buffer->empty()) {
LOG(error) << "Analyzer " << _name << " cannot read from sensor " << in->getName() << "!";
LOG(debug) << "Analyzer " << _name << " cannot read from sensor " << in->getName() << "!";
return;
}
......@@ -88,7 +88,7 @@ void AggregatorAnalyzer::computeAvg(int unitID) {
// Getting the most recent values as specified in _window
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer);
if(!_buffer || _buffer->empty()) {
LOG(error) << "Analyzer " << _name << " cannot read from sensor " << in->getName() << "!";
LOG(debug) << "Analyzer " << _name << " cannot read from sensor " << in->getName() << "!";
return;
}
......@@ -113,7 +113,7 @@ void AggregatorAnalyzer::computeMax(int unitID) {
// Getting the most recent values as specified in _window
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer);
if(!_buffer || _buffer->empty()) {
LOG(error) << "Analyzer " << _name << " cannot read from sensor " << in->getName() << "!";
LOG(debug) << "Analyzer " << _name << " cannot read from sensor " << in->getName() << "!";
return;
}
......@@ -136,7 +136,7 @@ void AggregatorAnalyzer::computeMin(int unitID) {
// Getting the most recent values as specified in _window
_buffer = _queryEngine.querySensor(in->getName(), _window, 0, _buffer);
if(!_buffer || _buffer->empty()) {
LOG(error) << "Analyzer " << _name << " cannot read from sensor " << in->getName() << "!";
LOG(debug) << "Analyzer " << _name << " cannot read from sensor " << in->getName() << "!";
return;
}
......
......@@ -34,7 +34,7 @@
static int dcdbdaemon()
{
#ifdef __linux__
return daemon(0, 0);
return daemon(1, 0);
#else
switch(fork()) {
case -1:
......
......@@ -170,7 +170,7 @@ serverError:
//this is not strictly necessary.
//Fails if client already disconnected from socket.
socket.shutdown(tcp::socket::shutdown_both, ec);
if(ec) { ServerLOG(warning) << "socket shutdown: " << ec.message(); }
if(ec) { ServerLOG(debug) << "socket shutdown: " << ec.message(); }
socket.close(ec);
if(ec) { ServerLOG(error) << "socket close: " << ec.message(); }
......
......@@ -6,5 +6,5 @@ DEFAULT_VERSION = 0.3
GIT_VERSION = $(shell git describe --tags 2>/dev/null|sed 's/-\([0-9]*\)/.\1/')
VERSION := $(if $(GIT_VERSION),$(GIT_VERSION),$(DEFAULT_VERSION))
CXXFLAGS = -std=c++11 -O2 -g -Wall -fmessage-length=0 -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_LOG_DYN_LINK -DVERSION=\"$(VERSION)\"
CXXFLAGS = -std=c++11 -O2 -g -Wall -fmessage-length=0 -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_LOG_DYN_LINK -DVERSION=\"$(VERSION)\" -Wno-unused-function -Wno-unused-variable
OS = $(shell uname -s)
set(CALIPER_DCDBPUSHER_SOURCES
DcdbPusher.cpp)
add_service_sources(${CALIPER_DCDBPUSHER_SOURCES})
add_caliper_service("dcdbpusher")
// This program is to be used with Caliper.
// Caliper belongs to Lawrence Livermore National Security, LLC.
// LLNL-CODE-678900
// All rights reserved.
//
// For details, see https://github.com/scalability-llnl/Caliper.
//================================================================================
// Name : DcdbPusher.cpp
// Author : Micha Mueller
// Copyright : Leibniz Supercomputing Centre
// Description : Caliper service to forward snapshot data to dcdb pusher.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2019-2019 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
/**
* @file DcdbPusher.cpp
*
* @brief Caliper service to forward snapshot data to dcdb pusher.
*
* @details Service for the runtime processing level. Relies on the sampler,
* symbollookup and timestamp service. To avoid a detour to the flush
* level the symbollookup service is invoked manually when processing a
* snapshot. This way all required information is available without
* having to explicitly trigger a flush and the relevant data can be
* instantly forwarded. Unneeded atttributes are stripped off and
* discarded.
*
*/
#include "caliper/CaliperService.h"
#include "caliper/Caliper.h"
#include "caliper/SnapshotRecord.h"
#include "caliper/common/Log.h"
#include "caliper/common/RuntimeConfig.h"
using namespace cali;
namespace {
class DcdbPusher {
private:
static const ConfigSet::Entry s_configdata[];
//TODO reduce snapshot frequency during runtime
unsigned snapshot_threshold = 10;
unsigned threshold_exceeded = 0;
unsigned snapshots_processed = 0;
unsigned snapshots_failed = 0;
bool initialized = false;
Attribute sampler { Attribute::invalid };
Attribute symbol_fun { Attribute::invalid };
Attribute timestamp { Attribute::invalid };
void post_init_cb(Caliper* c, Channel* chn) {
// manually issue a pre-flush event to set up the SymbolLookup service
chn->events().pre_flush_evt(c, chn, nullptr);
// Check if required services sampler, symbollookup and timestamp are
// active by searching for identifying attributes.
//TODO support data collection if event triggered snapshots are used
sampler = c->get_attribute("cali.sampler.pc");
symbol_fun = c->get_attribute("source.function#cali.sampler.pc");
timestamp = c->get_attribute("time.timestamp");
if (sampler == Attribute::invalid ||
symbol_fun == Attribute::invalid ||
timestamp == Attribute::invalid) {
Log(1).stream() << chn->name() << ": DcdbPusher: not all required services "
"sampler, symbollookup and timestamp are running." << std::endl;
return;
}
initialized = true;
}
void process_snapshot_cb(Caliper* c, Channel* chn, const SnapshotRecord*, const SnapshotRecord* sbuf) {
++snapshots_processed;
//TODO check if signal safety is required
//if (c->is_signal()) {
// ++snapshots_failed;
// return;
//}
if (!initialized) {
++snapshots_failed;
return;
}
SnapshotRecord::Sizes sizes = sbuf->size();
if ((sizes.n_nodes + sizes.n_immediate) == 0) {
++snapshots_failed;
return;
}
std::vector<Entry> rec = sbuf->to_entrylist();
// manually issue a postprocess-snapshot event to enrich snapshot with
// symbollookup data
chn->events().postprocess_snapshot(c, chn, rec);
Log(1).stream() << chn->name() << ": DcdbPusher: printing snapshot" << std::endl;
for (const Entry& e : rec) {
//print attribute values for timestamp and looked up function symbol name
cali_id_t entryId = c->get_attribute(e.attribute()).id();
if (entryId == symbol_fun.id() || entryId == timestamp.id()) {
Log(1).stream() << chn->name() << ": DcdbPusher: "
<< c->get_attribute(e.attribute()).name()
<< " = " << e.value().to_string()
<< std::endl;
}
}
Log(1).stream() << chn->name() << std::endl;
}
void finish_cb(Caliper* c, Channel* chn) {
if (threshold_exceeded > 0) {
Log(1).stream() << chn->name() << ": DcdbPusher: threshold was exceeded "
<< threshold_exceeded << " times." << std::endl;
}
Log(1).stream() << chn->name() << ": DcdbPusher: "
<< snapshots_processed << " snapshots processed of which "
<< snapshots_failed << " failed." << std::endl;
}
DcdbPusher(Caliper* c, Channel* chn) {
ConfigSet config = chn->config().init("dcdbpusher", s_configdata);
snapshot_threshold = config.get("snapshot_threshold").to_uint();
}
public:
~DcdbPusher() {
}
static void dcdbpusher_register(Caliper* c, Channel* chn) {
DcdbPusher* instance = new DcdbPusher(c, chn);
chn->events().post_init_evt.connect(
[instance](Caliper* c, Channel* chn){
instance->post_init_cb(c, chn);
});
chn->events().process_snapshot.connect(
[instance](Caliper* c, Channel* chn, const SnapshotRecord* trigger, const SnapshotRecord* snapshot){
instance->process_snapshot_cb(c, chn, trigger, snapshot);
});
chn->events().finish_evt.connect(
[instance](Caliper* c, Channel* chn){
instance->finish_cb(c, chn);
delete instance;
});
Log(1).stream() << chn->name() << ": Registered dcdbpusher service" << std::endl;
}
}; // class DcdbPusher
const ConfigSet::Entry DcdbPusher::s_configdata[] = {
{ "snapshot_threshold", CALI_TYPE_UINT, "10",
"Maximum number of snapshots per second we are allowed to process",
"Maximum number of snapshots per second we are allowed to process.\n"
"If the threshold is exceeded snapshots get discarded until the\n"
"limit is met again." },
ConfigSet::Terminator
};
} // namespace
namespace cali {
CaliperService dcdbpusher_service { "dcdbpusher", ::DcdbPusher::dcdbpusher_register };
}
--- a/src/services/CMakeLists.txt 2019-06-07 10:33:20.036516054 +0200
+++ b/src/services/CMakeLists.txt 2019-06-06 17:57:54.096623744 +0200
@@ -55,6 +55,7 @@
add_subdirectory(cupti)
add_subdirectory(cuptitrace)
endif()
+add_subdirectory(dcdbpusher)
add_subdirectory(debug)
add_subdirectory(env)
add_subdirectory(event)
......@@ -37,7 +37,9 @@ PluginManager::PluginManager(const pluginSettings_t& pluginSettings) :
_cfgFilePath("./") {}
PluginManager::~PluginManager() {
unloadPlugin();
for(const auto& p : _plugins)
p.destroy(p.configurator);
_plugins.clear();
}
bool PluginManager::loadPlugin(const string& name,
......
# Table of contents
# DCDB Pusher
### Table of contents
1. [Introduction](#introduction)
2. [dcdbpusher](#dcdbpusher)
1. [Global Configuration](#globalConfiguration)
......@@ -28,7 +30,7 @@ Harvesting of the data is task of the dcdbpusher.
# dcdbpusher <a name="dcdbpusher"></a>
This is a general MQTT pusher which sends values of various sensors to the DCDB-database.
It ships with plugins for BACnet, IPMI, PDU(proprietary Power Delivery Unit, but could be used as XML plugin) ,percounter, SNMP and sysFS.
It ships with plugins for BACnet, IPMI, PDU(proprietary Power Delivery Unit, but could be used as XML plugin), perfcounter, SNMP and sysFS.
Build it by simply running
```bash
......
......@@ -441,7 +441,7 @@ int main(int argc, char** argv) {
mqttThread.join();
LOG(info) << "MQTTPusher stopped";
LOG(info) << "Tear down objects...";
LOG(info) << "Tearing down objects...";
_sensorMap.clear();
delete _httpsServer;
delete _mqttPusher;
......
......@@ -54,7 +54,7 @@ ProcfsParser::ProcfsParser(std::string path) {
* Class destructor.
*
*/
ProcfsParser::~ProcfsParser() { close(); }
ProcfsParser::~ProcfsParser() { ProcfsParser::close(); }
/**
* Initializes the parser.
......@@ -512,6 +512,11 @@ bool ProcstatParser::_readMetrics() {
// **************************************************************
// Definitions for the SARParser class
/**
* Class destructor.
*
*/
SARParser::~SARParser() { SARParser::close(); }
/**
* Closes the parser.
......
......@@ -201,6 +201,7 @@ class SARParser : public ProcstatParser {
public:
SARParser(const std::string path = "") : ProcstatParser(path) {}
void close() override;
virtual ~SARParser();
protected:
bool _readMetrics() override;
......
......@@ -66,6 +66,11 @@ void TesterSensorGroup::start() {
}
void TesterSensorGroup::stop() {
if(_keepRunning == 0) {
LOG(info) << "Sensorgroup " << _groupName << " already stopped.";
return;
}
_keepRunning = 0;
//wait before closing _file
wait();
......
......@@ -771,7 +771,7 @@ WARN_LOGFILE = dox_stderr.txt
# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
# Note: If this tag is empty the current directory is searched.
INPUT = ./../analytics ./../collectagent ./../common ./../dcdbpusher ./../lib ./../tools
INPUT = ./../analytics ./../collectagent ./../common ./../dcdbpusher ./../lib ./../tools ./../README.md
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
......@@ -796,7 +796,7 @@ INPUT_ENCODING = UTF-8
# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl,
# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js.
FILE_PATTERNS = *.cpp *.h
FILE_PATTERNS = *.cpp *.h *.md
# The RECURSIVE tag can be used to specify whether or not subdirectories should
# be searched for input files as well.
......@@ -920,7 +920,7 @@ FILTER_SOURCE_PATTERNS =
# (index.html). This can be useful if you have a project on for instance GitHub
# and want to reuse the introduction page also for the doxygen output.
USE_MDFILE_AS_MAINPAGE =
USE_MDFILE_AS_MAINPAGE = ./../README.md
#---------------------------------------------------------------------------
# Configuration options related to source browsing
......
......@@ -24,31 +24,6 @@
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//================================================================================
/**
* @mainpage
* DataCenter DataBase (DCDB) is a modular, continuous and holistic monitoring
* framework for the HPC field. The framework consists of three main parts:
* pusher, collect agent and storage backend. Further on, DCDB ships with a
* small set of utility tools and allows access to the data store through
* libdcdb.
*
* See further:
* - @ref libdcdb "libdcdb" or
* - @subpage libdcdb "libdcdb"
* - @link pusher Pusher@endlink
*/
//
// TODO:
// write pages for:
// -how to use dcdb
// -architecture of dcdb
// -develop plugin for pusher/analyzer
// -develop for framework itself
// -dig deeper into doxygen to get a beatiful documentation. perhaps use
// markdown files for pages (for tutorials, architecture explanation)
//
/**
* @page libdcdb libdcdb
* The libdcdb library is a dynamic runtime library providing
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment