Notice to GitKraken users: A vulnerability has been found in the SSH key generation of GitKraken versions 7.6.0 to 8.0.0 (https://www.gitkraken.com/blog/weak-ssh-key-fix). If you use GitKraken and have generated a SSH key using one of these versions, please remove it both from your local workstation and from your LRZ GitLab profile.

21.10.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit 44ca1b2e authored by Daniele Tafani's avatar Daniele Tafani
Browse files

Stable version of Grafana Server (Works with Grafana 6.2.2)

parent 52eaf253
include ../config.mk include ../config.mk
CXXFLAGS = -O2 -g --std=c++11 -Wall -Wno-unused-local-typedefs -Wno-deprecated-declarations -Wno-unknown-warning-option -fmessage-length=0 -I../include/ -I../lib/include -I$(DCDBDEPLOYPATH)/include -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -I$(DCDBDEPSPATH)/cpp-netlib-0.12.0-final/deps/asio/asio/include -I$(DCDBDEPSPATH)/cpp-netlib-0.12.0-final -I$(DCDBDEPSPATH)/cpp-netlib-0.12.0-final/deps/cxxopts/src -DASIO_HEADER_ONLY -DBOOST_TEST_DYN_LINK CXXFLAGS = -O2 -g --std=c++11 -Wall -Wno-unused-function -Wno-unused-local-typedefs -Wno-deprecated-declarations -Wno-unknown-warning-option -fmessage-length=0 -I../common/include/ -I../lib/include -I$(DCDBDEPLOYPATH)/include -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_LOG_DYN_LINK -I$(DCDBDEPSPATH)/cpp-netlib-0.12.0-final/deps/asio/asio/include -DVERSION=\"$(VERSION)\"
OBJS = collectagent.o \ OBJS = ../common/src/logging.o \
sensorcache.o \ sensorcache.o \
simplemqttserver.o \ collectagent.o \
simplemqttserverthread.o \ configuration.o \
simplemqttservermessage.o simplemqttserver.o \
LIBS = -L$(DCDBDEPLOYPATH)/lib/ -L../lib -ldcdb -pthread -lcassandra -luv -lboost_system -lboost_random -lboost_thread -lboost_date_time -lboost_regex -lcppnetlib-server-parsers -lcppnetlib-uri simplemqttserverthread.o \
simplemqttservermessage.o
LIBS = -L$(DCDBDEPLOYPATH)/lib/ -L../lib -ldcdb -pthread -lcassandra -luv -lboost_system -lboost_random -lboost_thread -lboost_date_time -lboost_log_setup -lboost_log -lboost_regex -lcppnetlib-server-parsers -lcppnetlib-uri
TARGET = collectagent TARGET = collectagent
.PHONY : clean install .PHONY : clean install
...@@ -25,5 +27,8 @@ clean: ...@@ -25,5 +27,8 @@ clean:
rm -f $(TARGET) rm -f $(TARGET)
rm -f $(OBJS) rm -f $(OBJS)
install: $(TARGET) install_conf: config/collectagent.conf
install -m 644 $^ $(DCDBDEPLOYPATH)/etc/
install: $(TARGET) install_conf
install $(TARGET) $(DCDBDEPLOYPATH)/bin/ install $(TARGET) $(DCDBDEPLOYPATH)/bin/
This diff is collapsed.
...@@ -6,118 +6,51 @@ ...@@ -6,118 +6,51 @@
*/ */
#include "sensorcache.h" #include "sensorcache.h"
#include <dcdb/timestamp.h>
#include <exception>
#include <iostream>
#include <iomanip>
#include <algorithm>
namespace DCDB {
SensorCache::SensorCache() {
// TODO Auto-generated constructor stub
SensorCache::SensorCache(uint64_t maxHistory) {
this->_maxHistory = maxHistory;
} }
SensorCache::~SensorCache() { SensorCache::~SensorCache() {
// TODO Auto-generated destructor stub sensorCache.clear();
} }
void SensorCache::storeSensor(SensorId sid, uint64_t ts, int64_t val) { const sensorCache_t& SensorCache::getSensorMap() {
sensorReading_t s = { val, ts }; return sensorCache;
}
void SensorCache::storeSensor(SensorId sid, uint64_t ts, int64_t val) {
reading_t s = { val, ts };
/* Remove the reserved bytes to leverage the standard find function */ /* Remove the reserved bytes to leverage the standard find function */
sid.setRsvd(0); sid.setRsvd(0);
sensorCache_t::iterator it = sensorCache.find(sid); //TODO: Check for thread-safety
if (it != sensorCache.end()) { if(sensorCache.find(sid) == sensorCache.end())
if (it->second.size() && (it->second.front().timestamp+MAX_HISTORY_NS <= ts)) { sensorCache[sid] = CacheEntry(_maxHistory);
it->second.pop_front(); sensorCache[sid].store(s);
}
it->second.push_back(s);
} else {
sensorCache[sid].push_back(s);
}
}
bool SensorCache::checkValid(cacheEntry_t &entry) {
if (entry.size() > 2) {
TimeStamp ts;
uint64_t avg = 0;
cacheEntry_t::iterator it = entry.begin();
uint64_t prev = it->timestamp;
for (it++; it != entry.end(); it++) {
avg+= it->timestamp - prev;
prev = it->timestamp;
}
avg/= (entry.size()-1);
/*
* A SID is outdated if it's older than 5x the average sampling period.
*/
if ((ts.getRaw() - entry.back().timestamp) > 5 * avg) {
return false;
}
}
return true;
} }
int64_t SensorCache::getAverage(cacheEntry_t &entry, uint64_t avg) { int64_t SensorCache::getSensor(SensorId sid, uint64_t avg) {
TimeStamp ts;
if (entry.size() > 0) {
if (ts.getRaw() - entry.front().timestamp < avg * NS_PER_S) {
throw std::out_of_range("Sid outdated");
return 0;
}
double sum = 0;
cacheEntry_t::reverse_iterator it, prev;
it = prev = entry.rbegin();
it++;
while ((it != entry.rend()) && ((ts.getRaw() - it->timestamp) <= avg * NS_PER_S)) {
uint64_t deltaT = (prev->timestamp - it->timestamp);
sum+= ((it->val + prev->val) / 2) * deltaT;
//std::cout << "SensorCache::getAverage sum=" << sum << " deltaT=" <<deltaT << " it=(" << it->timestamp << "," <<it->val <<") prev=(" << prev->timestamp << "," << prev->val <<") " << (ts.getRaw() - it->timestamp) << std::endl;
prev = it++;
}
//std::cout << "SensorCache::getAverage (" << prev->timestamp << "," <<prev->val <<") (" << entry.back().timestamp << "," << entry.back().val << ") sum=" << sum << " deltaT=" << entry.back().timestamp - prev->timestamp << std::endl;
if (prev == entry.rbegin()) {
return entry.back().val;
} else {
return sum/(entry.back().timestamp - prev->timestamp);
}
}
throw std::invalid_argument("Sid not found");
return 0;
}
uint64_t SensorCache::getSensor(SensorId sid, uint64_t avg) {
/* Remove the reserved bytes to leverage the standard find function */ /* Remove the reserved bytes to leverage the standard find function */
sid.setRsvd(0); sid.setRsvd(0);
sensorCache_t::iterator it = sensorCache.find(sid); sensorCache_t::iterator it = sensorCache.find(sid);
if (it == sensorCache.end()) { if (it == sensorCache.end()) {
throw std::invalid_argument("Sid not found"); throw std::invalid_argument("Sid not found");
return 0;
} }
if (!checkValid(it->second)) if (!it->second.checkValid())
{ {
throw std::out_of_range("Sid outdated"); throw std::out_of_range("Sid outdated");
return 0;
} }
if (avg) { if (avg) {
return getAverage(it->second, avg); return it->second.getAverage(avg);
} else { } else {
return it->second.back().val; return it->second.getLatest().value;
} }
} }
uint64_t SensorCache::getSensor(std::string topic, uint64_t avg) { int64_t SensorCache::getSensor(std::string topic, uint64_t avg) {
topic.erase(std::remove(topic.begin(), topic.end(), '/'), topic.end()); topic.erase(std::remove(topic.begin(), topic.end(), '/'), topic.end());
size_t wp = topic.find("*"); size_t wp = topic.find("*");
...@@ -125,7 +58,7 @@ uint64_t SensorCache::getSensor(std::string topic, uint64_t avg) { ...@@ -125,7 +58,7 @@ uint64_t SensorCache::getSensor(std::string topic, uint64_t avg) {
return getSensor(DCDB::SensorId(topic), avg); return getSensor(DCDB::SensorId(topic), avg);
} }
int wl = 33 - topic.length(); int wl = 29 - topic.length();
/* Create SensorIds with the lowest and highest values matching the wildcard */ /* Create SensorIds with the lowest and highest values matching the wildcard */
DCDB::SensorId sidLow(std::string(topic).replace(wp, 1, std::string(wl, '0'))); DCDB::SensorId sidLow(std::string(topic).replace(wp, 1, std::string(wl, '0')));
...@@ -146,7 +79,7 @@ uint64_t SensorCache::getSensor(std::string topic, uint64_t avg) { ...@@ -146,7 +79,7 @@ uint64_t SensorCache::getSensor(std::string topic, uint64_t avg) {
/* We only return one value, even if multiple SensorIds would match. /* We only return one value, even if multiple SensorIds would match.
* At least make sure it's the most recent value * At least make sure it's the most recent value
*/ */
if (checkValid(it->second) && ((mostRecentSidIt == sensorCache.end()) || mostRecentSidIt->second.back().timestamp < it->second.back().timestamp)) { if (it->second.checkValid() && ((mostRecentSidIt == sensorCache.end()) || mostRecentSidIt->second.getLatest().timestamp < it->second.getLatest().timestamp)) {
mostRecentSidIt = it; mostRecentSidIt = it;
} }
} }
...@@ -163,9 +96,9 @@ uint64_t SensorCache::getSensor(std::string topic, uint64_t avg) { ...@@ -163,9 +96,9 @@ uint64_t SensorCache::getSensor(std::string topic, uint64_t avg) {
} }
if (avg) { if (avg) {
return getAverage(mostRecentSidIt->second, avg); return mostRecentSidIt->second.getAverage(avg);
} else { } else {
return mostRecentSidIt->second.back().val; return mostRecentSidIt->second.getLatest().value;
} }
} }
...@@ -173,16 +106,26 @@ void SensorCache::dump() { ...@@ -173,16 +106,26 @@ void SensorCache::dump() {
std::cout << "SensorCache Dump:" << std::endl; std::cout << "SensorCache Dump:" << std::endl;
for (sensorCache_t::iterator sit = sensorCache.begin(); sit != sensorCache.end(); sit++) { for (sensorCache_t::iterator sit = sensorCache.begin(); sit != sensorCache.end(); sit++) {
std::cout << " id=" << sit->first.toString() << " data=["; std::cout << " id=" << sit->first.toString() << " data=[";
for (cacheEntry_t::iterator eit = sit->second.begin(); eit != sit->second.end(); eit++) { for (std::vector<reading_t>::const_iterator eit = sit->second.getRaw()->begin(); eit != sit->second.getRaw()->end(); eit++) {
if (eit != sit->second.begin()) { if (eit != sit->second.getRaw()->begin()) {
std::cout << ","; std::cout << ",";
} }
std::cout << "(" << eit->val << "," << eit->timestamp/NS_PER_S << "." << std::setfill ('0') << std::setw (9) << eit->timestamp%NS_PER_S << ")"; std::cout << "(" << eit->value << "," << eit->timestamp/NS_PER_S << "." << std::setfill ('0') << std::setw (9) << eit->timestamp%NS_PER_S << ")";
} }
std::cout << "]" << std::endl; std::cout << "]" << std::endl;
} }
} }
} /* namespace DCDB */ uint64_t SensorCache::clean(uint64_t t) {
uint64_t thresh = getTimestamp() - t;
uint64_t ctr = 0;
for (auto it = sensorCache.cbegin(); it != sensorCache.cend();) {
uint64_t latestTs = it->second.getLatest().timestamp;
if (latestTs!=0 && latestTs < thresh) {
it = sensorCache.erase(it);
ctr++;
} else
++it;
}
return ctr;
}
...@@ -9,65 +9,96 @@ ...@@ -9,65 +9,96 @@
#define COLLECTAGENT_SENSORCACHE_H_ #define COLLECTAGENT_SENSORCACHE_H_
#include <map> #include <map>
#include <list>
#include <utility>
#include <dcdb/sensorid.h> #include <dcdb/sensorid.h>
#include <dcdb/timestamp.h>
#include "cacheentry.h"
#define MAX_HISTORY_NS 60000000000 // Store max 60s of historic data using namespace DCDB;
namespace DCDB { typedef std::map<SensorId, CacheEntry> sensorCache_t;
typedef struct {
int64_t val;
uint64_t timestamp;
} sensorReading_t;
typedef std::list<sensorReading_t> cacheEntry_t;
typedef std::map<SensorId, cacheEntry_t> sensorCache_t;
class SensorCache { class SensorCache {
public: public:
SensorCache(); SensorCache(uint64_t maxHistory=60000000000);
virtual ~SensorCache(); virtual ~SensorCache();
/** /**
* @brief Store a sensor reading in the SensorCache. * @brief Returns a constant reference to the internal sensor cache map.
* @param sid The SensorId of the sensor to be cached. **/
* @param ts The timestamp of the sensor reading. const sensorCache_t& getSensorMap();
* @param val The actual sensor reading.
* @return Returns true if the topic string was valid and the data field of the object was populated. /**
*/ * @brief Store a sensor reading in the SensorCache.
*
* @param sid The SensorId of the sensor to be cached.
* @param ts The timestamp of the sensor reading.
* @param val The actual sensor reading.
* @return Returns true if the topic string was valid and the data field of the object was populated.
**/
void storeSensor(SensorId sid, uint64_t ts, int64_t val); void storeSensor(SensorId sid, uint64_t ts, int64_t val);
/** /**
* @brief Return a sensor reading from the SensorCache. * @brief Return a sensor reading or the average of the last readings
* @param sid The SensorId of the sensor to be looked up in the cache. * from the SensorCache.
* @return The sensor reading of the corresponding cache entry. *
* @throws std::invalid_argument if the SensorId doesn't exist in the SensorCache. * @param sid The SensorId of the sensor to be looked up in the cache.
* @throws std::out_of_range if the sid was found in the cache entry but is outdated. * @param avg If avg > 0: denotes the length of the average aggregation window in nanoseconds.
*/ * @return If avg == 0 :The sensor reading of the corresponding cache entry.
uint64_t getSensor(SensorId sid, uint64_t avg=0); * If avg > 0 the average of the last readings is returned.
* @throws std::invalid_argument if the SensorId doesn't exist in the SensorCache.
/** * @throws std::out_of_range if the sid was found in the cache entry but is outdated.
* @brief Return a sensor reading from the SensorCache. **/
* @param topic The topic of the sensor to be looked up in the cache. May contain wildcards. int64_t getSensor(SensorId sid, uint64_t avg=0);
* @return The sensor reading of the corresponding cache entry.
* @throws std::invalid_argument if the topic couldn't be found in the SensorCache. /**
* @throws std::out_of_range if the topic was found in the cache entry but is outdated. * @brief Return a sensor reading or the average of the last readings
*/ * from the SensorCache.
uint64_t getSensor(std::string topic, uint64_t avg=0); *
* @param topic The topic of the sensor to be looked up in the cache. May contain wildcards.
/** * @param avg If avg > 0: denotes the length of the average aggregation window in nanoseconds.
* @brief Dump the contents of the SensorCache to stdout. * @return If avg == 0 :The sensor reading of the corresponding cache entry.
*/ * If avg > 0 the average of the last readings is returned.
* @throws std::invalid_argument if the topic couldn't be found in the SensorCache.
* @throws std::out_of_range if the topic was found in the cache entry but is outdated.
**/
int64_t getSensor(std::string topic, uint64_t avg=0);
/**
* @brief Dump the contents of the SensorCache to stdout.
**/
void dump(); void dump();
/**
* @brief Removes all obsolete entries from the cache
*
* All entries in the cache whose latest sensor reading is older than "now" - t nanoseconds are
* removed.
*
* @param t The threshold in nanoseconds for entries that must be removed
* @return The number of purged cache entries
**/
uint64_t clean(uint64_t t);
/**
* @brief Set a new maximum cache length.
*
* @param maxHistory: new sensor cache length value.
**/
void setMaxHistory(uint64_t maxHistory) { this->_maxHistory = maxHistory; }
/**
* @brief Returns the current maximum sensor cache length
*
* @return Current maximum sensor cache length
*/
uint64_t getMaxHistory() { return this->_maxHistory; }
private: private:
bool checkValid(cacheEntry_t &entry); // Map containing the single sensor caches
int64_t getAverage(cacheEntry_t &entry, uint64_t avg);
sensorCache_t sensorCache; sensorCache_t sensorCache;
// Global maximum allowed time frame for the sensor caches
uint64_t _maxHistory;
}; };
} /* namespace DCDB */
#endif /* COLLECTAGENT_SENSORCACHE_H_ */ #endif /* COLLECTAGENT_SENSORCACHE_H_ */
...@@ -104,12 +104,12 @@ void SimpleMQTTServer::initSockets(void) ...@@ -104,12 +104,12 @@ void SimpleMQTTServer::initSockets(void)
* Bind and listen on socket. * Bind and listen on socket.
*/ */
if (::bind(sock, ainfo_cur->ai_addr, ainfo_cur->ai_addrlen) == -1) { if (::bind(sock, ainfo_cur->ai_addr, ainfo_cur->ai_addrlen) == -1) {
cout << "Warning: could not bind to socket, ignoring socket.\n"; LOG(warning) << "Could not bind to socket, ignoring socket.";
close(sock); close(sock);
continue; continue;
} }
if (listen(sock, SimpleMQTTMaxBacklog) == -1) { if (listen(sock, SimpleMQTTMaxBacklog) == -1) {
cout << "Warning: could not listen on socket, ignoring socket.\n"; LOG(warning) << "Could not listen on socket, ignoring socket.";
close(sock); close(sock);
continue; continue;
} }
...@@ -137,7 +137,7 @@ void SimpleMQTTServer::start() ...@@ -137,7 +137,7 @@ void SimpleMQTTServer::start()
* Start one accept thread per socket. * Start one accept thread per socket.
*/ */
for (unsigned int i=0; i<listenSockets.size(); i++) for (unsigned int i=0; i<listenSockets.size(); i++)
acceptThreads.push_back(new SimpleMQTTServerAcceptThread(listenSockets[i], messageCallback)); acceptThreads.push_back(new SimpleMQTTServerAcceptThread(listenSockets[i], messageCallback, this->_maxThreads, this->_maxConnPerThread));
} }
void SimpleMQTTServer::stop() void SimpleMQTTServer::stop()
...@@ -191,11 +191,13 @@ SimpleMQTTServer::SimpleMQTTServer() ...@@ -191,11 +191,13 @@ SimpleMQTTServer::SimpleMQTTServer()
init("localhost", "1883"); init("localhost", "1883");
} }
SimpleMQTTServer::SimpleMQTTServer(std::string addr, std::string port) SimpleMQTTServer::SimpleMQTTServer(std::string addr, std::string port, uint64_t maxThreads, uint64_t maxConnPerThread)
{ {
/* /*
* Initialize server to listen on specified address and port. * Initialize server to listen on specified address and port.
*/ */
this->_maxThreads = maxThreads;
this->_maxConnPerThread = maxConnPerThread;
init(addr, port); init(addr, port);
} }
......
...@@ -42,30 +42,17 @@ ...@@ -42,30 +42,17 @@
#include <boost/thread/mutex.hpp> #include <boost/thread/mutex.hpp>
#include <boost/thread/thread.hpp> #include <boost/thread/thread.hpp>
#include "logging.h"
#ifndef SIMPLEMQTTSERVER_H_ #ifndef SIMPLEMQTTSERVER_H_
#define SIMPLEMQTTSERVER_H_ #define SIMPLEMQTTSERVER_H_
//#define SimpleMQTTVerbose 100 //#define SimpleMQTTVerbose 100
/*
* Define the number of connections that are handled by each thread.
* If the number of clients exceeds the specified value, a new thread will be spawned.
*/
#ifndef SimpleMQTTConnectionsPerThread
#define SimpleMQTTConnectionsPerThread 16
#endif
#ifndef SimpleMQTTConnectionsQueueLength #ifndef SimpleMQTTConnectionsQueueLength
#define SimpleMQTTConnectionsQueueLength 4 #define SimpleMQTTConnectionsQueueLength 4
#endif #endif
/*
* Define the maximum number of threads that will be spawned per socket.
*/
#ifndef SimpleMQTTMaxThreadsPerSocket
#define SimpleMQTTMaxThreadsPerSocket 128
#endif
/* /*
* Define the maximum backlog size for listen(). * Define the maximum backlog size for listen().
*/ */
...@@ -114,12 +101,16 @@ typedef int (*SimpleMQTTMessageCallback)(SimpleMQTTMessage*); ...@@ -114,12 +101,16 @@ typedef int (*SimpleMQTTMessageCallback)(SimpleMQTTMessage*);
class SimpleMQTTServer class SimpleMQTTServer
{ {
protected: protected:
uint64_t _maxThreads;
uint64_t _maxConnPerThread;
std::string listenAddress; std::string listenAddress;
std::string listenPort; std::string listenPort;
boost::ptr_vector<int> listenSockets; boost::ptr_vector<int> listenSockets;
boost::ptr_list<SimpleMQTTServerAcceptThread> acceptThreads; boost::ptr_list<SimpleMQTTServerAcceptThread> acceptThreads;
SimpleMQTTMessageCallback messageCallback; SimpleMQTTMessageCallback messageCallback;
logger_t lg;
void init(std::string addr, std::string port); void init(std::string addr, std::string port);
void initSockets(void); void initSockets(void);
...@@ -129,7 +120,7 @@ public: ...@@ -129,7 +120,7 @@ public:
void setMessageCallback(SimpleMQTTMessageCallback callback); void setMessageCallback(SimpleMQTTMessageCallback callback);
SimpleMQTTServer(); SimpleMQTTServer();
SimpleMQTTServer(std::string addr, std::string port); SimpleMQTTServer(std::string addr, std::string port, uint64_t maxThreads=128, uint64_t maxConnPerThread=16);
virtual virtual
~SimpleMQTTServer(); ~SimpleMQTTServer();
}; };
......
...@@ -44,6 +44,7 @@ SimpleMQTTMessage::SimpleMQTTMessage() ...@@ -44,6 +44,7 @@ SimpleMQTTMessage::SimpleMQTTMessage()
bytesProcessed = 0; bytesProcessed = 0;
remainingRaw = NULL; remainingRaw = NULL;
remainingLength = 0; remainingLength = 0;
bufferLength = 0;
fixedHeaderLength = 0; fixedHeaderLength = 0;
msgId = 0; msgId = 0;
payloadLength = 0; payloadLength = 0;
...@@ -59,6 +60,17 @@ SimpleMQTTMessage::~SimpleMQTTMessage() ...@@ -59,6 +60,17 @@ SimpleMQTTMessage::~SimpleMQTTMessage()
free(remainingRaw); free(remainingRaw);
} }
void SimpleMQTTMessage::clear() {
//We reset all variables except for the internal buffer, which is recycled
state = Empty;
bytesProcessed = 0;
remainingLength = 0;
fixedHeaderLength = 0;
msgId = 0;
payloadLength = 0;
payloadPtr = NULL;
}
ssize_t SimpleMQTTMessage::decodeFixedHeader(void* buf, size_t len) ssize_t SimpleMQTTMessage::decodeFixedHeader(void* buf, size_t len)