Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 0848af2f authored by Alessio Netti's avatar Alessio Netti
Browse files

Sensor name auto-publish support

- MQTT publish messages that have a topic preceded by the /DCDB_MAP/
keyword will be interpreted as sensor name auto-publish messages
- The payload of such messages is a string defining a sensor name, that
is used together with the MQTT topic to perform a "sensor publish"
action on the Cassandra db, like dcdbconfig does
parent 0f7909c7
......@@ -37,6 +37,7 @@
#include <dcdb/connection.h>
#include <dcdb/sensordatastore.h>
#include <dcdb/sensorconfig.h>
#include "simplemqttserver.h"
#include "messaging.h"
......@@ -62,8 +63,11 @@ int keepRunning;
bool statistics;
uint64_t msgCtr;
uint64_t pmsgCtr;
DCDB::Connection* dcdbConn;
DCDB::SensorDataStore *mySensorDataStore;
DCDB::SensorConfig *mySensorConfig;
DCDB::SensorCache mySensorCache;
DCDB::SCError err;
/* Normal termination (SIGINT, CTRL+C) */
void sigHandler(int sig)
......@@ -130,64 +134,93 @@ int mqttCallback(SimpleMQTTMessage *msg)
if (msg->isPublish())
pmsgCtr++;
uint64_t len;
/*
* Decode the message and put into the database.
*/
if (msg->isPublish()) {
mqttPayload buf, *payload;
uint64_t len;
len = msg->getPayloadLength();
//In the 64 bit message case, the collect agent provides a timestamp
if (len == sizeof(uint64_t)) {
payload = &buf;
payload->value = *((uint64_t*)msg->getPayload());
payload->timestamp = Messaging::calculateTimestamp();
len = sizeof(uint64_t) * 2;
}
//...otherwise it just retrieves it from the MQTT message payload.
else if((len%sizeof(mqttPayload)==0) && (len>0)){
payload = (mqttPayload*)msg->getPayload();
}
//...otherwise this message is malformed -> ignore...
else {
cout << "Message malformed\n";
return 1;
}
const char *topic = msg->getTopic().c_str();
// We check whether the topic includes the \DCDB_MAP\ keyword, indicating that the payload will contain the
// sensor's name. In that case, we set the mappingMessage flag to true, and filter the keyword out of the prefix
// We use strncmp as it is the most efficient way to do it
if (strlen(topic) > DCDB_MAP_LEN && strncmp(topic, DCDB_MAP, DCDB_MAP_LEN) == 0) {
if ((len = msg->getPayloadLength()) == 0) {
cout << "Empty topic-to-name mapping message received\n";
return 1;
}
/*
* Check if we can decode the message topic
* into a valid SensorId. If successful, store
* the record in the database.
*/
DCDB::SensorId sid;
if (sid.mqttTopicConvert(msg->getTopic())) {
#if 0
cout << "Topic decode successful:" << endl
<< " Raw: " << hex << setw(16) << setfill('0') << sid.getRaw()[0] << hex << setw(16) << setfill('0') << sid.getRaw()[1] << endl
<< " DeviceLocation: " << hex << setw(16) << setfill('0') << sid.getDeviceLocation() << endl
<< " device_id: " << hex << setw(8) << setfill('0') << sid.getDeviceSensorId().device_id << endl
<< " sensor_number: " << hex << setw(4) << setfill('0') << sid.getDeviceSensorId().sensor_number << endl << dec;
cout << "Payload (" << len/sizeof(mqttPayload) << " messages):"<< endl;
for (uint64_t i=0; i<len/sizeof(mqttPayload); i++) {
cout << " " << i << ": ts=" << payload[i].timestamp << " val=" << payload[i].value << endl;
string sensorName((char *) msg->getPayload(), len);
err = mySensorConfig->publishSensor(sensorName.c_str(), topic + DCDB_MAP_LEN);
// PublishSensor does most of the error checking for us
switch (err) {
case DCDB::SC_INVALIDPATTERN:
std::cout << "Invalid sensor topic : " << msg->getTopic() << std::endl;
return 1;
case DCDB::SC_INVALIDPUBLICNAME:
std::cout << "Invalid sensor public name: " << sensorName << std::endl;
return 1;
case DCDB::SC_INVALIDSESSION:
std::cout << "Cannot reach sensor data store." << std::endl;
return 1;
default:
break;
}
} else {
mqttPayload buf, *payload;
len = msg->getPayloadLength();
//In the 64 bit message case, the collect agent provides a timestamp
if (len == sizeof(uint64_t)) {
payload = &buf;
payload->value = *((uint64_t *) msg->getPayload());
payload->timestamp = Messaging::calculateTimestamp();
len = sizeof(uint64_t) * 2;
}
cout << endl;
//...otherwise it just retrieves it from the MQTT message payload.
else if ((len % sizeof(mqttPayload) == 0) && (len > 0)) {
payload = (mqttPayload *) msg->getPayload();
}
//...otherwise this message is malformed -> ignore...
else {
cout << "Message malformed\n";
return 1;
}
/*
* Check if we can decode the message topic
* into a valid SensorId. If successful, store
* the record in the database.
*/
DCDB::SensorId sid;
if (sid.mqttTopicConvert(msg->getTopic())) {
#if 0
cout << "Topic decode successful:" << endl
<< " Raw: " << hex << setw(16) << setfill('0') << sid.getRaw()[0] << hex << setw(16) << setfill('0') << sid.getRaw()[1] << endl
<< " DeviceLocation: " << hex << setw(16) << setfill('0') << sid.getDeviceLocation() << endl
<< " device_id: " << hex << setw(8) << setfill('0') << sid.getDeviceSensorId().device_id << endl
<< " sensor_number: " << hex << setw(4) << setfill('0') << sid.getDeviceSensorId().sensor_number << endl << dec;
cout << "Payload (" << len/sizeof(mqttPayload) << " messages):"<< endl;
for (uint64_t i=0; i<len/sizeof(mqttPayload); i++) {
cout << " " << i << ": ts=" << payload[i].timestamp << " val=" << payload[i].value << endl;
}
cout << endl;
#endif
for (uint64_t i=0; i<len/sizeof(mqttPayload); i++) {
mySensorDataStore->insert(&sid, payload[i].timestamp, payload[i].value);
mySensorCache.storeSensor(sid, payload[i].timestamp, payload[i].value);
for (uint64_t i = 0; i < len / sizeof(mqttPayload); i++) {
mySensorDataStore->insert(&sid, payload[i].timestamp, payload[i].value);
mySensorCache.storeSensor(sid, payload[i].timestamp, payload[i].value);
}
//mySensorCache.dump();
}
//mySensorCache.dump();
}
#if 1
else {
cout << "Wrong topic format: " << msg->getTopic() << "\n";
return 1;
}
else {
cout << "Wrong topic format: " << msg->getTopic() << "\n";
return 1;
}
#endif
}
}
return 0;
}
......@@ -318,7 +351,6 @@ int main(int argc, char* const argv[]) {
/*
* Allocate and initialize connection to Cassandra.
*/
DCDB::Connection* dcdbConn;
dcdbConn = new DCDB::Connection(cassandraHost, atoi(cassandraPort.c_str()), cassandraUser, cassandraPassword);
if (!dcdbConn->connect()) {
......@@ -335,6 +367,7 @@ int main(int argc, char* const argv[]) {
* Allocate the SensorDataStore.
*/
mySensorDataStore = new DCDB::SensorDataStore(dcdbConn);
mySensorConfig = new DCDB::SensorConfig(dcdbConn);
/*
* Set TTL for data store inserts if TTL > 0.
......@@ -405,6 +438,7 @@ int main(int argc, char* const argv[]) {
httpThread.join();
cout << "HTTP Server stopped..." << std::endl;
delete mySensorDataStore;
delete mySensorConfig;
dcdbConn->disconnect();
delete dcdbConn;
cout << "Collect Agent closed. Bye bye..." << std::endl;
......
......@@ -43,6 +43,9 @@
#define MQTT_PINGRESP 0xd
#define MQTT_DISCONNECT 0xe
#define DCDB_MAP "/DCDB_MAP/"
#define DCDB_MAP_LEN 10
#pragma pack(push,1)
typedef union {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment