Commit 3a25db65 authored by Alessio Netti's avatar Alessio Netti
Browse files

ProcFS plugin overhaul

- Now single sensors for distinct metrics can be specified within configuration files for each sensor group
- Major streamlining of code
- Documentation updated to reflect the new config options
parent f5212211
......@@ -140,7 +140,10 @@ void ProcfsConfigurator::sensorGroup(ProcfsSensorGroup& sGroup, CFG_VAL config)
// Read additional configuration parameters supported by the Procfs plugin
std::string filePath="", fileType="", mqttStart="00";
// Set of cpu ids read during configuration
std::set<short> cpuSet;
std::set<int> cpuSet;
std::vector<ProcfsSensorBase*> derivedSensors;
ProcfsParser *parser;
BOOST_FOREACH(boost::property_tree::iptree::value_type &val, config) {
if (boost::iequals(val.first, "type")) {
fileType = val.second.data();
......@@ -152,8 +155,7 @@ void ProcfsConfigurator::sensorGroup(ProcfsSensorGroup& sGroup, CFG_VAL config)
cpuSet = parseCpuString(val.second.data());
}
}
ProcfsParser *parser;
// The "type" parameter must refer to either vmstat, procstat or meminfo. If not, the sensor group is not initialized
// The only other case is when an empty string is found, which can happen for template groups
if(fileType == "")
......@@ -168,39 +170,26 @@ void ProcfsConfigurator::sensorGroup(ProcfsSensorGroup& sGroup, CFG_VAL config)
LOG(warning) << _groupName << " " << sGroup.getGroupName() << "::" << "Unspecified or invalid type! Available types are vmstat, meminfo, procstat";
return; }
// The sensor map contains all sensor objects specified by users in the configuration file; if any are present,
// if any sensor objects specified by users in the configuration file are present,
// automatic MQTT topic assignment is disabled and the topics supplied in the config are used
std::map<std::string, ProcfsSensorBase*> *sensorMap = sGroup.buildMap();
parser->init(sensorMap, &cpuSet);
// Probing the target file to be monitored and counting the available metrics
std::vector<std::string> *availableMetrics = parser->readNames();
// Reading the number of CPU cores present in the system (if needed by the sensor group)
std::vector<short> *metricsCores = parser->readCPUs();
int numMetrics = parser->getNumMetrics();
// Automatic MQTT topic assignment is enabled only if no sensors were specified in the config file
bool autoMQTT = sensorMap->empty();
bool autoMQTT = sGroup.getSensors().empty();
// After getting the vector of available metrics (subset of the sensor map, if any) from the parsed file, we delete
// all sensors not matching any parsed metrics, and arrange their order to respect the one in the file
derivedSensors = sGroup.getDerivedSensors();
parser->init(&derivedSensors, &cpuSet);
sGroup.setType(fileType);
// If no metrics were found in the file (or the file is unreadable) the configuration aborts
int numMetrics = parser->getNumMetrics();
if(numMetrics == 0) {
LOG(warning) << _groupName << " " << sGroup.getGroupName() << "::" << "Unable to parse file " << filePath << ", please check your configuration!";
sGroup.getSensors().clear();
sGroup.getDerivedSensors().clear();
return; }
LOG(debug) << " Number of metrics found: " << numMetrics;
// After getting the vector of available metrics (subset of the sensor map, if any) from the parsed file, we delete
// all sensors not matching any parsed metrics, and arrange their order to respect the one in the file
if(cpuSet.empty() && metricsCores != NULL && !metricsCores->empty()) {
cpuSet.insert(metricsCores->begin(), metricsCores->end());
cpuSet.erase(-1);
}
// Sensors specified in the config are duplicated, so that each CPU core has its own set of sensors
sGroup.prepareCpuSensors(&cpuSet, sensorMap);
unsigned int orphanSensors = sGroup.setupSensors(availableMetrics, sensorMap);
if( orphanSensors > 0 )
LOG(warning) << _groupName << " " << sGroup.getGroupName() << ":: " << orphanSensors << " sensors could not be matched to metrics";
// Assigning the parser to the sensor group
// We assign the final sensor objects as parsed in the target file, and assign the parser object
sGroup.replaceSensors(parser->getSensors());
sGroup.setParser(parser);
// Vector that keeps track of the current number of metrics for each CPU core for which a MQTT topic has been assigned
// All core-independent metrics refer to position 0 in the vector. Used only on automatic MQTT assignment
......@@ -210,21 +199,15 @@ void ProcfsConfigurator::sensorGroup(ProcfsSensorGroup& sGroup, CFG_VAL config)
// Adding the sensors corresponding to availableMetrics
for(int i=0;i < numMetrics; i++) {
ProcfsSensorBase *sensor = (ProcfsSensorBase*)sGroup.getSensors().at(i);
if ( autoMQTT ) sensor->setMqtt(this->increaseMqtt(mqttStart, metricsCounter->at(sensor->getCPUId() + 1)++ ));
// If the metric does not refer to a specific CPU core, the topic is prefix + default mqttPart + suffix
// The suffix is increased automatically through the metricsCounter vector
if( metricsCores == NULL || ( metricsCores != NULL && metricsCores->at(i) == -1 )) {
if ( autoMQTT ) sensor->setMqtt(this->increaseMqtt(mqttStart, metricsCounter->at(0)));
if( sensor->getCPUId() == -1 )
sensor->setMqtt(_mqttPrefix + sGroup.getMqttPart() + sensor->getMqtt());
metricsCounter->at(0)++;
}
// If the metrics refers to a specific CPU core, the topic is prefix + CPU core ID + suffix
// The suffix is automatically increased for each core through the metricsCounter vector
else {
if ( autoMQTT ) sensor->setMqtt(this->increaseMqtt(mqttStart, metricsCounter->at(metricsCores->at(i) + 1)));
sensor->setMqtt(_mqttPrefix + this->formatMqttCPU(sGroup.getMqttPart(), metricsCores->at(i)) + sensor->getMqtt());
metricsCounter->at(metricsCores->at(i) + 1)++;
}
//LOG(debug) << sensor->getName() << " : " << sensor->getMqtt() << " -> " << metricsValues->at(i).value;
else
sensor->setMqtt(_mqttPrefix + this->formatMqttCPU(sGroup.getMqttPart(), sensor->getCPUId()) + sensor->getMqtt());
}
// De-allocating memory
......@@ -234,8 +217,8 @@ void ProcfsConfigurator::sensorGroup(ProcfsSensorGroup& sGroup, CFG_VAL config)
}
//TODO: move this up in ConfiguratorTemplate
std::set<short> ProcfsConfigurator::parseCpuString(const std::string& cpuString) {
std::set<short> cpus;
std::set<int> ProcfsConfigurator::parseCpuString(const std::string& cpuString) {
std::set<int> cpus;
int maxCpu = 512;
std::vector<std::string> subStrings;
......@@ -259,7 +242,7 @@ std::set<short> ProcfsConfigurator::parseCpuString(const std::string& cpuString)
for (int i = minVal; i <= maxVal; i++) {
if (i >= 0 && i < maxCpu) {
cpus.insert((short)i);
cpus.insert((int)i);
}
}
} catch (const std::exception& e) {
......@@ -269,7 +252,7 @@ std::set<short> ProcfsConfigurator::parseCpuString(const std::string& cpuString)
try {
int val = stoi(s);
if (val >= 0 && val < maxCpu) {
cpus.insert((short)val);
cpus.insert((int)val);
}
} catch (const std::exception& e) {
LOG(debug) << "Could not parse value \"" << s << "\"";
......
......@@ -39,7 +39,7 @@ protected:
// Similar to the above, formats a numerical CPU core ID into an hex string
const std::string formatMqttCPU(const std::string& mqttPart, unsigned int val);
// Copy-paste of the method in the perfevent plugin to parse CPU id strings
std::set<short> parseCpuString(const std::string& cpuString);
std::set<int> parseCpuString(const std::string& cpuString);
};
// Functions required to correctly generate Configurator objects from linked libraries
......
This diff is collapsed.
......@@ -10,6 +10,7 @@
#define PROCFSPARSER_H_
#include "ProcfsSensorBase.h"
#include "timestamp.h"
#include <boost/regex.hpp>
#include <vector>
#include <map>
......@@ -30,7 +31,7 @@ public:
~ProcfsParser();
// Init/close methods
bool init(std::map<std::string, ProcfsSensorBase*> *sensorMap = NULL, std::set<short> *cpuSet = NULL);
bool init(std::vector<ProcfsSensorBase*> *sensorVec = NULL, std::set<int> *cpuSet = NULL);
virtual void close();
// Setters and getters
......@@ -38,25 +39,22 @@ public:
void setPath(std::string new_path) { this->_path = new_path; this->close(); }
unsigned int getNumMetrics() { return this->_numMetrics; }
unsigned int getNumCPUs() { return this->_numCPUs; }
// Public parsing methods
std::vector<std::string> *readNames();
std::vector<reading_t> *readMetrics();
std::vector<short> *readCPUs();
unsigned int getCacheIndex() { return this->_cacheIndex; }
void setCacheIndex(unsigned int c) { this->_cacheIndex = c; }
std::vector<ProcfsSensorBase*> *getSensors();
std::vector<ProcfsSensorBase*> *readSensors();
protected:
// Private parsing methods that implement all necessary logic to parse metrics' names (and associated CPU cores, if any)
// and values. MUST be overridden by children classes
virtual std::vector<std::string> *_readNames(std::map<std::string, ProcfsSensorBase*> *sensorMap, std::set<short> *cpuSet) { return NULL; };
virtual std::vector<reading_t> *_readMetrics() { return NULL; };
// Vectors containing the names of parsed metrics, associated CPU cores (if any), and values.
// After initialization, _metricsNames and _metricsReadings are always non-NULL.
// _metricsCores can be NULL if the file does not contain any CPU information (like in vmstat or meminfo)
std::vector<std::string> *_metricsNames;
std::vector<short> *_metricsCores;
std::vector<reading_t> *_metricsReadings;
virtual bool _readNames(std::map<std::string, ProcfsSensorBase*> *sensorMap, std::set<int> *cpuSet) { return NULL; };
virtual bool _readMetrics() { return NULL; };
// Vector containing the names of parsed metrics, associated CPU cores (if any), and values.
// After initialization, _sensors is always non-NULL.
std::vector<ProcfsSensorBase*> *_sensors;
// Keeps track of which lines and columns in the parsed file must be skipped
std::vector<bool> _skipLine;
// Every element in skipColumn can be either 0 (skip column), 1 (parse for all CPUs) or 2 (parse only at node level)
......@@ -64,6 +62,7 @@ protected:
// Internal variables
bool _initialized;
unsigned int _cacheIndex;
unsigned int _numMetrics;
unsigned int _numCPUs;
std::string _path;
......@@ -85,8 +84,8 @@ public:
VmstatParser(const std::string path = "") : ProcfsParser(path) { this->LINE_SEP = " \t"; if(path == "") this->_path = "/proc/vmstat"; else this->_path = path; }
protected:
std::vector<std::string> *_readNames(std::map<std::string, ProcfsSensorBase*> *sensorMap, std::set<short> *cpuSet) override;
std::vector<reading_t> *_readMetrics() override;
bool _readNames(std::map<std::string, ProcfsSensorBase*> *sensorMap, std::set<int> *cpuSet) override;
bool _readMetrics() override;
};
// **************************************************************
......@@ -115,8 +114,8 @@ public:
ProcstatParser(const std::string path = "") : ProcfsParser(path) { this->LINE_SEP = " \t"; if(path == "") this->_path = "/proc/stat"; else this->_path = path; }
protected:
std::vector<std::string> *_readNames(std::map<std::string, ProcfsSensorBase*> *sensorMap, std::set<short> *cpuSet) override;
std::vector<reading_t> *_readMetrics() override;
bool _readNames(std::map<std::string, ProcfsSensorBase*> *sensorMap, std::set<int> *cpuSet) override;
bool _readMetrics() override;
// Internal variables
unsigned short _numColumns;
......
......@@ -25,10 +25,17 @@ public:
this->_cpuId = -1;
}
ProcfsSensorBase(const std::string& name, const std::string& metric, bool percpu = false, int cpuid = -1) : SensorBase(name) {
this->_metric = metric;
this->_perCPU = percpu;
this->_cpuId = cpuid;
}
// Copy constructor
ProcfsSensorBase(ProcfsSensorBase& other) : SensorBase(other) {
this->_metric = other.getMetric();
this->_perCPU = other.isPerCPU();
this->_cpuId = other.getCPUId();
}
virtual ~ProcfsSensorBase() {}
......
......@@ -19,98 +19,28 @@ ProcfsSensorGroup::~ProcfsSensorGroup() {
}
/**
* Builds a STL map data structure that has sensor names of the group as keys, and sensor pointers as values.
* Replaces all sensor objects stored in the group with those contained in newSensors
*
* The map is dynamically allocated. Remember to delete it after use.
* Old sensors are deleted and cleared, therefore make sure that newSensors does not contain any object previously used
* by the sensor group.
*
* @return a map associating sensor names to sensor objects
* @param newSensors: Vector of ProcfsSensorBase objects to replace the old set
*
*/
std::map<std::string, ProcfsSensorBase*> *ProcfsSensorGroup::buildMap() {
std::map<std::string, ProcfsSensorBase*> *sensorMap = new std::map<std::string, ProcfsSensorBase*>();
for( auto s : this->_sensors )
sensorMap->insert(std::make_pair(s->getMetric(), s));
return sensorMap;
}
/**
* Duplicates sensor objects related to per-CPU metrics according to the input cpu set.
*
* Input must be a map built by the buildMap() method, and a set of CPU ids. Then, each sensor object in the map that
* has the perCPU flag set to true, is duplicated N times and renamed accordingly, with N being the number of CPU ids
* contained in the CPU set. No action is performed for sensors that have perCPU set to false.
*
* @param sensorMap: A map of sensor names -> sensor objects built by the buildMap() method
* @param cpuSet: a set of CPU ids
*
*/
void ProcfsSensorGroup::prepareCpuSensors(std::set<short> *cpuSet, std::map<std::string, ProcfsSensorBase*> *sensorMap) {
// The map does not need to be modified if it is empty (obviously), or if no CPU ids are specified in cpuSet
if( sensorMap == NULL || sensorMap->empty() || cpuSet == NULL || cpuSet->empty() )
return;
// The duplicated sensors are stored in a temporary map
std::map<std::string, ProcfsSensorBase*> *tempMap = new std::map<std::string, ProcfsSensorBase*>();
for( auto s : *sensorMap ) {
// If the sensor must be sampled per-cpu core we duplicate it, otherwise we keep it as it is
tempMap->insert(std::make_pair(s.first, s.second));
if (s.second->isPerCPU()) {
for (auto cpu : *cpuSet) {
ProcfsSensorBase *tempSensor = new ProcfsSensorBase(*s.second);
tempSensor->setName("cpu" + std::to_string(cpu) + "_" + tempSensor->getName());
tempSensor->setMetric("cpu" + std::to_string(cpu) + "_" + tempSensor->getMetric());
tempMap->insert(std::make_pair(tempSensor->getMetric(), tempSensor));
}
}
void ProcfsSensorGroup::replaceSensors(std::vector<ProcfsSensorBase*> *newSensors) {
std::set<std::string> sensorSet = std::set<std::string>();
for( auto s_new : *newSensors ) sensorSet.insert( s_new->getMetric() );
for( auto s : this->_sensors ) {
if( sensorSet.find(s->getMetric()) == sensorSet.end() )
LOG(warning) << _groupName << "::Sensor " << s->getName() << " could not be matched to any metric!";
// We clean up the original sensor
delete s;
}
// Transferring the new sensors to the original sensorMap and deleting the tempMap
sensorMap->clear();
for( auto s : *tempMap ) sensorMap->insert(std::make_pair(s.first, s.second));
tempMap->clear();
delete tempMap;
}
/**
* Builds the internal vector of sensor objects so that it respects the order given by the "names" input vector.
*
* Input must be a vector of strings indicating the final order of sensors based on their name, and a map built
* by the buildMap() method. Be aware that in the process the original sensor vector is cleared, and only
* the information in the map is used. At the end of the procedure the map is cleared and deleted.
*
* @param names: a vector of strings indicating the final order of sensors based on their names
* @param sensorMap: A map of sensor names -> sensor objects built by the buildMap() method
*
* @return 0 on success, or the number of orphaned sensors (not matched in the names vector) if any
*/
unsigned int ProcfsSensorGroup::setupSensors(std::vector<std::string> *names, std::map<std::string, ProcfsSensorBase*> *sensorMap) {
// We clean up the original sensors (delete will be performed only for the sensors that must be discarded, later)
sensorSet.clear();
this->_sensors.clear();
this->_baseSensors.clear();
unsigned int orphans = 0;
// If a sensor map was specified, only these sensors are added to the group
if( sensorMap != NULL && !sensorMap->empty() ) {
for(int i = 0; i < names->size(); i++) {
// We assume that names is a subset of sensorMap, and that all entries of the first are in the second
this->pushBackSensor(sensorMap->at(names->at(i)));
sensorMap->erase(names->at(i));
}
// We delete all sensors that were not present in the "names" vector and were therefore not found in the file
if ( ( orphans = sensorMap->size() ) > 0 ) {
for (auto s : *sensorMap)
delete s.second;
sensorMap->clear();
}
}
// If no map was specified, we build all-new sensors depending on the "names" vector, as parsed from the file
else {
for(int i=0; i < names->size(); i++)
this->pushBackSensor(new ProcfsSensorBase(names->at(i)));
}
if ( sensorMap != NULL ) delete sensorMap;
return orphans;
for( auto s : *newSensors ) this->pushBackSensor(s);
}
/**
......@@ -127,6 +57,12 @@ void ProcfsSensorGroup::start() {
// If a parser object has not been set, the sensor group cannot be initialized
if( this->_parser != NULL ) {
//this->_parser->init();
// Crude debugging stuff
//this->_parser->readSensors();
//for( auto sensor : _sensors)
// LOG(debug) << sensor->getName() << " : " << sensor->getMqtt() << " -> " << sensor->getLatestValue().value;
_keepRunning = 1;
_pendingTasks++;
_timer->async_wait(std::bind(&ProcfsSensorGroup::readAsync, this));
......@@ -161,25 +97,20 @@ void ProcfsSensorGroup::stop() {
*
*/
void ProcfsSensorGroup::read() {
reading_t reading;
reading.timestamp = getTimestamp();
std::vector<reading_t> *metricsReadings = NULL;
// Read values from the target file using the parser's _readMetrics() method
// If an error is encountered (NULL return value) we abort here
metricsReadings = this->_parser->readMetrics();
if( metricsReadings == NULL ) {
// If an error is encountered (false return value) we abort here
// Sensors are automatically updated from within the parser
if( !this->_parser->readSensors() ) {
LOG(error) << _groupName << "::" << "Could not read values from " << this->_type << "!";
return;
}
for(int i=0; i < this->_parser->getNumMetrics(); i++) {
// Values are read from the vector returned by _readMetrics() and pushed into the sensors
reading.value = metricsReadings->at(i).value;
#ifdef DEBUG
LOG(debug) << _groupName << "::" << _sensors[i]->getName() << ": \"" << reading.value << "\"";
for(int i=0; i < this->_parser->getNumMetrics(); i++)
LOG(debug) << _groupName << "::" << _sensors[i]->getName() << ": \"" << reading.value << "\"";
#endif
_sensors[i]->storeReading(reading, _cacheIndex);
}
_cacheIndex = (_cacheIndex + 1) % _cacheSize;
// Updating the cache index used to store readings within the parser
this->_parser->setCacheIndex(_cacheIndex);
}
/**
......
......@@ -9,7 +9,6 @@
#define PROCFSSENSORGROUP_H_
#include "ProcfsParser.h"
#include "timestamp.h"
#include "../../includes/SensorGroupTemplate.h"
/**
......@@ -35,10 +34,9 @@ public:
void setType(std::string t) { this->_type = t; }
std::string getType() { return this->_type; }
// Methods to build and manage map data structures for sensors, used during configuration
std::map<std::string, ProcfsSensorBase*> *buildMap();
void prepareCpuSensors(std::set<short> *cpuSet, std::map<std::string, ProcfsSensorBase*> *sensorMap);
unsigned int setupSensors(std::vector<std::string> *names, std::map<std::string, ProcfsSensorBase*> *sensorMap = NULL);
// Method to get the internal sensor vector used by the sensor group (different from getSensors)
std::vector<ProcfsSensorBase*> getDerivedSensors() { return this->_sensors; }
void replaceSensors(std::vector<ProcfsSensorBase*> *newSensors);
private:
// Methods inherited from SensorGroupTemplate
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment