Merge branch 'msr' into 'master'

Msr

See merge request dcdb/dcdbpusher!2

Makefile

@@ -19,7 +19,7 @@ CXXFLAGS =	-std=c++11 -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_NETWORK_EN
 LIBS = -L../deps/mosquitto_build/lib -L$(DCDBDEPLOYPATH)/lib/ -ldl -lmosquitto -lboost_system -lboost_thread -lboost_log_setup -lboost_log -lboost_regex -lpthread -lcrypto -lssl -lcppnetlib-server-parsers -lcppnetlib-uri -rdynamic
 OBJS = src/dcdbpusher.o src/Configuration.o src/MQTTPusher.o src/HttpsServer.o src/analytics/AnalyticsManager.o src/analytics/SensorNavigator.o
 
-PLUGINS = procfs pdu sysfs ipmi bacnet snmp gpfsmon tester
+PLUGINS = procfs pdu sysfs ipmi bacnet snmp gpfsmon tester msr
 ANALYZERS = average
 
 ifeq ($(OS),Darwin)
@@ -116,6 +116,9 @@ libdcdbplugin_tester.$(LIBEXT): src/sensors/tester/TesterSensorGroup.o src/senso
 libdcdbplugin_gpfsmon.$(LIBEXT): src/sensors/gpfsmon/GpfsmonSensorGroup.o src/sensors/gpfsmon/GpfsmonConfigurator.o
 	$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system -lboost_regex
 
+libdcdbplugin_msr.$(LIBEXT): src/sensors/msr/MSRSensorGroup.o src/sensors/msr/MSRConfigurator.o
+	$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system
+
 #libdcdbplugin_opa.$(LIBEXT): src/sensors/opa/OpaSensorGroup.o src/sensors/opa/OpaConfigurator.o
 #	$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system -lboost_regex -lopamgt -libverbs -libumad -lssl
 

config/msr.conf

+;comments in config files are indicated by a semicolon
+
+global {
+	mqttPrefix	/FF112233445566778899AABBFFFF
+	
+	;add here other global attributes for your plugin
+}
+
+group g1 {
+	interval	1000
+	mqttprefix	01
+	cpus		0-95
+
+	sensor Instructions {
+		mqttsuffix	00
+		metric		0x309
+	}
+
+	sensor Cycles {
+		mqttsuffix	01
+		metric		0x30A
+	}
+
+	sensor RefCycles {
+		mqttsuffix	02
+		metric		0x30B
+	}
+}
+
+

src/includes/ConfiguratorTemplate.h

@@ -382,8 +382,7 @@ protected:
 		if (!isTemplate) {
 			boost::optional<boost::property_tree::iptree&> def = config.get_child_optional("default");
 			if(def) {
-				//we copy all values from default (including copy constructing its sensors)
-				//if own sensors are specified they are appended
+				//we copy all values from default
 				LOG(debug) << "  Using \"" << def.get().data() << "\" as default.";
 				auto it = _templateSensorBases.find(def.get().data());
 				if(it != _templateSensorBases.end()) {

src/sensors/msr/MSRConfigurator.cpp

+/*
+ * MSRConfigurator.cpp
+ *
+ *  Created on: 28.01.2019
+ *      Author: Carla Guillen
+ */
+
+#include "MSRConfigurator.h"
+#include <iomanip>
+
+MSRConfigurator::MSRConfigurator() {
+	_groupName = "group";
+	_baseName = "sensor";
+}
+
+MSRConfigurator::~MSRConfigurator() {}
+
+void MSRConfigurator::sensorBase(MSRSensorBase& s, CFG_VAL config) {
+	ADD {
+		if (boost::iequals(val.first, "metric")){
+			//TODO try catch...
+			uint64_t metric = std::stoull(val.second.data(),nullptr,16);
+			s.setMetric(metric);
+		}
+	}
+}
+
+void MSRConfigurator::sensorGroup(MSRSensorGroup& s, CFG_VAL config) {
+	ADD {
+		if (boost::iequals(val.first, "cpus")){
+			std::set<int> cpus = ConfiguratorTemplate::parseCpuString(val.second.data());
+			for(int cpu: cpus){
+				s.addCpu(static_cast<unsigned int>(cpu));
+			}
+		}
+	}
+}
+
+/**
+ * Custom readConfig, as MSR has to copy sensors for each CPU
+ */
+bool MSRConfigurator::readConfig(std::string cfgPath) {
+    _cfgPath = cfgPath;
+
+    boost::property_tree::iptree cfg;
+    boost::property_tree::read_info(cfgPath, cfg);
+
+    //read global variables (if present overwrite those from global.conf)
+    readGlobal(cfg);
+
+    //read groups and templates for groups
+    BOOST_FOREACH(boost::property_tree::iptree::value_type &val, cfg) {
+        //template group
+        if (boost::iequals(val.first, "template_" + _groupName)) {
+            LOG(debug) << "Template " << _groupName << " \"" << val.second.data() << "\"";
+            if (!val.second.empty()) {
+                MSRSensorGroup* group = new MSRSensorGroup(val.second.data());
+                if (readSensorGroup(*group, val.second, true)) {
+                    auto ret = _templateSensorGroups.insert(std::pair<std::string, MSRSensorGroup*>(val.second.data(), group));
+                    if(!ret.second) {
+                        LOG(warning) << "Template " << _groupName << " " << val.second.data() << " already exists! Omitting...";
+                        delete group;
+                    }
+                } else {
+                    LOG(warning) << "Template " << _groupName << " \"" << val.second.data() << "\" has bad values! Ignoring...";
+                    delete group;
+                }
+            }
+        //template base
+        } else if (boost::iequals(val.first, "template_" + _baseName)) {
+            LOG(debug) << "Template " << _baseName << " \"" << val.second.data() << "\"";
+            if (!val.second.empty()) {
+                MSRSensorBase* base = new MSRSensorBase(val.second.data());
+                if (readSensorBase(*base, val.second, true)) {
+                    auto ret = _templateSensorBases.insert(std::pair<std::string, MSRSensorBase*>(val.second.data(), base));
+                    if(!ret.second) {
+                        LOG(warning) << "Template " << _baseName << " " << val.second.data() << " already exists! Omitting...";
+                        delete base;
+                    }
+                } else {
+                    LOG(warning) << "Template " << _baseName << " \"" << val.second.data() << "\" has bad values! Ignoring...";
+                    delete base;
+                }
+            }
+        //template single sensor
+        } else if (boost::iequals(val.first, "template_single_" + _baseName)) {
+            LOG(debug) << "Template single " << _baseName << " \"" << val.second.data() << "\"";
+            if (!val.second.empty()) {
+                MSRSensorGroup* group = new MSRSensorGroup(val.second.data());
+                if (readSensorGroup(*group, val.second, true)) {
+                    //group which consists of only one sensor
+                    SB_Ptr sensor = std::make_shared<MSRSensorBase>(val.second.data());
+                    if (readSensorBase(*sensor, val.second, true)) {
+                        group->pushBackSensor(sensor);
+                        auto ret = _templateSensorGroups.insert(std::pair<std::string, MSRSensorGroup*>(val.second.data(), group));
+                        if(!ret.second) {
+                            LOG(warning) << "Template single " << _baseName << " " << val.second.data() << " already exists! Omitting...";
+                            delete group;
+                        }
+                    } else {
+                        LOG(warning) << "Template single " << _baseName << " " << val.second.data() << " could not be read! Omitting";
+                        delete group;
+                    }
+                } else {
+                    LOG(warning) << "Template single " << _baseName << " \"" << val.second.data() << "\" has bad values! Ignoring...";
+                    delete group;
+                }
+            }
+        //group
+        } else if (boost::iequals(val.first, _groupName)) {
+            LOG(debug) << _groupName << " \"" << val.second.data() << "\"";
+            if (!val.second.empty()) {
+                SG_Ptr group = std::make_shared<MSRSensorGroup>(val.second.data());
+                if (readSensorGroup(*group, val.second)) {
+                    customizeAndStore(group);
+                } else {
+                    LOG(warning) << _groupName << " \"" << val.second.data() << "\" has bad values! Ignoring...";
+                }
+            }
+        //single sensor
+        } else if (boost::iequals(val.first, "single_" + _baseName)) {
+            LOG(debug) << "Single " << _baseName << " \"" << val.second.data() << "\"";
+            if (!val.second.empty()) {
+                SG_Ptr group = std::make_shared<MSRSensorGroup>(val.second.data());
+                if (readSensorGroup(*group, val.second)) {
+                    //group which consists of only one sensor
+                    SB_Ptr sensor;
+                    //perhaps one sensor is already present because it was copied from the template group
+                    if (group->getSensors().size() != 0) {
+                        sensor = std::dynamic_pointer_cast<MSRSensorBase>(group->getSensors()[0]);
+                        //check if cast was successful (sensor != nullptr)
+                        if (sensor) {
+                            sensor->setName(val.second.data());
+                            if (readSensorBase(*sensor, val.second)) {
+                                customizeAndStore(group);
+                            } else {
+                                LOG(warning) << "Single " << _baseName << " " << val.second.data() << " could not be read! Omitting";
+                            }
+                        } else {
+                            LOG(warning) << "Single " << _baseName << " " << val.second.data() << " had a type mismatch when casting! Omitting";
+                        }
+                    } else {
+                        sensor = std::make_shared<MSRSensorBase>(val.second.data());
+                        if (readSensorBase(*sensor, val.second)) {
+                            group->pushBackSensor(sensor);
+                            customizeAndStore(group);
+                        } else {
+                            LOG(warning) << "Single " << _baseName << " " << val.second.data() << " could not be read! Omitting";
+                        }
+                    }
+                } else {
+                    LOG(warning) << "Single " << _baseName << " \"" << val.second.data() << "\" has bad values! Ignoring...";
+                }
+            }
+        } else if( !boost::iequals(val.first, "global") ) {
+            LOG(error) << "\"" << val.first << "\": unknown construct!";
+            return false;
+        }
+    }
+    //read of config finished. Now we build the mqtt-topic for every sensor
+    constructSensorNames();
+    for(const auto& g : _sensorGroups) {
+        for(const auto& s : g->getSensors()) {
+            s->setMqtt(_mqttPrefix + g->getMqttPart() + s->getMqtt());
+            LOG(debug) << g->getGroupName() << "::" << s->getName() << " using MQTT-topic \"" << s->getMqtt() << "\"";
+        }
+    }
+    return true;
+}
+
+void MSRConfigurator::customizeAndStore(SG_Ptr g) {
+  bool begin = true;
+
+  std::vector<SB_Ptr> original;
+
+  for(auto cpu : g->getCpus()) {
+      if (begin){
+          for(auto s : g->getSensors()) {
+              SB_Ptr sensor = std::dynamic_pointer_cast<MSRSensorBase>(s);
+              sensor->setCpu(cpu);
+              s->setName(s->getName(), cpu);
+              auto size = s->getMqtt().size();
+              s->setMqtt(formatMqttCPU("XX", cpu) + s->getMqtt().substr(size-2));
+              original.push_back(sensor);
+          }
+          begin = false;
+      } else {
+          for (auto s: original) {
+              auto s_otherCPUs = std::make_shared<MSRSensorBase>(s->getName());
+	      std::size_t found = s->getName().find_first_of(".");
+	      if( found != std::string::npos){
+		found++; //to skip the point
+		s_otherCPUs->setName(s->getName().substr(found), cpu);
+	      }
+              s_otherCPUs->setCpu(cpu);
+              s_otherCPUs->setMetric(s->getMetric());
+              auto size = s->getMqtt().size();
+              s_otherCPUs->setMqtt(formatMqttCPU("XX", cpu) + s->getMqtt().substr(size-2));
+              g->pushBackSensor(s_otherCPUs);
+          }
+      }
+  }
+  storeSensorGroup(g);
+}
+

src/sensors/msr/MSRConfigurator.h

+/*
+ * MSRConfigurator.h
+ *
+ *  Created on: 28.01.2019
+ *      Author: Carla Guillen
+ */
+
+#ifndef MSR_MSRCONFIGURATOR_H_
+#define MSR_MSRCONFIGURATOR_H_
+
+#include "../../includes/ConfiguratorTemplate.h"
+#include "MSRSensorGroup.h"
+
+class MSRConfigurator : public ConfiguratorTemplate<MSRSensorBase, MSRSensorGroup> {
+
+public:
+	MSRConfigurator();
+	virtual ~MSRConfigurator();
+
+protected:
+	/* Overwritten from ConfiguratorTemplate */
+	void sensorBase(MSRSensorBase& s, CFG_VAL config) override;
+	void sensorGroup(MSRSensorGroup& s, CFG_VAL config) override;
+	bool readConfig(std::string cfgPath) override;
+
+private:
+	/**
+     *  Takes a MSRSensorGroup and duplicates its sensors for every CPU.
+     *  Assigns one CPU value to every newly constructed sensor and stores the
+     *  group afterwards.
+     *
+     *  @param g    MSRSensorGroup which is to be customized for every CPU
+     */
+	void customizeAndStore(SG_Ptr g);
+};
+
+extern "C" ConfiguratorInterface* create() {
+	return new MSRConfigurator;
+}
+
+extern "C" void destroy(ConfiguratorInterface* c) {
+	delete c;
+}
+
+#endif /* MSR_MSRCONFIGURATOR_H_ */

src/sensors/msr/MSRSensorBase.h

+/*
+ * MSRSensorBase.h
+ *
+ *  Created on: 28.01.2019
+ *      Author: Carla Guillen
+ */
+
+#ifndef MSR_MSRSENSORBASE_H_
+#define MSR_MSRSENSORBASE_H_
+
+#include "../../includes/SensorBase.h"
+
+class MSRSensorBase : public SensorBase {
+public:
+	MSRSensorBase(const std::string& name) :
+		SensorBase(name), _cpu(0), _metric(0) {
+	  //default delta to true, as msr has only monotonic sensors usually
+      _delta = true;
+	}
+	
+	MSRSensorBase(const MSRSensorBase& other)=default;
+
+	virtual ~MSRSensorBase() {
+	}
+	
+	MSRSensorBase& operator=(const MSRSensorBase& other)=default;
+
+	unsigned int getCpu() const {
+		return _cpu;
+	}
+
+	void setCpu(unsigned int cpu) {
+		_cpu = cpu;
+	}
+
+	uint64_t getMetric() const {
+		return _metric;
+	}
+
+	void setMetric(uint64_t metric) {
+		_metric = metric;
+	}
+
+	void printConfig(LOG_LEVEL ll, LOGGER& lg) {
+        	LOG_VAR(ll) << "     CPU:    " << _cpu;
+        	LOG_VAR(ll) << "     Metric: " << _metric;
+    	}
+
+protected:
+	unsigned int _cpu;
+	uint64_t _metric;
+
+};
+
+#endif /* MSR_MSRSENSORBASE_H_ */

src/sensors/msr/MSRSensorGroup.cpp

+/*
+ * MSRSensorGroup.cpp
+ *
+ *  Created on: 28.01.2019
+ *      Author: Carla Guillen
+ */
+
+#include "MSRSensorGroup.h"
+
+#include <boost/log/core/record.hpp>
+#include <boost/log/sources/record_ostream.hpp>
+#include <boost/log/trivial.hpp>
+#include <boost/log/utility/formatting_ostream.hpp>
+#include <boost/parameter/keyword.hpp>
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <atomic>
+#include <exception>
+#include <utility>
+
+#include "../../includes/Logging.h"
+#include "../../includes/SensorBase.h"
+#include "timestamp.h"
+#include "Types.h"
+#include <sstream>
+#include <iomanip>
+
+
+MSRSensorGroup::MSRSensorGroup(const std::string& name) :
+	SensorGroupTemplate(name) {
+}
+
+MSRSensorGroup::~MSRSensorGroup() {
+}
+
+void MSRSensorGroup::start() {
+	if (_keepRunning) {
+		//we have been started already
+		LOG(info) << "Sensorgroup " << _groupName << " already running.";
+		return;
+	}
+
+	for (auto &kv : cpuToFd) {
+		int cpu = kv.first;
+		char * path = new char[200];
+		snprintf(path, 200, "/dev/cpu/%d/msr", cpu);
+		int handle = open(path, O_RDWR);
+		if (handle < 0) {   // try msr_safe
+			snprintf(path, 200, "/dev/cpu/%d/msr_safe", cpu);
+			handle = open(path, O_RDWR);
+		}
+		if (handle < 0){
+			LOG(error) << "Can't open msr device " << path;
+			delete [] path; // TODO do this with RAII
+			continue;
+		}
+		delete [] path; //TODO do this with RAII
+		cpuToFd[cpu] = handle;
+	}
+
+	program_fixed();
+
+	_keepRunning = 1;
+	_pendingTasks++;
+	_timer->async_wait(std::bind(&MSRSensorGroup::readAsync, this));
+	LOG(info) << "Sensorgroup " << _groupName << " started.";
+}
+
+void MSRSensorGroup::stop() {
+    _keepRunning = 0;
+    //close file descriptors and leave counters running freely
+    for (auto &kv: cpuToFd) {
+    	close(kv.second);
+        kv.second = -1;
+    }
+
+    LOG(info) << "Sensorgroup " << _groupName << " stopped.";
+}
+
+void MSRSensorGroup::read() {
+	ureading_t reading;
+	reading.timestamp = getTimestamp();
+
+	try {
+		for(auto s : _sensors) {
+			auto ret_val = msr_read(s->getMetric(), &reading.value, s->getCpu());
+			if(ret_val != -1){
+				 s->storeReading(reading);
+#ifdef DEBUG
+				LOG(debug) << _groupName << "::" << s->getName() << " raw reading: \"" << reading.value << "\"";
+#endif
+			}
+		}
+	} catch (const std::exception& e) {
+		LOG(error) << "Sensorgroup" << _groupName << " could not read value: " << e.what();
+	}
+}
+
+void MSRSensorGroup::readAsync() {
+	uint64_t now = getTimestamp();
+	read();
+	if (_timer && _keepRunning) {
+		_timer->expires_at(timestamp2ptime(nextReadingTime()));
+		_pendingTasks++;
+		_timer->async_wait(std::bind(&MSRSensorGroup::readAsync, this));
+	}
+	_pendingTasks--;
+}
+
+int32_t MSRSensorGroup::msr_read(uint64_t msr_number, uint64_t * value, unsigned int cpu){
+	return pread(cpuToFd[cpu], (void *) value, sizeof(uint64_t), msr_number);
+}
+
+int32_t MSRSensorGroup::msr_write(uint64_t msr_number, uint64_t value, unsigned int cpu){
+	return pwrite(cpuToFd[cpu], (const void *) &value, sizeof(uint64_t), msr_number);
+}
+
+/**
+ * Program the fixed MSR as required for this plugin.
+ *
+ * @return  True if counters programmed successfully, false otherwise, e.g.
+ *          because the counters are already in use.
+ */
+void MSRSensorGroup::program_fixed(){
+
+	for (auto &kv : cpuToFd) {
+		// program core counters
+
+	    //we do not want to interrupt other services already doing measurements with MSRs
+        //therefore check if any fixed counter is currently enabled
+		struct FixedEventControlRegister ctrl_reg;
+
+		msr_read(IA32_CR_FIXED_CTR_CTRL, &ctrl_reg.value, kv.first);
+		//are they all enabled?
+		if (ctrl_reg.fields.os0 && ctrl_reg.fields.usr0 && ctrl_reg.fields.os1
+						&& ctrl_reg.fields.usr1 && ctrl_reg.fields.os2
+						&& ctrl_reg.fields.usr2) {
+			//yes! Free running counters were set by someone else => we don't need to program them, just read them.
+			LOG(debug) << "CPU" << kv.first << " has free running counter, so there will be no fixed counter programming";
+			continue;
+		}
+		//not all of them (or none) are enabled => we program them again
+
+		// disable counters while programming
+        msr_write(IA32_CR_PERF_GLOBAL_CTRL, 0, kv.first);
+
+		ctrl_reg.fields.os0 = 1;
+		ctrl_reg.fields.usr0 = 1;
+		ctrl_reg.fields.any_thread0 = 0;
+		ctrl_reg.fields.enable_pmi0 = 0;
+
+		ctrl_reg.fields.os1 = 1;
+		ctrl_reg.fields.usr1 = 1;
+		ctrl_reg.fields.any_thread1 = 0;
+		ctrl_reg.fields.enable_pmi1 = 0;
+
+		ctrl_reg.fields.os2 = 1;
+		ctrl_reg.fields.usr2 = 1;
+		ctrl_reg.fields.any_thread2 = 0;
+		ctrl_reg.fields.enable_pmi2 = 0;
+
+		ctrl_reg.fields.reserved1 = 0;
+
+		// program them
+		msr_write(IA32_CR_FIXED_CTR_CTRL, ctrl_reg.value, kv.first);
+
+		// start counting, enable 3 fixed counters (enable also the programmables counters)
+		uint64_t value = (1ULL << 0) + (1ULL << 1) + (1ULL << 2) + (1ULL << 3) + (1ULL << 32) + (1ULL << 33) + (1ULL << 34);
+		//uint64_t value = (1ULL << 32) + (1ULL << 33) + (1ULL << 34);
+		msr_write(IA32_CR_PERF_GLOBAL_CTRL, value, kv.first);
+	}
+}
+
+void MSRSensorGroup::addCpu(unsigned int cpu){
+	cpuToFd[cpu] = -1; /* -1 because no file descriptor has been assigned yet. */
+}
+
+std::vector<unsigned> MSRSensorGroup::getCpus() {
+  std::vector<unsigned> cpus;
+  for(auto kv : cpuToFd) {
+    cpus.push_back(kv.first);
+  }
+  return cpus;
+}
+
+void MSRSensorGroup::printConfig(LOG_LEVEL ll) {
+    std::stringstream ss;
+    const char* separator = "";
+    for (auto &kv : cpuToFd) {
+      ss << separator << kv.first;
+      separator = ", ";
+    }
+
+    LOG_VAR(ll) << "   CPUs:  " << ss.str();
+}

src/sensors/msr/MSRSensorGroup.h

+/*
+ * MSRSensorGroup.h
+ *
+ *  Created on: 28.01.2019
+ *      Author: Carla Guillen
+ */
+
+#ifndef MSR_MSRSENSORGROUP_H_
+#define MSR_MSRSENSORGROUP_H_
+
+#include "../../includes/SensorGroupTemplate.h"
+
+#include <map>
+#include <vector>
+#include <string>
+#include <cstdint>
+
+#include "MSRSensorBase.h"
+
+class MSRSensorGroup : public SensorGroupTemplate<MSRSensorBase> {
+public:
+	MSRSensorGroup(const std::string& name);
+	MSRSensorGroup(const MSRSensorGroup& other)=default;
+	virtual ~MSRSensorGroup();
+	MSRSensorGroup& operator=(const MSRSensorGroup& other)=default;
+
+	void start() override;
+	void stop() override;
+	void addCpu(unsigned int cpu);
+	std::vector<unsigned> getCpus();
+
+	void printConfig(LOG_LEVEL ll) override;
+