development of msr plugin

src/sensors/msr/MSRConfigurator.cpp

+/*
+ * MSRConfigurator.cpp
+ *
+ *  Created on: 28.01.2019
+ *      Author: Your name goes here!
+ */
+
+#include "MSRConfigurator.h"
+
+MSRConfigurator::MSRConfigurator() {
+	_groupName = "group";
+	_baseName = "sensor";
+}
+
+MSRConfigurator::~MSRConfigurator() {}
+
+void MSRConfigurator::sensorBase(MSRSensorBase& s, CFG_VAL config) {
+	ADD {
+		if (boost::iequals(val.first, "delta")) {
+			//it is explicitly stated to be off --> set it to false
+			s.setDelta( !(val.second.data() == "off") );
+		} else if (boost::iequals(val.first, "metric")){
+			s.setMetric(std::stoull(val.second.data(),nullptr,16));
+		}
+	}
+}
+
+void MSRConfigurator::sensorGroup(MSRSensorGroup& s, CFG_VAL config) {
+	ADD {
+		if (boost::iequals(val.first, "cpus")){
+			std::set<int> cpus = ConfiguratorTemplate::parseCpuString(val.second.data());
+			for(int cpu: cpus){
+				s.addCpu(static_cast<unsigned int>(cpu));
+			}
+		}
+	}
+}
+

src/sensors/msr/MSRConfigurator.h

+/*
+ * MSRConfigurator.h
+ *
+ *  Created on: 28.01.2019
+ *      Author: Your name goes here!
+ */
+
+#ifndef MSR_MSRCONFIGURATOR_H_
+#define MSR_MSRCONFIGURATOR_H_
+
+#include "../../includes/ConfiguratorTemplate.h"
+#include "MSRSensorGroup.h"
+
+class MSRConfigurator : public ConfiguratorTemplate<MSRSensorBase, MSRSensorGroup> {
+
+public:
+	MSRConfigurator();
+	virtual ~MSRConfigurator();
+
+protected:
+	/* Overwritten from ConfiguratorTemplate */
+	void sensorBase(MSRSensorBase& s, CFG_VAL config) override;
+	void sensorGroup(MSRSensorGroup& s, CFG_VAL config) override;
+};
+
+extern "C" ConfiguratorInterface* create() {
+	return new MSRConfigurator;
+}
+
+extern "C" void destroy(ConfiguratorInterface* c) {
+	delete c;
+}
+
+#endif /* MSR_MSRCONFIGURATOR_H_ */

src/sensors/msr/MSRSensorBase.h

+/*
+ * MSRSensorBase.h
+ *
+ *  Created on: 28.01.2019
+ *      Author: Your name goes here!
+ */
+
+#ifndef MSR_MSRSENSORBASE_H_
+#define MSR_MSRSENSORBASE_H_
+
+#include "../../includes/SensorBase.h"
+
+class MSRSensorBase : public SensorBase {
+public:
+	MSRSensorBase(const std::string& name) :
+		SensorBase(name), _cpu(0), _metric(0) {
+
+	}
+	
+	MSRSensorBase(const MSRSensorBase& other)=default;
+
+	virtual ~MSRSensorBase() {
+	}
+	
+	MSRSensorBase& operator=(const MSRSensorBase& other)=default;
+
+	unsigned int getCpu() const {
+		return _cpu;
+	}
+
+	void setCpu(unsigned int cpu) {
+		_cpu = cpu;
+	}
+
+	uint64_t getMetric() const {
+		return _metric;
+	}
+
+	void setMetric(uint64_t metric) {
+		_metric = metric;
+	}
+
+protected:
+	unsigned int _cpu;
+	uint64_t _metric;
+
+};
+
+#endif /* MSR_MSRSENSORBASE_H_ */

src/sensors/msr/MSRSensorGroup.cpp

+/*
+ * MSRSensorGroup.cpp
+ *
+ *  Created on: 28.01.2019
+ *      Author: Your name goes here!
+ */
+
+#include "MSRSensorGroup.h"
+
+#include <boost/log/core/record.hpp>
+#include <boost/log/sources/record_ostream.hpp>
+#include <boost/log/trivial.hpp>
+#include <boost/log/utility/formatting_ostream.hpp>
+#include <boost/parameter/keyword.hpp>
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <atomic>
+#include <exception>
+#include <utility>
+
+#include "../../includes/Logging.h"
+#include "../../includes/SensorBase.h"
+#include "timestamp.h"
+#include "Types.h"
+#include <sstream>
+#include <iomanip>
+
+
+MSRSensorGroup::MSRSensorGroup(const std::string& name) :
+	SensorGroupTemplate(name) {
+}
+
+MSRSensorGroup::~MSRSensorGroup() {
+}
+
+void MSRSensorGroup::init(boost::asio::io_service& io) {
+	SensorGroupTemplate::init(io);
+	bool begin = true;
+
+	std::vector<S_Ptr> original;
+	for(auto &kv: cpuToFd){
+		if (begin){
+			for(auto s : _sensors) {
+				s->setCpu(kv.first);
+				s->setName(s->getName(),kv.first);
+				std::stringstream ss;
+				auto size = s->getMqtt().size();
+				ss << _mqttPrefix << std::setw(2) << std::setfill('0') << kv.first << s->getMqtt().substr(size-2);
+				std::string mqtt = ss.str();
+				s->setMqtt(mqtt);
+				original.push_back(s);
+			}
+			begin = false;
+		} else {
+			for (auto s: original) {
+				auto s_otherCPUs = std::make_shared<MSRSensorBase>(s->getName());
+				s_otherCPUs->setName(s->getName(), kv.first);
+				s_otherCPUs->setCpu(kv.first);
+				s_otherCPUs->setMetric(s->getMetric());
+//				SBasePtr sensorBaseShrd = std::dynamic_pointer_cast<SBasePtr>(s_otherCPUs);
+				pushBackSensor(s_otherCPUs);
+			}
+		}
+	}
+}
+
+void MSRSensorGroup::start() {
+	if (_keepRunning) {
+		//we have been started already
+		LOG(info) << "Sensorgroup " << _groupName << " already running.";
+		return;
+	}
+
+	for (auto &kv : cpuToFd) {
+		int cpu = kv.first;
+		char * path = new char[200];
+		snprintf(path, 200, "/dev/cpu/%d/msr", cpu);
+		int handle = open(path, O_RDWR);
+		if (handle < 0) {   // try msr_safe
+			snprintf(path, 200, "/dev/cpu/%d/msr_safe", cpu);
+			handle = open(path, O_RDWR);
+		}
+		if (handle < 0){
+			LOG(error) << "Can't open msr device " << path;
+			delete [] path; // TODO do this with RAII
+			continue;
+		}
+		delete [] path; //TODO do this with RAII
+		cpuToFd[cpu] = handle;
+	}
+
+	program_fixed();
+
+	_keepRunning = 1;
+	_pendingTasks++;
+	_timer->async_wait(std::bind(&MSRSensorGroup::readAsync, this));
+	LOG(info) << "Sensorgroup " << _groupName << " started.";
+}
+
+void MSRSensorGroup::stop() {
+	_keepRunning = 0;
+	
+	for (auto &kv: cpuToFd) {
+		close(kv.second);
+		kv.second = -1;
+	}
+	 
+	LOG(info) << "Sensorgroup " << _groupName << " stopped.";
+}
+
+void MSRSensorGroup::read() {
+	ureading_t reading;
+	reading.timestamp = getTimestamp();
+
+	try {
+		for(auto s : _sensors) {
+			msr_read(s->getMetric(), &reading.value, s->getCpu());
+			s->storeReading(reading);
+#ifdef DEBUG
+			LOG(debug) << _groupName << "::" << s->getName() << " raw reading: \"" << reading.value << "\"";
+#endif
+		}
+	} catch (const std::exception& e) {
+		LOG(error) << "Sensorgroup" << _groupName << " could not read value: " << e.what();
+	}
+}
+
+void MSRSensorGroup::readAsync() {
+	uint64_t now = getTimestamp();
+	read();
+	if (_timer && _keepRunning) {
+		_timer->expires_at(timestamp2ptime(nextReadingTime()));
+		_pendingTasks++;
+		_timer->async_wait(std::bind(&MSRSensorGroup::readAsync, this));
+	}
+	_pendingTasks--;
+}
+
+int32_t MSRSensorGroup::msr_read(uint64_t msr_number, uint64_t * value, unsigned int cpu){
+	return pread(cpuToFd[cpu], (void *) value, sizeof(uint64_t), msr_number);
+}
+
+int32_t MSRSensorGroup::msr_write(uint64_t msr_number, uint64_t value, unsigned int cpu){
+	return pwrite(cpuToFd[cpu], (const void *) &value, sizeof(uint64_t), msr_number);
+}
+
+void MSRSensorGroup::program_fixed(){
+	for (auto &kv : cpuToFd) {
+		// program core counters
+
+		struct FixedEventControlRegister ctrl_reg;
+		// disable counters while programming
+		msr_write(IA32_CR_PERF_GLOBAL_CTRL, 0, kv.first);
+		msr_read(IA32_CR_FIXED_CTR_CTRL, &ctrl_reg.value, kv.first);
+
+		ctrl_reg.fields.os0 = 1;
+		ctrl_reg.fields.usr0 = 1;
+		ctrl_reg.fields.any_thread0 = 0;
+		ctrl_reg.fields.enable_pmi0 = 0;
+
+		ctrl_reg.fields.os1 = 1;
+		ctrl_reg.fields.usr1 = 1;
+		ctrl_reg.fields.any_thread1 = 0;
+		ctrl_reg.fields.enable_pmi1 = 0;
+
+		ctrl_reg.fields.os2 = 1;
+		ctrl_reg.fields.usr2 = 1;
+		ctrl_reg.fields.any_thread2 = 0;
+		ctrl_reg.fields.enable_pmi2 = 0;
+
+		ctrl_reg.fields.reserved1 = 0;
+		msr_write(IA32_CR_FIXED_CTR_CTRL, ctrl_reg.value, kv.first);
+
+		// start counting, enable 3 fixed counters
+		//uint64 value = (1ULL << 0) + (1ULL << 1) + (1ULL << 2) + (1ULL << 3) + (1ULL << 32) + (1ULL << 33) + (1ULL << 34);
+		uint64_t value = (1ULL << 32) + (1ULL << 33) + (1ULL << 34);
+		msr_write(IA32_CR_PERF_GLOBAL_CTRL, value, kv.first);
+	}
+
+}
+
+void MSRSensorGroup::addCpu(unsigned int cpu){
+	cpuToFd[cpu] = -1; /* -1 because no file descriptor has been assigned yet. */
+}

src/sensors/msr/MSRSensorGroup.h

+/*
+ * MSRSensorGroup.h
+ *
+ *  Created on: 28.01.2019
+ *      Author: Your name goes here!
+ */
+
+#ifndef MSR_MSRSENSORGROUP_H_
+#define MSR_MSRSENSORGROUP_H_
+
+#include "../../includes/SensorGroupTemplate.h"
+
+#include <map>
+#include <string>
+#include <cstdint>
+
+#include "MSRSensorBase.h"
+
+class MSRSensorGroup : public SensorGroupTemplate<MSRSensorBase> {
+public:
+	MSRSensorGroup(const std::string& name);
+	MSRSensorGroup(const MSRSensorGroup& other)=default;
+	virtual ~MSRSensorGroup();
+	MSRSensorGroup& operator=(const MSRSensorGroup& other)=default;
+
+	void start() override;
+	void stop() override;
+	void addCpu(unsigned int cpu);
+	void init(boost::asio::io_service& io) override;
+
+private:
+	void read() override;
+	void readAsync() override;
+
+	void program_fixed();
+	std::map<unsigned int,int> cpuToFd;
+	int32_t msr_read(uint64_t msr_number, uint64_t * value, unsigned int cpu);
+	int32_t msr_write(uint64_t msr_number, uint64_t value, unsigned int cpu);
+
+};
+
+#endif /* MSR_MSRSENSORGROUP_H_ */

src/sensors/msr/Types.h

+/*
+ * Types.h
+ *
+ *  Created on: 28.01.2019
+ *      Author: Carla Guillen
+ */
+
+#ifndef SRC_SENSORS_MSR_TYPES_H_
+#define SRC_SENSORS_MSR_TYPES_H_
+
+
+/*
+        MSR addreses from
+        "Intel 64 and IA-32 Architectures Software Developers Manual Volume 3B:
+        System Programming Guide, Part 2", Appendix A "PERFORMANCE-MONITORING EVENTS"
+*/
+
+#define INST_RETIRED_ANY_ADDR           (0x309)
+#define CPU_CLK_UNHALTED_THREAD_ADDR    (0x30A)
+#define CPU_CLK_UNHALTED_REF_ADDR       (0x30B)
+#define IA32_CR_PERF_GLOBAL_CTRL        (0x38F)
+#define IA32_CR_FIXED_CTR_CTRL          (0x38D)
+
+
+/* \brief Fixed Event Control Register format
+
+        According to
+        "Intel 64 and IA-32 Architectures Software Developers Manual Volume 3B:
+        System Programming Guide, Part 2", Figure 30-7. Layout of
+        IA32_FIXED_CTR_CTRL MSR Supporting Architectural Performance Monitoring Version 3
+*/
+struct FixedEventControlRegister
+{
+    union
+    {
+        struct
+        {
+            // CTR0
+            uint64_t os0 : 1;
+            uint64_t usr0 : 1;
+            uint64_t any_thread0 : 1;
+            uint64_t enable_pmi0 : 1;
+            // CTR1
+            uint64_t os1 : 1;
+            uint64_t usr1 : 1;
+            uint64_t any_thread1 : 1;
+            uint64_t enable_pmi1 : 1;
+            // CTR2
+            uint64_t os2 : 1;
+            uint64_t usr2 : 1;
+            uint64_t any_thread2 : 1;
+            uint64_t enable_pmi2 : 1;
+
+            uint64_t reserved1 : 52;
+        } fields;
+        uint64_t value;
+    };
+};
+
+#endif /* SRC_SENSORS_MSR_TYPES_H_ */

src/sensors/msr/appendToMakefile.txt

+Append to PLUGINS variable: msr
+
+Append at end of Makefile:
+
+libdcdbplugin_msr.$(LIBEXT): src/sensors/msr/MSRSensorGroup.o src/sensors/msr/MSRConfigurator.o
+	$(CXX) $(LIBFLAGS)$@ -o $@ $^ -L$(DCDBDEPLOYPATH)/lib/ -lboost_log -lboost_system
+	
+NOTE: Probably you will have to append further libraries to the linker for your plugin to compile

src/sensors/msr/msr.conf

+;comments in config files are indicated by a semicolon
+
+global {
+	mqttPrefix	/FF112233445566778899AABB
+	
+	;add here other global attributes for your plugin
+}
+
+template_group def1 {
+	;define template groups by appending "template_"
+	interval    1000
+	minValues   3
+	groupAtt	1234
+	;add other attributes your plugin requires for a sensor group
+	
+	;define sensors belonging to the group below
+	sensor temp1 {
+		sensorAtt	5678
+		;add other attributes your plugin requires for a sensor
+	}
+}
+
+single_sensor sens1 {
+	;if you want a group with only one sensor you can use a single_sensor
+	
+	default     temp1
+	mqttsuffix	0001
+	;add other attributes your plugin requires for a sensor
+}
+
+group g1 {
+	interval	1000
+	mqttprefix	01
+	default		def1
+	
+	;sensor temp1 is taken from def1 and does not need to be redefined
+	sensor gSens {
+		mqttsuffix	00
+	}
+}