MSRSensorGroup.cpp 6.55 KB
Newer Older
1
2
3
4
/*
 * MSRSensorGroup.cpp
 *
 *  Created on: 28.01.2019
5
 *      Author: Carla Guillen
6
7
8
9
10
11
12
13
14
15
16
17
18
 */

#include "MSRSensorGroup.h"

#include <boost/log/core/record.hpp>
#include <boost/log/sources/record_ostream.hpp>
#include <boost/log/trivial.hpp>
#include <boost/log/utility/formatting_ostream.hpp>
#include <boost/parameter/keyword.hpp>
#include <fcntl.h>
#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
Micha Mueller's avatar
Micha Mueller committed
19
#include <bitset>
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
#include <atomic>
#include <exception>
#include <utility>

#include "../../includes/Logging.h"
#include "../../includes/SensorBase.h"
#include "timestamp.h"
#include "Types.h"
#include <sstream>
#include <iomanip>


MSRSensorGroup::MSRSensorGroup(const std::string& name) :
	SensorGroupTemplate(name) {
}

MSRSensorGroup::~MSRSensorGroup() {
}

void MSRSensorGroup::start() {
	if (_keepRunning) {
		//we have been started already
		LOG(info) << "Sensorgroup " << _groupName << " already running.";
		return;
	}

Micha Mueller's avatar
Micha Mueller committed
46
47
48
49
50
	if (!program_fixed()) {
        LOG(error) << "MSR fixed counters already in use! Can not start counters";
        return;
    }

51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
	for (auto &kv : cpuToFd) {
		int cpu = kv.first;
		char * path = new char[200];
		snprintf(path, 200, "/dev/cpu/%d/msr", cpu);
		int handle = open(path, O_RDWR);
		if (handle < 0) {   // try msr_safe
			snprintf(path, 200, "/dev/cpu/%d/msr_safe", cpu);
			handle = open(path, O_RDWR);
		}
		if (handle < 0){
			LOG(error) << "Can't open msr device " << path;
			delete [] path; // TODO do this with RAII
			continue;
		}
		delete [] path; //TODO do this with RAII
		cpuToFd[cpu] = handle;
	}

	_keepRunning = 1;
	_pendingTasks++;
	_timer->async_wait(std::bind(&MSRSensorGroup::readAsync, this));
	LOG(info) << "Sensorgroup " << _groupName << " started.";
}

void MSRSensorGroup::stop() {
Micha Mueller's avatar
Micha Mueller committed
76
77
    int temp = _keepRunning;

78
79
	_keepRunning = 0;
	
Micha Mueller's avatar
Micha Mueller committed
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
	//only close this stuff if we have been properly started before
    if (temp) {
        //close file descriptors
        for (auto &kv: cpuToFd) {
            close(kv.second);
            kv.second = -1;
        }

        //reset counters
        for (auto& kv : cpuToFd) {
            //disable counters before writing anything else
            msr_write(IA32_CR_PERF_GLOBAL_CTRL, 0, kv.first);
            //reset all bits to 0
            msr_write(IA32_CR_FIXED_CTR_CTRL, 0, kv.first);
            //set all global enable bits to 1
            uint64_t value = (1ULL << 0) + (1ULL << 1) + (1ULL << 2) + (1ULL << 3) + (1ULL << 32) + (1ULL << 33) + (1ULL << 34);
            msr_write(IA32_CR_PERF_GLOBAL_CTRL, value, kv.first);
        }
    }

100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
	LOG(info) << "Sensorgroup " << _groupName << " stopped.";
}

void MSRSensorGroup::read() {
	ureading_t reading;
	reading.timestamp = getTimestamp();

	try {
		for(auto s : _sensors) {
			msr_read(s->getMetric(), &reading.value, s->getCpu());
			s->storeReading(reading);
#ifdef DEBUG
			LOG(debug) << _groupName << "::" << s->getName() << " raw reading: \"" << reading.value << "\"";
#endif
		}
	} catch (const std::exception& e) {
		LOG(error) << "Sensorgroup" << _groupName << " could not read value: " << e.what();
	}
}

void MSRSensorGroup::readAsync() {
	uint64_t now = getTimestamp();
	read();
	if (_timer && _keepRunning) {
		_timer->expires_at(timestamp2ptime(nextReadingTime()));
		_pendingTasks++;
		_timer->async_wait(std::bind(&MSRSensorGroup::readAsync, this));
	}
	_pendingTasks--;
}

int32_t MSRSensorGroup::msr_read(uint64_t msr_number, uint64_t * value, unsigned int cpu){
	return pread(cpuToFd[cpu], (void *) value, sizeof(uint64_t), msr_number);
}

int32_t MSRSensorGroup::msr_write(uint64_t msr_number, uint64_t value, unsigned int cpu){
	return pwrite(cpuToFd[cpu], (const void *) &value, sizeof(uint64_t), msr_number);
}

Micha Mueller's avatar
Micha Mueller committed
139
140
141
142
143
144
145
146
147
/**
 * Program the fixed MSR as required for this plugin.
 *
 * @return  True if counters programmed successfully, false otherwise, e.g.
 *          because the counters are already in use.
 */
bool MSRSensorGroup::program_fixed(){
    unsigned failedCpu = 9999;

148
149
150
	for (auto &kv : cpuToFd) {
		// program core counters

Micha Mueller's avatar
Micha Mueller committed
151
152
	    //we do not want to interrupt other services already doing measurements with MSRs
        //therefore check if any fixed counter is currently enabled
153
		struct FixedEventControlRegister ctrl_reg;
Micha Mueller's avatar
Micha Mueller committed
154
155
        std::bitset<64> cur_reg(ctrl_reg.value);

156
157
		msr_read(IA32_CR_FIXED_CTR_CTRL, &ctrl_reg.value, kv.first);

Micha Mueller's avatar
Micha Mueller committed
158
159
160
161
162
163
164
165
166
167
		if (cur_reg[0] || cur_reg[1] || cur_reg[4] || cur_reg[5] || cur_reg[8] || cur_reg[9]) {
		  //at least one is enabled. Do not overwrite them.
		  //Instead we revert the already programmed counters and return
		  failedCpu = kv.first;
		  break;
		}

		// disable counters while programming
        msr_write(IA32_CR_PERF_GLOBAL_CTRL, 0, kv.first);

168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
		ctrl_reg.fields.os0 = 1;
		ctrl_reg.fields.usr0 = 1;
		ctrl_reg.fields.any_thread0 = 0;
		ctrl_reg.fields.enable_pmi0 = 0;

		ctrl_reg.fields.os1 = 1;
		ctrl_reg.fields.usr1 = 1;
		ctrl_reg.fields.any_thread1 = 0;
		ctrl_reg.fields.enable_pmi1 = 0;

		ctrl_reg.fields.os2 = 1;
		ctrl_reg.fields.usr2 = 1;
		ctrl_reg.fields.any_thread2 = 0;
		ctrl_reg.fields.enable_pmi2 = 0;

		ctrl_reg.fields.reserved1 = 0;
184

185
186
187
188
189
190
191
192
		msr_write(IA32_CR_FIXED_CTR_CTRL, ctrl_reg.value, kv.first);

		// start counting, enable 3 fixed counters
		//uint64 value = (1ULL << 0) + (1ULL << 1) + (1ULL << 2) + (1ULL << 3) + (1ULL << 32) + (1ULL << 33) + (1ULL << 34);
		uint64_t value = (1ULL << 32) + (1ULL << 33) + (1ULL << 34);
		msr_write(IA32_CR_PERF_GLOBAL_CTRL, value, kv.first);
	}

Micha Mueller's avatar
Micha Mueller committed
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
	//One of the fixed counters was already enabled.
	//Reset all counters which were already programmed before returning.
	if (failedCpu != 9999) {
	  for (auto& kv : cpuToFd) {
	    if (kv.first != failedCpu) {
	      //disable counters before writing anything else
          msr_write(IA32_CR_PERF_GLOBAL_CTRL, 0, kv.first);
          //reset all bits to 0
          msr_write(IA32_CR_FIXED_CTR_CTRL, 0, kv.first);
          //set all global enable bits to 1
          uint64_t value = (1ULL << 0) + (1ULL << 1) + (1ULL << 2) + (1ULL << 3) + (1ULL << 32) + (1ULL << 33) + (1ULL << 34);
          msr_write(IA32_CR_PERF_GLOBAL_CTRL, value, kv.first);
	    } else {
	      return false;
	    }
	  }
	}

	return true;
212
213
214
215
216
}

void MSRSensorGroup::addCpu(unsigned int cpu){
	cpuToFd[cpu] = -1; /* -1 because no file descriptor has been assigned yet. */
}
217

218
219
220
221
222
223
224
225
std::vector<unsigned> MSRSensorGroup::getCpus() {
  std::vector<unsigned> cpus;
  for(auto kv : cpuToFd) {
    cpus.push_back(kv.first);
  }
  return cpus;
}

226
227
228
229
230
231
232
233
234
235
void MSRSensorGroup::printConfig(LOG_LEVEL ll) {
    std::stringstream ss;
    const char* separator = "";
    for (auto &kv : cpuToFd) {
      ss << separator << kv.first;
      separator = ", ";
    }

    LOG_VAR(ll) << "   CPUs:  " << ss.str();
}