Notice to GitKraken users: A vulnerability has been found in the SSH key generation of GitKraken versions 7.6.0 to 8.0.0 (https://www.gitkraken.com/blog/weak-ssh-key-fix). If you use GitKraken and have generated a SSH key using one of these versions, please remove it both from your local workstation and from your LRZ GitLab profile.

21.10.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit c98e5434 authored by Carla Guillen's avatar Carla Guillen
Browse files

Using doubles for most calculations

parent e9f50eaf
...@@ -36,6 +36,7 @@ ...@@ -36,6 +36,7 @@
#include <cstdint> #include <cstdint>
#include <memory> #include <memory>
#include <string> #include <string>
#include <numeric>
#include "../../../common/include/logging.h" #include "../../../common/include/logging.h"
#include "../../../common/include/sensorbase.h" #include "../../../common/include/sensorbase.h"
...@@ -77,39 +78,33 @@ void PerSystSqlOperator::compute(U_Ptr unit, qeJobData& jobData) { ...@@ -77,39 +78,33 @@ void PerSystSqlOperator::compute(U_Ptr unit, qeJobData& jobData) {
compute_internal(unit, _buffer); compute_internal(unit, _buffer);
} }
void PerSystSqlOperator::compute_internal(U_Ptr unit, vector<reading_t> buffer) { void PerSystSqlOperator::compute_internal(U_Ptr unit, vector<reading_t>& buffer) {
_quantileSensors.clear(); _quantileSensors.clear();
reading_t reading; reading_t reading;
AggregatorSensorBase::aggregationOps_t op; AggregatorSensorBase::aggregationOps_t op;
reading.timestamp = getTimestamp() - 10e9; reading.timestamp = getTimestamp() - 10e9;
std::vector<double> douBuffer;
punToDoubles(buffer, douBuffer);
// Performing the actual aggregation operation // Performing the actual aggregation operation
for(const auto& out : unit->getOutputs()) { for(const auto& out : unit->getOutputs()) {
op = out->getOperation(); op = out->getOperation();
if(op!=AggregatorSensorBase::QTL) { if(op!=AggregatorSensorBase::QTL) {
switch (op) { switch (op) {
case AggregatorSensorBase::SUM: case AggregatorSensorBase::SUM:
reading.value = computeSum(buffer); reading.value = punDoubleToLL(std::accumulate(douBuffer.begin(), douBuffer.end(), 0.0));
break; break;
case AggregatorSensorBase::AVG: case AggregatorSensorBase::AVG:
reading.value = computeAvg(buffer); reading.value = punDoubleToLL(std::accumulate(douBuffer.begin(), douBuffer.end(), 0.0)/douBuffer.size());
break;
case AggregatorSensorBase::MIN:
reading.value = computeMin(buffer);
break;
case AggregatorSensorBase::MAX:
reading.value = computeMax(buffer);
break;
case AggregatorSensorBase::STD:
reading.value = computeStd(buffer);
break; break;
case AggregatorSensorBase::OBS: case AggregatorSensorBase::OBS:
reading.value = computeObs(buffer); reading.value = computeObs(buffer);
break; break;
case AggregatorSensorBase::AVG_SEV: case AggregatorSensorBase::AVG_SEV:
reading.value = computeSeverityAverage(buffer); //ToDo an appropriate cast needed here reading.value = punDoubleToLL(computeSeverityAverage(douBuffer));
break; break;
default: default:
LOG(warning) << _name << ": Encountered unknown operation!"; LOG(warning) << _name << ": Operation " << op << " not supported!";
reading.value = 0; reading.value = 0;
break; break;
} }
...@@ -120,10 +115,10 @@ void PerSystSqlOperator::compute_internal(U_Ptr unit, vector<reading_t> buffer) ...@@ -120,10 +115,10 @@ void PerSystSqlOperator::compute_internal(U_Ptr unit, vector<reading_t> buffer)
} }
if(!_quantileSensors.empty()) { if(!_quantileSensors.empty()) {
vector<int64_t> quantiles; vector<double> quantiles;
computeEvenQuantiles(buffer, _number_of_even_quantiles, quantiles); computeEvenQuantiles(douBuffer, _number_of_even_quantiles, quantiles);
for(unsigned idx=0; idx<quantiles.size(); idx++) { for(unsigned idx=0; idx<quantiles.size(); idx++) {
reading.value = quantiles[idx]; reading.value = punDoubleToLL(quantiles[idx]);
_quantileSensors[idx]->storeReading(reading); _quantileSensors[idx]->storeReading(reading);
} }
} }
...@@ -133,10 +128,10 @@ void PerSystSqlOperator::compute(U_Ptr unit){ ...@@ -133,10 +128,10 @@ void PerSystSqlOperator::compute(U_Ptr unit){
//nothing here! //nothing here!
} }
float severity_formula1(float metric, float threshold, float exponent){ double severity_formula1(double metric, double threshold, double exponent){
float val = metric - threshold; double val = metric - threshold;
if (val > 0) { if (val > 0) {
float ret = (pow(val, exponent)); double ret = (pow(val, exponent));
if(ret > 1){ if(ret > 1){
return 1; return 1;
} }
...@@ -145,13 +140,13 @@ float severity_formula1(float metric, float threshold, float exponent){ ...@@ -145,13 +140,13 @@ float severity_formula1(float metric, float threshold, float exponent){
return 0; return 0;
} }
float severity_formula2(float metric, float threshold, float exponent){ double severity_formula2(double metric, double threshold, double exponent){
if(!threshold){ if(!threshold){
return -1; return -1;
} }
float val = metric / threshold - 1; double val = metric / threshold - 1;
if (val > 0) { if (val > 0) {
float ret= (pow(val, exponent)); double ret= (pow(val, exponent));
if(ret > 1){ if(ret > 1){
return 1; return 1;
} }
...@@ -160,13 +155,13 @@ float severity_formula2(float metric, float threshold, float exponent){ ...@@ -160,13 +155,13 @@ float severity_formula2(float metric, float threshold, float exponent){
return 0; return 0;
} }
float severity_formula3(float metric, float threshold, float exponent){ double severity_formula3(double metric, double threshold, double exponent){
if (!threshold) { if (!threshold) {
return -1; return -1;
} }
float val = metric / threshold; double val = metric / threshold;
if (val > 0) { if (val > 0) {
float ret= (1 - pow(val, exponent)); double ret= (1 - pow(val, exponent));
if(ret > 1 ){ if(ret > 1 ){
return 1; return 1;
} }
...@@ -178,9 +173,9 @@ float severity_formula3(float metric, float threshold, float exponent){ ...@@ -178,9 +173,9 @@ float severity_formula3(float metric, float threshold, float exponent){
return 0; return 0;
} }
float severity_memory(float metric, float threshold, float max_memory){ double severity_memory(double metric, double threshold, double max_memory){
float denominator = max_memory - threshold; double denominator = max_memory - threshold;
float severity = -1; double severity = -1;
if(denominator){ if(denominator){
severity = metric - threshold/(max_memory - threshold); severity = metric - threshold/(max_memory - threshold);
if(severity > 1) { if(severity > 1) {
...@@ -192,40 +187,90 @@ float severity_memory(float metric, float threshold, float max_memory){ ...@@ -192,40 +187,90 @@ float severity_memory(float metric, float threshold, float max_memory){
return severity; return severity;
} }
float PerSystSqlOperator::computeSeverityAverage(vector<reading_t> & buffer){ double PerSystSqlOperator::computeSeverityAverage(std::vector<double> & buffer){
std::vector<float> severities; std::vector<double> severities;
switch( _severity_formula ) { switch( _severity_formula ) {
case (FORMULA1): case (FORMULA1):
for(auto reading: buffer){ for(auto val : buffer){
auto severity = severity_formula1(reading.value, _severity_threshold, _severity_exponent); auto severity = severity_formula1(val, _severity_threshold, _severity_exponent);
severities.push_back(severity); severities.push_back(severity);
} }
break; break;
case (FORMULA2): case (FORMULA2):
for(auto reading: buffer){ for(auto val: buffer){
auto severity = severity_formula2(reading.value, _severity_threshold, _severity_exponent); auto severity = severity_formula2(val, _severity_threshold, _severity_exponent);
severities.push_back(severity); severities.push_back(severity);
} }
break; break;
case (FORMULA3): case (FORMULA3):
for(auto reading: buffer){ for(auto val: buffer){
auto severity = severity_formula3(reading.value, _severity_threshold, _severity_exponent); auto severity = severity_formula3(val, _severity_threshold, _severity_exponent);
severities.push_back(severity); severities.push_back(severity);
} }
break; break;
case (MEMORY_FORMULA): case (MEMORY_FORMULA):
for(auto reading: buffer){ for(auto val: buffer){
auto severity = severity_memory(reading.value, _severity_threshold, _severity_max_memory); auto severity = severity_memory(val, _severity_threshold, _severity_max_memory);
severities.push_back(severity); severities.push_back(severity);
} }
break; break;
case (NOFORMULA): case (NOFORMULA):
for(auto val: buffer){
severities.push_back(severity_noformula());
}
default: default:
return 0.0; return 0.0;
break; break;
} }
if (severities.size()){ if (severities.size()){
return (std::accumulate(severities.begin(),severities.end(), 0.0f) / severities.size()); return (std::accumulate(severities.begin(),severities.end(), 0.0) / severities.size());
} }
return 0.0; return 0.0;
} }
void punToDoubles(std::vector<reading_t> & buffer, std::vector<double> & outDoubleVec){
for(auto & reading: buffer){
outDoubleVec.push_back(punLLToDouble(reading.value));
}
}
double punLLToDouble(long long value){
double * returnval;
returnval = (double *)(&value);
return *returnval;
}
long long punDoubleToLL(double value){
long long * returnval;
returnval = (long long *)(&value);
return *returnval;
}
void computeEvenQuantiles(std::vector<double> &data, const unsigned int NUMBER_QUANTILES, std::vector<double> &quantiles) {
if (data.empty() || NUMBER_QUANTILES == 0) {
return;
}
std::sort(data.begin(), data.end());
int elementNumber = data.size();
quantiles.resize(NUMBER_QUANTILES + 1); //+min
double factor = elementNumber/static_cast<double>(NUMBER_QUANTILES);
quantiles[0] = data[0]; //minimum
quantiles[NUMBER_QUANTILES] = data[data.size() - 1]; //maximum
for (unsigned int i = 1; i < NUMBER_QUANTILES; i++) {
if (elementNumber > 1) {
int idx = static_cast<int>(std::floor(i * factor));
if(idx == 0){
quantiles[i] = data[0];
} else {
double rest = (i * factor) - idx;
quantiles[i] = data[idx - 1] + rest * (data[idx] - data[idx - 1]); //ToDo scaling factor??
}
} else { //optimization, we don't need to calculate all the quantiles
quantiles[i] = data[0];
}
}
}
...@@ -60,7 +60,7 @@ public: ...@@ -60,7 +60,7 @@ public:
_quantileSensors.push_back(qSensor); _quantileSensors.push_back(qSensor);
} }
void setSeverityExponent(float severityExponent) { void setSeverityExponent(double severityExponent) {
_severity_exponent = severityExponent; _severity_exponent = severityExponent;
} }
...@@ -68,33 +68,39 @@ public: ...@@ -68,33 +68,39 @@ public:
_severity_formula = severityFormula; _severity_formula = severityFormula;
} }
void setSeverityMaxMemory(float severityMaxMemory) { void setSeverityMaxMemory(double severityMaxMemory) {
_severity_max_memory = severityMaxMemory; _severity_max_memory = severityMaxMemory;
} }
void setSeverityThreshold(float severityThreshold) { void setSeverityThreshold(double severityThreshold) {
_severity_threshold = severityThreshold; _severity_threshold = severityThreshold;
} }
private: private:
std::vector<reading_t> _buffer; std::vector<reading_t> _buffer;
std::vector<double> _severities;
protected: protected:
virtual void compute(U_Ptr unit) override; virtual void compute(U_Ptr unit) override;
void compute_internal(U_Ptr unit, vector<reading_t> buffer); void compute_internal(U_Ptr unit, vector<reading_t>& buffer);
float computeSeverityAverage(vector<reading_t> & buffer); double computeSeverityAverage(vector<double> & buffer);
std::vector<AggregatorSBPtr> _quantileSensors; std::vector<AggregatorSBPtr> _quantileSensors;
unsigned int _number_of_even_quantiles; unsigned int _number_of_even_quantiles;
Formula _severity_formula; Formula _severity_formula;
float _severity_threshold; double _severity_threshold;
float _severity_exponent; double _severity_exponent;
float _severity_max_memory; double _severity_max_memory;
}; };
float severity_formula1(float metric, float threshold, float exponent); double severity_formula1(double metric, double threshold, double exponent);
float severity_formula2(float metric, float threshold, float exponent); double severity_formula2(double metric, double threshold, double exponent);
float severity_formula3(float metric, float threshold, float exponent); double severity_formula3(double metric, double threshold, double exponent);
float severity_memory(float metric, float threshold, float max_memory); double severity_memory(double metric, double threshold, double max_memory);
constexpr float severity_noformula(){return 0;} //No severity constexpr double severity_noformula(){return 0.0;} //No severity
void punToDoubles(std::vector<reading_t> & buffer, std::vector<double> & outDoubleVec);
void computeEvenQuantiles(std::vector<double> &data, const unsigned int NUMBER_QUANTILES, std::vector<double> &quantiles);
double punLLToDouble(long long value);
long long punDoubleToLL(double value);
#endif /* ANALYTICS_OPERATORS_PERSYSTSQL_PERSYSTSQLOPERATOR_H_ */ #endif /* ANALYTICS_OPERATORS_PERSYSTSQL_PERSYSTSQLOPERATOR_H_ */
...@@ -28,39 +28,54 @@ ...@@ -28,39 +28,54 @@
#include "SKXPMUMetrics.h" #include "SKXPMUMetrics.h"
double punLLToDouble(long long value){
double * returnval;
returnval = (double *)(&value);
return *returnval;
}
long long punDoubleToLL(double value){
long long * returnval;
returnval = (long long *)(&value);
return *returnval;
}
bool calculateFlops(reading_t &scalarDB, reading_t & scalarSP, bool calculateFlopsPerSec(reading_t &scalarDB, reading_t & scalarSP,
reading_t & packedDP128, reading_t & packedSP128, reading_t & packedDP128, reading_t & packedSP128,
reading_t & packedDP256, reading_t & packedSP256, reading_t & packedDP256, reading_t & packedSP256,
reading_t & packedDP512, reading_t & packedSP512, reading_t & result) { reading_t & packedDP512, reading_t & packedSP512, reading_t & result, double measuring_interval_s) {
result.value = (packedDP128.value * 2 + packedSP128.value * 4 if(!measuring_interval_s) return false;
result.value = punDoubleToLL(packedDP128.value * 2 + packedSP128.value * 4
+ packedDP256.value * 4 + packedSP256.value * 8 + packedDP256.value * 4 + packedSP256.value * 8
+ packedDP512.value * 8 + packedSP512.value * 16 + scalarDB.value + packedDP512.value * 8 + packedSP512.value * 16 + scalarDB.value
+ scalarSP.value); + scalarSP.value)/measuring_interval_s;
return getTimestampFromReadings(result.timestamp, scalarDB, scalarSP, packedDP128, packedSP128, packedDP256, packedSP256, packedDP512, packedSP512); return getTimestampFromReadings(result.timestamp, scalarDB, scalarSP, packedDP128, packedSP128, packedDP256, packedSP256, packedDP512, packedSP512);
} }
bool calculatePackedFlops(reading_t & packedDP128, reading_t & packedSP128, bool calculatePackedFlopsPerSec(reading_t & packedDP128, reading_t & packedSP128,
reading_t & packedDP256, reading_t & packedSP256, reading_t & packedDP256, reading_t & packedSP256,
reading_t & packedDP512, reading_t & packedSP512, reading_t & packedDP512, reading_t & packedSP512,
reading_t & result) { reading_t & result, double measuring_interval_s) {
result.value = (packedDP128.value * 2 + packedSP128.value * 4 result.value = punDoubleToLL((packedDP128.value * 2 + packedSP128.value * 4
+ packedDP256.value * 4 + packedSP256.value * 8 + packedDP256.value * 4 + packedSP256.value * 8
+ packedDP512.value * 8 + packedSP512.value * 16); + packedDP512.value * 8 + packedSP512.value * 16)/measuring_interval_s);
return getTimestampFromReadings(result.timestamp, packedDP128, packedSP128, packedDP256, packedSP256, packedDP512, packedSP512); return getTimestampFromReadings(result.timestamp, packedDP128, packedSP128, packedDP256, packedSP256, packedDP512, packedSP512);
} }
bool calculateVectorizationRatio(reading_t & scalarDB, reading_t & scalarSP, bool calculateVectorizationRatio(reading_t & scalarDB, reading_t & scalarSP,
reading_t & packedDP128, reading_t & packedSP128, reading_t & packedDP128, reading_t & packedSP128,
reading_t & packedDP256, reading_t & packedSP256, reading_t & packedDP256, reading_t & packedSP256,
reading_t & packedDP512, reading_t & packedSP512, unsigned int scaling_factor, reading_t & result) { reading_t & packedDP512, reading_t & packedSP512, reading_t & result) {
float denominator = static_cast<float>(scalarDB.value + scalarSP.value + double denominator = static_cast<double>(scalarDB.value + scalarSP.value +
packedDP128.value + packedSP128.value + packedDP128.value + packedSP128.value +
packedDP256.value + packedSP256.value + packedDP256.value + packedSP256.value +
packedDP512.value + packedSP512.value); packedDP512.value + packedSP512.value);
if(denominator > 0 ){ if(denominator > 0 ){
result.value = (packedDP128.value + packedSP128.value + packedDP256.value result.value = punDoubleToLL((packedDP128.value + packedSP128.value + packedDP256.value
+ packedSP256.value + packedDP256.value + packedSP512.value + packedDP512.value)/denominator; + packedSP256.value + packedDP256.value + packedSP512.value + packedDP512.value)/denominator);
return getTimestampFromReadings(result.timestamp, scalarDB, scalarSP, return getTimestampFromReadings(result.timestamp, scalarDB, scalarSP,
packedDP128, packedSP128, packedDP256, packedSP256, packedDP512, packedDP128, packedSP128, packedDP256, packedSP256, packedDP512,
packedSP512); packedSP512);
...@@ -70,13 +85,13 @@ bool calculateVectorizationRatio(reading_t & scalarDB, reading_t & scalarSP, ...@@ -70,13 +85,13 @@ bool calculateVectorizationRatio(reading_t & scalarDB, reading_t & scalarSP,
bool calculateAVX512FlopsToVectorizedRatio(reading_t & packedDP128, reading_t & packedSP128, bool calculateAVX512FlopsToVectorizedRatio(reading_t & packedDP128, reading_t & packedSP128,
reading_t & packedDP256, reading_t & packedSP256, reading_t & packedDP256, reading_t & packedSP256,
reading_t & packedDP512, reading_t & packedSP512, unsigned int scaling_factor, reading_t & packedDP512, reading_t & packedSP512,
reading_t & result){ reading_t & result){
float denominator = static_cast<float>(packedDP128.value + packedSP128.value + double denominator = static_cast<double>(packedDP128.value + packedSP128.value +
packedDP256.value + packedSP256.value + packedDP256.value + packedSP256.value +
packedDP512.value + packedSP512.value); packedDP512.value + packedSP512.value);
if(denominator > 0){ if(denominator > 0){
result.value = (packedDP512.value * 8 + packedSP512.value * 16)/denominator; result.value = punDoubleToLL((packedDP512.value * 8 + packedSP512.value * 16)/denominator);
return getTimestampFromReadings(result.timestamp, packedDP128, packedSP128, packedDP256, packedSP256, packedDP512, packedSP512); return getTimestampFromReadings(result.timestamp, packedDP128, packedSP128, packedDP256, packedSP256, packedDP512, packedSP512);
} }
return false; return false;
...@@ -86,13 +101,13 @@ bool calculateAVX512FlopsToVectorizedRatio(reading_t & packedDP128, reading_t & ...@@ -86,13 +101,13 @@ bool calculateAVX512FlopsToVectorizedRatio(reading_t & packedDP128, reading_t &
bool calculateSP_TO_TOTAL_RATIO(reading_t &scalarDB, reading_t & scalarSP, bool calculateSP_TO_TOTAL_RATIO(reading_t &scalarDB, reading_t & scalarSP,
reading_t & packedDP128, reading_t & packedSP128, reading_t & packedDP128, reading_t & packedSP128,
reading_t & packedDP256, reading_t & packedSP256, reading_t & packedDP256, reading_t & packedSP256,
reading_t & packedDP512, reading_t & packedSP512, unsigned int scaling_factor, reading_t & packedDP512, reading_t & packedSP512,
reading_t & result){ reading_t & result){
auto single_precision = packedSP128.value * 4 + packedSP256.value * 8 auto single_precision = packedSP128.value * 4 + packedSP256.value * 8
+ packedSP512.value * 16 + scalarSP.value; + packedSP512.value * 16 + scalarSP.value;
float total = single_precision = packedDP128.value * 2 + packedDP256.value * 4 + packedDP512.value * 8 + scalarDB.value; double total = single_precision = packedDP128.value * 2 + packedDP256.value * 4 + packedDP512.value * 8 + scalarDB.value;
if(total > 0){ if(total > 0){
result.value = single_precision/total; result.value = punDoubleToLL(single_precision/total);
return getTimestampFromReadings(result.timestamp, scalarDB, scalarSP, return getTimestampFromReadings(result.timestamp, scalarDB, scalarSP,
packedDP128, packedSP128, packedDP256, packedSP256, packedDP512, packedDP128, packedSP128, packedDP256, packedSP256, packedDP512,
packedSP512); packedSP512);
...@@ -101,16 +116,16 @@ bool calculateSP_TO_TOTAL_RATIO(reading_t &scalarDB, reading_t & scalarSP, ...@@ -101,16 +116,16 @@ bool calculateSP_TO_TOTAL_RATIO(reading_t &scalarDB, reading_t & scalarSP,
} }
bool calculateL3HitToL3MissRatio(reading_t & l3_misses, reading_t& l3_load_hits, bool calculateL3HitToL3MissRatio(reading_t & l3_misses, reading_t& l3_load_hits,
reading_t & l3_load_misses, reading_t & result, unsigned int scaling_factor){ reading_t & l3_load_misses, reading_t & result){
float denominator = static_cast<float>(l3_load_hits.value + l3_load_misses.value); double denominator = static_cast<double>(l3_load_hits.value + l3_load_misses.value);
if(denominator > 0){ if(denominator > 0){
result.value = (l3_misses.value/denominator) * scaling_factor; result.value = punDoubleToLL(l3_misses.value/denominator);
return getTimestampFromReadings(result.timestamp, l3_misses, l3_load_hits, l3_load_misses); return getTimestampFromReadings(result.timestamp, l3_misses, l3_load_hits, l3_load_misses);
} }
return false; return false;
} }
bool calculateMemoryBandwidth(std::vector<reading_t> membw_counts, reading_t & result) { bool calculateMemoryBandwidth(std::vector<reading_t> membw_counts, reading_t & result, double measuring_interval_s ) {
result.value = 0; result.value = 0;
const unsigned int SIZEPERACCESS = 64; const unsigned int SIZEPERACCESS = 64;
bool ret_val = false; bool ret_val = false;
...@@ -121,6 +136,7 @@ bool calculateMemoryBandwidth(std::vector<reading_t> membw_counts, reading_t & r ...@@ -121,6 +136,7 @@ bool calculateMemoryBandwidth(std::vector<reading_t> membw_counts, reading_t & r
ret_val = true; ret_val = true;
} }
} }
result.value = punDoubleToLL(result.value/measuring_interval_s);
return ret_val; return ret_val;
} }
...@@ -128,19 +144,18 @@ bool calculateMemoryBandwidth(std::vector<reading_t> membw_counts, reading_t & r ...@@ -128,19 +144,18 @@ bool calculateMemoryBandwidth(std::vector<reading_t> membw_counts, reading_t & r
/** /**
* For CPI, LoadsToStore, Branch rate, miss branch ratio, etc.. * For CPI, LoadsToStore, Branch rate, miss branch ratio, etc..
*/ */
bool calculateMetricRatio(reading_t & dividend, reading_t & divisor, bool calculateMetricRatio(reading_t & dividend, reading_t & divisor, reading_t & result) {
unsigned int scaling_factor, reading_t & result) {
if(divisor.value > 0){ if(divisor.value > 0){
result.value = (dividend.value / static_cast<float>(divisor.value))*scaling_factor; result.value = punDoubleToLL(dividend.value / static_cast<double>(divisor.value));
return getTimestampFromReadings(result.timestamp, dividend, divisor);