Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 62e58e94 authored by Carla Guillen's avatar Carla Guillen
Browse files

Fixing issue with connection details and template for the persystsql operator....

Fixing issue with connection details and template for the persystsql operator. Updating examples of config files.
parent e8979586
template_persystsql def1 { template_persystsql def1 {
interval 1000 ;every 2,5 minutes
minValues 3 interval 150000
duplicate false minValues 1
streaming true duplicate false
relaxed true streaming true
relaxed true
delay 2000
backend mariadb
mariadb_host some.server.lrz.de
mariadb_user me_the_user
mariadb_password secret
mariadb_database_name hpctest
mariadb_port 3306
mariadb_rotation EVERY_MONTH
number_quantiles 10
} }
persystsql cpi_p { persystsql cpi {
default def1 default def1
number_quantiles 10
topic_quantiles property_id 50
severity_threshold 0.25
severity_threshold 1.5 severity_exponent 0.1
severity_exponent 2 severity_formula FORMULA1
severity_formula formula1
severity_max_memory 100000000
input { input {
;/.../computenode/cpux/cpi sensor "<bottomup>CPI"
sensor "<bottomup 1>cpi"
} }
output { globalOutput {
; In this case "bottomup 1" is the sensor tree level associated to compute nodes ; In this case "bottomup 1" is the sensor tree level associated to compute nodes
; /slurmjobid/cpi/avg ; /slurmjobid/cpi.avg
sensor "<bottomup 1>cpi.avg" { sensor "<bottomup 1>CPI.avg" {
mqttsuffix /cpi.avg mqttsuffix /CPI.avg
operation average operation average
} }
sensor "<bottomup 1>cpi.numobs" { sensor "<bottomup 1>CPI.numobs" {
mqttsuffix /cpi.numobs mqttsuffix /CPI.numobs
operation numobs operation numobs
} }
; sensor "<bottomup 1>cpi.avgseverity" { sensor "<bottomup 1>CPI.avgseverity" {
; mqttsuffix /cpi.avgseverity mqttsuffix /CPI.avgseverity
; operation average_severity operation average_severity
; } }
; sensor "<bottomup 1>cpi.quantile" { sensor "<bottomup 1>CPI.quantile" {
; mqttsuffix /cpi.quantile mqttsuffix /CPI.quantile
; operation deciles operation deciles
; } }
} }
} }
persystsql frequency_p { persystsql frequency {
default def1 default def1
number_quantiles 10
input { property_id 51
severity_threshold 2700
severity_exponent 0.1
severity_formula FORMULA1
sensor "<bottomup 1>frequency" input {
sensor "<bottomup>FREQUENCY"
} }
output { globalOutput {
sensor "<bottomup 1>FREQUENCY.avg" {
; In this case "bottomup 1" is the sensor tree level associated to compute nodes mqttsuffix /FREQUENCY.avg
sensor "<bottomup 1>frequency.avg" {
mqttsuffix /frequency.avg
operation average operation average
} }
sensor "<bottomup 1>frequency.numobs" { sensor "<bottomup 1>FREQUENCY.numobs" {
mqttsuffix /frequency.numobs mqttsuffix /FREQUENCY.numobs
operation numobs operation numobs
} }
sensor "<bottomup 1>FREQUENCY.avgseverity" {
mqttsuffix /FREQUENCY.avgseverity
operation average_severity
}
sensor "<bottomup 1>FREQUENCY.quantile" {
mqttsuffix /FREQUENCY.quantile
operation deciles
}
}
}
persystsql membandwt {
default def1
property_id 73
severity_threshold 20000
severity_exponent 1
severity_formula FORMULA2
input {
sensor "<bottomup 1>MEMORY_BANDWIDTH"
}
globalOutput {
sensor "<bottomup 1>MEMORY_BANDWIDTH.avg" {
mqttsuffix /MEMORY_BANDWIDTH.avg
operation average
}
sensor "<bottomup 1>MEMORY_BANDWIDTH.numbobs" {
mqttsuffix /MEMORY_BANDWIDTH.numobs
operation numobs
}
sensor "<bottomup 1>MEMORY_BANDWIDTH.avgseverity" {
mqttsuffix /MEMORY_BANDWIDTH.avgseverity
operation average_severity
}
sensor "<bottomup 1>MEMORY_BANDWIDTH.quantile" {
mqttsuffix /MEMORY_BANDWIDTH.quantile
operation deciles
}
}
}
persystsql instr_per_second {
default def1
property_id 52
severity_threshold 2000000000
severity_exponent 1
severity_formula FORMULA2
input {
sensor "<bottomup>INSTRUCTIONS_PER_SECOND"
}
globalOutput {
sensor "<bottomup 1>INSTRUCTIONS_PER_SECOND.avg" {
mqttsuffix /INSTRUCTIONS_PER_SECOND.avg
operation average
}
sensor "<bottomup 1>INSTRUCTIONS_PER_SECOND.numobs" {
mqttsuffix /INSTRUCTIONS_PER_SECOND.numobs
operation numobs
}
sensor "<bottomup 1>INSTRUCTIONS_PER_SECOND.avgseverity" {
mqttsuffix /INSTRUCTIONS_PER_SECOND.avgseverity
operation average_severity
}
sensor "<bottomup 1>INSTRUCTIONS_PER_SECOND.quantile" {
mqttsuffix /INSTRUCTIONS_PER_SECOND.quantile
operation deciles
}
} }
} }
global {
mqttPrefix /test
}
template_supermucngperf def1 { template_supermucngperf def1 {
interval 10000 ; every 2.5 min
minValues 3 interval 150000
duplicate false minValues 1
streaming true duplicate false
streaming true
} }
supermucngperf cpi { supermucngperf msr_derived {
default def1 default def1
measurement_interval 10000 go_back_ms 10000
measuring_interval 10000
input { input {
sensor "<bottomup>clocks"{ sensor "<bottomup, filter cpu([0-9]|[1-3][0-9]|4[0-7])>CLOCKS"
position 0 sensor "<bottomup, filter cpu([0-9]|[1-3][0-9]|4[0-7])>CLOCKS_REF"
} sensor "<bottomup, filter cpu([0-9]|[1-3][0-9]|4[0-7])>INSTRUCTIONS"
sensor "<bottomup>ref_clocks"{
position 1
}
sensor "<bottomup>instructions" {
position 2
}
} }
output { output {
sensor "<bottomup, filter /login08/cpu>cpi" { sensor "<bottomup, filter /login08/cpu>CPI" {
mqttsuffix /cpi mqttsuffix /CPI
position 3 metadata {
scaling_factor 100 scale 0.00001
}
} }
sensor "<bottomup, filter /login08/cpu>frequency" { sensor "<bottomup, filter /login08/cpu>FREQUENCY" {
mqttsuffix /frequency mqttsuffix /FREQUENCY
position 4 metadata {
scaling_factor 1 scale 0.0001
}
} }
sensor "<bottomup, filter /login08/cpu>INSTRUCTIONS_PER_SECOND" {
mqttsuffix /INSTRUCTIONS_PER_SECOND
metadata {
scale 0.01
}
}
} }
} }
...@@ -69,14 +69,6 @@ void PerSystSqlConfigurator::sensorBase(AggregatorSensorBase& s, CFG_VAL config) ...@@ -69,14 +69,6 @@ void PerSystSqlConfigurator::sensorBase(AggregatorSensorBase& s, CFG_VAL config)
} }
void PerSystSqlConfigurator::operatorAttributes(PerSystSqlOperator& op, CFG_VAL config) { void PerSystSqlConfigurator::operatorAttributes(PerSystSqlOperator& op, CFG_VAL config) {
std::string host;
std::string user;
std::string password;
std::string database_name;
int port = -1;
MariaDB::Rotation_t rotation = MariaDB::EVERY_MONTH;
PerSystSqlOperator::Backend_t backend = PerSystSqlOperator::DEFAULT;
unsigned int every_x_days = 0;
BOOST_FOREACH(boost::property_tree::iptree::value_type &val, config) { BOOST_FOREACH(boost::property_tree::iptree::value_type &val, config) {
if (boost::iequals(val.first, "number_quantiles")) { if (boost::iequals(val.first, "number_quantiles")) {
unsigned int num_quantiles = std::stoul(val.second.data()); unsigned int num_quantiles = std::stoul(val.second.data());
...@@ -106,39 +98,36 @@ void PerSystSqlConfigurator::operatorAttributes(PerSystSqlOperator& op, CFG_VAL ...@@ -106,39 +98,36 @@ void PerSystSqlConfigurator::operatorAttributes(PerSystSqlOperator& op, CFG_VAL
op.setSeverityMaxMemory(max_memory); op.setSeverityMaxMemory(max_memory);
} else if (boost::iequals(val.first, "backend")) { } else if (boost::iequals(val.first, "backend")) {
if(val.second.data() == "cassandra") { if(val.second.data() == "cassandra") {
backend = PerSystSqlOperator::CASSANDRA;
op.setBackend(PerSystSqlOperator::CASSANDRA); op.setBackend(PerSystSqlOperator::CASSANDRA);
} else if(val.second.data() == "mariadb") { } else if(val.second.data() == "mariadb") {
backend = PerSystSqlOperator::MARIADB;
op.setBackend(PerSystSqlOperator::MARIADB); op.setBackend(PerSystSqlOperator::MARIADB);
} }
} else if(boost::iequals(val.first, "property_id")){ } else if(boost::iequals(val.first, "property_id")){
auto property_id = std::stoi(val.second.data()); auto property_id = std::stoi(val.second.data());
op.setPropertyId(property_id); op.setPropertyId(property_id);
} else if (boost::iequals(val.first, "mariadb_host")) { } else if (boost::iequals(val.first, "mariadb_host")) {
host = val.second.data(); op.setMariaDBHost(val.second.data());
} else if (boost::iequals(val.first, "mariadb_user")){ } else if (boost::iequals(val.first, "mariadb_user")){
user = val.second.data(); op.setMariaDBUser(val.second.data());
} else if (boost::iequals(val.first, "mariadb_password")){ } else if (boost::iequals(val.first, "mariadb_password")){
password = val.second.data(); op.setMariaDBPassword(val.second.data());
} else if (boost::iequals(val.first, "mariadb_database_name")){ } else if (boost::iequals(val.first, "mariadb_database_name")){
database_name = val.second.data(); op.setMariaDBDatabaseName(val.second.data());
} else if (boost::iequals(val.first, "mariadb_port")){ } else if (boost::iequals(val.first, "mariadb_port")){
port = std::stoi(val.second.data()); int port = std::stoi(val.second.data());
op.setMariaDBPort(port);
} else if (boost::iequals(val.first, "mariadb_rotation")){ } else if (boost::iequals(val.first, "mariadb_rotation")){
auto found = _rotation_map.find(val.second.data()); auto found = _rotation_map.find(val.second.data());
if (found != _rotation_map.end()){ if (found != _rotation_map.end()){
rotation = found->second; op.setMariaDBRotation(found->second);
} else { } else {
LOG(error) << " Rotation strategy (" << val.second.data() << ") not found."; LOG(error) << " Rotation strategy (" << val.second.data() << ") not found.";
} }
} else if (boost::iequals(val.first, "mariadb_every_x_days")){ } else if (boost::iequals(val.first, "mariadb_every_x_days")){
every_x_days = std::stoi(val.second.data()); unsigned int every_x_days = std::stoi(val.second.data());
op.setMariaDBEveryXDays(every_x_days);
} }
} }
if(backend == PerSystSqlOperator::MARIADB) {
op.setMariaDBConnection(host, user, password, database_name, port, rotation, every_x_days);
}
} }
bool PerSystSqlConfigurator::unit(UnitTemplate<AggregatorSensorBase>& u) { bool PerSystSqlConfigurator::unit(UnitTemplate<AggregatorSensorBase>& u) {
......
...@@ -106,7 +106,9 @@ void PerSystSqlOperator::printConfig(LOG_LEVEL ll) { ...@@ -106,7 +106,9 @@ void PerSystSqlOperator::printConfig(LOG_LEVEL ll) {
LOG_VAR(ll) << "\tDatabase=" << _conn.database_name; LOG_VAR(ll) << "\tDatabase=" << _conn.database_name;
LOG_VAR(ll) << "\tPort=" << _conn.port; LOG_VAR(ll) << "\tPort=" << _conn.port;
LOG_VAR(ll) << "\tRotation=" << _conn.rotation; LOG_VAR(ll) << "\tRotation=" << _conn.rotation;
LOG_VAR(ll) << "\tEvery_X_days=" << _conn.every_x_days; if(_conn.rotation == MariaDB::EVERY_XDAYS){
LOG_VAR(ll) << "\tEvery_X_days=" << _conn.every_x_days;
}
} }
LOG_VAR(ll) << "Property Configuration:"; LOG_VAR(ll) << "Property Configuration:";
LOG_VAR(ll) << "\tnumber_of_even_quantiles=" << _number_of_even_quantiles; LOG_VAR(ll) << "\tnumber_of_even_quantiles=" << _number_of_even_quantiles;
......
...@@ -101,14 +101,32 @@ public: ...@@ -101,14 +101,32 @@ public:
_backend = backend; _backend = backend;
} }
void setMariaDBConnection(const std::string & host, const std::string & user, const std::string & password, const std::string & database_name, int port, MariaDB::Rotation_t rotation, unsigned int every_x_days=0){ void setMariaDBEveryXDays(unsigned int every_x_days){
_conn.every_x_days = every_x_days;
}
void setMariaDBHost(const std::string & host){
_conn.host = host; _conn.host = host;
}
void setMariaDBUser(const std::string & user){
_conn.user = user; _conn.user = user;
}
void setMariaDBPassword(const std::string & password){
_conn.password = password; _conn.password = password;
}
void setMariaDBDatabaseName(const std::string & database_name){
_conn.database_name = database_name; _conn.database_name = database_name;
}
void setMariaDBPort(int port){
_conn.port = port; _conn.port = port;
}
void setMariaDBRotation(MariaDB::Rotation_t rotation){
_conn.rotation = rotation; _conn.rotation = rotation;
_conn.every_x_days = every_x_days;
} }
void setPropertyId(int property_id){ void setPropertyId(int property_id){
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment