Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
9.2.2023: Due to updates GitLab will be unavailable for some minutes between 9:00 and 11:00.
Open sidebar
dcdb
dcdb
Commits
3a66b50a
Commit
3a66b50a
authored
Jun 29, 2021
by
Michael Ott
Browse files
Merge branch 'REST' into development
parents
71b60c6b
79adb9c1
Changes
9
Hide whitespace changes
Inline
Side-by-side
config.mk
View file @
3a66b50a
...
...
@@ -3,7 +3,7 @@ DCDBDEPSPATH ?= $(DCDBBASEPATH)/deps
DCDBDEPLOYPATH
?=
$(DCDBBASEPATH)
/install
# dcdbpusher plugins to be built
PLUGINS
=
sysfs ipmi pdu bacnet snmp procfs tester gpfsmon msr
PLUGINS
=
sysfs ipmi pdu bacnet snmp procfs tester gpfsmon msr
rest
# data analytics plugins to be built
OPERATORS
=
aggregator smoothing regressor classifier clustering cssignatures job_aggregator testeroperator filesink smucngperf persystsql coolingcontrol healthchecker
...
...
dcdbpusher/Makefile
View file @
3a66b50a
...
...
@@ -121,3 +121,7 @@ libdcdbplugin_caliper.$(LIBEXT): sensors/caliper/CaliperSensorGroup.o sensors/ca
libdcdbplugin_nvml.$(LIBEXT)
:
sensors/nvml/nvmlSensorGroup.o sensors/nvml/nvmlConfigurator.o
$(NVCC)
-shared
--compiler-options
'-fPIC'
-o
$@
$^
-L
$(DCDBDEPLOYPATH)
/lib/
-lboost_log
-lboost_system
-lnvidia-ml
libdcdbplugin_rest.$(LIBEXT)
:
sensors/rest/RESTSensorGroup.o sensors/rest/RESTUnit.o sensors/rest/RESTConfigurator.o
$(CXX)
$(LIBFLAGS)$@
-o
$@
$^
-L
$(DCDBDEPLOYPATH)
/lib/
-lcrypto
-lssl
-lboost_regex
-lboost_log
-lboost_system
dcdbpusher/sensors/rest/RESTConfigurator.cpp
0 → 100644
View file @
3a66b50a
//================================================================================
// Name : RESTConfigurator.cpp
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Source file for REST plugin configurator class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include
"RESTConfigurator.h"
#include
<iostream>
#include
<sstream>
RESTConfigurator
::
RESTConfigurator
()
{
_entityName
=
"host"
;
_groupName
=
"group"
;
_baseName
=
"sensor"
;
}
RESTConfigurator
::~
RESTConfigurator
()
{}
void
RESTConfigurator
::
sensorBase
(
RESTSensorBase
&
s
,
CFG_VAL
config
)
{
ADD
{
ATTRIBUTE
(
"path"
,
setXMLPath
);
}
}
void
RESTConfigurator
::
sensorGroup
(
RESTSensorGroup
&
s
,
CFG_VAL
config
)
{
ADD
{
ATTRIBUTE
(
"endpoint"
,
setEndpoint
);
ATTRIBUTE
(
"request"
,
setRequest
);
}
}
void
RESTConfigurator
::
sensorEntity
(
RESTUnit
&
s
,
CFG_VAL
config
)
{
ADD
{
ATTRIBUTE
(
"baseurl"
,
setBaseURL
);
ATTRIBUTE
(
"authendpoint"
,
setAuthEndpoint
);
ATTRIBUTE
(
"authdata"
,
setAuthData
);
}
}
dcdbpusher/sensors/rest/RESTConfigurator.h
0 → 100644
View file @
3a66b50a
//================================================================================
// Name : RESTConfigurator.h
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Header file for REST plugin configurator class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#ifndef RESTCONFIGURATOR_H_
#define RESTCONFIGURATOR_H_
#include
"../../includes/ConfiguratorTemplateEntity.h"
#include
"RESTSensorGroup.h"
#include
"RESTUnit.h"
/**
* @brief ConfiguratorTemplate specialization for this plugin.
*
* @ingroup rest
*/
class
RESTConfigurator
:
public
ConfiguratorTemplateEntity
<
RESTSensorBase
,
RESTSensorGroup
,
RESTUnit
>
{
public:
RESTConfigurator
();
virtual
~
RESTConfigurator
();
protected:
/* Overwritten from ConfiguratorTemplate */
void
sensorBase
(
RESTSensorBase
&
s
,
CFG_VAL
config
)
override
;
void
sensorGroup
(
RESTSensorGroup
&
s
,
CFG_VAL
config
)
override
;
void
sensorEntity
(
RESTUnit
&
s
,
CFG_VAL
config
)
override
;
private:
/**
* Split the given string into multiple parts which are later required to find the sensor value in the boost property tree.
* Store the individual parts into the sensors _xmlPathVector
* @param sensor Sensor where the path belongs to
* @param pathString Complete unrefined string as read from the config file
*/
void
parsePathString
(
RESTSensorBase
&
sensor
,
const
std
::
string
&
pathString
);
};
extern
"C"
ConfiguratorInterface
*
create
()
{
return
new
RESTConfigurator
;
}
extern
"C"
void
destroy
(
ConfiguratorInterface
*
c
)
{
delete
c
;
}
#endif
/* RESTCONFIGURATOR_H_ */
dcdbpusher/sensors/rest/RESTSensorBase.h
0 → 100644
View file @
3a66b50a
//================================================================================
// Name : RESTSensorBase.h
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Sensor base class for REST plugin.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
/**
* @defgroup rest REST plugin
* @ingroup pusherplugins
*
* @brief Collect data from REST APIs
*/
#ifndef RESTSENSORBASE_H_
#define RESTSENSORBASE_H_
#include
"RESTUnit.h"
#include
"sensorbase.h"
#include
<sstream>
/**
* @brief SensorBase specialization for this plugin.
*
* @ingroup rest
*/
class
RESTSensorBase
:
public
SensorBase
{
public:
RESTSensorBase
(
const
std
::
string
&
name
)
:
SensorBase
(
name
)
{}
RESTSensorBase
(
const
RESTSensorBase
&
other
)
:
SensorBase
(
other
),
_xmlPath
(
other
.
_xmlPath
)
{}
virtual
~
RESTSensorBase
()
{}
RESTSensorBase
&
operator
=
(
const
RESTSensorBase
&
other
)
{
SensorBase
::
operator
=
(
other
);
_xmlPath
=
other
.
_xmlPath
;
return
*
this
;
}
const
xmlPathVector_t
&
getXMLPath
()
const
{
return
_xmlPath
;
}
std
::
string
getXMLPathString
()
const
{
std
::
stringstream
ss
;
for
(
const
auto
&
i
:
_xmlPath
)
{
ss
<<
"."
<<
std
::
get
<
0
>
(
i
)
<<
"."
<<
std
::
get
<
1
>
(
i
);
for
(
const
auto
&
j
:
std
::
get
<
2
>
(
i
))
{
ss
<<
"("
<<
j
.
first
<<
"="
<<
j
.
second
<<
")"
;
}
}
return
ss
.
str
();
}
void
setXMLPath
(
const
std
::
string
&
path
)
{
std
::
vector
<
std
::
string
>
subStrings
;
std
::
stringstream
pathStream
(
path
);
std
::
string
item
;
//split into parts if a attribute (indicated by '(' ')' ) was defined
while
(
std
::
getline
(
pathStream
,
item
,
')'
))
{
subStrings
.
push_back
(
item
);
}
for
(
auto
subStr
:
subStrings
)
{
//extract the attributes from the path-parts
if
(
subStr
.
find
(
'('
)
!=
std
::
string
::
npos
)
{
//attribute specified
std
::
stringstream
pathWithAttributesSStream
(
subStr
);
//split into path and attributes string
std
::
string
subPath
,
attributeString
;
std
::
getline
(
pathWithAttributesSStream
,
subPath
,
'('
);
std
::
getline
(
pathWithAttributesSStream
,
attributeString
);
if
(
subPath
.
front
()
==
'.'
)
{
subPath
.
erase
(
0
,
1
);
}
//now further split the attributes string as multiple attributes could be defined
std
::
vector
<
std
::
string
>
attributes
;
std
::
stringstream
attributeStream
(
attributeString
);
while
(
std
::
getline
(
attributeStream
,
item
,
','
))
{
attributes
.
push_back
(
item
);
}
attributesVector_t
attrs
;
for
(
auto
att
:
attributes
)
{
//part attributes into name and value
if
(
att
.
find
(
'='
)
!=
std
::
string
::
npos
)
{
std
::
stringstream
attStream
(
att
);
std
::
string
attName
,
attVal
;
std
::
getline
(
attStream
,
attName
,
'='
);
std
::
getline
(
attStream
,
attVal
);
attrs
.
push_back
(
std
::
make_pair
(
attName
,
attVal
));
}
else
{
//should not happen. If it does the path was malformed
//LOG(error) << " Could not parse XML-path!";
return
;
}
}
//split of the last child in the path. Required to iterate over multiple nodes which only differ in the attributes with BOOST_FOREACH
auto
index
=
subPath
.
find_last_of
(
'.'
);
if
(
index
!=
std
::
string
::
npos
)
{
std
::
string
subPathChild
(
subPath
.
substr
(
++
index
));
subPath
.
erase
(
--
index
);
_xmlPath
.
push_back
(
std
::
make_tuple
(
subPath
,
subPathChild
,
attrs
));
}
else
{
//the path contained only one node
_xmlPath
.
push_back
(
std
::
make_tuple
(
""
,
subPath
,
attrs
));
}
}
else
{
//no attributes specified. Last (sub)path
if
(
subStr
.
front
()
==
'.'
)
{
subStr
.
erase
(
0
,
1
);
}
_xmlPath
.
push_back
(
std
::
make_tuple
(
subStr
,
""
,
attributesVector_t
()));
break
;
}
}
}
void
printConfig
(
LOG_LEVEL
ll
,
LOGGER
&
lg
,
unsigned
leadingSpaces
=
16
)
{
std
::
string
leading
(
leadingSpaces
,
' '
);
LOG_VAR
(
ll
)
<<
leading
<<
" XML Path: "
<<
getXMLPathString
();
}
protected:
xmlPathVector_t
_xmlPath
;
};
#endif
/* RESTSENSORBASE_H_ */
dcdbpusher/sensors/rest/RESTSensorGroup.cpp
0 → 100644
View file @
3a66b50a
//================================================================================
// Name : RESTSensorGroup.cpp
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Source file for REST sensor group class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include
"RESTSensorGroup.h"
#include
<sstream>
#include
<boost/foreach.hpp>
#include
<boost/property_tree/xml_parser.hpp>
RESTSensorGroup
::
RESTSensorGroup
(
const
std
::
string
&
name
)
:
SensorGroupTemplateEntity
(
name
)
{
}
RESTSensorGroup
::
RESTSensorGroup
(
const
RESTSensorGroup
&
other
)
:
SensorGroupTemplateEntity
(
other
),
_endpoint
(
other
.
_endpoint
),
_request
(
other
.
_request
)
{}
RESTSensorGroup
::~
RESTSensorGroup
()
{}
RESTSensorGroup
&
RESTSensorGroup
::
operator
=
(
const
RESTSensorGroup
&
other
)
{
SensorGroupTemplate
::
operator
=
(
other
);
_endpoint
=
other
.
_endpoint
;
_request
=
other
.
_request
;
return
*
this
;
}
void
RESTSensorGroup
::
read
()
{
//send request
std
::
string
response
;
if
(
!
_entity
->
sendRequest
(
_endpoint
,
_request
,
response
))
{
LOG
(
error
)
<<
_groupName
<<
" could not send request!"
;
return
;
}
//parse response
boost
::
property_tree
::
ptree
ptree
;
std
::
string
xml
=
response
.
substr
(
response
.
find
(
"<"
));
std
::
istringstream
treeStream
(
xml
);
try
{
boost
::
property_tree
::
read_xml
(
treeStream
,
ptree
);
}
catch
(
const
std
::
exception
&
e
)
{
LOG
(
error
)
<<
_groupName
<<
" got malformed XML response"
;
return
;
}
//read values for every sensor from response
reading_t
reading
;
reading
.
timestamp
=
getTimestamp
();
for
(
const
auto
&
s
:
_sensors
)
{
try
{
std
::
string
readStr
;
const
xmlPathVector_t
&
xmlPath
=
s
->
getXMLPath
();
boost
::
property_tree
::
ptree
node
=
ptree
;
for
(
size_t
i
=
0
;
i
<
xmlPath
.
size
();
i
++
)
{
const
std
::
string
&
path
=
std
::
get
<
0
>
(
xmlPath
[
i
]);
const
std
::
string
&
child
=
std
::
get
<
1
>
(
xmlPath
[
i
]);
const
attributesVector_t
&
attVec
=
std
::
get
<
2
>
(
xmlPath
[
i
]);
unsigned
matchCount
;
if
(
child
!=
""
)
{
BOOST_FOREACH
(
boost
::
property_tree
::
ptree
::
value_type
&
v
,
node
.
get_child
(
path
))
{
if
(
v
.
first
==
child
)
{
matchCount
=
0
;
for
(
size_t
j
=
0
;
j
<
attVec
.
size
();
j
++
)
{
std
::
string
attributeVal
=
v
.
second
.
get_child
(
"<xmlattr>."
+
attVec
[
j
].
first
).
data
();
if
(
attributeVal
!=
attVec
[
j
].
second
)
{
//attribute values don't match
break
;
}
else
{
matchCount
++
;
}
}
if
(
matchCount
==
attVec
.
size
())
{
//all attributes matched
readStr
=
v
.
second
.
data
();
node
=
v
.
second
;
break
;
}
}
}
}
else
{
//child == ""
readStr
=
node
.
get
(
path
,
""
);
break
;
//last (part of the) path
}
}
if
(
readStr
==
""
)
{
throw
std
::
runtime_error
(
"Value not found!"
);
}
size_t
idx
;
uint64_t
ival
=
stoll
(
readStr
,
&
idx
);
double
dval
=
.0
;
bool
have_float
=
false
;
if
(
idx
<
readStr
.
size
())
{
if
(
readStr
[
idx
]
==
'.'
)
{
dval
=
stod
(
readStr
,
&
idx
);
have_float
=
true
;
}
}
uint64_t
factor
=
1
;
if
(
idx
<
readStr
.
size
())
{
switch
(
readStr
[
idx
])
{
case
'k'
:
case
'K'
:
factor
=
1000ll
;
break
;
case
'm'
:
case
'M'
:
factor
=
1000000ll
;
break
;
default:
break
;
}
}
if
(
have_float
)
{
reading
.
value
=
dval
*
factor
;
}
else
{
reading
.
value
=
ival
*
factor
;
}
#ifdef DEBUG
LOG
(
debug
)
<<
_groupName
<<
"::"
<<
s
->
getName
()
<<
" raw reading:
\"
"
<<
reading
.
value
<<
"
\"
"
;
#endif
s
->
storeReading
(
reading
);
}
catch
(
const
std
::
exception
&
e
)
{
LOG
(
error
)
<<
_groupName
<<
"::"
<<
s
->
getName
()
<<
" could not read value: "
<<
e
.
what
();
continue
;
}
}
}
void
RESTSensorGroup
::
printGroupConfig
(
LOG_LEVEL
ll
,
unsigned
int
leadingSpaces
)
{
std
::
string
leading
(
leadingSpaces
,
' '
);
LOG_VAR
(
ll
)
<<
leading
<<
"Endpoint: "
<<
_endpoint
;
LOG_VAR
(
ll
)
<<
leading
<<
"Request: "
<<
_request
;
}
dcdbpusher/sensors/rest/RESTSensorGroup.h
0 → 100644
View file @
3a66b50a
//================================================================================
// Name : RESTSensorGroup.h
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Header file for REST sensor group class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#ifndef RESTSENSORGROUP_H_
#define RESTSENSORGROUP_H_
#include
"../../includes/SensorGroupTemplateEntity.h"
#include
"RESTSensorBase.h"
/**
* @brief SensorGroupTemplate specialization for this plugin.
*
* @ingroup rest
*/
class
RESTSensorGroup
:
public
SensorGroupTemplateEntity
<
RESTSensorBase
,
RESTUnit
>
{
public:
RESTSensorGroup
(
const
std
::
string
&
name
);
RESTSensorGroup
(
const
RESTSensorGroup
&
other
);
virtual
~
RESTSensorGroup
();
RESTSensorGroup
&
operator
=
(
const
RESTSensorGroup
&
other
);
void
setEndpoint
(
const
std
::
string
&
endpoint
)
{
_endpoint
=
endpoint
;
}
const
std
::
string
&
getEndpoint
()
{
return
_endpoint
;
}
void
setRequest
(
const
std
::
string
&
request
)
{
_request
=
request
;
}
const
std
::
string
&
getRequest
()
{
return
_request
;
}
void
printGroupConfig
(
LOG_LEVEL
ll
,
unsigned
int
leadingSpaces
)
final
override
;
private:
void
read
()
final
override
;
std
::
string
_endpoint
;
std
::
string
_request
;
};
#endif
/* RESTSENSORGROUP_H_ */
dcdbpusher/sensors/rest/RESTUnit.cpp
0 → 100644
View file @
3a66b50a
//================================================================================
// Name : RESTUnit.cpp
// Author : Michael Ott
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Source file for RESTUnit class.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2018-2021 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//================================================================================
#include
"RESTUnit.h"
#include
"globalconfiguration.h"
#include
<iostream>
#include
<openssl/bio.h>
#include
<openssl/err.h>
RESTUnit
::
RESTUnit
(
const
std
::
string
&
name
)
:
EntityInterface
(
name
),
_ctx
(
ssl
::
context
::
tlsv12_client
),
_ssl
(
false
)
{
}
RESTUnit
::
RESTUnit
(
const
RESTUnit
&
other
)
:
EntityInterface
(
other
),
_ctx
(
ssl
::
context
::
tlsv12_client
),
_baseURL
(
other
.
_baseURL
),
_hostname
(
other
.
_hostname
),
_port
(
other
.
_port
),
_path
(
other
.
_path
),
_authEndpoint
(
other
.
_authEndpoint
),
_authData
(
other
.
_authData
),
_ssl
(
false
)
{
}
RESTUnit
::~
RESTUnit
()
{
}
RESTUnit
&
RESTUnit
::
operator
=
(
const
RESTUnit
&
other
)
{
EntityInterface
::
operator
=
(
other
);
_baseURL
=
other
.
_baseURL
;
_hostname
=
other
.
_hostname
;
_port
=
other
.
_port
;
_path
=
other
.
_path
;
_authEndpoint
=
other
.
_authEndpoint
;
_authData
=
other
.
_authData
;
_ssl
=
other
.
_ssl
;
return
*
this
;
}
void
RESTUnit
::
execOnInit
()
{
}
bool
RESTUnit
::
authenticate
()
{
tcp
::
resolver
resolver
(
_ioc
);
auto
const
hosts
=
resolver
.
resolve
(
_hostname
,
_port
);
beast
::
ssl_stream
<
beast
::
tcp_stream
>
stream
(
_ioc
,
_ctx
);
beast
::
get_lowest_layer
(
stream
).
connect
(
hosts
);