11.3.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

metadatastore.h 14.7 KB
Newer Older
Alessio Netti's avatar
Alessio Netti committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
//================================================================================
// Name        : metadatastore.h
// Author      : Alessio Netti
// Contact     : info@dcdb.it
// Copyright   : Leibniz Supercomputing Centre
// Description : A meta-data store for sensors
//================================================================================

//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2011-2019 Leibniz Supercomputing Centre
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public License
// as published by the Free Software Foundation; either version 2
// of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
//================================================================================

#ifndef PROJECT_METADATASTORE_H
#define PROJECT_METADATASTORE_H

#include <vector>
32
#include <unordered_map>
Alessio Netti's avatar
Alessio Netti committed
33 34 35 36 37 38 39
#include <string>
#include <boost/property_tree/ptree.hpp>
#include <boost/foreach.hpp>
#include <boost/property_tree/info_parser.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string.hpp>
40
#include <sstream>
Alessio Netti's avatar
Alessio Netti committed
41 42 43 44
#include "globalconfiguration.h"

using namespace std;

45 46
class SensorMetadata {
    
Alessio Netti's avatar
Alessio Netti committed
47
public:
48
    
49
    SensorMetadata() :
50
        isOperation(false),
51
        isVirtual(false),
52
        integrable(true),
53 54
        monotonic(false),
        publicName(""),
55
        pattern(""),
56
        unit(""),
57
        scale(1.0),
58
        ttl(0),
59 60
        interval(0),
        operations("") {}
61 62

    SensorMetadata(const SensorMetadata& other) {
63
        this->isOperation = other.isOperation;
64 65 66 67
        this->isVirtual = other.isVirtual;
        this->integrable = other.integrable;
        this->monotonic = other.monotonic;
        this->publicName = other.publicName;
68
        this->pattern = other.pattern;
69 70 71 72 73 74
        this->unit = other.unit;
        this->scale = other.scale;
        this->ttl = other.ttl;
        this->interval = other.interval;
        this->operations = other.operations;
    }
75

76
    SensorMetadata& operator=(const SensorMetadata& other) {
77
        this->isOperation = other.isOperation;
78 79 80 81
        this->isVirtual = other.isVirtual;
        this->integrable = other.integrable;
        this->monotonic = other.monotonic;
        this->publicName = other.publicName;
82
        this->pattern = other.pattern;
83 84 85 86 87 88 89 90
        this->unit = other.unit;
        this->scale = other.scale;
        this->ttl = other.ttl;
        this->interval = other.interval;
        this->operations = other.operations;
        return *this;
    }
    
91 92 93 94 95 96 97 98 99 100 101 102 103
    
    bool addOperation(const string& opName) {
        if (publicName.length()>0 && opName.length()>publicName.length() && !opName.compare(0, publicName.length(), publicName)) {
            if(operations.length()==0)
                operations = opName.substr(publicName.length());
            else
                operations += "," + opName.substr(publicName.length());
            return true;
        }
        else
            return false;
    }
    
104 105 106 107 108 109 110
    /**
     * @brief               Parses a JSON string and stores the content in this object.
     * 
     *                      If parsing fails, a InvalidArgument exception is thrown.
     * 
     * @param payload       JSON-encoded string containing metadata information
     */
111 112 113 114 115 116
    void parseJSON(const string& payload) {
        boost::property_tree::iptree config;
        std::istringstream str(payload);
        boost::property_tree::read_json(str, config);
        parsePTREE(config);
    }
117 118 119 120 121 122 123 124

    /**
     * @brief               Parses a PTREE INFO block and stores the content in this object.
     *
     *                      If parsing fails, a InvalidArgument exception is thrown.
     *                      
     * @param config        PTREE block containing metadata
     */
125 126 127 128 129 130
    void parsePTREE(boost::property_tree::iptree& config) {
        BOOST_FOREACH(boost::property_tree::iptree::value_type &val, config) {
            if (boost::iequals(val.first, "monotonic")) {
                this->monotonic = to_bool(val.second.data());
            } else if (boost::iequals(val.first, "isVirtual")) {
                this->isVirtual = to_bool(val.second.data());
131 132
            } else if (boost::iequals(val.first, "isOperation")) {
                this->isOperation = to_bool(val.second.data());
133 134 135 136 137 138
            } else if (boost::iequals(val.first, "integrable")) {
                this->integrable = to_bool(val.second.data());
            } else if (boost::iequals(val.first, "unit")) {
                this->unit = val.second.data();
            } else if (boost::iequals(val.first, "publicName")) {
                this->publicName = val.second.data();
139 140
            } else if (boost::iequals(val.first, "pattern")) {
                this->pattern = val.second.data();
141
            } else if (boost::iequals(val.first, "scale")) {
142
                this->scale = stod(val.second.data());
143 144
            } else if (boost::iequals(val.first, "interval")) {
                this->interval = stoull(val.second.data()) * 1000000;
Alessio Netti's avatar
Alessio Netti committed
145 146
            } else if (boost::iequals(val.first, "ttl")) {
                this->ttl = stoull(val.second.data()) * 1000000;
147
            } else if (boost::iequals(val.first, "operations")) {
148
                this->operations = _sanitizeOperations(val.second.data());
149 150 151
            }
        }
    }
152 153 154 155 156 157
    
    /**
     * @brief               Converts the content of this object into JSON format.      
     * 
     * @return              String containing the JSON representation of this object
     */
158 159 160 161 162 163 164
    string getJSON() const {
        boost::property_tree::ptree config;
        std::ostringstream output;
        _dumpPTREE(config);
        boost::property_tree::write_json(output, config, true);
        return output.str();
    }
165 166 167 168 169 170

    /**
     * @brief               Returns a sensorMetadata_t object from the internal map, converted into PTREE format.     
     * 
     * @return              A PTREE object representing this SensorMetadata object
     */
171 172 173 174 175
    boost::property_tree::ptree getPTREE() const {
        boost::property_tree::ptree config;
        _dumpPTREE(config);
        return config;
    }
Alessio Netti's avatar
Alessio Netti committed
176
    
177
    // Public class members
178
    bool                isOperation;
179 180 181 182
    bool                isVirtual;
    bool                integrable;
    bool                monotonic;
    string              publicName;
183
    string              pattern;
184
    string              unit;
185
    double              scale;
186 187
    uint64_t            ttl;
    uint64_t            interval;
188
    string              operations;
189 190 191
    
protected:

192 193 194 195 196 197
    // Parses a operations string and sanitizes it from excess whitespace
    string _sanitizeOperations(const string& str, const char sep=',') {
        vector<string> v;
        string out="";
        
        // We split the string into the comma-separated tokens
198 199 200 201 202 203 204 205
        std::stringstream ss(str);
        std::string token;
        while (std::getline(ss, token, sep)) {
            if(!token.empty()) {
                boost::algorithm::trim(token);
                v.push_back(token);
            }
        }
206 207 208 209 210 211 212 213 214
        
        // We re-write the vector into a string, this time properly formatted
        string sepStr = string(1,sep);
        for(const auto& el : v)
            out += el + sepStr;
        if(!out.empty() && out.back() == sep)
            out.erase(out.size()-1, 1);
        
        return out;
215 216
    }
    
217
    // Dumps the contents of "s" in "config"
218
    void _dumpPTREE(boost::property_tree::ptree& config) const {
219 220
        std::ostringstream scaleStream;
        scaleStream << this->scale;
221
        config.clear();
222
        config.push_back(boost::property_tree::ptree::value_type("isOperation", boost::property_tree::ptree(this->isOperation ? "true" : "false")));
223 224 225 226 227
        config.push_back(boost::property_tree::ptree::value_type("isVirtual", boost::property_tree::ptree(this->isVirtual ? "true" : "false")));
        config.push_back(boost::property_tree::ptree::value_type("monotonic", boost::property_tree::ptree(this->monotonic ? "true" : "false")));
        config.push_back(boost::property_tree::ptree::value_type("integrable", boost::property_tree::ptree(this->integrable ? "true" : "false")));
        config.push_back(boost::property_tree::ptree::value_type("unit", boost::property_tree::ptree(this->unit)));
        config.push_back(boost::property_tree::ptree::value_type("publicName", boost::property_tree::ptree(this->publicName)));
228
        config.push_back(boost::property_tree::ptree::value_type("pattern", boost::property_tree::ptree(this->pattern)));
229
        config.push_back(boost::property_tree::ptree::value_type("scale", boost::property_tree::ptree(scaleStream.str())));
230
        config.push_back(boost::property_tree::ptree::value_type("interval", boost::property_tree::ptree(to_string(this->interval / 1000000))));
Alessio Netti's avatar
Alessio Netti committed
231
        config.push_back(boost::property_tree::ptree::value_type("ttl", boost::property_tree::ptree(to_string(this->ttl / 1000000))));
232
        config.push_back(boost::property_tree::ptree::value_type("operations", boost::property_tree::ptree(this->operations)));
233
    }
234
    
Alessio Netti's avatar
Alessio Netti committed
235 236
};

237 238 239 240
// ---------------------------------------------------------------------------
// --------------------------- METADATASTORE CLASS --------------------------- 
// ---------------------------------------------------------------------------

Alessio Netti's avatar
Alessio Netti committed
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
class MetadataStore {
    
public:
    
    /**
     * @brief               Public class constructor.
     */
    MetadataStore() {}
    
    /**
     * @brief               Class destructor.
     */
    ~MetadataStore() {}
    
    /**
     * @brief               Clears the internal metadata map.
     */
258 259 260
    void clear() { 
        _metadata.clear(); 
    }
Alessio Netti's avatar
Alessio Netti committed
261 262 263 264 265 266
    
    /**
     * @brief               Returns the internal metadata map.
     * 
     * @return              A string to Metadata_t map
     */
267
    const unordered_map<string, SensorMetadata>& getMap() { 
268 269
        return _metadata; 
    }
Alessio Netti's avatar
Alessio Netti committed
270 271 272 273 274 275 276 277 278 279
    
    /**
     * @brief               Stores a sensorMetadata_t object in the internal map.
     * 
     *                      If the input key already exists in the map, the entry is overwritten.
     * 
     * @param key           Sensor key under which metadata must be stored
     * @param s             Object containing sensor metadata
     * @return              True if "key" is unique, False if there was a collision
     */
280 281 282 283 284
    bool store(const string& key, const SensorMetadata& s) {
        bool overwritten = !_metadata.count(key);
        _metadata[key] = s;
        return overwritten;
    }
Alessio Netti's avatar
Alessio Netti committed
285 286 287 288 289 290 291 292 293 294 295
    
    /**
     * @brief               Stores a sensorMetadata_t object in the internal map, after parsing it from a JSON string.
     * 
     *                      If the input key already exists in the map, the entry is overwritten. If parsing fails,
     *                      a InvalidArgument exception is thrown.
     * 
     * @param key           Sensor key under which metadata must be stored
     * @param payload       JSON-encoded string containing metadata information
     * @return              True if "key" is unique, False if there was a collision
     */
296 297 298 299 300
    bool storeFromJSON(const string& key, const string& payload) {
        SensorMetadata metadata;
        metadata.parseJSON(payload);
        return store(key, metadata);
    }
Alessio Netti's avatar
Alessio Netti committed
301 302 303 304 305 306 307 308 309 310 311
    
    /**
     * @brief               Stores a sensorMetadata_t object in the internal map, after parsing it from a INFO block.
     *
     *                      If the input key already exists in the map, the entry is overwritten. If parsing fails,
     *                      a InvalidArgument exception is thrown.
     *                      
     * @param key           Sensor key under which metadata must be stored
     * @param config        PTREE block containing metadata
     * @return              True if "key" is unique, False if there was a collision
     */
312 313 314 315 316
    bool storeFromPTREE(const string& key, boost::property_tree::iptree& config) {
        SensorMetadata metadata;
        metadata.parsePTREE(config);
        return store(key, metadata);
    }
Alessio Netti's avatar
Alessio Netti committed
317 318 319 320 321 322 323 324 325
    
    /**
     * @brief               Returns a sensorMetadata_t object from the internal map. 
     * 
     *                      If the input key does not exist in the map, a InvalidArgument exception is thrown.
     * 
     * @param key           Sensor key to be queried
     * @return              A reference to a sensorMetadata_t object
     */
326 327 328 329 330
    const SensorMetadata& get(const string& key) {
        if(!_metadata.count(key))
            throw invalid_argument("MetadataStore: key " + key + " does not exist!");
        return _metadata[key];
    }
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345

    /**
     * @brief               Returns the TTL of a sensorMetadata_t object from the internal map. 
     * 
     *                      If the input key does not exist in the map, the value -1 is returned. This method exists
     *                      to boost (slightly) look-up performance in the CollectAgent, which requires TTL values
     *                      when performing database inserts.
     * 
     * @param key           Sensor key to be queried
     * @return              TTL value in seconds
     */
    int64_t getTTL(const string& key) {
        auto it = _metadata.find(key);
        return it==_metadata.end() ? -1 : it->second.ttl/1000000000;
    }
Alessio Netti's avatar
Alessio Netti committed
346 347 348 349 350 351 352 353 354
    
    /**
     * @brief               Returns a sensorMetadata_t object from the internal map, converted into JSON format.      
     * 
     *                      If the input key does not exist in the map, a InvalidArgument exception is thrown.
     * 
     * @param key           Sensor key to be queried
     * @return              String containing the JSON representation of the sensorMetadata_t object
     */
355 356 357
    string getJSON(const string& key) {
        return this->get(key).getJSON();
    }
Alessio Netti's avatar
Alessio Netti committed
358 359 360 361 362 363 364 365 366
    
    /**
     * @brief               Returns a sensorMetadata_t object from the internal map, converted into PTREE format.     
     * 
     *                      If the input key does not exist in the map, a InvalidArgument exception is thrown.
     * 
     * @param key           Sensor key to be queried
     * @return              A PTREE object representing the sensorMetadata_t object
     */
367 368 369
     boost::property_tree::ptree getPTREE(const string& key) {
        return this->get(key).getPTREE();
    }
Alessio Netti's avatar
Alessio Netti committed
370 371 372
    
protected:
    
373
    unordered_map<string, SensorMetadata> _metadata;
Alessio Netti's avatar
Alessio Netti committed
374 375 376 377 378
    
};


#endif //PROJECT_METADATASTORE_H