The expiration time for new job artifacts in CI/CD pipelines is now 30 days (GitLab default). Previously generated artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit e6f162cb authored by Micha Müller's avatar Micha Müller
Browse files

WIP: Support Caliper Event Data in Storage Backend

-CED should go into separate table
-adds header files for required libdcdb extension
-add other sideshow stuff required for new Cassandra table
-actual implementation outstanding
parent f1314feb
//================================================================================
// Name : calievtdatastore.h
// Author : Micha Mueller
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : C++ API for inserting and querying Caliper event data.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2011-2019 Leibniz Supercomputing Centre
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//================================================================================
/**
* @file calievtdatastore.h
* @brief This file contains parts of the public API for the libdcdb library.
* It contains the class definition of the CaliEvtDataStore class, that handles
* database operations for Caliper Event data.
*
* @ingroup libdcdb
*/
#ifndef DCDB_CALIEVTDATASTORE_H
#define DCDB_CALIEVTDATASTORE_H
#include <cstdint>
#include <list>
#include <string>
#include "cassandra.h"
#include "connection.h"
#include "sensorid.h"
#include "timestamp.h"
namespace DCDB {
/* Forward-declaration of the implementation-internal classes */
class CaliEvtDataStoreImpl;
/**
* @brief This struct is a container for the information DCDB keeps about
* a Caliper Event.
*/
struct CaliEvtData {
SensorId evtId; /**< We abuse the SensorId to identify events that
occurred on the same CPU. */
TimeStamp timeStamp;/**< Time when the event occurred. */
std::string event; /**< String representation of the event that occurred. */
};
/**
* @brief %CaliEvtDataStore is the class of the libdcdb library
* to write and read Caliper Event data.
*/
class CaliEvtDataStore {
private:
CaliEvtDataStoreImpl* impl;
public:
/**
* @brief This function inserts a single event into the database.
*
* @param sid A SensorId object
* @param ts The timestamp of the Caliper Event
* @param event String representation of the encountered event
* @param ttl Time to live (in seconds) for the inserted reading
*/
void insert(SensorId* sid, uint64_t ts, const std::string& event, int64_t ttl=-1);
/**
* @brief This function inserts a single event into the database.
*
* @param data A CaliEvtData object
* @param ttl Time to live (in seconds) for the inserted reading
*/
void insert(CaliEvtData& data, int64_t ttl=-1);
/**
* @brief This function inserts a batch of Caliper Events into
* the database.
* @param readings A list of CaliEvtData objects
* @param ttl Time to live (in seconds) for the inserted readings
*/
void insertBatch(std::list<CaliEvtData>& datas, int64_t ttl=-1);
/**
* @brief Set the TTL for newly inserted event data.
* @param ttl The TTL for the event data in seconds.
*/
void setTTL(uint64_t ttl);
/**
* @brief Enables or disables logging of Cassandra insert errors
* @param dl true to enable logging, false otherwise
*/
void setDebugLog(bool dl);
/**
* @brief This function queries Caliper event data in the given time
* range.
* @param result The list where the results will be stored.
* @param sid The SensorId to query.
* @param start Start of the time series.
* @param end End of the time series.
*/
void query(std::list<CaliEvtData>& result, SensorId& sid, TimeStamp& start, TimeStamp& end);
/**
* @brief This function performs a fuzzy query and returns the
* closest Caliper event data to the input timestamp.
* @param result The list where the results will be stored.
* @param sid The SensorId to query.
* @param ts The target timestamp.
* @param tol_ns Tolerance of the fuzzy query in nanoseconds.
*/
void fuzzyQuery(std::list<CaliEvtData>& result, SensorId& sid, TimeStamp& ts, uint64_t tol_ns=10000000000);
typedef void (*QueryCECbFunc)(CaliEvtData& data, void* userData);
/**
* @brief This function queries Caliper event data in the given time
* range and calls a function for each reading.
* @param cbFunc A function to called for each reading.
* @param userData Pointer to user data that will be returned in the callback.
* @param sid The SensorId to query.
* @param start Start of the time series.
* @param end End of the time series.
*/
void queryCB(QueryCECbFunc cbFunc, void* userData, SensorId& sid, TimeStamp& start, TimeStamp& end);
/**
* @brief A shortcut constructor for a CaliEvtDataStore object that allows
* accessing the data store through a connection that is already
* established.
* @param conn The Connection object of an established connection to
* Cassandra.
*/
CaliEvtDataStore(Connection* conn);
/**
* @brief The standard destructor for a CaliEvtDatStore object.
*/
virtual ~CaliEvtDataStore();
};
} /* End of namespace DCDB */
#endif /* DCDB_CALIEVTDATASTORE_H */
//================================================================================
// Name : calievtdatastore_internal.h
// Author : Micha Mueller
// Contact : info@dcdb.it
// Copyright : Leibniz Supercomputing Centre
// Description : Internal interface for inserting and querying Caliper Event data.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2011-2019 Leibniz Supercomputing Centre
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//================================================================================
/*
* @file
* @brief This file contains the internal functions of the
* Caliper Event Data Store which are provided by the
* CaliEvtDataStoreImpl class.
*/
#ifndef DCDB_CALIEVTDATASTORE_INTERNAL_H
#define DCDB_CALIEVTDATASTORE_INTERNAL_H
#include "dcdb/calievtdatastore.h"
#include "dcdb/connection.h"
namespace DCDB {
/**
* @brief The CaliEvtDataStoreImpl class contains all protected
* functions belonging to CaliEvtDataStore which are
* hidden from the user of the libdcdb library.
*/
class CaliEvtDataStoreImpl {
protected:
Connection* connection; /**< The Connection object that does the low-level stuff for us. */
CassSession* session; /**< The CassSession object given by the connection. */
const CassPrepared* preparedInsert; /**< The prepared statement for fast insertions. */
const CassPrepared* preparedInsert_noTTL; /**< The prepared statement for fast insertions, without TTL. */
bool debugLog;
uint64_t defaultTTL;
/**
* @brief Prepare for insertions.
* @param ttl A TTL that will be set for newly inserted values. Set to 0 to insert without TTL.
*/
void prepareInsert(uint64_t ttl);
public:
/* See calievtdatastore.h for documentation */
void insert(SensorId* sid, uint64_t ts, const std::string& event, int64_t ttl=-1);
void insert(CaliEvtData& data, int64_t ttl=-1);
void insertBatch(std::list<CaliEvtData>& datas, int64_t ttl=-1);
void setTTL(uint64_t ttl);
void setDebugLog(bool dl);
void query(std::list<CaliEvtData>& result, SensorId& sid, TimeStamp& start, TimeStamp& end);
void fuzzyQuery(std::list<CaliEvtData>& result, SensorId& sid, TimeStamp& ts, uint64_t tol_ns=10000000000);
void queryCB(CaliEvtDataStore::QueryCECbFunc cbFunc, void* userData, SensorId& sid, TimeStamp& start, TimeStamp& end);
CaliEvtDataStoreImpl(Connection* conn);
virtual ~CaliEvtDataStoreImpl();
};
} /* End of namespace DCDB */
#endif /* DCDB_CALIEVTDATASTORE_INTERNAL_H */
......@@ -28,15 +28,16 @@
/**
* @file
* @brief This file contains some global definitions and names
* use by the libdcdb library.
* used by the libdcdb library.
*/
#ifndef DCDB_GLOBALS_H
#define DCDB_GLOBALS_H
/* Legend:
* CF = Column Family
* JD = Job Data
* CED = Caliper Event Data
* CF = Column Family
* JD = Job Data
*/
#define KEYSPACE_NAME "dcdb"
......@@ -48,6 +49,9 @@
#define CF_PUBLISHEDSENSORS "publishedsensors"
#define CF_VIRTUALSENSORS "virtualsensors"
#define CED_KEYSPACE_NAME KEYSPACE_NAME "_calievtdata"
#define CF_CALIEVTDATA "calievtdata"
#define JD_KEYSPACE_NAME KEYSPACE_NAME "_jobdata"
#define CF_JOBDATA "jobdata"
......
......@@ -502,6 +502,31 @@ bool ConnectionImpl::initSchema() {
" AND compaction = " SENSORDATA_COMPACTION);
}
/* Keyspace and column family for Caliper Event data */
if (!existsKeyspace(CED_KEYSPACE_NAME)) {
std::cout << "Creating Keyspace " << CED_KEYSPACE_NAME << "...\n";
createKeyspace(CED_KEYSPACE_NAME, 1);
}
selectKeyspace(CED_KEYSPACE_NAME);
if (!(getActiveKeyspace().compare(CED_KEYSPACE_NAME) == 0)) {
std::cout << "Cannot select keyspace " << CED_KEYSPACE_NAME << "\n";
return false;
}
if (!existsColumnFamily(CF_CALIEVTDATA)) {
std::cout << "Creating Column Family " CF_CALIEVTDATA "...\n";
createColumnFamily(CF_CALIEVTDATA,
"name varchar, " /* Public name */
"ts bigint, " /* Timestamp of a Caliper Event */
"value varchar ", /* String representation of the Event */
"name, ts", /* Make the "name" and "ts" column together the primary key */
"COMPACT STORAGE" /* Enable compact storage */
); /* No further options required */
}
/* Keyspace and column family for job data */
if (!existsKeyspace(JD_KEYSPACE_NAME)) {
std::cout << "Creating Keyspace " << JD_KEYSPACE_NAME << "...\n";
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment