Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit f25fe159 authored by Alessio Netti's avatar Alessio Netti
Browse files

libdcdb: overhaul to JobDataStore interface and schema

- Now using an optional "domain ID" as Cassandra partition key
- Can correspond to HPC system, SLURM partition, etc.
- Using materialized views to optimize performance on range queries
parent 48ce9d0a
......@@ -46,11 +46,15 @@
#include "connection.h"
#include "timestamp.h"
// Default domain ID for the jobs if none is specified
#define JOB_DEFAULT_DOMAIN "default"
namespace DCDB {
/* Forward-declaration of the implementation-internal classes */
class JobDataStoreImpl;
using DomainId = std::string;
using JobId = std::string;
using UserId = std::string;
using NodeList = std::list<std::string>;
......@@ -61,6 +65,7 @@ namespace DCDB {
* job.
*/
struct JobData {
DomainId domainId; /**< Domain of the job (e.g., system, partition) **/
JobId jobId; /**< SLURM job id of the job. */
UserId userId; /**< Id of the user who submitted the job. */
TimeStamp startTime;/**< Time when the job started (started != submitted)
......@@ -92,6 +97,7 @@ namespace DCDB {
JobDataStoreImpl* impl;
public:
/**
* @brief This function inserts a single job into the database.
*
......@@ -124,44 +130,63 @@ namespace DCDB {
* primary key together with startTime.
* @param startTime Start time of the job. Part of the primary key.
* @param endTime New endTime to be inserted.
* @param domainId Domain ID of the job (optional).
*
* return See JDError
*/
JDError updateEndtime(JobId jobId, TimeStamp startTs,
TimeStamp endTime);
JDError updateEndtime(JobId jobId, TimeStamp startTs, TimeStamp endTime, std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Update the start time of the job with matching primary key.
*
* @details Since the start time is part of the primary key, the job has
* to be deleted and then re-inserted into the table again.
*
* @param jobId JobId of the job to be updated. Makes up the
* primary key together with startTime.
* @param startTime Start time of the job. Part of the primary key.
* @param newStartTs New start time to be inserted.
* @param domainId Domain ID of the job (optional).
*
* return See JDError
*/
JDError updateStartTime(JobId jobId, TimeStamp startTs, TimeStamp newStartTs, std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Deletes a job from the job data list.
*
* @param jid JobId. Makes up the primary key together with startTs.
* @param startTs Start timestamp of the job. Part of the primary key.
* @return See JDError.
* @param jid JobId. Makes up the primary key together with startTs.
* @param startTs Start timestamp of the job. Part of the primary key.
* @param domainId Domain ID of the job (optional).
* @return See JDError.
*/
JDError deleteJob(JobId jid, TimeStamp startTs);
JDError deleteJob(JobId jid, TimeStamp startTs, std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Retrieve a job by its primary key.
*
* @param job Reference to a JobData object that will be populated
* with the job data.
* @param jid Id of the job to be retrieved. Makes up the primary key
* together with startTs.
* @param startTs Start time of the job. Part of the primary key.
* @return See JDError.
* @param job Reference to a JobData object that will be populated
* with the job data.
* @param jid Id of the job to be retrieved. Makes up the primary key
* together with startTs.
* @param startTs Start time of the job. Part of the primary key.
* @param domainId Domain ID of the job (optional).
* @return See JDError.
*/
JDError getJobByPrimaryKey(JobData& job, JobId jid, TimeStamp startTs);
JDError getJobByPrimaryKey(JobData& job, JobId jid, TimeStamp startTs, std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Retrieve the most recent job with jid.
*
* @param job Reference to a JobData object that will be populated with
* the job data.
* @param jid Id of the job whose information should be retrieved. If
* multiple jobs with the same jid are present the most
* recent one is returned.
* @return See JDError.
* @param job Reference to a JobData object that will be populated with
* the job data.
* @param jid Id of the job whose information should be retrieved. If
* multiple jobs with the same jid are present the most
* recent one is returned.
* @param domainId Domain ID of the job (optional).
* @return See JDError.
*/
JDError getJobById(JobData& job, JobId jid);
JDError getJobById(JobData& job, JobId jid, std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Retrieve an exclusive list of jobs which were run in the given
......@@ -175,11 +200,13 @@ namespace DCDB {
* populated with the jobs.
* @param intervalStart Start time of the interval.
* @param intervalEnd End time of the interval.
* @param domainId Domain ID of the job (optional).
* @return See JDError.
*/
JDError getJobsInIntervalExcl(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
/**
......@@ -194,11 +221,13 @@ namespace DCDB {
* populated with the jobs.
* @param intervalStart Start time of the interval.
* @param intervalEnd End time of the interval.
* @param domainId Domain ID of the job (optional).
* @return See JDError.
*/
JDError getJobsInIntervalIncl(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Retrieve a list of jobs which were run in the given time interval.
......@@ -211,11 +240,13 @@ namespace DCDB {
* populated with the jobs.
* @param intervalStart Start time of the interval.
* @param intervalEnd End time of the interval.
* @param domainId Domain ID of the job (optional).
* @return See JDError.
*/
JDError getJobsInIntervalRunning(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Retrieve a list of jobs which terminated in the given time interval.
......@@ -227,11 +258,13 @@ namespace DCDB {
* populated with the jobs.
* @param intervalStart Start time of the interval.
* @param intervalEnd End time of the interval.
* @param domainId Domain ID of the job (optional).
* @return See JDError.
*/
JDError getJobsInIntervalFinished(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Retrieve an inclusive list of jobs which were in pending state in the given
......@@ -245,11 +278,13 @@ namespace DCDB {
* populated with the jobs.
* @param intervalStart Start time of the interval.
* @param intervalEnd End time of the interval.
* @param domainId Domain ID of the job (optional).
* @return See JDError.
*/
JDError getJobsInIntervalPending(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
/**
* @brief Retrieve the list of nodes which were used by a job.
......
......@@ -119,6 +119,20 @@ protected:
*/
void createColumnFamily(std::string name, std::string fields, std::string primaryKey, std::string options = "");
/**
* @brief Create a new materialized view in the
* selected keyspace.
*
* @param name The name of the materialized view
* @param select A comma-separated list of columns from the original table
* @param fromTable Name of the original table from which the view must be created
* @param where List of selection clauses (usually of the IS NOT NULL type)
* @param primaryKey A primary key definition (one or more fields)
* @param options A Cassandra WITH statement for keyspace generation
*/
void createMaterializedView(std::string name, std::string select, std::string fromTable,
std::string where, std::string primaryKey, std::string options);
public:
/**
......
......@@ -55,5 +55,6 @@
#define JD_KEYSPACE_NAME KEYSPACE_NAME "_jobdata"
#define CF_JOBDATA "jobdata"
#define CF_JOBDATAVIEW "jobdata_bytime"
#endif /* DCDB_GLOBALS_H */
......@@ -67,26 +67,33 @@ namespace DCDB {
JDError insertJob(JobData& jdata);
JDError updateJob(JobData& jdata);
JDError updateEndtime(JobId jobId, TimeStamp startTs, TimeStamp endTime);
JDError deleteJob(JobId jid, TimeStamp startTs);
JDError updateEndtime(JobId jobId, TimeStamp startTs, TimeStamp endTime, std::string domainId=JOB_DEFAULT_DOMAIN);
JDError updateStartTime(JobId jobId, TimeStamp startTs, TimeStamp newStartTs, std::string domainId=JOB_DEFAULT_DOMAIN);
JDError deleteJob(JobId jid, TimeStamp startTs, std::string domainId=JOB_DEFAULT_DOMAIN);
JDError getJobByPrimaryKey(JobData& job, JobId jid, TimeStamp startTs);
JDError getJobById(JobData& job, JobId jid);
JDError getJobByPrimaryKey(JobData& job, JobId jid, TimeStamp startTs, std::string domainId=JOB_DEFAULT_DOMAIN);
JDError getJobById(JobData& job, JobId jid, std::string domainId=JOB_DEFAULT_DOMAIN);
JDError getJobsInIntervalExcl(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
JDError getJobsInIntervalIncl(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
JDError getJobsInIntervalRunning(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalStart,
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
JDError getJobsInIntervalFinished(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalStart,
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
JDError getJobsInIntervalPending(std::list<JobData>& jobs,
TimeStamp intervalStart,
TimeStamp intervalEnd);
TimeStamp intervalStart,
TimeStamp intervalEnd,
std::string domainId=JOB_DEFAULT_DOMAIN);
JDError getNodeList(NodeList& nodes, JobId jid, TimeStamp startTs);
JobDataStoreImpl(Connection* conn);
......
......@@ -288,6 +288,29 @@ void ConnectionImpl::createColumnFamily(std::string name, std::string fields, st
executeSimpleQuery(query.str());
}
/**
* @details
* This functions assembles the parameters into a CQL CREATE
* TABLE query and submits the query to the server.
*/
void ConnectionImpl::createMaterializedView(std::string name, std::string select, std::string fromTable,
std::string where, std::string primaryKey, std::string options)
{
std::stringstream query;
query << "CREATE MATERIALIZED VIEW " << name << " AS"
<< " SELECT " << select << " FROM " << fromTable
<< " WHERE " << where << " PRIMARY KEY (" << primaryKey << ")";
if (options != "") {
query << " WITH " << options;
}
query << ";";
executeSimpleQuery(query.str());
}
/*
* Definitions of the public ConnectionImpl class functions
*/
......@@ -547,15 +570,26 @@ bool ConnectionImpl::initSchema() {
if (!existsColumnFamily(CF_JOBDATA)) {
std::cout << "Creating Column Family " CF_JOBDATA "...\n";
createColumnFamily(CF_JOBDATA,
"domain varchar, " /* Job Domain */
"jid varchar, " /* Job Id */
"uid varchar, " /* User Id */
"start_ts bigint, " /* Start timestamp of the job */
"end_ts bigint, " /* End timestamp of the job */
"nodes set<varchar>", /* Set of nodes used by the job */
"jid, start_ts" /* Make jid + start_ts columns the primary key*/
/* Together they should be unique */
); /* No further options required */
"domain, jid, start_ts", /* Make start_ts + jid columns the primary key */
"CLUSTERING ORDER BY (jid DESC, start_ts DESC)" /* Setting the sorting criteria - recent jobs first */
);
std::cout << "Creating Materialized View " CF_JOBDATAVIEW "...\n";
createMaterializedView(CF_JOBDATAVIEW,
"domain,jid,start_ts,end_ts,uid,nodes", /* Picking all job data fields */
"jobdata", /* From the jobdata table */
"domain IS NOT NULL AND end_ts IS NOT NULL AND start_ts IS NOT NULL and jid IS NOT NULL", /* Primary key components are not null */
"domain,end_ts,start_ts,jid", /* We use end_ts in the primary key to improve selectivity in range queries */
"CLUSTERING ORDER BY (end_ts DESC, start_ts DESC)" /* Using descending order to get more recent jobs */
);
}
return true;
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment