Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 6cfb09d3 authored by Micha Mueller's avatar Micha Mueller
Browse files

Add first mockup for upcoming functionality to store SLURM job data

parent 6ac6d9ee
......@@ -20,3 +20,13 @@ cscope.out
## Ignore editor temp files
*~
## Ignore binaries and other build files
CollectAgent/collectagent
lib/.header-check
lib/libdcdb.so
tools/dcdbconfig/dcdbconfig
tools/dcdbcsvimport/dcdbcsvimport
tools/dcdbquery/dcdbquery
tools/dcdbquerysum/dcdbquerysum
tools/dcdbunitconv/dcdbunitconv
//================================================================================
// Name : jobdatastore.h
// Author : Axel Auweter, Micha Mueller
// Copyright : Leibniz Supercomputing Centre
// Description : C++ API for inserting and querying DCDB job data.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2011-2018 Leibniz Supercomputing Centre
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//================================================================================
/**
* @file
* @brief This file contains parts of the public API for the libdcdb library.
* It contains the class definition of the JobDataStore class, that handles
* database operations for job data.
*/
#ifndef DCDB_JOBDATASTORE_H
#define DCDB_JOBDATASTORE_H
#include <cstdint>
#include <list>
#include <string>
#include "cassandra.h"
#include "connection.h"
#include "timestamp.h"
namespace DCDB {
/* Forward-declaration of the implementation-internal classes */
class JobDataStoreImpl;
using JobId = uint32_t;
using UserId = uint32_t;
using NodeList = std::list<std::string>;
/**
* @brief This class is a container for the information DCDB keeps about
* SLURM jobs.
*/
class JobData {
public:
JobId jobId; /**< SLURM job id of the job. */
TimeStamp startTime;/**< Time when the job started (started != submitted) */
TimeStamp endTime; /**< Time when the job finished. */
NodeList nodes; /**< List of nodes the job occupied. */
UserId userId; /**< Id of the user who submitted the job. */
//TODO
};
typedef enum {
JD_OK,
JD_UNKNOWNERROR
//TODO
} JDError;
/**
* @brief JobDataStore is the class of the libdcdb library
* to write and read job data.
*/
class JobDataStore {
private:
JobDataStoreImpl* impl;
public:
/**
* @brief This function inserts a single job into the database.
* @param jid Id of the job.
* @param startTs The starting timestamp of the job.
* @param endTs The end timestamp of the job.
* @param nodes List of nodes which were used by the job.
* @param uid Id of the user the job belongs to.
*/
JDError insertJob(JobId jid, TimeStamp startTs, TimeStamp endTs,
NodeList nodes, UserId uid);
/**
* @brief This function inserts a single job which was submitted to SLURM
* but not yet executed into the database.
* @param jid Id of the job.
* @param uid Id of the user the job belongs to.
*/
JDError insertSubmittedJob(JobId jid, UserId uid);
/**
* @brief Removes a job from the job data list.
*
* @param jid JobId.
* @return See JDError.
*/
JDError removeJob(JobId jid);
/**
* @brief Get the entire list of jobs.
*
* @param jobData Reference to a list of JobData that will be populated
* with the jobs.
* @return See JDError.
*/
JDError getJobs(std::list<JobData>& jobs);
/**
* @brief Retrieve a job by id.
*
* @param job Reference to a JobData object that will be populated with
* the job data.
* @param jid Id of the job whose information should be retrieved.
* @return See JDError.
*/
JDError getJobById(JobData& job, JobId jid);
/**
* @brief Retrieve a list of jobs which belong to a user.
*
* @param jobs Reference to a list of JobData that will be populated with
* the jobs.
* @param uid Id of the user whose jobs should be retrieved.
* @return See JDError.
*/
JDError getJobsByUser(std::list<JobData>& jobs, UserId uid);
/**
* @brief Retrieve the list of nodes which were used by a job.
*
* @param nodes
* @param jid
* @return
*/
JDError getNodeList(NodeList& nodes, JobId jid);
/**
* @brief Set the startTime for a job. (Intended for jobs which were
* inserted via insertSubmittedJob())
* @param jid SLURM id of the job.
* @param startTime New starting time of the job.
* @return See JDError.
*/
JDError setStartTime(JobId jid, TimeStamp startTime);
/**
* @brief Set the endTime for a job. (Intended for jobs which were
* inserted via insertSubmittedJob())
* @param jid SLURM id of the job.
* @param endTime New end time of the job.
* @return See JDError.
*/
JDError setEndTime(JobId jid, TimeStamp endTime);
/**
* @brief Set the list of nodes for a job. (Intended for jobs which were
* inserted via insertSubmittedJob())
* @param jid SLURM id of the job.
* @param nodes List of nodes which were used by the job.
* @return See JDError.
*/
JDError setNodeList(JobId jid, NodeList nodes);
/**
* @brief A shortcut constructor for a JobDataStore object that allows
* accessing the data store through a connection that is already
* established.
* @param conn The Connection object of an established connection to
* Cassandra.
*/
JobDataStore(Connection* conn);
/**
* @brief The standard destructor for a JobDatStore object.
*/
virtual ~JobDataStore();
};
} /* End of namespace DCDB */
#endif /* DCDB_JOBDATASTORE_H */
......@@ -41,4 +41,6 @@
#define CF_PUBLISHEDSENSORS "publishedsensors"
#define CF_VIRTUALSENSORS "virtualsensors"
#define JD_KEYSPACE_NAME KEYSPACE_NAME "_jobdata"
#endif /* DCDB_GLOBALS_H */
//================================================================================
// Name : jobdatastore_internal.h
// Author : Axel Auweter, Micha Mueller
// Copyright : Leibniz Supercomputing Centre
// Description : Internal interface for inserting and querying DCDB job data.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2011-2018 Leibniz Supercomputing Centre
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//================================================================================
/*
* @file
* @brief This file contains the internal functions of the
* Job Data Store which are provided by the
* JobDataStoreImpl class.
*/
#ifndef DCDB_JOBDATASTORE_INTERNAL_H
#define DCDB_JOBDATASTORE_INTERNAL_H
#include <list>
#include <string>
#include "dcdb/jobdatastore.h"
#include "dcdb/connection.h"
#include "dcdb/timestamp.h"
namespace DCDB {
/**
* @brief The JobDataStoreImpl class contains all protected
* functions belonging to JobDataStore which are
* hidden from the user of the libdcdb library.
*/
class JobDataStoreImpl {
protected:
Connection* connection; /**< The Connection object that does the low-level stuff for us. */
CassSession* session; /**< The CassSession object given by the connection. */
public:
/* See jobdatastore.h for documentation */
JDError insertJob(JobId jid, TimeStamp startTs, TimeStamp endTs,
NodeList nodes, UserId uid);
JDError insertSubmittedJob(JobId jid, UserId uid);
JDError removeJob(JobId jid);
JDError getJobs(std::list<JobData>& jobs);
JDError getJobById(JobData& job, JobId jid);
JDError getJobsByUser(std::list<JobData>& jobs, UserId uid);
JDError getNodeList(NodeList& nodes, JobId jid);
JDError setStartTime(JobId jid, TimeStamp startTime);
JDError setEndTime(JobId jid, TimeStamp endTime);
JDError setNodeList(JobId jid, NodeList nodes);
JobDataStoreImpl(Connection* conn);
virtual ~JobDataStoreImpl();
};
} /* End of namespace DCDB */
#endif /* DCDB_JOBDATASTORE_INTERNAL_H */
//================================================================================
// Name : jobdatastore.cpp
// Author : Axel Auweter, Micha Mueller
// Copyright : Leibniz Supercomputing Centre
// Description : C++ API implementation for inserting and querying DCDB job data.
//================================================================================
//================================================================================
// This file is part of DCDB (DataCenter DataBase)
// Copyright (C) 2011-2016 Leibniz Supercomputing Centre
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//================================================================================
/**
* //TODO documentary
*/
#include <list>
#include <string>
#include "cassandra.h"
#include "dcdb/jobdatastore.h"
#include "jobdatastore_internal.h"
#include "dcdb/connection.h"
#include "dcdbglobals.h"
using namespace DCDB;
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::insertJob(JobId jid, TimeStamp startTs, TimeStamp endTs,
NodeList nodes, UserId uid) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::insertSubmittedJob(JobId jid, UserId uid) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::removeJob(JobId jid) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::getJobs(std::list<JobData>& jobs) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::getJobById(JobData& job, JobId jid) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::getJobsByUser(std::list<JobData>& jobs, UserId uid) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::getNodeList(NodeList& nodes, JobId jid) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::setStartTime(JobId jid, TimeStamp startTime) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::setEndTime(JobId jid, TimeStamp endTime) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* //TODO
*/
JDError JobDataStoreImpl::setNodeList(JobId jid, NodeList nodes) {
//TODO
return JD_UNKNOWNERROR;
}
/**
* @details
* This constructor sets the internal connection variable to
* the externally provided Connection object and also
* retrieves the CassSession pointer of the connection.
*/
JobDataStoreImpl::JobDataStoreImpl(Connection* conn) {
connection = conn;
session = connection->getSessionHandle();
}
/**
* @details
* The destructor just resets the internal pointers. Deletion of the pointers
* is not our responsibility.
*/
JobDataStoreImpl::~JobDataStoreImpl() {
connection = nullptr;
session = nullptr;
}
/* ########################################################################## */
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::insertJob(JobId jid, TimeStamp startTs, TimeStamp endTs,
NodeList nodes, UserId uid) {
return impl->insertJob(jid, startTs, endTs, nodes, uid);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::insertSubmittedJob(JobId jid, UserId uid) {
return impl->insertSubmittedJob(jid, uid);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::removeJob(JobId jid) {
return impl->removeJob(jid);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::getJobs(std::list<JobData>& jobs) {
return impl->getJobs(jobs);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::getJobById(JobData& job, JobId jid) {
return impl->getJobById(job, jid);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::getJobsByUser(std::list<JobData>& jobs, UserId uid) {
return impl->getJobsByUser(jobs, uid);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::getNodeList(NodeList& nodes, JobId jid) {
return impl->getNodeList(nodes, jid);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::setStartTime(JobId jid, TimeStamp startTime) {
return impl->setStartTime(jid, startTime);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::setEndTime(JobId jid, TimeStamp endTime) {
return impl->setEndTime(jid, endTime);
}
/**
* @details
* Instead of doing the actual work, this function simply forwards to the
* corresponding function of the JobDataStoreImpl class.
*/
JDError JobDataStore::setNodeList(JobId jid, NodeList nodes) {
return impl->setNodeList(jid, nodes);
}
/**
* @details
* This constructor allocates the implementation class which
* holds the actual implementation of the class functionality.
*/
JobDataStore::JobDataStore(Connection* conn) {
impl = new JobDataStoreImpl(conn);
}
/**
* @details
* The JobDataStore desctructor deallocates the
* JobDataStoreImpl and CassandraBackend objects.
*/
JobDataStore::~JobDataStore() {
/* Clean up... */
if (impl) {
delete impl;
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment