Commit 66710f33 authored by Michael Ott's avatar Michael Ott
Browse files

Allow for specifiying the timestamp column and the columns to be imported

parent 33071ef1
......@@ -35,6 +35,10 @@
#include <boost/tokenizer.hpp>
#include <algorithm>
#include <exception>
#include <set>
#include <map>
#include <string>
#include <pthread.h>
......@@ -53,9 +57,11 @@ typedef struct {
void usage(int argc, char* argv[])
{
std::cout << "Usage: " << argv[0] << " [-h <host>] <CSV File> <SensorPrefix>" << std::endl << std::endl;
std::cout << " -h <host> - Database hostname" << std::endl;
std::cout << " CSV File - CSV file with sensor readings. First row has to contain sensor names" << std::endl;
std::cout << "Usage: " << argv[0] << " [-h <host>] [-t <col>] [-c <col[,col,col]>] <CSV File> <SensorPrefix>" << std::endl << std::endl;
std::cout << " -h <host> - Database hostname" << std::endl;
std::cout << " -t <col> - Column in the CSV that contains the timestamp [default: 0]" << std::endl;
std::cout << " -c <col[,col,col]> - Column in the CSV to use [default: all]" << std::endl;
std::cout << " CSV File - CSV file with sensor readings. First row has to contain sensor names" << std::endl;
std::cout << " SensorPrefix - Prefix to use for sensor names" << std::endl;
}
......@@ -73,15 +79,27 @@ int main(int argc, char** argv)
host = "localhost";
}
int indexOffset = 0;
while ((ret=getopt(argc, argv, "+h:i:"))!=-1) {
int offset = 0;
int tsColumn = 0;
std::set<int> columns;
while ((ret=getopt(argc, argv, "+h:c:o:t:"))!=-1) {
switch(ret) {
case 'h':
host = optarg;
break;
case 'i':
indexOffset = atoi(optarg);
std::cout << "i: " << optarg << std::endl;
case 'c': {
std::string s(optarg);
boost::tokenizer<boost::escaped_list_separator<char> > tk(s, boost::escaped_list_separator<char>('\\', ',', '\"'));
for (boost::tokenizer<boost::escaped_list_separator<char> >::iterator i=tk.begin(); i!=tk.end();++i)
columns.insert(std::stoi(*i));
} break;
case 'o':
offset = atoi(optarg);
std::cout << ret << ": " << optarg << std::endl;
break;
case 't':
tsColumn = atoi(optarg);
std::cout << ret << ": " << optarg << std::endl;
break;
default:
usage(argc, argv);
......@@ -108,20 +126,34 @@ int main(int argc, char** argv)
std::ifstream fs;
std::string s;
std::vector<std::string> vec;
std::vector<sensor_t> sensors;
std::map<int,sensor_t> sensors;
std::string csvFilename = argv[optind];
std::string prefix = argv[optind+1];
int suffixLen = 32-(prefix.size()-std::count(prefix.begin(), prefix.end(), '/'));
uint64_t lineno = 0;
/* Read header line from CSV to obtain sensor names and topics */
std::cout << std::endl;
std::cout << "Parsing CSV file: " << csvFilename << std::endl;
std::cout << "Using sensor name prefix: " << prefix << std::endl;
std::cout << "Columns:";
if (columns.size()) {
std::set<int>::iterator it;
for (it=columns.begin(); it!=columns.end(); ++it) {
std::cout << " " << *it;
}
std::cout << std::endl;
} else {
std::cout << "all" << std::endl;
}
std::cout << "Timestamp Column: " << tsColumn << std::endl;
fs.open (csvFilename, std::fstream::in);
std::getline(fs, s);
lineno++;
boost::tokenizer<boost::escaped_list_separator<char> > tk(s, boost::escaped_list_separator<char>('\\', ',', '\"'));
int topics = indexOffset;
int topics = offset;
int col = 0;
for (boost::tokenizer<boost::escaped_list_separator<char> >::iterator i=tk.begin(); i!=tk.end();++i)
{
if (i == tk.begin()) {
......@@ -129,54 +161,53 @@ int main(int argc, char** argv)
}
sensor_t sensor;
sensor.name = *i;
std::stringstream ss;
if (prefix.size() < 12) {
ss << std::setfill('0') << std::setw((12-prefix.size())*2) << "";
}
const char *c = prefix.substr(0, 12).c_str();
while (*c) {
uint32_t x = *c++;
ss << std::setfill('0') << std::setw(2) << std::hex << x;
}
ss << "/" << std::setfill('0') << std::setw(8) << std::hex << topics;
sensor.topic = ss.str();
ss << std::setfill('0') << std::setw(suffixLen) << std::hex << topics;;
sensor.topic = prefix + ss.str();
sensor.publicName = prefix + "." + sensor.name;
std::replace(sensor.publicName.begin(), sensor.publicName.end(), ' ', '_');
sensors.push_back(sensor);
sensors.insert(std::pair<int,sensor_t>(col, sensor));
topics++;
}
/* Read actual sensor readings */
while (std::getline(fs, s)) {
int col = 0;
lineno++;
col = 0;
DCDB::TimeStamp ts(UINT64_C(0));
boost::tokenizer<boost::escaped_list_separator<char> > tk(s, boost::escaped_list_separator<char>('\\', ',', '\"'));
for (boost::tokenizer<boost::escaped_list_separator<char> >::iterator i=tk.begin(); i!=tk.end();++i) {
if (col == tsColumn) {
ts = DCDB::TimeStamp(*i);
}
col++;
}
col = 0;
for (boost::tokenizer<boost::escaped_list_separator<char> >::iterator i=tk.begin(); i!=tk.end();++i)
{
if (0 == col) {
ts = DCDB::TimeStamp(*i);
} else {
// std::cout << ts.getRaw() << " " << sensors[col-1].topic << ":" << *i << std::endl;
try {
DCDB::SensorId sid(sensors[col-1].topic);
sensorDataStore.insert(&sid, ts.getRaw(), std::stoll(*i));
}
catch (std::exception &e) {
std::cerr << "Error parsing CSV line " << lineno << " column " << col+1 << ": \"" << *i << "\"" << std::endl;
}
}
if ((columns.size() == 0) || (columns.find(col) != columns.end())) {
std::cout << ts.getRaw() << " " << col << " " << sensors[col].topic << " " << *i << std::endl;
try {
DCDB::SensorId sid(sensors[col-1].topic);
sensorDataStore.insert(&sid, ts.getRaw(), std::stoll(*i));
}
catch (std::exception &e) {
std::cerr << "Error parsing CSV line " << lineno << " column " << col+1 << ": \"" << *i << "\"" << std::endl;
}
}
col++;
}
}
fs.close();
/* Create public sensor names */
/* Create public sensor names
std::cout << std::endl;
std::cout << "Publishing sensors..." << std::endl;
std::vector<sensor_t>::iterator it;
for (it = sensors.begin(); it != sensors.end(); it++) {
std::cout << it->name << " " << it->topic << " " << it->publicName << std::endl;
sensorConfig.publishSensor(it->publicName.c_str(), it->topic.c_str());
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment