Allow for specifiying the timestamp column and the columns to be imported

tools/dcdbcsvimport/dcdbcsvimport.cpp

@@ -35,6 +35,10 @@
 #include <boost/tokenizer.hpp>
 #include <algorithm>
 #include <exception>
+#include <set>
+#include <map>
+#include <string>
+
 
 #include <pthread.h>
 
@@ -53,9 +57,11 @@ typedef struct {
 
 void usage(int argc, char* argv[])
 {
-  std::cout << "Usage: " << argv[0] << " [-h <host>] <CSV File> <SensorPrefix>" << std::endl << std::endl;
-  std::cout << "    -h <host>      - Database hostname" << std::endl;
-  std::cout << "    CSV File       - CSV file with sensor readings. First row has to contain sensor names" << std::endl;
+  std::cout << "Usage: " << argv[0] << " [-h <host>] [-t <col>] [-c <col[,col,col]>] <CSV File> <SensorPrefix>" << std::endl << std::endl;
+  std::cout << "    -h <host>           - Database hostname" << std::endl;
+  std::cout << "    -t <col>            - Column in the CSV that contains the timestamp [default: 0]" << std::endl;
+  std::cout << "    -c <col[,col,col]>  - Column in the CSV to use [default: all]" << std::endl;
+  std::cout << "    CSV File            - CSV file with sensor readings. First row has to contain sensor names" << std::endl;
   std::cout << "    SensorPrefix   - Prefix to use for sensor names" << std::endl;
 }
 
@@ -73,15 +79,27 @@ int main(int argc, char** argv)
       host = "localhost";
   }
 
-  int indexOffset = 0;
-  while ((ret=getopt(argc, argv, "+h:i:"))!=-1) {
+  int offset = 0;
+  int tsColumn = 0;
+  std::set<int> columns;
+  while ((ret=getopt(argc, argv, "+h:c:o:t:"))!=-1) {
       switch(ret) {
           case 'h':
               host = optarg;
               break;
-          case 'i':
-              indexOffset = atoi(optarg);
-		std::cout << "i: " << optarg << std::endl;
+          case 'c': {
+        	  std::string s(optarg);
+        	  boost::tokenizer<boost::escaped_list_separator<char> > tk(s, boost::escaped_list_separator<char>('\\', ',', '\"'));
+        	  for (boost::tokenizer<boost::escaped_list_separator<char> >::iterator i=tk.begin(); i!=tk.end();++i)
+        		  columns.insert(std::stoi(*i));
+        	  } break;
+          case 'o':
+              offset = atoi(optarg);
+              std::cout << ret << ": " << optarg << std::endl;
+              break;
+          case 't':
+              tsColumn = atoi(optarg);
+              std::cout << ret << ": " << optarg << std::endl;
               break;
           default:
               usage(argc, argv);
@@ -108,20 +126,34 @@ int main(int argc, char** argv)
   std::ifstream fs;
   std::string s;
   std::vector<std::string> vec;
-  std::vector<sensor_t> sensors;
+  std::map<int,sensor_t> sensors;
   std::string csvFilename = argv[optind];
   std::string prefix = argv[optind+1];
+  int suffixLen = 32-(prefix.size()-std::count(prefix.begin(), prefix.end(), '/'));
   uint64_t lineno = 0;
 
   /* Read header line from CSV to obtain sensor names and topics */
   std::cout << std::endl;
   std::cout << "Parsing CSV file: " << csvFilename << std::endl;
   std::cout << "Using sensor name prefix: " << prefix << std::endl;
+  std::cout << "Columns:";
+  if (columns.size()) {
+	  std::set<int>::iterator it;
+	  for (it=columns.begin(); it!=columns.end(); ++it) {
+	      std::cout << " " << *it;
+	  }
+	  std::cout << std::endl;
+  } else {
+	  std::cout << "all" << std::endl;
+  }
+  std::cout << "Timestamp Column: " << tsColumn << std::endl;
+
   fs.open (csvFilename, std::fstream::in);
   std::getline(fs, s);
   lineno++;
   boost::tokenizer<boost::escaped_list_separator<char> > tk(s, boost::escaped_list_separator<char>('\\', ',', '\"'));
-  int topics = indexOffset;
+  int topics = offset;
+  int col = 0;
   for (boost::tokenizer<boost::escaped_list_separator<char> >::iterator i=tk.begin(); i!=tk.end();++i) 
   {
 	  if (i == tk.begin()) {
@@ -129,54 +161,53 @@ int main(int argc, char** argv)
 	  }
 	sensor_t sensor;
 	sensor.name = *i;
+
 	std::stringstream ss;
-	if (prefix.size() < 12) {
-		ss << std::setfill('0') << std::setw((12-prefix.size())*2) << "";
-	}
-	const char *c = prefix.substr(0, 12).c_str();
-	while (*c) {
-		uint32_t x = *c++;
-		ss << std::setfill('0') << std::setw(2) << std::hex << x;
-	}
-	ss << "/" << std::setfill('0') << std::setw(8) << std::hex << topics;
-	sensor.topic = ss.str();
+	ss << std::setfill('0') << std::setw(suffixLen) << std::hex << topics;;
+	sensor.topic = prefix + ss.str();
 	sensor.publicName = prefix + "." + sensor.name;
 	std::replace(sensor.publicName.begin(), sensor.publicName.end(), ' ', '_');
-	sensors.push_back(sensor);
+	sensors.insert(std::pair<int,sensor_t>(col, sensor));
     topics++;
   }
-  
+
   /* Read actual sensor readings */
   while (std::getline(fs, s)) {
-    int col = 0;
     lineno++;
+
+    col = 0;
 	DCDB::TimeStamp ts(UINT64_C(0));
 	boost::tokenizer<boost::escaped_list_separator<char> > tk(s, boost::escaped_list_separator<char>('\\', ',', '\"'));
+    for (boost::tokenizer<boost::escaped_list_separator<char> >::iterator i=tk.begin(); i!=tk.end();++i) {
+    	if (col == tsColumn) {
+    		ts = DCDB::TimeStamp(*i);
+    	}
+    	col++;
+    }
+
+    col = 0;
     for (boost::tokenizer<boost::escaped_list_separator<char> >::iterator i=tk.begin(); i!=tk.end();++i) 
     {
-		if (0 == col) {
-			ts = DCDB::TimeStamp(*i);
-		} else {
-			// std::cout << ts.getRaw() << " " << sensors[col-1].topic << ":" << *i << std::endl;
-			try {
-				DCDB::SensorId sid(sensors[col-1].topic);
-				sensorDataStore.insert(&sid, ts.getRaw(), std::stoll(*i));
-			}
-			catch (std::exception &e) {
-				std::cerr << "Error parsing CSV line " << lineno << " column " << col+1 << ": \"" << *i << "\"" << std::endl;
-			}
-		}
+  		if ((columns.size() == 0) || (columns.find(col) != columns.end())) {
+   			std::cout << ts.getRaw() << " " << col << " " << sensors[col].topic << " " << *i << std::endl;
+   			try {
+   				DCDB::SensorId sid(sensors[col-1].topic);
+   				sensorDataStore.insert(&sid, ts.getRaw(), std::stoll(*i));
+   			}
+   			catch (std::exception &e) {
+   				std::cerr << "Error parsing CSV line " << lineno << " column " << col+1 << ": \"" << *i << "\"" << std::endl;
+   			}
+   		}
 		col++;
     }	  
   }
   
   fs.close();
 
-  /* Create public sensor names */
+  /* Create public sensor names
   std::cout << std::endl;
   std::cout << "Publishing sensors..." << std::endl;
 
-  std::vector<sensor_t>::iterator it;
   for (it = sensors.begin(); it != sensors.end(); it++) {
 	  std::cout << it->name << " " << it->topic << " " << it->publicName << std::endl;
 	  sensorConfig.publishSensor(it->publicName.c_str(), it->topic.c_str());