Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
dcdb
dcdb
Commits
66710f33
Commit
66710f33
authored
Dec 07, 2017
by
Michael Ott
Browse files
Allow for specifiying the timestamp column and the columns to be imported
parent
33071ef1
Changes
1
Hide whitespace changes
Inline
Side-by-side
tools/dcdbcsvimport/dcdbcsvimport.cpp
View file @
66710f33
...
...
@@ -35,6 +35,10 @@
#include
<boost/tokenizer.hpp>
#include
<algorithm>
#include
<exception>
#include
<set>
#include
<map>
#include
<string>
#include
<pthread.h>
...
...
@@ -53,9 +57,11 @@ typedef struct {
void
usage
(
int
argc
,
char
*
argv
[])
{
std
::
cout
<<
"Usage: "
<<
argv
[
0
]
<<
" [-h <host>] <CSV File> <SensorPrefix>"
<<
std
::
endl
<<
std
::
endl
;
std
::
cout
<<
" -h <host> - Database hostname"
<<
std
::
endl
;
std
::
cout
<<
" CSV File - CSV file with sensor readings. First row has to contain sensor names"
<<
std
::
endl
;
std
::
cout
<<
"Usage: "
<<
argv
[
0
]
<<
" [-h <host>] [-t <col>] [-c <col[,col,col]>] <CSV File> <SensorPrefix>"
<<
std
::
endl
<<
std
::
endl
;
std
::
cout
<<
" -h <host> - Database hostname"
<<
std
::
endl
;
std
::
cout
<<
" -t <col> - Column in the CSV that contains the timestamp [default: 0]"
<<
std
::
endl
;
std
::
cout
<<
" -c <col[,col,col]> - Column in the CSV to use [default: all]"
<<
std
::
endl
;
std
::
cout
<<
" CSV File - CSV file with sensor readings. First row has to contain sensor names"
<<
std
::
endl
;
std
::
cout
<<
" SensorPrefix - Prefix to use for sensor names"
<<
std
::
endl
;
}
...
...
@@ -73,15 +79,27 @@ int main(int argc, char** argv)
host
=
"localhost"
;
}
int
indexOffset
=
0
;
while
((
ret
=
getopt
(
argc
,
argv
,
"+h:i:"
))
!=-
1
)
{
int
offset
=
0
;
int
tsColumn
=
0
;
std
::
set
<
int
>
columns
;
while
((
ret
=
getopt
(
argc
,
argv
,
"+h:c:o:t:"
))
!=-
1
)
{
switch
(
ret
)
{
case
'h'
:
host
=
optarg
;
break
;
case
'i'
:
indexOffset
=
atoi
(
optarg
);
std
::
cout
<<
"i: "
<<
optarg
<<
std
::
endl
;
case
'c'
:
{
std
::
string
s
(
optarg
);
boost
::
tokenizer
<
boost
::
escaped_list_separator
<
char
>
>
tk
(
s
,
boost
::
escaped_list_separator
<
char
>
(
'\\'
,
','
,
'\"'
));
for
(
boost
::
tokenizer
<
boost
::
escaped_list_separator
<
char
>
>::
iterator
i
=
tk
.
begin
();
i
!=
tk
.
end
();
++
i
)
columns
.
insert
(
std
::
stoi
(
*
i
));
}
break
;
case
'o'
:
offset
=
atoi
(
optarg
);
std
::
cout
<<
ret
<<
": "
<<
optarg
<<
std
::
endl
;
break
;
case
't'
:
tsColumn
=
atoi
(
optarg
);
std
::
cout
<<
ret
<<
": "
<<
optarg
<<
std
::
endl
;
break
;
default:
usage
(
argc
,
argv
);
...
...
@@ -108,20 +126,34 @@ int main(int argc, char** argv)
std
::
ifstream
fs
;
std
::
string
s
;
std
::
vector
<
std
::
string
>
vec
;
std
::
vector
<
sensor_t
>
sensors
;
std
::
map
<
int
,
sensor_t
>
sensors
;
std
::
string
csvFilename
=
argv
[
optind
];
std
::
string
prefix
=
argv
[
optind
+
1
];
int
suffixLen
=
32
-
(
prefix
.
size
()
-
std
::
count
(
prefix
.
begin
(),
prefix
.
end
(),
'/'
));
uint64_t
lineno
=
0
;
/* Read header line from CSV to obtain sensor names and topics */
std
::
cout
<<
std
::
endl
;
std
::
cout
<<
"Parsing CSV file: "
<<
csvFilename
<<
std
::
endl
;
std
::
cout
<<
"Using sensor name prefix: "
<<
prefix
<<
std
::
endl
;
std
::
cout
<<
"Columns:"
;
if
(
columns
.
size
())
{
std
::
set
<
int
>::
iterator
it
;
for
(
it
=
columns
.
begin
();
it
!=
columns
.
end
();
++
it
)
{
std
::
cout
<<
" "
<<
*
it
;
}
std
::
cout
<<
std
::
endl
;
}
else
{
std
::
cout
<<
"all"
<<
std
::
endl
;
}
std
::
cout
<<
"Timestamp Column: "
<<
tsColumn
<<
std
::
endl
;
fs
.
open
(
csvFilename
,
std
::
fstream
::
in
);
std
::
getline
(
fs
,
s
);
lineno
++
;
boost
::
tokenizer
<
boost
::
escaped_list_separator
<
char
>
>
tk
(
s
,
boost
::
escaped_list_separator
<
char
>
(
'\\'
,
','
,
'\"'
));
int
topics
=
indexOffset
;
int
topics
=
offset
;
int
col
=
0
;
for
(
boost
::
tokenizer
<
boost
::
escaped_list_separator
<
char
>
>::
iterator
i
=
tk
.
begin
();
i
!=
tk
.
end
();
++
i
)
{
if
(
i
==
tk
.
begin
())
{
...
...
@@ -129,54 +161,53 @@ int main(int argc, char** argv)
}
sensor_t
sensor
;
sensor
.
name
=
*
i
;
std
::
stringstream
ss
;
if
(
prefix
.
size
()
<
12
)
{
ss
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
((
12
-
prefix
.
size
())
*
2
)
<<
""
;
}
const
char
*
c
=
prefix
.
substr
(
0
,
12
).
c_str
();
while
(
*
c
)
{
uint32_t
x
=
*
c
++
;
ss
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
2
)
<<
std
::
hex
<<
x
;
}
ss
<<
"/"
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
8
)
<<
std
::
hex
<<
topics
;
sensor
.
topic
=
ss
.
str
();
ss
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
suffixLen
)
<<
std
::
hex
<<
topics
;;
sensor
.
topic
=
prefix
+
ss
.
str
();
sensor
.
publicName
=
prefix
+
"."
+
sensor
.
name
;
std
::
replace
(
sensor
.
publicName
.
begin
(),
sensor
.
publicName
.
end
(),
' '
,
'_'
);
sensors
.
push_back
(
sensor
);
sensors
.
insert
(
std
::
pair
<
int
,
sensor_t
>
(
col
,
sensor
)
)
;
topics
++
;
}
/* Read actual sensor readings */
while
(
std
::
getline
(
fs
,
s
))
{
int
col
=
0
;
lineno
++
;
col
=
0
;
DCDB
::
TimeStamp
ts
(
UINT64_C
(
0
));
boost
::
tokenizer
<
boost
::
escaped_list_separator
<
char
>
>
tk
(
s
,
boost
::
escaped_list_separator
<
char
>
(
'\\'
,
','
,
'\"'
));
for
(
boost
::
tokenizer
<
boost
::
escaped_list_separator
<
char
>
>::
iterator
i
=
tk
.
begin
();
i
!=
tk
.
end
();
++
i
)
{
if
(
col
==
tsColumn
)
{
ts
=
DCDB
::
TimeStamp
(
*
i
);
}
col
++
;
}
col
=
0
;
for
(
boost
::
tokenizer
<
boost
::
escaped_list_separator
<
char
>
>::
iterator
i
=
tk
.
begin
();
i
!=
tk
.
end
();
++
i
)
{
if
(
0
==
col
)
{
ts
=
DCDB
::
TimeStamp
(
*
i
);
}
else
{
// std::cout << ts.getRaw() << " " << sensors[col-1].topic << ":" << *i << std::endl;
try
{
DCDB
::
SensorId
sid
(
sensors
[
col
-
1
].
topic
);
sensorDataStore
.
insert
(
&
sid
,
ts
.
getRaw
(),
std
::
stoll
(
*
i
));
}
catch
(
std
::
exception
&
e
)
{
std
::
cerr
<<
"Error parsing CSV line "
<<
lineno
<<
" column "
<<
col
+
1
<<
":
\"
"
<<
*
i
<<
"
\"
"
<<
std
::
endl
;
}
}
if
((
columns
.
size
()
==
0
)
||
(
columns
.
find
(
col
)
!=
columns
.
end
()))
{
std
::
cout
<<
ts
.
getRaw
()
<<
" "
<<
col
<<
" "
<<
sensors
[
col
].
topic
<<
" "
<<
*
i
<<
std
::
endl
;
try
{
DCDB
::
SensorId
sid
(
sensors
[
col
-
1
].
topic
);
sensorDataStore
.
insert
(
&
sid
,
ts
.
getRaw
(),
std
::
stoll
(
*
i
));
}
catch
(
std
::
exception
&
e
)
{
std
::
cerr
<<
"Error parsing CSV line "
<<
lineno
<<
" column "
<<
col
+
1
<<
":
\"
"
<<
*
i
<<
"
\"
"
<<
std
::
endl
;
}
}
col
++
;
}
}
fs
.
close
();
/* Create public sensor names
*/
/* Create public sensor names
std::cout << std::endl;
std::cout << "Publishing sensors..." << std::endl;
std
::
vector
<
sensor_t
>::
iterator
it
;
for (it = sensors.begin(); it != sensors.end(); it++) {
std::cout << it->name << " " << it->topic << " " << it->publicName << std::endl;
sensorConfig.publishSensor(it->publicName.c_str(), it->topic.c_str());
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment