11.08., 9:00 - 11:00: Due to updates GitLab will be unavailable for some minutes between 09:00 and 11:00.

Commit b26d14b8 authored by Rebecca's avatar Rebecca

added od-response learning for sbb data

parent 7cd52209
Pipeline #215369 failed with stages
in 102 minutes and 34 seconds
......@@ -588,7 +588,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
"version": "3.7.3"
},
"toc": {
"base_numbering": 1,
......
This source diff could not be displayed because it is too large. You can view the blob instead.
source,destination
-1,3
0,3
1,1
1,2
1,3
1,7
1,8
2,1
2,2
2,3
2,7
2,8
3,3
3,5
4,-1
4,0
4,1
4,2
4,3
4,4
4,5
4,6
4,7
4,8
5,1
5,2
5,3
5,4
5,6
6,2
6,3
6,4
6,5
6,6
7,1
7,2
7,3
7,7
8,1
8,2
8,3
8,7
8,8
6,1
8,7
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"df = pd.DataFrame({'Origin' : ['A','A','A','B','B','A'], \n",
" 'Destination' :['B','C','D','C','D','D'], \n",
" 'X' : ['a','b', 'c', 'd', 'e', 'f']})\n",
"\n",
"\n",
"I = pd.DataFrame({'Origin' : ['A','A','B'], 'Destination' : ['B','D','C']})\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Origin Destination X\n",
"0 A B a\n",
"1 A D c\n",
"2 A D f\n",
"3 B C d\n"
]
}
],
"source": [
"print (pd.merge(df,I))"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Origin Destination X\n",
"0 A B a\n",
"1 A D c\n",
"2 A D f\n",
"3 B C d\n"
]
}
],
"source": [
"print (pd.merge(I,df, on=['Origin','Destination'])[['Origin','Destination','X']])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Split by Pedestiran Id\n",
"\n",
"This notebook can be used to split the data of a hole day into separate files, each containing the trajectories of a single pedestrian.\n",
"\n",
"Usage: Modify the data directory to where the raw data file lies and set the save directory to the location where the resulting files should be stored.\n",
"\n",
"Beaware: Data files are quite big, for day 2019-09-06 there were 141,417 pedestrians, totaling of 2GB. \n",
"\n",
"For testing: Pass the argument block_size to pandas read_csv and only read the first e.g. 10000 lines."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as mpl\n",
"import matplotlib.patches as patches\n",
"import pandas as pd\n",
"import numpy as np\n",
"import os\n",
"from tqdm import tqdm_notebook as tqdm\n",
"from shapely.geometry import Point, Polygon, LineString\n",
"from descartes import PolygonPatch\n",
"import random\n",
"from collections import OrderedDict,Counter\n",
"from math import floor"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def split_by_pedid(filename, path, store_dir,mode='all', time_frame=0):\n",
" \n",
" block_size = 10000\n",
" block = pd.read_csv(os.path.join(path, filename+'.csv'), sep=',',header=None, skiprows=0)\n",
" \n",
" print(block.head(block_size))\n",
" \n",
" for i, x in block.groupby(1):\n",
" p_filename = \"pedestrian_{}.csv\".format(i)\n",
" p_path = os.path.join(store_dir, p_filename)\n",
" print(p_filename)\n",
" x.to_csv(p_path, index=False)\n",
" \n",
" return\n",
"\n",
"def remove_first_line(path):\n",
" \n",
" files = os.listdir(path)\n",
" \n",
" for file in files:\n",
" print(file)\n",
" ped_traj = pd.read_csv(os.path.join(path,file),sep=',')\n",
" ped_traj.to_csv(os.path.join(path,file),sep=',',index=False, header=False)\n",
" \n",
" return"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"base = os.getcwd()\n",
"print(base)\n",
"\n",
"data_dir = os.path.join(base,'../')\n",
"print(data_dir)\n",
"\n",
"folder = 'objectstream_20190906/pedestrians_20190906/'\n",
"store_dir = os.path.join(data_dir,folder)\n",
"print(store_dir)\n",
"#file_names = ['objectstream_20190906_BS_sensor6_reduced']"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### split by pedestrian id\n",
"### only execute once in same directory or files will be over written!!!\n",
"#peds = split_by_pedid(file_names[0],data_dir,store_dir)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### only execute once to remove header, better pass header=False\n",
"#remove_first_line(store_dir)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as mpl\n",
"import matplotlib.patches as patches\n",
"import pandas as pd\n",
"import numpy as np\n",
"import os\n",
"from tqdm import tqdm_notebook as tqdm\n",
"from shapely.geometry import Point, Polygon, LineString\n",
"from descartes import PolygonPatch\n",
"import random\n",
"from collections import OrderedDict,Counter\n",
"from math import floor"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def random_color():\n",
" r = lambda: random.randint(0,255)\n",
" return '#%02X%02X%02X' % (r(),r(),r())\n",
"\n",
"\n",
"def to_point(pedestrian):\n",
" return Point(pedestrian[2] / 1000, pedestrian[3] / -1000)\n",
"\n",
"\n",
"def to_path(points):\n",
" return \n",
"\n",
"\n",
"class Pedestrian:\n",
" def __init__(self, frame):\n",
" self.frame = frame\n",
" \n",
" self.frame[['x','y']] = self.frame[['x','y']] / 1000\n",
" self.frame['y'] *= -1\n",
" \n",
" self.id = self.frame.pedestrianId[0]\n",
" self.start = Point(self.frame.x[0], self.frame.y[0])\n",
" \n",
" self.path = None\n",
" \n",
"\n",
" def has_path(self):\n",
" return self.path is not None or len(self.frame.timestep) > 1\n",
" \n",
" def get_pedestrianId(self):\n",
" return self.id\n",
" \n",
" def get_start(self):\n",
" return self.start\n",
" \n",
" def get_path(self):\n",
" if self.path is None and self.has_path():\n",
" self.path = LineString(list(zip(self.frame.x.values, self.frame.y.values)))\n",
" \n",
" return self.path\n",
"\n",
"def plot_source(sourceId, mapping, path, sources, skip=1, mode='all'):\n",
" pedestrians = mapping[mapping['source'] == sourceId].pedestrianId.values\n",
" plot_trajectories(pedestrians, path, sources, skip=skip, mode=mode)\n",
"\n",
"def stripMillies(row):\n",
" row = row.replace(microsecond=0)\n",
" return row \n",
" \n",
"\n",
"def load_pedestrians(filename, path, mode='all',time_frame=0):\n",
" # read chuncks of the csv file\n",
" block_size = 10000\n",
" \n",
" block = pd.read_csv(os.path.join(path, filename+'.csv'), sep=',',header=None, skiprows=0, nrows=block_size)\n",
" block.insert(1,'datetime',pd.to_datetime(block[0],unit='ms'))\n",
" \n",
" block['datetime'] = block['datetime'].apply(stripMillies)\n",
" \n",
" #determine dts\n",
" t0 = block['datetime'][0]\n",
" tend = pd.to_datetime(block['datetime'].tail(1).item())\n",
" \n",
" print(\"t0:\",t0, '\\ntend:',tend)\n",
" print('\\n\\n')\n",
" \n",
" block.insert(2,'dt',block[0].diff())\n",
" print(block.head(20))\n",
" \n",
" block[0] = block[0].divide(1000)\n",
" block[0] = block[0].apply(floor)\n",
" \n",
" \n",
" block = block.drop_duplicates(0,keep='first')\n",
" print(block.head(20))\n",
" \n",
" \n",
" \n",
"def plot_trajectories(pedestrians, path, sources, skip=1, mode='all', showsources=True):\n",
" minx = -46;\n",
" maxx = 10;\n",
" miny = -7;\n",
" maxy = 8;\n",
" \n",
" pedestrians = pedestrians[::skip]\n",
" \n",
" fig, ax = mpl.subplots(ncols=1, nrows=1, figsize=(11, 6), dpi=96)\n",
" \n",
" for ped in tqdm(pedestrians, total=len(pedestrians)):\n",
" print(pd.read_csv(os.path.join(path, ped+'.csv')))\n",
" pedestrian = Pedestrian(pd.read_csv(os.path.join(path, ped), names=['timestep', 'pedestrianId', 'x', 'y']))\n",
" x, y = pedestrian.get_path().xy\n",
" \n",
" if 'start' in mode:\n",
" ax.plot(x[0], y[0], 'c.', markersize=5, label='start')\n",
" \n",
" if 'end' in mode:\n",
" ax.plot(x[-1], y[-1], 'y.', markersize=5, label='end')\n",
" \n",
" if mode == 'traj':\n",
" ax.plot(x, y, 'k', linewidth=0.2)\n",
" \n",
" \n",
" \n",
" img = mpl.imread(\"../Layout2.png\")\n",
" \n",
" ax.imshow(img, extent=[minx, maxx, miny - 3, maxy + 4])\n",
" \n",
" if showsources:\n",
" colors = ['b', 'g', 'r', 'c', 'm', 'y']\n",
" for index, s in enumerate(sources):\n",
" ax.add_patch(PolygonPatch(s, linewidth=2, edgecolor=colors[index % len(colors)], facecolor='none', label=str(index)))\n",
" \n",
" handles, labels = fig.gca().get_legend_handles_labels()\n",
" by_label = OrderedDict(zip(labels, handles))\n",
" \n",
" ax.legend(by_label.values(), by_label.keys(), loc='lower left', bbox_to_anchor=(0, 1.02, 1, 0.2), ncol=len(sources) + 2)\n",
" \n",
" ax.grid()\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"base = os.getcwd()\n",
"print(base)\n",
"\n",
"data_dir = os.path.join(base,'../test')\n",
"print(data_dir)\n",
"\n",
"file_names = ['objectstream_20190905_BS_sensor6_by_timestep']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"mpl.rcParams['figure.figsize'] = [20, 5]\n",
"pets = load_pedestrians(file_names[0],data_dir)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"l1 = np.linspace(0,100,1001)\n",
"print(len(l1))\n",
"dt2 = 0.5\n",
"dt1 = 0.1\n",
"\n",
"factor = int(0.5/0.1)\n",
"print(factor)\n",
"l2 = l1[0::5]\n",
"print(l2[-1])\n",
"print(len(l2))\n",
"\n",
"1000/5 + 1\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Generate data on commute times.\n",
"size, scale = 1000, 10\n",
"commutes = pd.Series(np.random.gamma(scale, size=size) ** 1.5)\n",
"\n",
"commutes.plot.hist(grid=True, bins=20, rwidth=0.9,\n",
" color='#607c8e')\n",
"mpl.title('Commute Times for 1,000 Commuters')\n",
"mpl.xlabel('Counts')\n",
"mpl.ylabel('Commute Time')\n",
"mpl.grid(axis='y', alpha=0.75)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
This source diff could not be displayed because it is too large. You can view the blob instead.
source,destination
-1,3
0,3
1,1
1,2
1,3
1,7
1,8
2,1
2,2
2,3
2,7
2,8
3,3
3,5
4,-1
4,0
4,1
4,2
4,3
4,4
4,5
4,6
4,7
4,8
5,1
5,2
5,3
5,4
5,6
6,2
6,3
6,4
6,5
6,6
7,1
7,2
7,3
7,7
8,1
8,2
8,3
8,7
8,8
6,1
8,7
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -353,7 +353,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
"version": "3.7.3"
},
"toc": {
"base_numbering": 1,
......
{
"dataset": {
"path": "../../../notebooks/datasets/densitymaps/experiment/combined",
"shuffle": true,
"test_size": 0.2
},
"number_of_cores": 4,
"number_of_targets": 2,
"number_of_trees": 50,
"reader": {
"densitymap": {
"seperator": ";"
}
},
"save": true,
"seed": 69757228
}