The expiration time for new job artifacts in CI/CD pipelines is now 30 days (GitLab default). Previously generated artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 628b0183 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

python helper package for analysis

parent 63274a48
# Vadere Analysis Tool
Simplify import and analysis of simulation output in python an juypter-notebooks
\ No newline at end of file
from vadere_analysis_tool.scenario_output import ScenarioOutput
from vadere_analysis_tool.scenario_output import NamedFiles
from vadere_analysis_tool.vadere_project import VadereProject
from vadere_analysis_tool.vadere_project import NamedOutput
name = 'vadere_analysis_tool'
import unittest
from vadere_analysis_tool import ScenarioOutput
class ScenarioOutputTests(unittest.TestCase):
def test_output_dir(self):
out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
self.assertEqual(out.output_dir, "testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
def test_scenario_dict(self):
out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
self.assertEqual(out.scenario['name'], 'bridge_coordinates_kai')
def test_invalid_scenario_dict(self):
self.assertRaises(FileNotFoundError, ScenarioOutput, "testData/s2ucre/output/invalid")
def test_scenario_output_dict(self):
out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
self.assertEqual(len(out.files), 3)
def test_settattr_works(self):
out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
self.assertTrue('df_overlapCount_txt' in dir(out.named_files))
self.assertTrue('df_overlaps_csv' in dir(out.named_files))
self.assertTrue('df_postvis_trajectories' in dir(out.named_files))
self.assertTrue('info_overlapCount_txt' in dir(out.named_files))
self.assertTrue('info_overlaps_csv' in dir(out.named_files))
self.assertTrue('info_postvis_trajectories' in dir(out.named_files))
import unittest
from vadere_analysis_tool import vadere_project
class VadereProjectTests(unittest.TestCase):
def test_project_name(self):
project = vadere_project.VadereProject("testData/s2ucre")
self.assertEqual(project.project_name, 's2ucre_scenarios')
def test_wrong_project_dir(self):
self.assertRaises(FileNotFoundError, vadere_project.VadereProject, "testData/s2uSSScre")
def test_no_output_dir(self):
self.assertRaises(FileNotFoundError, vadere_project.VadereProject, "testData/s2ucreInvalid")
def test_load_output_dir(self):
project = vadere_project.VadereProject("testData/s2ucre")
self.assertEqual(len(project.err_info()), 1)
self.assertEqual(len(project.output_dirs), 30)
import os
import json
def read_lines(path):
if os.path.exists(path):
with open(path, 'r') as f:
return f.readlines()
else:
raise FileNotFoundError("File not found {}".format(path))
def read_json_to_dict(path):
if os.path.exists(path):
with open(path, 'r') as f:
return json.load(f)
else:
raise FileNotFoundError("File not found {}".format(path))
def clean_dir_name(dir_name):
ret = dir_name.replace('.', '_')
ret = ret.replace('-','_')
return ret
\ No newline at end of file
import os
from vadere_analysis_tool import helper
import pandas as pd
import json
class NamedFiles:
def __init__(self):
pass
class ScenarioOutput:
def __init__(self, output_dir):
if os.path.exists(output_dir):
self.output_dir = output_dir
self.output_dir_name = os.path.basename(output_dir)
else:
raise FileNotFoundError("Directory at {} does not exist.".format(output_dir))
output_files = os.listdir(output_dir)
scenario_files = [f for f in output_files if f.endswith(".scenario")]
if len(scenario_files) < 1:
raise FileNotFoundError("Output directory has no scenario")
self.scenario = helper.read_json_to_dict(os.path.join(output_dir, scenario_files[0]))
# add attributes for output files programmatically to the ScenarioOutput object. These attributes
# are recognized by the code completion tool of jupyter-notebook and allow easy access to the each
# output file in one simulation run.
self.files = dict()
self.named_files = NamedFiles()
for file in self.scenario['processWriters']['files']:
f_name = file['filename']
f_path = os.path.join(self.output_dir, f_name)
if os.path.exists(f_path):
attr_df = helper.clean_dir_name(f_name)
setattr(self.named_files, "df_" + attr_df, self._load_df_(f_path))
self.files[f_name] = self._load_df_(f_path)
attr_info = "info_{}".format(attr_df)
attr_info_dict = dict()
attr_info_dict['keyType'] = file['type'].split('.')[-1]
attr_info_dict['dataprocessors'] = self._get_used_processors_(file['processors'])
attr_info_dict['path'] = os.path.abspath(f_path)
setattr(self.named_files, attr_info, attr_info_dict)
def _get_used_processors_(self, ids):
"""
:param ids: list of processor ids used in one output file
:return: the names of DataProcessors corresponding to the given ids. Only the last component of the
dataProcessor name is returned
"""
processor_list = [p for p in self.scenario['processWriters']['processors'] if p['id'] in ids]
return [p['type'].split('.')[-1] for p in processor_list]
def _load_df_(self, path):
"""
:return: lambda function to lazy load pandas DataFrame. This reduces the load time of a vadere_analysis_tool project in
a jupyter-notebook because the DataFrames of an output file is loaded only when needed.
"""
return lambda: self._get_dataframe_(path)
def info(self):
"""
:return: print important scenario settings based on scenario file from the output directory
"""
print("mainModel:", self.scenario['scenario']['mainModel'])
print("attributesSimulation:")
print(json.dumps(self.scenario['scenario']['attributesSimulation'], indent=2))
print("attributesModel:")
print(json.dumps(self.scenario['scenario']['attributesModel'], indent=2))
@staticmethod
def _get_dataframe_(path):
df = pd.read_csv(filepath_or_buffer=path, sep=" ", header=0, decimal=".", index_col=False, encoding="utf-8")
return df
import setuptools
with open("README.md", "r") as fh:
long_description = fh.read()
setuptools.setup(
name="vadere_analysis_tool",
version="0.0.1",
author="Stefan Schuhbäck",
author_email="stefan.schuhbaeck@hm.edu",
description="Import VadereProject to ease analysis",
long_description=long_description,
long_description_content_type="text/markdown",
packages=setuptools.find_packages(),
classifiers=[
"Programming Language :: Python :: 3",
"Operating System :: OS Independent",
],
)
\ No newline at end of file
import unittest
from vadere_analysis_tool import ScenarioOutput
class ScenarioOutputTests(unittest.TestCase):
def test_output_dir(self):
out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
self.assertEqual(out.output_dir, "testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
def test_scenario_dict(self):
out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
self.assertEqual(out.scenario['name'], 'bridge_coordinates_kai')
def test_invalid_scenario_dict(self):
self.assertRaises(FileNotFoundError, ScenarioOutput, "testData/s2ucre/output/invalid")
def test_scenario_output_dict(self):
out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
self.assertEqual(len(out.files), 3)
def test_settattr_works(self):
out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
self.assertTrue('df_overlapCount_txt' in dir(out.named_files))
self.assertTrue('df_overlaps_csv' in dir(out.named_files))
self.assertTrue('df_postvis_trajectories' in dir(out.named_files))
self.assertTrue('info_overlapCount_txt' in dir(out.named_files))
self.assertTrue('info_overlaps_csv' in dir(out.named_files))
self.assertTrue('info_postvis_trajectories' in dir(out.named_files))
{
"name" : "bridge_coordinates_kai",
"description" : "",
"release" : "0.6",
"commithash" : "topographyWarning: no commit hash",
"processWriters" : {
"files" : [ {
"type" : "org.vadere.simulator.projects.dataprocessing.outputfile.TimestepPedestrianIdOutputFile",
"filename" : "postvis.trajectories",
"processors" : [ 1, 2 ]
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.outputfile.TimestepPedestrianIdOverlapOutputFile",
"filename" : "overlaps.csv",
"processors" : [ 3 ]
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.outputfile.NoDataKeyOutputFile",
"filename" : "overlapCount.txt",
"processors" : [ 4 ]
} ],
"processors" : [ {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianPositionProcessor",
"id" : 1
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianTargetIdProcessor",
"id" : 2
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianOverlapProcessor",
"id" : 3
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.NumberOverlapsProcessor",
"id" : 4,
"attributesType" : "org.vadere.state.attributes.processor.AttributesNumberOverlapsProcessor",
"attributes" : {
"pedestrianOverlapProcessorId" : 3
}
} ],
"isTimestamped" : true
},
"scenario" : {
"mainModel" : "org.vadere.simulator.models.osm.OptimalStepsModel",
"attributesModel" : {
"org.vadere.state.attributes.models.AttributesOSM" : {
"stepCircleResolution" : 18,
"numberOfCircles" : 1,
"varyStepDirection" : false,
"stepLengthIntercept" : 0.4625,
"stepLengthSlopeSpeed" : 0.2345,
"stepLengthSD" : 0.036,
"movementThreshold" : 0.0,
"optimizationType" : "NELDER_MEAD",
"movementType" : "ARBITRARY",
"dynamicStepLength" : false,
"updateType" : "EVENT_DRIVEN",
"seeSmallWalls" : false,
"minimumStepLength" : false,
"targetPotentialModel" : "org.vadere.simulator.models.potential.fields.PotentialFieldTargetGrid",
"pedestrianPotentialModel" : "org.vadere.simulator.models.potential.PotentialFieldPedestrianCompactSoftshell",
"obstaclePotentialModel" : "org.vadere.simulator.models.potential.PotentialFieldObstacleCompactSoftshell",
"submodels" : [ ]
},
"org.vadere.state.attributes.models.AttributesPotentialCompactSoftshell" : {
"pedPotentialIntimateSpaceWidth" : 0.45,
"pedPotentialPersonalSpaceWidth" : 1.2,
"pedPotentialHeight" : 50.0,
"obstPotentialWidth" : 0.8,
"obstPotentialHeight" : 6.0,
"intimateSpaceFactor" : 1.2,
"personalSpacePower" : 1,
"intimateSpacePower" : 1
},
"org.vadere.state.attributes.models.AttributesFloorField" : {
"createMethod" : "HIGH_ACCURACY_FAST_MARCHING",
"potentialFieldResolution" : 0.1,
"obstacleGridPenalty" : 0.1,
"targetAttractionStrength" : 1.0,
"timeCostAttributes" : {
"standardDeviation" : 0.7,
"type" : "UNIT",
"obstacleDensityWeight" : 3.5,
"pedestrianSameTargetDensityWeight" : 3.5,
"pedestrianOtherTargetDensityWeight" : 3.5,
"pedestrianWeight" : 3.5,
"queueWidthLoading" : 1.0,
"pedestrianDynamicWeight" : 6.0,
"loadingType" : "CONSTANT"
}
}
},
"attributesSimulation" : {
"finishTime" : 200.0,
"simTimeStepLength" : 0.4,
"realTimeSimTimeRatio" : 0.0,
"writeSimulationData" : true,
"visualizationEnabled" : true,
"printFPS" : false,
"digitsPerCoordinate" : 2,
"useFixedSeed" : true,
"fixedSeed" : -3213925745664992646,
"simulationSeed" : -3213925745664992646
},
"topography" : {
"attributes" : {
"bounds" : {
"x" : 564280.0,
"y" : 5933391.0,
"width" : 50.0,
"height" : 50.0
},
"boundingBoxWidth" : 0.5,
"bounded" : true
},
"obstacles" : [ {
"shape" : {
"type" : "POLYGON",
"points" : [ {
"x" : 564330.2959240791,
"y" : 5933433.506680049
}, {
"x" : 564311.2959240791,
"y" : 5933433.506680049
}, {
"x" : 564300.9318720899,
"y" : 5933401.544835466
}, {
"x" : 564312.1291207125,
"y" : 5933396.936819093
}, {
"x" : 564330.1291207125,
"y" : 5933396.936819093
} ]
},
"id" : 1
}, {
"shape" : {
"type" : "POLYGON",
"points" : [ {
"x" : 564309.9431192318,
"y" : 5933389.478288522
}, {
"x" : 564309.9431192318,
"y" : 5933391.478288522
}, {
"x" : 564287.7114777192,
"y" : 5933401.030658626
}, {
"x" : 564280.3263510215,
"y" : 5933402.349890053
}, {
"x" : 564280.3263510215,
"y" : 5933390.349890053
} ]
},
"id" : 2
}, {
"shape" : {
"type" : "POLYGON",
"points" : [ {
"x" : 564283.8886836024,
"y" : 5933411.251412248
}, {
"x" : 564290.0683166787,
"y" : 5933406.240937642
}, {
"x" : 564295.4593781466,
"y" : 5933403.808810473
}, {
"x" : 564303.4638894079,
"y" : 5933436.907734532
}, {
"x" : 564280.4638894079,
"y" : 5933436.907734532
} ]
},
"id" : -1
} ],
"stairs" : [ ],
"targets" : [ {
"id" : 1,
"absorbing" : true,
"shape" : {
"type" : "POLYGON",
"points" : [ {
"x" : 564280.3,
"y" : 5933402.3
}, {
"x" : 564280.1,
"y" : 5933403.7
}, {
"x" : 564280.5,
"y" : 5933414.2
}, {
"x" : 564283.9,
"y" : 5933411.2
} ]
},
"waitingTime" : 0.0,
"waitingTimeYellowPhase" : 0.0,
"parallelWaiters" : 0,
"individualWaiting" : true,
"deletionDistance" : 0.1,
"startingWithRedLight" : false,
"nextSpeed" : -1.0
}, {
"id" : 1,
"absorbing" : true,
"shape" : {
"type" : "POLYGON",
"points" : [ {
"x" : 564313.9,
"y" : 5933391.2
}, {
"x" : 564309.9,
"y" : 5933391.5
}, {
"x" : 564312.3,
"y" : 5933397.0
}, {
"x" : 564316.4,
"y" : 5933395.9
} ]
},
"waitingTime" : 0.0,
"waitingTimeYellowPhase" : 0.0,
"parallelWaiters" : 0,
"individualWaiting" : true,
"deletionDistance" : 0.1,
"startingWithRedLight" : false,
"nextSpeed" : -1.0
} ],
"sources" : [ {
"id" : -1,
"shape" : {
"x" : 564304.3,
"y" : 5933435.5,
"width" : 8.0,
"height" : 4.4,
"type" : "RECTANGLE"
},
"interSpawnTimeDistribution" : "org.vadere.state.scenario.ConstantDistribution",
"distributionParameters" : [ 1.0 ],
"spawnNumber" : 4,
"maxSpawnNumberTotal" : -1,
"startTime" : 0.0,
"endTime" : 100.0,
"spawnAtRandomPositions" : false,
"useFreeSpaceOnly" : true,
"targetIds" : [ 1 ],
"groupSizeDistribution" : [ 1.0 ],
"dynamicElementType" : "PEDESTRIAN"
} ],
"dynamicElements" : [ ],
"attributesPedestrian" : {
"radius" : 0.195,
"densityDependentSpeed" : false,
"speedDistributionMean" : 1.34,
"speedDistributionStandardDeviation" : 0.26,
"minimumSpeed" : 0.5,
"maximumSpeed" : 2.2,
"acceleration" : 2.0
},
"attributesCar" : {
"id" : -1,
"radius" : 0.195,
"densityDependentSpeed" : false,
"speedDistributionMean" : 1.34,
"speedDistributionStandardDeviation" : 0.26,
"minimumSpeed" : 0.5,
"maximumSpeed" : 2.2,
"acceleration" : 2.0,
"length" : 4.5,
"width" : 1.7,
"direction" : {
"x" : 1.0,
"y" : 0.0
}
}
}
}
}
timeStep pedestrianId overlapPedestrianId distance overlaps
196 57 180 0.38946202210407527 5.379778959247461E-4
{
"name" : "bridge_coordinates_kai",
"description" : "",
"release" : "0.6",
"commithash" : "topographyWarning: no commit hash",
"processWriters" : {
"files" : [ {
"type" : "org.vadere.simulator.projects.dataprocessing.outputfile.TimestepPedestrianIdOutputFile",
"filename" : "postvis.trajectories",
"processors" : [ 1, 2 ]
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.outputfile.TimestepPedestrianIdOverlapOutputFile",
"filename" : "overlaps.csv",
"processors" : [ 3 ]
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.outputfile.NoDataKeyOutputFile",
"filename" : "overlapCount.txt",
"processors" : [ 4 ]
} ],
"processors" : [ {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianPositionProcessor",
"id" : 1
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianTargetIdProcessor",
"id" : 2
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianOverlapProcessor",
"id" : 3
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.NumberOverlapsProcessor",
"id" : 4,
"attributesType" : "org.vadere.state.attributes.processor.AttributesNumberOverlapsProcessor",
"attributes" : {
"pedestrianOverlapProcessorId" : 3
}
} ],
"isTimestamped" : true
},
"scenario" : {
"mainModel" : "org.vadere.simulator.models.osm.OptimalStepsModel",
"attributesModel" : {
"org.vadere.state.attributes.models.AttributesOSM" : {
"stepCircleResolution" : 18,
"numberOfCircles" : 1,
"varyStepDirection" : false,
"stepLengthIntercept" : 0.4625,
"stepLengthSlopeSpeed" : 0.2345,
"stepLengthSD" : 0.036,