python helper package for analysis

Tools/vadere_analysis_tool/README.md

+# Vadere Analysis Tool
+
+Simplify import and analysis of simulation output in python an juypter-notebooks
\ No newline at end of file

Tools/vadere_analysis_tool/__init__.py

+from vadere_analysis_tool.scenario_output import ScenarioOutput
+from vadere_analysis_tool.scenario_output import NamedFiles
+from vadere_analysis_tool.vadere_project import VadereProject
+from vadere_analysis_tool.vadere_project import NamedOutput
+
+
+name = 'vadere_analysis_tool'

Tools/vadere_analysis_tool/build/lib/tests/__init__.py

+

Tools/vadere_analysis_tool/build/lib/tests/scenario_output_test.py

+import unittest
+from vadere_analysis_tool import ScenarioOutput
+
+
+class ScenarioOutputTests(unittest.TestCase):
+
+    def test_output_dir(self):
+        out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+        self.assertEqual(out.output_dir, "testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+
+    def test_scenario_dict(self):
+        out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+        self.assertEqual(out.scenario['name'], 'bridge_coordinates_kai')
+
+    def test_invalid_scenario_dict(self):
+        self.assertRaises(FileNotFoundError, ScenarioOutput, "testData/s2ucre/output/invalid")
+
+    def test_scenario_output_dict(self):
+        out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+        self.assertEqual(len(out.files), 3)
+
+    def test_settattr_works(self):
+        out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+        self.assertTrue('df_overlapCount_txt' in dir(out.named_files))
+        self.assertTrue('df_overlaps_csv' in dir(out.named_files))
+        self.assertTrue('df_postvis_trajectories' in dir(out.named_files))
+        self.assertTrue('info_overlapCount_txt' in dir(out.named_files))
+        self.assertTrue('info_overlaps_csv' in dir(out.named_files))
+        self.assertTrue('info_postvis_trajectories' in dir(out.named_files))

Tools/vadere_analysis_tool/build/lib/tests/vadere_project_test.py

+import unittest
+from vadere_analysis_tool import vadere_project
+
+
+class VadereProjectTests(unittest.TestCase):
+
+    def test_project_name(self):
+        project = vadere_project.VadereProject("testData/s2ucre")
+        self.assertEqual(project.project_name, 's2ucre_scenarios')
+
+    def test_wrong_project_dir(self):
+        self.assertRaises(FileNotFoundError, vadere_project.VadereProject, "testData/s2uSSScre")
+
+    def test_no_output_dir(self):
+        self.assertRaises(FileNotFoundError, vadere_project.VadereProject, "testData/s2ucreInvalid")
+
+    def test_load_output_dir(self):
+        project = vadere_project.VadereProject("testData/s2ucre")
+        self.assertEqual(len(project.err_info()), 1)
+        self.assertEqual(len(project.output_dirs), 30)

Tools/vadere_analysis_tool/helper.py

+import os
+import json
+
+
+def read_lines(path):
+    if os.path.exists(path):
+        with open(path, 'r') as f:
+            return f.readlines()
+    else:
+        raise FileNotFoundError("File not found {}".format(path))
+
+
+def read_json_to_dict(path):
+    if os.path.exists(path):
+        with open(path, 'r') as f:
+            return json.load(f)
+    else:
+        raise FileNotFoundError("File not found {}".format(path))
+
+
+def clean_dir_name(dir_name):
+    ret = dir_name.replace('.', '_')
+    ret = ret.replace('-','_')
+    return ret
\ No newline at end of file

Tools/vadere_analysis_tool/scenario_output.py

+import os
+from vadere_analysis_tool import helper
+import pandas as pd
+import json
+
+
+class NamedFiles:
+
+    def __init__(self):
+        pass
+
+
+class ScenarioOutput:
+
+    def __init__(self, output_dir):
+
+        if os.path.exists(output_dir):
+            self.output_dir = output_dir
+            self.output_dir_name = os.path.basename(output_dir)
+        else:
+            raise FileNotFoundError("Directory at {} does not exist.".format(output_dir))
+
+        output_files = os.listdir(output_dir)
+        scenario_files = [f for f in output_files if f.endswith(".scenario")]
+
+        if len(scenario_files) < 1:
+            raise FileNotFoundError("Output directory has no scenario")
+        self.scenario = helper.read_json_to_dict(os.path.join(output_dir, scenario_files[0]))
+
+        # add attributes for output files programmatically to the ScenarioOutput object. These attributes
+        # are recognized by the code completion tool of jupyter-notebook and allow easy access to the each
+        # output file in one simulation run.
+        self.files = dict()
+        self.named_files = NamedFiles()
+
+        for file in self.scenario['processWriters']['files']:
+            f_name = file['filename']
+            f_path = os.path.join(self.output_dir, f_name)
+            if os.path.exists(f_path):
+                attr_df = helper.clean_dir_name(f_name)
+                setattr(self.named_files, "df_" + attr_df, self._load_df_(f_path))
+                self.files[f_name] = self._load_df_(f_path)
+
+                attr_info = "info_{}".format(attr_df)
+                attr_info_dict = dict()
+                attr_info_dict['keyType'] = file['type'].split('.')[-1]
+                attr_info_dict['dataprocessors'] = self._get_used_processors_(file['processors'])
+                attr_info_dict['path'] = os.path.abspath(f_path)
+                setattr(self.named_files, attr_info, attr_info_dict)
+
+    def _get_used_processors_(self, ids):
+        """
+        :param ids: list of processor ids used in one output file
+        :return:    the names of DataProcessors corresponding to the given ids. Only the last component of the
+                    dataProcessor name is returned
+        """
+        processor_list = [p for p in self.scenario['processWriters']['processors'] if p['id'] in ids]
+        return [p['type'].split('.')[-1] for p in processor_list]
+
+    def _load_df_(self, path):
+        """
+        :return: lambda function to lazy load pandas DataFrame. This reduces the load time of a vadere_analysis_tool project in
+        a jupyter-notebook because the DataFrames of an output file is loaded only when needed.
+        """
+        return lambda: self._get_dataframe_(path)
+
+    def info(self):
+        """
+        :return: print important scenario settings based on scenario file from the output directory
+        """
+        print("mainModel:", self.scenario['scenario']['mainModel'])
+        print("attributesSimulation:")
+        print(json.dumps(self.scenario['scenario']['attributesSimulation'], indent=2))
+        print("attributesModel:")
+        print(json.dumps(self.scenario['scenario']['attributesModel'], indent=2))
+
+    @staticmethod
+    def _get_dataframe_(path):
+        df = pd.read_csv(filepath_or_buffer=path, sep=" ", header=0, decimal=".", index_col=False, encoding="utf-8")
+        return df

Tools/vadere_analysis_tool/setup.py

+import setuptools
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name="vadere_analysis_tool",
+    version="0.0.1",
+    author="Stefan Schuhbäck",
+    author_email="stefan.schuhbaeck@hm.edu",
+    description="Import VadereProject to ease analysis",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    packages=setuptools.find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "Operating System :: OS Independent",
+    ],
+)
\ No newline at end of file

Tools/vadere_analysis_tool/tests/__init__.py

+

Tools/vadere_analysis_tool/tests/scenario_output_test.py

+import unittest
+from vadere_analysis_tool import ScenarioOutput
+
+
+class ScenarioOutputTests(unittest.TestCase):
+
+    def test_output_dir(self):
+        out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+        self.assertEqual(out.output_dir, "testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+
+    def test_scenario_dict(self):
+        out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+        self.assertEqual(out.scenario['name'], 'bridge_coordinates_kai')
+
+    def test_invalid_scenario_dict(self):
+        self.assertRaises(FileNotFoundError, ScenarioOutput, "testData/s2ucre/output/invalid")
+
+    def test_scenario_output_dict(self):
+        out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+        self.assertEqual(len(out.files), 3)
+
+    def test_settattr_works(self):
+        out = ScenarioOutput("testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43")
+        self.assertTrue('df_overlapCount_txt' in dir(out.named_files))
+        self.assertTrue('df_overlaps_csv' in dir(out.named_files))
+        self.assertTrue('df_postvis_trajectories' in dir(out.named_files))
+        self.assertTrue('info_overlapCount_txt' in dir(out.named_files))
+        self.assertTrue('info_overlaps_csv' in dir(out.named_files))
+        self.assertTrue('info_postvis_trajectories' in dir(out.named_files))

Tools/vadere_analysis_tool/tests/testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43/bridge_coordinates_kai.scenario

+{
+  "name" : "bridge_coordinates_kai",
+  "description" : "",
+  "release" : "0.6",
+  "commithash" : "topographyWarning: no commit hash",
+  "processWriters" : {
+    "files" : [ {
+      "type" : "org.vadere.simulator.projects.dataprocessing.outputfile.TimestepPedestrianIdOutputFile",
+      "filename" : "postvis.trajectories",
+      "processors" : [ 1, 2 ]
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.outputfile.TimestepPedestrianIdOverlapOutputFile",
+      "filename" : "overlaps.csv",
+      "processors" : [ 3 ]
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.outputfile.NoDataKeyOutputFile",
+      "filename" : "overlapCount.txt",
+      "processors" : [ 4 ]
+    } ],
+    "processors" : [ {
+      "type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianPositionProcessor",
+      "id" : 1
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianTargetIdProcessor",
+      "id" : 2
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianOverlapProcessor",
+      "id" : 3
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.processor.NumberOverlapsProcessor",
+      "id" : 4,
+      "attributesType" : "org.vadere.state.attributes.processor.AttributesNumberOverlapsProcessor",
+      "attributes" : {
+        "pedestrianOverlapProcessorId" : 3
+      }
+    } ],
+    "isTimestamped" : true
+  },
+  "scenario" : {
+    "mainModel" : "org.vadere.simulator.models.osm.OptimalStepsModel",
+    "attributesModel" : {
+      "org.vadere.state.attributes.models.AttributesOSM" : {
+        "stepCircleResolution" : 18,
+        "numberOfCircles" : 1,
+        "varyStepDirection" : false,
+        "stepLengthIntercept" : 0.4625,
+        "stepLengthSlopeSpeed" : 0.2345,
+        "stepLengthSD" : 0.036,
+        "movementThreshold" : 0.0,
+        "optimizationType" : "NELDER_MEAD",
+        "movementType" : "ARBITRARY",
+        "dynamicStepLength" : false,
+        "updateType" : "EVENT_DRIVEN",
+        "seeSmallWalls" : false,
+        "minimumStepLength" : false,
+        "targetPotentialModel" : "org.vadere.simulator.models.potential.fields.PotentialFieldTargetGrid",
+        "pedestrianPotentialModel" : "org.vadere.simulator.models.potential.PotentialFieldPedestrianCompactSoftshell",
+        "obstaclePotentialModel" : "org.vadere.simulator.models.potential.PotentialFieldObstacleCompactSoftshell",
+        "submodels" : [ ]
+      },
+      "org.vadere.state.attributes.models.AttributesPotentialCompactSoftshell" : {
+        "pedPotentialIntimateSpaceWidth" : 0.45,
+        "pedPotentialPersonalSpaceWidth" : 1.2,
+        "pedPotentialHeight" : 50.0,
+        "obstPotentialWidth" : 0.8,
+        "obstPotentialHeight" : 6.0,
+        "intimateSpaceFactor" : 1.2,
+        "personalSpacePower" : 1,
+        "intimateSpacePower" : 1
+      },
+      "org.vadere.state.attributes.models.AttributesFloorField" : {
+        "createMethod" : "HIGH_ACCURACY_FAST_MARCHING",
+        "potentialFieldResolution" : 0.1,
+        "obstacleGridPenalty" : 0.1,
+        "targetAttractionStrength" : 1.0,
+        "timeCostAttributes" : {
+          "standardDeviation" : 0.7,
+          "type" : "UNIT",
+          "obstacleDensityWeight" : 3.5,
+          "pedestrianSameTargetDensityWeight" : 3.5,
+          "pedestrianOtherTargetDensityWeight" : 3.5,
+          "pedestrianWeight" : 3.5,
+          "queueWidthLoading" : 1.0,
+          "pedestrianDynamicWeight" : 6.0,
+          "loadingType" : "CONSTANT"
+        }
+      }
+    },
+    "attributesSimulation" : {
+      "finishTime" : 200.0,
+      "simTimeStepLength" : 0.4,
+      "realTimeSimTimeRatio" : 0.0,
+      "writeSimulationData" : true,
+      "visualizationEnabled" : true,
+      "printFPS" : false,
+      "digitsPerCoordinate" : 2,
+      "useFixedSeed" : true,
+      "fixedSeed" : -3213925745664992646,
+      "simulationSeed" : -3213925745664992646
+    },
+    "topography" : {
+      "attributes" : {
+        "bounds" : {
+          "x" : 564280.0,
+          "y" : 5933391.0,
+          "width" : 50.0,
+          "height" : 50.0
+        },
+        "boundingBoxWidth" : 0.5,
+        "bounded" : true
+      },
+      "obstacles" : [ {
+        "shape" : {
+          "type" : "POLYGON",
+          "points" : [ {
+            "x" : 564330.2959240791,
+            "y" : 5933433.506680049
+          }, {
+            "x" : 564311.2959240791,
+            "y" : 5933433.506680049
+          }, {
+            "x" : 564300.9318720899,
+            "y" : 5933401.544835466
+          }, {
+            "x" : 564312.1291207125,
+            "y" : 5933396.936819093
+          }, {
+            "x" : 564330.1291207125,
+            "y" : 5933396.936819093
+          } ]
+        },
+        "id" : 1
+      }, {
+        "shape" : {
+          "type" : "POLYGON",
+          "points" : [ {
+            "x" : 564309.9431192318,
+            "y" : 5933389.478288522
+          }, {
+            "x" : 564309.9431192318,
+            "y" : 5933391.478288522
+          }, {
+            "x" : 564287.7114777192,
+            "y" : 5933401.030658626
+          }, {
+            "x" : 564280.3263510215,
+            "y" : 5933402.349890053
+          }, {
+            "x" : 564280.3263510215,
+            "y" : 5933390.349890053
+          } ]
+        },
+        "id" : 2
+      }, {
+        "shape" : {
+          "type" : "POLYGON",
+          "points" : [ {
+            "x" : 564283.8886836024,
+            "y" : 5933411.251412248
+          }, {
+            "x" : 564290.0683166787,
+            "y" : 5933406.240937642
+          }, {
+            "x" : 564295.4593781466,
+            "y" : 5933403.808810473
+          }, {
+            "x" : 564303.4638894079,
+            "y" : 5933436.907734532
+          }, {
+            "x" : 564280.4638894079,
+            "y" : 5933436.907734532
+          } ]
+        },
+        "id" : -1
+      } ],
+      "stairs" : [ ],
+      "targets" : [ {
+        "id" : 1,
+        "absorbing" : true,
+        "shape" : {
+          "type" : "POLYGON",
+          "points" : [ {
+            "x" : 564280.3,
+            "y" : 5933402.3
+          }, {
+            "x" : 564280.1,
+            "y" : 5933403.7
+          }, {
+            "x" : 564280.5,
+            "y" : 5933414.2
+          }, {
+            "x" : 564283.9,
+            "y" : 5933411.2
+          } ]
+        },
+        "waitingTime" : 0.0,
+        "waitingTimeYellowPhase" : 0.0,
+        "parallelWaiters" : 0,
+        "individualWaiting" : true,
+        "deletionDistance" : 0.1,
+        "startingWithRedLight" : false,
+        "nextSpeed" : -1.0
+      }, {
+        "id" : 1,
+        "absorbing" : true,
+        "shape" : {
+          "type" : "POLYGON",
+          "points" : [ {
+            "x" : 564313.9,
+            "y" : 5933391.2
+          }, {
+            "x" : 564309.9,
+            "y" : 5933391.5
+          }, {
+            "x" : 564312.3,
+            "y" : 5933397.0
+          }, {
+            "x" : 564316.4,
+            "y" : 5933395.9
+          } ]
+        },
+        "waitingTime" : 0.0,
+        "waitingTimeYellowPhase" : 0.0,
+        "parallelWaiters" : 0,
+        "individualWaiting" : true,
+        "deletionDistance" : 0.1,
+        "startingWithRedLight" : false,
+        "nextSpeed" : -1.0
+      } ],
+      "sources" : [ {
+        "id" : -1,
+        "shape" : {
+          "x" : 564304.3,
+          "y" : 5933435.5,
+          "width" : 8.0,
+          "height" : 4.4,
+          "type" : "RECTANGLE"
+        },
+        "interSpawnTimeDistribution" : "org.vadere.state.scenario.ConstantDistribution",
+        "distributionParameters" : [ 1.0 ],
+        "spawnNumber" : 4,
+        "maxSpawnNumberTotal" : -1,
+        "startTime" : 0.0,
+        "endTime" : 100.0,
+        "spawnAtRandomPositions" : false,
+        "useFreeSpaceOnly" : true,
+        "targetIds" : [ 1 ],
+        "groupSizeDistribution" : [ 1.0 ],
+        "dynamicElementType" : "PEDESTRIAN"
+      } ],
+      "dynamicElements" : [ ],
+      "attributesPedestrian" : {
+        "radius" : 0.195,
+        "densityDependentSpeed" : false,
+        "speedDistributionMean" : 1.34,
+        "speedDistributionStandardDeviation" : 0.26,
+        "minimumSpeed" : 0.5,
+        "maximumSpeed" : 2.2,
+        "acceleration" : 2.0
+      },
+      "attributesCar" : {
+        "id" : -1,
+        "radius" : 0.195,
+        "densityDependentSpeed" : false,
+        "speedDistributionMean" : 1.34,
+        "speedDistributionStandardDeviation" : 0.26,
+        "minimumSpeed" : 0.5,
+        "maximumSpeed" : 2.2,
+        "acceleration" : 2.0,
+        "length" : 4.5,
+        "width" : 1.7,
+        "direction" : {
+          "x" : 1.0,
+          "y" : 0.0
+        }
+      }
+    }
+  }
+}

Tools/vadere_analysis_tool/tests/testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43/overlapCount.txt

+nr_overlaps
+1

Tools/vadere_analysis_tool/tests/testData/s2ucre/output/bridge_coordinates_kai_2018-10-11_17-50-48.43/overlaps.csv

+timeStep pedestrianId overlapPedestrianId distance overlaps
+196 57 180 0.38946202210407527 5.379778959247461E-4

Tools/vadere_analysis_tool/tests/testData/s2ucre/output/bridge_coordinates_kai_2018-10-15_10-34-43.80/bridge_coordinates_kai.scenario

+{
+  "name" : "bridge_coordinates_kai",
+  "description" : "",
+  "release" : "0.6",
+  "commithash" : "topographyWarning: no commit hash",
+  "processWriters" : {
+    "files" : [ {
+      "type" : "org.vadere.simulator.projects.dataprocessing.outputfile.TimestepPedestrianIdOutputFile",
+      "filename" : "postvis.trajectories",
+      "processors" : [ 1, 2 ]
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.outputfile.TimestepPedestrianIdOverlapOutputFile",
+      "filename" : "overlaps.csv",
+      "processors" : [ 3 ]
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.outputfile.NoDataKeyOutputFile",
+      "filename" : "overlapCount.txt",
+      "processors" : [ 4 ]
+    } ],
+    "processors" : [ {
+      "type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianPositionProcessor",
+      "id" : 1
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianTargetIdProcessor",
+      "id" : 2
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianOverlapProcessor",
+      "id" : 3
+    }, {
+      "type" : "org.vadere.simulator.projects.dataprocessing.processor.NumberOverlapsProcessor",
+      "id" : 4,
+      "attributesType" : "org.vadere.state.attributes.processor.AttributesNumberOverlapsProcessor",
+      "attributes" : {
+        "pedestrianOverlapProcessorId" : 3
+      }
+    } ],
+    "isTimestamped" : true
+  },
+  "scenario" : {
+    "mainModel" : "org.vadere.simulator.models.osm.OptimalStepsModel",
+    "attributesModel" : {
+      "org.vadere.state.attributes.models.AttributesOSM" : {
+        "stepCircleResolution" : 18,
+        "numberOfCircles" : 1,
+        "varyStepDirection" : false,
+        "stepLengthIntercept" : 0.4625,
+        "stepLengthSlopeSpeed" : 0.2345,
+        "stepLengthSD" : 0.036,
+        "movementThreshold" : 0.0,