run_seed_comparison_test.py 3.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
import os
import shutil
import subprocess
import time

from vadere_analysis_tool import VadereProject
from vadere_analysis_tool import SameSeedTrajectory


def run_scenario_files_with_vadere_console(project, number_of_runs=3, vadere_console="VadereSimulator/target/vadere-console.jar",
                                           scenario_timeout_in_sec=60*5):

    scenario_files = project.scenario_files
    total_scenario_files = len(scenario_files)
    output_dir = project.output_path

    passed_scenarios = []
    failed_scenarios_with_exception = []
    for run in range(number_of_runs):
        for i, scenario_file in enumerate(scenario_files):
            try:
                scenario_name = os.path.basename(scenario_file)
                print("Running scenario file ({}/{}) of run {}/{}: {}".format(i + 1, total_scenario_files, run + 1, number_of_runs, scenario_name))

                # Measure wall time and not CPU time simply because it is the simplest method.
                wall_time_start = time.time()



                # Use timout feature, check return value and capture stdout/stderr to a PIPE (use
                # completed_process.stdout to get it).
                # print("subprocess call: {}".format(' '.join(
                #     ["java", "-enableassertions", "-jar", vadere_console, "scenario-run", "-f", scenario_file, "-o", output_dir])))
                completed_process = subprocess.run(
                    args=["java", "-enableassertions", "-jar", vadere_console, "scenario-run", "-f", scenario_file, "-o",
                          output_dir],
                    timeout=scenario_timeout_in_sec,
                    check=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE)

                wall_time_end = time.time()
                wall_time_delta = wall_time_end - wall_time_start

                print("Finished scenario file ({:.1f} s): {}".format(wall_time_delta, scenario_file))

                passed_scenarios.append(scenario_file)
            except subprocess.TimeoutExpired as exception:
                print("Scenario file failed: {}".format(scenario_file))
                print("->  Reason: timeout after {} s".format(exception.timeout))
                failed_scenarios_with_exception.append((scenario_file, exception))
            except subprocess.CalledProcessError as exception:
                print("Scenario file failed: {}".format(scenario_file))
                print("->  Reason: non-zero return value {}".format(exception.returncode))
                failed_scenarios_with_exception.append((scenario_file, exception))

    return {"passed": passed_scenarios, "failed": failed_scenarios_with_exception}


if __name__ == '__main__':
    project = VadereProject('Tools/ContinuousIntegration/run_seed_comparison_test.d')
    project.clear_output_dir()
    run_scenario_files_with_vadere_console(project)

    # this will reload project
    seed_test = SameSeedTrajectory('Tools/ContinuousIntegration/run_seed_comparison_test.d')
    seed_results = seed_test.get_trajectory_comparison_result()

    seed_err = [res for res in seed_results if not res['hash_is_equal']]
    seed_ok = [res for res in seed_results if  res['hash_is_equal']]

    if seed_err:
        for res in seed_err:
            print("Scenario {} with scenario-hash {} - ERROR".format(res['scenario_name'], res['scenario_hash']))
            for res_out in res['scenario_outputs']:
                print("   {}--{}".format(res_out[1], res_out[0]))

    if seed_ok:
        for res in seed_ok:
            print("Scenario {} with scenario-hash {} - OK".format(res['scenario_name'], res['scenario_hash']))