run_vadere_console_with_all_scenario_files.py 7.53 KB
Newer Older
1
2
3
4
# Use "vadere-console.jar", which is created by "mvn package", to run all
# scenario files under "VadereModelTests" subdirectory.
#
# Note: script contains some print statements so that progress can be tracked
5
# a little bit while script is running in continuous integration pipeline.
6

7
# Watch out: call this script from root directory of project. E.g.
8
9
10
11
12
#
#   python Tools/my_script.py

import fnmatch
import os
13
import re
14
15
import shutil
import subprocess
16
import time
17
18
import argparse

19

20
def find_scenario_files(path="VadereModelTests", scenario_search_pattern = "*.scenario", exclude_patterns = ["TESTOVM", "output","legacy"]):
21
22
23
24
25
    scenario_files = []

    for root, dirnames, filenames in os.walk(path):
        for filename in fnmatch.filter(filenames, scenario_search_pattern):
            scenario_path = os.path.join(root, filename)
26
            scenario_path_excluded = False
27
28
29
30

            for exclude_pattern in exclude_patterns:
                regex_pattern = re.compile(exclude_pattern)
                match = regex_pattern.search(scenario_path)
Marion Goedel's avatar
Marion Goedel committed
31
                if match:
32
                    scenario_path_excluded = True
33

34
            if scenario_path_excluded == False:
Marion Goedel's avatar
Marion Goedel committed
35
                scenario_files.append(scenario_path)
36

37
    return sorted(scenario_files)
38

39
def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereSimulator/target/vadere-console.jar", scenario_timeout_in_sec=60):
40
    output_dir = "output"
41
    log_dir = "log_dir"
42
43
44
45

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

46
47
48
    if not os.path.exists(log_dir):
        os.makedirs(log_dir)

49
50
    total_scenario_files = len(scenario_files)

51
52
    passed_scenarios = []
    failed_scenarios_with_exception = []
53
    failed_summary = []
54

55
    for i, scenario_file in enumerate(scenario_files):
56

57
        try:
58
            print("Running scenario file ({}/{}): {}".format(i + 1, total_scenario_files, scenario_file))
59
            scenario_name = os.path.basename(scenario_file).split('.')[0]
60
            log_file = os.path.join(log_dir, scenario_name + ".log")
61

62
            # Measure wall time and not CPU time simply because it is the simplest method.
63
64
65
            wall_time_start = time.time()

            # Use timout feature, check return value and capture stdout/stderr to a PIPE (use completed_process.stdout to get it).
66
67
68
            completed_process = subprocess.run(args=["java", "-enableassertions", "-jar", vadere_console,
                                                     "--logname", log_file, "scenario-run", "-f",
                                                     scenario_file, "-o", output_dir],
69
70
71
72
73
74
75
76
                                           timeout=scenario_timeout_in_sec,
                                           check=True,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)

            wall_time_end = time.time()
            wall_time_delta = wall_time_end - wall_time_start

77
            print("Finished scenario file ({:.1f} s): {}".format(wall_time_delta, scenario_file))
78

79

80
81
            passed_scenarios.append(scenario_file)
        except subprocess.TimeoutExpired as exception:
82
            print("Scenario file failed: {}".format(scenario_file))
Marion Goedel's avatar
Marion Goedel committed
83
            print("->  Reason: timeout after {} s".format(exception.timeout))
84
85
            failed_summary.append("Scenario file failed: {}".format(scenario_file))
            failed_summary.append("->  Reason: timeout after {} s".format(exception.timeout))
86
87
            failed_scenarios_with_exception.append((scenario_file, exception))
        except subprocess.CalledProcessError as exception:
Marion Goedel's avatar
Marion Goedel committed
88
            prefix = ""
89
90
            if "TestOSM" in scenario_file:
                prefix = " * OSM * "
Marion Goedel's avatar
Marion Goedel committed
91
92
            print(prefix + "Scenario file failed: {}".format(scenario_file))
            print("->  Reason: non-zero return value {}".format(exception.returncode))
93
94
95
96
97
            print("            {}".format(read_first_error_linies(exception.stderr)))
            failed_summary.append(prefix + "Scenario file failed: {}".format(scenario_file))
            failed_summary.append("->  Reason: non-zero return value {}".format(exception.returncode))
            failed_summary.append("            {}".format(read_first_error_linies(exception.stderr)))

98
            failed_scenarios_with_exception.append((scenario_file, exception))
99
100
101
102

    if os.path.exists(output_dir):
        shutil.rmtree(output_dir)

103
104
105
106
107
108
109
110
111
    return {"passed": passed_scenarios, "failed": failed_scenarios_with_exception, "failed_summary": failed_summary}

def read_first_error_linies(error_byte_string):
    err_string_lines = error_byte_string.decode('utf-8').split('\n')
    if len(err_string_lines) >= 2:
        return re.sub('\s+', ' ', ' '.join(err_string_lines[:2]))
    else:
        return 'unknown error see vadere log file.'

112

113
114
115
116
def print_summary(passed_and_failed_scenarios):
    total_passed_scenarios = len(passed_and_failed_scenarios["passed"])
    total_failed_scenarios = + len(passed_and_failed_scenarios["failed"])
    total_scenarios = total_passed_scenarios + total_failed_scenarios
117
118
119
120
    faild_summary = passed_and_failed_scenarios["failed_summary"]

    if len(faild_summary) > 0:
        print("#################")
121
        print("# Failed Summary #")
122
123
124
125
        print("#################")
        for line in faild_summary:
            print(line)

Marion Goedel's avatar
Marion Goedel committed
126

127
128
129
130
131
132
133
134
    print("###########")
    print("# Summary #")
    print("###########")
    print("")
    print("Total scenario runs: {}".format(total_scenarios))
    print("Passed: {}".format(total_passed_scenarios))
    print("Failed: {}".format(total_failed_scenarios))

135

136
def run_all():
137
    long_running_scenarios = [
138
139
140
141
142
143
144
145
        "basic_4_1_wall_gnm1",
        "queueing",
        "rimea_09",
        "rimea_11",
        "TestSFM",
        "thin_wall_and_closer_source_nelder_mead_ok",
        "thin_wall_and_closer_source_pso_could_fail",
        "rimea_04_flow_osm1_550_up",
146
        "stairs_diagonal_both_1_2_+1.scenario",
147
148
        "04-NarrowedStreet-Pollichtrasse",
        "05-Guimaraes-Platz"
149
    ]
150

151
152
    excluded_scenarios = ["TESTOVM", "output", "legacy"]
    excluded_scenarios.extend(long_running_scenarios)
Marion Goedel's avatar
Marion Goedel committed
153

154
    scenario_files_regular_length = find_scenario_files(exclude_patterns=excluded_scenarios)
155
    passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files_regular_length)
Marion Goedel's avatar
Marion Goedel committed
156

157
    for scenario in long_running_scenarios:
Marion Goedel's avatar
Marion Goedel committed
158
159
        search_pattern = "*" + scenario + "*.scenario"
        scenario_files_long = find_scenario_files(scenario_search_pattern=search_pattern)
160
        tmp_passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files_long,
161
                                                                                 scenario_timeout_in_sec=540)
Marion Goedel's avatar
Marion Goedel committed
162
163
        passed_and_failed_scenarios["passed"].extend(tmp_passed_and_failed_scenarios["passed"])
        passed_and_failed_scenarios["failed"].extend(tmp_passed_and_failed_scenarios["failed"])
164
165
166
167
168
169
170
171
172
173
174
175
176
177
        passed_and_failed_scenarios["failed_summary"].extend(tmp_passed_and_failed_scenarios["failed_summary"])

    return passed_and_failed_scenarios


if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='Run ModelTest')
    parser.add_argument('--scenario', '-s', nargs='?', default='', metavar='S', help='only run one selected scenario')
    args = vars(parser.parse_args())
    if args['scenario'] == '':
        passed_and_failed_scenarios = run_all()
    else:
        passed_and_failed_scenarios = run_scenario_files_with_vadere_console([args['scenario']])
Marion Goedel's avatar
Marion Goedel committed
178

179
    print_summary(passed_and_failed_scenarios)
180
181
182
183
184

    if len(passed_and_failed_scenarios["failed"]) > 0:
        exit(1)
    else:
        exit(0)