2.12.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

run_vadere_console_with_all_scenario_files.py 10.9 KB
Newer Older
1
2
3
4
# Use "vadere-console.jar", which is created by "mvn package", to run all
# scenario files under "VadereModelTests" subdirectory.
#
# Note: script contains some print statements so that progress can be tracked
5
# a little bit while script is running in continuous integration pipeline.
6

7
# Watch out: call this script from root directory of project. E.g.
8
9
10
#
#   python Tools/my_script.py

11
import argparse
12
import copy
13
14
import fnmatch
import os
15
import re
16
17
import shutil
import subprocess
18
import time
19

20
21
22
long_timeout_in_seconds = 12 * 60
short_timeout_in_seconds = 2 * 60

23

24
25
def parse_command_line_arguments():
    parser = argparse.ArgumentParser(description="Run all scenario files.")
26

27
28
29
    parser.add_argument("scenario", type=str, nargs="?",
                        help="Run only the given scenario file and not all. E.g., "
                             "\"VadereModelTests/TestOSM/scenarios/basic_2_density_discrete_ca.scenario\"")
30

31
    return parser.parse_args()
32

33
34

def run_all_model_tests():
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
    long_running_scenarios = [
        "basic_4_1_wall_gnm1",
        "queueing",
        "rimea_09",
        "rimea_11",
        "TestSFM",
        "thin_wall_and_closer_source_nelder_mead_ok",
        "thin_wall_and_closer_source_pso_could_fail",
        "rimea_04_flow_osm1_550_up",
        "stairs_diagonal_both_1_2_+1.scenario",
        "05_bang_event_narrowed_street",
        "06_bang_event_guimaraes_platz"
    ]

    excluded_scenarios = ["TestOVM", "output", "legacy"]
    excluded_scenarios.extend(long_running_scenarios)

52
    scenario_files_regular_length = find_scenario_files(path="VadereModelTests", exclude_patterns=excluded_scenarios)
53
54
55

    passed_and_failed_scenarios = run_scenario_files_with_vadere_console(
        scenario_files_regular_length, scenario_timeout_in_sec=short_timeout_in_seconds)
56

57
58
    for scenario in long_running_scenarios:
        search_pattern = "*" + scenario + "*.scenario"
59
        scenario_files_long = find_scenario_files(path="VadereModelTests", scenario_search_pattern=search_pattern)
60
61
62
63
64
65

        tmp_passed_and_failed_scenarios = run_scenario_files_with_vadere_console(
            scenario_files_long, scenario_timeout_in_sec=long_timeout_in_seconds)

        passed_and_failed_scenarios = result_dict_merge(merge_into=passed_and_failed_scenarios,
                                                        merge_from=tmp_passed_and_failed_scenarios)
66
67
68

    return passed_and_failed_scenarios

69
70
71

def run_all_optimization_tests():
    scenario_files = find_scenario_files(path="VadereOptimizationTests")
72
73
74
75
76
77
78

    # enables flag to compare optimization with brute force solution
    config_filepath = os.path.join("VadereOptimizationTests", "TestNelderMead", "vadere.conf")

    # NOTE: it is likely that the set config file is not required by new optimization tests
    passed_and_failed_scenarios = run_scenario_files_with_vadere_console(
        scenario_files, scenario_timeout_in_sec=short_timeout_in_seconds, config_filepath=config_filepath)
79
80
81
82
83
84
85
86
87
88
89
90
91
92

    return passed_and_failed_scenarios


def find_scenario_files(path, scenario_search_pattern="*.scenario", exclude_patterns=None):

    if exclude_patterns is None:
        exclude_patterns = ["output", "legacy"]  # default values
    else:  # always insert output and legacy
        if "output" not in exclude_patterns:
            exclude_patterns.append("output")
        if "legacy" not in exclude_patterns:
            exclude_patterns.append("legacy")

93
94
95
96
97
    scenario_files = []

    for root, dirnames, filenames in os.walk(path):
        for filename in fnmatch.filter(filenames, scenario_search_pattern):
            scenario_path = os.path.join(root, filename)
98
            scenario_path_excluded = False
99
100
101
102

            for exclude_pattern in exclude_patterns:
                regex_pattern = re.compile(exclude_pattern)
                match = regex_pattern.search(scenario_path)
Marion Goedel's avatar
Marion Goedel committed
103
                if match:
104
                    scenario_path_excluded = True
105

106
            if not scenario_path_excluded:
Marion Goedel's avatar
Marion Goedel committed
107
                scenario_files.append(scenario_path)
108

109
    return sorted(scenario_files)
110

111
112

def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereSimulator/target/vadere-console.jar",
113
                                           scenario_timeout_in_sec=short_timeout_in_seconds, config_filepath=None):
114
    output_dir = "output"
115
    log_base_dir = "vadere_logs"
116

117
118
    makedirs_if_non_existing(output_dir)
    makedirs_if_non_existing(log_base_dir)
119

120
121
    total_scenario_files = len(scenario_files)

122
123
    passed_scenarios = []
    failed_scenarios_with_exception = []
124
    failed_summary = []
125

126
    for i, scenario_file in enumerate(scenario_files):
127

128
        try:
129
            print(f"Running scenario file ({i + 1}/{total_scenario_files}): {scenario_file}")
130
131
132
133
134
135
136
137
            
            # A scenario filename has the form "VadereModelTests/TestOSM/scenarios/chicken_floorfield_ok.scenario"
            # Use second-level directory as subdirectory for logging (e.g., "TestOSM").
            log_sub_dir = scenario_file.split(os.path.sep)[1]
            log_dir = os.path.join(".", log_base_dir, log_sub_dir)
            
            makedirs_if_non_existing(log_dir)
            
138
            scenario_name = os.path.basename(scenario_file).split('.')[0]
139
            log_file = os.path.join(log_dir, ".".join([scenario_name, "log"]))
140

141
            print("  Log file: " + log_file)
142
143

            # Measure wall time and not CPU time.
144
145
            wall_time_start = time.time()

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
            # Build subprocess args:
            # Basic, always enable asserttions for tests
            subprocess_args = ["java",
                               "-enableassertions",
                               "-jar", vadere_console,
                               "--logname", log_file]
            # add config file if required
            if config_filepath is not None:
                subprocess_args += ["--config-file", config_filepath]

            # run per scenario given the scenario path and output directory
            subprocess_args += ["scenario-run", "-f", scenario_file, "-o", output_dir]

            # Use timout feature, check return value and capture stdout/stderr to a PIPE (use completed_process.stdout
            # to get it).
            completed_process = subprocess.run(args=subprocess_args,
162
163
164
165
                                               timeout=scenario_timeout_in_sec,
                                               check=True,
                                               stdout=subprocess.PIPE,
                                               stderr=subprocess.PIPE)
166
167
168
169

            wall_time_end = time.time()
            wall_time_delta = wall_time_end - wall_time_start

170
            print("Finished scenario file ({:.1f} s): {}".format(wall_time_delta, scenario_file))
171
            passed_scenarios.append(scenario_file)
172

173
        except subprocess.TimeoutExpired as exception:
174
            print("Scenario file failed: {}".format(scenario_file))
Marion Goedel's avatar
Marion Goedel committed
175
            print("->  Reason: timeout after {} s".format(exception.timeout))
176
            
177
178
            failed_summary.append("Scenario file failed: {}".format(scenario_file))
            failed_summary.append("->  Reason: timeout after {} s".format(exception.timeout))
179
180
            failed_scenarios_with_exception.append((scenario_file, exception))
        except subprocess.CalledProcessError as exception:
181
            print("Scenario file failed: {}".format(scenario_file))
Marion Goedel's avatar
Marion Goedel committed
182
            print("->  Reason: non-zero return value {}".format(exception.returncode))
183
            print("            {}".format(read_first_error_lines(exception.stderr)))
184
185
            
            failed_summary.append("Scenario file failed: {}".format(scenario_file))
186
            failed_summary.append("->  Reason: non-zero return value {}".format(exception.returncode))
187
            failed_summary.append("            {}".format(read_first_error_lines(exception.stderr)))
188

189
            failed_scenarios_with_exception.append((scenario_file, exception))
190
191
192
193

    if os.path.exists(output_dir):
        shutil.rmtree(output_dir)

194
    return result_dict_create(passed_scenarios, failed_scenarios_with_exception, failed_summary)
195

196

197
198
199
200
def makedirs_if_non_existing(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)

201

202
def read_first_error_lines(error_byte_string):
203
204
205
206
    err_string_lines = error_byte_string.decode('utf-8').split('\n')
    if len(err_string_lines) >= 2:
        return re.sub('\s+', ' ', ' '.join(err_string_lines[:2]))
    else:
207
        return 'Unknown error see Vadere log file.'
208

209

210
211
212
213
214
def result_dict_create(passed, failed, failed_summary):
    return {"passed": passed, "failed": failed, "failed_summary": failed_summary}


def result_dict_print_summary(passed_and_failed_scenarios):
215
    total_passed_scenarios = len(passed_and_failed_scenarios["passed"])
216
    total_failed_scenarios = len(passed_and_failed_scenarios["failed"])
217
    total_scenarios = total_passed_scenarios + total_failed_scenarios
218
219
220
    faild_summary = passed_and_failed_scenarios["failed_summary"]

    if len(faild_summary) > 0:
221
        print("##################")
222
        print("# Failed Summary #")
223
        print("##################")
224
225
226
        for line in faild_summary:
            print(line)

227
228
229
230
231
232
233
234
    print("###########")
    print("# Summary #")
    print("###########")
    print("")
    print("Total scenario runs: {}".format(total_scenarios))
    print("Passed: {}".format(total_passed_scenarios))
    print("Failed: {}".format(total_failed_scenarios))

235

236
def result_dict_has_failed_tests(passed_and_failed_scenarios):
237
238
239
    return len(passed_and_failed_scenarios["failed"]) > 0


240
241
242
243
244
245
246
247
248
249
250
251
def result_dict_merge(merge_into, merge_from):
    # both dict and list (values in result_dict) are mutable, so it is safer to make a copy of both and return a fresh
    # instance
    merge_into = copy.deepcopy(merge_into)
    merge_from = copy.deepcopy(merge_from)

    merge_into["failed"].extend(merge_from["failed"])
    merge_into["passed"].extend(merge_from["passed"])
    merge_into["failed_summary"].extend(merge_from["failed_summary"])
    return merge_into


252
if __name__ == "__main__":
253
    args = parse_command_line_arguments()
254

255
256
257
258
    if args.scenario is None:

        passed_and_failed_scenarios_model = run_all_model_tests()

259
        if not result_dict_has_failed_tests(passed_and_failed_scenarios_model):
260
261
            passed_and_failed_scenarios_optimization = run_all_optimization_tests()
        else:
Daniel Lehmberg's avatar
Daniel Lehmberg committed
262
263
            print("Skipping optimization tests...")
            passed_and_failed_scenarios_optimization = {"failed": [], "passed": [], "failed_summary": []}  # empty
264
265

        # Make a summery of all scenario files
266
267
        all_passed_and_failed_scenarios = result_dict_merge(passed_and_failed_scenarios_model,
                                                            passed_and_failed_scenarios_optimization)
268

269
    else:
Daniel Lehmberg's avatar
Daniel Lehmberg committed
270
        all_passed_and_failed_scenarios = run_scenario_files_with_vadere_console([args.scenario])
Marion Goedel's avatar
Marion Goedel committed
271

272
    result_dict_print_summary(all_passed_and_failed_scenarios)
273

274
    if result_dict_has_failed_tests(all_passed_and_failed_scenarios):
275
276
277
        exit(1)
    else:
        exit(0)