run_vadere_console_with_all_scenario_files.py 5.03 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
# Use "vadere-console.jar", which is created by "mvn package", to run all
# scenario files under "VadereModelTests" subdirectory.
#
# Note: script contains some print statements so that progress can be tracked
# a little bit

# Wach out: call this script from root directory of project. E.g.
#
#   python Tools/my_script.py

import fnmatch
import os
13
import re
14
15
import shutil
import subprocess
16
import time
17

18
def find_scenario_files(path="VadereModelTests", scenario_search_pattern = "*.scenario", exclude_patterns = ["TESTOVM", "output","legacy"]):
19
20
21
22
23
    scenario_files = []

    for root, dirnames, filenames in os.walk(path):
        for filename in fnmatch.filter(filenames, scenario_search_pattern):
            scenario_path = os.path.join(root, filename)
24
            scenario_path_excluded = False
25
26
27
28

            for exclude_pattern in exclude_patterns:
                regex_pattern = re.compile(exclude_pattern)
                match = regex_pattern.search(scenario_path)
Marion Goedel's avatar
Marion Goedel committed
29
                if match:
30
                    scenario_path_excluded = True
31

32
            if scenario_path_excluded == False:
Marion Goedel's avatar
Marion Goedel committed
33
                scenario_files.append(scenario_path)
34

35
    print("Total scenario files: {}".format(len(scenario_files)))
Marion Goedel's avatar
Marion Goedel committed
36
    # print("Exclude patterns: {}".format(exclude_patterns))
37

38
    return sorted(scenario_files)
39

40
def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereGui/target/vadere-console.jar", scenario_timeout_in_sec=60):
41
42
43
44
45
    output_dir = "output"

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

46
47
    total_scenario_files = len(scenario_files)

48
49
    passed_scenarios = []
    failed_scenarios_with_exception = []
50

51
52
    for i, scenario_file in enumerate(scenario_files):
        try:
53
            # Surpress this output to print only failed scenarios.
Marion Goedel's avatar
Marion Goedel committed
54
            # print("Running scenario file ({}/{}): {}".format(i + 1, total_scenario_files, scenario_file))
55
56
57
58

            # Measure wall time and not cpu because it is the easiest.
            wall_time_start = time.time()

Marion Goedel's avatar
Marion Goedel committed
59

60
            # Use timout feature, check return value and capture stdout/stderr to a PIPE (use completed_process.stdout to get it).
Stefan Schuhbaeck's avatar
Stefan Schuhbaeck committed
61
            completed_process = subprocess.run(args=["java", "-enableassertions", "-jar", vadere_console, "scenario-run", "-f", scenario_file, "-o", output_dir],
62
63
64
65
66
67
68
69
                                           timeout=scenario_timeout_in_sec,
                                           check=True,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE)

            wall_time_end = time.time()
            wall_time_delta = wall_time_end - wall_time_start

70
            # Surpress this output to print only failed scenarios.
Marion Goedel's avatar
Marion Goedel committed
71
            # print("Finished scenario file ({:.1f} s): {}".format(wall_time_delta, scenario_file))
72
73
74

            passed_scenarios.append(scenario_file)
        except subprocess.TimeoutExpired as exception:
Marion Goedel's avatar
Marion Goedel committed
75
            prefix = ""
76
            if "TestOSM" in scenario_file:
Marion Goedel's avatar
Marion Goedel committed
77
78
                prefix = " * OSM * "

79
            print(prefix + "Scenario file failed: {}".format(scenario_file))
Marion Goedel's avatar
Marion Goedel committed
80
            print("->  Reason: timeout after {} s".format(exception.timeout))
81
82
            failed_scenarios_with_exception.append((scenario_file, exception))
        except subprocess.CalledProcessError as exception:
Marion Goedel's avatar
Marion Goedel committed
83
            prefix = ""
84
85
            if "TestOSM" in scenario_file:
                prefix = " * OSM * "
Marion Goedel's avatar
Marion Goedel committed
86
87
            print(prefix + "Scenario file failed: {}".format(scenario_file))
            print("->  Reason: non-zero return value {}".format(exception.returncode))
88
            failed_scenarios_with_exception.append((scenario_file, exception))
89

Marion Goedel's avatar
Marion Goedel committed
90

91
92
93
    if os.path.exists(output_dir):
        shutil.rmtree(output_dir)

94
95
    return {"passed": passed_scenarios, "failed": failed_scenarios_with_exception}

96
if __name__ == "__main__":
97
    print("Output is only shown for failed scenarios!")
Marion Goedel's avatar
Marion Goedel committed
98

99
100
101
102
103
104
105
106
107
    long_running_scenarios = [
            "basic_4_1_wall_gnm1",
            "queueing",
            "rimea_09",
            "rimea_11",
            "TestSFM",
            "thin_wall_and_closer_source_nelder_mead_ok",
            "thin_wall_and_closer_source_pso_could_fail"
    ]
108
109
    excluded_scenarios = ["TESTOVM", "output", "legacy"]
    excluded_scenarios.extend(long_running_scenarios)
Marion Goedel's avatar
Marion Goedel committed
110

111
    scenario_files_regular_length = find_scenario_files(exclude_patterns=excluded_scenarios)
112
    passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files_regular_length)
Marion Goedel's avatar
Marion Goedel committed
113

114
    for scenario in long_running_scenarios:
Marion Goedel's avatar
Marion Goedel committed
115
116
        search_pattern = "*" + scenario + "*.scenario"
        scenario_files_long = find_scenario_files(scenario_search_pattern=search_pattern)
117
        tmp_passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files_long, scenario_timeout_in_sec=480)
Marion Goedel's avatar
Marion Goedel committed
118
119
120
        passed_and_failed_scenarios["passed"].extend(tmp_passed_and_failed_scenarios["passed"])
        passed_and_failed_scenarios["failed"].extend(tmp_passed_and_failed_scenarios["failed"])

121
122
123
124
125

    if len(passed_and_failed_scenarios["failed"]) > 0:
        exit(1)
    else:
        exit(0)