Commit 4ae168cb authored by Benedikt Kleinmeier's avatar Benedikt Kleinmeier

In "run_seed_comparison_test.py" and...

In "run_seed_comparison_test.py" and "run_vadere_console_with_all_scenario_files.py", increased timeouts to avoid errors during CI pipeline.

Further changes:
- Renamed test directory "TESTOVM" to "TestOVM".
- Renamed test directory "TestOSM_Group" to "TestOSMGroup".
parent 49d5419a
......@@ -8,7 +8,7 @@ from vadereanalysistool import SameSeedTrajectory
def run_scenario_files_with_vadere_console(project, number_of_runs=3, vadere_console="VadereSimulator/target/vadere-console.jar",
scenario_timeout_in_sec=60*9):
scenario_timeout_in_sec=12*60):
if not os.path.exists(vadere_console):
raise ValueError("vadere console could not be found at path: {}".format(os.path.abspath(vadere_console)))
......
......@@ -8,16 +8,58 @@
#
# python Tools/my_script.py
import argparse
import fnmatch
import os
import re
import shutil
import subprocess
import time
import argparse
long_timeout_in_seconds = 12 * 60
short_timeout_in_seconds = 2 * 60
def parse_command_line_arguments():
parser = argparse.ArgumentParser(description="Run all scenario files.")
parser.add_argument("--scenario", "-s", type=str, nargs="?", help="Run only the given scenario file and not all. E.g. \"VadereModelTests/TestOSM/scenarios/basic_2_density_discrete_ca.scenario\"")
args = parser.parse_args()
return args
def run_all():
long_running_scenarios = [
"basic_4_1_wall_gnm1",
"queueing",
"rimea_09",
"rimea_11",
"TestSFM",
"thin_wall_and_closer_source_nelder_mead_ok",
"thin_wall_and_closer_source_pso_could_fail",
"rimea_04_flow_osm1_550_up",
"stairs_diagonal_both_1_2_+1.scenario",
"05_bang_event_narrowed_street",
"06_bang_event_guimaraes_platz"
]
excluded_scenarios = ["TestOVM", "output", "legacy"]
excluded_scenarios.extend(long_running_scenarios)
scenario_files_regular_length = find_scenario_files(exclude_patterns=excluded_scenarios)
passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files_regular_length, scenario_timeout_in_sec=short_timeout_in_seconds)
def find_scenario_files(path="VadereModelTests", scenario_search_pattern = "*.scenario", exclude_patterns = ["TESTOVM", "output","legacy"]):
for scenario in long_running_scenarios:
search_pattern = "*" + scenario + "*.scenario"
scenario_files_long = find_scenario_files(scenario_search_pattern=search_pattern)
tmp_passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files_long,
scenario_timeout_in_sec=long_timeout_in_seconds)
passed_and_failed_scenarios["passed"].extend(tmp_passed_and_failed_scenarios["passed"])
passed_and_failed_scenarios["failed"].extend(tmp_passed_and_failed_scenarios["failed"])
passed_and_failed_scenarios["failed_summary"].extend(tmp_passed_and_failed_scenarios["failed_summary"])
return passed_and_failed_scenarios
def find_scenario_files(path="VadereModelTests", scenario_search_pattern = "*.scenario", exclude_patterns = ["TestOVM", "output","legacy"]):
scenario_files = []
for root, dirnames, filenames in os.walk(path):
......@@ -36,7 +78,7 @@ def find_scenario_files(path="VadereModelTests", scenario_search_pattern = "*.sc
return sorted(scenario_files)
def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereSimulator/target/vadere-console.jar", scenario_timeout_in_sec=60):
def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereSimulator/target/vadere-console.jar", scenario_timeout_in_sec=short_timeout_in_seconds):
output_dir = "output"
log_dir = "log_dir"
......@@ -57,9 +99,11 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader
try:
print("Running scenario file ({}/{}): {}".format(i + 1, total_scenario_files, scenario_file))
scenario_name = os.path.basename(scenario_file).split('.')[0]
log_file = os.path.join(log_dir, scenario_name + ".log")
log_file = os.path.join(".", log_dir, scenario_name + ".log")
# Measure wall time and not CPU time simply because it is the simplest method.
print(log_file)
# Measure wall time and not CPU time.
wall_time_start = time.time()
# Use timout feature, check return value and capture stdout/stderr to a PIPE (use completed_process.stdout to get it).
......@@ -75,9 +119,8 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader
wall_time_delta = wall_time_end - wall_time_start
print("Finished scenario file ({:.1f} s): {}".format(wall_time_delta, scenario_file))
passed_scenarios.append(scenario_file)
except subprocess.TimeoutExpired as exception:
print("Scenario file failed: {}".format(scenario_file))
print("-> Reason: timeout after {} s".format(exception.timeout))
......@@ -109,7 +152,6 @@ def read_first_error_linies(error_byte_string):
else:
return 'unknown error see vadere log file.'
def print_summary(passed_and_failed_scenarios):
total_passed_scenarios = len(passed_and_failed_scenarios["passed"])
total_failed_scenarios = + len(passed_and_failed_scenarios["failed"])
......@@ -117,9 +159,9 @@ def print_summary(passed_and_failed_scenarios):
faild_summary = passed_and_failed_scenarios["failed_summary"]
if len(faild_summary) > 0:
print("#################")
print("##################")
print("# Failed Summary #")
print("#################")
print("##################")
for line in faild_summary:
print(line)
......@@ -132,49 +174,13 @@ def print_summary(passed_and_failed_scenarios):
print("Passed: {}".format(total_passed_scenarios))
print("Failed: {}".format(total_failed_scenarios))
def run_all():
long_running_scenarios = [
"basic_4_1_wall_gnm1",
"queueing",
"rimea_09",
"rimea_11",
"TestSFM",
"thin_wall_and_closer_source_nelder_mead_ok",
"thin_wall_and_closer_source_pso_could_fail",
"rimea_04_flow_osm1_550_up",
"stairs_diagonal_both_1_2_+1.scenario",
"05_bang_event_narrowed_street",
"06_bang_event_guimaraes_platz"
]
excluded_scenarios = ["TESTOVM", "output", "legacy"]
excluded_scenarios.extend(long_running_scenarios)
scenario_files_regular_length = find_scenario_files(exclude_patterns=excluded_scenarios)
passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files_regular_length)
for scenario in long_running_scenarios:
search_pattern = "*" + scenario + "*.scenario"
scenario_files_long = find_scenario_files(scenario_search_pattern=search_pattern)
tmp_passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files_long,
scenario_timeout_in_sec=540)
passed_and_failed_scenarios["passed"].extend(tmp_passed_and_failed_scenarios["passed"])
passed_and_failed_scenarios["failed"].extend(tmp_passed_and_failed_scenarios["failed"])
passed_and_failed_scenarios["failed_summary"].extend(tmp_passed_and_failed_scenarios["failed_summary"])
return passed_and_failed_scenarios
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Run ModelTest')
parser.add_argument('--scenario', '-s', nargs='?', default='', metavar='S', help='only run one selected scenario')
args = vars(parser.parse_args())
if args['scenario'] == '':
args = parse_command_line_arguments()
if args.scenario == None:
passed_and_failed_scenarios = run_all()
else:
passed_and_failed_scenarios = run_scenario_files_with_vadere_console([args['scenario']])
passed_and_failed_scenarios = run_scenario_files_with_vadere_console([args.scenario])
print_summary(passed_and_failed_scenarios)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment