run_orign_translation_test.py 6.94 KB
Newer Older
1 2 3 4 5
import os
import shutil
import subprocess
import time

6 7
from vadereanalysistool import VadereProject
from vadereanalysistool.analysis import OriginDeviation
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65


def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereSimulator/target/vadere-console.jar",
                                           scenario_timeout_in_sec=60):
    output_dir = "Tools/ContinuousIntegration/run_orign_translation_test.d/output"

    if os.path.exists(output_dir):
        shutil.rmtree(output_dir)

    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    total_scenario_files = len(scenario_files)

    passed_scenarios = []
    failed_scenarios_with_exception = []

    for i, scenario_file in enumerate(scenario_files):
        try:
            print("Running scenario file ({}/{}): {}".format(i + 1, total_scenario_files, scenario_file))

            # Measure wall time and not CPU time simply because it is the simplest method.
            wall_time_start = time.time()

            out = os.path.join(output_dir, os.path.basename(scenario_file).split('.')[0])

            # Use timout feature, check return value and capture stdout/stderr to a PIPE (use
            # completed_process.stdout to get it).
            print("subprocess call: {}".format(' '.join(
                ["java", "-enableassertions", "-jar", vadere_console, "scenario-run", "-f", scenario_file, "-o", out])))
            completed_process = subprocess.run(
                args=["java", "-enableassertions", "-jar", vadere_console, "scenario-run", "-f", scenario_file, "-o",
                      out, "--override-timestep-setting"],
                timeout=scenario_timeout_in_sec,
                check=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)

            wall_time_end = time.time()
            wall_time_delta = wall_time_end - wall_time_start

            print("Finished scenario file ({:.1f} s): {}".format(wall_time_delta, scenario_file))

            passed_scenarios.append(scenario_file)
        except subprocess.TimeoutExpired as exception:
            print("Scenario file failed: {}".format(scenario_file))
            print("->  Reason: timeout after {} s".format(exception.timeout))
            failed_scenarios_with_exception.append((scenario_file, exception))
        except subprocess.CalledProcessError as exception:
            print("Scenario file failed: {}".format(scenario_file))
            print("->  Reason: non-zero return value {}".format(exception.returncode))
            failed_scenarios_with_exception.append((scenario_file, exception))

    return {"passed": passed_scenarios, "failed": failed_scenarios_with_exception}


def get_output_pairs(path="Tools/ContinuousIntegration/run_orign_translation_test.d/output"):
    _output_pairs = []
Stefan Schuhbaeck's avatar
fixbug  
Stefan Schuhbaeck committed
66
    output_without_offset = [d for d in os.listdir(path) if d.endswith("without_offset")]
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
    for dir_without in output_without_offset:
        dir_with_offset = dir_without.replace("without_offset", "with_offset")
        name = dir_with_offset[:-len("without_offset")]
        pair = [os.path.join(path, dir_without), os.path.join(path, dir_with_offset)]
        _output_pairs.append(OriginDeviation(pair, name))

    return _output_pairs


def run_simulations(test_project_path):
    vadere_project = VadereProject(test_project_path)
    scenario_files = vadere_project.scenario_files
    passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files)

    if len(passed_and_failed_scenarios['failed']) > 0:
        print("Error in simulation runs: " + passed_and_failed_scenarios['failed'])
        exit(1)


if __name__ == '__main__':
    test_project_path = "Tools/ContinuousIntegration/run_orign_translation_test.d"

89
    run_simulations(test_project_path)
90 91 92

    output_pairs = get_output_pairs()
    print("comparing {} output pairs".format(len(output_pairs)))
93
    # results = [pair.get_origin_deviation_result() for pair in output_pairs]
94 95
    return_val = 0
    space_offset = ''
96 97 98 99
    all_results_count = len(output_pairs)
    err_results_count = 0
    warn_results_count = 0
    ok_results_coutn = 0
100 101 102 103 104

    for pair in output_pairs:
        result = pair.get_origin_deviation_result()
        print("*** compare trajectories for {} ... ".format(result['output_name']), end='')
        if result['err_count'] > 0:
105
            err_results_count += 1
106 107 108 109 110 111 112 113 114 115 116 117 118
            print("ERROR")
            return_val = 1
            print("    Error: Found {} pedestrians with maximal diff bigger than {}.".format(result['err_count'],
                                                                                             pair.max_diff_warn))
            print("    First occurrence of error for pedestrian at timestep \n    [pedestrianId, timeStep, diff]")
            for i, (row_id, row) in enumerate(result['err_dict'].items()):
                if i > 2:
                    print("    ...")
                    break
                print("    [{}, {}, {}]".format(row['pedestrianId'], row['timeStep'], row['diff']))

        if result['warn_count'] > 0:
            if result['err_count'] == 0:
119
                warn_results_count += 1
120 121 122 123 124 125 126 127 128 129 130 131
                print("WARN")
            print("    Warning: Found {} pedestrians with maximal diff between {} and {}.".format(result['warn_count'],
                                                                                                  pair.max_diff_ok,
                                                                                                  pair.max_diff_warn))
            print("    First occurrence of warning for pedestrian at \n    [pedestrianId, timeStep, diff]")
            for i, (row_id, row) in enumerate(result['warn_dict'].items()):
                if i > 2:
                    print("    ...")
                    break
                print("    [{}, {}, {}]".format(row['pedestrianId'], row['timeStep'], row['diff']))

        if result['err_count'] == 0 and result['warn_count'] == 0:
132
            ok_results_coutn += 1
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
            print("OK")

        if result['err_count'] > 0 or result['warn_count'] > 0:
            stats_at_max_diff = result['stats_at_max_diff']
            print("    Max diff Stats      | mean: {} std: {}, median: {}, min: {}, max: {}".format(
                stats_at_max_diff['mean'], stats_at_max_diff['std'], stats_at_max_diff['50%'],
                stats_at_max_diff['min'], stats_at_max_diff['max']
            ))

            stats_at_start = result['stats_at_start']
            print("    Start position Stats| mean: {} std: {}, median: {}, min: {}, max: {}".format(
                stats_at_start['mean'], stats_at_start['std'], stats_at_start['50%'],
                stats_at_start['min'], stats_at_start['max']
            ))


149 150 151 152 153 154 155 156
    print("###########")
    print("# Summary #")
    print("###########")
    print("")
    print("Total scenario runs: {}".format(all_results_count))
    print("Passed: {}".format(ok_results_coutn))
    print("Warning: {}".format(warn_results_count))
    print("Failed: {}".format(err_results_count))
157
    exit(return_val)