Commit b06951da authored by Benedikt Kleinmeier's avatar Benedikt Kleinmeier

In "run_vadere_console_with_all_scenario_files.py", store log files in subdirs...

In "run_vadere_console_with_all_scenario_files.py", store log files in subdirs (e.g., "TestGNM", "TestOSM" etc.).

Tools/ContinuousIntegration/run_vadere_console_with_all_scenario_files.py:
- Argument parser uses a **positional**, but optional, argument to run a single scenario file instead of an **optional** argument provided with `--scenario <name>`.

Tools/collect_line_and_branch_coverage.py:
- Deleted this duplicated file.
parent ad2d8105
Pipeline #115163 canceled with stages
...@@ -70,7 +70,7 @@ stages: ...@@ -70,7 +70,7 @@ stages:
artifacts: artifacts:
when: on_failure when: on_failure
paths: paths:
- "log_dir" - "vadere_logs"
expire_in: 1 week expire_in: 1 week
when: on_success when: on_success
......
...@@ -21,7 +21,7 @@ short_timeout_in_seconds = 2 * 60 ...@@ -21,7 +21,7 @@ short_timeout_in_seconds = 2 * 60
def parse_command_line_arguments(): def parse_command_line_arguments():
parser = argparse.ArgumentParser(description="Run all scenario files.") parser = argparse.ArgumentParser(description="Run all scenario files.")
parser.add_argument("--scenario", "-s", type=str, nargs="?", help="Run only the given scenario file and not all. E.g. \"VadereModelTests/TestOSM/scenarios/basic_2_density_discrete_ca.scenario\"") parser.add_argument("scenario", type=str, nargs="?", help="Run only the given scenario file and not all. E.g., \"VadereModelTests/TestOSM/scenarios/basic_2_density_discrete_ca.scenario\"")
args = parser.parse_args() args = parser.parse_args()
...@@ -80,13 +80,10 @@ def find_scenario_files(path="VadereModelTests", scenario_search_pattern = "*.sc ...@@ -80,13 +80,10 @@ def find_scenario_files(path="VadereModelTests", scenario_search_pattern = "*.sc
def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereSimulator/target/vadere-console.jar", scenario_timeout_in_sec=short_timeout_in_seconds): def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereSimulator/target/vadere-console.jar", scenario_timeout_in_sec=short_timeout_in_seconds):
output_dir = "output" output_dir = "output"
log_dir = "log_dir" log_base_dir = "vadere_logs"
if not os.path.exists(output_dir): makedirs_if_non_existing(output_dir)
os.makedirs(output_dir) makedirs_if_non_existing(log_base_dir)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
total_scenario_files = len(scenario_files) total_scenario_files = len(scenario_files)
...@@ -98,10 +95,18 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader ...@@ -98,10 +95,18 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader
try: try:
print("Running scenario file ({}/{}): {}".format(i + 1, total_scenario_files, scenario_file)) print("Running scenario file ({}/{}): {}".format(i + 1, total_scenario_files, scenario_file))
# A scenario filename has the form "VadereModelTests/TestOSM/scenarios/chicken_floorfield_ok.scenario"
# Use second-level directory as subdirectory for logging (e.g., "TestOSM").
log_sub_dir = scenario_file.split(os.path.sep)[1]
log_dir = os.path.join(".", log_base_dir, log_sub_dir)
makedirs_if_non_existing(log_dir)
scenario_name = os.path.basename(scenario_file).split('.')[0] scenario_name = os.path.basename(scenario_file).split('.')[0]
log_file = os.path.join(".", log_dir, scenario_name + ".log") log_file = os.path.join(log_dir, scenario_name + ".log")
print(log_file) print(" Log file: " + log_file)
# Measure wall time and not CPU time. # Measure wall time and not CPU time.
wall_time_start = time.time() wall_time_start = time.time()
...@@ -124,17 +129,16 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader ...@@ -124,17 +129,16 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader
except subprocess.TimeoutExpired as exception: except subprocess.TimeoutExpired as exception:
print("Scenario file failed: {}".format(scenario_file)) print("Scenario file failed: {}".format(scenario_file))
print("-> Reason: timeout after {} s".format(exception.timeout)) print("-> Reason: timeout after {} s".format(exception.timeout))
failed_summary.append("Scenario file failed: {}".format(scenario_file)) failed_summary.append("Scenario file failed: {}".format(scenario_file))
failed_summary.append("-> Reason: timeout after {} s".format(exception.timeout)) failed_summary.append("-> Reason: timeout after {} s".format(exception.timeout))
failed_scenarios_with_exception.append((scenario_file, exception)) failed_scenarios_with_exception.append((scenario_file, exception))
except subprocess.CalledProcessError as exception: except subprocess.CalledProcessError as exception:
prefix = "" print("Scenario file failed: {}".format(scenario_file))
if "TestOSM" in scenario_file:
prefix = " * OSM * "
print(prefix + "Scenario file failed: {}".format(scenario_file))
print("-> Reason: non-zero return value {}".format(exception.returncode)) print("-> Reason: non-zero return value {}".format(exception.returncode))
print(" {}".format(read_first_error_linies(exception.stderr))) print(" {}".format(read_first_error_linies(exception.stderr)))
failed_summary.append(prefix + "Scenario file failed: {}".format(scenario_file))
failed_summary.append("Scenario file failed: {}".format(scenario_file))
failed_summary.append("-> Reason: non-zero return value {}".format(exception.returncode)) failed_summary.append("-> Reason: non-zero return value {}".format(exception.returncode))
failed_summary.append(" {}".format(read_first_error_linies(exception.stderr))) failed_summary.append(" {}".format(read_first_error_linies(exception.stderr)))
...@@ -145,6 +149,10 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader ...@@ -145,6 +149,10 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader
return {"passed": passed_scenarios, "failed": failed_scenarios_with_exception, "failed_summary": failed_summary} return {"passed": passed_scenarios, "failed": failed_scenarios_with_exception, "failed_summary": failed_summary}
def makedirs_if_non_existing(directory):
if not os.path.exists(directory):
os.makedirs(directory)
def read_first_error_linies(error_byte_string): def read_first_error_linies(error_byte_string):
err_string_lines = error_byte_string.decode('utf-8').split('\n') err_string_lines = error_byte_string.decode('utf-8').split('\n')
if len(err_string_lines) >= 2: if len(err_string_lines) >= 2:
......
# Extract line and branch coverage (in percentage) from HTML coverage reports
# which are created by Maven's jacoco plugin.
# Use top-level pom.xml to search in correct subdirectories.
#
# Wach out: call this script from root directory of project. E.g.
#
# python Tools/my_script.py
import xml.etree.ElementTree as ET
import os
import re
def get_modules_from_pom_file(filename="pom.xml"):
"""Return a list of submodules which where found in passed "pom.xml"."""
xml_name_spaces = {
"default": "http://maven.apache.org/POM/4.0.0"
}
xml_tree = ET.parse(filename)
xml_root = xml_tree.getroot()
exclude_list = ["./VadereAnnotation", "./VadereGui"]
modules = xml_root.findall("default:modules/default:module", xml_name_spaces)
module_names = [module.text for module in modules if module.text not in exclude_list]
return module_names
def extract_line_and_branch_coverage(module_names):
"""Return a dictionary which maps module name to line and branch coverage tuple."""
module_to_coverage = dict()
default_coverage_file = "target/site/coverage-reports/index.html"
for module in module_names:
coverage_path = os.path.join(module, default_coverage_file)
with open(coverage_path, "r") as file:
coverage_report = file.read()
regex_pattern = re.compile(r"Total.*?([0-9]{1,3})%.*?([0-9]{1,3})%")
match = regex_pattern.search(coverage_report)
if match:
line_coverage = float(match.group(1))
branch_coverage = float(match.group(2))
module_to_coverage[module] = (line_coverage, branch_coverage)
else:
raise Exception("Coverage data not found for module: {}".format(module))
return module_to_coverage
def print_averaged_line_coverage(coverage_data):
"""GitLab CI tools read out the stdout output of the build process. Therefore, print coverage info to stdout."""
total_modules = len(coverage_data)
line_coverage_data = [line_coverage for (line_coverage, branch_coverage) in coverage_data.values()]
branch_coverage_data = [branch_coverage for (line_coverage, branch_coverage) in coverage_data.values()]
summed_line_coverage_data = sum(line_coverage_data)
summed_branch_coverage_data = sum(branch_coverage_data)
# By default, GitLab CI parses only integers.
averaged_line_coverage = int(round(summed_line_coverage_data / total_modules, 0))
averaged_branch_coverage = int(round(summed_branch_coverage_data / total_modules, 0))
print("Line Coverage: Total {}%".format(averaged_line_coverage))
print("Branch Coverage: {}%".format(averaged_branch_coverage))
if __name__ == "__main__":
module_names = get_modules_from_pom_file()
module_to_coverage = extract_line_and_branch_coverage(module_names)
print_averaged_line_coverage(module_to_coverage)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment