Merge remote-tracking branch 'origin/develop' into TL-BA-Target-Learning

# Conflicts:
#	VadereSimulator/src/org/vadere/simulator/control/SourceController.java
#	VadereState/src/org/vadere/state/attributes/scenario/AttributesSource.java

.gitignore

@@ -23,10 +23,12 @@ VadereSimulator/resources/current_commit_hash.txt
 
 #model test output
 VadereModelTests/TestOSM/output/*
+VadereModelTests/TestOSM_Group/output/*
 VadereModelTests/TestSFM/output/*
 VadereModelTests/TestGNM/output/*
 VadereModelTests/TestOVM/output/*
 VadereModelTests/TestOSM/processed output/*
+VadereModelTests/TestOSM_Group/processed output/*
 VadereModelTests/TestSFM/processed output/*
 VadereModelTests/TestGNM/processed output/*
 VadereModelTests/TestOVM/processed output/*
@@ -41,3 +43,4 @@ VadereGui/output/
 
 #linux deskopt dotfiles
 **/.directory
+/VadereModelTests/TestStairs/output

.gitlab-ci.yml

+# # GitLab CI Jobs
+#
+# Each of the jobs defined here is triggered by GitLab's CI pipeline.
+#
+# ## GitLab YAML Job Description Syntax
+#
+# <job_name>:                   # Job name which is displayed in web interface.
+#   stage: <stage_name>         # <build|test|deploy>: these stages are executed
+#                               # one after another. A stage can contain multiple
+#                               # jobs.
+#   script:
+#      - <arbitrary_commands>   # Here you can execute arbitratry terminal
+#                               # commands. If any of the commands return a
+#                               # non-zero exit code, the job fails.
+#
+# ## Maven Syntax for Skipping Unit Tests
+#
+# General syntax to execute unit tests:
+#
+# ```
+# mvn test
+# ```
+#
+# The comma-separated list of tests after "-Dtest=<list>" is skipped. All other
+# tests are executed normally:
+#
+# ```
+# mvn -Dtest=!TestClassA,!TestClassB,!TestClassC
+# ```
+#
+# **Note:** on some CI instances the exclamation mark must be escaped by a
+# backslash. Otherwise, it is interpreted by the executing shell as special
+# command.
+#
+# ## Unit Test Limitations
+#
+# **Watch out:** OpenCL code cannot be run on virtual machines where tests are
+# executed. Therefore, skip this tests.
+
 unit_tests_with_coverage:
-  # A job is always executed within a stage.
-  # If no stage is set, it defaults to 'test'.
   stage: test
   
-  # Here you can execute arbitrate terminal commands.
-  # If any of the commands returns a non zero exit code the job fails.
   script:
     - mvn clean
-    - mvn test
-    
\ No newline at end of file
+    - mvn -Dtest=!TestConvolution,!TestBitonicSort,!TestCLLinkedList test
+    - python3 Tools/ContinuousIntegration/collect_line_and_branch_coverage.py
+
+run_scenario_files:
+  stage: deploy
+
+  script:
+      - mvn clean
+      - mvn -Dmaven.test.skip=true package
+      - python3 Tools/ContinuousIntegration/run_vadere_console_with_all_scenario_files.py
+

Tools/ContinuousIntegration/collect_line_and_branch_coverage.py

+# Extract line and branch coverage (in percentage) from HTML coverage reports
+# which are created by Maven's jacoco plugin.
+# Use top-level pom.xml to search in correct subdirectories.
+#
+# Wach out: call this script from root directory of project. E.g.
+#
+#   python Tools/my_script.py
+
+import xml.etree.ElementTree as ET
+
+import os
+import re
+
+def get_modules_from_pom_file(filename="pom.xml"):
+    """Return a list of submodules which where found in passed "pom.xml"."""
+
+    xml_name_spaces = {
+            "default": "http://maven.apache.org/POM/4.0.0"
+    }
+    
+    xml_tree = ET.parse(filename)
+    xml_root = xml_tree.getroot()
+    
+    exclude_list = ["./VadereAnnotation", "./VadereGui"]
+    
+    modules = xml_root.findall("default:modules/default:module", xml_name_spaces)
+    module_names = [module.text for module in modules if module.text not in exclude_list]
+    
+    return module_names
+
+def extract_line_and_branch_coverage(module_names):
+    """Return a dictionary which maps module name to line and branch coverage tuple."""
+
+    module_to_coverage = dict()
+
+    default_coverage_file = "target/site/coverage-reports/index.html"
+
+    for module in module_names:
+        coverage_path = os.path.join(module, default_coverage_file)
+
+        with open(coverage_path, "r") as file:
+            coverage_report = file.read()
+
+            regex_pattern = re.compile(r"Total.*?([0-9]{1,3})%.*?([0-9]{1,3})%")
+            match = regex_pattern.search(coverage_report)
+
+            if match:
+                line_coverage = float(match.group(1))
+                branch_coverage = float(match.group(2))
+                module_to_coverage[module] = (line_coverage, branch_coverage)
+            else:
+                raise Exception("Coverage data not found for module: {}".format(module))
+
+    return module_to_coverage
+
+def print_averaged_line_coverage(coverage_data):
+    """GitLab CI tools read out the stdout output of the build process. Therefore, print coverage info to stdout."""
+
+    total_modules = len(coverage_data)
+    line_coverage_data = [line_coverage for (line_coverage, branch_coverage) in coverage_data.values()]
+    branch_coverage_data = [branch_coverage for (line_coverage, branch_coverage) in coverage_data.values()]
+
+    summed_line_coverage_data = sum(line_coverage_data)
+    summed_branch_coverage_data = sum(branch_coverage_data)
+
+    # By default, GitLab CI parses only integers.
+    averaged_line_coverage = int(round(summed_line_coverage_data / total_modules, 0))
+    averaged_branch_coverage = int(round(summed_branch_coverage_data / total_modules, 0))
+
+    print("Analyzed modules: {}".format(sorted(coverage_data.keys())))
+    print("Line Coverage: Total {}%".format(averaged_line_coverage))
+    print("Branch Coverage: Total {}%".format(averaged_branch_coverage))
+
+if __name__ == "__main__":
+    module_names = get_modules_from_pom_file()
+    module_to_coverage = extract_line_and_branch_coverage(module_names)
+    print_averaged_line_coverage(module_to_coverage)

Tools/ContinuousIntegration/run_vadere_console_with_all_scenario_files.py

+# Use "vadere-console.jar", which is created by "mvn package", to run all
+# scenario files under "VadereModelTests" subdirectory.
+#
+# Note: script contains some print statements so that progress can be tracked
+# a little bit
+
+# Wach out: call this script from root directory of project. E.g.
+#
+#   python Tools/my_script.py
+
+import fnmatch
+import os
+import re
+import shutil
+import subprocess
+import time
+
+def find_scenario_files(path="VadereModelTests"):
+    scenario_search_pattern = "*.scenario"
+    scenario_files = []
+    exclude_patterns = ["TESTOVM"]
+
+    for root, dirnames, filenames in os.walk(path):
+        for filename in fnmatch.filter(filenames, scenario_search_pattern):
+            scenario_path = os.path.join(root, filename)
+
+            for exclude_pattern in exclude_patterns:
+                regex_pattern = re.compile(exclude_pattern)
+                match = regex_pattern.search(scenario_path)
+
+                if match is None:
+                    scenario_files.append(scenario_path)
+
+    print("Total scenario files: {}".format(len(scenario_files)))
+    print("Exclude patterns: {}".format(exclude_patterns))
+
+    return sorted(scenario_files)
+
+def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereGui/target/vadere-console.jar", scenario_timeout_in_sec=60):
+    output_dir = "output"
+
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    total_scenario_files = len(scenario_files)
+
+    passed_scenarios = []
+    failed_scenarios_with_exception = []
+
+    for i, scenario_file in enumerate(scenario_files):
+        try:
+            print("Running scenario file ({}/{}): {}".format(i + 1, total_scenario_files, scenario_file))
+
+            # Measure wall time and not cpu because it is the easiest.
+            wall_time_start = time.time()
+
+            # Use timout feature, check return value and capture stdout/stderr to a PIPE (use completed_process.stdout to get it).
+            completed_process = subprocess.run(args=["java", "-enableassertions", "-jar", vadere_console, scenario_file, output_dir],
+                                           timeout=scenario_timeout_in_sec,
+                                           check=True,
+                                           stdout=subprocess.PIPE,
+                                           stderr=subprocess.PIPE)
+
+            wall_time_end = time.time()
+            wall_time_delta = wall_time_end - wall_time_start
+
+            print("Finished scenario file ({:.1f} s): {}".format(wall_time_delta, scenario_file))
+
+            passed_scenarios.append(scenario_file)
+        except subprocess.TimeoutExpired as exception:
+            print("Scenario file failed: {}".format(scenario_file))
+            print("->  Reason: timeout after {} s ({})".format(exception.timeout, exception.cmd))
+            failed_scenarios_with_exception.append((scenario_file, exception))
+        except subprocess.CalledProcessError as exception:
+            print("Scenario file failed: {}".format(scenario_file))
+            print("->  Reason: non-zero return value {} ({})".format(exception.returncode, exception.cmd))
+            failed_scenarios_with_exception.append((scenario_file, exception))
+
+    if os.path.exists(output_dir):
+        shutil.rmtree(output_dir)
+
+    return {"passed": passed_scenarios, "failed": failed_scenarios_with_exception}
+
+if __name__ == "__main__":
+    scenario_files = find_scenario_files()
+    passed_and_failed_scenarios = run_scenario_files_with_vadere_console(scenario_files)
+
+    if len(passed_and_failed_scenarios["failed"]) > 0:
+        exit(1)
+    else:
+        exit(0)
+

VadereAnnotation/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>vadere</artifactId>
+        <groupId>org.vadere</groupId>
+        <version>0.1-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>annotation</artifactId>
+    <name>Vadere Annotation</name>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>com.google.auto.service</groupId>
+            <artifactId>auto-service</artifactId>
+            <version>${auto-service.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <sourceDirectory>src</sourceDirectory>
+        <testSourceDirectory>tests</testSourceDirectory>
+        <resources>
+            <resource>
+                <directory>src</directory>
+                <excludes>
+                    <exclude>**/*.java</exclude>
+                </excludes>
+            </resource>
+            <resource>
+                <directory>resources</directory>
+            </resource>
+        </resources>
+        <testResources>
+            <testResource>
+                <directory>testResources</directory>
+            </testResource>
+            <testResource>
+                <directory>tests</directory>
+                <excludes>
+                    <exclude>**/*.java</exclude>
+                </excludes>
+            </testResource>
+        </testResources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>${maven-compiler.version}</version>
+                <configuration>
+                    <source>${maven-compiler.source.version}</source>
+                    <target>${maven-compiler.target.version}</target>
+                </configuration>
+            </plugin>
+            <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>0.7.9</version>
+            <executions>
+                <!-- Use JaCoCo's Java agents to record coverage data by using on-the-fly class file instrumentation. -->
+                <execution>
+                    <id>pre-unit-test</id>
+                    <goals>
+                        <goal>prepare-agent</goal>
+                    </goals>
+                    <configuration>
+                        <output>file</output>
+                        <destFile>${project.build.directory}/coverage-data/jacoco-unit-tests.coverage</destFile>
+                        <propertyName>surefireArgLine</propertyName>
+                    </configuration>
+                </execution>
+                <!-- Use JaCoCo agent output to create coverage reports in CSV, HTML and XML format. -->
+                <execution>
+                    <id>post-unit-test</id>
+                    <phase>test</phase>
+                    <goals>
+                        <goal>report</goal>
+                    </goals>
+                    <configuration>
+                        <dataFile>${project.build.directory}/coverage-data/jacoco-unit-tests.coverage</dataFile>
+                        <outputDirectory>${project.reporting.outputDirectory}/coverage-reports</outputDirectory>
+                    </configuration>
+                </execution>
+            </executions>    
+        </plugin>
+        <plugin>
+            <!-- Maven Surefire is used to execute unit tests. -->
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>2.19</version>
+            <configuration>
+                <argLine>${surefireArgLine}</argLine>
+                <enableAssertions>true</enableAssertions>
+                <excludes>
+                    <exclude></exclude>
+                </excludes>
+                <failIfNoTests>false</failIfNoTests>
+                <skipTests>${skip.unit.tests}</skipTests>
+            </configuration>
+        </plugin>
+        </plugins>
+
+    </build>
+
+</project>
\ No newline at end of file

VadereAnnotation/src/org/vadere/annotation/factories/AbstractFactoryProcessor.java

+package org.vadere.annotation.factories;
+
+import org.vadere.annotation.factories.dataprocessors.DataProcessorClass;
+import org.vadere.annotation.factories.outputfiles.OutputFileClass;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.RoundEnvironment;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.ElementKind;
+import javax.lang.model.element.Modifier;
+import javax.lang.model.element.TypeElement;
+
+/**
+ * Base FactoryProcessor to generate Factories
+ *
+ * Factories are crated with two annotations: {@link FactoryType} and a specific Annotation Interface
+ * declaring a new Factory. {@link FactoryType} is used to annotate the specific annotation used
+ * for factories (i.e. {@link OutputFileClass}, {@link DataProcessorClass},
+ * {@link org.vadere.annotation.factories.models.ModelClass} and
+ * {@link org.vadere.annotation.factories.attributes.ModelAttributeClass})
+ *
+ * The {@link AbstractFactoryProcessor} defines process loop  used to
+ * generate java source files. It also provides abstract methods as hooks to allow
+ * child classes to add additional elements into the generated java source file.
+ * All hooks have access to the PrintWriter used to create the java source file as well
+ * the Set of TypElements annotated with the specific annotation used in this {@link javax.annotation.processing.Processor}
+ *
+ * <ul>
+ * <li>{@link #addImports(Set, PrintWriter)}: </br>
+ * This hook allows the implementing class to add additional import statements
+ * which where not mentioned in the {@link FactoryType} annotation interface.
+ * </li>
+ * <li>{@link #addMembers(Set, PrintWriter)}: </br>
+ * This hook allows the implementing class to add additional members to the new
+ * java source file.
+ * </li>
+ * <li>{@link #addLastConstructor(Set, PrintWriter)}: </br>
+ * This hook allows the implementing class to add additional statement to the
+ * Constructor of the new Factory. Only the Default constructor is implemented!
+ * </li>
+ * <li>{@link #addLastConstructor(Set, PrintWriter)}: </br>
+ * This hook allows the implementing class to add additional function to the
+ * new java source file.
+ * </li>
+ * </ul>
+ *
+ *
+ * {@link #addImports(Set, PrintWriter)}
+ */
+public abstract class AbstractFactoryProcessor extends AbstractProcessor {
+
+	private static final String QUOTE = "\"";
+
+	protected String factoryClassName;      //Name of the new Factory
+	protected String extendedClassName;     //Name of the Factory which needs to be extended.
+	protected String genericFactoryTypes;   //Generics if extendedClass needs it.
+	protected String factoryPackage;        //Where to place the new Factory.
+	protected String[] factoryImports;      //Imports needed by the new Factory.
+
+
+	/**
+	 * This method is called from the java compiler directly*
+	 */
+	@Override
+	public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
+
+		for (TypeElement annotation : annotations) {
+			//only no abstract classes...
+			Set<? extends Element> annotatedElements =
+					roundEnv.getElementsAnnotatedWith(annotation)
+							.stream()
+							.filter(e -> e.getKind().isClass()
+									&& !e.getKind().equals(ElementKind.ENUM)
+									&& !e.getModifiers().contains(Modifier.ABSTRACT)
+							)
+							.collect(Collectors.toSet());
+
+			if (annotatedElements.isEmpty())
+				continue;
+
+			setup(annotation);
+
+			try {
+				writeFactory(annotatedElements);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+
+		}
+
+		return true;
+	}
+
+	protected abstract void addImports(Set<? extends Element> elements, PrintWriter writer);
+
+	protected abstract void addMembers(Set<? extends Element> elements, PrintWriter writer);
+
+	protected abstract void addLastConstructor(Set<? extends Element> elements, PrintWriter writer);
+
+	protected abstract void addLast(Set<? extends Element> elements, PrintWriter writer);
+
+	protected String name(Element e) {
+		return e.getSimpleName().toString();
+	}
+
+	/**
+	 * This method is called for each SupportedAnnotationTypes from the {@link #process(Set, RoundEnvironment)}
+	 * method. The current implementation only allows ONE SupportedAnnotationTypes because this
+	 * method build the whole java source file.
+	 *
+	 * @param elements Elements annotated with one of the SupportedAnnotationTypes
+	 */
+	protected abstract void writeFactory(Set<? extends Element> elements) throws IOException;
+
+
+	/**
+	 * Create class statement based on information provided by the {@link FactoryType} annotation
+	 *
+	 * @param elements Elements annotated with one of the SupportedAnnotationTypes
+	 * @param writer   PrintWriter used to create the source file.
+	 */
+	protected void buildClass(Set<? extends Element> elements, PrintWriter writer) {
+		writer.append("public class ").append(factoryClassName);
+		if (!extendedClassName.isEmpty()) {
+			writer.append(" extends ").append(extendedClassName);
+			if (!genericFactoryTypes.isEmpty()) {
+				writer.append("<").append(genericFactoryTypes).append("> ");
+			}
+		}
+		writer.println("{");
+	}
+
+	/**
+	 * Create a getter for each Element in elements set. The underling TypeElement
+	 * is a Class and must have a constructor without parameters.
+	 *
+	 * @param elements Elements annotated with one of the SupportedAnnotationTypes
+	 * @param writer   PrintWriter used to create the source file.
+	 */
+	protected void buildGetters(Set<? extends Element> elements, PrintWriter writer) {
+		for (Element element : elements) {
+			TypeElement p = (TypeElement) element;
+			writer.append("	public ").append(p.getSimpleName())
+					.append(" get").append(p.getSimpleName()).append("()").println("{");
+			writer.append("		return new ").append(p.getSimpleName()).println("();");
+			writer.append("	}").println();
+			writer.println();
+		}
+	}
+
+	/**
+	 * Creates a static method to provide a thread safe singletone implementation.
+	 */
+	protected void createSingletone(final String instanceType, PrintWriter writer){
+		writer.append("	private static ").append(instanceType).append(" instance;").println();
+		writer.println();
+		writer.println("	//performance threadsafe Singletone. Sync block will only be used once");
+		writer.append("	public static ").append(instanceType).append(" instance(){").println();
+		writer.println("		if(instance ==  null){");
+		writer.append("			synchronized (").append(instanceType).append(".class){").println();
+		writer.println("				if(instance == null){");
+		writer.append("					instance = new ").append(instanceType).append("();").println();
+		writer.println("				}");
+		writer.println("			}");
+		writer.println("		}");
+		writer.println("		return instance;");
+		writer.println("	}");
+		writer.println();
+	}
+
+	protected String quote(final String data){
+		return QUOTE + data + QUOTE;
+	}
+
+	private void setup(TypeElement annotation) {
+		FactoryType factoryType = annotation.getAnnotation(FactoryType.class);
+		assert factoryType != null;
+		this.factoryClassName = factoryType.factoryClassName();
+		this.extendedClassName = factoryType.extendedClassName();
+		this.genericFactoryTypes = factoryType.genericFactoryTypes();
+		this.factoryPackage = factoryType.factoryPackage();
+		this.factoryImports = factoryType.factoryImports();
+	}
+}

VadereAnnotation/src/org/vadere/annotation/factories/BaseFactoryProcessor.java

+package org.vadere.annotation.factories;
+
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Set;
+
+import javax.lang.model.element.Element;
+import javax.lang.model.element.TypeElement;
+import javax.tools.JavaFileObject;
+
+/**
+ * Implements the default layout of the java source file.
+ */
+public abstract class BaseFactoryProcessor extends AbstractFactoryProcessor {
+
+	@Override
+	protected void writeFactory(Set<? extends Element> elements) throws IOException {
+		JavaFileObject jFile = processingEnv.getFiler().createSourceFile(factoryClassName);
+
+		try (PrintWriter out = new PrintWriter(jFile.openWriter())) {
+			out.append("package ").append(factoryPackage).append(";").println();
+			out.println();
+			// Add Imports start
+			for (String factoryImport : factoryImports) {
+				out.append("import ").append(factoryImport).println(";");
+			}
+			out.println();
+			for (Element e : elements) {
+				TypeElement p = (TypeElement) e;
+				out.append("import ").append(p.getQualifiedName()).println(";");
+			}
+			out.println();
+
+			//add additional import defined by subclass
+			addImports(elements, out);