Merge branch 'master' into salient_behavior

# Conflicts:
#	VadereSimulator/src/org/vadere/simulator/projects/io/TrajectoryReader.java

.gitlab-ci.yml

@@ -70,7 +70,7 @@ stages:
     artifacts:
         when: on_failure
         paths:
-            - "log_dir"
+            - "vadere_logs"
         expire_in: 1 week
     when: on_success
 

CHANGELOG.md

@@ -69,6 +69,7 @@
   * vadere.jar -> vadere-gui.jar
   * postvis.jar -> vadere-postvis.jar
   * vadere-console.jar -> remained unchanged
+- Header in output file have now the following form "[NAME]-PID[ID]". This avoids name conflicts and makes mapping to the relevant processor easy and fast.
 - Migration to Java 11 (OpenJDK).
 - Removed directory `Documentation/version-control` which contained the Git hooks. The Git hooks are not required anymore. Instead, added `git rev-parse HEAD` to file `VadereSimulator/pom.xml` to create `VadereSimulator/resources/current_commit_hash.txt` during each build via `mvn compile`.
   **Note:** The file `current_commit_hash.txt` is created during Maven's validation phase, i.e., before the actual build. 

Tools/ContinuousIntegration/run_vadere_console_with_all_scenario_files.py

@@ -21,7 +21,7 @@ short_timeout_in_seconds = 2 * 60
 
 def parse_command_line_arguments():
     parser = argparse.ArgumentParser(description="Run all scenario files.")
-    parser.add_argument("--scenario", "-s", type=str, nargs="?", help="Run only the given scenario file and not all. E.g. \"VadereModelTests/TestOSM/scenarios/basic_2_density_discrete_ca.scenario\"")
+    parser.add_argument("scenario", type=str, nargs="?", help="Run only the given scenario file and not all. E.g., \"VadereModelTests/TestOSM/scenarios/basic_2_density_discrete_ca.scenario\"")
 
     args = parser.parse_args()
 
@@ -80,13 +80,10 @@ def find_scenario_files(path="VadereModelTests", scenario_search_pattern = "*.sc
 
 def run_scenario_files_with_vadere_console(scenario_files, vadere_console="VadereSimulator/target/vadere-console.jar", scenario_timeout_in_sec=short_timeout_in_seconds):
     output_dir = "output"
-    log_dir = "log_dir"
+    log_base_dir = "vadere_logs"
 
-    if not os.path.exists(output_dir):
-        os.makedirs(output_dir)
-
-    if not os.path.exists(log_dir):
-        os.makedirs(log_dir)
+    makedirs_if_non_existing(output_dir)
+    makedirs_if_non_existing(log_base_dir)
 
     total_scenario_files = len(scenario_files)
 
@@ -98,10 +95,18 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader
 
         try:
             print("Running scenario file ({}/{}): {}".format(i + 1, total_scenario_files, scenario_file))
+            
+            # A scenario filename has the form "VadereModelTests/TestOSM/scenarios/chicken_floorfield_ok.scenario"
+            # Use second-level directory as subdirectory for logging (e.g., "TestOSM").
+            log_sub_dir = scenario_file.split(os.path.sep)[1]
+            log_dir = os.path.join(".", log_base_dir, log_sub_dir)
+            
+            makedirs_if_non_existing(log_dir)
+            
             scenario_name = os.path.basename(scenario_file).split('.')[0]
-            log_file = os.path.join(".", log_dir, scenario_name + ".log")
+            log_file = os.path.join(log_dir, scenario_name + ".log")
 
-            print(log_file)
+            print("  Log file: " + log_file)
 
             # Measure wall time and not CPU time.
             wall_time_start = time.time()
@@ -124,17 +129,16 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader
         except subprocess.TimeoutExpired as exception:
             print("Scenario file failed: {}".format(scenario_file))
             print("->  Reason: timeout after {} s".format(exception.timeout))
+            
             failed_summary.append("Scenario file failed: {}".format(scenario_file))
             failed_summary.append("->  Reason: timeout after {} s".format(exception.timeout))
             failed_scenarios_with_exception.append((scenario_file, exception))
         except subprocess.CalledProcessError as exception:
-            prefix = ""
-            if "TestOSM" in scenario_file:
-                prefix = " * OSM * "
-            print(prefix + "Scenario file failed: {}".format(scenario_file))
+            print("Scenario file failed: {}".format(scenario_file))
             print("->  Reason: non-zero return value {}".format(exception.returncode))
             print("            {}".format(read_first_error_linies(exception.stderr)))
-            failed_summary.append(prefix + "Scenario file failed: {}".format(scenario_file))
+            
+            failed_summary.append("Scenario file failed: {}".format(scenario_file))
             failed_summary.append("->  Reason: non-zero return value {}".format(exception.returncode))
             failed_summary.append("            {}".format(read_first_error_linies(exception.stderr)))
 
@@ -145,6 +149,10 @@ def run_scenario_files_with_vadere_console(scenario_files, vadere_console="Vader
 
     return {"passed": passed_scenarios, "failed": failed_scenarios_with_exception, "failed_summary": failed_summary}
 
+def makedirs_if_non_existing(directory):
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
 def read_first_error_linies(error_byte_string):
     err_string_lines = error_byte_string.decode('utf-8').split('\n')
     if len(err_string_lines) >= 2:

Tools/collect_line_and_branch_coverage.py

-# Extract line and branch coverage (in percentage) from HTML coverage reports
-# which are created by Maven's jacoco plugin.
-# Use top-level pom.xml to search in correct subdirectories.
-#
-# Wach out: call this script from root directory of project. E.g.
-#
-#   python Tools/my_script.py
-
-import xml.etree.ElementTree as ET
-
-import os
-import re
-
-def get_modules_from_pom_file(filename="pom.xml"):
-    """Return a list of submodules which where found in passed "pom.xml"."""
-
-    xml_name_spaces = {
-            "default": "http://maven.apache.org/POM/4.0.0"
-    }
-    
-    xml_tree = ET.parse(filename)
-    xml_root = xml_tree.getroot()
-    
-    exclude_list = ["./VadereAnnotation", "./VadereGui"]
-    
-    modules = xml_root.findall("default:modules/default:module", xml_name_spaces)
-    module_names = [module.text for module in modules if module.text not in exclude_list]
-    
-    return module_names
-
-def extract_line_and_branch_coverage(module_names):
-    """Return a dictionary which maps module name to line and branch coverage tuple."""
-
-    module_to_coverage = dict()
-
-    default_coverage_file = "target/site/coverage-reports/index.html"
-
-    for module in module_names:
-        coverage_path = os.path.join(module, default_coverage_file)
-
-        with open(coverage_path, "r") as file:
-            coverage_report = file.read()
-
-            regex_pattern = re.compile(r"Total.*?([0-9]{1,3})%.*?([0-9]{1,3})%")
-            match = regex_pattern.search(coverage_report)
-
-            if match:
-                line_coverage = float(match.group(1))
-                branch_coverage = float(match.group(2))
-                module_to_coverage[module] = (line_coverage, branch_coverage)
-            else:
-                raise Exception("Coverage data not found for module: {}".format(module))
-
-    return module_to_coverage
-
-def print_averaged_line_coverage(coverage_data):
-    """GitLab CI tools read out the stdout output of the build process. Therefore, print coverage info to stdout."""
-
-    total_modules = len(coverage_data)
-    line_coverage_data = [line_coverage for (line_coverage, branch_coverage) in coverage_data.values()]
-    branch_coverage_data = [branch_coverage for (line_coverage, branch_coverage) in coverage_data.values()]
-
-    summed_line_coverage_data = sum(line_coverage_data)
-    summed_branch_coverage_data = sum(branch_coverage_data)
-
-    # By default, GitLab CI parses only integers.
-    averaged_line_coverage = int(round(summed_line_coverage_data / total_modules, 0))
-    averaged_branch_coverage = int(round(summed_branch_coverage_data / total_modules, 0))
-
-    print("Line Coverage: Total {}%".format(averaged_line_coverage))
-    print("Branch Coverage: {}%".format(averaged_branch_coverage))
-
-if __name__ == "__main__":
-    module_names = get_modules_from_pom_file()
-    module_to_coverage = extract_line_and_branch_coverage(module_names)
-    print_averaged_line_coverage(module_to_coverage)

VadereGui/src/org/vadere/gui/postvisualization/view/PostvisualizationWindow.java

@@ -338,8 +338,13 @@ public class PostvisualizationWindow extends JPanel implements Observer {
 
 	public void loadOutputFile(final File trajectoryFile, final Scenario scenario) throws IOException {
 		Player.getInstance(model).stop();
-		model.init(IOOutput.readTrajectories(trajectoryFile.toPath(), scenario), scenario, trajectoryFile.getParent());
-		model.notifyObservers();
+
+		try {
+			model.init(IOOutput.readTrajectories(trajectoryFile.toPath(), scenario), scenario, trajectoryFile.getParent());
+			model.notifyObservers();
+		} catch (Exception ex) {
+			JOptionPane.showMessageDialog(this, ex.getMessage(), Messages.getString("Error.text"), JOptionPane.ERROR_MESSAGE);
+		}
 	}
 
 	public void loadOutputFile(final Scenario scenario) {

VadereSimulator/src/org/vadere/simulator/projects/dataprocessing/outputfile/OutputFile.java

@@ -2,6 +2,7 @@
 
 package org.vadere.simulator.projects.dataprocessing.outputfile;
 
+import org.vadere.simulator.projects.dataprocessing.DataProcessingJsonManager;
 import org.vadere.simulator.projects.dataprocessing.datakey.DataKey;
 import org.vadere.simulator.projects.dataprocessing.processor.DataProcessor;
 import org.vadere.simulator.projects.dataprocessing.writer.VadereWriter;
@@ -54,7 +55,11 @@ public abstract class OutputFile<K extends DataKey<K>> {
 	private boolean isWriteMetaData;
 
 	private String separator;
-	private final static String nameConflictAdd = "-Proc?"; // the # is replaced with the processor id
+
+	// Check also the PostVis where there is a dependency
+	public final static String headerProcSep = "-";
+	public final static String headerNameAdd = headerProcSep + "PID?"; // the # is replaced with the processor id
+
 	private VadereWriterFactory writerFactory;
 	private VadereWriter writer;
 
@@ -129,7 +134,7 @@ public abstract class OutputFile<K extends DataKey<K>> {
 		// characters
 		String md = "#IDXCOL=" + dataIndices.length +
 				",DATACOL="+(getEntireHeader().size()-dataIndices.length)+","+
-				"SEP=\'"+ this.separator+"\'";
+				"SEP=\'"+ DataProcessingJsonManager.DEFAULT_SEPARATOR +"\'";
 
 		//Make a list with one element to reuse 'writeLine' function
 		List<String> line = new LinkedList<>();
@@ -166,16 +171,41 @@ public abstract class OutputFile<K extends DataKey<K>> {
 	}
 
 	public String getHeaderLine() {
-		return String.join(this.separator, this.getEntireHeader());
+		return String.join(DataProcessingJsonManager.DEFAULT_SEPARATOR, this.getEntireHeader());
 	}
 
 	public String getIndicesLine() {
-		return String.join(this.separator, this.getIndices());
+		return String.join(DataProcessingJsonManager.DEFAULT_SEPARATOR, this.getIndices());
+	}
+
+	public static String addHeaderProcInfo(String columnName, int procId){
+		return columnName + headerNameAdd.replace("?", "" + procId);
+	}
+
+	private List<String> uniqueHeaderNames(){
+		// This function adds to every header "headerNameAdd", for ALL headers EVERY time
+		// (cmp. headersWithNameMangling)
+		LinkedList<String> headers = new LinkedList<>();
+
+		for (DataProcessor dataProcessor: dataProcessors) {
+			List<String> allProcHeaders = Arrays.asList(dataProcessor.getHeaders());
+
+			for (String singleHeader: allProcHeaders) {
+				// add the processor id to make header unique
+				String adaptedColumnName = OutputFile.addHeaderProcInfo(singleHeader, dataProcessor.getId());
+				headers.addLast(adaptedColumnName);
+			}
+		}
+
+		return headers;
 	}
 
 	private List<String> headersWithNameMangling(){
+		// This function adds to every header "headerNameAdd", ONLY if there is a name conflict detected
+		// (cmp. uniqueHeaderNames)
+
 		LinkedList<String> headers = new LinkedList<>();
-		boolean isNameMangle = false; // assume there is no nameing conflict
+		boolean isNameMangleDetected = false; // assume there is no nameing conflict
 
 		mainloop:
 		for (DataProcessor l: dataProcessors) {
@@ -183,7 +213,7 @@ public abstract class OutputFile<K extends DataKey<K>> {
 
 			for(String el: list) {
 				if(headers.contains(el)){
-					isNameMangle = true;  // conflict found: stop collecting headers
+					isNameMangleDetected = true;  // conflict found: stop collecting and name make every header unique
 					break mainloop;
 				}else{
 					headers.addLast(el);
@@ -191,24 +221,16 @@ public abstract class OutputFile<K extends DataKey<K>> {
 			}
 		}
 
-		if(isNameMangle){
-			headers.clear();  //start from new...
-			for (DataProcessor l: dataProcessors) {
-				List<String> list = Arrays.asList(l.getHeaders());
-
-				for (String h: list) {
-					// ... but now add the processor id
-					headers.addLast(h +
-							nameConflictAdd.replace('?', (char) (l.getId()+'0')));
-				}
-			}
+		if(isNameMangleDetected){
+			headers = (LinkedList<String>) uniqueHeaderNames();
 		}
+
 		return headers;
 	}
 
 	private List<String> composeHeaderLine(){
 		final List<String> allHeaders = new LinkedList<>(Arrays.asList(dataIndices));
-		List<String> procHeaders = this.headersWithNameMangling();
+		List<String> procHeaders = this.uniqueHeaderNames();
 
 		allHeaders.addAll(procHeaders);
 

VadereSimulator/src/org/vadere/simulator/projects/io/IOOutput.java

@@ -76,7 +76,9 @@ public abstract class IOOutput {
 	private static boolean testTrajectories (final VadereProject project, final File directory) {
 		try {
 			TrajectoryReader reader = new TrajectoryReader(getPathToOutputFile(project, directory.getName(), IOUtils.TRAJECTORY_FILE_EXTENSION));
-			return reader.checkFile();
+			reader.checkFile();
+			return true;
+
 		} catch (IOException | VadereClassNotFoundException e) {
 			logger.error("Error in output file " + directory.getName());
 			return false;

VadereSimulator/src/org/vadere/simulator/projects/io/TrajectoryReader.java

@@ -3,6 +3,7 @@ package org.vadere.simulator.projects.io;
 import org.apache.commons.math3.util.Pair;
 import org.jetbrains.annotations.NotNull;
 import org.vadere.simulator.projects.Scenario;
+import org.vadere.simulator.projects.dataprocessing.outputfile.OutputFile;
 import org.vadere.simulator.projects.dataprocessing.processor.PedestrianPositionProcessor;
 import org.vadere.state.attributes.scenario.AttributesAgent;
 import org.vadere.state.behavior.SalientBehavior;
@@ -72,6 +73,8 @@ public class TrajectoryReader {
 	private int mostImportantEventIndex;
 	private int salientBehaviorIndex;
 
+	private static final int notSetColumnIndexIdentifier = -1;
+
 	public TrajectoryReader(final Path trajectoryFilePath, final Scenario scenario) {
 		this(trajectoryFilePath, scenario.getAttributesPedestrian());
 	}
@@ -109,27 +112,31 @@ public class TrajectoryReader {
 		mostImportantEventKeys.add("mostImportantEvent");
 		salientBehaviorKeys.add("salientBehavior");
 
-		pedIdIndex = -1;
-		stepIndex = -1;
-		xIndex = -1;
-		yIndex = -1;
-		targetIdIndex = -1;
-		groupIdIndex = -1;
-		groupSizeIndex = -1;
-		stridesIndex = -1;
-		mostImportantEventIndex = -1;
-		salientBehaviorIndex = -1;
+		pedIdIndex = notSetColumnIndexIdentifier;
+		stepIndex = notSetColumnIndexIdentifier;
+		xIndex = notSetColumnIndexIdentifier;
+		yIndex = notSetColumnIndexIdentifier;
+		targetIdIndex = notSetColumnIndexIdentifier;
+		groupIdIndex = notSetColumnIndexIdentifier;
+		groupSizeIndex = notSetColumnIndexIdentifier;
+		stridesIndex = notSetColumnIndexIdentifier;
+		mostImportantEventIndex = notSetColumnIndexIdentifier;
+		salientBehaviorIndex = notSetColumnIndexIdentifier;
 	}
 
 	public Map<Step, List<Agent>> readFile() throws IOException {
-		if (checkFile()){
-			return readStandardTrajectoryFile();
-		} else {
-			throw new IOException("could not read trajectory file, some colums are missing.");
+		checkFile();
+		return readStandardTrajectoryFile();
+	}
+
+	private void errorWhenNotUniqueColumn(int currentValue, String columnName) throws IOException{
+		if(currentValue != notSetColumnIndexIdentifier){
+			throw new IOException("The header " + columnName + " is not unique in the file. This is likely to have " +
+					     "unwanted side effects");
 		}
 	}
 
-	public boolean checkFile () throws IOException {
+	public void checkFile () throws IOException {
 		// 1. Get the correct column
 		String header;
 		//read only first line.
@@ -139,40 +146,52 @@ public class TrajectoryReader {
 		String[] columns = header.split(SPLITTER);
 
 		for (int index = 0; index < columns.length; index++) {
-			if (pedestrianIdKeys.contains(columns[index])) {
+
+			// header name without processor ID
+			String headerName = columns[index].split(OutputFile.headerProcSep)[0];
+
+			if (pedestrianIdKeys.contains(headerName)) {
+				errorWhenNotUniqueColumn(pedIdIndex, headerName);
 				pedIdIndex = index;
-			} else if (stepKeys.contains(columns[index])) {
+			} else if (stepKeys.contains(headerName)) {
+				errorWhenNotUniqueColumn(stepIndex, headerName);
 				stepIndex = index;
-			} else if (xKeys.contains(columns[index])) {
+			} else if (xKeys.contains(headerName)) {
+				errorWhenNotUniqueColumn(xIndex, headerName);
 				xIndex = index;
-			} else if (yKeys.contains(columns[index])) {
+			} else if (yKeys.contains(headerName)) {
+				errorWhenNotUniqueColumn(yIndex, headerName);
 				yIndex = index;
-			} else if (targetIdKeys.contains(columns[index])) {
+			} else if (targetIdKeys.contains(headerName)) {
+				errorWhenNotUniqueColumn(targetIdIndex, headerName);
 				targetIdIndex = index;
-			} else if (groupIdKeys.contains(columns[index])){
+			} else if (groupIdKeys.contains(headerName)){
+				errorWhenNotUniqueColumn(groupIdIndex, headerName);
 				groupIdIndex = index;
-			} else if (groupSizeKeys.contains(columns[index])){
+			}
+			else if (groupSizeKeys.contains(headerName)){
+				errorWhenNotUniqueColumn(groupSizeIndex, headerName);
 				groupSizeIndex = index;
-			} else if(stridesKeys.contains(columns[index])) {
+			}
+			else if(stridesKeys.contains(headerName)) {
+				errorWhenNotUniqueColumn(stridesIndex, headerName);
 				stridesIndex = index;
-			} else if (mostImportantEventKeys.contains(columns[index])) {
+			} else if (mostImportantEventKeys.contains(headerName)) {
+				errorWhenNotUniqueColumn(mostImportantEventIndex, headerName);
 				mostImportantEventIndex = index;
-			} else if (salientBehaviorKeys.contains(columns[index])) {
+			} else if (salientBehaviorKeys.contains(headerName)) {
+				errorWhenNotUniqueColumn(salientBehaviorIndex, headerName);
 				salientBehaviorIndex = index;
 			}
 		}
-		try {
-			if (pedIdIndex != -1 && xIndex != -1 && yIndex != -1 && stepIndex != -1) {
-				// load default values with no groups
-				return true;
-			} else {
-				return false;
-			}
-		} catch (Exception e) {
-			logger.warn("could not read trajectory file. The file format might not be compatible or it is missing.");
-			throw e;
-		}
 
+		if (! (pedIdIndex != notSetColumnIndexIdentifier && xIndex != notSetColumnIndexIdentifier &&
+				yIndex != notSetColumnIndexIdentifier && stepIndex != notSetColumnIndexIdentifier)) {
+			// load default values with no groups
+			throw new IOException(String.format("All columns with " + notSetColumnIndexIdentifier + " value could " +
+							"not be found in the trajectory file pedIdIndex=%d, x-values=%d, y-values=%d, step " +
+							"values=%d", pedIdIndex, xIndex, yIndex, stepIndex));
+		}
 	}
 
 	private Map<Step, List<Agent>> readStandardTrajectoryFile() throws IOException {
@@ -184,7 +203,7 @@ public class TrajectoryReader {
 					.collect(Collectors.groupingBy(Pair::getKey,    // group all agent objects by the step.
 							Collectors.mapping(Pair::getValue, Collectors.toList())));
 		} catch (Exception e){
-			logger.warn("could not read trajectory file. The file format might not be compatible or it is missing.");
+			logger.warn("Could not read trajectory file. The file format might not be compatible or it is missing.");
 			throw e;
 		}
 	}

VadereSimulator/tests/org/vadere/simulator/dataprocessing/TestOutputFile.java

@@ -7,6 +7,7 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.vadere.simulator.models.MainModel;
 import org.vadere.simulator.models.MainModelBuilder;
@@ -68,13 +69,13 @@ public class TestOutputFile {
 
 		List<String> header = testScenario.getDataProcessingJsonManager().getOutputFiles().get(0).getEntireHeader();
 
-		//Note these fail if the name conflict is handled differently, for now hard coded.
 		assertTrue(header.contains("timeStep"));
 		assertTrue(header.contains("pedestrianId"));
-		assertTrue(header.contains("x-Proc1"));
-		assertTrue(header.contains("y-Proc1"));
-		assertTrue(header.contains("x-Proc2"));
-		assertTrue(header.contains("y-Proc2"));
+
+		assertTrue(header.contains(OutputFile.addHeaderProcInfo("x", 1)));
+		assertTrue(header.contains(OutputFile.addHeaderProcInfo("y", 1)));
+		assertTrue(header.contains(OutputFile.addHeaderProcInfo("x", 2)));
+		assertTrue(header.contains(OutputFile.addHeaderProcInfo("y", 2)));
 	}
 
 

VadereSimulator/tests/org/vadere/simulator/projects/dataprocessing/processor/PedestrianLastPositionProcessorTest.java

 package org.vadere.simulator.projects.dataprocessing.processor;
 
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 public class PedestrianLastPositionProcessorTest extends ProcessorTest {

VadereSimulator/tests/org/vadere/simulator/projects/dataprocessing/processor/ProcessorTest.java

@@ -8,7 +8,6 @@ import org.vadere.simulator.projects.dataprocessing.outputfile.OutputFile;
 import org.vadere.simulator.utils.reflection.ReflectionHelper;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * Base Test for all Processors.
@@ -51,10 +50,14 @@ public abstract class ProcessorTest {
 		}
 		int l = processorTestEnv.getSimStates().size();
 		p.postLoop(processorTestEnv.getSimStates().get(l - 1));
-		processorTestEnv.getOutputFile().write();
+
+		OutputFile outputFile = processorTestEnv.getOutputFile();
+		outputFile.write();
+
+		// NOTE: these are the column names that have the additional information of the data processor ID.
+		assertEquals(processorTestEnv.getHeader(), outputFile.getHeaderLine());
 
 		String header = String.join(processorTestEnv.getDelimiter(), p.getHeaders());
-		assertTrue(processorTestEnv.getHeader().contains(header));
 		if (header.equals("")){
 			assertEquals(processorTestEnv.getExpectedOutputAsList(), processorTestEnv.getOutput(0));
 		} else {