merge local migration branch

VadereSimulator/src/org/vadere/simulator/entrypoints/ScenarioFactory.java

@@ -5,6 +5,7 @@ import org.vadere.simulator.projects.io.IOVadere;
 import org.vadere.util.io.IOUtils;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 
 public class ScenarioFactory {
@@ -31,4 +32,21 @@ public class ScenarioFactory {
 		return scenario;
 	}
 
+
+	public static Scenario createScenarioWithScenarioFilePath(
+			final Path scenarioPath) throws IOException {
+
+		String json = IOUtils.readTextFile(scenarioPath);
+		Scenario scenario = IOVadere.fromJson(json);
+
+		return scenario;
+	}
+
+	public static Scenario createScenarioWithScenarioJson(
+			final String scenarioJson) throws IOException {
+
+		Scenario scenario = IOVadere.fromJson(scenarioJson);
+
+		return scenario;
+	}
 }

VadereSimulator/src/org/vadere/simulator/entrypoints/Version.java

 package org.vadere.simulator.entrypoints;
 
 import org.jetbrains.annotations.NotNull;
+import org.vadere.simulator.projects.migration.incidents.VersionBumpIncident;
 
 import java.util.Optional;
 
@@ -30,6 +31,34 @@ public enum Version {
 		return null;
 	}
 
+	private static int versionId(Version curr){
+		Version[] versions = values();
+		for ( int i = 0 ; i < versions.length ; i++){
+			if (curr.equals(versions[i])){
+				return  i;
+			}
+		}
+		throw new IllegalArgumentException("Value not in Version Enumeration " + curr.toString());
+	}
+
+	public static Version nextVersion(Version curr){
+		int nextId = versionId(curr) == (values().length -1) ? versionId(curr) : versionId(curr) + 1;
+		return values()[nextId];
+	}
+
+	public static Version[] listToLatest (Version v){
+		int start = versionId(v) == (values().length -1) ? versionId(v) : versionId(v) + 1;
+		int end = values().length;
+		Version[] ret = new Version[end-start];
+		System.arraycopy(values(), start, ret, 0, end - start );
+		return ret;
+
+	}
+
+	public boolean equalOrSamller(Version test){
+		return versionId(this) <= versionId(test);
+	}
+
 	public static Version latest() {
 		return values()[values().length - 1];
 	}

VadereSimulator/src/org/vadere/simulator/projects/ScenarioRun.java

@@ -13,6 +13,7 @@ import java.util.Random;
 
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
+import org.apache.log4j.MDC;
 import org.jetbrains.annotations.Nullable;
 import org.vadere.simulator.control.PassiveCallback;
 import org.vadere.simulator.control.Simulation;
@@ -69,6 +70,9 @@ public class ScenarioRun implements Runnable {
 	@Override
 	public void run() {
 		try {
+			//add Scenario Name to Log4j Mapped Diagnostic Context to filter log by ScenarioRun
+//			MDC.put("scenario.Name", outputPath.getFileName().toString());
+
 			/**
 			 * To make sure that no other Thread changes the scenarioStore object during the initialization of a scenario run
 			 * this is an atomic operation with respect to the scenarioStore. We observed that with Linux 18.04 KUbunto
@@ -77,7 +81,7 @@ public class ScenarioRun implements Runnable {
 			synchronized (scenarioStore) {
 				logger.info(String.format("Initializing scenario. Start of scenario '%s'...", scenario.getName()));
 				scenarioStore.getTopography().reset();
-
+				System.out.println("StartIt " + scenario.getName());
 				MainModelBuilder modelBuilder = new MainModelBuilder(scenarioStore);
 				modelBuilder.createModelAndRandom();
 
@@ -106,6 +110,8 @@ public class ScenarioRun implements Runnable {
 			throw new RuntimeException("Simulation failed.", e);
 		} finally {
 			doAfterSimulation();
+			//remove Log4j Mapped Diagnostic Context after ScenarioRun
+//			MDC.remove("scenario.Name");
 		}
 	}
 	
@@ -138,6 +144,10 @@ public class ScenarioRun implements Runnable {
 		}
 	}
 
+	public Path getOutputPath() {
+		return Paths.get(this.outputPath.toString());
+	}
+
 	public void pause() {
 		if (simulation != null) { // TODO throw an illegal state exception if simulation is not running
 			simulation.pause();

VadereSimulator/src/org/vadere/simulator/projects/dataprocessing/DataProcessingJsonManager.java

@@ -278,4 +278,8 @@ public class DataProcessingJsonManager {
 		return maxId;
 	}
 
+	public boolean containsOutputFile(String name){
+		return  this.outputFiles.stream().anyMatch(f -> f.getFileName().equals(name));
+	}
+
 }

VadereSimulator/src/org/vadere/simulator/projects/migration/JoltTest.java

@@ -30,7 +30,7 @@ public class JoltTest {
 	public void run2() {
 
 
-		List chainrSpecJson = JsonUtils.classpathToList("/transfrom_v2_to_v3.json");
+		List chainrSpecJson = JsonUtils.classpathToList("/transform_v2_to_v3.json");
 		Chainr chainr = Chainr.fromSpec(chainrSpecJson);
 
 		Object inputJson = JsonUtils.classpathToObject("/scenario_test.json");

VadereSimulator/src/org/vadere/simulator/projects/migration/Migration.java

+package org.vadere.simulator.projects.migration;
+
+import com.bazaarvoice.jolt.Chainr;
+import com.bazaarvoice.jolt.Diffy;
+import com.bazaarvoice.jolt.JsonUtils;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import org.apache.log4j.Logger;
+import org.vadere.simulator.entrypoints.Version;
+import org.vadere.state.util.StateJsonConverter;
+import org.vadere.util.io.IOUtils;
+import org.vadere.util.logging.LogBufferAppender;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Optional;
+
+import static org.vadere.util.io.IOUtils.LEGACY_DIR;
+import static org.vadere.util.io.IOUtils.OUTPUT_DIR;
+import static org.vadere.util.io.IOUtils.SCENARIO_DIR;
+
+
+public class Migration {
+
+	private final static Logger logger = Logger.getLogger(Migration.class);
+	private final static LogBufferAppender appender;
+
+	private static HashMap<Version, Chainr> identityTransformation = new LinkedHashMap<>();
+	// Version is the target version. Only Incremental Transformation supported 1->2->3 (not 1->3)
+	private static HashMap<Version, Chainr> transformations = new LinkedHashMap<>();
+	private static final String LEGACY_EXTENSION = "legacy";
+	private static final String NONMIGRATABLE_EXTENSION = "nonmigratable";
+
+	static {
+		identityTransformation.put(Version.V0_1, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v1.json")));
+		identityTransformation.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v2.json")));
+
+		transformations.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/transform_v1_to_v2.json")));
+
+		appender = new LogBufferAppender();
+	}
+
+
+	private Version baseVersion = null;
+	private boolean reapplyLatestMigrationFlag = false;
+	private Diffy transformationDiff;
+
+	public Migration() {
+		this.transformationDiff = new Diffy();
+		logger.addAppender(appender);
+	}
+
+	public String getLog() {
+		return appender.getMigrationLog();
+	}
+
+	public void setReapplyLatestMigrationFlag() {
+		reapplyLatestMigrationFlag = true;
+		baseVersion = null;
+	}
+
+	public void setReapplyLatestMigrationFlag(final Version version) {
+		reapplyLatestMigrationFlag = true;
+		baseVersion = version;
+	}
+
+	public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
+		MigrationResult stats = new MigrationResult();
+
+		Path scenarioDir = Paths.get(projectFolderPath, SCENARIO_DIR);
+		if (Files.exists(scenarioDir)) {
+			stats = analyzeDirectory(scenarioDir, SCENARIO_DIR);
+		}
+
+		Path outputDir = Paths.get(projectFolderPath, OUTPUT_DIR);
+		if (Files.exists(outputDir)) {
+			MigrationResult outputDirStats = analyzeDirectory(outputDir, OUTPUT_DIR);
+			stats.add(outputDirStats);
+		}
+
+		baseVersion = null;
+
+		return stats;
+	}
+
+
+	public MigrationResult analyzeDirectory(Path dir, String dirName) throws IOException {
+
+		Path legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(dirName);
+
+		File[] scenarioFiles = dirName.equals(SCENARIO_DIR) ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
+		MigrationResult stats = new MigrationResult(scenarioFiles.length);
+
+		for (File file : scenarioFiles) {
+
+			if (dirName.equals(OUTPUT_DIR)) {
+				String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies
+				legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(OUTPUT_DIR).resolve(fileFolder);
+			}
+
+			Path scenarioFilePath = Paths.get(file.getAbsolutePath());
+			try {
+				if (analyzeScenario(scenarioFilePath, legacyDir, dirName)) {
+					stats.legacy++;
+				} else {
+					stats.upToDate++;
+				}
+			} catch (MigrationException e) {
+				moveFileAddExtension(scenarioFilePath, legacyDir, NONMIGRATABLE_EXTENSION, dirName.equals(SCENARIO_DIR));
+				logger.error("!> Can't migrate the scenario to latest version, removed it from the directory (" +
+						e.getMessage() + ") If you can fix this problem manually, do so and then remove ." +
+						NONMIGRATABLE_EXTENSION + " from the file in the " + LEGACY_DIR + "-directory "
+						+ "and move it back into the scenarios-directory, it will be checked again when the GUI restarts.");
+				stats.notmigratable++;
+			}
+		}
+		return stats;
+	}
+
+	public JsonNode transform(JsonNode currentJson, Version targetVersion) throws MigrationException {
+		try {
+			return transform(StateJsonConverter.convertJsonNodeToObject(currentJson), targetVersion);
+		} catch (IOException e) {
+			logger.error("Error in converting JsonNode To Map of Object representation");
+			throw new MigrationException("Error in converting JsonNode To Map of Object representation", e);
+		}
+	}
+
+	private JsonNode transform(Object currentJson, Version targetVersion) throws MigrationException {
+		Chainr t = transformations.get(targetVersion);
+		if (t == null) {
+			logger.error("No Transformation defined for Version " + targetVersion.toString());
+			throw new MigrationException("No Transformation defined for Version " + targetVersion.toString());
+		}
+
+		Object scenarioTransformed = t.transform(currentJson);
+
+		Chainr identity = identityTransformation.get(targetVersion);
+		if (identity == null) {
+			logger.error("No IdentityTransformation defined for Version " + targetVersion.toString());
+			throw new MigrationException("No IdentityTransformation definde for Version " + targetVersion.toString());
+		}
+
+		Object scenarioTransformedTest = identity.transform(scenarioTransformed);
+		Diffy.Result diffResult = transformationDiff.diff(scenarioTransformed, scenarioTransformedTest);
+		if (diffResult.isEmpty()) {
+			return StateJsonConverter.deserializeToNode(scenarioTransformed);
+		} else {
+			logger.error("Error in Transformation " + diffResult.toString());
+			throw new MigrationException("Error in Transformation " + diffResult.toString());
+		}
+	}
+
+	private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, String dirName)
+			throws IOException, MigrationException {
+		String json = IOUtils.readTextFile(scenarioFilePath);
+		JsonNode node = StateJsonConverter.deserializeToNode(json);
+
+		String parentPath = dirName.equals(SCENARIO_DIR) ? SCENARIO_DIR + "/" : OUTPUT_DIR + "/" + scenarioFilePath.getParent().getFileName().toString() + "/";
+
+		logger.info(">> analyzing JSON tree of scenario <" + parentPath + node.get("name").asText() + ">");
+
+		Version version = Version.UNDEFINED;
+
+		if (node.get("release") != null) {
+			version = Version.fromString(node.get("release").asText());
+
+			if (version == null || version.equalOrSamller(Version.NOT_A_RELEASE)) {
+				logger.error("release version " + node.get("release").asText() + " is unknown or not " +
+						"supported. If this is a valid release create a version transformation and a new idenity transformation");
+				throw new MigrationException("release version " + node.get("release").asText() + " is unknown or not " +
+						"supported. If this is a valid releasecreate a version transformation and a new idenity transformation");
+			}
+
+			// if enforced migration should be done from baseVersion to latestVersion
+			if (reapplyLatestMigrationFlag && baseVersion != null) {
+				version = baseVersion;
+
+			} else if (reapplyLatestMigrationFlag) { // if enforced migration should be done from prev version to latest
+				Optional<Version> optVersion = Version.getPrevious(version);
+				if (optVersion.isPresent()) {
+					version = optVersion.get();
+				} else {
+					return false;
+				}
+			} // if no enforced migration should be done and we are at the latest version, no migration is required.
+			else if (version == Version.latest()) {
+				return false;
+			}
+		} else {
+			logger.error("Version is unknown");
+			throw new MigrationException("Version is unknown");
+		}
+
+		JsonNode transformedNode = node;
+		// apply all transformation from current to latest version.
+		for (Version v : Version.listToLatest(version)) {
+			transformedNode = transform(transformedNode, v);
+		}
+		if (legacyDir != null) {
+			moveFileAddExtension(scenarioFilePath, legacyDir, LEGACY_EXTENSION, false);
+		}
+		IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(transformedNode));
+		return true;
+	}
+
+	private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder)
+			throws IOException {
+		Path source = scenarioFilePath;
+		Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension);
+
+		if (moveOutputFolder) {
+			source = source.getParent();
+			target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension);
+		}
+
+		IOUtils.createDirectoryIfNotExisting(target);
+		Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten?
+	}
+}

VadereSimulator/src/org/vadere/simulator/projects/migration/MigrationAssistant.java

@@ -36,7 +36,7 @@ public class MigrationAssistant {
 	private static boolean reapplyLatestMigrationFlag = false;
 	private static Version baseVersion = null;
 
-	public static void setReapplyLatestMigrationFlag() {
+		public static void setReapplyLatestMigrationFlag() {
 		reapplyLatestMigrationFlag = true;
 		baseVersion = null;
 	}
@@ -46,8 +46,14 @@ public class MigrationAssistant {
 		baseVersion = version;
 	}
 
-	public static void analyzeSingleScenario(Path path) {
+	public static void analyzeSingleScenario(Path path) throws IOException {
 		// TODO [priority=high] [task=implement] for runs initiated not from GUI... where to hook in?
+		StringBuilder log = new StringBuilder();
+		try {
+			analyzeScenario(path, null, log, true)	;
+		} catch (MigrationException e) {
+			logger.error(log.toString());
+		}
 	}
 
 	public static int[] analyzeProject(String projectFolderPath) throws IOException {

VadereSimulator/src/org/vadere/simulator/projects/migration/MigrationException.java

@@ -6,12 +6,17 @@ public class MigrationException extends Exception {
 
 	private static final long serialVersionUID = 1L;
 
-	public MigrationException() {}
+	public MigrationException() {
+	}
 
 	public MigrationException(String message) {
 		super(message);
 	}
 
+	public MigrationException(String message, Throwable cause) {
+		super(message, cause);
+	}
+
 	public MigrationException(Incident incident, String message) {
 		super(incident.getClass().getSimpleName() + ": " + message);
 	}

VadereSimulator/src/org/vadere/simulator/projects/migration/MigrationResult.java

+package org.vadere.simulator.projects.migration;
+
+import java.util.Objects;
+
+public class MigrationResult {
+	public int total;
+	public int upToDate;
+	public int legacy;
+	public int notmigratable;
+
+	public MigrationResult() {
+	}
+
+	public MigrationResult(int total, int upToDate, int legacy, int notmigratable) {
+		this.total = total;
+		this.upToDate = upToDate;
+		this.legacy = legacy;
+		this.notmigratable = notmigratable;
+	}
+
+	public MigrationResult(int total) {
+		this.total = total;
+	}
+
+	boolean checkTotal() {
+		return (upToDate + legacy + notmigratable) == total;
+	}
+
+	public MigrationResult add(MigrationResult other) {
+		this.total = this.total + other.total;
+		this.upToDate = this.upToDate + other.upToDate;
+		this.legacy = this.legacy + other.legacy;
+		this.notmigratable = this.notmigratable + other.notmigratable;
+		return this;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) return true;
+		if (o == null || getClass() != o.getClass()) return false;
+		MigrationResult result = (MigrationResult) o;
+		return total == result.total &&
+				upToDate == result.upToDate &&
+				legacy == result.legacy &&
+				notmigratable == result.notmigratable;
+	}
+
+	@Override
+	public int hashCode() {
+
+		return Objects.hash(total, upToDate, legacy, notmigratable);
+	}
+
+	@Override
+	public String toString() {
+		return "MigrationResult{" +
+				"total=" + total +
+				", upToDate=" + upToDate +
+				", legacy=" + legacy +
+				", notmigratable=" + notmigratable +
+				'}';
+	}
+}

VadereSimulator/testResources/migration/testProject_v0.1.bak/output/basic_1_chicken_osm1_2018-07-17_16-56-52.794/basic_1_chicken_osm1.scenario

+{
+  "name" : "basic_1_chicken_osm1",
+  "description" : "",
+  "release" : "0.1",
+  "topographyhash" : "cde764736cfb41b5cd5e4ff40e0adfca0fee6370",
+  "attributeshash" : "75778572f188cffcdedc74a7a9b60ddbe42359fc",
+  "processWriters" : [ {
+    "formatString" : "%s",
+    "columnNames" : [ "waitingTimeTest" ],
+    "attributes" : {
+      "startTime" : 0.0,
+      "endTime" : 1.7976931348623157E308
+    },
+    "processor" : {
+      "pedestrianWaitingTimeProcessor" : {
+        "attributes" : {
+          "waitingArea" : {
+            "x" : 10.0,
+            "y" : 32.0,
+            "width" : 15.0,
+            "height" : 8.0
+          }
+        },
+        "columnNames" : [ "totalWaitingTime", "time", "id" ],
+        "clazz" : "PedestrianWaitingTimeProcessor"
+      },
+      "attributes" : {
+        "maxWaitingTime" : 0.0,
+        "maxWaitingTimeMean" : 0.0,
+        "minWaitingTime" : 0.0,
+        "minWaitingTimeMean" : 0.0,
+        "expectFailure" : false
+      },
+      "columnNames" : [ ],
+      "clazz" : "PedestrianWaitingTimeTest"
+    }
+  }, {
+    "formatString" : "%s",
+    "columnNames" : [ "waitingTimeTest" ],
+    "attributes" : {
+      "startTime" : 0.0,
+      "endTime" : 1.7976931348623157E308
+    },
+    "processor" : {
+      "pedestrianWaitingTimeProcessor" : {
+        "attributes" : {
+          "waitingArea" : {
+            "x" : 10.0,
+            "y" : 23.5,
+            "width" : 15.0,
+            "height" : 7.5
+          }
+        },
+        "columnNames" : [ "totalWaitingTime", "time", "id" ],
+        "clazz" : "PedestrianWaitingTimeProcessor"
+      },
+      "attributes" : {
+        "maxWaitingTime" : 10.0,
+        "maxWaitingTimeMean" : 10.0,
+        "minWaitingTime" : 0.0,
+        "minWaitingTimeMean" : 0.0,
+        "expectFailure" : false
+      },
+      "columnNames" : [ ],
+      "clazz" : "PedestrianWaitingTimeTest"
+    }
+  }, {
+    "formatString" : "%s",
+    "columnNames" : [ "evacuationTimeTest" ],
+    "attributes" : {
+      "startTime" : 0.0,
+      "endTime" : 1.7976931348623157E308
+    },
+    "processor" : {
+      "attributes" : {
+        "maxEvacuationTime" : 100.0,
+        "maxEvacuationTimeMean" : 100.0,
+        "minEvacuationTime" : 0.0,
+        "minEvacuationTimeMean" : 0.0,
+        "expectFailure" : false
+      },
+      "errorTime" : -1.0,
+      "columnNames" : [ ],
+      "clazz" : "PedestrianEvacuationTimeTest"
+    }
+  } ],
+  "vadere" : {
+    "mainModel" : "org.vadere.simulator.models.osm.OptimalStepsModel",
+