Commit 9afe5ec3 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck

merge local migration branch

parents 5c092be5 bb44e786
Pipeline #62846 failed with stages
in 46 seconds
......@@ -5,6 +5,7 @@ import org.vadere.simulator.projects.io.IOVadere;
import org.vadere.util.io.IOUtils;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
public class ScenarioFactory {
......@@ -31,4 +32,21 @@ public class ScenarioFactory {
return scenario;
}
public static Scenario createScenarioWithScenarioFilePath(
final Path scenarioPath) throws IOException {
String json = IOUtils.readTextFile(scenarioPath);
Scenario scenario = IOVadere.fromJson(json);
return scenario;
}
public static Scenario createScenarioWithScenarioJson(
final String scenarioJson) throws IOException {
Scenario scenario = IOVadere.fromJson(scenarioJson);
return scenario;
}
}
package org.vadere.simulator.entrypoints;
import org.jetbrains.annotations.NotNull;
import org.vadere.simulator.projects.migration.incidents.VersionBumpIncident;
import java.util.Optional;
......@@ -30,6 +31,34 @@ public enum Version {
return null;
}
private static int versionId(Version curr){
Version[] versions = values();
for ( int i = 0 ; i < versions.length ; i++){
if (curr.equals(versions[i])){
return i;
}
}
throw new IllegalArgumentException("Value not in Version Enumeration " + curr.toString());
}
public static Version nextVersion(Version curr){
int nextId = versionId(curr) == (values().length -1) ? versionId(curr) : versionId(curr) + 1;
return values()[nextId];
}
public static Version[] listToLatest (Version v){
int start = versionId(v) == (values().length -1) ? versionId(v) : versionId(v) + 1;
int end = values().length;
Version[] ret = new Version[end-start];
System.arraycopy(values(), start, ret, 0, end - start );
return ret;
}
public boolean equalOrSamller(Version test){
return versionId(this) <= versionId(test);
}
public static Version latest() {
return values()[values().length - 1];
}
......
......@@ -13,6 +13,7 @@ import java.util.Random;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.MDC;
import org.jetbrains.annotations.Nullable;
import org.vadere.simulator.control.PassiveCallback;
import org.vadere.simulator.control.Simulation;
......@@ -69,6 +70,9 @@ public class ScenarioRun implements Runnable {
@Override
public void run() {
try {
//add Scenario Name to Log4j Mapped Diagnostic Context to filter log by ScenarioRun
// MDC.put("scenario.Name", outputPath.getFileName().toString());
/**
* To make sure that no other Thread changes the scenarioStore object during the initialization of a scenario run
* this is an atomic operation with respect to the scenarioStore. We observed that with Linux 18.04 KUbunto
......@@ -77,7 +81,7 @@ public class ScenarioRun implements Runnable {
synchronized (scenarioStore) {
logger.info(String.format("Initializing scenario. Start of scenario '%s'...", scenario.getName()));
scenarioStore.getTopography().reset();
System.out.println("StartIt " + scenario.getName());
MainModelBuilder modelBuilder = new MainModelBuilder(scenarioStore);
modelBuilder.createModelAndRandom();
......@@ -106,6 +110,8 @@ public class ScenarioRun implements Runnable {
throw new RuntimeException("Simulation failed.", e);
} finally {
doAfterSimulation();
//remove Log4j Mapped Diagnostic Context after ScenarioRun
// MDC.remove("scenario.Name");
}
}
......@@ -138,6 +144,10 @@ public class ScenarioRun implements Runnable {
}
}
public Path getOutputPath() {
return Paths.get(this.outputPath.toString());
}
public void pause() {
if (simulation != null) { // TODO throw an illegal state exception if simulation is not running
simulation.pause();
......
......@@ -278,4 +278,8 @@ public class DataProcessingJsonManager {
return maxId;
}
public boolean containsOutputFile(String name){
return this.outputFiles.stream().anyMatch(f -> f.getFileName().equals(name));
}
}
......@@ -30,7 +30,7 @@ public class JoltTest {
public void run2() {
List chainrSpecJson = JsonUtils.classpathToList("/transfrom_v2_to_v3.json");
List chainrSpecJson = JsonUtils.classpathToList("/transform_v2_to_v3.json");
Chainr chainr = Chainr.fromSpec(chainrSpecJson);
Object inputJson = JsonUtils.classpathToObject("/scenario_test.json");
......
package org.vadere.simulator.projects.migration;
import com.bazaarvoice.jolt.Chainr;
import com.bazaarvoice.jolt.Diffy;
import com.bazaarvoice.jolt.JsonUtils;
import com.fasterxml.jackson.databind.JsonNode;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.state.util.StateJsonConverter;
import org.vadere.util.io.IOUtils;
import org.vadere.util.logging.LogBufferAppender;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Optional;
import static org.vadere.util.io.IOUtils.LEGACY_DIR;
import static org.vadere.util.io.IOUtils.OUTPUT_DIR;
import static org.vadere.util.io.IOUtils.SCENARIO_DIR;
public class Migration {
private final static Logger logger = Logger.getLogger(Migration.class);
private final static LogBufferAppender appender;
private static HashMap<Version, Chainr> identityTransformation = new LinkedHashMap<>();
// Version is the target version. Only Incremental Transformation supported 1->2->3 (not 1->3)
private static HashMap<Version, Chainr> transformations = new LinkedHashMap<>();
private static final String LEGACY_EXTENSION = "legacy";
private static final String NONMIGRATABLE_EXTENSION = "nonmigratable";
static {
identityTransformation.put(Version.V0_1, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v1.json")));
identityTransformation.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v2.json")));
transformations.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/transform_v1_to_v2.json")));
appender = new LogBufferAppender();
}
private Version baseVersion = null;
private boolean reapplyLatestMigrationFlag = false;
private Diffy transformationDiff;
public Migration() {
this.transformationDiff = new Diffy();
logger.addAppender(appender);
}
public String getLog() {
return appender.getMigrationLog();
}
public void setReapplyLatestMigrationFlag() {
reapplyLatestMigrationFlag = true;
baseVersion = null;
}
public void setReapplyLatestMigrationFlag(final Version version) {
reapplyLatestMigrationFlag = true;
baseVersion = version;
}
public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
MigrationResult stats = new MigrationResult();
Path scenarioDir = Paths.get(projectFolderPath, SCENARIO_DIR);
if (Files.exists(scenarioDir)) {
stats = analyzeDirectory(scenarioDir, SCENARIO_DIR);
}
Path outputDir = Paths.get(projectFolderPath, OUTPUT_DIR);
if (Files.exists(outputDir)) {
MigrationResult outputDirStats = analyzeDirectory(outputDir, OUTPUT_DIR);
stats.add(outputDirStats);
}
baseVersion = null;
return stats;
}
public MigrationResult analyzeDirectory(Path dir, String dirName) throws IOException {
Path legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(dirName);
File[] scenarioFiles = dirName.equals(SCENARIO_DIR) ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
MigrationResult stats = new MigrationResult(scenarioFiles.length);
for (File file : scenarioFiles) {
if (dirName.equals(OUTPUT_DIR)) {
String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies
legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(OUTPUT_DIR).resolve(fileFolder);
}
Path scenarioFilePath = Paths.get(file.getAbsolutePath());
try {
if (analyzeScenario(scenarioFilePath, legacyDir, dirName)) {
stats.legacy++;
} else {
stats.upToDate++;
}
} catch (MigrationException e) {
moveFileAddExtension(scenarioFilePath, legacyDir, NONMIGRATABLE_EXTENSION, dirName.equals(SCENARIO_DIR));
logger.error("!> Can't migrate the scenario to latest version, removed it from the directory (" +
e.getMessage() + ") If you can fix this problem manually, do so and then remove ." +
NONMIGRATABLE_EXTENSION + " from the file in the " + LEGACY_DIR + "-directory "
+ "and move it back into the scenarios-directory, it will be checked again when the GUI restarts.");
stats.notmigratable++;
}
}
return stats;
}
public JsonNode transform(JsonNode currentJson, Version targetVersion) throws MigrationException {
try {
return transform(StateJsonConverter.convertJsonNodeToObject(currentJson), targetVersion);
} catch (IOException e) {
logger.error("Error in converting JsonNode To Map of Object representation");
throw new MigrationException("Error in converting JsonNode To Map of Object representation", e);
}
}
private JsonNode transform(Object currentJson, Version targetVersion) throws MigrationException {
Chainr t = transformations.get(targetVersion);
if (t == null) {
logger.error("No Transformation defined for Version " + targetVersion.toString());
throw new MigrationException("No Transformation defined for Version " + targetVersion.toString());
}
Object scenarioTransformed = t.transform(currentJson);
Chainr identity = identityTransformation.get(targetVersion);
if (identity == null) {
logger.error("No IdentityTransformation defined for Version " + targetVersion.toString());
throw new MigrationException("No IdentityTransformation definde for Version " + targetVersion.toString());
}
Object scenarioTransformedTest = identity.transform(scenarioTransformed);
Diffy.Result diffResult = transformationDiff.diff(scenarioTransformed, scenarioTransformedTest);
if (diffResult.isEmpty()) {
return StateJsonConverter.deserializeToNode(scenarioTransformed);
} else {
logger.error("Error in Transformation " + diffResult.toString());
throw new MigrationException("Error in Transformation " + diffResult.toString());
}
}
private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, String dirName)
throws IOException, MigrationException {
String json = IOUtils.readTextFile(scenarioFilePath);
JsonNode node = StateJsonConverter.deserializeToNode(json);
String parentPath = dirName.equals(SCENARIO_DIR) ? SCENARIO_DIR + "/" : OUTPUT_DIR + "/" + scenarioFilePath.getParent().getFileName().toString() + "/";
logger.info(">> analyzing JSON tree of scenario <" + parentPath + node.get("name").asText() + ">");
Version version = Version.UNDEFINED;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
if (version == null || version.equalOrSamller(Version.NOT_A_RELEASE)) {
logger.error("release version " + node.get("release").asText() + " is unknown or not " +
"supported. If this is a valid release create a version transformation and a new idenity transformation");
throw new MigrationException("release version " + node.get("release").asText() + " is unknown or not " +
"supported. If this is a valid releasecreate a version transformation and a new idenity transformation");
}
// if enforced migration should be done from baseVersion to latestVersion
if (reapplyLatestMigrationFlag && baseVersion != null) {
version = baseVersion;
} else if (reapplyLatestMigrationFlag) { // if enforced migration should be done from prev version to latest
Optional<Version> optVersion = Version.getPrevious(version);
if (optVersion.isPresent()) {
version = optVersion.get();
} else {
return false;
}
} // if no enforced migration should be done and we are at the latest version, no migration is required.
else if (version == Version.latest()) {
return false;
}
} else {
logger.error("Version is unknown");
throw new MigrationException("Version is unknown");
}
JsonNode transformedNode = node;
// apply all transformation from current to latest version.
for (Version v : Version.listToLatest(version)) {
transformedNode = transform(transformedNode, v);
}
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, LEGACY_EXTENSION, false);
}
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(transformedNode));
return true;
}
private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder)
throws IOException {
Path source = scenarioFilePath;
Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension);
if (moveOutputFolder) {
source = source.getParent();
target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension);
}
IOUtils.createDirectoryIfNotExisting(target);
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten?
}
}
......@@ -36,7 +36,7 @@ public class MigrationAssistant {
private static boolean reapplyLatestMigrationFlag = false;
private static Version baseVersion = null;
public static void setReapplyLatestMigrationFlag() {
public static void setReapplyLatestMigrationFlag() {
reapplyLatestMigrationFlag = true;
baseVersion = null;
}
......@@ -46,8 +46,14 @@ public class MigrationAssistant {
baseVersion = version;
}
public static void analyzeSingleScenario(Path path) {
public static void analyzeSingleScenario(Path path) throws IOException {
// TODO [priority=high] [task=implement] for runs initiated not from GUI... where to hook in?
StringBuilder log = new StringBuilder();
try {
analyzeScenario(path, null, log, true) ;
} catch (MigrationException e) {
logger.error(log.toString());
}
}
public static int[] analyzeProject(String projectFolderPath) throws IOException {
......
......@@ -6,12 +6,17 @@ public class MigrationException extends Exception {
private static final long serialVersionUID = 1L;
public MigrationException() {}
public MigrationException() {
}
public MigrationException(String message) {
super(message);
}
public MigrationException(String message, Throwable cause) {
super(message, cause);
}
public MigrationException(Incident incident, String message) {
super(incident.getClass().getSimpleName() + ": " + message);
}
......
package org.vadere.simulator.projects.migration;
import java.util.Objects;
public class MigrationResult {
public int total;
public int upToDate;
public int legacy;
public int notmigratable;
public MigrationResult() {
}
public MigrationResult(int total, int upToDate, int legacy, int notmigratable) {
this.total = total;
this.upToDate = upToDate;
this.legacy = legacy;
this.notmigratable = notmigratable;
}
public MigrationResult(int total) {
this.total = total;
}
boolean checkTotal() {
return (upToDate + legacy + notmigratable) == total;
}
public MigrationResult add(MigrationResult other) {
this.total = this.total + other.total;
this.upToDate = this.upToDate + other.upToDate;
this.legacy = this.legacy + other.legacy;
this.notmigratable = this.notmigratable + other.notmigratable;
return this;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
MigrationResult result = (MigrationResult) o;
return total == result.total &&
upToDate == result.upToDate &&
legacy == result.legacy &&
notmigratable == result.notmigratable;
}
@Override
public int hashCode() {
return Objects.hash(total, upToDate, legacy, notmigratable);
}
@Override
public String toString() {
return "MigrationResult{" +
"total=" + total +
", upToDate=" + upToDate +
", legacy=" + legacy +
", notmigratable=" + notmigratable +
'}';
}
}
{
"name" : "basic_1_chicken_osm1",
"description" : "",
"release" : "0.1",
"topographyhash" : "cde764736cfb41b5cd5e4ff40e0adfca0fee6370",
"attributeshash" : "75778572f188cffcdedc74a7a9b60ddbe42359fc",
"processWriters" : [ {
"formatString" : "%s",
"columnNames" : [ "waitingTimeTest" ],
"attributes" : {
"startTime" : 0.0,
"endTime" : 1.7976931348623157E308
},
"processor" : {
"pedestrianWaitingTimeProcessor" : {
"attributes" : {
"waitingArea" : {
"x" : 10.0,
"y" : 32.0,
"width" : 15.0,
"height" : 8.0
}
},
"columnNames" : [ "totalWaitingTime", "time", "id" ],
"clazz" : "PedestrianWaitingTimeProcessor"
},
"attributes" : {
"maxWaitingTime" : 0.0,
"maxWaitingTimeMean" : 0.0,
"minWaitingTime" : 0.0,
"minWaitingTimeMean" : 0.0,
"expectFailure" : false
},
"columnNames" : [ ],
"clazz" : "PedestrianWaitingTimeTest"
}
}, {
"formatString" : "%s",
"columnNames" : [ "waitingTimeTest" ],
"attributes" : {
"startTime" : 0.0,
"endTime" : 1.7976931348623157E308
},
"processor" : {
"pedestrianWaitingTimeProcessor" : {
"attributes" : {
"waitingArea" : {
"x" : 10.0,
"y" : 23.5,
"width" : 15.0,
"height" : 7.5
}
},
"columnNames" : [ "totalWaitingTime", "time", "id" ],
"clazz" : "PedestrianWaitingTimeProcessor"
},
"attributes" : {
"maxWaitingTime" : 10.0,
"maxWaitingTimeMean" : 10.0,
"minWaitingTime" : 0.0,
"minWaitingTimeMean" : 0.0,
"expectFailure" : false
},
"columnNames" : [ ],
"clazz" : "PedestrianWaitingTimeTest"
}
}, {