Commit 58cd4211 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

add new migration implementation with tests.

parent 97083f4f
...@@ -4,9 +4,12 @@ import com.bazaarvoice.jolt.Chainr; ...@@ -4,9 +4,12 @@ import com.bazaarvoice.jolt.Chainr;
import com.bazaarvoice.jolt.Diffy; import com.bazaarvoice.jolt.Diffy;
import com.bazaarvoice.jolt.JsonUtils; import com.bazaarvoice.jolt.JsonUtils;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version; import org.vadere.simulator.entrypoints.Version;
import org.vadere.state.util.StateJsonConverter; import org.vadere.state.util.StateJsonConverter;
import org.vadere.util.io.IOUtils; import org.vadere.util.io.IOUtils;
import org.vadere.util.logging.LogBufferAppender;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
...@@ -25,177 +28,199 @@ import static org.vadere.util.io.IOUtils.SCENARIO_DIR; ...@@ -25,177 +28,199 @@ import static org.vadere.util.io.IOUtils.SCENARIO_DIR;
public class Migration { public class Migration {
private static HashMap<Version, Chainr> identityTransformation = new LinkedHashMap<>(); private final static Logger logger = Logger.getLogger(Migration.class);
// Version is the target version. Only Incremental Transformation supported 1->2->3 (not 1->3) private final static LogBufferAppender appender;
private static HashMap<Version, Chainr> transformations = new LinkedHashMap<>();
private static final String LEGACY_EXTENSION = "legacy"; private static HashMap<Version, Chainr> identityTransformation = new LinkedHashMap<>();
private static final String NONMIGRATABLE_EXTENSION = "nonmigratable"; // Version is the target version. Only Incremental Transformation supported 1->2->3 (not 1->3)
static { private static HashMap<Version, Chainr> transformations = new LinkedHashMap<>();
identityTransformation.put(Version.V0_1, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v1.json"))); private static final String LEGACY_EXTENSION = "legacy";
identityTransformation.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v2.json"))); private static final String NONMIGRATABLE_EXTENSION = "nonmigratable";
transformations.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/transform_v1_to_v2.json"))); static {
} identityTransformation.put(Version.V0_1, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v1.json")));
identityTransformation.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v2.json")));
transformations.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/transform_v1_to_v2.json")));
private Version baseVersion = null;
private boolean reapplyLatestMigrationFlag = false; appender = new LogBufferAppender();
private StringBuilder migrationLog; }
private Diffy transformationDiff;
public Migration(){ private Version baseVersion = null;
this.migrationLog = new StringBuilder(); private boolean reapplyLatestMigrationFlag = false;
this.transformationDiff = new Diffy(); private Diffy transformationDiff;
}
public Migration() {
public Migration(StringBuilder migrationLog){ this.transformationDiff = new Diffy();
this.migrationLog = migrationLog; logger.addAppender(appender);
this.transformationDiff = new Diffy(); }
}
public String getLog() {
public void setReapplyLatestMigrationFlag() { return appender.getMigrationLog();
reapplyLatestMigrationFlag = true; }
baseVersion = null;
} public void setReapplyLatestMigrationFlag() {
reapplyLatestMigrationFlag = true;
public void setReapplyLatestMigrationFlag(final Version version) { baseVersion = null;
reapplyLatestMigrationFlag = true; }
baseVersion = version;
} public void setReapplyLatestMigrationFlag(final Version version) {
reapplyLatestMigrationFlag = true;
public MigrationResult analyzeProject(String projectFolderPath) throws IOException { baseVersion = version;
MigrationResult stats = new MigrationResult(); }
Path scenarioDir = Paths.get(projectFolderPath, SCENARIO_DIR); public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
if (Files.exists(scenarioDir)) { MigrationResult stats = new MigrationResult();
stats = analyzeDirectory(scenarioDir, SCENARIO_DIR);
} Path scenarioDir = Paths.get(projectFolderPath, SCENARIO_DIR);
if (Files.exists(scenarioDir)) {
Path outputDir = Paths.get(projectFolderPath, OUTPUT_DIR); stats = analyzeDirectory(scenarioDir, SCENARIO_DIR);
if (Files.exists(outputDir)) { }
MigrationResult outputDirStats = analyzeDirectory(outputDir, OUTPUT_DIR);
stats.add(outputDirStats); Path outputDir = Paths.get(projectFolderPath, OUTPUT_DIR);
} if (Files.exists(outputDir)) {
MigrationResult outputDirStats = analyzeDirectory(outputDir, OUTPUT_DIR);
baseVersion = null; stats.add(outputDirStats);
}
return stats;
} baseVersion = null;
return stats;
public MigrationResult analyzeDirectory(Path dir, String dirName) throws IOException{ }
Path legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(dirName);
public MigrationResult analyzeDirectory(Path dir, String dirName) throws IOException {
File[] scenarioFiles = dirName.equals(SCENARIO_DIR) ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
MigrationResult stats = new MigrationResult(scenarioFiles.length); Path legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(dirName);
for (File file : scenarioFiles) { File[] scenarioFiles = dirName.equals(SCENARIO_DIR) ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
MigrationResult stats = new MigrationResult(scenarioFiles.length);
if (dirName.equals(OUTPUT_DIR)) {
String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies for (File file : scenarioFiles) {
legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(OUTPUT_DIR).resolve(fileFolder);
} if (dirName.equals(OUTPUT_DIR)) {
String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies
Path scenarioFilePath = Paths.get(file.getAbsolutePath()); legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(OUTPUT_DIR).resolve(fileFolder);
try { }
if (analyzeScenario(scenarioFilePath, legacyDir, migrationLog, dirName.equals(SCENARIO_DIR))) {
stats.legacy++; Path scenarioFilePath = Paths.get(file.getAbsolutePath());
} else { try {
stats.upToDate++; if (analyzeScenario(scenarioFilePath, legacyDir, dirName)) {
} stats.legacy++;
} catch (MigrationException e) { } else {
moveFileAddExtension(scenarioFilePath, legacyDir, NONMIGRATABLE_EXTENSION, dirName.equals(SCENARIO_DIR)); stats.upToDate++;
migrationLog.append("! --> Can't migrate the scenario to latest version, removed it from the directory (") }
.append(e.getMessage()).append(")\n") } catch (MigrationException e) {
.append("If you can fix this problem manually, do so and then remove .") moveFileAddExtension(scenarioFilePath, legacyDir, NONMIGRATABLE_EXTENSION, dirName.equals(SCENARIO_DIR));
.append(NONMIGRATABLE_EXTENSION).append(" from the file in the ") logger.error("!> Can't migrate the scenario to latest version, removed it from the directory (" +
.append(LEGACY_DIR).append("-directory ") e.getMessage() + ") If you can fix this problem manually, do so and then remove ." +
.append("and move it back into the scenarios-directory, it will be checked again when the GUI restarts.\n"); NONMIGRATABLE_EXTENSION + " from the file in the " + LEGACY_DIR + "-directory "
stats.nomigratable++; + "and move it back into the scenarios-directory, it will be checked again when the GUI restarts.");
} stats.notmigratable++;
} }
return stats; }
} return stats;
}
public JsonNode transform (JsonNode currentJson, Version targetVersion) throws MigrationException {
Chainr t = transformations.get(targetVersion); public JsonNode transform(JsonNode currentJson, Version targetVersion) throws MigrationException {
if (t == null) try {
throw new MigrationException("No Transformation defined for Version " + targetVersion.toString()); return transform(StateJsonConverter.convertJsonNodeToObject(currentJson), targetVersion);
} catch (IOException e) {
Object scenarioTransformed = t.transform(currentJson); logger.error("Error in converting JsonNode To Map of Object representation");
throw new MigrationException("Error in converting JsonNode To Map of Object representation", e);
Chainr identity = identityTransformation.get(targetVersion); }
if (identity == null) }
throw new MigrationException("No IdentityTransformation definde for Version " + targetVersion.toString());
private JsonNode transform(Object currentJson, Version targetVersion) throws MigrationException {
Object scenarioTransformedTest = identity.transform(scenarioTransformed); Chainr t = transformations.get(targetVersion);
Diffy.Result diffResult = transformationDiff.diff(scenarioTransformed, scenarioTransformedTest); if (t == null) {
if (diffResult.isEmpty()){ logger.error("No Transformation defined for Version " + targetVersion.toString());
return (JsonNode) scenarioTransformed; throw new MigrationException("No Transformation defined for Version " + targetVersion.toString());
} else { }
throw new MigrationException("Error in Transformation " + diffResult.toString());
} Object scenarioTransformed = t.transform(currentJson);
}
Chainr identity = identityTransformation.get(targetVersion);
private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, StringBuilder log, boolean isScenario) if (identity == null) {
throws IOException, MigrationException { logger.error("No IdentityTransformation defined for Version " + targetVersion.toString());
String json = IOUtils.readTextFile(scenarioFilePath); throw new MigrationException("No IdentityTransformation definde for Version " + targetVersion.toString());
JsonNode node = StateJsonConverter.deserializeToNode(json); }
Tree tree = new Tree(node);
Object scenarioTransformedTest = identity.transform(scenarioTransformed);
String outputScenarioParentFolderName = isScenario ? "" : scenarioFilePath.getParent().getFileName().toString() + " _ "; Diffy.Result diffResult = transformationDiff.diff(scenarioTransformed, scenarioTransformedTest);
if (diffResult.isEmpty()) {
log.append("\n>> analyzing JSON tree of scenario <").append(outputScenarioParentFolderName).append(node.get("name").asText()).append(">\n"); return StateJsonConverter.deserializeToNode(scenarioTransformed);
} else {
Version version = Version.UNDEFINED; logger.error("Error in Transformation " + diffResult.toString());
throw new MigrationException("Error in Transformation " + diffResult.toString());
if (node.get("release") != null) { }
version = Version.fromString(node.get("release").asText()); }
if (version == null) { private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, String dirName)
throw new MigrationException("release version " + node.get("release").asText() + " is unknown. If this " + throws IOException, MigrationException {
"is a valid release, update the version-list in MigrationAssistant accordingly"); String json = IOUtils.readTextFile(scenarioFilePath);
} JsonNode node = StateJsonConverter.deserializeToNode(json);
// if enforced migration should be done from baseVersion to latestVersion String parentPath = dirName.equals(SCENARIO_DIR) ? SCENARIO_DIR + "/" : OUTPUT_DIR + "/" + scenarioFilePath.getParent().getFileName().toString() + "/";
if (reapplyLatestMigrationFlag && baseVersion != null) {
version = baseVersion; logger.info(">> analyzing JSON tree of scenario <" + parentPath + node.get("name").asText() + ">");
} else if(reapplyLatestMigrationFlag) { // if enforced migration should be done from prev version to latest Version version = Version.UNDEFINED;
Optional<Version> optVersion = Version.getPrevious(version);
if(optVersion.isPresent()) { if (node.get("release") != null) {
version = optVersion.get(); version = Version.fromString(node.get("release").asText());
}
else { if (version == null || version.equalOrSamller(Version.NOT_A_RELEASE)) {
return false; logger.error("release version " + node.get("release").asText() + " is unknown or not " +
} "supported. If this is a valid release create a version transformation and a new idenity transformation");
} // if no enforced migration should be done and we are at the latest version, no migration is required. throw new MigrationException("release version " + node.get("release").asText() + " is unknown or not " +
else if(version == Version.latest()) { "supported. If this is a valid releasecreate a version transformation and a new idenity transformation");
return false; }
}
} // if enforced migration should be done from baseVersion to latestVersion
if (reapplyLatestMigrationFlag && baseVersion != null) {
JsonNode transformedNode = transform(node, version); version = baseVersion;
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, LEGACY_EXTENSION, false); } else if (reapplyLatestMigrationFlag) { // if enforced migration should be done from prev version to latest
} Optional<Version> optVersion = Version.getPrevious(version);
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(transformedNode)); if (optVersion.isPresent()) {
return true; version = optVersion.get();
} } else {
return false;
private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder) }
throws IOException { } // if no enforced migration should be done and we are at the latest version, no migration is required.
Path source = scenarioFilePath; else if (version == Version.latest()) {
Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension); return false;
}
if (moveOutputFolder) { } else {
source = source.getParent(); logger.error("Version is unknown");
target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension); throw new MigrationException("Version is unknown");
} }
IOUtils.createDirectoryIfNotExisting(target); JsonNode transformedNode = node;
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten? // apply all transformation from current to latest version.
} for (Version v : Version.listToLatest(version)) {
transformedNode = transform(transformedNode, v);
}
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, LEGACY_EXTENSION, false);
}
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(transformedNode));
return true;
}
private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder)
throws IOException {
Path source = scenarioFilePath;
Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension);
if (moveOutputFolder) {
source = source.getParent();
target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension);
}
IOUtils.createDirectoryIfNotExisting(target);
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten?
}
} }
...@@ -6,12 +6,17 @@ public class MigrationException extends Exception { ...@@ -6,12 +6,17 @@ public class MigrationException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public MigrationException() {} public MigrationException() {
}
public MigrationException(String message) { public MigrationException(String message) {
super(message); super(message);
} }
public MigrationException(String message, Throwable cause) {
super(message, cause);
}
public MigrationException(Incident incident, String message) { public MigrationException(Incident incident, String message) {
super(incident.getClass().getSimpleName() + ": " + message); super(incident.getClass().getSimpleName() + ": " + message);
} }
......
package org.vadere.simulator.projects.migration; package org.vadere.simulator.projects.migration;
import java.util.Objects;
public class MigrationResult { public class MigrationResult {
public int total; public int total;
public int upToDate; public int upToDate;
public int legacy; public int legacy;
public int nomigratable; public int notmigratable;
public MigrationResult(){ public MigrationResult() {
} }
public MigrationResult(int total){ public MigrationResult(int total, int upToDate, int legacy, int notmigratable) {
this.total = total; this.total = total;
} this.upToDate = upToDate;
this.legacy = legacy;
boolean checkTotal(){ this.notmigratable = notmigratable;
return (upToDate + legacy + nomigratable) == total; }
}
public MigrationResult(int total) {
public MigrationResult add(MigrationResult other){ this.total = total;
this.total =+ other.total; }
this.upToDate =+ other.upToDate;
this.legacy =+ other.legacy; boolean checkTotal() {
this.nomigratable =+ other.nomigratable; return (upToDate + legacy + notmigratable) == total;
return this; }
}
public MigrationResult add(MigrationResult other) {
this.total = this.total + other.total;
this.upToDate = this.upToDate + other.upToDate;
this.legacy = this.legacy + other.legacy;
this.notmigratable = this.notmigratable + other.notmigratable;
return this;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
MigrationResult result = (MigrationResult) o;
return total == result.total &&
upToDate == result.upToDate &&
legacy == result.legacy &&
notmigratable == result.notmigratable;
}
@Override
public int hashCode() {
return Objects.hash(total, upToDate, legacy, notmigratable);
}
@Override
public String toString() {
return "MigrationResult{" +
"total=" + total +
", upToDate=" + upToDate +
", legacy=" + legacy +
", notmigratable=" + notmigratable +
'}';
}
} }
package org.vadere.simulator.projects.migration; package org.vadere.simulator.projects.migration;
import com.bazaarvoice.jolt.JsonUtils;
import com.fasterxml.jackson.databind.JsonNode;
import org.junit.After;
import org.junit.Test; import org.junit.Test;
import org.junit.Assert.*;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.state.util.StateJsonConverter;
import org.vadere.util.io.IOUtils;
import org.vadere.util.io.RecursiveCopy;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import joptsimple.internal.Strings;
import static org.junit.Assert.*; import static org.junit.Assert.assertEquals;
public class MigrationTest { public class MigrationTest {
// clean up after test
@After
public void resetTestStructure() throws URISyntaxException {
String dest = getClass().getResource("/migration/testProject_v0.1").toURI().getPath();
String source = getClass().getResource("/migration/testProject_v0.1.bak").toURI().getPath();
try {
if (Paths.get(dest).toFile().exists()) {
Files.walk(Paths.get(dest))
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.forEach(File::delete);
}
Files.walkFileTree(Paths.get(source), new RecursiveCopy(source, dest));
} catch (IOException e) {
e.printStackTrace();
}
}
// Test transformation of single scenario file
@Test
public void TestTransform() throws IOException {
String json = IOUtils.readTextFileFromResources("/migration/testProject_v0.1/scenarios/basic_1_chicken_osm1.scenario");
JsonNode node = StateJsonConverter.deserializeToNode(json);
Migration migration = new Migration();
try {
JsonNode newNode = migration.transform(node, Version.V0_2);
} catch (MigrationException e) {
e.printStackTrace();
}
}
// Test project transformation
@Test
public void TestTransformProject() throws URISyntaxException, IOException {
String projectPath = getClass().getResource("/migration/testProject_v0.1").toURI().getPath();
@Test Migration migration = new Migration();
public void TestTransform(){
} MigrationResult res = migration.analyzeProject(projectPath);
assertEquals("", new MigrationResult(12, 1, 10, 1), res);
System.out.println(Strings.repeat('#', 80));
}
} }