The expiration time for new job artifacts in CI/CD pipelines is now 30 days (GitLab default). Previously generated artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 58cd4211 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

add new migration implementation with tests.

parent 97083f4f
......@@ -4,9 +4,12 @@ import com.bazaarvoice.jolt.Chainr;
import com.bazaarvoice.jolt.Diffy;
import com.bazaarvoice.jolt.JsonUtils;
import com.fasterxml.jackson.databind.JsonNode;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.state.util.StateJsonConverter;
import org.vadere.util.io.IOUtils;
import org.vadere.util.logging.LogBufferAppender;
import java.io.File;
import java.io.IOException;
......@@ -25,177 +28,199 @@ import static org.vadere.util.io.IOUtils.SCENARIO_DIR;
public class Migration {
private static HashMap<Version, Chainr> identityTransformation = new LinkedHashMap<>();
// Version is the target version. Only Incremental Transformation supported 1->2->3 (not 1->3)
private static HashMap<Version, Chainr> transformations = new LinkedHashMap<>();
private static final String LEGACY_EXTENSION = "legacy";
private static final String NONMIGRATABLE_EXTENSION = "nonmigratable";
static {
identityTransformation.put(Version.V0_1, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v1.json")));
identityTransformation.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v2.json")));
transformations.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/transform_v1_to_v2.json")));
}
private Version baseVersion = null;
private boolean reapplyLatestMigrationFlag = false;
private StringBuilder migrationLog;
private Diffy transformationDiff;
public Migration(){
this.migrationLog = new StringBuilder();
this.transformationDiff = new Diffy();
}
public Migration(StringBuilder migrationLog){
this.migrationLog = migrationLog;
this.transformationDiff = new Diffy();
}
public void setReapplyLatestMigrationFlag() {
reapplyLatestMigrationFlag = true;
baseVersion = null;
}
public void setReapplyLatestMigrationFlag(final Version version) {
reapplyLatestMigrationFlag = true;
baseVersion = version;
}
public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
MigrationResult stats = new MigrationResult();
Path scenarioDir = Paths.get(projectFolderPath, SCENARIO_DIR);
if (Files.exists(scenarioDir)) {
stats = analyzeDirectory(scenarioDir, SCENARIO_DIR);
}
Path outputDir = Paths.get(projectFolderPath, OUTPUT_DIR);
if (Files.exists(outputDir)) {
MigrationResult outputDirStats = analyzeDirectory(outputDir, OUTPUT_DIR);
stats.add(outputDirStats);
}
baseVersion = null;
return stats;
}
public MigrationResult analyzeDirectory(Path dir, String dirName) throws IOException{
Path legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(dirName);
File[] scenarioFiles = dirName.equals(SCENARIO_DIR) ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
MigrationResult stats = new MigrationResult(scenarioFiles.length);
for (File file : scenarioFiles) {
if (dirName.equals(OUTPUT_DIR)) {
String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies
legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(OUTPUT_DIR).resolve(fileFolder);
}
Path scenarioFilePath = Paths.get(file.getAbsolutePath());
try {
if (analyzeScenario(scenarioFilePath, legacyDir, migrationLog, dirName.equals(SCENARIO_DIR))) {
stats.legacy++;
} else {
stats.upToDate++;
}
} catch (MigrationException e) {
moveFileAddExtension(scenarioFilePath, legacyDir, NONMIGRATABLE_EXTENSION, dirName.equals(SCENARIO_DIR));
migrationLog.append("! --> Can't migrate the scenario to latest version, removed it from the directory (")
.append(e.getMessage()).append(")\n")
.append("If you can fix this problem manually, do so and then remove .")
.append(NONMIGRATABLE_EXTENSION).append(" from the file in the ")
.append(LEGACY_DIR).append("-directory ")
.append("and move it back into the scenarios-directory, it will be checked again when the GUI restarts.\n");
stats.nomigratable++;
}
}
return stats;
}
public JsonNode transform (JsonNode currentJson, Version targetVersion) throws MigrationException {
Chainr t = transformations.get(targetVersion);
if (t == null)
throw new MigrationException("No Transformation defined for Version " + targetVersion.toString());
Object scenarioTransformed = t.transform(currentJson);
Chainr identity = identityTransformation.get(targetVersion);
if (identity == null)
throw new MigrationException("No IdentityTransformation definde for Version " + targetVersion.toString());
Object scenarioTransformedTest = identity.transform(scenarioTransformed);
Diffy.Result diffResult = transformationDiff.diff(scenarioTransformed, scenarioTransformedTest);
if (diffResult.isEmpty()){
return (JsonNode) scenarioTransformed;
} else {
throw new MigrationException("Error in Transformation " + diffResult.toString());
}
}
private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, StringBuilder log, boolean isScenario)
throws IOException, MigrationException {
String json = IOUtils.readTextFile(scenarioFilePath);
JsonNode node = StateJsonConverter.deserializeToNode(json);
Tree tree = new Tree(node);
String outputScenarioParentFolderName = isScenario ? "" : scenarioFilePath.getParent().getFileName().toString() + " _ ";
log.append("\n>> analyzing JSON tree of scenario <").append(outputScenarioParentFolderName).append(node.get("name").asText()).append(">\n");
Version version = Version.UNDEFINED;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
if (version == null) {
throw new MigrationException("release version " + node.get("release").asText() + " is unknown. If this " +
"is a valid release, update the version-list in MigrationAssistant accordingly");
}
// if enforced migration should be done from baseVersion to latestVersion
if (reapplyLatestMigrationFlag && baseVersion != null) {
version = baseVersion;
} else if(reapplyLatestMigrationFlag) { // if enforced migration should be done from prev version to latest
Optional<Version> optVersion = Version.getPrevious(version);
if(optVersion.isPresent()) {
version = optVersion.get();
}
else {
return false;
}
} // if no enforced migration should be done and we are at the latest version, no migration is required.
else if(version == Version.latest()) {
return false;
}
}
JsonNode transformedNode = transform(node, version);
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, LEGACY_EXTENSION, false);
}
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(transformedNode));
return true;
}
private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder)
throws IOException {
Path source = scenarioFilePath;
Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension);
if (moveOutputFolder) {
source = source.getParent();
target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension);
}
IOUtils.createDirectoryIfNotExisting(target);
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten?
}
private final static Logger logger = Logger.getLogger(Migration.class);
private final static LogBufferAppender appender;
private static HashMap<Version, Chainr> identityTransformation = new LinkedHashMap<>();
// Version is the target version. Only Incremental Transformation supported 1->2->3 (not 1->3)
private static HashMap<Version, Chainr> transformations = new LinkedHashMap<>();
private static final String LEGACY_EXTENSION = "legacy";
private static final String NONMIGRATABLE_EXTENSION = "nonmigratable";
static {
identityTransformation.put(Version.V0_1, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v1.json")));
identityTransformation.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v2.json")));
transformations.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/transform_v1_to_v2.json")));
appender = new LogBufferAppender();
}
private Version baseVersion = null;
private boolean reapplyLatestMigrationFlag = false;
private Diffy transformationDiff;
public Migration() {
this.transformationDiff = new Diffy();
logger.addAppender(appender);
}
public String getLog() {
return appender.getMigrationLog();
}
public void setReapplyLatestMigrationFlag() {
reapplyLatestMigrationFlag = true;
baseVersion = null;
}
public void setReapplyLatestMigrationFlag(final Version version) {
reapplyLatestMigrationFlag = true;
baseVersion = version;
}
public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
MigrationResult stats = new MigrationResult();
Path scenarioDir = Paths.get(projectFolderPath, SCENARIO_DIR);
if (Files.exists(scenarioDir)) {
stats = analyzeDirectory(scenarioDir, SCENARIO_DIR);
}
Path outputDir = Paths.get(projectFolderPath, OUTPUT_DIR);
if (Files.exists(outputDir)) {
MigrationResult outputDirStats = analyzeDirectory(outputDir, OUTPUT_DIR);
stats.add(outputDirStats);
}
baseVersion = null;
return stats;
}
public MigrationResult analyzeDirectory(Path dir, String dirName) throws IOException {
Path legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(dirName);
File[] scenarioFiles = dirName.equals(SCENARIO_DIR) ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
MigrationResult stats = new MigrationResult(scenarioFiles.length);
for (File file : scenarioFiles) {
if (dirName.equals(OUTPUT_DIR)) {
String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies
legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(OUTPUT_DIR).resolve(fileFolder);
}
Path scenarioFilePath = Paths.get(file.getAbsolutePath());
try {
if (analyzeScenario(scenarioFilePath, legacyDir, dirName)) {
stats.legacy++;
} else {
stats.upToDate++;
}
} catch (MigrationException e) {
moveFileAddExtension(scenarioFilePath, legacyDir, NONMIGRATABLE_EXTENSION, dirName.equals(SCENARIO_DIR));
logger.error("!> Can't migrate the scenario to latest version, removed it from the directory (" +
e.getMessage() + ") If you can fix this problem manually, do so and then remove ." +
NONMIGRATABLE_EXTENSION + " from the file in the " + LEGACY_DIR + "-directory "
+ "and move it back into the scenarios-directory, it will be checked again when the GUI restarts.");
stats.notmigratable++;
}
}
return stats;
}
public JsonNode transform(JsonNode currentJson, Version targetVersion) throws MigrationException {
try {
return transform(StateJsonConverter.convertJsonNodeToObject(currentJson), targetVersion);
} catch (IOException e) {
logger.error("Error in converting JsonNode To Map of Object representation");
throw new MigrationException("Error in converting JsonNode To Map of Object representation", e);
}
}
private JsonNode transform(Object currentJson, Version targetVersion) throws MigrationException {
Chainr t = transformations.get(targetVersion);
if (t == null) {
logger.error("No Transformation defined for Version " + targetVersion.toString());
throw new MigrationException("No Transformation defined for Version " + targetVersion.toString());
}
Object scenarioTransformed = t.transform(currentJson);
Chainr identity = identityTransformation.get(targetVersion);
if (identity == null) {
logger.error("No IdentityTransformation defined for Version " + targetVersion.toString());
throw new MigrationException("No IdentityTransformation definde for Version " + targetVersion.toString());
}
Object scenarioTransformedTest = identity.transform(scenarioTransformed);
Diffy.Result diffResult = transformationDiff.diff(scenarioTransformed, scenarioTransformedTest);
if (diffResult.isEmpty()) {
return StateJsonConverter.deserializeToNode(scenarioTransformed);
} else {
logger.error("Error in Transformation " + diffResult.toString());
throw new MigrationException("Error in Transformation " + diffResult.toString());
}
}
private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, String dirName)
throws IOException, MigrationException {
String json = IOUtils.readTextFile(scenarioFilePath);
JsonNode node = StateJsonConverter.deserializeToNode(json);
String parentPath = dirName.equals(SCENARIO_DIR) ? SCENARIO_DIR + "/" : OUTPUT_DIR + "/" + scenarioFilePath.getParent().getFileName().toString() + "/";
logger.info(">> analyzing JSON tree of scenario <" + parentPath + node.get("name").asText() + ">");
Version version = Version.UNDEFINED;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
if (version == null || version.equalOrSamller(Version.NOT_A_RELEASE)) {
logger.error("release version " + node.get("release").asText() + " is unknown or not " +
"supported. If this is a valid release create a version transformation and a new idenity transformation");
throw new MigrationException("release version " + node.get("release").asText() + " is unknown or not " +
"supported. If this is a valid releasecreate a version transformation and a new idenity transformation");
}
// if enforced migration should be done from baseVersion to latestVersion
if (reapplyLatestMigrationFlag && baseVersion != null) {
version = baseVersion;
} else if (reapplyLatestMigrationFlag) { // if enforced migration should be done from prev version to latest
Optional<Version> optVersion = Version.getPrevious(version);
if (optVersion.isPresent()) {
version = optVersion.get();
} else {
return false;
}
} // if no enforced migration should be done and we are at the latest version, no migration is required.
else if (version == Version.latest()) {
return false;
}
} else {
logger.error("Version is unknown");
throw new MigrationException("Version is unknown");
}
JsonNode transformedNode = node;
// apply all transformation from current to latest version.
for (Version v : Version.listToLatest(version)) {
transformedNode = transform(transformedNode, v);
}
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, LEGACY_EXTENSION, false);
}
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(transformedNode));
return true;
}
private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder)
throws IOException {
Path source = scenarioFilePath;
Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension);
if (moveOutputFolder) {
source = source.getParent();
target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension);
}
IOUtils.createDirectoryIfNotExisting(target);
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten?
}
}
......@@ -6,12 +6,17 @@ public class MigrationException extends Exception {
private static final long serialVersionUID = 1L;
public MigrationException() {}
public MigrationException() {
}
public MigrationException(String message) {
super(message);
}
public MigrationException(String message, Throwable cause) {
super(message, cause);
}
public MigrationException(Incident incident, String message) {
super(incident.getClass().getSimpleName() + ": " + message);
}
......
package org.vadere.simulator.projects.migration;
import java.util.Objects;
public class MigrationResult {
public int total;
public int upToDate;
public int legacy;
public int nomigratable;
public MigrationResult(){
}
public MigrationResult(int total){
this.total = total;
}
boolean checkTotal(){
return (upToDate + legacy + nomigratable) == total;
}
public MigrationResult add(MigrationResult other){
this.total =+ other.total;
this.upToDate =+ other.upToDate;
this.legacy =+ other.legacy;
this.nomigratable =+ other.nomigratable;
return this;
}
public int total;
public int upToDate;
public int legacy;
public int notmigratable;
public MigrationResult() {
}
public MigrationResult(int total, int upToDate, int legacy, int notmigratable) {
this.total = total;
this.upToDate = upToDate;
this.legacy = legacy;
this.notmigratable = notmigratable;
}
public MigrationResult(int total) {
this.total = total;
}
boolean checkTotal() {
return (upToDate + legacy + notmigratable) == total;
}
public MigrationResult add(MigrationResult other) {
this.total = this.total + other.total;
this.upToDate = this.upToDate + other.upToDate;
this.legacy = this.legacy + other.legacy;
this.notmigratable = this.notmigratable + other.notmigratable;
return this;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
MigrationResult result = (MigrationResult) o;
return total == result.total &&
upToDate == result.upToDate &&
legacy == result.legacy &&
notmigratable == result.notmigratable;
}
@Override
public int hashCode() {
return Objects.hash(total, upToDate, legacy, notmigratable);
}
@Override
public String toString() {
return "MigrationResult{" +
"total=" + total +
", upToDate=" + upToDate +
", legacy=" + legacy +
", notmigratable=" + notmigratable +
'}';
}
}
package org.vadere.simulator.projects.migration;
import com.bazaarvoice.jolt.JsonUtils;
import com.fasterxml.jackson.databind.JsonNode;
import org.junit.After;
import org.junit.Test;
import org.junit.Assert.*;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.state.util.StateJsonConverter;
import org.vadere.util.io.IOUtils;
import org.vadere.util.io.RecursiveCopy;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import joptsimple.internal.Strings;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
public class MigrationTest {
// clean up after test
@After
public void resetTestStructure() throws URISyntaxException {
String dest = getClass().getResource("/migration/testProject_v0.1").toURI().getPath();
String source = getClass().getResource("/migration/testProject_v0.1.bak").toURI().getPath();
try {
if (Paths.get(dest).toFile().exists()) {
Files.walk(Paths.get(dest))
.sorted(Comparator.reverseOrder())
.map(Path::toFile)
.forEach(File::delete);
}
Files.walkFileTree(Paths.get(source), new RecursiveCopy(source, dest));
} catch (IOException e) {
e.printStackTrace();
}
}
// Test transformation of single scenario file
@Test
public void TestTransform() throws IOException {
String json = IOUtils.readTextFileFromResources("/migration/testProject_v0.1/scenarios/basic_1_chicken_osm1.scenario");
JsonNode node = StateJsonConverter.deserializeToNode(json);
Migration migration = new Migration();
try {
JsonNode newNode = migration.transform(node, Version.V0_2);
} catch (MigrationException e) {
e.printStackTrace();
}
}
// Test project transformation
@Test
public void TestTransformProject() throws URISyntaxException, IOException {
String projectPath = getClass().getResource("/migration/testProject_v0.1").toURI().getPath();
@Test
public void TestTransform(){
Migration migration = new Migration();
}
MigrationResult res = migration.analyzeProject(projectPath);
assertEquals("", new MigrationResult(12,</