Commit 8375f258 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

move migration logic back to MigrationAssistant, add MigrationUtil

Features (MigrationUtil):

* create new Transformation and Identity files: This will use the
current identity    Transformation as starting point for the new
version. The new version string is automatically included in the new
files. The files will be created based   on the defined naming
convention and saved in VadereSimulator/resources

* automate scenario migration without gui: The MigrationUtil will
migrate   all scenario files within the specified directory and all
child directories.

To exclude specific sub-trees or only specific
directories the igonoreDirs List can be expanded.

  To exclude sub-trees or specific directories without code change add
  one of the following marker-files to the directory.

  * DO_NOT_MIGRATE or .DO_NOT_MIGRATE: Ignore current directory but
    continue with existing child directories.

  * DO_NOT_MIGRATE_TREE or .DO_NOT_MIGRATE_TREE: Ignore the directories
    and the complete sub-tree.

  Use the marker files from the Tools directory. These files have a
  comment to explain for what they are used for.

  The content of the file is not used, only the file name is needed.
  These files must be added to the git cache to work.

 * automate migration revert: This will use the revert function of the
   MigrationAssistant to revert a migration if the <file-name>.legacy
   files still exist. For this to work the legacy file must be in the
   same directory as the scenario-file
parent bbe6bc97
Pipeline #64519 failed with stages
in 44 minutes and 27 seconds
# Marker-File for MigrationUtil
The content of this file is ignored
All scenario files in this directory will NOT be migrated.
This Marker-File will allow traversal down the directory tree.
Use DO_NOT_MIGRATE_TREE or .DO_NOT_MIGRATE_TREE to stop migration for
the complete subtree of the current directory.
\ No newline at end of file
# Marker-File for MigrationUtil
The content of this file is ignored.
All scenario files in this directory and all child directories will _NOT_ be migrated.
Use DO_NOT_MIGRATE or .DO_NOT_MIGRATE to stop migration only for the current directory and
allow traversal down to child directories.
\ No newline at end of file
......@@ -444,7 +444,6 @@
"spec": {
"name": "",
"description": "",
// set new version
"release": "0.3",
"commithash": "warning: no commit hash",
"processWriters": {
......
......@@ -7,12 +7,9 @@ import org.apache.log4j.Logger;
import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationException;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.util.io.IOUtils;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
public class MigrationSubCommand implements SubCommandRunner{
private final static Logger logger = Logger.getLogger(MigrationSubCommand.class);
......@@ -48,17 +45,14 @@ public class MigrationSubCommand implements SubCommandRunner{
try{
logger.info("Scenario file" + scenarioFile.getFileName().toString());
logger.info("Try to migrate to version " + targetVersion);
out = ma.convertFile(scenarioFile, targetVersion);
if (outputPathString == null){
//overwrite inptu
Files.copy(scenarioFile, addSuffix(scenarioFile, "." + IOUtils.LEGACY_DIR), StandardCopyOption.REPLACE_EXISTING);
logger.info("write new verison to " + scenarioFile.toString());
IOUtils.writeTextFile(scenarioFile.toString(), out);
} else {
logger.info("write new version to " + outputPathString);
IOUtils.writeTextFile(outputPathString, out);
Path outputFile = null;
if (outputPathString != null){
outputFile = Paths.get(outputPathString);
}
ma.migrateFile(scenarioFile, targetVersion, outputFile);
} catch (Exception e) {
logger.error("Migration failed", e);
}
......
......@@ -2,16 +2,6 @@ package org.vadere.simulator.projects.migration;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
......@@ -23,6 +13,17 @@ import org.vadere.util.io.IOUtils;
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
public class IncidentMigrationAssistant extends MigrationAssistant {
private static Logger logger = LogManager.getLogger(IncidentMigrationAssistant.class);
......@@ -76,6 +77,12 @@ public class IncidentMigrationAssistant extends MigrationAssistant {
throw new NotImplementedException();
}
@Override
public void migrateFile(Path scenarioFilePath, Version targetVersion, Path outputFile) {
throw new NotImplementedException();
}
@Override
public void revertFile(Path scenarioFile) {
throw new NotImplementedException();
......
......@@ -67,6 +67,7 @@ public class JoltMigrationAssistant extends MigrationAssistant {
return stats;
}
// will return null if current and target version match
@Override
public String convertFile(Path scenarioFilePath, Version targetVersion) throws MigrationException {
JsonNode node;
......@@ -79,7 +80,7 @@ public class JoltMigrationAssistant extends MigrationAssistant {
}
restLog();
logger.info(">> analyzing JSON tree of scenario <" + node.get("name").asText() + ">");
Version version = Version.UNDEFINED;
Version version;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
......@@ -94,6 +95,12 @@ public class JoltMigrationAssistant extends MigrationAssistant {
version = Version.NOT_A_RELEASE;
}
if (version.equals(targetVersion)) {
logger.info("Nothing to do current version and target version match");
restLog();
return null;
}
JsonNode transformedNode = node;
// apply all transformation from current to latest version.
for (Version v : Version.listToLatest(version)) {
......@@ -102,6 +109,7 @@ public class JoltMigrationAssistant extends MigrationAssistant {
}
try {
restLog();
return StateJsonConverter.serializeJsonNode(transformedNode);
} catch (JsonProcessingException e) {
logger.error("could not serializeJsonNode after Transformation: " + e.getMessage());
......@@ -109,6 +117,35 @@ public class JoltMigrationAssistant extends MigrationAssistant {
}
}
@Override
public void migrateFile(Path scenarioFilePath, Version targetVersion, Path outputFile) throws MigrationException {
String json = convertFile(scenarioFilePath, targetVersion);
if (json == null){
logger.info("Nothing todo scenarioFile up-to-date");
return;
}
if (outputFile == null || scenarioFilePath.equals(outputFile)){
//overwrite scenarioFile
Path backupPath = getBackupPath(scenarioFilePath);
try {
Files.copy(scenarioFilePath, backupPath, StandardCopyOption.REPLACE_EXISTING);
IOUtils.writeTextFile(scenarioFilePath.toString(), json);
} catch (IOException e) {
logger.error("Cannot overwrite scenarioFile or cannot write new file new version: " + e.getMessage(), e);
throw new MigrationException("Cannot overwrite scenarioFile or cannot write new file new version: " + e.getMessage(), e);
}
} else {
try {
IOUtils.writeTextFile(outputFile.toString(), json);
} catch (IOException e) {
throw new MigrationException("Cannot write to output file: " + e.getMessage(), e);
}
}
}
@Override
public void revertFile(Path scenarioFile) throws MigrationException {
Path backupFile = MigrationAssistant.getBackupPath(scenarioFile);
......
......@@ -33,7 +33,6 @@ public abstract class MigrationAssistant {
public abstract void restLog();
// public abstract void analyzeSingleScenario(Path path) throws IOException;
public abstract MigrationResult analyzeProject(String projectFolderPath) throws IOException;
......@@ -43,5 +42,7 @@ public abstract class MigrationAssistant {
public abstract String convertFile(Path scenarioFilePath, Version targetVersion) throws MigrationException;
public abstract void migrateFile(Path scenarioFilePath, Version targetVersion, Path outputFile) throws MigrationException;
public abstract void revertFile(Path scenarioFile) throws MigrationException;
}
package org.vadere.simulator.projects.migration.helper;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationException;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.simulator.projects.migration.jolttranformation.JoltTransformation;
import org.vadere.util.io.IOUtils;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.function.Function;
import java.util.stream.Collectors;
public class MigrationUtil {
private final static Logger logger = Logger.getLogger(MigrationUtil.class);
public static void main(String[] args) throws URISyntaxException, IOException {
MigrationUtil migrationUtil = new MigrationUtil();
// migrationUtil.migrateTestData(Paths.get("/home/lphex/hm.d/vadere/"));
migrationUtil.generateNewVersionTransform(Paths.get("/home/lphex/hm.d/vadere/VadereSimulator/resources"), "0.4");
}
private ArrayList<String> createList(String... addAll){
return Arrays.stream(addAll).collect(Collectors.toCollection(ArrayList::new));
}
private void generateNewVersionTransform(Path resourceDir, String newVersionLabel) throws URISyntaxException, IOException {
Path oldTransform = JoltTransformation.getTransforamtionFile(Version.latest());
Path oldIdentity = JoltTransformation.getIdenityFile(Version.latest());
String newTransformString = JoltTransformation
.getTransforamtionResourcePath(Version.latest().label('-'), newVersionLabel);
String newIdenityString = JoltTransformation
.getIdentiyResoucrePath(newVersionLabel);
Path newTransform = resourceDir.resolve(newTransformString.substring(1));
Path newIdenity = resourceDir.resolve(newIdenityString.substring(1));
String json = IOUtils.readTextFile(oldIdentity);
json = json.replace("\"release\": \"" + Version.latest().label('-') + "\",", "\"release\": \"" + newVersionLabel + "\",");
IOUtils.writeTextFile(newIdenity.toString(), json);
json = json.replace("\"release\": \"&\",", "// no relase here to overwrite it with default at the default operation down below");
IOUtils.writeTextFile(newTransform.toString(), json);
}
/**
*
* @param p root path of repo
*/
private void migrateTestData(Path p) {
ArrayList<String> ignoreDirs = createList("VadereModelTests", "target", "Documentation");
ArrayList<String> dirMarker = createList("DO_NOT_MIGRATE", ".DO_NOT_MIGRATE");
ArrayList<String> treeMarker = createList("DO_NOT_MIGRATE_TREE", ".DO_NOT_MIGRATE_TREE");
FileVisitor<Path> visitor = getVisitor(ignoreDirs, treeMarker, dirMarker, this::migrateToLatestVersion);
try {
Files.walkFileTree(p, visitor);
} catch (IOException e) {
e.printStackTrace();
}
}
private boolean migrateToLatestVersion(Path path){
MigrationAssistant ms = MigrationAssistant.getNewInstance(MigrationOptions.defaultOptions());
try {
ms.migrateFile(path, Version.latest(), null);
return true;
} catch (MigrationException e) {
logger.error("Error in MigrationUtil stop: " + e.getMessage());
e.printStackTrace();
return false;
}
}
private boolean revert(Path path){
MigrationAssistant ms = MigrationAssistant.getNewInstance(MigrationOptions.defaultOptions());
try{
ms.revertFile(path);
return true;
} catch (MigrationException e){
logger.error("Error in MigraionUtil: " + e.getMessage());
return false;
}
}
FileVisitor<Path> getVisitor(ArrayList<String> ignoreDirs, ArrayList<String> treeMarker, ArrayList<String> dirMarker, Function<Path, Boolean> func) {
return new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
boolean hasTreeMarker = treeMarker.stream().anyMatch( m -> dir.resolve(m).toFile().exists());
if (ignoreDirs.contains(dir.getFileName().toString())){ // skip ignored directories-trees.
return FileVisitResult.SKIP_SUBTREE;
} else if (hasTreeMarker) { // skip directory tree if treeMarker is present.
return FileVisitResult.SKIP_SUBTREE;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
// continue traversal if not scenario
if (! file.getFileName().toString().endsWith("scenario"))
return FileVisitResult.CONTINUE;
// if dirMarker is set do not migrate any scenario files in this dir
// but continue with sub-dirs.
boolean hasDirMarker = dirMarker.stream().anyMatch(m -> file.getParent().resolve(m).toFile().exists());
if (hasDirMarker){
return FileVisitResult.CONTINUE;
}
boolean ret = func.apply(file);
if (ret){
return FileVisitResult.CONTINUE;
} else {
return FileVisitResult.TERMINATE;
}
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
return null;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
return FileVisitResult.CONTINUE;
}
};
}
}
package org.vadere.simulator.projects.migration.helper;
public class Util {
public static void main(String[] args){
}
private void listScenarioFiles(){
}
}
......@@ -11,6 +11,10 @@ import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.migration.MigrationException;
import org.vadere.state.util.StateJsonConverter;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.concurrent.ConcurrentHashMap;
......@@ -35,8 +39,8 @@ public abstract class JoltTransformation {
throw new MigrationException("No Transformation needed. Already latest Version!");
String transformationResource = "/transform_v" + currentVersion.label('-').toUpperCase() + "_to_v" + currentVersion.nextVersion().label('-').toUpperCase() + ".json";
String identityResource = "/identity_v" + currentVersion.nextVersion().label('-').toUpperCase() + ".json";
String transformationResource = getTransforamtionResourcePath(currentVersion.label('-'), currentVersion.nextVersion().label('-'));
String identityResource = getIdentiyResoucrePath(currentVersion.nextVersion().label('-'));
JoltTransformation ret = transformations.getOrDefault(currentVersion, null);
if ( ret == null) {
......@@ -58,6 +62,28 @@ public abstract class JoltTransformation {
return ret;
}
public static Path getTransforamtionFile(Version toVersion) throws URISyntaxException {
String transformString = getTransforamtionResourcePath(
toVersion.previousVersion().label('-'),
toVersion.label('-'));
URI res = JoltTransformation.class.getResource(transformString).toURI();
return Paths.get(res);
}
public static Path getIdenityFile(Version v) throws URISyntaxException {
String idenityString = getIdentiyResoucrePath(v.label('-'));
URI res = JoltTransformation.class.getResource(idenityString).toURI();
return Paths.get(res);
}
public static String getTransforamtionResourcePath(String from, String to){
return "/transform_v" +from.toUpperCase() + "_to_v" + to.toUpperCase() + ".json";
}
public static String getIdentiyResoucrePath(String to){
return "/identity_v" + to.toUpperCase() + ".json";
}
public JoltTransformation(String transformation, String identity, Version version) throws MigrationException {
this.chainr = Chainr.fromSpec(JsonUtils.classpathToList(transformation));
this.identity = Chainr.fromSpec(JsonUtils.classpathToList(identity));
......
# Marker-File for MigrationUtil
The content of this file is ignored.
All scenario files in this directory and all child directories will _NOT_ be migrated.
Use DO_NOT_MIGRATE or .DO_NOT_MIGRATE to stop migration only for the current directory and
allow traversal down to child directories.
\ No newline at end of file
package org.vadere.simulator.entrypoints;
import net.sourceforge.argparse4j.inf.ArgumentParserException;
import org.hamcrest.core.StringContains;
import org.junit.After;
import org.junit.Before;
......@@ -68,6 +66,18 @@ public class MigrationSubCommandTest {
}
@Test
public void testMigrationSameVersion() throws IOException {
assertThat("Old Version must be 0.1", getText(baseScenario), v01);
String[] args = new String[]{SubCommand.MIGRATE.getCmdName(), "-f", baseScenario.toString(), "--target-version", Version.V0_1.label()};
VadereConsole.main(args);
Path legacyFile = MigrationAssistant.getBackupPath(baseScenario);
assertThat("New Version must be the same" , getText(baseScenario), v01);
assertFalse("No Transformation performed thus the should not be a legacyFile", legacyFile.toFile().exists());
}
private String getText (Path path) throws IOException {
return IOUtils.readTextFile(path);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment