Commit 37fde165 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

Merge branch 'migration_helper' into develop

parents 57d90e93 8375f258
Pipeline #64545 failed with stages
in 44 minutes and 24 seconds
# Marker-File for MigrationUtil
The content of this file is ignored
All scenario files in this directory will NOT be migrated.
This Marker-File will allow traversal down the directory tree.
Use DO_NOT_MIGRATE_TREE or .DO_NOT_MIGRATE_TREE to stop migration for
the complete subtree of the current directory.
\ No newline at end of file
# Marker-File for MigrationUtil
The content of this file is ignored.
All scenario files in this directory and all child directories will _NOT_ be migrated.
Use DO_NOT_MIGRATE or .DO_NOT_MIGRATE to stop migration only for the current directory and
allow traversal down to child directories.
\ No newline at end of file
......@@ -444,7 +444,6 @@
"spec": {
"name": "",
"description": "",
// set new version
"release": "0.3",
"commithash": "warning: no commit hash",
"processWriters": {
......
......@@ -5,48 +5,57 @@ import net.sourceforge.argparse4j.inf.Namespace;
import org.apache.log4j.Logger;
import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationException;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.util.io.IOUtils;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
public class MigrationSubCommand implements SubCommandRunner{
private final static Logger logger = Logger.getLogger(MigrationSubCommand.class);
@Override
public void run(Namespace ns, ArgumentParser parser) {
public void run(Namespace ns, ArgumentParser parser) throws Exception {
Path scenarioFile = Paths.get(ns.getString("scenario-file"));
if (!scenarioFile.toFile().exists() || !scenarioFile.toFile().isFile()){
logger.error("scenario-file does not exist, is not a regular file or you do not have read permissions:" +
scenarioFile.toFile().toString());
System.exit(-1);
}
String outputPathString = ns.getString("output-file");
String outputPathString = ns.getString("output-file");
Version targetVersion = Version.fromString(ns.getString("target-version"));
boolean revertMode = ns.getBoolean("revert-migration");
if (revertMode){
revert(scenarioFile);
} else {
migrate(scenarioFile, targetVersion, outputPathString);
}
}
private void revert(Path scenarioFile) throws MigrationException {
MigrationAssistant ma = MigrationAssistant.getNewInstance(MigrationOptions.defaultOptions());
ma.revertFile(scenarioFile);
}
private void migrate(Path scenarioFile, Version targetVersion, String outputPathString){
MigrationAssistant ma = MigrationAssistant.getNewInstance(MigrationOptions.defaultOptions());
String out;
try{
logger.info("Scenario file" + scenarioFile.getFileName().toString());
logger.info("Try to migrate to version " + targetVersion);
out = ma.convertFile(scenarioFile, targetVersion);
if (outputPathString == null){
//overwrite inptu
Files.copy(scenarioFile, addSuffix(scenarioFile, "." + IOUtils.LEGACY_DIR), StandardCopyOption.REPLACE_EXISTING);
logger.info("write new verison to " + scenarioFile.toString());
IOUtils.writeTextFile(scenarioFile.toString(), out);
} else {
logger.info("write new version to " + outputPathString);
IOUtils.writeTextFile(outputPathString, out);
Path outputFile = null;
if (outputPathString != null){
outputFile = Paths.get(outputPathString);
}
ma.migrateFile(scenarioFile, targetVersion, outputFile);
} catch (Exception e) {
logger.error("Migration failed", e);
}
}
private Path addSuffix(Path p, String suffix){
......
package org.vadere.simulator.entrypoints;
public enum SubCommand {
PROJECT_RUN("project-run"),
SCENARO_RUN("scenario-run"),
SUQ("suq"),
MIGRATE("migrate");
private String cmdName;
SubCommand(String s) {
this.cmdName = s;
}
public String getCmdName(){
return this.cmdName;
}
}
......@@ -9,6 +9,6 @@ import net.sourceforge.argparse4j.inf.Namespace;
*/
public interface SubCommandRunner {
void run(Namespace ns, ArgumentParser parser);
void run(Namespace ns, ArgumentParser parser) throws Exception;
}
package org.vadere.simulator.entrypoints;
import org.apache.log4j.Logger;
import org.vadere.simulator.projects.Scenario;
import org.vadere.simulator.projects.ScenarioRun;
import org.vadere.simulator.projects.migration.incidents.VersionBumpIncident;
import net.sourceforge.argparse4j.ArgumentParsers;
import net.sourceforge.argparse4j.impl.Arguments;
import net.sourceforge.argparse4j.inf.ArgumentParser;
import net.sourceforge.argparse4j.inf.ArgumentParserException;
import net.sourceforge.argparse4j.inf.Namespace;
import net.sourceforge.argparse4j.inf.Subparser;
import net.sourceforge.argparse4j.inf.Subparsers;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Locale;
import org.vadere.simulator.entrypoints.Version;
import org.apache.log4j.Logger;
/**
* Provides the possibility to start VADERE in console mode.
*
......@@ -24,20 +17,20 @@ public class VadereConsole {
private final static Logger logger = Logger.getLogger(VadereConsole.class);
/**
* @param args
*/
public static void main(String[] args) throws ArgumentParserException {
public static void main(String[] args) {
ArgumentParser parser = createArgumentParser();
// args = new String[]{"migrate", "-f", "/home/lphex/hm.d/vadere/VadereSimulator/testResources/data/simpleProject/output/test_postvis_2018-01-17_16-56-37.307/test_postvis.scenario"};
try {
Namespace ns = parser.parseArgs(args);
SubCommandRunner sRunner = (SubCommandRunner) ns.get("func");
SubCommandRunner sRunner = ns.get("func");
sRunner.run(ns, parser);
} catch (ArgumentParserException e) {
parser.handleError(e);
System.exit(1);
} catch (Exception e) {
logger.error("error in command:" + e.getMessage());
System.exit(1);
}
}
......@@ -53,7 +46,7 @@ public class VadereConsole {
// Run Project
Subparser projectRun = subparsers
.addParser("project-run")
.addParser(SubCommand.PROJECT_RUN.getCmdName())
.help("This command uses a Vadere Project and runs selected scenario.")
.setDefault("func", new ProjectRunSubCommand());
projectRun.addArgument("--project-dir", "-p")
......@@ -69,7 +62,7 @@ public class VadereConsole {
// Run Scenario
Subparser scenarioRun = subparsers
.addParser("scenario-run")
.addParser(SubCommand.SCENARO_RUN.getCmdName())
.help("Run scenario without a project")
.setDefault("func", new ScenarioRunSubCommand());
scenarioRun.addArgument("--output-dir", "-o")
......@@ -87,7 +80,7 @@ public class VadereConsole {
// Run SUQ
Subparser suqRun = subparsers
.addParser("suq")
.addParser(SubCommand.SUQ.getCmdName())
.help("Run a single scenario file to specify to fully controll folder structure for input and output.")
.setDefault("func", new SuqSubCommand());
......@@ -107,7 +100,7 @@ public class VadereConsole {
// Run Migration Assistant
Subparser migrationAssistant = subparsers
.addParser("migrate")
.addParser(SubCommand.MIGRATE.getCmdName())
.help("Run migration assistant on single sceanrio file")
.setDefault("func", new MigrationSubCommand());
......@@ -133,6 +126,13 @@ public class VadereConsole {
.choices(versions)
.help("Write new version to this file. If not specified backup input file and overwrite it.");
migrationAssistant.addArgument("--revert-migration")
.required(false)
.action(Arguments.storeTrue())
.dest("revert-migration")
.help("if set vadere will search for a <scenario-file>.legacy and will replace the current version with this backup." +
" The Backup must be in the same directory");
return parser;
}
......
......@@ -26,8 +26,8 @@ public enum Version {
return label;
}
public String label(char repalce) {
return label.replace(' ', repalce);
public String label(char replaceSpaceWith) {
return label.replace(' ', replaceSpaceWith);
}
public static Version fromString(String versionStr) {
......
......@@ -2,16 +2,6 @@ package org.vadere.simulator.projects.migration;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
......@@ -23,6 +13,17 @@ import org.vadere.util.io.IOUtils;
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
public class IncidentMigrationAssistant extends MigrationAssistant {
private static Logger logger = LogManager.getLogger(IncidentMigrationAssistant.class);
......@@ -72,7 +73,18 @@ public class IncidentMigrationAssistant extends MigrationAssistant {
}
@Override
public String convertFile(Path scenarioFilePath, Version targetVersion) throws IOException, MigrationException {
public String convertFile(Path scenarioFilePath, Version targetVersion) {
throw new NotImplementedException();
}
@Override
public void migrateFile(Path scenarioFilePath, Version targetVersion, Path outputFile) {
throw new NotImplementedException();
}
@Override
public void revertFile(Path scenarioFile) {
throw new NotImplementedException();
}
......
package org.vadere.simulator.projects.migration;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import org.apache.log4j.Logger;
......@@ -66,13 +67,20 @@ public class JoltMigrationAssistant extends MigrationAssistant {
return stats;
}
// will return null if current and target version match
@Override
public String convertFile(Path scenarioFilePath, Version targetVersion) throws IOException, MigrationException {
String json = IOUtils.readTextFile(scenarioFilePath);
JsonNode node = StateJsonConverter.deserializeToNode(json);
public String convertFile(Path scenarioFilePath, Version targetVersion) throws MigrationException {
JsonNode node;
try {
String json = IOUtils.readTextFile(scenarioFilePath);
node = StateJsonConverter.deserializeToNode(json);
} catch (IOException e){
logger.error("Error converting File: " + e.getMessage());
throw new MigrationException("Could not read JsonFile or create Json representation" + e.getMessage());
}
restLog();
logger.info(">> analyzing JSON tree of scenario <" + node.get("name").asText() + ">");
Version version = Version.UNDEFINED;
Version version;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
......@@ -87,6 +95,12 @@ public class JoltMigrationAssistant extends MigrationAssistant {
version = Version.NOT_A_RELEASE;
}
if (version.equals(targetVersion)) {
logger.info("Nothing to do current version and target version match");
restLog();
return null;
}
JsonNode transformedNode = node;
// apply all transformation from current to latest version.
for (Version v : Version.listToLatest(version)) {
......@@ -94,9 +108,67 @@ public class JoltMigrationAssistant extends MigrationAssistant {
transformedNode = transform(transformedNode, v);
}
return StateJsonConverter.serializeJsonNode(transformedNode);
try {
restLog();
return StateJsonConverter.serializeJsonNode(transformedNode);
} catch (JsonProcessingException e) {
logger.error("could not serializeJsonNode after Transformation: " + e.getMessage());
throw new MigrationException("could not serializeJsonNode after Transformation: " + e.getMessage());
}
}
@Override
public void migrateFile(Path scenarioFilePath, Version targetVersion, Path outputFile) throws MigrationException {
String json = convertFile(scenarioFilePath, targetVersion);
if (json == null){
logger.info("Nothing todo scenarioFile up-to-date");
return;
}
if (outputFile == null || scenarioFilePath.equals(outputFile)){
//overwrite scenarioFile
Path backupPath = getBackupPath(scenarioFilePath);
try {
Files.copy(scenarioFilePath, backupPath, StandardCopyOption.REPLACE_EXISTING);
IOUtils.writeTextFile(scenarioFilePath.toString(), json);
} catch (IOException e) {
logger.error("Cannot overwrite scenarioFile or cannot write new file new version: " + e.getMessage(), e);
throw new MigrationException("Cannot overwrite scenarioFile or cannot write new file new version: " + e.getMessage(), e);
}
} else {
try {
IOUtils.writeTextFile(outputFile.toString(), json);
} catch (IOException e) {
throw new MigrationException("Cannot write to output file: " + e.getMessage(), e);
}
}
}
@Override
public void revertFile(Path scenarioFile) throws MigrationException {
Path backupFile = MigrationAssistant.getBackupPath(scenarioFile);
if (!backupFile.toFile().exists()){
logger.error("There does not exist a Backup for the given file");
logger.error("File: " + scenarioFile.toString());
logger.error("Backup does not exist: " + backupFile.toString());
}
try {
Files.copy(backupFile, scenarioFile, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
logger.error("Could not copy legacy backup to current version: " + e.getMessage(), e);
throw new MigrationException("Could not copy legacy backup to current version: " + e.getMessage(), e);
}
try {
Files.deleteIfExists(backupFile);
} catch (IOException e) {
logger.error("Cold not delete old legacy file after reverting File: " + e.getMessage(), e);
}
}
public MigrationResult analyzeDirectory(Path dir, String dirName) throws IOException {
Path legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(dirName);
......@@ -150,7 +222,7 @@ public class JoltMigrationAssistant extends MigrationAssistant {
logger.info(">> analyzing JSON tree of scenario <" + parentPath + node.get("name").asText() + ">");
Version version = Version.UNDEFINED;
Version version;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
......
package org.vadere.simulator.projects.migration;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.util.io.IOUtils;
import java.io.IOException;
import java.nio.file.Path;
......@@ -24,11 +25,14 @@ public abstract class MigrationAssistant {
}
}
public static Path getBackupPath(Path scenarioFile){
return IOUtils.addSuffix(scenarioFile, "." + IOUtils.LEGACY_DIR, false);
}
public abstract String getLog();
public abstract void restLog();
// public abstract void analyzeSingleScenario(Path path) throws IOException;
public abstract MigrationResult analyzeProject(String projectFolderPath) throws IOException;
......@@ -36,5 +40,9 @@ public abstract class MigrationAssistant {
return new SimpleDateFormat("yyyyMMddHHmmss").format(new java.util.Date());
}
public abstract String convertFile(Path scenarioFilePath, Version targetVersion) throws IOException, MigrationException;
public abstract String convertFile(Path scenarioFilePath, Version targetVersion) throws MigrationException;
public abstract void migrateFile(Path scenarioFilePath, Version targetVersion, Path outputFile) throws MigrationException;
public abstract void revertFile(Path scenarioFile) throws MigrationException;
}
package org.vadere.simulator.projects.migration.helper;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationException;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.simulator.projects.migration.jolttranformation.JoltTransformation;
import org.vadere.util.io.IOUtils;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.function.Function;
import java.util.stream.Collectors;
public class MigrationUtil {
private final static Logger logger = Logger.getLogger(MigrationUtil.class);
public static void main(String[] args) throws URISyntaxException, IOException {
MigrationUtil migrationUtil = new MigrationUtil();
// migrationUtil.migrateTestData(Paths.get("/home/lphex/hm.d/vadere/"));
migrationUtil.generateNewVersionTransform(Paths.get("/home/lphex/hm.d/vadere/VadereSimulator/resources"), "0.4");
}
private ArrayList<String> createList(String... addAll){
return Arrays.stream(addAll).collect(Collectors.toCollection(ArrayList::new));
}
private void generateNewVersionTransform(Path resourceDir, String newVersionLabel) throws URISyntaxException, IOException {
Path oldTransform = JoltTransformation.getTransforamtionFile(Version.latest());
Path oldIdentity = JoltTransformation.getIdenityFile(Version.latest());
String newTransformString = JoltTransformation
.getTransforamtionResourcePath(Version.latest().label('-'), newVersionLabel);
String newIdenityString = JoltTransformation
.getIdentiyResoucrePath(newVersionLabel);
Path newTransform = resourceDir.resolve(newTransformString.substring(1));
Path newIdenity = resourceDir.resolve(newIdenityString.substring(1));
String json = IOUtils.readTextFile(oldIdentity);
json = json.replace("\"release\": \"" + Version.latest().label('-') + "\",", "\"release\": \"" + newVersionLabel + "\",");
IOUtils.writeTextFile(newIdenity.toString(), json);
json = json.replace("\"release\": \"&\",", "// no relase here to overwrite it with default at the default operation down below");
IOUtils.writeTextFile(newTransform.toString(), json);
}
/**
*
* @param p root path of repo
*/
private void migrateTestData(Path p) {
ArrayList<String> ignoreDirs = createList("VadereModelTests", "target", "Documentation");
ArrayList<String> dirMarker = createList("DO_NOT_MIGRATE", ".DO_NOT_MIGRATE");
ArrayList<String> treeMarker = createList("DO_NOT_MIGRATE_TREE", ".DO_NOT_MIGRATE_TREE");
FileVisitor<Path> visitor = getVisitor(ignoreDirs, treeMarker, dirMarker, this::migrateToLatestVersion);
try {
Files.walkFileTree(p, visitor);
} catch (IOException e) {
e.printStackTrace();
}
}
private boolean migrateToLatestVersion(Path path){
MigrationAssistant ms = MigrationAssistant.getNewInstance(MigrationOptions.defaultOptions());
try {
ms.migrateFile(path, Version.latest(), null);
return true;
} catch (MigrationException e) {
logger.error("Error in MigrationUtil stop: " + e.getMessage());
e.printStackTrace();
return false;
}
}
private boolean revert(Path path){
MigrationAssistant ms = MigrationAssistant.getNewInstance(MigrationOptions.defaultOptions());
try{
ms.revertFile(path);
return true;
} catch (MigrationException e){
logger.error("Error in MigraionUtil: " + e.getMessage());
return false;
}
}
FileVisitor<Path> getVisitor(ArrayList<String> ignoreDirs, ArrayList<String> treeMarker, ArrayList<String> dirMarker, Function<Path, Boolean> func) {
return new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
boolean hasTreeMarker = treeMarker.stream().anyMatch( m -> dir.resolve(m).toFile().exists());
if (ignoreDirs.contains(dir.getFileName().toString())){ // skip ignored directories-trees.
return FileVisitResult.SKIP_SUBTREE;
} else if (hasTreeMarker) { // skip directory tree if treeMarker is present.
return FileVisitResult.SKIP_SUBTREE;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
// continue traversal if not scenario
if (! file.getFileName().toString().endsWith("scenario"))