Commit 5a38031c authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

add Jolt (JsonTransformation) based implementation of the MigrationAssistant

The MigrationAssistant know has tow implementation which can be used
interchangeably. The Selection of the the right implementation is
triggerd by a flag within the MigrationOptions Class.
parent 5da1ea8e
......@@ -11,6 +11,8 @@ import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.VadereProject;
import org.vadere.simulator.projects.io.IOVadere;
import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.simulator.projects.migration.MigrationResult;
import javax.swing.*;
import java.awt.event.ActionEvent;
......@@ -54,6 +56,7 @@ public class ActionLoadProject extends AbstractAction {
//TODO: [refactoring]: static call which has side-effect to the following call!
if (isRemigrationLoading) {
MigrationOptions migrationOptions;
Object option = JOptionPane.showInputDialog(null,
Messages.getString("ProjectView.chooseMigrationBaseDialog.text"),
Messages.getString("ProjectView.chooseMigrationBaseDialog.title"),
......@@ -61,17 +64,19 @@ public class ActionLoadProject extends AbstractAction {
options, options[options.length-1]);
if(option.equals(options[options.length-1])) {
MigrationAssistant.setReapplyLatestMigrationFlag();
migrationOptions = MigrationOptions.reapplyWithAutomaticVersionDiscorvery();
}
else {
Version version = (Version)option;
MigrationAssistant.setReapplyLatestMigrationFlag(version);
migrationOptions = MigrationOptions.reapplyFromVersion((Version)option);
}
// 3. load project
loadProjectByPath(model, projectFilePath, migrationOptions);
} else {
// 3. load project
loadProjectByPath(model, projectFilePath);
}
// 3. load project
loadProjectByPath(model, projectFilePath);
} else {
logger.info(String.format("user canceled load project."));
......@@ -97,9 +102,12 @@ public class ActionLoadProject extends AbstractAction {
ProjectView.getMainWindow().updateRecentProjectsMenu();
}
public static void loadProjectByPath(ProjectViewModel projectViewModel, String projectFilePath) {
public static void loadProjectByPath(ProjectViewModel projectViewModel, String projectFilePath){
loadProjectByPath(projectViewModel, projectFilePath, MigrationOptions.defaultOptions());
}
public static void loadProjectByPath(ProjectViewModel projectViewModel, String projectFilePath, MigrationOptions options) {
try {
VadereProject project = IOVadere.readProjectJson(projectFilePath);
VadereProject project = IOVadere.readProjectJson(projectFilePath, options);
projectViewModel.setCurrentProjectPath(projectFilePath);
projectViewModel.setProject(project);
......@@ -118,15 +126,15 @@ public class ActionLoadProject extends AbstractAction {
logger.info(String.format("project '%s' loaded.", projectViewModel.getProject().getName()));
// results from migration assistant if he was active
int[] stats = project.getMigrationStats();
MigrationResult stats = project.getMigrationStats();
if (stats[1] > 0 || stats[2] > 0) { // scenarios: [0] total, [1] legacy'ed, [2] unmigratable
if (stats.total > 0 || stats.notmigratable > 0) { // scenarios: [0] total, [1] legacy'ed, [2] unmigratable
SwingWorker<Void, Void> worker = new SwingWorker<Void, Void>() {
@Override
public Void doInBackground() {
int total = stats[0];
int migrated = stats[1];
int nonmigratable = stats[2];
int total = stats.total;
int migrated = stats.legacy;
int nonmigratable = stats.notmigratable;
int untouched = total - migrated - nonmigratable;
// TODO pull this text from the language files
......@@ -148,7 +156,7 @@ public class ActionLoadProject extends AbstractAction {
JOptionPane.showMessageDialog(
ProjectView.getMainWindow(),
message, "Migration assistant",
message, "JoltMigrationAssistant assistant",
JOptionPane.INFORMATION_MESSAGE);
return null;
}
......@@ -157,7 +165,7 @@ public class ActionLoadProject extends AbstractAction {
}
} catch (Exception e) {
JOptionPane.showMessageDialog(null, e.getMessage(), "Migration assistant",
JOptionPane.showMessageDialog(null, e.getMessage(), "JoltMigrationAssistant assistant",
JOptionPane.ERROR_MESSAGE);
logger.error("could not load project: " + e.getMessage());
e.printStackTrace();
......
......@@ -3,6 +3,7 @@ package org.vadere.simulator.projects;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.vadere.simulator.control.PassiveCallback;
import org.vadere.simulator.projects.migration.MigrationResult;
import java.nio.file.Path;
import java.nio.file.Paths;
......@@ -19,7 +20,6 @@ import java.util.stream.Collectors;
/**
* A VadereProject holds a list of {@link Scenario}s and functionality to manage them.
*
*/
public class VadereProject {
......@@ -37,8 +37,7 @@ public class VadereProject {
private Path outputDirectory;
private ProjectOutput projectOutput; //TODO initialize and wire up with rest ....
// TODO should be encapsulated in a class (we are not programming in C):
private int[] migrationStats; // scenarios: [0] total, [1] legacy'ed, [2] nonmigratable
private MigrationResult migrationStats;
public VadereProject(final String name, final Iterable<Scenario> scenarios) {
this.name = name;
......@@ -98,7 +97,7 @@ public class VadereProject {
currentScenarioRun.simulationFailed(ex);
notifySimulationListenersSimulationError(currentScenarioRun.getScenario(), ex);
});
notifySimulationListenersSimulationStarted(getCurrentScenario());
currentScenarioThread.start();
}
......@@ -222,8 +221,8 @@ public class VadereProject {
public int getScenarioIndexByName(final Scenario srm) {
int index = -1;
int currentIndex = 0;
for(Scenario csrm : getScenarios()) {
if(csrm.getName().equals(srm.getName())) {
for (Scenario csrm : getScenarios()) {
if (csrm.getName().equals(srm.getName())) {
return currentIndex;
} else {
currentIndex++;
......@@ -237,7 +236,7 @@ public class VadereProject {
}
public Scenario getScenario(int index) {
return getScenarios().toArray(new Scenario[] {})[index];
return getScenarios().toArray(new Scenario[]{})[index];
}
public void removeScenario(final Scenario scenario) {
......@@ -256,15 +255,17 @@ public class VadereProject {
return currentScenarioRun.getScenario();
}
public void setMigrationStats(int[] migrationStats) {
public void setMigrationStats(MigrationResult migrationStats) {
this.migrationStats = migrationStats;
}
public int[] getMigrationStats() {
public MigrationResult getMigrationStats() {
return migrationStats;
}
/** Starts the next simulation if any. */
/**
* Starts the next simulation if any.
*/
private RunnableFinishedListener scenarioFinishedListener = new RunnableFinishedListener() {
@Override
public void finished(Runnable runnable) {
......
......@@ -6,12 +6,15 @@ import org.vadere.simulator.projects.ProjectOutput;
import org.vadere.simulator.projects.Scenario;
import org.vadere.simulator.projects.VadereProject;
import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.simulator.projects.migration.MigrationResult;
import org.vadere.simulator.projects.migration.incidents.ExceptionIncident;
import org.vadere.util.io.IOUtils;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
......@@ -30,7 +33,12 @@ public class IOVadere {
return JsonConverter.deserializeScenarioRunManager(json);
}
public static VadereProject readProjectJson(final String filepath)
public static VadereProject readProjectJson(final String filepath) throws ParserConfigurationException, SAXException,
IOException, TransformerException {
return readProjectJson(filepath, MigrationOptions.defaultOptions());
}
public static VadereProject readProjectJson(final String filepath, final MigrationOptions options)
throws ParserConfigurationException, SAXException,
IOException, TransformerException {
......@@ -38,21 +46,26 @@ public class IOVadere {
if (!Files.isDirectory(p))
p = p.getParent();
return IOVadere.readProject(p.toString());
return IOVadere.readProject(p.toString(), options);
}
public static VadereProject readProject(final String folderpath) throws IOException {
return readProject(folderpath, MigrationOptions.defaultOptions());
}
public static VadereProject readProject(final String folderpath, final MigrationOptions options) throws IOException {
String name = IOUtils.readTextFile(Paths.get(folderpath, IOUtils.VADERE_PROJECT_FILENAME).toString());
logger.info("read .project file");
List<Scenario> scenarios = new ArrayList<>();
Set<String> scenarioNames = new HashSet<>();
Path p = Paths.get(folderpath, IOUtils.SCENARIO_DIR);
int[] migrationStats = {0, 0, 0};
MigrationResult migrationStats = new MigrationResult();
if (Files.isDirectory(p)) {
migrationStats = MigrationAssistant.analyzeProject(folderpath);
logger.info("analysed .scenario files");
MigrationAssistant migrationAssistant = MigrationAssistant.getNewInstance(options);
migrationStats = migrationAssistant.analyzeProject(folderpath);
logger.info("analysed .scenario files");
for (File file : IOUtils.getFilesInScenarioDirectory(p)) {
try {
Scenario scenario =
......@@ -62,8 +75,7 @@ public class IOVadere {
throw new IOException("Found two scenarios with the same name.");
}
scenarios.add(scenario);
}
catch (Exception e) {
} catch (Exception e) {
logger.error("could not read " + file.getName());
throw e;
}
......@@ -71,11 +83,12 @@ public class IOVadere {
}
VadereProject project = new VadereProject(name, scenarios);
logger.info(migrationStats.toString());
project.setMigrationStats(migrationStats); // TODO [priority=low] [task=refactoring] better way to tunnel those results to the GUI?
project.setOutputDir(Paths.get(folderpath, IOUtils.OUTPUT_DIR));
ProjectOutput projectOutput = new ProjectOutput(project);
project.setProjectOutput(projectOutput);
logger.info("project loaded: " + project.getName());
logger.info("project loaded: " + project.getName());
return project;
}
......
package org.vadere.simulator.projects.migration;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.migration.incidents.ExceptionIncident;
import org.vadere.simulator.projects.migration.incidents.Incident;
import org.vadere.simulator.projects.migration.incidents.VersionBumpIncident;
import org.vadere.state.util.StateJsonConverter;
import org.vadere.util.io.IOUtils;
public class IncidentMigrationAssistant extends MigrationAssistant {
private static Logger logger = LogManager.getLogger(IncidentMigrationAssistant.class);
StringBuilder log;
public IncidentMigrationAssistant() {
super(MigrationOptions.defaultOptions());
StringBuilder log = new StringBuilder();
}
public IncidentMigrationAssistant(final MigrationOptions migrationOptions) {
super(migrationOptions);
log = new StringBuilder();
}
@Override
public String getLog() {
return log.toString();
}
// @Override
// public void analyzeSingleScenario(Path path) throws IOException {
// try {
// analyzeScenario(path, null, log, true) ;
// } catch (MigrationException e) {
// logger.error(log.toString());
// }
// }
@Override
public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
MigrationResult stats = new MigrationResult();
Path scenarioDir = Paths.get(projectFolderPath, IOUtils.SCENARIO_DIR);
if (Files.exists(scenarioDir)) {
stats = analyzeDirectory(scenarioDir, true);
}
Path outputDir = Paths.get(projectFolderPath, IOUtils.OUTPUT_DIR);
if (Files.exists(outputDir)) {
MigrationResult outputDirStats = analyzeDirectory(outputDir, false);
stats.add(outputDirStats);
}
return stats;
}
// if isScenario is false, its output
private MigrationResult analyzeDirectory(Path dir, boolean isScenario) throws IOException {
Path legacyDir = null;
if (isScenario) {
legacyDir = dir.getParent().resolve(IOUtils.LEGACY_DIR).resolve("scenarios");
}
File[] scenarioFiles = isScenario ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
MigrationResult stats = new MigrationResult(scenarioFiles.length); // scenarios: [0] total, [1] legacy'ed, [2] nonmigratable
int legacyedCount = 0;
int nonmigratableCount = 0;
for (File file : scenarioFiles) {
if (!isScenario) {
String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies
legacyDir = dir.getParent().resolve(IOUtils.LEGACY_DIR).resolve("output").resolve(fileFolder);
}
Path scenarioFilePath = Paths.get(file.getAbsolutePath());
try {
if (analyzeScenario(scenarioFilePath, legacyDir, isScenario)) {
stats.legacy++;
} else {
stats.upToDate++;
}
} catch (MigrationException e) {
moveFileAddExtension(scenarioFilePath, legacyDir, migrationOptions.getNonmigratabelExtension(), !isScenario);
log.append(
"! --> Can't migrate the scenario to latest version, removed it from the directory ("
+ e.getMessage() + ")\n" +
"If you can fix this problem manually, do so and then remove ."
+ migrationOptions.getNonmigratabelExtension() + " from the file in the " + IOUtils.LEGACY_DIR + "-directory " +
"and move it back into the scenarios-directory, it will be checked again when the GUI restarts.\n");
stats.notmigratable++;
}
}
if (!isScenario) {
legacyDir = dir.getParent().resolve(IOUtils.LEGACY_DIR).resolve("output");
}
if (stats.legacy + stats.notmigratable > 0)
IOUtils.writeTextFile(legacyDir.resolve("_LOG-" + getTimestamp() + ".txt").toString(), log.toString());
return stats;
}
private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, boolean isScenario) throws IOException, MigrationException {
String json = IOUtils.readTextFile(scenarioFilePath);
JsonNode node = StateJsonConverter.deserializeToNode(json);
Tree tree = new Tree(node);
String outputScenarioParentFolderName = isScenario ? "" : scenarioFilePath.getParent().getFileName().toString() + " _ ";
log.append("\n>> analyzing JSON tree of scenario <" + outputScenarioParentFolderName + node.get("name").asText() + ">\n");
Version version = Version.UNDEFINED;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
if (version == null) {
throw new MigrationException("release version " + node.get("release").asText() + " is unknown. If this " +
"is a valid release, update the version-list in MigrationAssistant accordingly");
}
// if enforced migration should be done from baseVersion to latestVersion
if (migrationOptions.isReapplyLatestMigrationFlag() && migrationOptions.getBaseVersion() != null) {
version = migrationOptions.getBaseVersion();
} else if (migrationOptions.isReapplyLatestMigrationFlag()) { // if enforced migration should be done from prev version to latest
Optional<Version> optVersion = Version.getPrevious(version);
if (optVersion.isPresent()) {
version = optVersion.get();
} else {
return false;
}
} // if no enforced migration should be done and we are at the latest version, no migration is required.
else if (version == Version.latest()) {
return false;
}
}
// 1. collect possible incidents
List<Incident> possibleIncidents = new ArrayList<>();
for (int versionIndex = version.ordinal(); versionIndex < Version.latest().ordinal(); versionIndex++) {
Version ver = Version.values()[versionIndex];
log.append(" > checking possible incidents from version \"")
.append(ver.label()).append("\" to version \"")
.append(Version.values()[versionIndex + 1].label()).append("\"\n");
possibleIncidents.addAll(IncidentDatabase.getInstance().getPossibleIncidentsFor(ver));
}
possibleIncidents.add(new ExceptionIncident(node));
possibleIncidents.add(new VersionBumpIncident(node, version));
// 2. filter those out that don't apply
List<Incident> applicableIncidents = possibleIncidents.stream()
.filter(incident -> incident.applies(tree))
.collect(Collectors.toList());
// 3. resolve the applicable incidents (step 2 and 3 are intentionally separated to uncover
// potentially dangerous flaws in the order of the incidents in the IncidentDatabase)
for (Incident incident : applicableIncidents)
incident.resolve(tree, log);
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, migrationOptions.getLegacyExtension(), false);
}
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(node));
return true;
}
private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder) throws IOException {
Path source = scenarioFilePath;
Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension);
if (moveOutputFolder) {
source = source.getParent();
target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension);
}
IOUtils.createDirectoryIfNotExisting(target);
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten?
}
private String getTimestamp() {
return new SimpleDateFormat("yyyyMMddHHmmss").format(new java.util.Date());
}
}
......@@ -26,50 +26,41 @@ import static org.vadere.util.io.IOUtils.OUTPUT_DIR;
import static org.vadere.util.io.IOUtils.SCENARIO_DIR;
public class Migration {
public class JoltMigrationAssistant extends MigrationAssistant {
private final static Logger logger = Logger.getLogger(Migration.class);
private final static Logger logger = Logger.getLogger(JoltMigrationAssistant.class);
private final static LogBufferAppender appender;
private static HashMap<Version, Chainr> identityTransformation = new LinkedHashMap<>();
// Version is the target version. Only Incremental Transformation supported 1->2->3 (not 1->3)
private static HashMap<Version, Chainr> transformations = new LinkedHashMap<>();
private static final String LEGACY_EXTENSION = "legacy";
private static final String NONMIGRATABLE_EXTENSION = "nonmigratable";
static {
identityTransformation.put(Version.V0_1, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v1.json")));
identityTransformation.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v2.json")));
transformations.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/transform_v1_to_v2.json")));
appender = new LogBufferAppender();
}
private Version baseVersion = null;
private boolean reapplyLatestMigrationFlag = false;
private Diffy transformationDiff;
public Migration() {
public JoltMigrationAssistant(MigrationOptions options) {
super(options);
this.transformationDiff = new Diffy();
logger.addAppender(appender);
}
public String getLog() {
return appender.getMigrationLog();
public JoltMigrationAssistant() {
this(MigrationOptions.defaultOptions());
}
public void setReapplyLatestMigrationFlag() {
reapplyLatestMigrationFlag = true;
baseVersion = null;
@Override
public String getLog() {
return appender.getMigrationLog();
}
public void setReapplyLatestMigrationFlag(final Version version) {
reapplyLatestMigrationFlag = true;
baseVersion = version;
}
@Override
public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
MigrationResult stats = new MigrationResult();
......@@ -83,9 +74,6 @@ public class Migration {
MigrationResult outputDirStats = analyzeDirectory(outputDir, OUTPUT_DIR);
stats.add(outputDirStats);
}
baseVersion = null;
return stats;
}
......@@ -112,10 +100,10 @@ public class Migration {
stats.upToDate++;
}
} catch (MigrationException e) {
moveFileAddExtension(scenarioFilePath, legacyDir, NONMIGRATABLE_EXTENSION, !dirName.equals(SCENARIO_DIR));
moveFileAddExtension(scenarioFilePath, legacyDir, migrationOptions.getNonmigratabelExtension(), !dirName.equals(SCENARIO_DIR));
logger.error("!> Can't migrate the scenario to latest version, removed it from the directory (" +
e.getMessage() + ") If you can fix this problem manually, do so and then remove ." +
NONMIGRATABLE_EXTENSION + " from the file in the " + LEGACY_DIR + "-directory "
migrationOptions.getNonmigratabelExtension() + " from the file in the " + LEGACY_DIR + "-directory "
+ "and move it back into the scenarios-directory, it will be checked again when the GUI restarts.");
stats.notmigratable++;
}
......@@ -179,10 +167,10 @@ public class Migration {
}
// if enforced migration should be done from baseVersion to latestVersion
if (reapplyLatestMigrationFlag && baseVersion != null) {
version = baseVersion;
if (migrationOptions.isReapplyLatestMigrationFlag() && migrationOptions.getBaseVersion() != null) {
version = migrationOptions.getBaseVersion();
} else if (reapplyLatestMigrationFlag) { // if enforced migration should be done from prev version to latest
} else if (migrationOptions.isReapplyLatestMigrationFlag()) { // if enforced migration should be done from prev version to latest
Optional<Version> optVersion = Version.getPrevious(version);
if (optVersion.isPresent()) {
version = optVersion.get();
......@@ -204,7 +192,8 @@ public class Migration {
transformedNode = transform(transformedNode, v);
}
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, LEGACY_EXTENSION, false);
logger.info("Scenario Migrated. Move olde version to legacyDir");
moveFileAddExtension(scenarioFilePath, legacyDir, migrationOptions.getLegacyExtension(), false);
}
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(transformedNode));
return true;
......
......@@ -3,13 +3,22 @@ package org.vadere.simulator.projects.migration;
import com.bazaarvoice.jolt.Chainr;
import com.bazaarvoice.jolt.Diffy;
import com.bazaarvoice.jolt.JsonUtils;