Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 5a38031c authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

add Jolt (JsonTransformation) based implementation of the MigrationAssistant

The MigrationAssistant know has tow implementation which can be used
interchangeably. The Selection of the the right implementation is
triggerd by a flag within the MigrationOptions Class.
parent 5da1ea8e
...@@ -11,6 +11,8 @@ import org.vadere.simulator.entrypoints.Version; ...@@ -11,6 +11,8 @@ import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.VadereProject; import org.vadere.simulator.projects.VadereProject;
import org.vadere.simulator.projects.io.IOVadere; import org.vadere.simulator.projects.io.IOVadere;
import org.vadere.simulator.projects.migration.MigrationAssistant; import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.simulator.projects.migration.MigrationResult;
import javax.swing.*; import javax.swing.*;
import java.awt.event.ActionEvent; import java.awt.event.ActionEvent;
...@@ -54,6 +56,7 @@ public class ActionLoadProject extends AbstractAction { ...@@ -54,6 +56,7 @@ public class ActionLoadProject extends AbstractAction {
//TODO: [refactoring]: static call which has side-effect to the following call! //TODO: [refactoring]: static call which has side-effect to the following call!
if (isRemigrationLoading) { if (isRemigrationLoading) {
MigrationOptions migrationOptions;
Object option = JOptionPane.showInputDialog(null, Object option = JOptionPane.showInputDialog(null,
Messages.getString("ProjectView.chooseMigrationBaseDialog.text"), Messages.getString("ProjectView.chooseMigrationBaseDialog.text"),
Messages.getString("ProjectView.chooseMigrationBaseDialog.title"), Messages.getString("ProjectView.chooseMigrationBaseDialog.title"),
...@@ -61,17 +64,19 @@ public class ActionLoadProject extends AbstractAction { ...@@ -61,17 +64,19 @@ public class ActionLoadProject extends AbstractAction {
options, options[options.length-1]); options, options[options.length-1]);
if(option.equals(options[options.length-1])) { if(option.equals(options[options.length-1])) {
MigrationAssistant.setReapplyLatestMigrationFlag(); migrationOptions = MigrationOptions.reapplyWithAutomaticVersionDiscorvery();
} }
else { else {
Version version = (Version)option; migrationOptions = MigrationOptions.reapplyFromVersion((Version)option);
MigrationAssistant.setReapplyLatestMigrationFlag(version);
} }
// 3. load project
loadProjectByPath(model, projectFilePath, migrationOptions);
} else {
// 3. load project
loadProjectByPath(model, projectFilePath);
} }
// 3. load project
loadProjectByPath(model, projectFilePath);
} else { } else {
logger.info(String.format("user canceled load project.")); logger.info(String.format("user canceled load project."));
...@@ -97,9 +102,12 @@ public class ActionLoadProject extends AbstractAction { ...@@ -97,9 +102,12 @@ public class ActionLoadProject extends AbstractAction {
ProjectView.getMainWindow().updateRecentProjectsMenu(); ProjectView.getMainWindow().updateRecentProjectsMenu();
} }
public static void loadProjectByPath(ProjectViewModel projectViewModel, String projectFilePath) { public static void loadProjectByPath(ProjectViewModel projectViewModel, String projectFilePath){
loadProjectByPath(projectViewModel, projectFilePath, MigrationOptions.defaultOptions());
}
public static void loadProjectByPath(ProjectViewModel projectViewModel, String projectFilePath, MigrationOptions options) {
try { try {
VadereProject project = IOVadere.readProjectJson(projectFilePath); VadereProject project = IOVadere.readProjectJson(projectFilePath, options);
projectViewModel.setCurrentProjectPath(projectFilePath); projectViewModel.setCurrentProjectPath(projectFilePath);
projectViewModel.setProject(project); projectViewModel.setProject(project);
...@@ -118,15 +126,15 @@ public class ActionLoadProject extends AbstractAction { ...@@ -118,15 +126,15 @@ public class ActionLoadProject extends AbstractAction {
logger.info(String.format("project '%s' loaded.", projectViewModel.getProject().getName())); logger.info(String.format("project '%s' loaded.", projectViewModel.getProject().getName()));
// results from migration assistant if he was active // results from migration assistant if he was active
int[] stats = project.getMigrationStats(); MigrationResult stats = project.getMigrationStats();
if (stats[1] > 0 || stats[2] > 0) { // scenarios: [0] total, [1] legacy'ed, [2] unmigratable if (stats.total > 0 || stats.notmigratable > 0) { // scenarios: [0] total, [1] legacy'ed, [2] unmigratable
SwingWorker<Void, Void> worker = new SwingWorker<Void, Void>() { SwingWorker<Void, Void> worker = new SwingWorker<Void, Void>() {
@Override @Override
public Void doInBackground() { public Void doInBackground() {
int total = stats[0]; int total = stats.total;
int migrated = stats[1]; int migrated = stats.legacy;
int nonmigratable = stats[2]; int nonmigratable = stats.notmigratable;
int untouched = total - migrated - nonmigratable; int untouched = total - migrated - nonmigratable;
// TODO pull this text from the language files // TODO pull this text from the language files
...@@ -148,7 +156,7 @@ public class ActionLoadProject extends AbstractAction { ...@@ -148,7 +156,7 @@ public class ActionLoadProject extends AbstractAction {
JOptionPane.showMessageDialog( JOptionPane.showMessageDialog(
ProjectView.getMainWindow(), ProjectView.getMainWindow(),
message, "Migration assistant", message, "JoltMigrationAssistant assistant",
JOptionPane.INFORMATION_MESSAGE); JOptionPane.INFORMATION_MESSAGE);
return null; return null;
} }
...@@ -157,7 +165,7 @@ public class ActionLoadProject extends AbstractAction { ...@@ -157,7 +165,7 @@ public class ActionLoadProject extends AbstractAction {
} }
} catch (Exception e) { } catch (Exception e) {
JOptionPane.showMessageDialog(null, e.getMessage(), "Migration assistant", JOptionPane.showMessageDialog(null, e.getMessage(), "JoltMigrationAssistant assistant",
JOptionPane.ERROR_MESSAGE); JOptionPane.ERROR_MESSAGE);
logger.error("could not load project: " + e.getMessage()); logger.error("could not load project: " + e.getMessage());
e.printStackTrace(); e.printStackTrace();
......
...@@ -3,6 +3,7 @@ package org.vadere.simulator.projects; ...@@ -3,6 +3,7 @@ package org.vadere.simulator.projects;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.vadere.simulator.control.PassiveCallback; import org.vadere.simulator.control.PassiveCallback;
import org.vadere.simulator.projects.migration.MigrationResult;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
...@@ -19,7 +20,6 @@ import java.util.stream.Collectors; ...@@ -19,7 +20,6 @@ import java.util.stream.Collectors;
/** /**
* A VadereProject holds a list of {@link Scenario}s and functionality to manage them. * A VadereProject holds a list of {@link Scenario}s and functionality to manage them.
*
*/ */
public class VadereProject { public class VadereProject {
...@@ -37,8 +37,7 @@ public class VadereProject { ...@@ -37,8 +37,7 @@ public class VadereProject {
private Path outputDirectory; private Path outputDirectory;
private ProjectOutput projectOutput; //TODO initialize and wire up with rest .... private ProjectOutput projectOutput; //TODO initialize and wire up with rest ....
// TODO should be encapsulated in a class (we are not programming in C): private MigrationResult migrationStats;
private int[] migrationStats; // scenarios: [0] total, [1] legacy'ed, [2] nonmigratable
public VadereProject(final String name, final Iterable<Scenario> scenarios) { public VadereProject(final String name, final Iterable<Scenario> scenarios) {
this.name = name; this.name = name;
...@@ -98,7 +97,7 @@ public class VadereProject { ...@@ -98,7 +97,7 @@ public class VadereProject {
currentScenarioRun.simulationFailed(ex); currentScenarioRun.simulationFailed(ex);
notifySimulationListenersSimulationError(currentScenarioRun.getScenario(), ex); notifySimulationListenersSimulationError(currentScenarioRun.getScenario(), ex);
}); });
notifySimulationListenersSimulationStarted(getCurrentScenario()); notifySimulationListenersSimulationStarted(getCurrentScenario());
currentScenarioThread.start(); currentScenarioThread.start();
} }
...@@ -222,8 +221,8 @@ public class VadereProject { ...@@ -222,8 +221,8 @@ public class VadereProject {
public int getScenarioIndexByName(final Scenario srm) { public int getScenarioIndexByName(final Scenario srm) {
int index = -1; int index = -1;
int currentIndex = 0; int currentIndex = 0;
for(Scenario csrm : getScenarios()) { for (Scenario csrm : getScenarios()) {
if(csrm.getName().equals(srm.getName())) { if (csrm.getName().equals(srm.getName())) {
return currentIndex; return currentIndex;
} else { } else {
currentIndex++; currentIndex++;
...@@ -237,7 +236,7 @@ public class VadereProject { ...@@ -237,7 +236,7 @@ public class VadereProject {
} }
public Scenario getScenario(int index) { public Scenario getScenario(int index) {
return getScenarios().toArray(new Scenario[] {})[index]; return getScenarios().toArray(new Scenario[]{})[index];
} }
public void removeScenario(final Scenario scenario) { public void removeScenario(final Scenario scenario) {
...@@ -256,15 +255,17 @@ public class VadereProject { ...@@ -256,15 +255,17 @@ public class VadereProject {
return currentScenarioRun.getScenario(); return currentScenarioRun.getScenario();
} }
public void setMigrationStats(int[] migrationStats) { public void setMigrationStats(MigrationResult migrationStats) {
this.migrationStats = migrationStats; this.migrationStats = migrationStats;
} }
public int[] getMigrationStats() { public MigrationResult getMigrationStats() {
return migrationStats; return migrationStats;
} }
/** Starts the next simulation if any. */ /**
* Starts the next simulation if any.
*/
private RunnableFinishedListener scenarioFinishedListener = new RunnableFinishedListener() { private RunnableFinishedListener scenarioFinishedListener = new RunnableFinishedListener() {
@Override @Override
public void finished(Runnable runnable) { public void finished(Runnable runnable) {
......
...@@ -6,12 +6,15 @@ import org.vadere.simulator.projects.ProjectOutput; ...@@ -6,12 +6,15 @@ import org.vadere.simulator.projects.ProjectOutput;
import org.vadere.simulator.projects.Scenario; import org.vadere.simulator.projects.Scenario;
import org.vadere.simulator.projects.VadereProject; import org.vadere.simulator.projects.VadereProject;
import org.vadere.simulator.projects.migration.MigrationAssistant; import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.simulator.projects.migration.MigrationResult;
import org.vadere.simulator.projects.migration.incidents.ExceptionIncident; import org.vadere.simulator.projects.migration.incidents.ExceptionIncident;
import org.vadere.util.io.IOUtils; import org.vadere.util.io.IOUtils;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerException;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
...@@ -30,7 +33,12 @@ public class IOVadere { ...@@ -30,7 +33,12 @@ public class IOVadere {
return JsonConverter.deserializeScenarioRunManager(json); return JsonConverter.deserializeScenarioRunManager(json);
} }
public static VadereProject readProjectJson(final String filepath) public static VadereProject readProjectJson(final String filepath) throws ParserConfigurationException, SAXException,
IOException, TransformerException {
return readProjectJson(filepath, MigrationOptions.defaultOptions());
}
public static VadereProject readProjectJson(final String filepath, final MigrationOptions options)
throws ParserConfigurationException, SAXException, throws ParserConfigurationException, SAXException,
IOException, TransformerException { IOException, TransformerException {
...@@ -38,21 +46,26 @@ public class IOVadere { ...@@ -38,21 +46,26 @@ public class IOVadere {
if (!Files.isDirectory(p)) if (!Files.isDirectory(p))
p = p.getParent(); p = p.getParent();
return IOVadere.readProject(p.toString()); return IOVadere.readProject(p.toString(), options);
} }
public static VadereProject readProject(final String folderpath) throws IOException { public static VadereProject readProject(final String folderpath) throws IOException {
return readProject(folderpath, MigrationOptions.defaultOptions());
}
public static VadereProject readProject(final String folderpath, final MigrationOptions options) throws IOException {
String name = IOUtils.readTextFile(Paths.get(folderpath, IOUtils.VADERE_PROJECT_FILENAME).toString()); String name = IOUtils.readTextFile(Paths.get(folderpath, IOUtils.VADERE_PROJECT_FILENAME).toString());
logger.info("read .project file"); logger.info("read .project file");
List<Scenario> scenarios = new ArrayList<>(); List<Scenario> scenarios = new ArrayList<>();
Set<String> scenarioNames = new HashSet<>(); Set<String> scenarioNames = new HashSet<>();
Path p = Paths.get(folderpath, IOUtils.SCENARIO_DIR); Path p = Paths.get(folderpath, IOUtils.SCENARIO_DIR);
int[] migrationStats = {0, 0, 0}; MigrationResult migrationStats = new MigrationResult();
if (Files.isDirectory(p)) { if (Files.isDirectory(p)) {
migrationStats = MigrationAssistant.analyzeProject(folderpath); MigrationAssistant migrationAssistant = MigrationAssistant.getNewInstance(options);
logger.info("analysed .scenario files"); migrationStats = migrationAssistant.analyzeProject(folderpath);
logger.info("analysed .scenario files");
for (File file : IOUtils.getFilesInScenarioDirectory(p)) { for (File file : IOUtils.getFilesInScenarioDirectory(p)) {
try { try {
Scenario scenario = Scenario scenario =
...@@ -62,8 +75,7 @@ public class IOVadere { ...@@ -62,8 +75,7 @@ public class IOVadere {
throw new IOException("Found two scenarios with the same name."); throw new IOException("Found two scenarios with the same name.");
} }
scenarios.add(scenario); scenarios.add(scenario);
} } catch (Exception e) {
catch (Exception e) {
logger.error("could not read " + file.getName()); logger.error("could not read " + file.getName());
throw e; throw e;
} }
...@@ -71,11 +83,12 @@ public class IOVadere { ...@@ -71,11 +83,12 @@ public class IOVadere {
} }
VadereProject project = new VadereProject(name, scenarios); VadereProject project = new VadereProject(name, scenarios);
logger.info(migrationStats.toString());
project.setMigrationStats(migrationStats); // TODO [priority=low] [task=refactoring] better way to tunnel those results to the GUI? project.setMigrationStats(migrationStats); // TODO [priority=low] [task=refactoring] better way to tunnel those results to the GUI?
project.setOutputDir(Paths.get(folderpath, IOUtils.OUTPUT_DIR)); project.setOutputDir(Paths.get(folderpath, IOUtils.OUTPUT_DIR));
ProjectOutput projectOutput = new ProjectOutput(project); ProjectOutput projectOutput = new ProjectOutput(project);
project.setProjectOutput(projectOutput); project.setProjectOutput(projectOutput);
logger.info("project loaded: " + project.getName()); logger.info("project loaded: " + project.getName());
return project; return project;
} }
......
package org.vadere.simulator.projects.migration;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.migration.incidents.ExceptionIncident;
import org.vadere.simulator.projects.migration.incidents.Incident;
import org.vadere.simulator.projects.migration.incidents.VersionBumpIncident;
import org.vadere.state.util.StateJsonConverter;
import org.vadere.util.io.IOUtils;
public class IncidentMigrationAssistant extends MigrationAssistant {
private static Logger logger = LogManager.getLogger(IncidentMigrationAssistant.class);
StringBuilder log;
public IncidentMigrationAssistant() {
super(MigrationOptions.defaultOptions());
StringBuilder log = new StringBuilder();
}
public IncidentMigrationAssistant(final MigrationOptions migrationOptions) {
super(migrationOptions);
log = new StringBuilder();
}
@Override
public String getLog() {
return log.toString();
}
// @Override
// public void analyzeSingleScenario(Path path) throws IOException {
// try {
// analyzeScenario(path, null, log, true) ;
// } catch (MigrationException e) {
// logger.error(log.toString());
// }
// }
@Override
public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
MigrationResult stats = new MigrationResult();
Path scenarioDir = Paths.get(projectFolderPath, IOUtils.SCENARIO_DIR);
if (Files.exists(scenarioDir)) {
stats = analyzeDirectory(scenarioDir, true);
}
Path outputDir = Paths.get(projectFolderPath, IOUtils.OUTPUT_DIR);
if (Files.exists(outputDir)) {
MigrationResult outputDirStats = analyzeDirectory(outputDir, false);
stats.add(outputDirStats);
}
return stats;
}
// if isScenario is false, its output
private MigrationResult analyzeDirectory(Path dir, boolean isScenario) throws IOException {
Path legacyDir = null;
if (isScenario) {
legacyDir = dir.getParent().resolve(IOUtils.LEGACY_DIR).resolve("scenarios");
}
File[] scenarioFiles = isScenario ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
MigrationResult stats = new MigrationResult(scenarioFiles.length); // scenarios: [0] total, [1] legacy'ed, [2] nonmigratable
int legacyedCount = 0;
int nonmigratableCount = 0;
for (File file : scenarioFiles) {
if (!isScenario) {
String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies
legacyDir = dir.getParent().resolve(IOUtils.LEGACY_DIR).resolve("output").resolve(fileFolder);
}
Path scenarioFilePath = Paths.get(file.getAbsolutePath());
try {
if (analyzeScenario(scenarioFilePath, legacyDir, isScenario)) {
stats.legacy++;
} else {
stats.upToDate++;
}
} catch (MigrationException e) {
moveFileAddExtension(scenarioFilePath, legacyDir, migrationOptions.getNonmigratabelExtension(), !isScenario);
log.append(
"! --> Can't migrate the scenario to latest version, removed it from the directory ("
+ e.getMessage() + ")\n" +
"If you can fix this problem manually, do so and then remove ."
+ migrationOptions.getNonmigratabelExtension() + " from the file in the " + IOUtils.LEGACY_DIR + "-directory " +
"and move it back into the scenarios-directory, it will be checked again when the GUI restarts.\n");
stats.notmigratable++;
}
}
if (!isScenario) {
legacyDir = dir.getParent().resolve(IOUtils.LEGACY_DIR).resolve("output");
}
if (stats.legacy + stats.notmigratable > 0)
IOUtils.writeTextFile(legacyDir.resolve("_LOG-" + getTimestamp() + ".txt").toString(), log.toString());
return stats;
}
private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, boolean isScenario) throws IOException, MigrationException {
String json = IOUtils.readTextFile(scenarioFilePath);
JsonNode node = StateJsonConverter.deserializeToNode(json);
Tree tree = new Tree(node);
String outputScenarioParentFolderName = isScenario ? "" : scenarioFilePath.getParent().getFileName().toString() + " _ ";
log.append("\n>> analyzing JSON tree of scenario <" + outputScenarioParentFolderName + node.get("name").asText() + ">\n");
Version version = Version.UNDEFINED;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
if (version == null) {
throw new MigrationException("release version " + node.get("release").asText() + " is unknown. If this " +
"is a valid release, update the version-list in MigrationAssistant accordingly");
}
// if enforced migration should be done from baseVersion to latestVersion
if (migrationOptions.isReapplyLatestMigrationFlag() && migrationOptions.getBaseVersion() != null) {
version = migrationOptions.getBaseVersion();
} else if (migrationOptions.isReapplyLatestMigrationFlag()) { // if enforced migration should be done from prev version to latest
Optional<Version> optVersion = Version.getPrevious(version);
if (optVersion.isPresent()) {
version = optVersion.get();
} else {
return false;
}
} // if no enforced migration should be done and we are at the latest version, no migration is required.
else if (version == Version.latest()) {
return false;
}
}
// 1. collect possible incidents
List<Incident> possibleIncidents = new ArrayList<>();
for (int versionIndex = version.ordinal(); versionIndex < Version.latest().ordinal(); versionIndex++) {
Version ver = Version.values()[versionIndex];
log.append(" > checking possible incidents from version \"")
.append(ver.label()).append("\" to version \"")
.append(Version.values()[versionIndex + 1].label()).append("\"\n");
possibleIncidents.addAll(IncidentDatabase.getInstance().getPossibleIncidentsFor(ver));
}
possibleIncidents.add(new ExceptionIncident(node));
possibleIncidents.add(new VersionBumpIncident(node, version));
// 2. filter those out that don't apply
List<Incident> applicableIncidents = possibleIncidents.stream()
.filter(incident -> incident.applies(tree))
.collect(Collectors.toList());
// 3. resolve the applicable incidents (step 2 and 3 are intentionally separated to uncover
// potentially dangerous flaws in the order of the incidents in the IncidentDatabase)
for (Incident incident : applicableIncidents)
incident.resolve(tree, log);
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, migrationOptions.getLegacyExtension(), false);
}
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(node));
return true;
}
private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder) throws IOException {
Path source = scenarioFilePath;
Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension);
if (moveOutputFolder) {
source = source.getParent();
target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension);
}
IOUtils.createDirectoryIfNotExisting(target);
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten?
}