Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 9afe5ec3 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

merge local migration branch

parents 5c092be5 bb44e786
Pipeline #62846 failed with stages
in 46 seconds
......@@ -5,6 +5,7 @@ import org.vadere.simulator.projects.io.IOVadere;
import org.vadere.util.io.IOUtils;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
public class ScenarioFactory {
......@@ -31,4 +32,21 @@ public class ScenarioFactory {
return scenario;
}
public static Scenario createScenarioWithScenarioFilePath(
final Path scenarioPath) throws IOException {
String json = IOUtils.readTextFile(scenarioPath);
Scenario scenario = IOVadere.fromJson(json);
return scenario;
}
public static Scenario createScenarioWithScenarioJson(
final String scenarioJson) throws IOException {
Scenario scenario = IOVadere.fromJson(scenarioJson);
return scenario;
}
}
package org.vadere.simulator.entrypoints;
import org.jetbrains.annotations.NotNull;
import org.vadere.simulator.projects.migration.incidents.VersionBumpIncident;
import java.util.Optional;
......@@ -30,6 +31,34 @@ public enum Version {
return null;
}
private static int versionId(Version curr){
Version[] versions = values();
for ( int i = 0 ; i < versions.length ; i++){
if (curr.equals(versions[i])){
return i;
}
}
throw new IllegalArgumentException("Value not in Version Enumeration " + curr.toString());
}
public static Version nextVersion(Version curr){
int nextId = versionId(curr) == (values().length -1) ? versionId(curr) : versionId(curr) + 1;
return values()[nextId];
}
public static Version[] listToLatest (Version v){
int start = versionId(v) == (values().length -1) ? versionId(v) : versionId(v) + 1;
int end = values().length;
Version[] ret = new Version[end-start];
System.arraycopy(values(), start, ret, 0, end - start );
return ret;
}
public boolean equalOrSamller(Version test){
return versionId(this) <= versionId(test);
}
public static Version latest() {
return values()[values().length - 1];
}
......
......@@ -13,6 +13,7 @@ import java.util.Random;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.log4j.MDC;
import org.jetbrains.annotations.Nullable;
import org.vadere.simulator.control.PassiveCallback;
import org.vadere.simulator.control.Simulation;
......@@ -69,6 +70,9 @@ public class ScenarioRun implements Runnable {
@Override
public void run() {
try {
//add Scenario Name to Log4j Mapped Diagnostic Context to filter log by ScenarioRun
// MDC.put("scenario.Name", outputPath.getFileName().toString());
/**
* To make sure that no other Thread changes the scenarioStore object during the initialization of a scenario run
* this is an atomic operation with respect to the scenarioStore. We observed that with Linux 18.04 KUbunto
......@@ -77,7 +81,7 @@ public class ScenarioRun implements Runnable {
synchronized (scenarioStore) {
logger.info(String.format("Initializing scenario. Start of scenario '%s'...", scenario.getName()));
scenarioStore.getTopography().reset();
System.out.println("StartIt " + scenario.getName());
MainModelBuilder modelBuilder = new MainModelBuilder(scenarioStore);
modelBuilder.createModelAndRandom();
......@@ -106,6 +110,8 @@ public class ScenarioRun implements Runnable {
throw new RuntimeException("Simulation failed.", e);
} finally {
doAfterSimulation();
//remove Log4j Mapped Diagnostic Context after ScenarioRun
// MDC.remove("scenario.Name");
}
}
......@@ -138,6 +144,10 @@ public class ScenarioRun implements Runnable {
}
}
public Path getOutputPath() {
return Paths.get(this.outputPath.toString());
}
public void pause() {
if (simulation != null) { // TODO throw an illegal state exception if simulation is not running
simulation.pause();
......
......@@ -278,4 +278,8 @@ public class DataProcessingJsonManager {
return maxId;
}
public boolean containsOutputFile(String name){
return this.outputFiles.stream().anyMatch(f -> f.getFileName().equals(name));
}
}
......@@ -30,7 +30,7 @@ public class JoltTest {
public void run2() {
List chainrSpecJson = JsonUtils.classpathToList("/transfrom_v2_to_v3.json");
List chainrSpecJson = JsonUtils.classpathToList("/transform_v2_to_v3.json");
Chainr chainr = Chainr.fromSpec(chainrSpecJson);
Object inputJson = JsonUtils.classpathToObject("/scenario_test.json");
......
package org.vadere.simulator.projects.migration;
import com.bazaarvoice.jolt.Chainr;
import com.bazaarvoice.jolt.Diffy;
import com.bazaarvoice.jolt.JsonUtils;
import com.fasterxml.jackson.databind.JsonNode;
import org.apache.log4j.Logger;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.state.util.StateJsonConverter;
import org.vadere.util.io.IOUtils;
import org.vadere.util.logging.LogBufferAppender;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Optional;
import static org.vadere.util.io.IOUtils.LEGACY_DIR;
import static org.vadere.util.io.IOUtils.OUTPUT_DIR;
import static org.vadere.util.io.IOUtils.SCENARIO_DIR;
public class Migration {
private final static Logger logger = Logger.getLogger(Migration.class);
private final static LogBufferAppender appender;
private static HashMap<Version, Chainr> identityTransformation = new LinkedHashMap<>();
// Version is the target version. Only Incremental Transformation supported 1->2->3 (not 1->3)
private static HashMap<Version, Chainr> transformations = new LinkedHashMap<>();
private static final String LEGACY_EXTENSION = "legacy";
private static final String NONMIGRATABLE_EXTENSION = "nonmigratable";
static {
identityTransformation.put(Version.V0_1, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v1.json")));
identityTransformation.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/identity_v2.json")));
transformations.put(Version.V0_2, Chainr.fromSpec(JsonUtils.classpathToList("/transform_v1_to_v2.json")));
appender = new LogBufferAppender();
}
private Version baseVersion = null;
private boolean reapplyLatestMigrationFlag = false;
private Diffy transformationDiff;
public Migration() {
this.transformationDiff = new Diffy();
logger.addAppender(appender);
}
public String getLog() {
return appender.getMigrationLog();
}
public void setReapplyLatestMigrationFlag() {
reapplyLatestMigrationFlag = true;
baseVersion = null;
}
public void setReapplyLatestMigrationFlag(final Version version) {
reapplyLatestMigrationFlag = true;
baseVersion = version;
}
public MigrationResult analyzeProject(String projectFolderPath) throws IOException {
MigrationResult stats = new MigrationResult();
Path scenarioDir = Paths.get(projectFolderPath, SCENARIO_DIR);
if (Files.exists(scenarioDir)) {
stats = analyzeDirectory(scenarioDir, SCENARIO_DIR);
}
Path outputDir = Paths.get(projectFolderPath, OUTPUT_DIR);
if (Files.exists(outputDir)) {
MigrationResult outputDirStats = analyzeDirectory(outputDir, OUTPUT_DIR);
stats.add(outputDirStats);
}
baseVersion = null;
return stats;
}
public MigrationResult analyzeDirectory(Path dir, String dirName) throws IOException {
Path legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(dirName);
File[] scenarioFiles = dirName.equals(SCENARIO_DIR) ? IOUtils.getFilesInScenarioDirectory(dir) : IOUtils.getScenarioFilesInOutputDirectory(dir);
MigrationResult stats = new MigrationResult(scenarioFiles.length);
for (File file : scenarioFiles) {
if (dirName.equals(OUTPUT_DIR)) {
String fileFolder = Paths.get(file.getParent()).getFileName().toString(); // the folder in which the .scenario and the .trajectories file lies
legacyDir = dir.getParent().resolve(LEGACY_DIR).resolve(OUTPUT_DIR).resolve(fileFolder);
}
Path scenarioFilePath = Paths.get(file.getAbsolutePath());
try {
if (analyzeScenario(scenarioFilePath, legacyDir, dirName)) {
stats.legacy++;
} else {
stats.upToDate++;
}
} catch (MigrationException e) {
moveFileAddExtension(scenarioFilePath, legacyDir, NONMIGRATABLE_EXTENSION, dirName.equals(SCENARIO_DIR));
logger.error("!> Can't migrate the scenario to latest version, removed it from the directory (" +
e.getMessage() + ") If you can fix this problem manually, do so and then remove ." +
NONMIGRATABLE_EXTENSION + " from the file in the " + LEGACY_DIR + "-directory "
+ "and move it back into the scenarios-directory, it will be checked again when the GUI restarts.");
stats.notmigratable++;
}
}
return stats;
}
public JsonNode transform(JsonNode currentJson, Version targetVersion) throws MigrationException {
try {
return transform(StateJsonConverter.convertJsonNodeToObject(currentJson), targetVersion);
} catch (IOException e) {
logger.error("Error in converting JsonNode To Map of Object representation");
throw new MigrationException("Error in converting JsonNode To Map of Object representation", e);
}
}
private JsonNode transform(Object currentJson, Version targetVersion) throws MigrationException {
Chainr t = transformations.get(targetVersion);
if (t == null) {
logger.error("No Transformation defined for Version " + targetVersion.toString());
throw new MigrationException("No Transformation defined for Version " + targetVersion.toString());
}
Object scenarioTransformed = t.transform(currentJson);
Chainr identity = identityTransformation.get(targetVersion);
if (identity == null) {
logger.error("No IdentityTransformation defined for Version " + targetVersion.toString());
throw new MigrationException("No IdentityTransformation definde for Version " + targetVersion.toString());
}
Object scenarioTransformedTest = identity.transform(scenarioTransformed);
Diffy.Result diffResult = transformationDiff.diff(scenarioTransformed, scenarioTransformedTest);
if (diffResult.isEmpty()) {
return StateJsonConverter.deserializeToNode(scenarioTransformed);
} else {
logger.error("Error in Transformation " + diffResult.toString());
throw new MigrationException("Error in Transformation " + diffResult.toString());
}
}
private boolean analyzeScenario(Path scenarioFilePath, Path legacyDir, String dirName)
throws IOException, MigrationException {
String json = IOUtils.readTextFile(scenarioFilePath);
JsonNode node = StateJsonConverter.deserializeToNode(json);
String parentPath = dirName.equals(SCENARIO_DIR) ? SCENARIO_DIR + "/" : OUTPUT_DIR + "/" + scenarioFilePath.getParent().getFileName().toString() + "/";
logger.info(">> analyzing JSON tree of scenario <" + parentPath + node.get("name").asText() + ">");
Version version = Version.UNDEFINED;
if (node.get("release") != null) {
version = Version.fromString(node.get("release").asText());
if (version == null || version.equalOrSamller(Version.NOT_A_RELEASE)) {
logger.error("release version " + node.get("release").asText() + " is unknown or not " +
"supported. If this is a valid release create a version transformation and a new idenity transformation");
throw new MigrationException("release version " + node.get("release").asText() + " is unknown or not " +
"supported. If this is a valid releasecreate a version transformation and a new idenity transformation");
}
// if enforced migration should be done from baseVersion to latestVersion
if (reapplyLatestMigrationFlag && baseVersion != null) {
version = baseVersion;
} else if (reapplyLatestMigrationFlag) { // if enforced migration should be done from prev version to latest
Optional<Version> optVersion = Version.getPrevious(version);
if (optVersion.isPresent()) {
version = optVersion.get();
} else {
return false;
}
} // if no enforced migration should be done and we are at the latest version, no migration is required.
else if (version == Version.latest()) {
return false;
}
} else {
logger.error("Version is unknown");
throw new MigrationException("Version is unknown");
}
JsonNode transformedNode = node;
// apply all transformation from current to latest version.
for (Version v : Version.listToLatest(version)) {
transformedNode = transform(transformedNode, v);
}
if (legacyDir != null) {
moveFileAddExtension(scenarioFilePath, legacyDir, LEGACY_EXTENSION, false);
}
IOUtils.writeTextFile(scenarioFilePath.toString(), StateJsonConverter.serializeJsonNode(transformedNode));
return true;
}
private void moveFileAddExtension(Path scenarioFilePath, Path legacyDir, String additionalExtension, boolean moveOutputFolder)
throws IOException {
Path source = scenarioFilePath;
Path target = legacyDir.resolve(source.getFileName() + "." + additionalExtension);
if (moveOutputFolder) {
source = source.getParent();
target = Paths.get(legacyDir.toAbsolutePath() + "." + additionalExtension);
}
IOUtils.createDirectoryIfNotExisting(target);
Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); // ensure potential existing files aren't overwritten?
}
}
......@@ -36,7 +36,7 @@ public class MigrationAssistant {
private static boolean reapplyLatestMigrationFlag = false;
private static Version baseVersion = null;
public static void setReapplyLatestMigrationFlag() {
public static void setReapplyLatestMigrationFlag() {
reapplyLatestMigrationFlag = true;
baseVersion = null;
}
......@@ -46,8 +46,14 @@ public class MigrationAssistant {
baseVersion = version;
}
public static void analyzeSingleScenario(Path path) {
public static void analyzeSingleScenario(Path path) throws IOException {
// TODO [priority=high] [task=implement] for runs initiated not from GUI... where to hook in?
StringBuilder log = new StringBuilder();
try {
analyzeScenario(path, null, log, true) ;
} catch (MigrationException e) {
logger.error(log.toString());
}
}
public static int[] analyzeProject(String projectFolderPath) throws IOException {
......
......@@ -6,12 +6,17 @@ public class MigrationException extends Exception {
private static final long serialVersionUID = 1L;
public MigrationException() {}
public MigrationException() {
}
public MigrationException(String message) {
super(message);
}
public MigrationException(String message, Throwable cause) {
super(message, cause);
}
public MigrationException(Incident incident, String message) {
super(incident.getClass().getSimpleName() + ": " + message);
}
......
package org.vadere.simulator.projects.migration;
import java.util.Objects;
public class MigrationResult {
public int total;
public int upToDate;
public int legacy;
public int notmigratable;
public MigrationResult() {
}
public MigrationResult(int total, int upToDate, int legacy, int notmigratable) {
this.total = total;
this.upToDate = upToDate;
this.legacy = legacy;
this.notmigratable = notmigratable;
}
public MigrationResult(int total) {
this.total = total;
}
boolean checkTotal() {
return (upToDate + legacy + notmigratable) == total;
}
public MigrationResult add(MigrationResult other) {
this.total = this.total + other.total;
this.upToDate = this.upToDate + other.upToDate;
this.legacy = this.legacy + other.legacy;
this.notmigratable = this.notmigratable + other.notmigratable;
return this;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
MigrationResult result = (MigrationResult) o;
return total == result.total &&
upToDate == result.upToDate &&
legacy == result.legacy &&
notmigratable == result.notmigratable;
}
@Override
public int hashCode() {
return Objects.hash(total, upToDate, legacy, notmigratable);
}
@Override
public String toString() {
return "MigrationResult{" +
"total=" + total +
", upToDate=" + upToDate +
", legacy=" + legacy +
", notmigratable=" + notmigratable +
'}';
}
}
{
"name" : "basic_1_chicken_osm1",
"description" : "",
"release" : "0.1",
"topographyhash" : "cde764736cfb41b5cd5e4ff40e0adfca0fee6370",
"attributeshash" : "75778572f188cffcdedc74a7a9b60ddbe42359fc",
"processWriters" : [ {
"formatString" : "%s",
"columnNames" : [ "waitingTimeTest" ],
"attributes" : {
"startTime" : 0.0,
"endTime" : 1.7976931348623157E308
},
"processor" : {
"pedestrianWaitingTimeProcessor" : {
"attributes" : {
"waitingArea" : {
"x" : 10.0,
"y" : 32.0,
"width" : 15.0,
"height" : 8.0
}
},
"columnNames" : [ "totalWaitingTime", "time", "id" ],
"clazz" : "PedestrianWaitingTimeProcessor"
},
"attributes" : {
"maxWaitingTime" : 0.0,
"maxWaitingTimeMean" : 0.0,
"minWaitingTime" : 0.0,
"minWaitingTimeMean" : 0.0,
"expectFailure" : false
},
"columnNames" : [ ],
"clazz" : "PedestrianWaitingTimeTest"
}
}, {
"formatString" : "%s",
"columnNames" : [ "waitingTimeTest" ],
"attributes" : {
"startTime" : 0.0,
"endTime" : 1.7976931348623157E308
},
"processor" : {
"pedestrianWaitingTimeProcessor" : {
"attributes" : {
"waitingArea" : {
"x" : 10.0,
"y" : 23.5,
"width" : 15.0,
"height" : 7.5
}
},
"columnNames" : [ "totalWaitingTime", "time", "id" ],
"clazz" : "PedestrianWaitingTimeProcessor"
},
"attributes" : {
"maxWaitingTime" : 10.0,
"maxWaitingTimeMean" : 10.0,
"minWaitingTime" : 0.0,
"minWaitingTimeMean" : 0.0,
"expectFailure" : false
},
"columnNames" : [ ],
"clazz" : "PedestrianWaitingTimeTest"
}
}, {
"formatString" : "%s",
"columnNames" : [ "evacuationTimeTest" ],
"attributes" : {
"startTime" : 0.0,
"endTime" : 1.7976931348623157E308
},
"processor" : {
"attributes" : {
"maxEvacuationTime" : 100.0,
"maxEvacuationTimeMean" : 100.0,
"minEvacuationTime" : 0.0,
"minEvacuationTimeMean" : 0.0,
"expectFailure" : false
},
"errorTime" : -1.0,
"columnNames" : [ ],
"clazz" : "PedestrianEvacuationTimeTest"
}
} ],
"vadere" : {
"mainModel" : "org.vadere.simulator.models.osm.OptimalStepsModel",