Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 35205f58 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

add simulation seed and fixed seed to scenario file.

Changes:

* Add UseFixedSeed (bool switch) and fixedSeed Attribute values
  This allows to rerun the simulation with different but fixed seeds
  for random numbers.
* Add simulationSeed to save currently used seed while simulation is
  running. This is useful to determine the seed used for processor
  outputs in the output directory.
* Add Test for new version and refactor code duplicates
* Change Hash calculation to exclude UseFixedSeed, fixedSeed and
  simulationSeed to highlight (green) output to selected scenario
  within the gui.
parent 6f4d4ae2
...@@ -14,7 +14,8 @@ public enum Version { ...@@ -14,7 +14,8 @@ public enum Version {
NOT_A_RELEASE("not a release"), NOT_A_RELEASE("not a release"),
V0_1("0.1"), V0_1("0.1"),
V0_2("0.2"), V0_2("0.2"),
V0_3("0.3"); V0_3("0.3"),
V0_4("0.4");
private String label; private String label;
......
...@@ -26,10 +26,14 @@ public class MainModelBuilder { ...@@ -26,10 +26,14 @@ public class MainModelBuilder {
throws ClassNotFoundException, InstantiationException, IllegalAccessException { throws ClassNotFoundException, InstantiationException, IllegalAccessException {
final AttributesSimulation attributesSimulation = scenarioStore.getAttributesSimulation(); final AttributesSimulation attributesSimulation = scenarioStore.getAttributesSimulation();
if (attributesSimulation.isUseRandomSeed()) { if (attributesSimulation.isUseFixedSeed()) {
random = new Random(attributesSimulation.getRandomSeed()); long seed = attributesSimulation.getFixedSeed();
attributesSimulation.setSimulationSeed(seed);
random = new Random(seed);
} else { } else {
random = new Random(); long seed = new Random().nextLong();
attributesSimulation.setSimulationSeed(seed);
random = new Random(seed);
} }
model = instantiateMainModel(random); model = instantiateMainModel(random);
......
...@@ -74,7 +74,7 @@ public class ScenarioStore { ...@@ -74,7 +74,7 @@ public class ScenarioStore {
} }
public String hashOfJsonRepresentation() throws JsonProcessingException { public String hashOfJsonRepresentation() throws JsonProcessingException {
return DigestUtils.sha1Hex(StateJsonConverter.serializeObject(this)); return StateJsonConverter.getScenarioStoreHash(this);
} }
public void sealAllAttributes() { public void sealAllAttributes() {
......
...@@ -17,11 +17,11 @@ public class JoltTransformV1toV2 extends JoltTransformation { ...@@ -17,11 +17,11 @@ public class JoltTransformV1toV2 extends JoltTransformation {
@Override @Override
protected void initPostHooks() { protected void initPostHooks() {
postTransformHooks.add(this::sort); postTransformHooks.add(JoltTransformV1toV2::sort);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public JsonNode sort (JsonNode node) throws MigrationException{ public static JsonNode sort (JsonNode node) throws MigrationException{
LinkedHashMap source = (LinkedHashMap) StateJsonConverter.convertJsonNodeToObject(node); LinkedHashMap source = (LinkedHashMap) StateJsonConverter.convertJsonNodeToObject(node);
LinkedHashMap<Object, Object> sortedRoot = new LinkedHashMap<>(); LinkedHashMap<Object, Object> sortedRoot = new LinkedHashMap<>();
putObject(sortedRoot, source, "name"); putObject(sortedRoot, source, "name");
......
package org.vadere.simulator.projects.migration.jolttranformation; package org.vadere.simulator.projects.migration.jolttranformation;
import com.fasterxml.jackson.databind.JsonNode;
import org.vadere.simulator.entrypoints.Version; import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.migration.MigrationException; import org.vadere.simulator.projects.migration.MigrationException;
import org.vadere.state.util.StateJsonConverter;
import java.util.LinkedHashMap;
public class JoltTransformV2toV3 extends JoltTransformation { public class JoltTransformV2toV3 extends JoltTransformation {
public JoltTransformV2toV3(String transformation, String identity, Version version) throws MigrationException { public JoltTransformV2toV3(String transformation, String identity, Version version) throws MigrationException {
...@@ -15,20 +10,8 @@ public class JoltTransformV2toV3 extends JoltTransformation { ...@@ -15,20 +10,8 @@ public class JoltTransformV2toV3 extends JoltTransformation {
@Override @Override
protected void initPostHooks() { protected void initPostHooks() {
postTransformHooks.add(this::sort); postTransformHooks.add(JoltTransformV1toV2::sort);
} }
@SuppressWarnings("unchecked")
public JsonNode sort (JsonNode node) throws MigrationException{
LinkedHashMap source = (LinkedHashMap) StateJsonConverter.convertJsonNodeToObject(node);
LinkedHashMap<Object, Object> sortedRoot = new LinkedHashMap<>();
putObject(sortedRoot, source, "name");
putObject(sortedRoot, source, "description");
putObject(sortedRoot, source, "release");
putObject(sortedRoot, source, "commithash");
putObject(sortedRoot, source, "processWriters", "files", "processors", "isTimestamped");
putObject(sortedRoot, source, "scenario", "mainModel", "attributesModel", "attributesSimulation", "topography");
return StateJsonConverter.deserializeToNode(sortedRoot);
}
} }
package org.vadere.simulator.projects.migration.jolttranformation;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.vadere.simulator.entrypoints.Version;
import org.vadere.simulator.projects.migration.MigrationException;
import java.util.Random;
public class JoltTransformV3toV4 extends JoltTransformation {
public JoltTransformV3toV4(String transformation, String identity, Version version) throws MigrationException {
super(transformation, identity, version);
}
@Override
protected void initPostHooks() throws MigrationException {
postTransformHooks.add(this::presetSeedValues);
postTransformHooks.add(JoltTransformV1toV2::sort);
}
public JsonNode presetSeedValues(JsonNode node) throws MigrationException {
JsonNode attSim = node.findPath("scenario").findPath("attributesSimulation");
if (attSim.isMissingNode())
throw new MigrationException("attributesSimulation is not part of Scenario.");
JsonNode fixedSeed = attSim.findPath("fixedSeed");
if (fixedSeed.isMissingNode())
throw new MigrationException("scenario.attributesSimulation.fixedSeed must be present" +
"for Version V0.4");
Integer fixedSeedVal = fixedSeed.asInt(-1);
if (fixedSeedVal == -1){
((ObjectNode)attSim).put("fixedSeed", new Random().nextLong());
}
return node;
}
}
...@@ -54,6 +54,9 @@ public abstract class JoltTransformation { ...@@ -54,6 +54,9 @@ public abstract class JoltTransformation {
case V0_2: case V0_2:
ret = new JoltTransformV2toV3(transformationResource, identityResource, currentVersion); ret = new JoltTransformV2toV3(transformationResource, identityResource, currentVersion);
break; break;
case V0_3:
ret = new JoltTransformV3toV4(transformationResource, identityResource, currentVersion);
break;
default: default:
throw new MigrationException("No Transformation defined for Verson " + currentVersion.toString()); throw new MigrationException("No Transformation defined for Verson " + currentVersion.toString());
} }
...@@ -147,7 +150,7 @@ public abstract class JoltTransformation { ...@@ -147,7 +150,7 @@ public abstract class JoltTransformation {
* @param key key to add to new HashMap * @param key key to add to new HashMap
* @param children Specify Order on second level * @param children Specify Order on second level
*/ */
protected void putObject(LinkedHashMap<Object, Object> target, protected static void putObject(LinkedHashMap<Object, Object> target,
LinkedHashMap<Object, Object> source, LinkedHashMap<Object, Object> source,
String key, String... children) throws MigrationException { String key, String... children) throws MigrationException {
......
package org.vadere.simulator.projects.migration.jolttranformation;
import org.hamcrest.core.IsNot;
import org.hamcrest.core.StringContains;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.vadere.simulator.projects.migration.MigrationAssistant;
import org.vadere.simulator.projects.migration.MigrationException;
import org.vadere.simulator.projects.migration.MigrationOptions;
import org.vadere.util.io.IOUtils;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import org.vadere.simulator.entrypoints.Version;
import static org.hamcrest.core.IsNot.not;
import static org.junit.Assert.*;
public class JoltTransformV3toV4Test {
ArrayList<Path> scenarios;
ArrayList<Path> bakScenarios;
private StringContains v04 = new StringContains("\"release\" : \"0.4\"");
private StringContains useFixedSeed = new StringContains("useFixedSeed");
private StringContains fixedSeed = new StringContains("fixedSeed");
private StringContains simulationSeed = new StringContains("simulationSeed");
private StringContains useRandomSeed = new StringContains("useRandomSeed");
private StringContains randomSeed = new StringContains("randomSeed");
@Before
public void init() throws URISyntaxException {
scenarios = new ArrayList<>();
bakScenarios = new ArrayList<>();
scenarios.add(Paths.get(getClass()
.getResource("/migration/v03_to_v04/basic_1_chicken_osm1.scenario").toURI()));
scenarios.add(Paths.get(getClass()
.getResource("/migration/v03_to_v04/group_OSM_1Source1Place.scenario").toURI()));
scenarios.forEach(f -> {
try {
bakScenarios.add(IOUtils.makeBackup(f, ".bak", true));
} catch (IOException e) {
e.printStackTrace();
}
});
}
@After
public void cleaUp() throws IOException {
for (int i=0; i < scenarios.size(); i++){
if (scenarios.get(i) != null && bakScenarios.get(i) != null){
Files.move(bakScenarios.get(i), scenarios.get(i), StandardCopyOption.REPLACE_EXISTING);
}
}
}
@Test
public void TestTransform(){
MigrationAssistant ma = MigrationAssistant.getNewInstance(MigrationOptions.defaultOptions());
try {
String out = ma.convertFile(scenarios.get(0), Version.V0_4);
// System.out.println(out);
assertThat("New Version must be V0.4", out, v04);
assertThat("New Entry missing", out, useFixedSeed);
assertThat("New Entry missing", out, fixedSeed);
assertThat("New Entry missing", out, simulationSeed);
assertThat("Old Entry still there", out, not(useRandomSeed));
assertThat("Old Entry still there", out, not(randomSeed));
} catch (MigrationException e) {
e.printStackTrace();
fail();
}
}
}
\ No newline at end of file
...@@ -94,6 +94,11 @@ ...@@ -94,6 +94,11 @@
<artifactId>annotation</artifactId> <artifactId>annotation</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.8</version>
</dependency>
</dependencies> </dependencies>
</project> </project>
package org.vadere.state.attributes; package org.vadere.state.attributes;
import com.fasterxml.jackson.annotation.JsonFilter; import java.util.Objects;
import java.util.Random;
/** /**
* Provides attributes for the simulation, like visualizationEnabled and * Provides attributes for the simulation, like visualizationEnabled and
* writeSimulationData. * writeSimulationData.
* *
*/ */
@JsonFilter("testFilter")
public class AttributesSimulation extends Attributes { public class AttributesSimulation extends Attributes {
private double finishTime = 500; private double finishTime = 500;
...@@ -19,8 +19,9 @@ public class AttributesSimulation extends Attributes { ...@@ -19,8 +19,9 @@ public class AttributesSimulation extends Attributes {
private boolean printFPS = false; private boolean printFPS = false;
private boolean needsBoundary = false; private boolean needsBoundary = false;
private int digitsPerCoordinate = 2; private int digitsPerCoordinate = 2;
private boolean useRandomSeed = true; private boolean useFixedSeed = true;
private long randomSeed = 1; private long fixedSeed = new Random().nextLong();
private long simulationSeed;
// Getter... // Getter...
...@@ -56,12 +57,16 @@ public class AttributesSimulation extends Attributes { ...@@ -56,12 +57,16 @@ public class AttributesSimulation extends Attributes {
return digitsPerCoordinate; return digitsPerCoordinate;
} }
public boolean isUseRandomSeed() { public boolean isUseFixedSeed() {
return useRandomSeed; return useFixedSeed;
} }
public long getRandomSeed() { public long getFixedSeed() {
return randomSeed; return fixedSeed;
}
public long getSimulationSeed() {
return simulationSeed;
} }
public void setFinishTime(double finishTime) { public void setFinishTime(double finishTime) {
...@@ -104,66 +109,42 @@ public class AttributesSimulation extends Attributes { ...@@ -104,66 +109,42 @@ public class AttributesSimulation extends Attributes {
this.digitsPerCoordinate = digitsPerCoordinate; this.digitsPerCoordinate = digitsPerCoordinate;
} }
public void setUseRandomSeed(boolean useRandomSeed) { public void setUseFixedSeed(boolean useFixedSeed) {
checkSealed();
this.useFixedSeed = useFixedSeed;
}
public void setFixedSeed(long fixedSeed) {
checkSealed(); checkSealed();
this.useRandomSeed = useRandomSeed; this.fixedSeed = fixedSeed;
} }
public void setRandomSeed(long randomSeed) { public void setSimulationSeed(long simulationSeed) {
checkSealed(); checkSealed();
this.randomSeed = randomSeed; this.simulationSeed = simulationSeed;
} }
@Override @Override
public boolean equals(Object o) { public boolean equals(Object o) {
if (this == o) if (this == o) return true;
return true; if (o == null || getClass() != o.getClass()) return false;
if (o == null || getClass() != o.getClass())
return false;
AttributesSimulation that = (AttributesSimulation) o; AttributesSimulation that = (AttributesSimulation) o;
return Double.compare(that.finishTime, finishTime) == 0 &&
if (digitsPerCoordinate != that.digitsPerCoordinate) Double.compare(that.simTimeStepLength, simTimeStepLength) == 0 &&
return false; Double.compare(that.realTimeSimTimeRatio, realTimeSimTimeRatio) == 0 &&
if (needsBoundary != that.needsBoundary) writeSimulationData == that.writeSimulationData &&
return false; visualizationEnabled == that.visualizationEnabled &&
if (printFPS != that.printFPS) printFPS == that.printFPS &&
return false; needsBoundary == that.needsBoundary &&
if (randomSeed != that.randomSeed) digitsPerCoordinate == that.digitsPerCoordinate &&
return false; useFixedSeed == that.useFixedSeed &&
if (Double.compare(that.realTimeSimTimeRatio, realTimeSimTimeRatio) != 0) fixedSeed == that.fixedSeed &&
return false; simulationSeed == that.simulationSeed;
if (Double.compare(that.simTimeStepLength, simTimeStepLength) != 0)
return false;
if (useRandomSeed != that.useRandomSeed)
return false;
if (visualizationEnabled != that.visualizationEnabled)
return false;
if (writeSimulationData != that.writeSimulationData)
return false;
if (finishTime != that.finishTime)
return false;
return true;
} }
@Override @Override
public int hashCode() { public int hashCode() {
int result;
long temp; return Objects.hash(finishTime, simTimeStepLength, realTimeSimTimeRatio, writeSimulationData, visualizationEnabled, printFPS, needsBoundary, digitsPerCoordinate, useFixedSeed, fixedSeed, simulationSeed);
temp = Double.doubleToLongBits(simTimeStepLength);
result = (int) (temp ^ (temp >>> 32));
temp = Double.doubleToLongBits(realTimeSimTimeRatio);
result = 31 * result + (int) (temp ^ (temp >>> 32));
temp = Double.doubleToLongBits(finishTime);
result = 31 * result + (int) (temp ^ (temp >>> 32));
result = 31 * result + (writeSimulationData ? 1 : 0);
result = 31 * result + (visualizationEnabled ? 1 : 0);
result = 31 * result + (printFPS ? 1 : 0);
result = 31 * result + (needsBoundary ? 1 : 0);
result = 31 * result + digitsPerCoordinate;
result = 31 * result + (useRandomSeed ? 1 : 0);
result = 31 * result + (int) (randomSeed ^ (randomSeed >>> 32));
return result;
} }
} }
...@@ -43,6 +43,8 @@ import com.fasterxml.jackson.databind.ObjectWriter; ...@@ -43,6 +43,8 @@ import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.codec.digest.DigestUtils;
public abstract class StateJsonConverter { public abstract class StateJsonConverter {
public static final String SCENARIO_KEY = "scenario"; public static final String SCENARIO_KEY = "scenario";
...@@ -292,6 +294,23 @@ public abstract class StateJsonConverter { ...@@ -292,6 +294,23 @@ public abstract class StateJsonConverter {
} }
} }
public static String getScenarioStoreHash(Object object){
JsonNode jsonNode = mapper.convertValue(object, JsonNode.class);
JsonNode attrSimulation = jsonNode.findPath("attributesSimulation");
if (! attrSimulation.isMissingNode()){
((ObjectNode)attrSimulation).remove("simulationSeed");
((ObjectNode)attrSimulation).remove("useFixedSeed");
((ObjectNode)attrSimulation).remove("simulationSeed");
}
try {
String scenarioString = writer.writeValueAsString(jsonNode);
return DigestUtils.sha1Hex(scenarioString);
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
public static JsonNode toJsonNode(final Object object) { public static JsonNode toJsonNode(final Object object) {
return mapper.convertValue(object, JsonNode.class); return mapper.convertValue(object, JsonNode.class);
} }
......
package org.vadere.state.util;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.ser.FilterProvider;
import com.fasterxml.jackson.databind.ser.PropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
import org.junit.Test;
import org.vadere.state.attributes.AttributesSimulation;
import static org.junit.Assert.*;
public class StateJsonConverterTest {
@Test
public void testMapper(){
AttributesSimulation a = new AttributesSimulation();
PropertyFilter empty = SimpleBeanPropertyFilter.serializeAll();
PropertyFilter ignoreSeed = SimpleBeanPropertyFilter.serializeAllExcept("randomSeed");
SimpleFilterProvider fp = new SimpleFilterProvider().addFilter("testFilter", empty);
ObjectMapper mapper = StateJsonConverter.getMapper().setFilterProvider(fp);
ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
try {
JsonNode json = mapper.convertValue(a, JsonNode.class);
String str = writer.writeValueAsString(json);
System.out.println(str);
} catch (JsonProcessingException | IllegalArgumentException e) {
throw new RuntimeException(e);
}
fp.removeFilter("testFilter");
fp.addFilter("testFilter", ignoreSeed);
try {
JsonNode json = mapper.convertValue(a, JsonNode.class);
String str = writer.writeValueAsString(json);
System.out.println(str);
} catch (JsonProcessingException | IllegalArgumentException e) {
throw new RuntimeException(e);
}
}
}
\ No newline at end of file