Commit bb976657 authored by Stefan Schuhbaeck's avatar Stefan Schuhbaeck
Browse files

Create ProjectOutput to hold simulated data to reduce io

ProjectOutput holds a map of SimulationOutputs which were marked as
valid. Only the scenario file is kept in memory. The trajectory file
is only loaded once to see if it is valid but is not kept in memory.
SimulationOutputs can be marked as dirty. If so they will not be listed
and will be reloaded or removed with the next cleanup is done.
parent 31ffb963
package org.vadere.simulator.projects;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.vadere.simulator.projects.io.IOOutput;
import org.vadere.util.io.IOUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentMap;
import java.util.stream.Collectors;
/**
* Represents all run simulations within this project.
*
* @author Stefan Schuhbäck
*/
public class ProjectOutput {
private final VadereProject project;
private ConcurrentMap<String, SimulationOutput> simulationOutputs;
private OutputDirWatcher watcher;
public ProjectOutput(VadereProject project) {
this.project = project;
this.simulationOutputs = IOOutput.getSimulationOutputs(project);
// add watched directories manually to ensure only valid output directories
// and the root output directory are added.
// TODO exceptions?
try {
OutputDirWatcherBuilder builder = new OutputDirWatcherBuilder();
builder.initOutputDirWatcher(project);
builder.register(project.getOutputDir());
builder.addDefaultEventHandler();
Path out = project.getOutputDir();
List<Path> outputs = simulationOutputs.keySet().stream()
.map(k -> out.resolve(k))
.collect(Collectors.toList());
builder.register(outputs);
this.watcher = builder.build();
} catch (IOException e) {
e.printStackTrace();
}
}
public List<File> getAllOutputDirs() {
System.out.println("No FS Touch!");
Path out = project.getOutputDir();
// Keys of concurrentMap are the output directories for each simulation run.
// Resolve them against the project output dir gets you the needed paths/files.
List<File> outputs = simulationOutputs.keySet().stream()
.map(k -> out.resolve(k).toFile())
.collect(Collectors.toList());
return outputs;
}
public List<File> listSelectedOutputDirs(final Scenario scenario) {
List<File> out = new ArrayList<>();
try {
final String hash1 = scenario.getScenarioStore().hashOfJsonRepresentation();
for (Map.Entry<String, SimulationOutput> entry : simulationOutputs.entrySet()) {
if (!entry.getValue().isDirty()) {
String hash2 = entry.getValue().getScenarioHash();
if (hash1.equals(hash2))
out.add(project.getOutputDir().resolve(entry.getKey()).toFile());
}
}
} catch (JsonProcessingException e) {
e.printStackTrace();
}
return out;
}
synchronized public void reloadData() {
}
synchronized public void markDirty(String outputDir) {
Optional.ofNullable(this.simulationOutputs.get(outputDir)).ifPresent(SimulationOutput::setDirty);
}
public ConcurrentMap<String, SimulationOutput> getSimulationOutputs() {
return simulationOutputs;
}
public void setSimulationOutputs(ConcurrentMap<String, SimulationOutput> simulationOutputs) {
this.simulationOutputs = simulationOutputs;
}
public Optional<SimulationOutput> getSimulationOutput(String dirName) {
return Optional.ofNullable(simulationOutputs.get(dirName));
}
public void cleanOutputDirs() {
Iterator<Map.Entry<String, SimulationOutput>> iter = this.simulationOutputs.entrySet().iterator();
while (iter.hasNext()) {
Map.Entry<String, SimulationOutput> entry = iter.next();
if (entry.getValue().isDirty()) {
Optional<SimulationOutput> updated =
IOOutput.getSimulationOutput(project, entry.getValue().getOutputDir());
if (updated.isPresent()) {
entry.setValue(updated.get());
} else {
//existing dir went invalid.
iter.remove();
}
}
}
}
public void update() {
List<File> existingOutputDirs = getAllOutputDirs();
try {
Files.walkFileTree(project.getOutputDir(), new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
if (dir.endsWith(IOUtils.CORRUPT_DIR)) {
return FileVisitResult.SKIP_SUBTREE;
}
if (!existingOutputDirs.contains(dir.toFile())) {
Optional<SimulationOutput> newSim = IOOutput.getSimulationOutput(project, dir.toFile());
newSim.ifPresent(out -> simulationOutputs.put(dir.toFile().getName(), out));
}
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
e.printStackTrace();
}
}
}
package org.vadere.simulator.projects;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.File;
import java.nio.file.Path;
/**
......@@ -9,17 +12,47 @@ import java.nio.file.Path;
*/
public class SimulationOutput {
private Scenario simulatedScenario;private Path dir;
private final Scenario simulatedScenario;
private final Path outputDirectory;
private final String scenarioHash;
private boolean isDirty;
public SimulationOutput(Path directory, Scenario scenario){
this.dir = directory;
public SimulationOutput(Path directory, Scenario scenario) {
this.outputDirectory = directory;
this.simulatedScenario = scenario;
this.isDirty = false;
String tmpHash;
try {
tmpHash = scenario.getScenarioStore().hashOfJsonRepresentation();
} catch (JsonProcessingException e) {
tmpHash = "";
e.printStackTrace();
}
this.scenarioHash = tmpHash;
}
public String getScenarioHash() {
return this.scenarioHash;
}
public Scenario getSimulatedScenario() {
return simulatedScenario;
}
public boolean isDirty() {
return isDirty;
}
public void setDirty(boolean dirty) {
isDirty = dirty;
}
public void setDirty() {
isDirty = true;
}
public File getOutputDir() {
return outputDirectory.toFile();
}
}
......@@ -35,6 +35,7 @@ public class VadereProject {
new LinkedBlockingQueue<>();
private LinkedBlockingDeque<Scenario> scenariosLeft;
private Path outputDirectory;
private ProjectOutput projectOutput; //TODO initialize and wire up with rest ....
// TODO should be encapsulated in a class (we are not programming in C):
private int[] migrationStats; // scenarios: [0] total, [1] legacy'ed, [2] nonmigratable
......@@ -268,6 +269,8 @@ public class VadereProject {
public void finished(Runnable runnable) {
notifyScenarioRMListenerAboutPostRun(getCurrentScenario());
projectOutput.update();
if (scenariosLeft.isEmpty()) {
for (ProjectFinishedListener listener : projectFinishedListener) {
listener.postProjectRun(VadereProject.this);
......@@ -278,4 +281,11 @@ public class VadereProject {
}
};
public ProjectOutput getProjectOutput() {
return projectOutput;
}
public void setProjectOutput(ProjectOutput projectOutput) {
this.projectOutput = projectOutput;
}
}
......@@ -29,17 +29,16 @@ import java.util.stream.Collectors;
/**
* This IOUtility class provides all methods to load, delete, list, clean output directories.
* Each output directory contains two fiels *.scenario and *.trajectories.
* Each output directory contains two files *.scenario and *.trajectories.
*
*/
public abstract class IOOutput {
private static Logger logger = LogManager.getLogger(IOOutput.class);
private static final Logger logger = LogManager.getLogger(IOOutput.class);
public static List<File> listSelectedOutputDirs(final VadereProject project, final Scenario scenario) {
List<File> selectedOutputDirectories = new LinkedList<>();
selectedOutputDirectories = listAllOutputDirs(project).stream()
List<File> selectedOutputDirectories = listAllOutputDirs(project).stream()
.filter(dir -> isMatchingOutputDirectory(project, dir, scenario))
.collect(Collectors.toList());
......@@ -163,7 +162,7 @@ public abstract class IOOutput {
private static List<File> listAllDirs(final VadereProject project) {
List<File> outputDirectories = new LinkedList<>();
if (Files.exists(project.getOutputDir())) {
File[] files = new File(project.getOutputDir().toString()).listFiles(f -> f.isDirectory());
File[] files = new File(project.getOutputDir().toString()).listFiles(File::isDirectory);
if (files != null) {
outputDirectories = Arrays.stream(files).filter(dir -> !dir.getName().equals(IOUtils.CORRUPT_DIR))
.collect(Collectors.toList());
......@@ -172,22 +171,30 @@ public abstract class IOOutput {
return outputDirectories;
}
private static void cleanDirectory(final VadereProject project, final File directory) {
IOUtils.errorBox(
"The directory '"
+ directory.getName()
+ "' is corrupted and was moved to the '" + IOUtils.CORRUPT_DIR + "' folder.",
"Corrupt output file detected.");
private static void cleanDirectory(final VadereProject project, final File directory, boolean withGui){
final String info = "The directory '"
+ directory.getName()
+ "' is corrupted and was moved to the '" + IOUtils.CORRUPT_DIR + "' folder.";
if(withGui)
IOUtils.errorBox(info, "Corrupt output file detected.");
try {
Files.createDirectories(Paths.get(project.getOutputDir().toString(), IOUtils.CORRUPT_DIR));
Path sourcePath = directory.toPath();
Path targetPath = Paths.get(project.getOutputDir().toString(), IOUtils.CORRUPT_DIR, directory.getName());
Files.move(sourcePath, targetPath, StandardCopyOption.ATOMIC_MOVE);
logger.info(info);
} catch (IOException e1) {
logger.error(e1);
}
}
private static void cleanDirectory(final VadereProject project, final File directory) {
cleanDirectory(project, directory, true);
}
/**
* Returns {@link SimulationOutput} if supplied directory is a valid output directory.
* @param project VadereProject
......@@ -195,12 +202,16 @@ public abstract class IOOutput {
* @return SimulationOutput contained in selected directory
*/
public static Optional<SimulationOutput> getSimulationOutput(final VadereProject project, final File directory ){
if(!directory.exists())
return Optional.empty();
Optional<Scenario> scenario = readOutputFile(project, directory);
Optional<Map<Step, List<Agent>>> trajectories = readTrajectories(project, directory);
if (scenario.isPresent() && trajectories.isPresent()){
return Optional.of(new SimulationOutput(directory.toPath(), scenario.get()));
} else {
//if directory is not a valid OutputDirectory
cleanDirectory(project, directory, false);
return Optional.empty();
}
}
......@@ -219,6 +230,9 @@ public abstract class IOOutput {
if (scenario.isPresent() && trajectories.isPresent()){
SimulationOutput out = new SimulationOutput(f.toPath(), scenario.get());
simulationOutputs.put(f.getName(), out);
} else {
//invalid output directory move to corrupt.
cleanDirectory(project, f, false);
}
});
return simulationOutputs;
......
......@@ -2,6 +2,7 @@ package org.vadere.simulator.projects.io;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.vadere.simulator.projects.ProjectOutput;
import org.vadere.simulator.projects.Scenario;
import org.vadere.simulator.projects.VadereProject;
import org.vadere.simulator.projects.migration.MigrationAssistant;
......@@ -72,6 +73,8 @@ public class IOVadere {
VadereProject project = new VadereProject(name, scenarios);
project.setMigrationStats(migrationStats); // TODO [priority=low] [task=refactoring] better way to tunnel those results to the GUI?
project.setOutputDir(Paths.get(folderpath, IOUtils.OUTPUT_DIR));
ProjectOutput projectOutput = new ProjectOutput(project);
project.setProjectOutput(projectOutput);
logger.info("project loaded: " + project.getName());
return project;
}
......
timeStep pedestrianId x y
1 1 2.4 3.6
1 2 4.2 2.8
1 3 7.6 3.2
2 1 1.7563689308042068 3.8677692779348405
2 2 4.86205678414004 2.6352301505906714
2 3 7.067922981205282 3.5188806781618873
3 1 1.7563689308042068 3.8677692779348405
3 2 4.86205678414004 2.6352301505906714
3 3 7.067922981205282 3.5188806781618873
4 1 1.7563689308042068 4.604241082365638
4 2 5.579087200091227 2.6352301505906714
4 3 7.448936577289335 4.178815584953994
5 1 1.7563689308042068 5.340712886796436
5 2 6.252875390839607 2.88046899622316
5 3 7.687070074841869 4.893216077611594
6 1 1.7563689308042068 5.340712886796436
6 2 6.252875390839607 2.88046899622316
6 3 7.687070074841869 4.893216077611594
7 1 1.7563689308042068 6.077184691227233
7 2 6.802152556526303 3.341367263364968
7 3 7.819394708094508 5.6436663644648695
8 1 2.4484260508447093 6.329072883333968
8 2 7.0451430350372775 3.448659258329182
8 3 8.285439379047906 6.178283628351536
9 1 2.937337496203416 6.754134207514085
9 2 6.977202411434149 3.535681014603657
9 3 8.285439379047906 6.178283628351536
10 1 2.937337496203416 6.754134207514085
10 2 6.977202411434149 3.535681014603657
10 3 8.285439379047906 6.9403108205196435
11 1 2.937337496203416 6.754134207514085
11 2 7.06153231884398 3.528470420314763
11 3 8.285439379047906 7.70233801268775
12 1 2.937337496203416 6.754134207514085
12 2 7.442891030507138 4.130986307906705
12 3 8.666452975131959 8.083351608771803
13 1 2.937337496203416 6.754134207514085
13 2 7.567402055568822 4.837123420681001
14 1 2.937337496203416 6.754134207514085
14 2 7.567402055568822 4.837123420681001
15 1 2.937337496203416 6.754134207514085
15 2 7.925917263544415 5.458089976180851
16 1 2.937337496203416 6.754134207514085
16 2 8.284432471520008 6.079056531680701
17 1 2.937337496203416 6.754134207514085
17 2 8.284432471520008 6.796086947631887
18 1 2.937337496203416 6.754134207514085
18 2 8.284432471520008 6.796086947631887
19 1 2.937337496203416 6.754134207514085
19 2 8.284432471520008 7.513117363583072
20 1 2.937337496203416 6.754134207514085
20 2 8.284432471520008 8.230147779534258
21 1 2.937337496203416 6.754134207514085
22 1 2.937337496203416 6.754134207514085
23 1 2.937337496203416 6.754134207514085
24 1 2.937337496203416 6.754134207514085
25 1 2.937337496203416 6.754134207514085
26 1 2.937337496203416 6.754134207514085
27 1 2.937337496203416 6.754134207514085
28 1 2.937337496203416 6.754134207514085
29 1 2.937337496203416 6.754134207514085
30 1 2.937337496203416 6.754134207514085
31 1 2.937337496203416 6.754134207514085
32 1 2.937337496203416 6.754134207514085
33 1 2.937337496203416 6.754134207514085
34 1 2.937337496203416 6.754134207514085
35 1 2.937337496203416 6.754134207514085
36 1 2.937337496203416 6.754134207514085
37 1 2.937337496203416 6.754134207514085
38 1 2.937337496203416 6.754134207514085
39 1 2.937337496203416 6.754134207514085
40 1 2.937337496203416 6.754134207514085
41 1 2.937337496203416 6.754134207514085
42 1 2.937337496203416 6.754134207514085
43 1 2.937337496203416 6.754134207514085
44 1 2.937337496203416 6.754134207514085
45 1 2.937337496203416 6.754134207514085
46 1 2.937337496203416 6.754134207514085
47 1 2.937337496203416 6.754134207514085
48 1 2.937337496203416 6.754134207514085
49 1 2.937337496203416 6.754134207514085
50 1 2.937337496203416 6.754134207514085
51 1 2.937337496203416 6.754134207514085
52 1 2.937337496203416 6.754134207514085
53 1 2.937337496203416 6.754134207514085
54 1 2.937337496203416 6.754134207514085
55 1 2.937337496203416 6.754134207514085
56 1 2.937337496203416 6.754134207514085
57 1 2.937337496203416 6.754134207514085
58 1 2.937337496203416 6.754134207514085
59 1 2.937337496203416 6.754134207514085
60 1 2.937337496203416 6.754134207514085
61 1 2.937337496203416 6.754134207514085
62 1 2.937337496203416 6.754134207514085
63 1 2.937337496203416 6.754134207514085
64 1 2.937337496203416 6.754134207514085
65 1 2.937337496203416 6.754134207514085
66 1 2.937337496203416 6.754134207514085
67 1 2.937337496203416 6.754134207514085
68 1 2.937337496203416 6.754134207514085
69 1 2.937337496203416 6.754134207514085
70 1 2.937337496203416 6.754134207514085
71 1 2.937337496203416 6.754134207514085
72 1 2.937337496203416 6.754134207514085
73 1 2.937337496203416 6.754134207514085
74 1 2.937337496203416 6.754134207514085
75 1 2.937337496203416 6.754134207514085
76 1 2.937337496203416 6.754134207514085
77 1 2.937337496203416 6.754134207514085
78 1 2.937337496203416 6.754134207514085
79 1 2.937337496203416 6.754134207514085
80 1 2.937337496203416 6.754134207514085
81 1 2.937337496203416 6.754134207514085
82 1 2.937337496203416 6.754134207514085
83 1 2.937337496203416 6.754134207514085
84 1 2.937337496203416 6.754134207514085
85 1 2.937337496203416 6.754134207514085
86 1 2.937337496203416 6.754134207514085
87 1 2.937337496203416 6.754134207514085
88 1 2.937337496203416 6.754134207514085
89 1 2.937337496203416 6.754134207514085
90 1 2.937337496203416 6.754134207514085
91 1 2.937337496203416 6.754134207514085
92 1 2.937337496203416 6.754134207514085
93 1 2.937337496203416 6.754134207514085
94 1 2.937337496203416 6.754134207514085
95 1 2.937337496203416 6.754134207514085
96 1 2.937337496203416 6.754134207514085
97 1 2.937337496203416 6.754134207514085
98 1 2.937337496203416 6.754134207514085
99 1 2.937337496203416 6.754134207514085
100 1 2.937337496203416 6.754134207514085
101 1 2.937337496203416 6.754134207514085
102 1 2.937337496203416 6.754134207514085
103 1 2.937337496203416 6.754134207514085
104 1 2.937337496203416 6.754134207514085
105 1 2.937337496203416 6.754134207514085
106 1 2.937337496203416 6.754134207514085
107 1 2.937337496203416 6.754134207514085
108 1 2.937337496203416 6.754134207514085
109 1 2.937337496203416 6.754134207514085
110 1 2.937337496203416 6.754134207514085
111 1 2.937337496203416 6.754134207514085
112 1 2.937337496203416 6.754134207514085
113 1 2.937337496203416 6.754134207514085
114 1 2.937337496203416 6.754134207514085
115 1 2.937337496203416 6.754134207514085
116 1 2.937337496203416 6.754134207514085
117 1 2.937337496203416 6.754134207514085
118 1 2.937337496203416 6.754134207514085
119 1 2.937337496203416 6.754134207514085
120 1 2.937337496203416 6.754134207514085
121 1 2.937337496203416 6.754134207514085
122 1 2.937337496203416 6.754134207514085
123 1 2.937337496203416 6.754134207514085
124 1 2.937337496203416 6.754134207514085
125 1 2.937337496203416 6.754134207514085
126 1 2.937337496203416 6.754134207514085
127 1 2.937337496203416 6.754134207514085
128 1 2.937337496203416 6.754134207514085
129 1 2.937337496203416 6.754134207514085
130 1 2.937337496203416 6.754134207514085
131 1 2.937337496203416 6.754134207514085
132 1 2.937337496203416 6.754134207514085
133 1 2.937337496203416 6.754134207514085
134 1 2.937337496203416 6.754134207514085
135 1 2.937337496203416 6.754134207514085
136 1 2.937337496203416 6.754134207514085
137 1 2.937337496203416 6.754134207514085
138 1 2.937337496203416 6.754134207514085
139 1 2.937337496203416 6.754134207514085
140 1 2.937337496203416 6.754134207514085
141 1 2.937337496203416 6.754134207514085
142 1 2.937337496203416 6.754134207514085
143 1 2.937337496203416 6.754134207514085
144 1 2.937337496203416 6.754134207514085
145 1 2.937337496203416 6.754134207514085
146 1 2.937337496203416 6.754134207514085
147 1 2.937337496203416 6.754134207514085
148 1 2.937337496203416 6.754134207514085
149 1 2.937337496203416 6.754134207514085
150 1 2.937337496203416 6.754134207514085
151 1 2.937337496203416 6.754134207514085
152 1 2.937337496203416 6.754134207514085
153 1 2.937337496203416 6.754134207514085
154 1 2.937337496203416 6.754134207514085
155 1 2.937337496203416 6.754134207514085
156 1 2.937337496203416 6.754134207514085
157 1 2.937337496203416 6.754134207514085
158 1 2.937337496203416 6.754134207514085
159 1 2.937337496203416 6.754134207514085
160 1 2.937337496203416 6.754134207514085
161 1 2.937337496203416 6.754134207514085
162 1 2.937337496203416 6.754134207514085
163 1 2.937337496203416 6.754134207514085
164 1 2.937337496203416 6.754134207514085
165 1 2.937337496203416 6.754134207514085
166 1 2.937337496203416 6.754134207514085
167 1 2.937337496203416 6.754134207514085
168 1 2.937337496203416 6.754134207514085
169 1 2.937337496203416 6.754134207514085
170 1 2.937337496203416 6.754134207514085
171 1 2.937337496203416 6.754134207514085
172 1 2.937337496203416 6.754134207514085
173 1 2.937337496203416 6.754134207514085
174 1 2.937337496203416 6.754134207514085
175 1 2.937337496203416 6.754134207514085
176 1 2.937337496203416 6.754134207514085
177 1 2.937337496203416 6.754134207514085
178 1 2.937337496203416 6.754134207514085
179 1 2.937337496203416 6.754134207514085
180 1 2.937337496203416 6.754134207514085
181 1 2.937337496203416 6.754134207514085
182 1 2.937337496203416 6.754134207514085
183 1 2.937337496203416 6.754134207514085
184 1 2.937337496203416 6.754134207514085
185 1 2.937337496203416 6.754134207514085
186 1 2.937337496203416 6.754134207514085
187 1 2.937337496203416 6.754134207514085
188 1 2.937337496203416 6.754134207514085
189 1 2.937337496203416 6.754134207514085
190 1 2.937337496203416 6.754134207514085
191 1 2.937337496203416 6.754134207514085
192 1 2.937337496203416 6.754134207514085
193 1 2.937337496203416 6.754134207514085
194 1 2.937337496203416 6.754134207514085
195 1 2.937337496203416 6.754134207514085
196 1 2.937337496203416 6.754134207514085
197 1 2.937337496203416 6.754134207514085
198 1 2.937337496203416 6.754134207514085
199 1 2.937337496203416 6.754134207514085
200 1 2.937337496203416 6.754134207514085
201 1 2.937337496203416 6.754134207514085