11.08., 9:00 - 11:00: Due to updates GitLab will be unavailable for some minutes between 09:00 and 11:00.

Commit 7c0e3d37 authored by Daniel Lehmberg's avatar Daniel Lehmberg

Merge branch 'master' into generalize_source

# Conflicts:
#	Scenarios/OptimizationTests/TestNelderMead/scenarios/rimea_11_exit_selection_nelder_mead.scenario
parents 7bb473dc f604af19
Pipeline #219243 passed with stages
in 129 minutes and 7 seconds
......@@ -42,6 +42,25 @@ stages:
# 2. Variables from "variables" section are not expanded correctly in "url"
# section of an "environment".
# WATCH OUT:
#
# We use "scp -o StrictHostKeyChecking=no" to transfer data to
# "webdev02-mwn.lrz.de"! Not checking the host key allows man-in-the-middle
# attacks!
#
# Unfortunately, this workaround is necessary since LRZ moved Vadere's webspace
# from "webdev-mwn.lrz.de" to "webdev02-mwn.lrz.de". Since then, scp sometimes
# returns with following error: "host verification failed".
# I.e., the key in CI worker's "~/.ssh/known_hosts" and returned host key from
# "webdev02-mwn.lrz.de" do not match.
#
# Even deleting "~/.ssh/known_hosts" and re-adding "webdev02-mwn.lrz.de"'s key
# does not work via "ssh-keyscan -H webdev02-mwn.lrz.de > ~/.ssh/known_hosts".
#
# Maybe, worker's SSH version (OpenSSH_7.6p1) and host's SSH versions
# (OpenSSH_7.9p1) or LRZ has a severe problem because their servers always
# return different / unrelieable SSH keys!
# Job Template Definitions
.template_unit_tests:
stage: unit_test
......@@ -79,9 +98,9 @@ stages:
- mvn clean
- mvn -Dmaven.test.skip=true package
- python3 -m zipfile -c ${VADERE_PACKAGE_NAME_BRANCHES} CHANGELOG.md README.md Scenarios/ VadereGui/target/vadere-gui.jar VadereSimulator/target/vadere-console.jar
- scp ${VADERE_PACKAGE_NAME_BRANCHES} di49mur@webdev-mwn.lrz.de:~/webserver/htdocs/builds/master/${VADERE_PACKAGE_NAME_BRANCHES}
- scp -o StrictHostKeyChecking=no ${VADERE_PACKAGE_NAME_BRANCHES} di49mur@webdev02-mwn.lrz.de:./webserver/htdocs/builds/master/${VADERE_PACKAGE_NAME_BRANCHES}
- python3 -m zipfile -c ${EIKMESH_PACKAGE_NAME_BRANCHES} VadereMeshing/README.md VadereMeshing/target/meshing-0.1-SNAPSHOT.jar
- scp ${EIKMESH_PACKAGE_NAME_BRANCHES} di49mur@webdev-mwn.lrz.de:~/webserver/htdocs/builds/master/eikmesh/${EIKMESH_PACKAGE_NAME_BRANCHES}
- scp -o StrictHostKeyChecking=no ${EIKMESH_PACKAGE_NAME_BRANCHES} di49mur@webdev02-mwn.lrz.de:./webserver/htdocs/builds/master/eikmesh/${EIKMESH_PACKAGE_NAME_BRANCHES}
only:
refs:
- master
......@@ -93,7 +112,7 @@ stages:
- mvn clean
- mvn -Dmaven.test.skip=true package
- python3 -m zipfile -c ${VADERE_PACKAGE_NAME_RELEASES} CHANGELOG.md README.md Scenarios/ VadereGui/target/vadere-gui.jar VadereSimulator/target/vadere-console.jar
- scp ${VADERE_PACKAGE_NAME_RELEASES} di49mur@webdev-mwn.lrz.de:~/webserver/htdocs/builds/stable/${VADERE_PACKAGE_NAME_RELEASES}
- scp -o StrictHostKeyChecking=no ${VADERE_PACKAGE_NAME_RELEASES} di49mur@webdev02-mwn.lrz.de:./webserver/htdocs/builds/stable/${VADERE_PACKAGE_NAME_RELEASES}
# Run job only for "tags" (usually on the "stable" branch). Watch out: Multiple "only" tags are ORed together.
# See: https://gitlab.com/gitlab-org/gitlab-ce/issues/27818
only:
......@@ -106,7 +125,7 @@ stages:
- mvn clean
- mvn -Dmaven.test.skip=true package
- python3 -m zipfile -c ${VADERE_PACKAGE_NAME_BRANCHES} CHANGELOG.md README.md Scenarios/ VadereGui/target/vadere-gui.jar VadereSimulator/target/vadere-console.jar
- scp ${VADERE_PACKAGE_NAME_BRANCHES} di49mur@webdev-mwn.lrz.de:~/webserver/htdocs/builds/branches/${VADERE_PACKAGE_NAME_BRANCHES}
- scp -o StrictHostKeyChecking=no ${VADERE_PACKAGE_NAME_BRANCHES} di49mur@webdev02-mwn.lrz.de:./webserver/htdocs/builds/branches/${VADERE_PACKAGE_NAME_BRANCHES}
when: manual
.template_seed_test:
......
......@@ -12,14 +12,14 @@
"expectedResult" : "SUCCESS",
"optimizationMetricProcessorId" : 2,
"testEvacuationProcessorId" : 3,
"maxMeanPointDistance" : 0.07032687521241791,
"maxMeanDifferenceFuncValue" : 0.03480114233444013,
"maxMeanPointDistance" : 0.0741743541679023,
"maxMeanDifferenceFuncValue" : 0.03977588209021506,
"infoMinPointDistanceL2" : 0.0,
"infoMaxPointDistanceL2" : 1.3986496595476534,
"infoMinFuncDifference" : -0.010149427397823985,
"infoMaxFuncDifference" : 2.563180447354023,
"infoStddevPointDistance" : 0.12029822051457525,
"infoStddevDifferenceFuncValue" : 0.11224623990550493
"infoMaxPointDistanceL2" : 1.116456695995013,
"infoMinFuncDifference" : -0.014172405140369548,
"infoMaxFuncDifference" : 3.285814543687535,
"infoStddevPointDistance" : 0.12307022389816641,
"infoStddevDifferenceFuncValue" : 0.13840077341363527
}
}, {
"type" : "org.vadere.simulator.projects.dataprocessing.processor.PedestrianMetricOptimizationProcessor",
......@@ -234,15 +234,15 @@
"sources" : [ {
"id" : 2,
"shape" : {
"x" : 2.5,
"y" : 8.0,
"width" : 3.95,
"height" : 4.74,
"x" : 0.6,
"y" : 6.4,
"width" : 10.0,
"height" : 8.0,
"type" : "RECTANGLE"
},
"interSpawnTimeDistribution" : "org.vadere.state.scenario.ConstantDistribution",
"distributionParameters" : [ 1.0 ],
"spawnNumber" : 100,
"spawnNumber" : 70,
"maxSpawnNumberTotal" : -1,
"startTime" : 0.0,
"endTime" : 0.0,
......
Density.measurementRadius = 15
Density.measurementScale = 10.0
Density.standardDeviation = 0.5
Gui.dataProcessingViewMode = json
Gui.toolbar.size = 40
Messages.language = en
SettingsDialog.dataFormat = yyyy_MM_dd_HH_mm_ss
PostVis.maxNumberOfSaveDirectories = 5
Pedestrian.radius = 0.195
PostVis.SVGHeight = 768
PostVis.SVGWidth = 1024
PostVis.cellWidth = 1.0
Gui.lastSavePoint =
ProjectView.maxCellWidth = 10.0
Density.measurementRadius = 15
PostVis.minCellWidth = 0.01
Testing.stepCircleOptimization.compareBruteForceSolution = true
PostVis.enableJsonInformationPanel = true
PostVis.framesPerSecond = 5
PostVis.maxCellWidth = 10.0
PostVis.maxFramePerSecond = 30
PostVis.maxNumberOfSaveDirectories = 5
PostVis.minCellWidth = 0.01
PostVis.framesPerSecond = 20
Gui.toolbar.size = 40
Messages.language = en
ProjectView.icon.width.value = 35
ProjectView.cellWidth = 1.0
ProjectView.icon.height.value = 35
ProjectView.icon.width.value = 35
ProjectView.maxCellWidth = 10.0
ProjectView.minCellWidth = 0.01
SettingsDialog.dataFormat = yyyy_MM_dd_HH_mm_ss
SettingsDialog.maxNumberOfTargets = 10
SettingsDialog.outputDirectory.path = .
PostVis.SVGHeight = 768
ProjectView.defaultDirectoryAttributes =
TopographyCreator.dotRadius = 0.5
History.lastUsedProject =
History.recentProjects =
SettingsDialog.showLogo = false
Gui.dataProcessingViewMode = json
SettingsDialog.outputDirectory.path = .
SettingsDialog.snapshotDirectory.path = .
Testing.stepCircleOptimization.compareBruteForceSolution = true
TopographyCreator.dotRadius = 0.5
PostVis.SVGWidth = 1024
ProjectView.defaultDirectory =
ProjectView.defaultDirectoryScenarios =
Density.measurementScale = 10.0
PostVis.maxFramePerSecond = 40
ProjectView.minCellWidth = 0.01
PostVis.cellWidth = 1.0
ProjectView.defaultDirectoryOutputProcessors =
Gui.showNodes = false
Project.ScenarioChecker.active = false
Vadere.cache.globalCacheBaseDir =
Gui.node.radius = 0.3
Project.simulationResult.show = true
Vadere.cache.useGlobalCacheBaseDir = false
PostVis.timeResolution = 0.4
......@@ -69,6 +69,19 @@ cli2.startScenario(scenarioPath)
embed()
```
```python
if __name__ == "__main__":
c = Py4jClient.create(
project_path="VADERE_PATH/Scenarios/Demos/roVer/scenarios",
vadere_base_path="VADERE_PATH",
start_server=False,
debug=False,
)
c.entrypoint_jar = "/VADERE_PATH/Tools/PythonTraciWrapper/vadere-traci-entrypoint.jar"
c.connect()
c.startScenario("roVerTest002.scenario")
```
## Extend API
If new TraCI commands are implemented in vadere, these can also be used via a python client. To achieve this the following steps are required:
......
......@@ -46,6 +46,7 @@ class Py4jClient:
self.server_log = server_log
self.entrypoint_log = entry_point_log
self.debug = debug
self.entrypoint_jar = ''
self._build_argparser()
if isinstance(args, List):
......@@ -53,6 +54,7 @@ class Py4jClient:
else:
self._parser.parse_args(["--help"])
def connect(self):
self._start_entrypoint()
self._connect_gateway()
......@@ -111,12 +113,18 @@ class Py4jClient:
def _start_entrypoint(self):
if self.start_server:
if self.entrypoint_jar == '':
jar_path = os.path.join(
self.vadere_base_path,
"VadereManager/target/vadere-server.jar",
),
else:
jar_path = self.entrypoint_jar
vadere_server_cmd = [
"java",
"-jar",
os.path.join(
self.vadere_base_path, "VadereManager/target/vadere-server.jar",
),
jar_path,
]
vadere_server_cmd.extend(self._server_args())
self.server_thread = Runner(
......@@ -132,13 +140,18 @@ class Py4jClient:
self.server_thread = None
print("Connecting to existing Server...")
entrypoint_cmd = [
"java",
"-jar",
os.path.join(
if self.entrypoint_jar == '':
jar_path = os.path.join(
self.vadere_base_path,
"VadereManager/target/vadere-traci-entrypoint.jar",
),
else:
jar_path = self.entrypoint_jar
entrypoint_cmd = [
"java",
"-jar",
jar_path,
]
entrypoint_cmd.extend(self._entrypoint_args())
self.entrypoint_thread = Runner(
......@@ -252,13 +265,3 @@ class Py4jClient:
dest="defaultScenario",
help="Supply a default scenario",
)
if __name__ == "__main__":
c = Py4jClient.create(
project_path="/path/to/vadere/scenarios/",
vadere_base_path="/path/to/vadere",
start_server=False,
debug=False,
)
c.startScenario("Test001.scenario")
.idea
__pycache__
*.pyc
.mypy_cache/
*.csv
......@@ -9,8 +10,11 @@ log.out
# do not track Vadere outputs (folders and files and environments), every computer as different paths, which results in changes for every commit.
vadere_scenarios
*.scenarios
!tutorial/example.scenario
*.scenario
*.project
*.json
# will be created automatically:
suqc/suqc_envs/
......@@ -18,14 +22,13 @@ suqc/suqc_envs/
*.jar
# Configuration files (generated when importing the project)
suq_config.json
suq_controller.conf
update_server.sh
# pickle files
*.p
# Ignore virtual environment
venv
......@@ -36,5 +39,5 @@ suqc.egg-info
# tutorial output
/tutorial/example_output/
/tutorial/*/
/tutorial/example_multirun_output/
......@@ -9,35 +9,16 @@ git subtree pull --prefix Tools/SUQController git@gitlab.lrz.de:vadere/suq-contr
### WORK IN PROGRESS
The suq-controller connects the modules "Surrogate Model" (S) and "Uncertainty Quantification" (UQ) (see other vadere Repos).
The main functionality of the `suq-controller` is to query many differently parametrized VADERE scenarios and
return the result of specified quantity of interests (QoI) in a convenient format ([pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)).
This git repository uses git large file storage (git-lfs). This allows to ship default VADERE models (larger .jar files.)
with the git repository.
For developers: To install git-lfs follow the instructions [here](https://github.com/git-lfs/git-lfs/wiki/Installation)
In file `.gitattributes` in the repo shows the settings for git-lfs.
### Glossary
Other words were used in this project to not confuse terminology with VADERE (such as `scenario` and `project`).
* **container** is the parent folder of (multiple) environments
* **environment** is folder consisting of a specifed VADERE scenario that is intended to query
* **request** is an user request for a quantity of interest for the specific VADERE setting with the given the scenario
set in the environment. A query can simulate VADERE for multiple scenario settings for the parameter variation
(such as a full grid sampling).
The suq-controller connects the modules "Surrogate Model" (S) and "Uncertainty
Quantification" (UQ) (see other Vadere group repos).
The main functionality of the `suq-controller` is to sample parameters from Vadere
and return the result of specified quantity of interests (QoI) in a convenient format
([pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)).
## Getting started
Either install as a Python package or run the source code directly. Either way it is recommended to use Python>=3.6
### Using the code
#### Use source directly:
......@@ -47,7 +28,7 @@ Run the code from the source directly (without install), please check if you mee
To install as a package `suqc` run
```
python3 setup.py install
python setup.py install
```
from the command line. (Note: In Linux this may have to be executed with `sudo`).
......@@ -55,7 +36,7 @@ from the command line. (Note: In Linux this may have to be executed with `sudo`)
Test if the installed package was installed successfully by running:
```
python3 -c "import suqc; print(suqc.__version__)"
python -c "import suqc; print(suqc.__version__)"
```
This command should print the installed version number (and potentially infos to set up required folder) in the terminal. In case an error was thrown the package is
......
matplotlib==2.2.3
setuptools==39.1.0
setuptools>=39.1.0
fabric==2.4.0
pandas==0.23.4
numpy==1.15.4
scikit-learn==0.20.2
pandas>=0.24.0,<1.0.0
numpy>=1.17.0
scikit-learn>=0.21.0
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
import os
from setuptools import setup, find_packages
......@@ -9,13 +7,6 @@ from setuptools import setup, find_packages
from suqc.configuration import SuqcConfig
from suqc import __version__
# --------------------------------------------------
# people who contributed code
__authors__ = "Daniel Lehmberg"
# people who made suggestions or reported bugs but didn't contribute code
__credits__ = ["n/a"]
# --------------------------------------------------
# To generate a new requirements.txt file run in console (install vis 'pip3 install pipreqs'):
# pipreqs --use-local --force /home/daniel/REPOS/suq-controller
......@@ -40,4 +31,4 @@ setup(
)
os.remove(SuqcConfig.path_package_indicator_file())
assert not os.path.exists(SuqcConfig.path_package_indicator_file())
\ No newline at end of file
assert not os.path.exists(SuqcConfig.path_package_indicator_file())
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
# include imports after here:
# --------------------------------------------------
# people who contributed code
__authors__ = "Daniel Lehmberg"
# people who made suggestions or reported bugs but didn't contribute code
__credits__ = ["n/a"]
# --------------------------------------------------
from suqc.parameter.sampling import *
from suqc.parameter.postchanges import ScenarioChanges
from suqc.parameter.postchanges import PostScenarioChangesBase
from suqc.qoi import *
from suqc.request import *
__version__ = "1.01"
__version__ = "2.0"
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
import os
import json
import os.path as p
import pathlib
# --------------------------------------------------
# people who contributed code
__authors__ = "Daniel Lehmberg"
# people who made suggestions or reported bugs but didn't contribute code
__credits__ = ["n/a"]
# --------------------------------------------------
# configuration of the suq-controller
DEFAULT_SUQ_CONFIG = {"default_vadere_src_path": "TODO", # TODO Feature: compile Vadere before using the jar file
"server": {
"host": "",
"user": "",
"port": -1
}}
DEFAULT_SUQC_CONFIG = {"default_vadere_src_path": "TODO", # TODO Feature: #25
"server": {
"host": "",
"user": "",
"port": -1}}
def check_setup(_paths_class):
if not os.path.exists(_paths_class.path_cfg_folder()) and _paths_class.is_package_paths():
print(f"INFO: Setting up configuration folder {_paths_class.path_cfg_folder()}")
os.mkdir(
_paths_class.path_cfg_folder()) # the next two checks will fail automatically too, because the folder is empty
if not os.path.exists(_paths_class.path_models_folder()):
print(f"INFO: Setting up default model lookup folder {_paths_class.path_models_folder()}")
os.mkdir(_paths_class.path_models_folder())
# the next two checks will fail automatically too, because the folder is empty
os.mkdir(_paths_class.path_cfg_folder())
if not os.path.exists(_paths_class.path_suq_config_file()):
print(f"INFO: Setting up default configuration file located at {_paths_class.path_suq_config_file()}")
_paths_class.store_config(DEFAULT_SUQ_CONFIG)
_paths_class.store_config(DEFAULT_SUQC_CONFIG)
if not os.path.exists(_paths_class.path_container_folder()):
print(f"INFO: Setting up the default container path (which will store output of simulation runs). "
......@@ -47,12 +33,12 @@ def check_setup(_paths_class):
return _paths_class
# class annotation -> everythime the clsas is used, it will be checked if the folders are correctly configured
# class annotation -> everythime the class is used, it will be checked if the folders are correctly configured
@check_setup
class SuqcConfig(object):
NAME_PACKAGE = "suqc"
NAME_SUQ_CONFIG_FILE = "suq_config.json"
NAME_SUQ_CONFIG_FILE = "suq_controller.conf"
NAME_MODELS_FOLDER = "models"
NAME_CON_FOLDER = "suqc_envs"
NAME_PACKAGE_FILE = "PACKAGE.txt"
......@@ -73,7 +59,7 @@ class SuqcConfig(object):
@classmethod
def _name_cfg_folder(cls):
if cls.is_package_paths():
return ".suqc"
return ".config"
else:
raise RuntimeError("This should not be called when IS_PACKAGE=False.")
......@@ -92,7 +78,11 @@ class SuqcConfig(object):
@classmethod
def path_cfg_folder(cls):
if cls.is_package_paths():
return p.join(cls.path_usrhome_folder(), cls._name_cfg_folder())
dir2conf = p.join(cls.path_usrhome_folder(), cls._name_cfg_folder())
if not p.exists(dir2conf):
# TODO instead of raising error the directory can also be created.
raise ValueError(f"Directory {dir2conf} does not exist.")
return dir2conf
else:
return cls.path_src_folder()
......
This diff is collapsed.
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
#!/usr/bin/env python3
import multiprocessing
import warnings
import suqc.request # no "from suqc.request import ..." works because of circular imports
from suqc.environment import EnvironmentManager
from suqc.parameter.sampling import ParameterVariation
from suqc.parameter.postchanges import ScenarioChanges
from suqc.utils.general import create_folder, remove_folder, njobs_check_and_set
from suqc.utils.dict_utils import change_dict
# --------------------------------------------------
# people who contributed code
__authors__ = "Daniel Lehmberg"
# people who made suggestions or reported bugs but didn't contribute code
__credits__ = ["n/a"]
# --------------------------------------------------
from suqc.parameter.postchanges import PostScenarioChangesBase
from suqc.parameter.sampling import ParameterVariationBase
from suqc.utils.dict_utils import change_dict, deep_dict_lookup
from suqc.utils.general import create_folder, njobs_check_and_set, remove_folder
class VadereScenarioCreation(object):
def __init__(self, env_man: EnvironmentManager, par_var: ParameterVariation, sc_change: ScenarioChanges=None):
def __init__(self, env_man: EnvironmentManager, parameter_variation: ParameterVariationBase,
post_change: PostScenarioChangesBase = None):
self._env_man = env_man
self._par_var = par_var
self._sc_change = sc_change
self._parameter_variation = parameter_variation
self._post_changes = post_change
self._basis_scenario = self._env_man.basis_scenario
self._par_var.check_selected_keys(self._basis_scenario)
def _vars_dict(self, pid, fp, op):
return {"par_id": pid, "scenario_path": fp, "output_path": op}
self._parameter_variation.check_selected_keys(self._basis_scenario)
def _create_new_vadere_scenario(self, scenario: dict, par_id: int, par_var: dict):
def _create_new_vadere_scenario(self, scenario: dict, parameter_id: int, run_id: int, parameter_variation: dict):
par_var_scenario = change_dict(scenario, changes=par_var)
par_var_scenario = change_dict(scenario, changes=parameter_variation)
if self._sc_change is not None:
if self._post_changes is not None:
# Apply pre-defined changes to each scenario file
final_scenario = self._sc_change.change_scenario(scenario=par_var_scenario, par_id=par_id, par_var=par_var)
final_scenario = self._post_changes.change_scenario(scenario=par_var_scenario,
parameter_id=parameter_id,
run_id=run_id,
parameter_variation=parameter_variation)
else:
final_scenario = par_var_scenario
return final_scenario
def _save_vadere_scenario(self, par_id, s):
fp = self._env_man.save_scenario_variation(par_id, s)
def _print_scenario_warnings(self, scenario):
try:
real_time_sim_time_ratio, _ = deep_dict_lookup(scenario, "realTimeSimTimeRatio")
except Exception:
real_time_sim_time_ratio = 0 # ignore this warning if the lookup failed for whatever reason.
if real_time_sim_time_ratio > 1E-14:
warnings.warn(f"In a scenario the key 'realTimeSimTimeRatio={real_time_sim_time_ratio}'. Large values "
f"slow down the evaluation speed.")
def _save_vadere_scenario(self, parameter_id, run_id, scenario):
self._print_scenario_warnings(scenario)
fp = self._env_man.save_scenario_variation(parameter_id, run_id, scenario)
return fp
def _create_scenario(self, args): # TODO: how do multiple arguments work for pool.map functions? (see below)
"""Set up a new scenario and return info of parameter id and location."""
par_id = args[0] # TODO: this would kind of reduce this ugly code
par_change = args[1]
parameter_id = args[0] # TODO: this would kind of reduce this ugly code
run_id = args[1]
parameter_variation = args[2]
new_scenario = self._create_new_vadere_scenario(self._basis_scenario, par_id, par_change)
new_scenario = self._create_new_vadere_scenario(self._basis_scenario, parameter_id, run_id, parameter_variation)
output_path = self._env_man.get_par_id_output_path(par_id, create=False)
scenario_path = self._save_vadere_scenario(par_id, new_scenario)
output_folder = self._env_man.get_variation_output_folder(parameter_id, run_id)
scenario_path = self._save_vadere_scenario(parameter_id, run_id, new_scenario)
return self._vars_dict(par_id, scenario_path, output_path)
result_item = suqc.request.RequestItem(parameter_id=parameter_id,
run_id=run_id,
scenario_path=scenario_path,
base_path=self._env_man.base_path,
output_folder=output_folder)
return result_item
def _sp_creation(self):
"""Single process loop to create all requested scenarios."""
basis_scenario = self._env_man.basis_scenario
self._par_var.check_selected_keys(basis_scenario)
vars_ = list()
for par_id, par_change in self._par_var.par_iter():
vars_.append(self._create_scenario([par_id, par_change]))
return vars_
request_item_list = list()
for par_id, run_id, par_change in self._parameter_variation.par_iter():
request_item_list.append(self._create_scenario([par_id, run_id, par_change]))
return request_item_list
def _mp_creation(self, njobs):
"""Multi process function to create all requested scenarios."""
pool = multiprocessing.Pool(processes=njobs)
vars_ = pool.map(self._create_scenario, self._par_var.par_iter())
return vars_
request_item_list = pool.map(self._create_scenario, self._parameter_variation.par_iter())
return request_item_list
def _adapt_nr_digits_env_man(self, nr_variations, nr_runs):
self._env_man.nr_digits_variation = len(str(nr_variations))
self._env_man.nr_digits_runs = len(str(nr_runs))
def generate_vadere_scenarios(self, njobs):
njobs = njobs_check_and_set(njobs=njobs, ntasks=self._par_var.points.shape[0])
ntasks = self._parameter_variation.points.shape[0]
njobs = njobs_check_and_set(njobs=njobs, ntasks=ntasks)
# increases readability and promotes shorter paths (apparently lengthy paths can cause problems on Windows)
# see issue #76
self._adapt_nr_digits_env_man(nr_variations=self._parameter_variation.nr_parameter_variations(),
nr_runs=self._parameter_variation.nr_scenario_runs())
target_path = self._env_man.get_env_outputfolder_path()
# For security:
remove_folder(target_path)
create_folder(target_path)
if njobs == 1:
vars_ = self._sp_creation()
request_item_list = self._sp_creation()
else:
vars_ = self._mp_creation(njobs)
request_item_list = self._mp_creation(njobs)
return vars_
return request_item_list
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
#!/usr/bin/env python3
import abc
......@@ -8,20 +6,8 @@ import numpy as np
from suqc.utils.dict_utils import change_dict