Commit eed76a68 authored by Daniel Lehmberg's avatar Daniel Lehmberg
Browse files

Squashed 'Tools/SUQController/' changes from 285c69a13..f75af9422

f75af9422 Merge branch 'add_jvm_parameter' into 'master'
f2106b649 Add jvm parameter
faaf2c962 set version 2.0
7357a144e Merge branch 'develop' into 'master'
1d255f89c fix: errored if output_folder=None
5210fcb45 detail gitignore
5a95837b5 update tutorials
9c65cd608 integrate post_changes into VariationBase
377ce2158 change default value
145b95628 remove models folder
86f652d16 remove folder
5b8ae215b remove folder
c6c44992b Remote (2)
f086d5c0d Remote (1)
8421608b0 removal of remote environment folder in __exit__
21d1815da remote (1)
24c864027 solves #75
a3275bb79 remote (2)
b3d128f2a remote (1)
3e6cb6639 Merge branch 'improve_existing' into 'develop'
b9b4d6284 Improve existing
f8dd4816e remove header templates
1fa592368 order import paths

git-subtree-dir: Tools/SUQController
git-subtree-split: f75af94229264e126e21d8cbf6ee07a159c11556
parent 488d9311
.idea
__pycache__
*.pyc
.mypy_cache/
*.csv
......@@ -9,8 +10,11 @@ log.out
# do not track Vadere outputs (folders and files and environments), every computer as different paths, which results in changes for every commit.
vadere_scenarios
*.scenarios
!tutorial/example.scenario
*.scenario
*.project
*.json
# will be created automatically:
suqc/suqc_envs/
......@@ -18,14 +22,13 @@ suqc/suqc_envs/
*.jar
# Configuration files (generated when importing the project)
suq_config.json
suq_controller.conf
update_server.sh
# pickle files
*.p
# Ignore virtual environment
venv
......@@ -36,5 +39,5 @@ suqc.egg-info
# tutorial output
/tutorial/example_output/
/tutorial/*/
/tutorial/example_multirun_output/
......@@ -9,35 +9,16 @@ git subtree pull --prefix Tools/SUQController git@gitlab.lrz.de:vadere/suq-contr
### WORK IN PROGRESS
The suq-controller connects the modules "Surrogate Model" (S) and "Uncertainty Quantification" (UQ) (see other vadere Repos).
The main functionality of the `suq-controller` is to query many differently parametrized VADERE scenarios and
return the result of specified quantity of interests (QoI) in a convenient format ([pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)).
This git repository uses git large file storage (git-lfs). This allows to ship default VADERE models (larger .jar files.)
with the git repository.
For developers: To install git-lfs follow the instructions [here](https://github.com/git-lfs/git-lfs/wiki/Installation)
In file `.gitattributes` in the repo shows the settings for git-lfs.
### Glossary
Other words were used in this project to not confuse terminology with VADERE (such as `scenario` and `project`).
* **container** is the parent folder of (multiple) environments
* **environment** is folder consisting of a specifed VADERE scenario that is intended to query
* **request** is an user request for a quantity of interest for the specific VADERE setting with the given the scenario
set in the environment. A query can simulate VADERE for multiple scenario settings for the parameter variation
(such as a full grid sampling).
The suq-controller connects the modules "Surrogate Model" (S) and "Uncertainty
Quantification" (UQ) (see other Vadere group repos).
The main functionality of the `suq-controller` is to sample parameters from Vadere
and return the result of specified quantity of interests (QoI) in a convenient format
([pandas DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html)).
## Getting started
Either install as a Python package or run the source code directly. Either way it is recommended to use Python>=3.6
### Using the code
#### Use source directly:
......@@ -47,7 +28,7 @@ Run the code from the source directly (without install), please check if you mee
To install as a package `suqc` run
```
python3 setup.py install
python setup.py install
```
from the command line. (Note: In Linux this may have to be executed with `sudo`).
......@@ -55,7 +36,7 @@ from the command line. (Note: In Linux this may have to be executed with `sudo`)
Test if the installed package was installed successfully by running:
```
python3 -c "import suqc; print(suqc.__version__)"
python -c "import suqc; print(suqc.__version__)"
```
This command should print the installed version number (and potentially infos to set up required folder) in the terminal. In case an error was thrown the package is
......
matplotlib==2.2.3
setuptools==39.1.0
setuptools>=39.1.0
fabric==2.4.0
pandas==0.23.4
numpy==1.15.4
scikit-learn==0.20.2
pandas>=0.24.0,<1.0.0
numpy>=1.17.0
scikit-learn>=0.21.0
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
import os
from setuptools import setup, find_packages
......@@ -9,13 +7,6 @@ from setuptools import setup, find_packages
from suqc.configuration import SuqcConfig
from suqc import __version__
# --------------------------------------------------
# people who contributed code
__authors__ = "Daniel Lehmberg"
# people who made suggestions or reported bugs but didn't contribute code
__credits__ = ["n/a"]
# --------------------------------------------------
# To generate a new requirements.txt file run in console (install vis 'pip3 install pipreqs'):
# pipreqs --use-local --force /home/daniel/REPOS/suq-controller
......@@ -40,4 +31,4 @@ setup(
)
os.remove(SuqcConfig.path_package_indicator_file())
assert not os.path.exists(SuqcConfig.path_package_indicator_file())
\ No newline at end of file
assert not os.path.exists(SuqcConfig.path_package_indicator_file())
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
# include imports after here:
# --------------------------------------------------
# people who contributed code
__authors__ = "Daniel Lehmberg"
# people who made suggestions or reported bugs but didn't contribute code
__credits__ = ["n/a"]
# --------------------------------------------------
from suqc.parameter.sampling import *
from suqc.parameter.postchanges import ScenarioChanges
from suqc.parameter.postchanges import PostScenarioChangesBase
from suqc.qoi import *
from suqc.request import *
__version__ = "1.01"
__version__ = "2.0"
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
import os
import json
import os.path as p
import pathlib
# --------------------------------------------------
# people who contributed code
__authors__ = "Daniel Lehmberg"
# people who made suggestions or reported bugs but didn't contribute code
__credits__ = ["n/a"]
# --------------------------------------------------
# configuration of the suq-controller
DEFAULT_SUQ_CONFIG = {"default_vadere_src_path": "TODO", # TODO Feature: compile Vadere before using the jar file
"server": {
"host": "",
"user": "",
"port": -1
}}
DEFAULT_SUQC_CONFIG = {"default_vadere_src_path": "TODO", # TODO Feature: #25
"server": {
"host": "",
"user": "",
"port": -1}}
def check_setup(_paths_class):
if not os.path.exists(_paths_class.path_cfg_folder()) and _paths_class.is_package_paths():
print(f"INFO: Setting up configuration folder {_paths_class.path_cfg_folder()}")
os.mkdir(
_paths_class.path_cfg_folder()) # the next two checks will fail automatically too, because the folder is empty
if not os.path.exists(_paths_class.path_models_folder()):
print(f"INFO: Setting up default model lookup folder {_paths_class.path_models_folder()}")
os.mkdir(_paths_class.path_models_folder())
# the next two checks will fail automatically too, because the folder is empty
os.mkdir(_paths_class.path_cfg_folder())
if not os.path.exists(_paths_class.path_suq_config_file()):
print(f"INFO: Setting up default configuration file located at {_paths_class.path_suq_config_file()}")
_paths_class.store_config(DEFAULT_SUQ_CONFIG)
_paths_class.store_config(DEFAULT_SUQC_CONFIG)
if not os.path.exists(_paths_class.path_container_folder()):
print(f"INFO: Setting up the default container path (which will store output of simulation runs). "
......@@ -47,12 +33,12 @@ def check_setup(_paths_class):
return _paths_class
# class annotation -> everythime the clsas is used, it will be checked if the folders are correctly configured
# class annotation -> everythime the class is used, it will be checked if the folders are correctly configured
@check_setup
class SuqcConfig(object):
NAME_PACKAGE = "suqc"
NAME_SUQ_CONFIG_FILE = "suq_config.json"
NAME_SUQ_CONFIG_FILE = "suq_controller.conf"
NAME_MODELS_FOLDER = "models"
NAME_CON_FOLDER = "suqc_envs"
NAME_PACKAGE_FILE = "PACKAGE.txt"
......@@ -73,7 +59,7 @@ class SuqcConfig(object):
@classmethod
def _name_cfg_folder(cls):
if cls.is_package_paths():
return ".suqc"
return ".config"
else:
raise RuntimeError("This should not be called when IS_PACKAGE=False.")
......@@ -92,7 +78,11 @@ class SuqcConfig(object):
@classmethod
def path_cfg_folder(cls):
if cls.is_package_paths():
return p.join(cls.path_usrhome_folder(), cls._name_cfg_folder())
dir2conf = p.join(cls.path_usrhome_folder(), cls._name_cfg_folder())
if not p.exists(dir2conf):
# TODO instead of raising error the directory can also be created.
raise ValueError(f"Directory {dir2conf} does not exist.")
return dir2conf
else:
return cls.path_src_folder()
......
#!/usr/bin/env python3
# TODO: """ << INCLUDE DOCSTRING (one-line or multi-line) >> """
import json
import glob
import subprocess
......@@ -12,17 +10,11 @@ from shutil import rmtree
from typing import *
from suqc.configuration import SuqcConfig
from suqc.utils.general import user_query_yes_no, get_current_suqc_state, create_folder
from suqc.utils.general import user_query_yes_no, get_current_suqc_state, str_timestamp
# --------------------------------------------------
# people who contributed code
__authors__ = "Daniel Lehmberg"
# people who made suggestions or reported bugs but didn't contribute code
__credits__ = ["n/a"]
# --------------------------------------------------
# configuration of the suq-controller
DEFAULT_SUQ_CONFIG = {"default_vadere_src_path": "TODO", # TODO Feature: compile Vadere before using the jar file
DEFAULT_SUQ_CONFIG = {"default_vadere_src_path": "TODO",
"server": {
"host": "",
"user": "",
......@@ -30,40 +22,73 @@ DEFAULT_SUQ_CONFIG = {"default_vadere_src_path": "TODO", # TODO Feature: compi
}}
@DeprecationWarning
def get_suq_config():
assert os.path.exists(SuqcConfig.path_suq_config_file()), "Config file does not exist."
with open(SuqcConfig.path_suq_config_file(), "r") as f:
config_file = f.read()
return json.loads(config_file)
class VadereConsoleWrapper(object):
# Current log level choices, requires to manually add, if there are changes
ALLOWED_LOGLVL = ["OFF", "FATAL", "TOPOGRAPHY_ERROR", "TOPOGRAPHY_WARN", "INFO", "DEBUG", "ALL"]
def __init__(self, model_path: str, loglvl="ALL"):
def __init__(self, model_path: str, loglvl="INFO", jvm_flags: Optional[List] =
None, timeout_sec=None):
self.jar_path = os.path.abspath(model_path)
assert os.path.exists(self.jar_path)
self.loglvl = loglvl
if not os.path.exists(self.jar_path):
raise FileNotFoundError(
f"Vadere console .jar file {self.jar_path} does not exist.")
assert self.loglvl in self.ALLOWED_LOGLVL, f"set loglvl={self.loglvl} not contained in allowed: " \
f"{self.ALLOWED_LOGLVL}"
loglvl = loglvl.upper()
if loglvl not in self.ALLOWED_LOGLVL:
raise ValueError(f"set loglvl={loglvl} not contained "
f"in allowed: {self.ALLOWED_LOGLVL}")
if not os.path.exists(self.jar_path):
raise FileExistsError(f"Vadere console file {self.jar_path} does not exist.")
if jvm_flags is not None and not isinstance(jvm_flags, list):
raise TypeError(
f"jvm_flags are required to be a list. Got: {type(jvm_flags)}")
if timeout_sec is None:
pass # do nothing, no timeout
elif not isinstance(timeout_sec, int) or timeout_sec <= 0:
raise TypeError("vadere_run_timeout_sec must be of type int and positive "
"value")
self.loglvl = loglvl
# Additional Java Virtual Machine options / flags
self.jvm_flags = jvm_flags if jvm_flags is not None else []
self.timeout_sec = timeout_sec
def run_simulation(self, scenario_fp, output_path):
start = time.time()
ret_val = subprocess.call(["java", "-jar",
self.jar_path, "--loglevel",
self.loglvl, "suq", "-f", scenario_fp,
"-o", output_path])
return ret_val, time.time() - start
subprocess_cmd = ["java"]
subprocess_cmd += self.jvm_flags
subprocess_cmd += ["-jar", self.jar_path]
# Vadere console commands
subprocess_cmd += ["--loglevel", self.loglvl]
subprocess_cmd += ["suq", "-f", scenario_fp, "-o", output_path]
output_subprocess = dict()
try:
subprocess.check_output(subprocess_cmd,
timeout=self.timeout_sec,
stderr=subprocess.PIPE)
process_duration = time.time() - start
# if return_code != 0 a subprocess.CalledProcessError is raised
return_code = 0
output_subprocess = None
except subprocess.TimeoutExpired as exception:
return_code = 1
process_duration = self.timeout_sec
output_subprocess["stdout"] = exception.stdout
output_subprocess["stderr"] = None
except subprocess.CalledProcessError as exception:
return_code = exception.returncode
process_duration = time.time() - start
output_subprocess["stdout"] = exception.stdout
output_subprocess["stderr"] = exception.stderr
return return_code, process_duration, output_subprocess
@classmethod
def from_default_models(cls, model):
......@@ -75,10 +100,6 @@ class VadereConsoleWrapper(object):
def from_model_path(cls, model_path):
return cls(model_path)
@classmethod
def from_new_compiled_package(cls, src_path=None):
pass # TODO: use default src_path
@classmethod
def infer_model(cls, model):
if isinstance(model, str):
......@@ -89,23 +110,33 @@ class VadereConsoleWrapper(object):
elif isinstance(model, VadereConsoleWrapper):
return model
else:
raise ValueError("Failed to infer the Vadere model.")
raise ValueError(f"Failed to infer Vadere model. \n {model}")
class EnvironmentManager(object):
output_folder = "vadere_output"
PREFIX_BASIS_SCENARIO = "BASIS_"
VADERE_SCENARIO_FILE_TYPE = ".scenario"
vadere_output_folder = "vadere_output"
def __init__(self, base_path, env_name: str):
self.base_path, self.env_name = self.handle_path_and_env_input(base_path, env_name)
def __init__(self, env_name: str):
self.env_name = env_name
self.env_path = self.output_folder_path(self.base_path, self.env_name)
self.name = env_name
self.env_path = self.environment_path(self.name)
# output is usually of the following format:
# 000001_000002 for variation 1 and run_id 2
# Change these attributes externally, if less digits are required to have
# shorter/longer paths.
self.nr_digits_variation = 6
self.nr_digits_runs = 6
print(f"INFO: Set environment path to {self.env_path}")
if not os.path.exists(self.env_path):
raise FileNotFoundError(f"Environment {self.env_path} does not exist. Use function "
f"'EnvironmentManager.create_environment' or "
f"'EnvironmentManager.create_if_not_exist'")
f"'EnvironmentManager.create_new_environment'")
self._scenario_basis = None
@property
......@@ -121,47 +152,84 @@ class EnvironmentManager(object):
@property
def path_basis_scenario(self):
sc_files = glob.glob(os.path.join(self.env_path, "*.scenario"))
assert len(sc_files) == 1, "None or too many .scenario files found in environment."
sc_files = glob.glob(os.path.join(self.env_path, f"*{self.VADERE_SCENARIO_FILE_TYPE}"))
if len(sc_files) != 1:
raise RuntimeError(f"None or too many '{self.VADERE_SCENARIO_FILE_TYPE}' files "
"found in environment.")
return sc_files[0]
@classmethod
def create_if_not_exist(cls, env_name: str, basis_scenario: Union[str, dict]):
target_path = cls.environment_path(env_name)
if os.path.exists(target_path):
existing = cls(env_name)
# TODO: maybe it is good to compare if the handled file is the same as the existing
#exist_basis_file = existing.get_vadere_scenario_basis_file()
return existing
else:
return cls.create_environment(env_name, basis_scenario)
def from_full_path(cls, env_path):
assert os.path.isdir(env_path)
base_path = os.path.dirname(env_path)
if env_path.endswith(os.pathsep):
env_path = env_path.rstrip(os.path.sep)
env_name = os.path.basename(env_path)
cls(base_path=base_path, env_name=env_name)
@classmethod
def create_environment(cls, env_name: str, basis_scenario: Union[str, dict], replace: bool = False):
def create_new_environment(cls, base_path=None, env_name=None, handle_existing="ask_user_replace"):
# Check if environment already exists
target_path = cls.environment_path(env_name)
base_path, env_name = cls.handle_path_and_env_input(base_path, env_name)
if replace and os.path.exists(target_path):
if replace:
cls.remove_environment(env_name, force=True)
elif not cls.remove_environment(env_name):
print("Aborting to create a new scenario.")
return
# TODO: Refactor, make handle_existing an Enum
assert handle_existing in ["ask_user_replace", "force_replace", "write_in_if_exist_else_create", "write_in"]
# Create new environment folder
os.mkdir(target_path)
# set to True if env already exists, and it shouldn't be overwritten
about_creating_env = False
env_man = None
if isinstance(basis_scenario, str): # assume that this is a path
env_exists = os.path.exists(cls.output_folder_path(base_path, env_name))
if handle_existing == "ask_user_replace" and env_exists:
if not cls.remove_environment(base_path, env_name):
about_creating_env = True
elif handle_existing == "force_replace" and env_exists:
if env_exists:
cls.remove_environment(base_path, env_name, force=True)
elif handle_existing == "write_in":
assert env_exists, f"base_path={base_path} env_name={env_name} does not exist"
env_man = cls(base_path=base_path, env_name=env_name)
elif handle_existing == "write_in_if_exist_else_create":
if env_exists:
env_man = cls(base_path=base_path, env_name=env_name)
if about_creating_env:
raise ValueError("Could not create new environment.")
if env_man is None:
# Create new environment folder
os.mkdir(cls.output_folder_path(base_path, env_name))
env_man = cls(base_path=base_path, env_name=env_name)
return env_man
@classmethod
def create_variation_env(cls, basis_scenario: Union[str, dict], base_path=None, env_name=None,
handle_existing="ask_user_replace"):
assert os.path.isfile(basis_scenario), "Filepath to .scenario does not exist"
assert basis_scenario.split(".")[-1] == "scenario", "File has to be a Vadere '*.scenario' file"
# Check if environment already exists
env_man = cls.create_new_environment(base_path=base_path, env_name=env_name, handle_existing=handle_existing)
path_output_folder = env_man.env_path
# Add basis scenario used for the variation (i.e. sampling)
if isinstance(basis_scenario, str): # assume that this is a path
if not os.path.isfile(basis_scenario):
raise FileExistsError("Filepath to .scenario does not exist")
elif basis_scenario.split(".")[-1] != cls.VADERE_SCENARIO_FILE_TYPE[1:]:
raise ValueError("basis_scenario has to be a Vadere '*"
f"{cls.VADERE_SCENARIO_FILE_TYPE}' file")
with open(basis_scenario, "r") as file:
basis_scenario = file.read()
basis_fp = os.path.join(target_path, f"BASIS_{env_name}.scenario")
# add prefix to scenario file:
basis_fp = os.path.join(path_output_folder,
f"{cls.PREFIX_BASIS_SCENARIO}{env_name}.scenario")
# FILL IN THE STANDARD FILES IN THE NEW SCENARIO:
with open(basis_fp, "w") as file:
......@@ -173,21 +241,23 @@ class EnvironmentManager(object):
# Create and store the configuration file to the new folder
cfg = dict()
if not SuqcConfig.is_package_paths(): # TODO it may be good to write the git hash / version number in the file
# TODO it may be good to write the git hash / version number in the file
if not SuqcConfig.is_package_paths():
cfg["suqc_state"] = get_current_suqc_state()
with open(os.path.join(target_path, "suqc_commit_hash.json"), 'w') as outfile:
with open(os.path.join(path_output_folder, "suqc_commit_hash.json"), 'w') as outfile:
s = "\n".join(["commit hash at creation", cfg["suqc_state"]["git_hash"]])
outfile.write(s)
# Create the folder where the output is stored
os.mkdir(os.path.join(target_path, EnvironmentManager.output_folder))
# Create the folder where all output is stored
os.mkdir(os.path.join(path_output_folder, EnvironmentManager.vadere_output_folder))
return cls(env_name)
return cls(base_path, env_name)
@classmethod
def remove_environment(cls, name, force=False):
target_path = cls.environment_path(name)
def remove_environment(cls, base_path, name, force=False):
target_path = cls.output_folder_path(base_path, name)
if force or user_query_yes_no(question=f"Are you sure you want to remove the current environment? Path: \n "
f"{target_path}"):
try:
......@@ -198,32 +268,45 @@ class EnvironmentManager(object):
return False
@staticmethod
def environment_path(name):
path = os.path.join(SuqcConfig.path_container_folder(), name)
return path
def handle_path_and_env_input(base_path, env_name):
if env_name is None:
env_name = "_".join(["output", str_timestamp()])
if base_path is None:
base_path = SuqcConfig.path_container_folder()
return base_path, env_name
@staticmethod
def output_folder_path(base_path, env_name):
base_path, env_name = EnvironmentManager.handle_path_and_env_input(base_path, env_name)
assert os.path.isdir(base_path)
output_folder_path = os.path.join(base_path, env_name)
return output_folder_path
def get_env_outputfolder_path(self):
rel_path = os.path.join(self.env_path, EnvironmentManager.output_folder)
rel_path = os.path.join(self.env_path, EnvironmentManager.vadere_output_folder)
return os.path.abspath(rel_path)
def get_par_id_output_path(self, par_id, create):
scenario_filename = self._scenario_variation_filename(par_id=par_id)
scenario_filename = scenario_filename.replace(".scenario", "")
output_path = os.path.join(self.get_env_outputfolder_path(), "".join([scenario_filename, "_output"]))
if create:
create_folder(output_path)
return output_path
def get_variation_output_folder(self, parameter_id, run_id):