The name of the initial branch for new projects is now "main" instead of "master". Existing projects remain unchanged. More information: https://doku.lrz.de/display/PUBLIC/GitLab

Commit c9faf5a6 authored by Daniel Lehmberg's avatar Daniel Lehmberg
Browse files

Merge branch 'update_suqc' into 'master'

Update suqc

See merge request !136
parents 7a9937ed 974b435a
Pipeline #280200 passed with stages
in 129 minutes and 38 seconds
......@@ -45,3 +45,27 @@ not installed successfully.
### Introduction
See [SRC_PATH]/tutorial
#### Using SUQC and Vadere
Here a few hints for your .scenario file for Vadere:
1. ScenarioChecker
Before running your scenario automatically on suqc, activate the ``ScenarioChecker`` (Project > Activate ScenarioChecker) and run it in the ``VadereGui``.
The ScenarioChecker will point out potential problems with your scenario file.
2. Default parameters
Make sure to set ``realTimeSimTimeRatio`` to 0.0. (Values > 0.0 slow down the simulation for the visualisation)
Another point that may cost a lot of computation time is the ``optimizationType``, consider using ``DISCRETE`` (discrete optimization) instead of ``NELDER_MEAD``. Please note, that ``varyStepDirection`` should always be activated with discrete optimization.
Remove ``attributesCar`` from the .scenario file if you are not using any vehicles to avoid confusion of attributes.
3. Visual check
Visually check the results of your simulation, maybe check upper and lower parameter bounds.
4. Clean topography
Remove elements in your topography that are not used. Sometimes through the interaction with the mouse, tiny obstacles or targets are created unintentionally.
Check the elements in your topography, you can take a look at the ``ElementTree`` in the Topography creator tab. Remove all elements that are unused, especially focusing on targets.
5. Data processors
Remove all data processors and output files that you don't use. In particular, remove the overlap processors, they are intended for testing purposes.
6. Reproducibility
Make sure that your runs are reproducible - work with a ``fixedSeed`` by activating ``useFixedSeed`` or save all the ``simulationSeed``s that have been used.
(Another way is to provide a ``fixedSeed`` for each runs with suqc, in this case make sure that ``useFixedSeed`` is true.)
setuptools>=39.1.0
fabric==2.4.0
pandas>=0.24.0,<1.0.0
pandas>=1.0.3
numpy>=1.17.0
scikit-learn>=0.21.0
......@@ -2,15 +2,15 @@
import os
from setuptools import setup, find_packages
from setuptools import find_packages, setup
from suqc.configuration import SuqcConfig
from suqc import __version__
from suqc.configuration import SuqcConfig
# To generate a new requirements.txt file run in console (install vis 'pip3 install pipreqs'):
# pipreqs --use-local --force /home/daniel/REPOS/suq-controller
with open('requirements.txt', "r") as f:
with open("requirements.txt", "r") as f:
requirements = f.read().splitlines()
# Writes a file that gives information about the version such that "suqc.__version__" provides the current version,
......@@ -27,7 +27,7 @@ setup(
url="www.vadere.org",
packages=find_packages(),
install_requires=requirements,
data_files=[('suqc', ["suqc/PACKAGE.txt"])]
data_files=[("suqc", ["suqc/PACKAGE.txt"])],
)
os.remove(SuqcConfig.path_package_indicator_file())
......
......@@ -5,4 +5,4 @@ from suqc.parameter.postchanges import PostScenarioChangesBase
from suqc.qoi import *
from suqc.request import *
__version__ = "2.0"
__version__ = "2.1"
#!/usr/bin/env python3
import os
import json
import os
import os.path as p
import pathlib
# configuration of the suq-controller
DEFAULT_SUQC_CONFIG = {"default_vadere_src_path": "TODO", # TODO Feature: #25
"server": {
"host": "",
"user": "",
"port": -1}}
DEFAULT_SUQC_CONFIG = {
"default_vadere_src_path": "TODO", # TODO Feature: #25
"server": {"host": "", "user": "", "port": -1},
}
def check_setup(_paths_class):
if not os.path.exists(_paths_class.path_cfg_folder()) and _paths_class.is_package_paths():
if (
not os.path.exists(_paths_class.path_cfg_folder())
and _paths_class.is_package_paths()
):
print(f"INFO: Setting up configuration folder {_paths_class.path_cfg_folder()}")
# the next two checks will fail automatically too, because the folder is empty
os.mkdir(_paths_class.path_cfg_folder())
if not os.path.exists(_paths_class.path_suq_config_file()):
print(f"INFO: Setting up default configuration file located at {_paths_class.path_suq_config_file()}")
print(
f"INFO: Setting up default configuration file located at "
f"{_paths_class.path_suq_config_file()}"
)
_paths_class.store_config(DEFAULT_SUQC_CONFIG)
if not os.path.exists(_paths_class.path_container_folder()):
print(f"INFO: Setting up the default container path (which will store output of simulation runs). "
f"Location {_paths_class.path_container_folder()}")
print(
f"INFO: Setting up the default container path "
f"(which will store output of simulation runs). "
f"Location {_paths_class.path_container_folder()}"
)
os.mkdir(_paths_class.path_container_folder())
return _paths_class
......@@ -128,4 +135,4 @@ class SuqcConfig(object):
if __name__ == "__main__":
print(SuqcConfig)
\ No newline at end of file
print(SuqcConfig)
This diff is collapsed.
import enum
from collections.abc import MutableMapping
from configparser import ConfigParser, NoOptionError
class OppParser(ConfigParser):
def optionxform(self, optionstr):
return optionstr
class OppConfigType(enum.Enum):
"""
Set type on OppConfigFileBase to create read-only configurations if needed.
"""
READ_ONLY = 1
EDIT_LOCAL = 2
EXT_DEL_LOCAL = 3
def __lt__(self, other):
if self.__class__ is other.__class__:
return self.value < other.value
elif type(other) is int:
return self.value < other
return NotImplemented
def __gt__(self, other):
return self != other and other < self
def __eq__(self, other):
if self.__class__ is other.__class__:
return self.value == other.value
elif type(other) is int:
return self.value == other
return NotImplemented
def __le__(self, other):
return self < other or self == other
def __ge__(self, other):
return self > other or self == other
class OppConfigFileBase(MutableMapping):
"""
Represents an omnetpp.ini file. The extends logic is defined in SimulationManual.pdf p.282 ff.
Each OppConfigFileBase object has a reference to complete omnetpp.ini configuration file but
only access's its own options, as well as all options reachable by the search path build
using the 'extends' option.
Example(taken form [1]):
The search path for options for the configuration `SlottedAloha2b` is:
SlottedAloha2b->SlottedAloha2->SlottedAlohaBase->HighTrafficSettings->General
```
[General]
...
[Config SlottedAlohaBase]
...
[Config LowTrafficSettings]
...
[Config HighTrafficSettings]
...
[Config SlottedAloha1]
extends = SlottedAlohaBase, LowTrafficSettings
...
[Config SlottedAloha2]
extends = SlottedAlohaBase, HighTrafficSettings
...
[Config SlottedAloha2a]
extends = SlottedAloha2
...
[Config SlottedAloha2b]
extends = SlottedAloha2
```
[1]: https://doc.omnetpp.org/omnetpp/manual/#sec:config-sim:section-inheritance
"""
@classmethod
def from_path(
cls, ini_path, config, cfg_type=OppConfigType.EDIT_LOCAL, is_parent=False
):
_root = OppParser(inline_comment_prefixes="#")
_root.read(ini_path)
return cls(_root, config, cfg_type, is_parent)
def __init__(
self,
root_cfg: OppParser,
config_name: str,
cfg_type=OppConfigType.EDIT_LOCAL,
is_parent=False,
):
self._root: OppParser = root_cfg
self._cfg_type = cfg_type
self._sec = self._ensure_config_prefix(config_name)
if not self._has_section_(self._sec):
raise ValueError(f"no section found with name {self._sec}")
self._parent_cfg = []
self.section_hierarchy = [self._ensure_config_prefix(self._sec)]
self._is_parent = is_parent
if not self._is_parent:
stack = [iter(self.parents)]
while stack:
for p in stack[0]:
if p == "":
continue
_pp = OppConfigFileBase(self._root, p, is_parent=True)
self._parent_cfg.append(_pp)
self.section_hierarchy.append(self._ensure_config_prefix(p))
if len(_pp.parents) > 0:
stack.append(iter(_pp.parents))
else:
stack.pop(0)
if config_name != "General" and self._has_section_("General"):
self._parent_cfg.append(
OppConfigFileBase(self._root, "General", is_parent=True)
)
self.section_hierarchy.append("General")
def writer(self, fp):
""" write the current state to the given file descriptor. Caller must close file."""
self._root.write(fp)
@staticmethod
def _ensure_config_prefix(val):
""" All omnetpp configurations start with 'Config'. Add 'Config' if it is missing. """
if not val.startswith("Config") and val != "General":
return f"Config {val.strip()}"
return val
@property
def section(self):
""" Section managed by this OppConfigFileBase object (read-only) """
return self._sec
@property
def parents(self):
""" local parents i.e all configurations listed in the 'extends' option (read-only) """
return [
s.strip()
for s in self._getitem_local_("extends", default="").strip().split(",")
]
@property
def type(self):
return self._cfg_type
def is_local(self, option):
"""
Returns True if the given object belongs directly to the current section and False if
options is contained higher up the hierarchy OR does not exist.
"""
return self._contains_local_(option)
def get_config_for_option(self, option):
"""
Returns the name of the section the option first occurs search order: local --> general
or None if option does not exist
"""
if self._contains_local_(option):
return self.section
else:
for p in self._parent_cfg:
if p._contains_local_(option):
return p.get_config_for_option(option)
return None
def _has_section_(self, sec):
"""
True if section exist in the configuration. Note: Returns also True even if given section is not
in the section_hierarchy if the current section.
"""
return self._root.has_section(sec)
def _getitem_local_(self, k, default=None):
"""
Search for key in local configuration
"""
try:
return self._root.get(self._sec, k)
except NoOptionError:
if default is not None:
return default
else:
raise KeyError(f"key not found. Key: {k}")
def _set_local(self, k, v):
"""
Set new value for key. OppConfigType checks already done
"""
self._root.set(self._sec, k, v)
def _contains_local_(self, k):
"""
True if key exist in current section (parents are not searched) otherwise False
"""
return self._root.has_option(self._sec, k)
def _contained_by_parent(self, k):
"""
True if key exists in any parent. Note key my exist multiple time but only first occurrence of key
will be returned. See search path.
"""
return any([k in parent for parent in self._parent_cfg])
def _delitem_local(self, k):
"""
Delete local key.
"""
self._root.remove_option(self._sec, k)
def __setitem__(self, k, v):
if self._cfg_type is OppConfigType.READ_ONLY:
raise NotImplementedError("Cannot set value on read only config")
if self._contains_local_(k):
self._set_local(k, v)
elif self._contained_by_parent(k):
if self._cfg_type <= OppConfigType.EDIT_LOCAL:
raise NotImplementedError("Cannot edit value of parent config")
else:
for p in self._parent_cfg:
if p._contains_local_(k):
p._set_local(k, v)
return
else:
raise KeyError(f"key not found. Key: {k}")
def __delitem__(self, k):
if self._cfg_type.value < OppConfigType.EXT_DEL_LOCAL:
raise ValueError(
f"current object does not allow deletion. cfg_type={self._cfg_type}"
)
if k not in self:
raise KeyError(f"key not found. Key: {k}")
if self._contains_local_(k):
self._root.remove_option(self._sec, k)
else:
raise NotImplementedError(
f"deletion of parent config option not implemented"
)
def __getitem__(self, k):
if k not in self:
raise KeyError(f"key not found. Key: {k}")
if self._contains_local_(k):
return self._getitem_local_(k)
else:
for parent in self._parent_cfg:
try:
return parent._getitem_local_(k)
except KeyError:
pass
raise KeyError(f"key not found. Key: {k}")
def __contains__(self, k) -> bool:
if self._contains_local_(k):
return True
elif any([k in parent for parent in self._parent_cfg]):
return True
else:
return False
def __len__(self) -> int:
_len = 0
for s in self.section_hierarchy:
_len += len(self._root.items(s))
return _len
def __iter__(self):
for s in self.section_hierarchy:
for item in self._root.items(s):
yield item
def items(self):
return list(self.__iter__())
def keys(self):
return [k for k, _ in self.__iter__()]
def values(self):
return [v for _, v in self.__iter__()]
def get(self, k):
return self[k]
def setdefault(self, k, default=...):
if k in self:
return self[k]
else:
if self._cfg_type <= OppConfigType.EDIT_LOCAL:
raise NotImplementedError(
"Cannot set value on READ_ONLY or EDIT_LOCAL config. Use EXT_DEL_LOCAL "
)
else:
self._set_local(k, default)
return default
class OppConfigFile(OppConfigFileBase):
"""
Helpers to manage OMNeT++ specifics not part of the standard ini-Configuration
* Read/Write int and doubles
* specify units (i.e. s, dBm, m)
* Handle string quotes (are part of the value)
todo: implement
"""
def __init__(self, root_cfg: OppParser, config_name: str):
super().__init__(root_cfg, config_name)
import os
import unittest
from suqc.opp.config_parser import OppConfigFileBase, OppConfigType
class OppConfigFileBaseTest(unittest.TestCase):
NEW_FILE = "omnetpp_2.ini"
NEW_FILE_COMP = "omnetpp_compare.ini"
NEW_FILE_DEL = "omnetpp_del.ini"
@staticmethod
def get_object(
config, cfg_type=OppConfigType.EDIT_LOCAL, is_parent=False, path="omnetpp.ini"
):
return OppConfigFileBase.from_path(
os.path.join(os.path.split(__file__)[0], path), config, cfg_type, is_parent
)
@staticmethod
def save_object(obj: OppConfigFileBase, path):
with open(os.path.join(os.path.split(__file__)[0], path), "w") as f:
obj.writer(f)
@staticmethod
def get_lines(path):
with open(os.path.join(os.path.split(__file__)[0], path), "r") as f:
return f.readlines()
def tearDown(self) -> None:
f = os.path.join(os.path.split(__file__)[0], self.NEW_FILE)
if os.path.exists(f):
os.remove(f)
def test_set_default_exits(self):
opp = self.get_object("HighTrafficSettings", OppConfigType.READ_ONLY)
# set default on exiting must work
self.assertEqual(opp.setdefault("opt_3", '"val_3"'), '"val_3"')
# set new will ignore default on existing
self.assertEqual(opp.setdefault("opt_3", '"new"'), '"val_3"')
def test_set_default_not_exits(self):
opp = self.get_object("HighTrafficSettings", OppConfigType.READ_ONLY)
# must raise error on OppConfigType.READ_ONLY
self.assertRaises(NotImplementedError, opp.setdefault, "new_key", "42")
opp = self.get_object("HighTrafficSettings", OppConfigType.EDIT_LOCAL)
# must raise error on OppConfigType.EDIT_LOCAL
self.assertRaises(NotImplementedError, opp.setdefault, "new_key", "42")
opp = self.get_object("HighTrafficSettings", OppConfigType.EXT_DEL_LOCAL)
# must work on OppConfigType.EXT_DEL_LOCAL
ret = opp.setdefault("new_key", "42")
self.assertEqual(ret, "42")
self.assertEqual(opp["new_key"], "42")
# new key must be local
self.assertTrue(opp.is_local("new_key"))
def test_read_only(self):
opp = self.get_object("HighTrafficSettings", OppConfigType.READ_ONLY)
# reading must work
self.assertEqual(opp["opt_3"], '"val_3"')
self.assertEqual(opp["general_option"], '"VAL1"')
# setting new values must not work for (local and parent options)
self.assertRaises(NotImplementedError, opp.__setitem__, "opt_3", '"new_val"')
self.assertRaises(
NotImplementedError, opp.__setitem__, "general_option", '"new_val"'
)
def test_edit_local(self):
opp = self.get_object("HighTrafficSettings", OppConfigType.EDIT_LOCAL)
# reading must work
self.assertEqual(opp["opt_3"], '"val_3"')
self.assertEqual(opp["general_option"], '"VAL1"')
# setting new values must work for local options only
opp["opt_3"] = '"new_val"'
self.assertEqual(opp["opt_3"], '"new_val"')
# general_option belongs to parent config 'General'
self.assertRaises(
NotImplementedError, opp.__setitem__, "general_option", '"new_val"'
)
def test_hierarchy(self):
""" Ensure correct lookup order for extended configurations"""
opp = self.get_object("SlottedAloha2b", OppConfigType.EDIT_LOCAL)
self.assertListEqual(
opp.section_hierarchy,
[
"Config SlottedAloha2b",
"Config SlottedAloha2",
"Config SlottedAlohaBase",
"Config HighTrafficSettings",
"General",
],
)
opp = self.get_object("SlottedAloha1", OppConfigType.EDIT_LOCAL)
self.assertListEqual(
opp.section_hierarchy,
[
"Config SlottedAloha1",
"Config SlottedAlohaBase",
"Config LowTrafficSettings",
"General",
],
)
opp = self.get_object("General", OppConfigType.EDIT_LOCAL)
self.assertListEqual(opp.section_hierarchy, ["General"])
opp = self.get_object("SlottedAlohaBase", OppConfigType.EDIT_LOCAL)
self.assertListEqual(
opp.section_hierarchy, ["Config SlottedAlohaBase", "General"]
)
def test_override(self):
opp = self.get_object("SlottedAloha2", OppConfigType.EXT_DEL_LOCAL)
opp["opt_5"] = '"val_55"'
self.save_object(opp, self.NEW_FILE)
opp2 = opp = self.get_object(
"SlottedAloha2", OppConfigType.EXT_DEL_LOCAL, path=self.NEW_FILE
)
self.assertEqual(opp2["opt_5"], '"val_55"')
lines_new = [
line for line in self.get_lines(self.NEW_FILE) if not line.startswith("\n")
]
lines_comp = [
line
for line in self.get_lines(self.NEW_FILE_COMP)
if not line.startswith("\n")
]
self.assertListEqual(lines_new, lines_comp)
def test_safe_to_file(self):
sa_2 = self.get_object("SlottedAloha2", OppConfigType.EXT_DEL_LOCAL)
hts = self.get_object("HighTrafficSettings", OppConfigType.EXT_DEL_LOCAL)
self.assertEqual(sa_2["opt_HT"], '"overwritten_val_HT"')
self.assertEqual(