Merge branch 'master' into psychology

Tools/SUQController/README.md

@@ -45,3 +45,27 @@ not installed successfully.
 ### Introduction
 
 See [SRC_PATH]/tutorial
+
+
+#### Using SUQC and Vadere 
+Here a few hints for your .scenario file for Vadere:
+
+1.  ScenarioChecker  
+    Before running your scenario automatically on suqc, activate the ``ScenarioChecker`` (Project > Activate ScenarioChecker) and run it in the ``VadereGui``.
+   The ScenarioChecker will point out potential problems with your scenario file. 
+2.  Default parameters  
+    Make sure to set ``realTimeSimTimeRatio`` to 0.0. (Values > 0.0 slow down the simulation for the visualisation)  
+    Another point that may cost a lot of computation time is the ``optimizationType``, consider using ``DISCRETE`` (discrete optimization) instead of ``NELDER_MEAD``. Please note, that ``varyStepDirection`` should always be activated with discrete optimization.  
+    Remove ``attributesCar`` from the .scenario file if you are not using any vehicles to avoid confusion of attributes. 
+3.  Visual check   
+    Visually check the results of your simulation, maybe check upper and lower parameter bounds. 
+4.  Clean topography  
+    Remove elements in your topography that are not used. Sometimes through the interaction with the mouse, tiny obstacles or targets are created unintentionally. 
+    Check the elements in your topography, you can take a look at the ``ElementTree`` in the Topography creator tab. Remove all elements that are unused, especially focusing on targets. 
+5.  Data processors  
+    Remove all data processors and output files that you don't use. In particular, remove the overlap processors, they are intended for testing purposes. 
+6.  Reproducibility  
+    Make sure that your runs are reproducible - work with a ``fixedSeed`` by activating ``useFixedSeed`` or save all the ``simulationSeed``s that have been used. 
+   (Another way is to provide a ``fixedSeed`` for each runs with suqc, in this case make sure that ``useFixedSeed`` is true.)
+
+

Tools/SUQController/requirements.txt

 setuptools>=39.1.0
 fabric==2.4.0
-pandas>=0.24.0,<1.0.0
+pandas>=1.0.3
 numpy>=1.17.0
 scikit-learn>=0.21.0

Tools/SUQController/setup.py

@@ -2,15 +2,15 @@
 
 import os
 
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
-from suqc.configuration import SuqcConfig
 from suqc import __version__
+from suqc.configuration import SuqcConfig
 
 # To generate a new requirements.txt file run in console (install vis 'pip3 install pipreqs'):
 # pipreqs --use-local --force /home/daniel/REPOS/suq-controller
 
-with open('requirements.txt', "r") as f:
+with open("requirements.txt", "r") as f:
     requirements = f.read().splitlines()
 
 # Writes a file that gives information about the version such that "suqc.__version__" provides the current version,
@@ -27,7 +27,7 @@ setup(
     url="www.vadere.org",
     packages=find_packages(),
     install_requires=requirements,
-    data_files=[('suqc', ["suqc/PACKAGE.txt"])]
+    data_files=[("suqc", ["suqc/PACKAGE.txt"])],
 )
 
 os.remove(SuqcConfig.path_package_indicator_file())

Tools/SUQController/suqc/__init__.py

@@ -5,4 +5,4 @@ from suqc.parameter.postchanges import PostScenarioChangesBase
 from suqc.qoi import *
 from suqc.request import *
 
-__version__ = "2.0"
+__version__ = "2.1"

Tools/SUQController/suqc/configuration.py

 #!/usr/bin/env python3
 
-import os
 import json
+import os
 import os.path as p
 import pathlib
 
-
 # configuration of the suq-controller
-DEFAULT_SUQC_CONFIG = {"default_vadere_src_path": "TODO",  # TODO Feature: #25
-                       "server": {
-                        "host": "",
-                        "user": "",
-                        "port": -1}}
+DEFAULT_SUQC_CONFIG = {
+    "default_vadere_src_path": "TODO",  # TODO Feature: #25
+    "server": {"host": "", "user": "", "port": -1},
+}
 
 
 def check_setup(_paths_class):
 
-    if not os.path.exists(_paths_class.path_cfg_folder()) and _paths_class.is_package_paths():
+    if (
+        not os.path.exists(_paths_class.path_cfg_folder())
+        and _paths_class.is_package_paths()
+    ):
         print(f"INFO: Setting up configuration folder {_paths_class.path_cfg_folder()}")
         # the next two checks will fail automatically too, because the folder is empty
         os.mkdir(_paths_class.path_cfg_folder())
 
     if not os.path.exists(_paths_class.path_suq_config_file()):
-        print(f"INFO: Setting up default configuration file located at {_paths_class.path_suq_config_file()}")
+        print(
+            f"INFO: Setting up default configuration file located at "
+            f"{_paths_class.path_suq_config_file()}"
+        )
         _paths_class.store_config(DEFAULT_SUQC_CONFIG)
 
     if not os.path.exists(_paths_class.path_container_folder()):
-        print(f"INFO: Setting up the default container path (which will store output of simulation runs). "
-              f"Location {_paths_class.path_container_folder()}")
+        print(
+            f"INFO: Setting up the default container path "
+            f"(which will store output of simulation runs). "
+            f"Location {_paths_class.path_container_folder()}"
+        )
         os.mkdir(_paths_class.path_container_folder())
 
     return _paths_class
@@ -128,4 +135,4 @@ class SuqcConfig(object):
 
 
 if __name__ == "__main__":
-    print(SuqcConfig)
\ No newline at end of file
+    print(SuqcConfig)

Tools/SUQController/suqc/opp/config_parser.py

+import enum
+from collections.abc import MutableMapping
+from configparser import ConfigParser, NoOptionError
+
+
+class OppParser(ConfigParser):
+    def optionxform(self, optionstr):
+        return optionstr
+
+
+class OppConfigType(enum.Enum):
+    """
+    Set type on OppConfigFileBase to create read-only configurations if needed.
+    """
+
+    READ_ONLY = 1
+    EDIT_LOCAL = 2
+    EXT_DEL_LOCAL = 3
+
+    def __lt__(self, other):
+        if self.__class__ is other.__class__:
+            return self.value < other.value
+        elif type(other) is int:
+            return self.value < other
+        return NotImplemented
+
+    def __gt__(self, other):
+        return self != other and other < self
+
+    def __eq__(self, other):
+        if self.__class__ is other.__class__:
+            return self.value == other.value
+        elif type(other) is int:
+            return self.value == other
+        return NotImplemented
+
+    def __le__(self, other):
+        return self < other or self == other
+
+    def __ge__(self, other):
+        return self > other or self == other
+
+
+class OppConfigFileBase(MutableMapping):
+    """
+    Represents an omnetpp.ini file. The extends logic is defined in SimulationManual.pdf p.282 ff.
+    Each OppConfigFileBase object has a reference to complete omnetpp.ini configuration file but
+    only access's its own options, as well as all options reachable by the search path build
+    using the 'extends' option.
+
+    Example(taken form [1]):
+    The search path for options for the configuration `SlottedAloha2b` is:
+    SlottedAloha2b->SlottedAloha2->SlottedAlohaBase->HighTrafficSettings->General
+    ```
+    [General]
+    ...
+    [Config SlottedAlohaBase]
+    ...
+    [Config LowTrafficSettings]
+    ...
+    [Config HighTrafficSettings]
+    ...
+
+    [Config SlottedAloha1]
+    extends = SlottedAlohaBase, LowTrafficSettings
+    ...
+    [Config SlottedAloha2]
+    extends = SlottedAlohaBase, HighTrafficSettings
+    ...
+    [Config SlottedAloha2a]
+    extends = SlottedAloha2
+    ...
+    [Config SlottedAloha2b]
+    extends = SlottedAloha2
+    ```
+    [1]: https://doc.omnetpp.org/omnetpp/manual/#sec:config-sim:section-inheritance
+    """
+
+    @classmethod
+    def from_path(
+        cls, ini_path, config, cfg_type=OppConfigType.EDIT_LOCAL, is_parent=False
+    ):
+        _root = OppParser(inline_comment_prefixes="#")
+        _root.read(ini_path)
+        return cls(_root, config, cfg_type, is_parent)
+
+    def __init__(
+        self,
+        root_cfg: OppParser,
+        config_name: str,
+        cfg_type=OppConfigType.EDIT_LOCAL,
+        is_parent=False,
+    ):
+        self._root: OppParser = root_cfg
+        self._cfg_type = cfg_type
+        self._sec = self._ensure_config_prefix(config_name)
+
+        if not self._has_section_(self._sec):
+            raise ValueError(f"no section found with name {self._sec}")
+        self._parent_cfg = []
+        self.section_hierarchy = [self._ensure_config_prefix(self._sec)]
+        self._is_parent = is_parent
+        if not self._is_parent:
+            stack = [iter(self.parents)]
+            while stack:
+                for p in stack[0]:
+                    if p == "":
+                        continue
+                    _pp = OppConfigFileBase(self._root, p, is_parent=True)
+                    self._parent_cfg.append(_pp)
+                    self.section_hierarchy.append(self._ensure_config_prefix(p))
+                    if len(_pp.parents) > 0:
+                        stack.append(iter(_pp.parents))
+                else:
+                    stack.pop(0)
+
+        if config_name != "General" and self._has_section_("General"):
+            self._parent_cfg.append(
+                OppConfigFileBase(self._root, "General", is_parent=True)
+            )
+            self.section_hierarchy.append("General")
+
+    def writer(self, fp):
+        """ write the current state to the given file descriptor. Caller must close file."""
+        self._root.write(fp)
+
+    @staticmethod
+    def _ensure_config_prefix(val):
+        """ All omnetpp configurations start with 'Config'. Add 'Config' if it is missing.  """
+        if not val.startswith("Config") and val != "General":
+            return f"Config {val.strip()}"
+        return val
+
+    @property
+    def section(self):
+        """ Section managed by this OppConfigFileBase object (read-only) """
+        return self._sec
+
+    @property
+    def parents(self):
+        """ local parents i.e all configurations listed in the 'extends' option (read-only) """
+        return [
+            s.strip()
+            for s in self._getitem_local_("extends", default="").strip().split(",")
+        ]
+
+    @property
+    def type(self):
+        return self._cfg_type
+
+    def is_local(self, option):
+        """
+        Returns True if the given object belongs directly to the current section and False if
+        options is contained higher up the hierarchy OR does not exist.
+        """
+        return self._contains_local_(option)
+
+    def get_config_for_option(self, option):
+        """
+        Returns the name of the section the option first occurs search order: local --> general
+        or None if option does not exist
+        """
+        if self._contains_local_(option):
+            return self.section
+        else:
+            for p in self._parent_cfg:
+                if p._contains_local_(option):
+                    return p.get_config_for_option(option)
+        return None
+
+    def _has_section_(self, sec):
+        """
+        True if section exist in the configuration. Note: Returns also True even if given section is not
+        in the section_hierarchy if the current section.
+        """
+        return self._root.has_section(sec)
+
+    def _getitem_local_(self, k, default=None):
+        """
+        Search for key in local configuration
+        """
+        try:
+            return self._root.get(self._sec, k)
+        except NoOptionError:
+            if default is not None:
+                return default
+            else:
+                raise KeyError(f"key not found. Key: {k}")
+
+    def _set_local(self, k, v):
+        """
+        Set new value for key. OppConfigType checks already done
+        """
+        self._root.set(self._sec, k, v)
+
+    def _contains_local_(self, k):
+        """
+        True if key exist in current section (parents are not searched) otherwise False
+        """
+        return self._root.has_option(self._sec, k)
+
+    def _contained_by_parent(self, k):
+        """
+        True if key exists in any parent. Note key my exist multiple time but only first occurrence of key
+        will be returned. See search path.
+        """
+        return any([k in parent for parent in self._parent_cfg])
+
+    def _delitem_local(self, k):
+        """
+        Delete local key.
+        """
+        self._root.remove_option(self._sec, k)
+
+    def __setitem__(self, k, v):
+        if self._cfg_type is OppConfigType.READ_ONLY:
+            raise NotImplementedError("Cannot set value on read only config")
+
+        if self._contains_local_(k):
+            self._set_local(k, v)
+        elif self._contained_by_parent(k):
+            if self._cfg_type <= OppConfigType.EDIT_LOCAL:
+                raise NotImplementedError("Cannot edit value of parent config")
+            else:
+                for p in self._parent_cfg:
+                    if p._contains_local_(k):
+                        p._set_local(k, v)
+                        return
+        else:
+            raise KeyError(f"key not found. Key: {k}")
+
+    def __delitem__(self, k):
+        if self._cfg_type.value < OppConfigType.EXT_DEL_LOCAL:
+            raise ValueError(
+                f"current object does not allow deletion. cfg_type={self._cfg_type}"
+            )
+        if k not in self:
+            raise KeyError(f"key not found. Key: {k}")
+        if self._contains_local_(k):
+            self._root.remove_option(self._sec, k)
+        else:
+            raise NotImplementedError(
+                f"deletion of parent config option not implemented"
+            )
+
+    def __getitem__(self, k):
+        if k not in self:
+            raise KeyError(f"key not found. Key: {k}")
+
+        if self._contains_local_(k):
+            return self._getitem_local_(k)
+        else:
+            for parent in self._parent_cfg:
+                try:
+                    return parent._getitem_local_(k)
+                except KeyError:
+                    pass
+        raise KeyError(f"key not found. Key: {k}")
+
+    def __contains__(self, k) -> bool:
+        if self._contains_local_(k):
+            return True
+        elif any([k in parent for parent in self._parent_cfg]):
+            return True
+        else:
+            return False
+
+    def __len__(self) -> int:
+        _len = 0
+        for s in self.section_hierarchy:
+            _len += len(self._root.items(s))
+        return _len
+
+    def __iter__(self):
+        for s in self.section_hierarchy:
+            for item in self._root.items(s):
+                yield item
+
+    def items(self):
+        return list(self.__iter__())
+
+    def keys(self):
+        return [k for k, _ in self.__iter__()]
+
+    def values(self):
+        return [v for _, v in self.__iter__()]
+
+    def get(self, k):
+        return self[k]
+
+    def setdefault(self, k, default=...):
+        if k in self:
+            return self[k]
+        else:
+            if self._cfg_type <= OppConfigType.EDIT_LOCAL:
+                raise NotImplementedError(
+                    "Cannot set value on READ_ONLY or EDIT_LOCAL config. Use EXT_DEL_LOCAL "
+                )
+            else:
+                self._set_local(k, default)
+        return default
+
+
+class OppConfigFile(OppConfigFileBase):
+    """
+    Helpers to manage OMNeT++ specifics not part of the standard ini-Configuration
+    * Read/Write int and doubles
+    * specify units (i.e. s, dBm, m)
+    * Handle string quotes (are part of the value)
+    todo: implement
+    """
+
+    def __init__(self, root_cfg: OppParser, config_name: str):
+        super().__init__(root_cfg, config_name)

Tools/SUQController/suqc/opp/tests/config_parser_tests.py

+import os
+import unittest
+
+from suqc.opp.config_parser import OppConfigFileBase, OppConfigType
+
+
+class OppConfigFileBaseTest(unittest.TestCase):
+
+    NEW_FILE = "omnetpp_2.ini"
+    NEW_FILE_COMP = "omnetpp_compare.ini"
+    NEW_FILE_DEL = "omnetpp_del.ini"
+
+    @staticmethod
+    def get_object(
+        config, cfg_type=OppConfigType.EDIT_LOCAL, is_parent=False, path="omnetpp.ini"
+    ):
+        return OppConfigFileBase.from_path(
+            os.path.join(os.path.split(__file__)[0], path), config, cfg_type, is_parent
+        )
+
+    @staticmethod
+    def save_object(obj: OppConfigFileBase, path):
+        with open(os.path.join(os.path.split(__file__)[0], path), "w") as f:
+            obj.writer(f)
+
+    @staticmethod
+    def get_lines(path):
+        with open(os.path.join(os.path.split(__file__)[0], path), "r") as f:
+            return f.readlines()
+
+    def tearDown(self) -> None:
+        f = os.path.join(os.path.split(__file__)[0], self.NEW_FILE)
+        if os.path.exists(f):
+            os.remove(f)
+
+    def test_set_default_exits(self):
+        opp = self.get_object("HighTrafficSettings", OppConfigType.READ_ONLY)
+        # set default on exiting must work
+        self.assertEqual(opp.setdefault("opt_3", '"val_3"'), '"val_3"')
+        # set new will ignore default on existing
+        self.assertEqual(opp.setdefault("opt_3", '"new"'), '"val_3"')
+
+    def test_set_default_not_exits(self):
+        opp = self.get_object("HighTrafficSettings", OppConfigType.READ_ONLY)
+        # must raise error on OppConfigType.READ_ONLY
+        self.assertRaises(NotImplementedError, opp.setdefault, "new_key", "42")
+        opp = self.get_object("HighTrafficSettings", OppConfigType.EDIT_LOCAL)
+        # must raise error on OppConfigType.EDIT_LOCAL
+        self.assertRaises(NotImplementedError, opp.setdefault, "new_key", "42")
+        opp = self.get_object("HighTrafficSettings", OppConfigType.EXT_DEL_LOCAL)
+        # must work on OppConfigType.EXT_DEL_LOCAL
+        ret = opp.setdefault("new_key", "42")
+        self.assertEqual(ret, "42")
+        self.assertEqual(opp["new_key"], "42")
+        # new key must be local
+        self.assertTrue(opp.is_local("new_key"))
+
+    def test_read_only(self):
+        opp = self.get_object("HighTrafficSettings", OppConfigType.READ_ONLY)
+        # reading must work
+        self.assertEqual(opp["opt_3"], '"val_3"')
+        self.assertEqual(opp["general_option"], '"VAL1"')
+        # setting new values must not work for (local and parent options)
+        self.assertRaises(NotImplementedError, opp.__setitem__, "opt_3", '"new_val"')
+        self.assertRaises(
+            NotImplementedError, opp.__setitem__, "general_option", '"new_val"'
+        )
+
+    def test_edit_local(self):
+        opp = self.get_object("HighTrafficSettings", OppConfigType.EDIT_LOCAL)
+        # reading must work
+        self.assertEqual(opp["opt_3"], '"val_3"')
+        self.assertEqual(opp["general_option"], '"VAL1"')
+        # setting new values must work for local options only
+        opp["opt_3"] = '"new_val"'
+        self.assertEqual(opp["opt_3"], '"new_val"')
+        # general_option belongs to parent config 'General'
+        self.assertRaises(
+            NotImplementedError, opp.__setitem__, "general_option", '"new_val"'
+        )
+
+    def test_hierarchy(self):
+        """ Ensure correct lookup order for extended configurations"""
+        opp = self.get_object("SlottedAloha2b", OppConfigType.EDIT_LOCAL)
+        self.assertListEqual(
+            opp.section_hierarchy,
+            [
+                "Config SlottedAloha2b",
+                "Config SlottedAloha2",
+                "Config SlottedAlohaBase",
+                "Config HighTrafficSettings",
+                "General",
+            ],
+        )
+        opp = self.get_object("SlottedAloha1", OppConfigType.EDIT_LOCAL)
+        self.assertListEqual(
+            opp.section_hierarchy,
+            [
+                "Config SlottedAloha1",
+                "Config SlottedAlohaBase",
+                "Config LowTrafficSettings",
+                "General",
+            ],
+        )
+        opp = self.get_object("General", OppConfigType.EDIT_LOCAL)
+        self.assertListEqual(opp.section_hierarchy, ["General"])
+        opp = self.get_object("SlottedAlohaBase", OppConfigType.EDIT_LOCAL)
+        self.assertListEqual(
+            opp.section_hierarchy, ["Config SlottedAlohaBase", "General"]
+        )
+
+    def test_override(self):
+        opp = self.get_object("SlottedAloha2", OppConfigType.EXT_DEL_LOCAL)
+        opp["opt_5"] = '"val_55"'
+        self.save_object(opp, self.NEW_FILE)
+        opp2 = opp = self.get_object(
+            "SlottedAloha2", OppConfigType.EXT_DEL_LOCAL, path=self.NEW_FILE
+        )
+        self.assertEqual(opp2["opt_5"], '"val_55"')
+        lines_new = [
+            line for line in self.get_lines(self.NEW_FILE) if not line.startswith("\n")
+        ]
+        lines_comp = [
+            line
+            for line in self.get_lines(self.NEW_FILE_COMP)
+            if not line.startswith("\n")
+        ]
+        self.assertListEqual(lines_new, lines_comp)
+
+    def test_safe_to_file(self):
+        sa_2 = self.get_object("SlottedAloha2", OppConfigType.EXT_DEL_LOCAL)
+        hts = self.get_object("HighTrafficSettings", OppConfigType.EXT_DEL_LOCAL)
+        self.assertEqual(sa_2["opt_HT"], '"overwritten_val_HT"')
+        self.assertEqual(
+            sa_2["general_option"],
+            '"general_option option overwritten by SlottedAloha2"',