Commit 93c7ad10 authored by di68kap's avatar di68kap
Browse files

- configuration presets refactored in order support multiprocessing start...

- configuration presets refactored in order support multiprocessing start methods "spawn" and "fork-server"
parent 03925ae7
......@@ -32,7 +32,8 @@ program and before any DHParser-function is invoked.
from typing import Dict, Hashable, Any
__all__ = ('CONFIG_PRESET',
__all__ = ('access_presets',
'finalize_presets',
'XML_SERIALIZATION',
'SXPRESSION_SERIALIZATION',
'COMPACT_SERIALIZATION',
......@@ -40,7 +41,82 @@ __all__ = ('CONFIG_PRESET',
'JSON_SERIALIZATION',
'SERIALIZATIONS')
CONFIG_PRESET = dict() # type: Dict[Hashable, Any]
########################################################################
#
# multiprocessing-safe preset-handling
#
########################################################################
CONFIG_PRESET = dict() # type: Dict[str, Any]
CONFIG_PRESET['syncfile_path'] = ''
def get_syncfile_path(pid: int) -> str:
import os
import tempfile
return os.path.join(tempfile.gettempdir(), 'DHParser_%i.cfg' % pid)
def access_presets() -> Dict[str, Any]:
"""
Returns a dictionary of presets for configuration values.
If any preset values are changed after calling `access_presets()`,
`finalize_presets()` should be called to make sure that processes
spawned after changing the preset values, will be able to read
the changed values.
See: https://docs.python.org/3/library/multiprocessing.html#the-spawn-and-forkserver-start-methods
"""
import multiprocessing
global CONFIG_PRESET
if not CONFIG_PRESET['syncfile_path'] and multiprocessing.get_start_method() != 'fork':
import os
import pickle
syncfile_path = get_syncfile_path(os.getppid()) # assume this is a spawned process
if not os.path.exists(syncfile_path):
syncfile_path = get_syncfile_path(os.getpid()) # assume this is the root process
f = None
try:
f = open(syncfile_path, 'rb')
preset = pickle.load(f)
assert isinstance(preset, dict)
assert preset['syncfile_path'] == syncfile_path
CONFIG_PRESET = preset
except FileNotFoundError:
pass
finally:
if f is not None:
f.close()
return CONFIG_PRESET
def remove_cfg_tempfile(filename: str):
import os
os.remove(filename)
def finalize_presets():
"""
Finalizes changes of the presets of the configuration values.
This method should always be called after changing preset values to
make sure the changes will be visible to processes spawned later.
"""
import atexit
import multiprocessing
import os
import pickle
if multiprocessing.get_start_method() != 'fork':
syncfile_path = get_syncfile_path(os.getpid())
existing_syncfile = CONFIG_PRESET['syncfile_path']
assert ((not existing_syncfile or existing_syncfile == syncfile_path)
and (not os.path.exists((get_syncfile_path(os.getppid()))))), \
"finalize_presets() can only be called from the main process!"
with open(syncfile_path, 'wb') as f:
if not existing_syncfile:
CONFIG_PRESET['syncfile_path'] = syncfile_path
atexit.register(remove_cfg_tempfile, syncfile_path)
pickle.dump(CONFIG_PRESET, f)
########################################################################
......
......@@ -97,7 +97,7 @@ from DHParser import logging, is_filename, load_if_file, \\
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \\
error_on, recompile_grammar, left_associative, lean_left, set_config_value, \\
get_config_value, XML_SERIALIZATION, SXPRESSION_SERIALIZATION, COMPACT_SERIALIZATION, \\
JSON_SERIALIZATION, CONFIG_PRESET, THREAD_LOCALS
JSON_SERIALIZATION, THREAD_LOCALS
'''.format(dhparser_parentdir=DHPARSER_PARENTDIR)
......
......@@ -298,8 +298,8 @@ class Parser:
history_tracking__ = grammar.history_tracking__
if history_tracking__:
grammar.call_stack__.append(
(self.repr if self.tag_name in (':RegExp', ':Token', ':DropToken')
else self.tag_name, location))
((self.repr if self.tag_name in (':RegExp', ':Token', ':DropToken')
else self.tag_name), location))
grammar.moving_forward__ = True
error = None
......@@ -574,10 +574,11 @@ class Grammar:
constructor of the Parser object explicitly, but it suffices to
assign them to class variables, which results in better
readability of the Python code.
See classmethod `Grammar._assign_parser_names__()`
3. The parsers in the class do not necessarily need to be connected
to one single root parser, which is helpful for testing and
building up a parser successively of several components.
to one single root parser, which is helpful for testing and when
building up a parser gradually from several components.
As a consequence, though, it is highly recommended that a Grammar
class should not define any other variables or methods with names
......@@ -606,10 +607,9 @@ class Grammar:
'named' parser and its field `parser.pname` contains the variable
name after instantiation of the Grammar class. All other parsers,
i.e. parsers that are defined within a `named` parser, remain
"anonymous parsers" where `parser.pname` is the empty string, unless
a name has been passed explicitly upon instantiation.
"anonymous parsers" where `parser.pname` is the empty string.
If one and the same parser is assigned to several class variables
such as, for example the parser `expression` in the example above,
such as, for example, the parser `expression` in the example above,
the first name sticks.
Grammar objects are callable. Calling a grammar object with a UTF-8
......@@ -856,7 +856,7 @@ class Grammar:
self.rollback__ = [] # type: List[Tuple[int, Callable]]
self.last_rb__loc__ = -1 # type: int
# support for call stack tracing
self.call_stack__ = [] # type: List[str, int] # tag_name, location
self.call_stack__ = [] # type: List[Tuple[str, int]] # tag_name, location
# snapshots of call stacks
self.history__ = [] # type: List[HistoryRecord]
# also needed for call stack tracing
......@@ -927,7 +927,7 @@ class Grammar:
Checks if failure to match document was only due to a succeeding
lookahead parser, which is a common design pattern that can break test
cases. (Testing for this case allows to modify the error message, so
that the testing framework can know that the failure is only a
that the testing framework knows that the failure is only a
test-case-artifact and no real failure.
(See test/test_testing.TestLookahead !)
"""
......
......@@ -47,6 +47,11 @@ def run_grammar_tests(glob_pattern, get_grammar, get_transformer):
if __name__ == '__main__':
# from DHParser.configuration import access_presets, finalize_presets
# CONFIG_PRESETS = access_presets()
# CONFIG_PRESET['test_parallelization'] = True
# finalize_presets()
argv = sys.argv[:]
if len(argv) > 1 and sys.argv[1] == "--debug":
LOGGING = True
......
......@@ -48,7 +48,7 @@ except ImportError:
cython_optimized = False
import DHParser.shadow_cython as cython
from DHParser.configuration import CONFIG_PRESET
from DHParser.configuration import access_presets
__all__ = ('typing',
......@@ -94,6 +94,7 @@ def get_config_value(key: Hashable) -> Any:
:param key: the key (an immutable, usually a string)
:return: the value
"""
global THREAD_LOCALS
try:
cfg = THREAD_LOCALS.config
except AttributeError:
......@@ -102,6 +103,7 @@ def get_config_value(key: Hashable) -> Any:
try:
return cfg[key]
except KeyError:
CONFIG_PRESET = access_presets()
value = CONFIG_PRESET[key]
THREAD_LOCALS.config[key] = value
return value
......@@ -116,6 +118,9 @@ def set_config_value(key: Hashable, value: Any):
:param key: the key (an immutable, usually a string)
:param value: the value
"""
global THREAD_LOCALS
if THREAD_LOCALS is None:
THREAD_LOCALS = threading.local()
try:
_ = THREAD_LOCALS.config
except AttributeError:
......
......@@ -16,16 +16,13 @@ scriptpath = os.path.dirname(__file__)
try:
from DHParser import dsl
import DHParser.log
from DHParser import testing, create_test_templates, CONFIG_PRESET
from DHParser import testing, create_test_templates, access_presets, finalize_presets
except ModuleNotFoundError:
print('Could not import DHParser. Please adjust sys.path in file '
'"%s" manually' % __file__)
sys.exit(1)
CONFIG_PRESET['ast_serialization'] = "S-expression"
CONFIG_PRESET['test_parallelization'] = False
def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
if not os.path.exists(grammar_tests_dir) \
......@@ -54,6 +51,11 @@ def run_grammar_tests(glob_pattern):
if __name__ == '__main__':
CONFIG_PRESET = access_presets()
CONFIG_PRESET['ast_serialization'] = "S-expression"
CONFIG_PRESET['test_parallelization'] = False
finalize_presets()
argv = sys.argv[:]
if len(argv) > 1 and sys.argv[1] == "--debug":
LOGGING = True
......
......@@ -16,17 +16,13 @@ scriptpath = os.path.dirname(__file__)
try:
from DHParser import dsl
import DHParser.log
from DHParser import testing, create_test_templates, CONFIG_PRESET
from DHParser import testing, create_test_templates, access_presets, finalize_presets
except ModuleNotFoundError:
print('Could not import DHParser. Please adjust sys.path in file '
'"%s" manually' % __file__)
sys.exit(1)
CONFIG_PRESET['ast_serialization'] = "S-expression"
CONFIG_PRESET['test_parallelization'] = True
def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
create_test_templates(grammar_src, grammar_tests_dir)
......@@ -52,6 +48,11 @@ def run_grammar_tests(glob_pattern):
if __name__ == '__main__':
CONFIG_PRESET = access_presets()
CONFIG_PRESET['ast_serialization'] = "S-expression"
CONFIG_PRESET['test_parallelization'] = True
finalize_presets()
argv = sys.argv[:]
if len(argv) > 1 and sys.argv[1] == "--debug":
LOGGING = True
......
......@@ -16,16 +16,13 @@ scriptpath = os.path.dirname(__file__)
try:
from DHParser import dsl
import DHParser.log
from DHParser import testing, create_test_templates, CONFIG_PRESET
from DHParser import testing, create_test_templates, access_presets, finalize_presets
except ModuleNotFoundError:
print('Could not import DHParser. Please adjust sys.path in file '
'"%s" manually' % __file__)
sys.exit(1)
CONFIG_PRESET['ast_serialization'] = "S-expression"
CONFIG_PRESET['test_parallelization'] = True
def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
if not os.path.exists(grammar_tests_dir) \
......@@ -54,6 +51,11 @@ def run_grammar_tests(glob_pattern):
if __name__ == '__main__':
CONFIG_PRESET = access_presets()
CONFIG_PRESET['ast_serialization'] = "S-expression"
CONFIG_PRESET['test_parallelization'] = True
finalize_presets()
argv = sys.argv[:]
if len(argv) > 1 and sys.argv[1] == "--debug":
LOGGING = True
......
......@@ -27,7 +27,7 @@ from DHParser import configuration
import DHParser.dsl
from DHParser import testing
configuration.CONFIG_PRESET['test_parallelization'] = True
configuration.access_presets()['test_parallelization'] = True
if __name__ == "__main__":
if not DHParser.dsl.recompile_grammar('BibTeX.ebnf', force=False): # recompiles Grammar only if it has changed
......
......@@ -16,16 +16,13 @@ scriptpath = os.path.dirname(__file__)
try:
from DHParser import dsl
import DHParser.log
from DHParser import testing, create_test_templates, CONFIG_PRESET
from DHParser import testing, create_test_templates, access_presets, finalize_presets
except ModuleNotFoundError:
print('Could not import DHParser. Please adjust sys.path in file '
'"%s" manually' % __file__)
sys.exit(1)
CONFIG_PRESET['ast_serialization'] = "S-expression"
def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
if not os.path.exists(grammar_tests_dir) \
......@@ -54,6 +51,9 @@ def run_grammar_tests(glob_pattern):
if __name__ == '__main__':
access_presets()['ast_serialization'] = "S-expression"
finalize_presets()
argv = sys.argv[:]
if len(argv) > 1 and sys.argv[1] == "--debug":
LOGGING = True
......
......@@ -29,7 +29,7 @@ import DHParser.log
from DHParser import testing
configuration.CONFIG_PRESET['test_parallelization'] = True
configuration.access_presets()['test_parallelization'] = True
def recompile_grammar(grammar_src, force):
......
......@@ -35,7 +35,7 @@ from DHParser import logging, is_filename, load_if_file, \
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \
error_on, recompile_grammar, left_associative, lean_left, set_config_value, \
get_config_value, XML_SERIALIZATION, SXPRESSION_SERIALIZATION, COMPACT_SERIALIZATION, \
JSON_SERIALIZATION, CONFIG_PRESET, THREAD_LOCALS
JSON_SERIALIZATION, THREAD_LOCALS
#######################################################################
......
#!/usr/bin/python3
"""test_configuration.py - tests of the configuration-module of DHParser
Author: Eckhart Arnold <arnold@badw.de>
Copyright 2019 Bavarian Academy of Sciences and Humanities
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import multiprocessing
import sys
sys.path.extend(['../', './'])
from DHParser.configuration import access_presets, finalize_presets
def evaluate_presets(flag):
CONFIG_PRESET = access_presets()
if CONFIG_PRESET.get('test', 'failure') != 'failure' and \
CONFIG_PRESET.get('test2', 'failure') != 'failure':
flag.value = 1
else:
flag.value = 0
class TestConfigMultiprocessing:
def test_presets(self):
"""Checks whether changes to CONFIG_PRESET before spawning / forking
new processes will be present in spawned or forked processes
afterwards."""
CONFIG_PRESET = access_presets()
CONFIG_PRESET['test'] = 'multiprocessing presets test'
finalize_presets()
CONFIG_PRESET = access_presets()
CONFIG_PRESET['test2'] = 'multiprocessing presets test2'
finalize_presets()
flag = multiprocessing.Value('b', 0)
p = multiprocessing.Process(target=evaluate_presets, args=(flag,))
p.start()
p.join()
assert flag.value == 1
if __name__ == "__main__":
from DHParser.testing import runner
runner("", globals())
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment