Commit c7b7a76b authored by eckhart's avatar eckhart
Browse files

- Replaced process-local global variables by thread-local variables

parent efb7ef3d
......@@ -101,7 +101,7 @@ from DHParser import logging, is_filename, load_if_file, MockParser, \\
remove_nodes, remove_content, remove_brackets, replace_parser, remove_anonymous_tokens, \\
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \\
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \\
replace_content, replace_content_by, error_on, recompile_grammar
replace_content, replace_content_by, error_on, recompile_grammar, THREAD_LOCAL
'''.format(dhparserdir=dhparserdir)
......
......@@ -33,14 +33,14 @@ from DHParser.error import Error
from DHParser.parse import Grammar, mixin_comment, Forward, RegExp, Whitespace, \
NegativeLookahead, Alternative, Series, Option, OneOrMore, ZeroOrMore, Token
from DHParser.preprocess import nil_preprocessor, PreprocessorFunc
from DHParser.syntaxtree import Node, WHITESPACE_PTYPE, TOKEN_PTYPE
from DHParser.syntaxtree import Node, RootNode, WHITESPACE_PTYPE, TOKEN_PTYPE
from DHParser.toolkit import load_if_file, escape_re, md5, sane_parser_name, re, expand_table, \
typing
THREAD_LOCAL, typing
from DHParser.transform import TransformationFunc, traverse, remove_brackets, \
reduce_single_child, replace_by_single_child, remove_expendables, \
remove_tokens, flatten, forbid, assert_content, remove_infix_operator
from DHParser.versionnumber import __version__
from typing import Callable, Dict, List, Set, Tuple
from typing import Callable, Dict, List, Set, Tuple, Any
__all__ = ('get_ebnf_preprocessor',
......@@ -195,13 +195,12 @@ def grammar_changed(grammar_class, grammar_source: str) -> bool:
def get_ebnf_grammar() -> EBNFGrammar:
global thread_local_ebnf_grammar_singleton
try:
grammar = thread_local_ebnf_grammar_singleton
grammar = THREAD_LOCAL.ebnf_grammar_singleton
return grammar
except NameError:
thread_local_ebnf_grammar_singleton = EBNFGrammar()
return thread_local_ebnf_grammar_singleton
except AttributeError:
THREAD_LOCAL.ebnf_grammar_singleton = EBNFGrammar()
return THREAD_LOCAL.ebnf_grammar_singleton
########################################################################
......@@ -249,12 +248,11 @@ def EBNFTransform() -> TransformationFunc:
def get_ebnf_transformer() -> TransformationFunc:
global thread_local_EBNF_transformer_singleton
try:
transformer = thread_local_EBNF_transformer_singleton
except NameError:
thread_local_EBNF_transformer_singleton = EBNFTransform()
transformer = thread_local_EBNF_transformer_singleton
transformer = THREAD_LOCAL.EBNF_transformer_singleton
except AttributeError:
THREAD_LOCAL.EBNF_transformer_singleton = EBNFTransform()
transformer = THREAD_LOCAL.EBNF_transformer_singleton
return transformer
......@@ -278,12 +276,11 @@ def get_preprocessor() -> PreprocessorFunc:
GRAMMAR_FACTORY = '''
def get_grammar() -> {NAME}Grammar:
global thread_local_{NAME}_grammar_singleton
try:
grammar = thread_local_{NAME}_grammar_singleton
except NameError:
thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
grammar = thread_local_{NAME}_grammar_singleton
grammar = THREAD_LOCAL.{NAME}_{ID}_grammar_singleton
except AttributeError:
THREAD_LOCAL.{NAME}_{ID}_grammar_singleton = {NAME}Grammar()
grammar = THREAD_LOCAL.{NAME}_{ID}_grammar_singleton
return grammar
'''
......@@ -293,26 +290,24 @@ def {NAME}Transform() -> TransformationDict:
return partial(traverse, processing_table={NAME}_AST_transformation_table.copy())
def get_transformer() -> TransformationFunc:
global thread_local_{NAME}_transformer_singleton
try:
transformer = thread_local_{NAME}_transformer_singleton
except NameError:
thread_local_{NAME}_transformer_singleton = {NAME}Transform()
transformer = thread_local_{NAME}_transformer_singleton
transformer = THREAD_LOCAL.{NAME}_{ID}_transformer_singleton
except AttributeError:
THREAD_LOCAL.{NAME}_{ID}_transformer_singleton = {NAME}Transform()
transformer = THREAD_LOCAL.{NAME}_{ID}_transformer_singleton
return transformer
'''
COMPILER_FACTORY = '''
def get_compiler(grammar_name="{NAME}", grammar_source="") -> {NAME}Compiler:
global thread_local_{NAME}_compiler_singleton
try:
compiler = thread_local_{NAME}_compiler_singleton
compiler = THREAD_LOCAL.{NAME}_{ID}_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
except NameError:
thread_local_{NAME}_compiler_singleton = \\
except AttributeError:
THREAD_LOCAL.{NAME}_{ID}_compiler_singleton = \\
{NAME}Compiler(grammar_name, grammar_source)
compiler = thread_local_{NAME}_compiler_singleton
compiler = THREAD_LOCAL.{NAME}_{ID}_compiler_singleton
return compiler
'''
......@@ -369,19 +364,23 @@ class EBNFCompiler(Compiler):
definitions: A dictionary of definitions. Other than `rules`
this maps the symbols to their compiled definienda.
deferred_taks: A list of callables that is filled during
deferred_tasks: A list of callables that is filled during
compilatation, but that will be executed only after
compilation has finished. Typically, it contains
sementatic checks that require information that
is only available upon completion of compilation.
root: The name of the root symbol.
root_symbol: The name of the root symbol.
directives: A dictionary of all directives and their default
values.
re_flags: A set of regular expression flags to be added to all
regular expressions found in the current parsing process
grammar_id: a unique id for every compiled grammar. (Required for
disambiguation of of thread local variables storing
compiled texts.)
"""
COMMENT_KEYWORD = "COMMENT__"
WHITESPACE_KEYWORD = "WSP_RE__"
......@@ -401,8 +400,9 @@ class EBNFCompiler(Compiler):
def __init__(self, grammar_name="", grammar_source=""):
self.grammar_id = 0
super(EBNFCompiler, self).__init__(grammar_name, grammar_source)
self._reset()
# self._reset()
def _reset(self):
......@@ -424,6 +424,8 @@ class EBNFCompiler(Compiler):
'filter': dict()} # alt. 'filter'
# self.directives['ignorecase']: False
self.defined_directives = set() # type: Set[str]
self.grammar_id += 1
@property
def result(self) -> str:
......@@ -463,7 +465,7 @@ class EBNFCompiler(Compiler):
transtable.append(' "' + name + '": %s,' % transformations)
transtable.append(' ":Token": reduce_single_child,')
transtable += [' "*": replace_by_single_child', '}', '']
transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name, ID=self.grammar_id)]
return '\n'.join(transtable)
......@@ -495,7 +497,7 @@ class EBNFCompiler(Compiler):
else:
compiler += [' # def ' + method_name + '(self, node):',
' # return node', '']
compiler += [COMPILER_FACTORY.format(NAME=self.grammar_name)]
compiler += [COMPILER_FACTORY.format(NAME=self.grammar_name, ID=self.grammar_id)]
return '\n'.join(compiler)
def verify_transformation_table(self, transtable):
......@@ -618,7 +620,7 @@ class EBNFCompiler(Compiler):
declarations.append('root__ = ' + self.root_symbol)
declarations.append('')
self._result = '\n '.join(declarations) \
+ GRAMMAR_FACTORY.format(NAME=self.grammar_name)
+ GRAMMAR_FACTORY.format(NAME=self.grammar_name, ID=self.grammar_id)
return self._result
......@@ -985,11 +987,10 @@ class EBNFCompiler(Compiler):
def get_ebnf_compiler(grammar_name="", grammar_source="") -> EBNFCompiler:
global thread_local_ebnf_compiler_singleton
try:
compiler = thread_local_ebnf_compiler_singleton
compiler = THREAD_LOCAL.ebnf_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except NameError:
thread_local_ebnf_compiler_singleton = EBNFCompiler(grammar_name, grammar_source)
return thread_local_ebnf_compiler_singleton
except AttributeError:
THREAD_LOCAL.ebnf_compiler_singleton = EBNFCompiler(grammar_name, grammar_source)
return THREAD_LOCAL.ebnf_compiler_singleton
......@@ -52,11 +52,12 @@ import collections
import contextlib
import html
import os
import threading
from DHParser.error import line_col
from DHParser.stringview import StringView
from DHParser.syntaxtree import Node
from DHParser.toolkit import is_filename, escape_control_characters, typing
from DHParser.toolkit import is_filename, escape_control_characters, THREAD_LOCAL, typing
from typing import List, Tuple, Union
__all__ = ('log_dir',
......@@ -87,15 +88,14 @@ def log_dir() -> str:
Returns:
name of the logging directory
"""
# the try-except clauses in the following are precautions for multiprocessing
global LOGGING
# the try-except clauses in the following are precautions for multithreading
try:
dirname = LOGGING # raises a name error if LOGGING is not defined
dirname = THREAD_LOCAL.LOGGING # raises a name error if LOGGING is not defined
if not dirname:
raise NameError # raise a name error if LOGGING evaluates to False
except NameError:
raise NameError("No access to log directory before logging has been "
"turned on within the same thread/process.")
raise AttributeError # raise a name error if LOGGING evaluates to False
except AttributeError:
raise AttributeError("No access to log directory before logging has been "
"turned on within the same thread/process.")
if os.path.exists(dirname) and not os.path.isdir(dirname):
raise IOError('"' + dirname + '" cannot be used as log directory, '
'because it is not a directory!')
......@@ -123,24 +123,22 @@ def logging(dirname="LOGS"):
dirname: the name for the log directory or the empty string to
turn logging of
"""
global LOGGING
if dirname and not isinstance(dirname, str):
dirname = "LOGS" # be fail tolerant here...
try:
save = LOGGING
except NameError:
save = THREAD_LOCAL.LOGGING
except AttributeError:
save = ""
LOGGING = dirname or ""
THREAD_LOCAL.LOGGING = dirname or ""
yield
LOGGING = save
THREAD_LOCAL.LOGGING = save
def is_logging() -> bool:
"""-> True, if logging is turned on."""
global LOGGING
try:
return bool(LOGGING)
except NameError:
return bool(THREAD_LOCAL.LOGGING)
except AttributeError:
return False
......
......@@ -34,6 +34,7 @@ import inspect
import json
import os
import sys
import threading
from DHParser.error import Error, is_error, adjust_error_locations
from DHParser.log import is_logging, clear_logs, log_ST, log_parsing_history
......
......@@ -57,7 +57,8 @@ __all__ = ('escape_re',
'expand_table',
'compile_python_object',
'smart_list',
'sane_parser_name')
'sane_parser_name',
'THREAD_LOCAL')
#######################################################################
......@@ -67,6 +68,9 @@ __all__ = ('escape_re',
#######################################################################
THREAD_LOCAL = threading.local()
def escape_re(strg: str) -> str:
"""
Returns the string with all regular expression special characters escaped.
......
......@@ -124,8 +124,9 @@ class TestCompilerGeneration:
assert not errors
with open(self.compiler_name, 'r') as f:
compiler_suite_2nd_run = f.read()
assert compiler_suite == compiler_suite_2nd_run
diff = ''.join([a for a, b in zip(compiler_suite_2nd_run, compiler_suite) if a != b])
# differences should only be ID-Numbers
assert diff.isnumeric()
# test compiling with a generated compiler suite
# assert is_filename(self.text_name)
errors = compile_on_disk(self.text_name, self.compiler_name)
......
......@@ -157,14 +157,17 @@ class TestNode:
res1 = compiler(tree_copy)
t2 = copy.deepcopy(tree_copy)
res2 = compiler(t2)
assert res1 == res2
diff = ''.join([a for a, b in zip(res1, res2) if a != b])
assert diff.isnumeric() # differences should only be ID-Numbers
tree_copy = copy.deepcopy(tree)
transform(tree_copy)
res3 = compiler(tree_copy)
assert res3 == res2
diff = ''.join([a for a, b in zip(res2, res3) if a != b])
assert diff.isnumeric() # differences should only be ID-Numbers
transform(tree)
res4 = compiler(tree)
assert res4 == res3
diff = ''.join([a for a, b in zip(res3, res4) if a != b])
assert diff.isnumeric() # differences should only be ID-Numbers
def test_len_and_pos(self):
"""Test len-property of Node."""
......
......@@ -95,9 +95,9 @@ class TestLoggingAndLoading:
def test_logging(self):
try:
log_dir()
assert False, "Name error should be raised when log_dir() is called outside " \
assert False, "AttributeError should be raised when log_dir() is called outside " \
"a logging context."
except NameError:
except AttributeError:
pass
with logging("TESTLOGS"):
assert not os.path.exists("TESTSLOGS"), \
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment