- transform.py: thread-safe between call caching for function traverse()

DHParser/dsl.py

@@ -30,7 +30,7 @@ try:
 except ImportError:
     from .typing34 import Any, cast, Tuple, Union
 
-from DHParser.ebnf import EBNFTransformer, EBNFCompiler, grammar_changed, \
+from DHParser.ebnf import EBNFCompiler, grammar_changed, \
     get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
     PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
 from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object
@@ -80,7 +80,7 @@ from DHParser import logging, is_filename, load_if_file, \\
     Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
     last_value, counterpart, accumulate, PreprocessorFunc, \\
-    Node, TransformationFunc, TRUE_CONDITION, \\
+    Node, TransformationFunc, TransformationDict, TRUE_CONDITION, \\
     traverse, remove_children_if, merge_children, is_anonymous, \\
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
     remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
@@ -219,7 +219,8 @@ def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
     """
     grammar = get_ebnf_grammar()
     compiler = get_ebnf_compiler(branding , ebnf_src)
-    compileDSL(ebnf_src, nil_preprocessor, grammar, EBNFTransformer, compiler)
+    transformer = get_ebnf_transformer()
+    compileDSL(ebnf_src, nil_preprocessor, grammar, transformer, compiler)
     return compiler
 
 

DHParser/ebnf.py

@@ -18,6 +18,7 @@ permissions and limitations under the License.
 
 import keyword
 from collections import OrderedDict
+from functools import partial
 
 try:
     import regex as re
@@ -33,9 +34,9 @@ from DHParser.parser import Grammar, mixin_comment, nil_preprocessor, Forward, R
     Alternative, Series, Optional, Required, OneOrMore, ZeroOrMore, Token, Compiler, \
     PreprocessorFunc
 from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, Node, TransformationFunc
-from DHParser.transform import traverse, remove_brackets, \
+from DHParser.transform import TransformationDict, traverse, remove_brackets, \
     reduce_single_child, replace_by_single_child, remove_expendables, \
-    remove_tokens, flatten, forbid, assert_content, key_tag_name, remove_infix_operator
+    remove_tokens, flatten, forbid, assert_content, remove_infix_operator
 from DHParser.versionnumber import __version__
 
 __all__ = ('get_ebnf_preprocessor',
@@ -43,7 +44,7 @@ __all__ = ('get_ebnf_preprocessor',
            'get_ebnf_transformer',
            'get_ebnf_compiler',
            'EBNFGrammar',
-           'EBNFTransformer',
+           'EBNFTransform',
            'EBNFCompilerError',
            'EBNFCompiler',
            'grammar_changed',
@@ -191,7 +192,7 @@ def get_ebnf_grammar() -> EBNFGrammar:
 ########################################################################
 
 
-EBNF_transformation_table = {
+EBNF_AST_transformation_table = {
     # AST Transformations for EBNF-grammar
     "+":
         remove_expendables,
@@ -221,12 +222,17 @@ EBNF_transformation_table = {
 }
 
 
-def EBNFTransformer(syntax_tree: Node):
-    traverse(syntax_tree, EBNF_transformation_table, key_tag_name)
-
+def EBNFTransform() -> TransformationDict:
+    return partial(traverse, processing_table=EBNF_AST_transformation_table.copy())
 
 def get_ebnf_transformer() -> TransformationFunc:
-    return EBNFTransformer
+    global thread_local_EBNF_transformer_singleton
+    try:
+        transformer = thread_local_EBNF_transformer_singleton
+    except NameError:
+        thread_local_EBNF_transformer_singleton = EBNFTransform()
+        transformer = thread_local_EBNF_transformer_singleton
+    return transformer
 
 
 ########################################################################
@@ -252,16 +258,25 @@ def get_grammar() -> {NAME}Grammar:
     global thread_local_{NAME}_grammar_singleton
     try:
         grammar = thread_local_{NAME}_grammar_singleton
-        return grammar
     except NameError:
         thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
-        return thread_local_{NAME}_grammar_singleton
+        grammar = thread_local_{NAME}_grammar_singleton
+    return grammar
 '''
 
 
 TRANSFORMER_FACTORY = '''
+def {NAME}Transform() -> TransformationDict:
+    return partial(traverse, processing_table={NAME}_AST_transformation_table.copy())
+
 def get_transformer() -> TransformationFunc:
-    return {NAME}Transform
+    global thread_local_{NAME}_transformer_singleton
+    try:
+        transformer = thread_local_{NAME}_transformer_singleton
+    except NameError:
+        thread_local_{NAME}_transformer_singleton = {NAME}Transform()
+        transformer = thread_local_{NAME}_transformer_singleton
+    return transformer
 '''
 
 
@@ -271,11 +286,11 @@ def get_compiler(grammar_name="{NAME}", grammar_source="") -> {NAME}Compiler:
     try:
         compiler = thread_local_{NAME}_compiler_singleton
         compiler.set_grammar_name(grammar_name, grammar_source)
-        return compiler
     except NameError:
         thread_local_{NAME}_compiler_singleton = \\
             {NAME}Compiler(grammar_name, grammar_source)
-        return thread_local_{NAME}_compiler_singleton 
+        compiler = thread_local_{NAME}_compiler_singleton
+    return compiler
 '''
 
 
@@ -404,7 +419,6 @@ class EBNFCompiler(Compiler):
             raise EBNFCompilerError('Compiler must be run before calling '
                                     '"gen_transformer_Skeleton()"!')
         tt_name = self.grammar_name + '_AST_transformation_table'
-        tf_name = self.grammar_name + 'Transform'
         transtable = [tt_name + ' = {',
                       '    # AST Transformations for the ' +
                       self.grammar_name + '-grammar']
@@ -418,8 +432,7 @@ class EBNFCompiler(Compiler):
                 tf = '[replace_by_single_child]'
             transtable.append('    "' + name + '": %s,' % tf)
         transtable.append('    ":Token, :RE": reduce_single_child,')
-        transtable += ['    "*": replace_by_single_child', '}', '', tf_name +
-                       ' = partial(traverse, processing_table=%s)' % tt_name, '']
+        transtable += ['    "*": replace_by_single_child', '}', '']
         transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
         return '\n'.join(transtable)
 

DHParser/transform.py

@@ -35,7 +35,11 @@ except ImportError:
 
 from DHParser.toolkit import expand_table, smart_list
 
-__all__ = ('transformation_factory',
+__all__ = ('TransformationDict',
+           'TransformationProc',
+           'ConditionFunc',
+           'KeyFunc',
+           'transformation_factory',
            'key_parser_name',
            'key_tag_name',
            'traverse',
@@ -78,6 +82,7 @@ __all__ = ('transformation_factory',
 
 
 TransformationProc = Callable[[List[Node]], None]
+TransformationDict = Dict
 ConditionFunc = Callable  # Callable[[List[Node]], bool]
 KeyFunc = Callable[[Node], str]
 
@@ -191,8 +196,8 @@ def traverse(root_node: Node,
         root_node (Node): The root-node of the syntax tree to be traversed
         processing_table (dict): node key -> sequence of functions that
             will be applied to matching nodes in order. This dictionary
-            is interpreted as a ``compact_table``. See
-            ``toolkit.expand_table`` or ``EBNFCompiler.EBNFTransTable``
+            is interpreted as a `compact_table`. See
+            `toolkit.expand_table` or ``EBNFCompiler.EBNFTransTable`
         key_func (function): A mapping key_func(node) -> keystr. The default
             key_func yields node.parser.name.
 
@@ -201,11 +206,27 @@ def traverse(root_node: Node,
             "factor, flowmarker, retrieveop": replace_by_single_child }
         traverse(node, table)
     """
-    # normalize processing_table entries by turning single values into lists
-    # with a single value
-    table = {name: smart_list(call) for name, call in list(processing_table.items())}
-    table = expand_table(table)
-    cache = {}  # type: Dict[str, List[Callable]]
+    # Is this optimazation really needed?
+    if '__cache__' in processing_table:
+        # assume that processing table has already been expanded
+        table = processing_table
+        cache = processing_table['__cache__']
+    else:
+        # normalize processing_table entries by turning single values into lists
+        # with a single value
+        table = {name: smart_list(call) for name, call in list(processing_table.items())}
+        table = expand_table(table)
+        cache = table.setdefault('__cache__', {})  # type: Dict[str, List[Callable]]
+        # change processing table in place, so that table expansion does not get lost
+        # between calls
+        processing_table.clear();
+        processing_table.update(table)
+
+    # assert '__cache__' in processing_table
+    # # Code without optimization
+    # table = {name: smart_list(call) for name, call in list(processing_table.items())}
+    # table = expand_table(table)
+    # cache = {}
 
     def traverse_recursive(context):
         node = context[-1]
@@ -232,6 +253,8 @@ def traverse(root_node: Node,
             call(context)
 
     traverse_recursive([root_node])
+    # assert processing_table['__cache__']
+
 
 
 # ------------------------------------------------

examples/LaTeX/LaTeX.ebnf

@@ -77,7 +77,7 @@ tabular_config      = "{" /[lcr|]+/~ §"}"
 
 block_of_paragraphs = /{/~ sequence §/}/
 sequence            = { (paragraph | block_environment ) [PARSEP] }+
-paragraph           = { !blockcmd text_element //~ }+
+paragraph           = { !blockcmd (text_element | LINEFEED) //~ }+
 text_element        = text | block | inline_environment | command
 
 #### inline enivronments ####
@@ -98,7 +98,7 @@ inline_math         = /\$/ /[^$]*/ §/\$/
 
 command             = known_command | text_command | generic_command
 known_command       = footnote | includegraphics | caption | multicolumn
-text_command        = TXTCOMMAND | ESCAPED | BRACKETS | LINEFEED
+text_command        = TXTCOMMAND | ESCAPED | BRACKETS
 generic_command     = !no_command CMDNAME [[ //~ config ] //~ block ]
 
 footnote            = "\footnote" block_of_paragraphs

examples/LaTeX/LaTeXCompiler.py

@@ -19,7 +19,7 @@ from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, Altern
     Required, Token, Synonym, \
     Optional, NegativeLookbehind, OneOrMore, RegExp, Series, RE, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
-    PreprocessorFunc, \
+    PreprocessorFunc, TransformationDict, \
     Node, TransformationFunc, traverse, remove_children_if, is_anonymous, \
     reduce_single_child, replace_by_single_child, remove_whitespace, \
     flatten, is_empty, collapse, replace_content, remove_brackets, is_one_of, remove_first
@@ -126,7 +126,7 @@ class LaTeXGrammar(Grammar):
     
     block_of_paragraphs = /{/~ sequence §/}/
     sequence            = { (paragraph | block_environment ) [PARSEP] }+
-    paragraph           = { !blockcmd text_element //~ }+
+    paragraph           = { !blockcmd (text_element | LINEFEED) //~ }+
     text_element        = text | block | inline_environment | command
     
     #### inline enivronments ####
@@ -147,7 +147,7 @@ class LaTeXGrammar(Grammar):
     
     command             = known_command | text_command | generic_command
     known_command       = footnote | includegraphics | caption | multicolumn
-    text_command        = TXTCOMMAND | ESCAPED | BRACKETS | LINEFEED
+    text_command        = TXTCOMMAND | ESCAPED | BRACKETS
     generic_command     = !no_command CMDNAME [[ //~ config ] //~ block ]
     
     footnote            = "\footnote" block_of_paragraphs
@@ -216,7 +216,7 @@ class LaTeXGrammar(Grammar):
     paragraph = Forward()
     tabular_config = Forward()
     text_element = Forward()
-    source_hash__ = "fc3ee1800932b561e9cec1e22aab7157"
+    source_hash__ = "a99d24bcde48695ca003d544738b8057"
     parser_initialization__ = "upon instantiation"
     COMMENT__ = r'%.*(?:\n|$)'
     WSP__ = mixin_comment(whitespace=r'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?', comment=r'%.*(?:\n|$)')
@@ -250,7 +250,7 @@ class LaTeXGrammar(Grammar):
     includegraphics = Series(Token("\\includegraphics"), Optional(config), block)
     footnote = Series(Token("\\footnote"), block_of_paragraphs)
     generic_command = Series(NegativeLookahead(no_command), CMDNAME, Optional(Series(Optional(Series(RE(''), config)), RE(''), block)))
-    text_command = Alternative(TXTCOMMAND, ESCAPED, BRACKETS, LINEFEED)
+    text_command = Alternative(TXTCOMMAND, ESCAPED, BRACKETS)
     known_command = Alternative(footnote, includegraphics, caption, multicolumn)
     command = Alternative(known_command, text_command, generic_command)
     inline_math = Series(RegExp('\\$'), RegExp('[^$]*'), Required(RegExp('\\$')))
@@ -262,7 +262,8 @@ class LaTeXGrammar(Grammar):
     known_inline_env = Synonym(inline_math)
     inline_environment = Alternative(known_inline_env, generic_inline_env)
     text_element.set(Alternative(text, block, inline_environment, command))
-    paragraph.set(OneOrMore(Series(NegativeLookahead(blockcmd), text_element, RE(''))))
+    paragraph.set(
+        OneOrMore(Series(NegativeLookahead(blockcmd), Alternative(text_element, LINEFEED), RE(''))))
     sequence = OneOrMore(Series(Alternative(paragraph, block_environment), Optional(PARSEP)))
     block_of_paragraphs.set(Series(RE('{'), sequence, Required(RegExp('}'))))
     tabular_config.set(Series(Token("{"), RE('[lcr|]+'), Required(Token("}"))))
@@ -409,11 +410,19 @@ LaTeX_AST_transformation_table = {
     "*": replace_by_single_child
 }
 
-LaTeXTransform = partial(traverse, processing_table=LaTeX_AST_transformation_table)
 
+def LaTeXTransform() -> TransformationDict:
+    return partial(traverse, processing_table=LaTeX_AST_transformation_table.copy())
 
 def get_transformer() -> TransformationFunc:
-    return LaTeXTransform
+    global thread_local_LaTeX_transformer_singleton
+    try:
+        transformer = thread_local_LaTeX_transformer_singleton
+    except NameError:
+        thread_local_LaTeX_transformer_singleton = LaTeXTransform()
+        transformer = thread_local_LaTeX_transformer_singleton
+    return transformer
+
 
 
 #######################################################################

test/test_ebnf.py

@@ -31,7 +31,7 @@ sys.path.extend(['../', './'])
 
 from DHParser.toolkit import compile_python_object
 from DHParser.parser import compile_source, WHITESPACE_PTYPE, nil_preprocessor
-from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
+from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransform, get_ebnf_compiler
 from DHParser.dsl import CompilationError, compileDSL, DHPARSER_IMPORTS, grammar_provider
 
 
@@ -156,7 +156,7 @@ class TestSemanticValidation:
         grammar = get_ebnf_grammar()
         st = grammar(minilang)
         assert not st.collect_errors()
-        EBNFTransformer(st)
+        EBNFTransform()(st)
         assert bool_filter(st.collect_errors())
 
     def test_illegal_nesting(self):