Commit 337cc278 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

- transform.py: thread-safe between call caching for function traverse()

parent eaa04b6f
...@@ -30,7 +30,7 @@ try: ...@@ -30,7 +30,7 @@ try:
except ImportError: except ImportError:
from .typing34 import Any, cast, Tuple, Union from .typing34 import Any, cast, Tuple, Union
from DHParser.ebnf import EBNFTransformer, EBNFCompiler, grammar_changed, \ from DHParser.ebnf import EBNFCompiler, grammar_changed, \
get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \ get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object
...@@ -80,7 +80,7 @@ from DHParser import logging, is_filename, load_if_file, \\ ...@@ -80,7 +80,7 @@ from DHParser import logging, is_filename, load_if_file, \\
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\ Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\ ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
last_value, counterpart, accumulate, PreprocessorFunc, \\ last_value, counterpart, accumulate, PreprocessorFunc, \\
Node, TransformationFunc, TRUE_CONDITION, \\ Node, TransformationFunc, TransformationDict, TRUE_CONDITION, \\
traverse, remove_children_if, merge_children, is_anonymous, \\ traverse, remove_children_if, merge_children, is_anonymous, \\
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\ reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\ remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
...@@ -219,7 +219,8 @@ def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler: ...@@ -219,7 +219,8 @@ def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
""" """
grammar = get_ebnf_grammar() grammar = get_ebnf_grammar()
compiler = get_ebnf_compiler(branding , ebnf_src) compiler = get_ebnf_compiler(branding , ebnf_src)
compileDSL(ebnf_src, nil_preprocessor, grammar, EBNFTransformer, compiler) transformer = get_ebnf_transformer()
compileDSL(ebnf_src, nil_preprocessor, grammar, transformer, compiler)
return compiler return compiler
......
...@@ -18,6 +18,7 @@ permissions and limitations under the License. ...@@ -18,6 +18,7 @@ permissions and limitations under the License.
import keyword import keyword
from collections import OrderedDict from collections import OrderedDict
from functools import partial
try: try:
import regex as re import regex as re
...@@ -33,9 +34,9 @@ from DHParser.parser import Grammar, mixin_comment, nil_preprocessor, Forward, R ...@@ -33,9 +34,9 @@ from DHParser.parser import Grammar, mixin_comment, nil_preprocessor, Forward, R
Alternative, Series, Optional, Required, OneOrMore, ZeroOrMore, Token, Compiler, \ Alternative, Series, Optional, Required, OneOrMore, ZeroOrMore, Token, Compiler, \
PreprocessorFunc PreprocessorFunc
from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, Node, TransformationFunc from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, Node, TransformationFunc
from DHParser.transform import traverse, remove_brackets, \ from DHParser.transform import TransformationDict, traverse, remove_brackets, \
reduce_single_child, replace_by_single_child, remove_expendables, \ reduce_single_child, replace_by_single_child, remove_expendables, \
remove_tokens, flatten, forbid, assert_content, key_tag_name, remove_infix_operator remove_tokens, flatten, forbid, assert_content, remove_infix_operator
from DHParser.versionnumber import __version__ from DHParser.versionnumber import __version__
__all__ = ('get_ebnf_preprocessor', __all__ = ('get_ebnf_preprocessor',
...@@ -43,7 +44,7 @@ __all__ = ('get_ebnf_preprocessor', ...@@ -43,7 +44,7 @@ __all__ = ('get_ebnf_preprocessor',
'get_ebnf_transformer', 'get_ebnf_transformer',
'get_ebnf_compiler', 'get_ebnf_compiler',
'EBNFGrammar', 'EBNFGrammar',
'EBNFTransformer', 'EBNFTransform',
'EBNFCompilerError', 'EBNFCompilerError',
'EBNFCompiler', 'EBNFCompiler',
'grammar_changed', 'grammar_changed',
...@@ -191,7 +192,7 @@ def get_ebnf_grammar() -> EBNFGrammar: ...@@ -191,7 +192,7 @@ def get_ebnf_grammar() -> EBNFGrammar:
######################################################################## ########################################################################
EBNF_transformation_table = { EBNF_AST_transformation_table = {
# AST Transformations for EBNF-grammar # AST Transformations for EBNF-grammar
"+": "+":
remove_expendables, remove_expendables,
...@@ -221,12 +222,17 @@ EBNF_transformation_table = { ...@@ -221,12 +222,17 @@ EBNF_transformation_table = {
} }
def EBNFTransformer(syntax_tree: Node): def EBNFTransform() -> TransformationDict:
traverse(syntax_tree, EBNF_transformation_table, key_tag_name) return partial(traverse, processing_table=EBNF_AST_transformation_table.copy())
def get_ebnf_transformer() -> TransformationFunc: def get_ebnf_transformer() -> TransformationFunc:
return EBNFTransformer global thread_local_EBNF_transformer_singleton
try:
transformer = thread_local_EBNF_transformer_singleton
except NameError:
thread_local_EBNF_transformer_singleton = EBNFTransform()
transformer = thread_local_EBNF_transformer_singleton
return transformer
######################################################################## ########################################################################
...@@ -252,16 +258,25 @@ def get_grammar() -> {NAME}Grammar: ...@@ -252,16 +258,25 @@ def get_grammar() -> {NAME}Grammar:
global thread_local_{NAME}_grammar_singleton global thread_local_{NAME}_grammar_singleton
try: try:
grammar = thread_local_{NAME}_grammar_singleton grammar = thread_local_{NAME}_grammar_singleton
return grammar
except NameError: except NameError:
thread_local_{NAME}_grammar_singleton = {NAME}Grammar() thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
return thread_local_{NAME}_grammar_singleton grammar = thread_local_{NAME}_grammar_singleton
return grammar
''' '''
TRANSFORMER_FACTORY = ''' TRANSFORMER_FACTORY = '''
def {NAME}Transform() -> TransformationDict:
return partial(traverse, processing_table={NAME}_AST_transformation_table.copy())
def get_transformer() -> TransformationFunc: def get_transformer() -> TransformationFunc:
return {NAME}Transform global thread_local_{NAME}_transformer_singleton
try:
transformer = thread_local_{NAME}_transformer_singleton
except NameError:
thread_local_{NAME}_transformer_singleton = {NAME}Transform()
transformer = thread_local_{NAME}_transformer_singleton
return transformer
''' '''
...@@ -271,11 +286,11 @@ def get_compiler(grammar_name="{NAME}", grammar_source="") -> {NAME}Compiler: ...@@ -271,11 +286,11 @@ def get_compiler(grammar_name="{NAME}", grammar_source="") -> {NAME}Compiler:
try: try:
compiler = thread_local_{NAME}_compiler_singleton compiler = thread_local_{NAME}_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source) compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except NameError: except NameError:
thread_local_{NAME}_compiler_singleton = \\ thread_local_{NAME}_compiler_singleton = \\
{NAME}Compiler(grammar_name, grammar_source) {NAME}Compiler(grammar_name, grammar_source)
return thread_local_{NAME}_compiler_singleton compiler = thread_local_{NAME}_compiler_singleton
return compiler
''' '''
...@@ -404,7 +419,6 @@ class EBNFCompiler(Compiler): ...@@ -404,7 +419,6 @@ class EBNFCompiler(Compiler):
raise EBNFCompilerError('Compiler must be run before calling ' raise EBNFCompilerError('Compiler must be run before calling '
'"gen_transformer_Skeleton()"!') '"gen_transformer_Skeleton()"!')
tt_name = self.grammar_name + '_AST_transformation_table' tt_name = self.grammar_name + '_AST_transformation_table'
tf_name = self.grammar_name + 'Transform'
transtable = [tt_name + ' = {', transtable = [tt_name + ' = {',
' # AST Transformations for the ' + ' # AST Transformations for the ' +
self.grammar_name + '-grammar'] self.grammar_name + '-grammar']
...@@ -418,8 +432,7 @@ class EBNFCompiler(Compiler): ...@@ -418,8 +432,7 @@ class EBNFCompiler(Compiler):
tf = '[replace_by_single_child]' tf = '[replace_by_single_child]'
transtable.append(' "' + name + '": %s,' % tf) transtable.append(' "' + name + '": %s,' % tf)
transtable.append(' ":Token, :RE": reduce_single_child,') transtable.append(' ":Token, :RE": reduce_single_child,')
transtable += [' "*": replace_by_single_child', '}', '', tf_name + transtable += [' "*": replace_by_single_child', '}', '']
' = partial(traverse, processing_table=%s)' % tt_name, '']
transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)] transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
return '\n'.join(transtable) return '\n'.join(transtable)
......
...@@ -35,7 +35,11 @@ except ImportError: ...@@ -35,7 +35,11 @@ except ImportError:
from DHParser.toolkit import expand_table, smart_list from DHParser.toolkit import expand_table, smart_list
__all__ = ('transformation_factory', __all__ = ('TransformationDict',
'TransformationProc',
'ConditionFunc',
'KeyFunc',
'transformation_factory',
'key_parser_name', 'key_parser_name',
'key_tag_name', 'key_tag_name',
'traverse', 'traverse',
...@@ -78,6 +82,7 @@ __all__ = ('transformation_factory', ...@@ -78,6 +82,7 @@ __all__ = ('transformation_factory',
TransformationProc = Callable[[List[Node]], None] TransformationProc = Callable[[List[Node]], None]
TransformationDict = Dict
ConditionFunc = Callable # Callable[[List[Node]], bool] ConditionFunc = Callable # Callable[[List[Node]], bool]
KeyFunc = Callable[[Node], str] KeyFunc = Callable[[Node], str]
...@@ -191,8 +196,8 @@ def traverse(root_node: Node, ...@@ -191,8 +196,8 @@ def traverse(root_node: Node,
root_node (Node): The root-node of the syntax tree to be traversed root_node (Node): The root-node of the syntax tree to be traversed
processing_table (dict): node key -> sequence of functions that processing_table (dict): node key -> sequence of functions that
will be applied to matching nodes in order. This dictionary will be applied to matching nodes in order. This dictionary
is interpreted as a ``compact_table``. See is interpreted as a `compact_table`. See
``toolkit.expand_table`` or ``EBNFCompiler.EBNFTransTable`` `toolkit.expand_table` or ``EBNFCompiler.EBNFTransTable`
key_func (function): A mapping key_func(node) -> keystr. The default key_func (function): A mapping key_func(node) -> keystr. The default
key_func yields node.parser.name. key_func yields node.parser.name.
...@@ -201,11 +206,27 @@ def traverse(root_node: Node, ...@@ -201,11 +206,27 @@ def traverse(root_node: Node,
"factor, flowmarker, retrieveop": replace_by_single_child } "factor, flowmarker, retrieveop": replace_by_single_child }
traverse(node, table) traverse(node, table)
""" """
# normalize processing_table entries by turning single values into lists # Is this optimazation really needed?
# with a single value if '__cache__' in processing_table:
table = {name: smart_list(call) for name, call in list(processing_table.items())} # assume that processing table has already been expanded
table = expand_table(table) table = processing_table
cache = {} # type: Dict[str, List[Callable]] cache = processing_table['__cache__']
else:
# normalize processing_table entries by turning single values into lists
# with a single value
table = {name: smart_list(call) for name, call in list(processing_table.items())}
table = expand_table(table)
cache = table.setdefault('__cache__', {}) # type: Dict[str, List[Callable]]
# change processing table in place, so that table expansion does not get lost
# between calls
processing_table.clear();
processing_table.update(table)
# assert '__cache__' in processing_table
# # Code without optimization
# table = {name: smart_list(call) for name, call in list(processing_table.items())}
# table = expand_table(table)
# cache = {}
def traverse_recursive(context): def traverse_recursive(context):
node = context[-1] node = context[-1]
...@@ -232,6 +253,8 @@ def traverse(root_node: Node, ...@@ -232,6 +253,8 @@ def traverse(root_node: Node,
call(context) call(context)
traverse_recursive([root_node]) traverse_recursive([root_node])
# assert processing_table['__cache__']
# ------------------------------------------------ # ------------------------------------------------
......
...@@ -77,7 +77,7 @@ tabular_config = "{" /[lcr|]+/~ §"}" ...@@ -77,7 +77,7 @@ tabular_config = "{" /[lcr|]+/~ §"}"
block_of_paragraphs = /{/~ sequence §/}/ block_of_paragraphs = /{/~ sequence §/}/
sequence = { (paragraph | block_environment ) [PARSEP] }+ sequence = { (paragraph | block_environment ) [PARSEP] }+
paragraph = { !blockcmd text_element //~ }+ paragraph = { !blockcmd (text_element | LINEFEED) //~ }+
text_element = text | block | inline_environment | command text_element = text | block | inline_environment | command
#### inline enivronments #### #### inline enivronments ####
...@@ -98,7 +98,7 @@ inline_math = /\$/ /[^$]*/ §/\$/ ...@@ -98,7 +98,7 @@ inline_math = /\$/ /[^$]*/ §/\$/
command = known_command | text_command | generic_command command = known_command | text_command | generic_command
known_command = footnote | includegraphics | caption | multicolumn known_command = footnote | includegraphics | caption | multicolumn
text_command = TXTCOMMAND | ESCAPED | BRACKETS | LINEFEED text_command = TXTCOMMAND | ESCAPED | BRACKETS
generic_command = !no_command CMDNAME [[ //~ config ] //~ block ] generic_command = !no_command CMDNAME [[ //~ config ] //~ block ]
footnote = "\footnote" block_of_paragraphs footnote = "\footnote" block_of_paragraphs
......
...@@ -19,7 +19,7 @@ from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, Altern ...@@ -19,7 +19,7 @@ from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, Altern
Required, Token, Synonym, \ Required, Token, Synonym, \
Optional, NegativeLookbehind, OneOrMore, RegExp, Series, RE, Capture, \ Optional, NegativeLookbehind, OneOrMore, RegExp, Series, RE, Capture, \
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \ ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
PreprocessorFunc, \ PreprocessorFunc, TransformationDict, \
Node, TransformationFunc, traverse, remove_children_if, is_anonymous, \ Node, TransformationFunc, traverse, remove_children_if, is_anonymous, \
reduce_single_child, replace_by_single_child, remove_whitespace, \ reduce_single_child, replace_by_single_child, remove_whitespace, \
flatten, is_empty, collapse, replace_content, remove_brackets, is_one_of, remove_first flatten, is_empty, collapse, replace_content, remove_brackets, is_one_of, remove_first
...@@ -126,7 +126,7 @@ class LaTeXGrammar(Grammar): ...@@ -126,7 +126,7 @@ class LaTeXGrammar(Grammar):
block_of_paragraphs = /{/~ sequence §/}/ block_of_paragraphs = /{/~ sequence §/}/
sequence = { (paragraph | block_environment ) [PARSEP] }+ sequence = { (paragraph | block_environment ) [PARSEP] }+
paragraph = { !blockcmd text_element //~ }+ paragraph = { !blockcmd (text_element | LINEFEED) //~ }+
text_element = text | block | inline_environment | command text_element = text | block | inline_environment | command
#### inline enivronments #### #### inline enivronments ####
...@@ -147,7 +147,7 @@ class LaTeXGrammar(Grammar): ...@@ -147,7 +147,7 @@ class LaTeXGrammar(Grammar):
command = known_command | text_command | generic_command command = known_command | text_command | generic_command
known_command = footnote | includegraphics | caption | multicolumn known_command = footnote | includegraphics | caption | multicolumn
text_command = TXTCOMMAND | ESCAPED | BRACKETS | LINEFEED text_command = TXTCOMMAND | ESCAPED | BRACKETS
generic_command = !no_command CMDNAME [[ //~ config ] //~ block ] generic_command = !no_command CMDNAME [[ //~ config ] //~ block ]
footnote = "\footnote" block_of_paragraphs footnote = "\footnote" block_of_paragraphs
...@@ -216,7 +216,7 @@ class LaTeXGrammar(Grammar): ...@@ -216,7 +216,7 @@ class LaTeXGrammar(Grammar):
paragraph = Forward() paragraph = Forward()
tabular_config = Forward() tabular_config = Forward()
text_element = Forward() text_element = Forward()
source_hash__ = "fc3ee1800932b561e9cec1e22aab7157" source_hash__ = "a99d24bcde48695ca003d544738b8057"
parser_initialization__ = "upon instantiation" parser_initialization__ = "upon instantiation"
COMMENT__ = r'%.*(?:\n|$)' COMMENT__ = r'%.*(?:\n|$)'
WSP__ = mixin_comment(whitespace=r'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?', comment=r'%.*(?:\n|$)') WSP__ = mixin_comment(whitespace=r'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?', comment=r'%.*(?:\n|$)')
...@@ -250,7 +250,7 @@ class LaTeXGrammar(Grammar): ...@@ -250,7 +250,7 @@ class LaTeXGrammar(Grammar):
includegraphics = Series(Token("\\includegraphics"), Optional(config), block) includegraphics = Series(Token("\\includegraphics"), Optional(config), block)
footnote = Series(Token("\\footnote"), block_of_paragraphs) footnote = Series(Token("\\footnote"), block_of_paragraphs)
generic_command = Series(NegativeLookahead(no_command), CMDNAME, Optional(Series(Optional(Series(RE(''), config)), RE(''), block))) generic_command = Series(NegativeLookahead(no_command), CMDNAME, Optional(Series(Optional(Series(RE(''), config)), RE(''), block)))
text_command = Alternative(TXTCOMMAND, ESCAPED, BRACKETS, LINEFEED) text_command = Alternative(TXTCOMMAND, ESCAPED, BRACKETS)
known_command = Alternative(footnote, includegraphics, caption, multicolumn) known_command = Alternative(footnote, includegraphics, caption, multicolumn)
command = Alternative(known_command, text_command, generic_command) command = Alternative(known_command, text_command, generic_command)
inline_math = Series(RegExp('\\$'), RegExp('[^$]*'), Required(RegExp('\\$'))) inline_math = Series(RegExp('\\$'), RegExp('[^$]*'), Required(RegExp('\\$')))
...@@ -262,7 +262,8 @@ class LaTeXGrammar(Grammar): ...@@ -262,7 +262,8 @@ class LaTeXGrammar(Grammar):
known_inline_env = Synonym(inline_math) known_inline_env = Synonym(inline_math)
inline_environment = Alternative(known_inline_env, generic_inline_env) inline_environment = Alternative(known_inline_env, generic_inline_env)
text_element.set(Alternative(text, block, inline_environment, command)) text_element.set(Alternative(text, block, inline_environment, command))
paragraph.set(OneOrMore(Series(NegativeLookahead(blockcmd), text_element, RE('')))) paragraph.set(
OneOrMore(Series(NegativeLookahead(blockcmd), Alternative(text_element, LINEFEED), RE(''))))
sequence = OneOrMore(Series(Alternative(paragraph, block_environment), Optional(PARSEP))) sequence = OneOrMore(Series(Alternative(paragraph, block_environment), Optional(PARSEP)))
block_of_paragraphs.set(Series(RE('{'), sequence, Required(RegExp('}')))) block_of_paragraphs.set(Series(RE('{'), sequence, Required(RegExp('}'))))
tabular_config.set(Series(Token("{"), RE('[lcr|]+'), Required(Token("}")))) tabular_config.set(Series(Token("{"), RE('[lcr|]+'), Required(Token("}"))))
...@@ -409,11 +410,19 @@ LaTeX_AST_transformation_table = { ...@@ -409,11 +410,19 @@ LaTeX_AST_transformation_table = {
"*": replace_by_single_child "*": replace_by_single_child
} }
LaTeXTransform = partial(traverse, processing_table=LaTeX_AST_transformation_table)
def LaTeXTransform() -> TransformationDict:
return partial(traverse, processing_table=LaTeX_AST_transformation_table.copy())
def get_transformer() -> TransformationFunc: def get_transformer() -> TransformationFunc:
return LaTeXTransform global thread_local_LaTeX_transformer_singleton
try:
transformer = thread_local_LaTeX_transformer_singleton
except NameError:
thread_local_LaTeX_transformer_singleton = LaTeXTransform()
transformer = thread_local_LaTeX_transformer_singleton
return transformer
####################################################################### #######################################################################
......
...@@ -31,7 +31,7 @@ sys.path.extend(['../', './']) ...@@ -31,7 +31,7 @@ sys.path.extend(['../', './'])
from DHParser.toolkit import compile_python_object from DHParser.toolkit import compile_python_object
from DHParser.parser import compile_source, WHITESPACE_PTYPE, nil_preprocessor from DHParser.parser import compile_source, WHITESPACE_PTYPE, nil_preprocessor
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransform, get_ebnf_compiler
from DHParser.dsl import CompilationError, compileDSL, DHPARSER_IMPORTS, grammar_provider from DHParser.dsl import CompilationError, compileDSL, DHPARSER_IMPORTS, grammar_provider
...@@ -156,7 +156,7 @@ class TestSemanticValidation: ...@@ -156,7 +156,7 @@ class TestSemanticValidation:
grammar = get_ebnf_grammar() grammar = get_ebnf_grammar()
st = grammar(minilang) st = grammar(minilang)
assert not st.collect_errors() assert not st.collect_errors()
EBNFTransformer(st) EBNFTransform()(st)
assert bool_filter(st.collect_errors()) assert bool_filter(st.collect_errors())
def test_illegal_nesting(self): def test_illegal_nesting(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment