Skip to content

GitLab

D
DHParser
Commit 337cc278 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files
Options

- transform.py: thread-safe between call caching for function traverse()

parent eaa04b6f
Hide whitespace changes
Inline Side-by-side
Showing with 85 additions and 39 deletions
DHParser/dsl.py
View file @ 337cc278
......@@ -30,7 +30,7 @@ try:
except ImportError:
from .typing34 import Any, cast, Tuple, Union
from DHParser.ebnf import EBNFTransformer, EBNFCompiler, grammar_changed, \
from DHParser.ebnf import EBNFCompiler, grammar_changed, \
get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object
......@@ -80,7 +80,7 @@ from DHParser import logging, is_filename, load_if_file, \\
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
last_value, counterpart, accumulate, PreprocessorFunc, \\
Node, TransformationFunc, TRUE_CONDITION, \\
Node, TransformationFunc, TransformationDict, TRUE_CONDITION, \\
traverse, remove_children_if, merge_children, is_anonymous, \\
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
......@@ -219,7 +219,8 @@ def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
"""
grammar = get_ebnf_grammar()
compiler = get_ebnf_compiler(branding , ebnf_src)
compileDSL(ebnf_src, nil_preprocessor, grammar, EBNFTransformer, compiler)
transformer = get_ebnf_transformer()
compileDSL(ebnf_src, nil_preprocessor, grammar, transformer, compiler)
return compiler
......
DHParser/ebnf.py
View file @ 337cc278
......@@ -18,6 +18,7 @@ permissions and limitations under the License.
import keyword
from collections import OrderedDict
from functools import partial
try:
import regex as re
......@@ -33,9 +34,9 @@ from DHParser.parser import Grammar, mixin_comment, nil_preprocessor, Forward, R
Alternative, Series, Optional, Required, OneOrMore, ZeroOrMore, Token, Compiler, \
PreprocessorFunc
from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, Node, TransformationFunc
from DHParser.transform import traverse, remove_brackets, \
from DHParser.transform import TransformationDict, traverse, remove_brackets, \
reduce_single_child, replace_by_single_child, remove_expendables, \
remove_tokens, flatten, forbid, assert_content, key_tag_name, remove_infix_operator
remove_tokens, flatten, forbid, assert_content, remove_infix_operator
from DHParser.versionnumber import __version__
__all__ = ('get_ebnf_preprocessor',
......@@ -43,7 +44,7 @@ __all__ = ('get_ebnf_preprocessor',
'get_ebnf_transformer',
'get_ebnf_compiler',
'EBNFGrammar',
'EBNFTransformer',
'EBNFTransform',
'EBNFCompilerError',
'EBNFCompiler',
'grammar_changed',
......@@ -191,7 +192,7 @@ def get_ebnf_grammar() -> EBNFGrammar:
########################################################################
EBNF_transformation_table = {
EBNF_AST_transformation_table = {
# AST Transformations for EBNF-grammar
"+":
remove_expendables,
......@@ -221,12 +222,17 @@ EBNF_transformation_table = {
}
def EBNFTransformer(syntax_tree: Node):
traverse(syntax_tree, EBNF_transformation_table, key_tag_name)
def EBNFTransform() -> TransformationDict:
return partial(traverse, processing_table=EBNF_AST_transformation_table.copy())
def get_ebnf_transformer() -> TransformationFunc:
return EBNFTransformer
global thread_local_EBNF_transformer_singleton
try:
transformer = thread_local_EBNF_transformer_singleton
except NameError:
thread_local_EBNF_transformer_singleton = EBNFTransform()
transformer = thread_local_EBNF_transformer_singleton
return transformer
########################################################################
......@@ -252,16 +258,25 @@ def get_grammar() -> {NAME}Grammar:
global thread_local_{NAME}_grammar_singleton
try:
grammar = thread_local_{NAME}_grammar_singleton
return grammar
except NameError:
thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
return thread_local_{NAME}_grammar_singleton
grammar = thread_local_{NAME}_grammar_singleton
return grammar
'''
TRANSFORMER_FACTORY = '''
def {NAME}Transform() -> TransformationDict:
return partial(traverse, processing_table={NAME}_AST_transformation_table.copy())
def get_transformer() -> TransformationFunc:
return {NAME}Transform
global thread_local_{NAME}_transformer_singleton
try:
transformer = thread_local_{NAME}_transformer_singleton
except NameError:
thread_local_{NAME}_transformer_singleton = {NAME}Transform()
transformer = thread_local_{NAME}_transformer_singleton
return transformer
'''
......@@ -271,11 +286,11 @@ def get_compiler(grammar_name="{NAME}", grammar_source="") -> {NAME}Compiler:
try:
compiler = thread_local_{NAME}_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except NameError:
thread_local_{NAME}_compiler_singleton = \\
{NAME}Compiler(grammar_name, grammar_source)
return thread_local_{NAME}_compiler_singleton
compiler = thread_local_{NAME}_compiler_singleton
return compiler
'''
......@@ -404,7 +419,6 @@ class EBNFCompiler(Compiler):
raise EBNFCompilerError('Compiler must be run before calling '
'"gen_transformer_Skeleton()"!')
tt_name = self.grammar_name + '_AST_transformation_table'
tf_name = self.grammar_name + 'Transform'
transtable = [tt_name + ' = {',
' # AST Transformations for the ' +
self.grammar_name + '-grammar']
......@@ -418,8 +432,7 @@ class EBNFCompiler(Compiler):
tf = '[replace_by_single_child]'
transtable.append(' "' + name + '": %s,' % tf)
transtable.append(' ":Token, :RE": reduce_single_child,')
transtable += [' "*": replace_by_single_child', '}', '', tf_name +
' = partial(traverse, processing_table=%s)' % tt_name, '']
transtable += [' "*": replace_by_single_child', '}', '']
transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
return '\n'.join(transtable)
......
DHParser/transform.py
View file @ 337cc278
......@@ -35,7 +35,11 @@ except ImportError:
from DHParser.toolkit import expand_table, smart_list
__all__ = ('transformation_factory',
__all__ = ('TransformationDict',
'TransformationProc',
'ConditionFunc',
'KeyFunc',
'transformation_factory',
'key_parser_name',
'key_tag_name',
'traverse',
......@@ -78,6 +82,7 @@ __all__ = ('transformation_factory',
TransformationProc = Callable[[List[Node]], None]
TransformationDict = Dict
ConditionFunc = Callable # Callable[[List[Node]], bool]
KeyFunc = Callable[[Node], str]
......@@ -191,8 +196,8 @@ def traverse(root_node: Node,
root_node (Node): The root-node of the syntax tree to be traversed
processing_table (dict): node key -> sequence of functions that
will be applied to matching nodes in order. This dictionary
is interpreted as a ``compact_table``. See
``toolkit.expand_table`` or ``EBNFCompiler.EBNFTransTable``
is interpreted as a `compact_table`. See
`toolkit.expand_table` or ``EBNFCompiler.EBNFTransTable`
key_func (function): A mapping key_func(node) -> keystr. The default
key_func yields node.parser.name.
......@@ -201,11 +206,27 @@ def traverse(root_node: Node,
"factor, flowmarker, retrieveop": replace_by_single_child }
traverse(node, table)
"""
# normalize processing_table entries by turning single values into lists
# with a single value
table = {name: smart_list(call) for name, call in list(processing_table.items())}
table = expand_table(table)
cache = {} # type: Dict[str, List[Callable]]
# Is this optimazation really needed?
if '__cache__' in processing_table:
# assume that processing table has already been expanded
table = processing_table
cache = processing_table['__cache__']
else:
# normalize processing_table entries by turning single values into lists
# with a single value
table = {name: smart_list(call) for name, call in list(processing_table.items())}
table = expand_table(table)
cache = table.setdefault('__cache__', {}) # type: Dict[str, List[Callable]]
# change processing table in place, so that table expansion does not get lost
# between calls
processing_table.clear();
processing_table.update(table)
# assert '__cache__' in processing_table
# # Code without optimization
# table = {name: smart_list(call) for name, call in list(processing_table.items())}
# table = expand_table(table)
# cache = {}
def traverse_recursive(context):
node = context[-1]
......@@ -232,6 +253,8 @@ def traverse(root_node: Node,
call(context)
traverse_recursive([root_node])
# assert processing_table['__cache__']
# ------------------------------------------------
......
examples/LaTeX/LaTeX.ebnf
View file @ 337cc278
......@@ -77,7 +77,7 @@ tabular_config = "{" /[lcr|]+/~ §"}"
block_of_paragraphs = /{/~ sequence §/}/
sequence = { (paragraph | block_environment ) [PARSEP] }+
paragraph = { !blockcmd text_element //~ }+
paragraph = { !blockcmd (text_element | LINEFEED) //~ }+
text_element = text | block | inline_environment | command
#### inline enivronments ####
......@@ -98,7 +98,7 @@ inline_math = /\$/ /[^$]*/ §/\$/
command = known_command | text_command | generic_command
known_command = footnote | includegraphics | caption | multicolumn
text_command = TXTCOMMAND | ESCAPED | BRACKETS | LINEFEED
text_command = TXTCOMMAND | ESCAPED | BRACKETS
generic_command = !no_command CMDNAME [[ //~ config ] //~ block ]
footnote = "\footnote" block_of_paragraphs
......
examples/LaTeX/LaTeXCompiler.py
View file @ 337cc278
......@@ -19,7 +19,7 @@ from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, Altern
Required, Token, Synonym, \
Optional, NegativeLookbehind, OneOrMore, RegExp, Series, RE, Capture, \
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
PreprocessorFunc, \
PreprocessorFunc, TransformationDict, \
Node, TransformationFunc, traverse, remove_children_if, is_anonymous, \
reduce_single_child, replace_by_single_child, remove_whitespace, \
flatten, is_empty, collapse, replace_content, remove_brackets, is_one_of, remove_first
......@@ -126,7 +126,7 @@ class LaTeXGrammar(Grammar):
block_of_paragraphs = /{/~ sequence §/}/
sequence = { (paragraph | block_environment ) [PARSEP] }+
paragraph = { !blockcmd text_element //~ }+
paragraph = { !blockcmd (text_element | LINEFEED) //~ }+
text_element = text | block | inline_environment | command
#### inline enivronments ####
......@@ -147,7 +147,7 @@ class LaTeXGrammar(Grammar):
command = known_command | text_command | generic_command
known_command = footnote | includegraphics | caption | multicolumn
text_command = TXTCOMMAND | ESCAPED | BRACKETS | LINEFEED
text_command = TXTCOMMAND | ESCAPED | BRACKETS
generic_command = !no_command CMDNAME [[ //~ config ] //~ block ]
footnote = "\footnote" block_of_paragraphs
......@@ -216,7 +216,7 @@ class LaTeXGrammar(Grammar):
paragraph = Forward()
tabular_config = Forward()
text_element = Forward()
source_hash__ = "fc3ee1800932b561e9cec1e22aab7157"
source_hash__ = "a99d24bcde48695ca003d544738b8057"
parser_initialization__ = "upon instantiation"
COMMENT__ = r'%.*(?:\n|$)'
WSP__ = mixin_comment(whitespace=r'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?', comment=r'%.*(?:\n|$)')
......@@ -250,7 +250,7 @@ class LaTeXGrammar(Grammar):
includegraphics = Series(Token("\\includegraphics"), Optional(config), block)
footnote = Series(Token("\\footnote"), block_of_paragraphs)
generic_command = Series(NegativeLookahead(no_command), CMDNAME, Optional(Series(Optional(Series(RE(''), config)), RE(''), block)))
text_command = Alternative(TXTCOMMAND, ESCAPED, BRACKETS, LINEFEED)
text_command = Alternative(TXTCOMMAND, ESCAPED, BRACKETS)
known_command = Alternative(footnote, includegraphics, caption, multicolumn)
command = Alternative(known_command, text_command, generic_command)
inline_math = Series(RegExp('\\$'), RegExp('[^$]*'), Required(RegExp('\\$')))
......@@ -262,7 +262,8 @@ class LaTeXGrammar(Grammar):
known_inline_env = Synonym(inline_math)
inline_environment = Alternative(known_inline_env, generic_inline_env)
text_element.set(Alternative(text, block, inline_environment, command))
paragraph.set(OneOrMore(Series(NegativeLookahead(blockcmd), text_element, RE(''))))
paragraph.set(
OneOrMore(Series(NegativeLookahead(blockcmd), Alternative(text_element, LINEFEED), RE(''))))
sequence = OneOrMore(Series(Alternative(paragraph, block_environment), Optional(PARSEP)))
block_of_paragraphs.set(Series(RE('{'), sequence, Required(RegExp('}'))))
tabular_config.set(Series(Token("{"), RE('[lcr|]+'), Required(Token("}"))))
......@@ -409,11 +410,19 @@ LaTeX_AST_transformation_table = {
"*": replace_by_single_child
}
LaTeXTransform = partial(traverse, processing_table=LaTeX_AST_transformation_table)
def LaTeXTransform() -> TransformationDict:
return partial(traverse, processing_table=LaTeX_AST_transformation_table.copy())
def get_transformer() -> TransformationFunc:
return LaTeXTransform
global thread_local_LaTeX_transformer_singleton
try:
transformer = thread_local_LaTeX_transformer_singleton
except NameError:
thread_local_LaTeX_transformer_singleton = LaTeXTransform()
transformer = thread_local_LaTeX_transformer_singleton
return transformer
#######################################################################
......
test/test_ebnf.py
View file @ 337cc278
......@@ -31,7 +31,7 @@ sys.path.extend(['../', './'])
from DHParser.toolkit import compile_python_object
from DHParser.parser import compile_source, WHITESPACE_PTYPE, nil_preprocessor
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransform, get_ebnf_compiler
from DHParser.dsl import CompilationError, compileDSL, DHPARSER_IMPORTS, grammar_provider
......@@ -156,7 +156,7 @@ class TestSemanticValidation:
grammar = get_ebnf_grammar()
st = grammar(minilang)
assert not st.collect_errors()
EBNFTransformer(st)
EBNFTransform()(st)
assert bool_filter(st.collect_errors())
def test_illegal_nesting(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment