Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
D
DHParser
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Iterations
Merge Requests
0
Merge Requests
0
Requirements
Requirements
List
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Operations
Operations
Incidents
Analytics
Analytics
Code Review
Insights
Issue
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
badw-it
DHParser
Commits
337cc278
Commit
337cc278
authored
Aug 20, 2017
by
Eckhart Arnold
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
- transform.py: thread-safe between call caching for function traverse()
parent
eaa04b6f
Changes
6
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
85 additions
and
39 deletions
+85
-39
DHParser/dsl.py
DHParser/dsl.py
+4
-3
DHParser/ebnf.py
DHParser/ebnf.py
+29
-16
DHParser/transform.py
DHParser/transform.py
+31
-8
examples/LaTeX/LaTeX.ebnf
examples/LaTeX/LaTeX.ebnf
+2
-2
examples/LaTeX/LaTeXCompiler.py
examples/LaTeX/LaTeXCompiler.py
+17
-8
test/test_ebnf.py
test/test_ebnf.py
+2
-2
No files found.
DHParser/dsl.py
View file @
337cc278
...
...
@@ -30,7 +30,7 @@ try:
except
ImportError
:
from
.typing34
import
Any
,
cast
,
Tuple
,
Union
from
DHParser.ebnf
import
EBNF
Transformer
,
EBNF
Compiler
,
grammar_changed
,
\
from
DHParser.ebnf
import
EBNFCompiler
,
grammar_changed
,
\
get_ebnf_preprocessor
,
get_ebnf_grammar
,
get_ebnf_transformer
,
get_ebnf_compiler
,
\
PreprocessorFactoryFunc
,
ParserFactoryFunc
,
TransformerFactoryFunc
,
CompilerFactoryFunc
from
DHParser.toolkit
import
logging
,
load_if_file
,
is_python_code
,
compile_python_object
...
...
@@ -80,7 +80,7 @@ from DHParser import logging, is_filename, load_if_file, \\
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture,
\\
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source,
\\
last_value, counterpart, accumulate, PreprocessorFunc,
\\
Node, TransformationFunc, TRUE_CONDITION,
\\
Node, TransformationFunc, T
ransformationDict, T
RUE_CONDITION,
\\
traverse, remove_children_if, merge_children, is_anonymous,
\\
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace,
\\
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace,
\\
...
...
@@ -219,7 +219,8 @@ def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
"""
grammar
=
get_ebnf_grammar
()
compiler
=
get_ebnf_compiler
(
branding
,
ebnf_src
)
compileDSL
(
ebnf_src
,
nil_preprocessor
,
grammar
,
EBNFTransformer
,
compiler
)
transformer
=
get_ebnf_transformer
()
compileDSL
(
ebnf_src
,
nil_preprocessor
,
grammar
,
transformer
,
compiler
)
return
compiler
...
...
DHParser/ebnf.py
View file @
337cc278
...
...
@@ -18,6 +18,7 @@ permissions and limitations under the License.
import
keyword
from
collections
import
OrderedDict
from
functools
import
partial
try
:
import
regex
as
re
...
...
@@ -33,9 +34,9 @@ from DHParser.parser import Grammar, mixin_comment, nil_preprocessor, Forward, R
Alternative
,
Series
,
Optional
,
Required
,
OneOrMore
,
ZeroOrMore
,
Token
,
Compiler
,
\
PreprocessorFunc
from
DHParser.syntaxtree
import
WHITESPACE_PTYPE
,
TOKEN_PTYPE
,
Node
,
TransformationFunc
from
DHParser.transform
import
traverse
,
remove_brackets
,
\
from
DHParser.transform
import
TransformationDict
,
traverse
,
remove_brackets
,
\
reduce_single_child
,
replace_by_single_child
,
remove_expendables
,
\
remove_tokens
,
flatten
,
forbid
,
assert_content
,
key_tag_name
,
remove_infix_operator
remove_tokens
,
flatten
,
forbid
,
assert_content
,
remove_infix_operator
from
DHParser.versionnumber
import
__version__
__all__
=
(
'get_ebnf_preprocessor'
,
...
...
@@ -43,7 +44,7 @@ __all__ = ('get_ebnf_preprocessor',
'get_ebnf_transformer'
,
'get_ebnf_compiler'
,
'EBNFGrammar'
,
'EBNFTransform
er
'
,
'EBNFTransform'
,
'EBNFCompilerError'
,
'EBNFCompiler'
,
'grammar_changed'
,
...
...
@@ -191,7 +192,7 @@ def get_ebnf_grammar() -> EBNFGrammar:
########################################################################
EBNF_transformation_table
=
{
EBNF_
AST_
transformation_table
=
{
# AST Transformations for EBNF-grammar
"+"
:
remove_expendables
,
...
...
@@ -221,12 +222,17 @@ EBNF_transformation_table = {
}
def
EBNFTransformer
(
syntax_tree
:
Node
):
traverse
(
syntax_tree
,
EBNF_transformation_table
,
key_tag_name
)
def
EBNFTransform
()
->
TransformationDict
:
return
partial
(
traverse
,
processing_table
=
EBNF_AST_transformation_table
.
copy
())
def
get_ebnf_transformer
()
->
TransformationFunc
:
return
EBNFTransformer
global
thread_local_EBNF_transformer_singleton
try
:
transformer
=
thread_local_EBNF_transformer_singleton
except
NameError
:
thread_local_EBNF_transformer_singleton
=
EBNFTransform
()
transformer
=
thread_local_EBNF_transformer_singleton
return
transformer
########################################################################
...
...
@@ -252,16 +258,25 @@ def get_grammar() -> {NAME}Grammar:
global thread_local_{NAME}_grammar_singleton
try:
grammar = thread_local_{NAME}_grammar_singleton
return grammar
except NameError:
thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
return thread_local_{NAME}_grammar_singleton
grammar = thread_local_{NAME}_grammar_singleton
return grammar
'''
TRANSFORMER_FACTORY
=
'''
def {NAME}Transform() -> TransformationDict:
return partial(traverse, processing_table={NAME}_AST_transformation_table.copy())
def get_transformer() -> TransformationFunc:
return {NAME}Transform
global thread_local_{NAME}_transformer_singleton
try:
transformer = thread_local_{NAME}_transformer_singleton
except NameError:
thread_local_{NAME}_transformer_singleton = {NAME}Transform()
transformer = thread_local_{NAME}_transformer_singleton
return transformer
'''
...
...
@@ -271,11 +286,11 @@ def get_compiler(grammar_name="{NAME}", grammar_source="") -> {NAME}Compiler:
try:
compiler = thread_local_{NAME}_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except NameError:
thread_local_{NAME}_compiler_singleton =
\\
{NAME}Compiler(grammar_name, grammar_source)
return thread_local_{NAME}_compiler_singleton
compiler = thread_local_{NAME}_compiler_singleton
return compiler
'''
...
...
@@ -404,7 +419,6 @@ class EBNFCompiler(Compiler):
raise
EBNFCompilerError
(
'Compiler must be run before calling '
'"gen_transformer_Skeleton()"!'
)
tt_name
=
self
.
grammar_name
+
'_AST_transformation_table'
tf_name
=
self
.
grammar_name
+
'Transform'
transtable
=
[
tt_name
+
' = {'
,
' # AST Transformations for the '
+
self
.
grammar_name
+
'-grammar'
]
...
...
@@ -418,8 +432,7 @@ class EBNFCompiler(Compiler):
tf
=
'[replace_by_single_child]'
transtable
.
append
(
' "'
+
name
+
'": %s,'
%
tf
)
transtable
.
append
(
' ":Token, :RE": reduce_single_child,'
)
transtable
+=
[
' "*": replace_by_single_child'
,
'}'
,
''
,
tf_name
+
' = partial(traverse, processing_table=%s)'
%
tt_name
,
''
]
transtable
+=
[
' "*": replace_by_single_child'
,
'}'
,
''
]
transtable
+=
[
TRANSFORMER_FACTORY
.
format
(
NAME
=
self
.
grammar_name
)]
return
'
\n
'
.
join
(
transtable
)
...
...
DHParser/transform.py
View file @
337cc278
...
...
@@ -35,7 +35,11 @@ except ImportError:
from
DHParser.toolkit
import
expand_table
,
smart_list
__all__
=
(
'transformation_factory'
,
__all__
=
(
'TransformationDict'
,
'TransformationProc'
,
'ConditionFunc'
,
'KeyFunc'
,
'transformation_factory'
,
'key_parser_name'
,
'key_tag_name'
,
'traverse'
,
...
...
@@ -78,6 +82,7 @@ __all__ = ('transformation_factory',
TransformationProc
=
Callable
[[
List
[
Node
]],
None
]
TransformationDict
=
Dict
ConditionFunc
=
Callable
# Callable[[List[Node]], bool]
KeyFunc
=
Callable
[[
Node
],
str
]
...
...
@@ -191,8 +196,8 @@ def traverse(root_node: Node,
root_node (Node): The root-node of the syntax tree to be traversed
processing_table (dict): node key -> sequence of functions that
will be applied to matching nodes in order. This dictionary
is interpreted as a `
`compact_table`
`. See
`
`toolkit.expand_table`` or ``EBNFCompiler.EBNFTransTable`
`
is interpreted as a `
compact_table
`. See
`
toolkit.expand_table` or ``EBNFCompiler.EBNFTransTable
`
key_func (function): A mapping key_func(node) -> keystr. The default
key_func yields node.parser.name.
...
...
@@ -201,11 +206,27 @@ def traverse(root_node: Node,
"factor, flowmarker, retrieveop": replace_by_single_child }
traverse(node, table)
"""
# normalize processing_table entries by turning single values into lists
# with a single value
table
=
{
name
:
smart_list
(
call
)
for
name
,
call
in
list
(
processing_table
.
items
())}
table
=
expand_table
(
table
)
cache
=
{}
# type: Dict[str, List[Callable]]
# Is this optimazation really needed?
if
'__cache__'
in
processing_table
:
# assume that processing table has already been expanded
table
=
processing_table
cache
=
processing_table
[
'__cache__'
]
else
:
# normalize processing_table entries by turning single values into lists
# with a single value
table
=
{
name
:
smart_list
(
call
)
for
name
,
call
in
list
(
processing_table
.
items
())}
table
=
expand_table
(
table
)
cache
=
table
.
setdefault
(
'__cache__'
,
{})
# type: Dict[str, List[Callable]]
# change processing table in place, so that table expansion does not get lost
# between calls
processing_table
.
clear
();
processing_table
.
update
(
table
)
# assert '__cache__' in processing_table
# # Code without optimization
# table = {name: smart_list(call) for name, call in list(processing_table.items())}
# table = expand_table(table)
# cache = {}
def
traverse_recursive
(
context
):
node
=
context
[
-
1
]
...
...
@@ -232,6 +253,8 @@ def traverse(root_node: Node,
call
(
context
)
traverse_recursive
([
root_node
])
# assert processing_table['__cache__']
# ------------------------------------------------
...
...
examples/LaTeX/LaTeX.ebnf
View file @
337cc278
...
...
@@ -77,7 +77,7 @@ tabular_config = "{" /[lcr|]+/~ §"}"
block_of_paragraphs = /{/~ sequence §/}/
sequence = { (paragraph | block_environment ) [PARSEP] }+
paragraph = { !blockcmd
text_element
//~ }+
paragraph = { !blockcmd
(text_element | LINEFEED)
//~ }+
text_element = text | block | inline_environment | command
#### inline enivronments ####
...
...
@@ -98,7 +98,7 @@ inline_math = /\$/ /[^$]*/ §/\$/
command = known_command | text_command | generic_command
known_command = footnote | includegraphics | caption | multicolumn
text_command = TXTCOMMAND | ESCAPED | BRACKETS
| LINEFEED
text_command = TXTCOMMAND | ESCAPED | BRACKETS
generic_command = !no_command CMDNAME [[ //~ config ] //~ block ]
footnote = "\footnote" block_of_paragraphs
...
...
examples/LaTeX/LaTeXCompiler.py
View file @
337cc278
...
...
@@ -19,7 +19,7 @@ from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, Altern
Required
,
Token
,
Synonym
,
\
Optional
,
NegativeLookbehind
,
OneOrMore
,
RegExp
,
Series
,
RE
,
Capture
,
\
ZeroOrMore
,
Forward
,
NegativeLookahead
,
mixin_comment
,
compile_source
,
\
PreprocessorFunc
,
\
PreprocessorFunc
,
TransformationDict
,
\
Node
,
TransformationFunc
,
traverse
,
remove_children_if
,
is_anonymous
,
\
reduce_single_child
,
replace_by_single_child
,
remove_whitespace
,
\
flatten
,
is_empty
,
collapse
,
replace_content
,
remove_brackets
,
is_one_of
,
remove_first
...
...
@@ -126,7 +126,7 @@ class LaTeXGrammar(Grammar):
block_of_paragraphs = /{/~ sequence §/}/
sequence = { (paragraph | block_environment ) [PARSEP] }+
paragraph = { !blockcmd
text_element
//~ }+
paragraph = { !blockcmd
(text_element | LINEFEED)
//~ }+
text_element = text | block | inline_environment | command
#### inline enivronments ####
...
...
@@ -147,7 +147,7 @@ class LaTeXGrammar(Grammar):
command = known_command | text_command | generic_command
known_command = footnote | includegraphics | caption | multicolumn
text_command = TXTCOMMAND | ESCAPED | BRACKETS
| LINEFEED
text_command = TXTCOMMAND | ESCAPED | BRACKETS
generic_command = !no_command CMDNAME [[ //~ config ] //~ block ]
footnote = "\footnote" block_of_paragraphs
...
...
@@ -216,7 +216,7 @@ class LaTeXGrammar(Grammar):
paragraph
=
Forward
()
tabular_config
=
Forward
()
text_element
=
Forward
()
source_hash__
=
"
fc3ee1800932b561e9cec1e22aab71
57"
source_hash__
=
"
a99d24bcde48695ca003d544738b80
57"
parser_initialization__
=
"upon instantiation"
COMMENT__
=
r
'%.*(?:\n|$)'
WSP__
=
mixin_comment
(
whitespace
=
r
'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?'
,
comment
=
r
'%.*(?:\n|$)'
)
...
...
@@ -250,7 +250,7 @@ class LaTeXGrammar(Grammar):
includegraphics
=
Series
(
Token
(
"
\\
includegraphics"
),
Optional
(
config
),
block
)
footnote
=
Series
(
Token
(
"
\\
footnote"
),
block_of_paragraphs
)
generic_command
=
Series
(
NegativeLookahead
(
no_command
),
CMDNAME
,
Optional
(
Series
(
Optional
(
Series
(
RE
(
''
),
config
)),
RE
(
''
),
block
)))
text_command
=
Alternative
(
TXTCOMMAND
,
ESCAPED
,
BRACKETS
,
LINEFEED
)
text_command
=
Alternative
(
TXTCOMMAND
,
ESCAPED
,
BRACKETS
)
known_command
=
Alternative
(
footnote
,
includegraphics
,
caption
,
multicolumn
)
command
=
Alternative
(
known_command
,
text_command
,
generic_command
)
inline_math
=
Series
(
RegExp
(
'
\\
$'
),
RegExp
(
'[^$]*'
),
Required
(
RegExp
(
'
\\
$'
)))
...
...
@@ -262,7 +262,8 @@ class LaTeXGrammar(Grammar):
known_inline_env
=
Synonym
(
inline_math
)
inline_environment
=
Alternative
(
known_inline_env
,
generic_inline_env
)
text_element
.
set
(
Alternative
(
text
,
block
,
inline_environment
,
command
))
paragraph
.
set
(
OneOrMore
(
Series
(
NegativeLookahead
(
blockcmd
),
text_element
,
RE
(
''
))))
paragraph
.
set
(
OneOrMore
(
Series
(
NegativeLookahead
(
blockcmd
),
Alternative
(
text_element
,
LINEFEED
),
RE
(
''
))))
sequence
=
OneOrMore
(
Series
(
Alternative
(
paragraph
,
block_environment
),
Optional
(
PARSEP
)))
block_of_paragraphs
.
set
(
Series
(
RE
(
'{'
),
sequence
,
Required
(
RegExp
(
'}'
))))
tabular_config
.
set
(
Series
(
Token
(
"{"
),
RE
(
'[lcr|]+'
),
Required
(
Token
(
"}"
))))
...
...
@@ -409,11 +410,19 @@ LaTeX_AST_transformation_table = {
"*"
:
replace_by_single_child
}
LaTeXTransform
=
partial
(
traverse
,
processing_table
=
LaTeX_AST_transformation_table
)
def
LaTeXTransform
()
->
TransformationDict
:
return
partial
(
traverse
,
processing_table
=
LaTeX_AST_transformation_table
.
copy
())
def
get_transformer
()
->
TransformationFunc
:
return
LaTeXTransform
global
thread_local_LaTeX_transformer_singleton
try
:
transformer
=
thread_local_LaTeX_transformer_singleton
except
NameError
:
thread_local_LaTeX_transformer_singleton
=
LaTeXTransform
()
transformer
=
thread_local_LaTeX_transformer_singleton
return
transformer
#######################################################################
...
...
test/test_ebnf.py
View file @
337cc278
...
...
@@ -31,7 +31,7 @@ sys.path.extend(['../', './'])
from
DHParser.toolkit
import
compile_python_object
from
DHParser.parser
import
compile_source
,
WHITESPACE_PTYPE
,
nil_preprocessor
from
DHParser.ebnf
import
get_ebnf_grammar
,
get_ebnf_transformer
,
EBNFTransform
er
,
get_ebnf_compiler
from
DHParser.ebnf
import
get_ebnf_grammar
,
get_ebnf_transformer
,
EBNFTransform
,
get_ebnf_compiler
from
DHParser.dsl
import
CompilationError
,
compileDSL
,
DHPARSER_IMPORTS
,
grammar_provider
...
...
@@ -156,7 +156,7 @@ class TestSemanticValidation:
grammar
=
get_ebnf_grammar
()
st
=
grammar
(
minilang
)
assert
not
st
.
collect_errors
()
EBNFTransform
er
(
st
)
EBNFTransform
()
(
st
)
assert
bool_filter
(
st
.
collect_errors
())
def
test_illegal_nesting
(
self
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment