Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
9.2.2023: Due to updates GitLab will be unavailable for some minutes between 9:00 and 11:00.
Open sidebar
badw-it
DHParser
Commits
219a01a8
Commit
219a01a8
authored
Jul 06, 2017
by
Eckhart Arnold
Browse files
- parsrs.py optimization for guarded_call rollback-tests
parent
e4638fde
Changes
5
Hide whitespace changes
Inline
Side-by-side
DHParser/parsers.py
View file @
219a01a8
...
...
@@ -172,10 +172,9 @@ def add_parser_guard(parser_func):
if
not
grammar
.
moving_forward__
:
# rollback variable changes for discarded branch of parsing tree
while
grammar
.
rollback__
and
grammar
.
rollback__
[
-
1
][
0
]
<=
location
:
grammar
.
rollback__
[
-
1
][
1
]()
grammar
.
rollback__
.
pop
()
grammar
.
moving_forward__
=
True
if
grammar
.
last_rb__loc__
<=
location
:
grammar
.
rollback_to__
(
location
)
grammar
.
moving_forward__
=
True
if
grammar
.
history_tracking__
:
grammar
.
call_stack__
.
append
(
parser
)
...
...
@@ -197,7 +196,7 @@ def add_parser_guard(parser_func):
# in case of a recursive call saves the result of the first
# (or left-most) call that matches; but not for variable manipulating parsers,
# because caching would interfere with changes of variable state
if
not
(
grammar
.
rollback__
and
grammar
.
rollback__
[
-
1
][
0
]
<=
location
)
:
if
grammar
.
last_rb__loc__
>
location
:
parser
.
visited
[
location
]
=
(
node
,
rest
)
grammar
.
last_node__
=
node
# store last node for Lookbehind parser
elif
location
in
parser
.
visited
:
...
...
@@ -339,7 +338,7 @@ class Grammar:
self
.
all_parsers__
=
set
()
# type: Set[Parser]
self
.
dirty_flag__
=
False
self
.
history_tracking__
=
False
self
.
_reset
()
self
.
_reset
__
()
# prepare parsers in the class, first
self
.
_assign_parser_names
()
# then deep-copy the parser tree from class to instance;
...
...
@@ -359,7 +358,7 @@ class Grammar:
self
.
all_parsers__
.
add
(
self
.
wsp_right_parser__
)
# don't you forget about me...
else
:
self
.
wsp_right_parser__
=
ZOMBIE_PARSER
self
.
root__
.
apply
(
self
.
_add_parser
)
self
.
root__
.
apply
(
self
.
_add_parser
__
)
def
__getitem__
(
self
,
key
):
try
:
...
...
@@ -369,16 +368,17 @@ class Grammar:
if
parser_template
:
# add parser to grammar object on the fly...
parser
=
copy
.
deepcopy
(
parser_template
)
parser
.
apply
(
self
.
_add_parser
)
parser
.
apply
(
self
.
_add_parser
__
)
# assert self[key] == parser
return
self
[
key
]
raise
KeyError
(
'Unknown parser "%s" !'
%
key
)
def
_reset
(
self
):
def
_reset
__
(
self
):
self
.
document__
=
""
# type: str
# variables stored and recalled by Capture and Retrieve parsers
self
.
variables__
=
dict
()
# type: Dict[str, List[str]]
self
.
rollback__
=
[]
# type: List[Tuple[int, Callable]]
self
.
last_rb__loc__
=
-
1
# type: int
# previously parsed node, needed by Lookbehind parser
self
.
last_node__
=
None
# type: Node
# support for call stack tracing
...
...
@@ -388,7 +388,7 @@ class Grammar:
# also needed for call stack tracing
self
.
moving_forward__
=
True
# type: bool
def
_add_parser
(
self
,
parser
:
Parser
)
->
None
:
def
_add_parser
__
(
self
,
parser
:
Parser
)
->
None
:
"""Adds the particular copy of the parser object to this
particular instance of Grammar.
"""
...
...
@@ -417,13 +417,14 @@ class Grammar:
if
self
.
root__
is
None
:
raise
NotImplementedError
()
if
self
.
dirty_flag__
:
self
.
_reset
()
self
.
_reset
__
()
for
parser
in
self
.
all_parsers__
:
parser
.
reset
()
else
:
self
.
dirty_flag__
=
True
self
.
history_tracking__
=
is_logging
()
self
.
document__
=
document
self
.
last_rb__loc__
=
len
(
document
)
+
1
# rollback location
parser
=
self
[
start_parser
]
if
isinstance
(
start_parser
,
str
)
else
start_parser
assert
parser
.
grammar
==
self
,
"Cannot run parsers from a different grammar object!"
\
" %s vs. %s"
%
(
str
(
self
),
str
(
parser
.
grammar
))
...
...
@@ -469,7 +470,19 @@ class Grammar:
result
.
pos
=
0
# calculate all positions
return
result
def
log_parsing_history
(
self
,
log_file_name
:
str
=
''
)
->
None
:
def
push_rollback__
(
self
,
location
,
func
):
self
.
rollback__
.
append
((
location
,
func
))
self
.
last_rb__loc__
=
location
def
rollback_to__
(
self
,
location
):
while
self
.
rollback__
and
self
.
rollback__
[
-
1
][
0
]
<=
location
:
loc
,
rollback_func
=
self
.
rollback__
.
pop
()
assert
not
loc
>
self
.
last_rb__loc__
rollback_func
()
self
.
last_rb__loc__
==
self
.
rollback__
[
-
1
][
0
]
if
self
.
rollback__
\
else
(
len
(
self
.
document__
)
+
1
)
def
log_parsing_history__
(
self
,
log_file_name
:
str
=
''
)
->
None
:
"""Writes a log of the parsing history of the most recently parsed
document.
"""
...
...
@@ -1085,7 +1098,8 @@ class Capture(UnaryOperator):
if
node
:
stack
=
self
.
grammar
.
variables__
.
setdefault
(
self
.
name
,
[])
stack
.
append
(
str
(
node
))
self
.
grammar
.
rollback__
.
append
((
len
(
text
),
lambda
:
stack
.
pop
()))
self
.
grammar
.
push_rollback__
(
len
(
text
),
lambda
:
stack
.
pop
())
# self.grammar.rollback__.append((len(text), lambda : stack.pop()))
# block caching, because it would prevent recapturing of rolled back captures
return
Node
(
self
,
node
),
text_
else
:
...
...
@@ -1148,7 +1162,8 @@ class Pop(Retrieve):
if
nd
and
not
nd
.
error_flag
:
stack
=
self
.
grammar
.
variables__
[
self
.
symbol
.
name
]
value
=
stack
.
pop
()
self
.
grammar
.
rollback__
.
append
((
len
(
text
),
lambda
:
stack
.
append
(
value
)))
self
.
grammar
.
push_rollback__
(
len
(
text
),
lambda
:
stack
.
append
(
value
))
# self.grammar.rollback__.append((len(text), lambda : stack.append(value)))
return
nd
,
txt
def
__repr__
(
self
):
...
...
@@ -1306,7 +1321,7 @@ def compile_source(source: str,
syntax_tree
=
parser
(
source_text
)
if
is_logging
():
syntax_tree
.
log
(
log_file_name
+
'.cst'
)
parser
.
log_parsing_history
(
log_file_name
)
parser
.
log_parsing_history
__
(
log_file_name
)
assert
syntax_tree
.
error_flag
or
str
(
syntax_tree
)
==
source_text
,
str
(
syntax_tree
)
# only compile if there were no syntax errors, for otherwise it is
...
...
DHParser/testing.py
View file @
219a01a8
...
...
@@ -232,7 +232,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
cst
=
parser
(
test_code
,
parser_name
)
if
is_logging
():
cst
.
log
(
"match_%s_%s.cst"
%
(
parser_name
,
test_name
))
parser
.
log_parsing_history
(
"match_%s_%s.log"
%
(
parser_name
,
test_name
))
parser
.
log_parsing_history
__
(
"match_%s_%s.log"
%
(
parser_name
,
test_name
))
tests
.
setdefault
(
'__cst__'
,
{})[
test_name
]
=
cst
if
"ast"
in
tests
or
report
:
ast
=
copy
.
deepcopy
(
cst
)
...
...
@@ -270,7 +270,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
cst
=
parser
(
test_code
,
parser_name
)
if
is_logging
():
cst
.
log
(
"fail_%s_%s.cst"
%
(
parser_name
,
test_name
))
parser
.
log_parsing_history
(
"fail_%s_%s.log"
%
(
parser_name
,
test_name
))
parser
.
log_parsing_history
__
(
"fail_%s_%s.log"
%
(
parser_name
,
test_name
))
if
not
cst
.
error_flag
:
errata
.
append
(
'Fail test "%s" for parser "%s" yields match instead of '
'expected failure!'
%
(
test_name
,
parser_name
))
...
...
OLDSTUFF/ParserCombinators_obsolete.py
View file @
219a01a8
...
...
@@ -1406,7 +1406,7 @@ def full_compilation(source, grammar_base, AST_transformations, compiler):
syntax_tree
=
grammar_base
(
source
)
syntax_tree
.
log
(
grammar_base
.
log_file_name
,
ext
=
'.cst'
)
grammar_base
.
log_parsing_history
()
grammar_base
.
log_parsing_history
__
()
assert
syntax_tree
.
error_flag
or
str
(
syntax_tree
)
==
source
,
str
(
syntax_tree
)
# only compile if there were no syntax errors, for otherwise it is
...
...
dhparser.py
View file @
219a01a8
...
...
@@ -86,5 +86,5 @@ if __name__ == "__main__":
else
:
# run self test
# selftest('EBNF/EBNF.ebnf')
with
logging
(
Tru
e
):
with
logging
(
Fals
e
):
profile
(
partial
(
selftest
,
file_name
=
'EBNF/EBNF.ebnf'
))
test/test_parsers.py
View file @
219a01a8
...
...
@@ -25,8 +25,6 @@ from functools import partial
sys
.
path
.
extend
([
'../'
,
'./'
])
from
DHParser.toolkit
import
is_logging
,
logging
,
compile_python_object
from
DHParser.syntaxtree
import
traverse
,
remove_expendables
,
\
replace_by_single_child
,
reduce_single_child
,
flatten
,
TOKEN_PTYPE
from
DHParser.parsers
import
compile_source
,
Retrieve
from
DHParser.ebnf
import
get_ebnf_grammar
,
get_ebnf_transformer
,
get_ebnf_compiler
from
DHParser.dsl
import
parser_factory
,
DHPARSER_IMPORTS
...
...
@@ -86,7 +84,7 @@ class TestInfiLoopsAndRecursion:
assert
snippet
==
str
(
syntax_tree
)
if
is_logging
():
syntax_tree
.
log
(
"test_LeftRecursion_direct.cst"
)
# self.minilang_parser1.log_parsing_history("test_LeftRecursion_direct")
# self.minilang_parser1.log_parsing_history
__
("test_LeftRecursion_direct")
def
test_indirect_left_recursion
(
self
):
minilang
=
ARITHMETIC2_EBNF
...
...
@@ -143,7 +141,7 @@ class TestRegex:
assert
not
messages
parser
=
compile_python_object
(
DHPARSER_IMPORTS
+
result
,
'\w+Grammar$'
)()
result
=
parser
(
testdoc
)
# parser.log_parsing_history("test.log")
# parser.log_parsing_history
__
("test.log")
assert
not
result
.
error_flag
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment