Commit 5bf393d0 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

parser.py: disabled caching for variable manipulating parsers (and

containing parsers), because caching does not keep track of the state
of variables.
parent 872abf78
...@@ -194,8 +194,10 @@ def add_parser_guard(parser_func): ...@@ -194,8 +194,10 @@ def add_parser_guard(parser_func):
if node is not None: if node is not None:
# in case of a recursive call saves the result of the first # in case of a recursive call saves the result of the first
# (or left-most) call that matches # (or left-most) call that matches; but not for variable manipulating parsers,
parser.visited[location] = (node, rest) # because caching would interfere with changes of variable state
if not (grammar.rollback__ and grammar.rollback__[-1][0] <= location):
parser.visited[location] = (node, rest)
grammar.last_node__ = node # store last node for Lookbehind operator grammar.last_node__ = node # store last node for Lookbehind operator
elif location in parser.visited: elif location in parser.visited:
# if parser did non match but a saved result exits, assume # if parser did non match but a saved result exits, assume
...@@ -248,9 +250,9 @@ class Parser(ParserBase, metaclass=ParserMetaClass): ...@@ -248,9 +250,9 @@ class Parser(ParserBase, metaclass=ParserMetaClass):
return self.__class__(self.name) return self.__class__(self.name)
def reset(self): def reset(self):
self.visited = dict() # type: Dict[int, Tuple[Node, str]] self.visited = dict() # type: Dict[int, Tuple[Node, str]]
self.recursion_counter = dict() # type: Dict[int, int] self.recursion_counter = dict() # type: Dict[int, int]
self.cycle_detection = set() # type: Set[Callable] self.cycle_detection = set() # type: Set[Callable]
return self return self
def __call__(self, text: str) -> Tuple[Node, str]: def __call__(self, text: str) -> Tuple[Node, str]:
...@@ -372,16 +374,16 @@ class Grammar: ...@@ -372,16 +374,16 @@ class Grammar:
raise KeyError('Unknown parser "%s" !' % key) raise KeyError('Unknown parser "%s" !' % key)
def _reset(self): def _reset(self):
self.document__ = "" # type: str self.document__ = "" # type: str
# variables stored and recalled by Capture and Retrieve parsers # variables stored and recalled by Capture and Retrieve parsers
self.variables__ = dict() # type: Dict[str, List[str]] self.variables__ = dict() # type: Dict[str, List[str]]
self.rollback__ = [] # type: List[Tuple[int, Callable]] self.rollback__ = [] # type: List[Tuple[int, Callable]]
# previously parsed node, needed by Lookbehind parser # previously parsed node, needed by Lookbehind parser
self.last_node__ = None # type: Node self.last_node__ = None # type: Node
# support for call stack tracing # support for call stack tracing
self.call_stack__ = [] # type: List[Parser] self.call_stack__ = [] # type: List[Parser]
# snapshots of call stacks # snapshots of call stacks
self.history__ = [] # type: List[HistoryRecord] self.history__ = [] # type: List[HistoryRecord]
# also needed for call stack tracing # also needed for call stack tracing
self.moving_forward__ = True # type: bool self.moving_forward__ = True # type: bool
...@@ -1089,6 +1091,7 @@ class Capture(UnaryOperator): ...@@ -1089,6 +1091,7 @@ class Capture(UnaryOperator):
stack = self.grammar.variables__.setdefault(self.name, []) stack = self.grammar.variables__.setdefault(self.name, [])
stack.append(str(node)) stack.append(str(node))
self.grammar.rollback__.append((len(text), lambda : stack.pop())) self.grammar.rollback__.append((len(text), lambda : stack.pop()))
# block caching, because it would prevent recapturing of rolled back captures
return Node(self, node), text_ return Node(self, node), text_
else: else:
return None, text return None, text
......
...@@ -41,18 +41,39 @@ ARITHMETIC_EBNF = """ ...@@ -41,18 +41,39 @@ ARITHMETIC_EBNF = """
# example: "5 + 3 * 4" # example: "5 + 3 * 4"
""" """
ARITHMETIC2_EBNF = """
ARITHMETIC_EBNF_transformation_table = { @ whitespace = linefeed
# AST Transformations for the DSL-grammar formula = [ //~ ] expr
"formula": [remove_expendables], expr = ex
"term, expr": [replace_by_single_child, flatten], ex = expr ("+"|"-") term | term
"factor": [remove_expendables, reduce_single_child], term = term ("*"|"/") factor | factor
(TOKEN_PTYPE): [remove_expendables, reduce_single_child], factor = /[0-9]+/~
"*": [remove_expendables, replace_by_single_child] # example: "5 + 3 * 4"
} """
ARITHMETIC_EBNFTransform = partial(traverse, processing_table=ARITHMETIC_EBNF_transformation_table) # ARITHMETIC_EBNF_transformation_table = {
# # AST Transformations for the DSL-grammar
# "formula": [remove_expendables],
# "term, expr": [replace_by_single_child, flatten],
# "factor": [remove_expendables, reduce_single_child],
# (TOKEN_PTYPE): [remove_expendables, reduce_single_child],
# "*": [remove_expendables, replace_by_single_child]
# }
#
#
# ARITHMETIC2_EBNF_transformation_table = {
# # AST Transformations for the DSL-grammar
# "formula": [remove_expendables],
# "term, ex": [replace_by_single_child, flatten],
# "factor": [remove_expendables, reduce_single_child],
# (TOKEN_PTYPE): [remove_expendables, reduce_single_child],
# "*": [remove_expendables, replace_by_single_child]
# }
#
#
# ARITHMETIC_EBNFTransform = partial(traverse, processing_table=ARITHMETIC_EBNF_transformation_table)
# ARITHMETIC2_EBNFTransform = partial(traverse, processing_table=ARITHMETIC2_EBNF_transformation_table)
class TestInfiLoopsAndRecursion: class TestInfiLoopsAndRecursion:
...@@ -69,7 +90,15 @@ class TestInfiLoopsAndRecursion: ...@@ -69,7 +90,15 @@ class TestInfiLoopsAndRecursion:
# self.minilang_parser1.log_parsing_history("test_LeftRecursion_direct") # self.minilang_parser1.log_parsing_history("test_LeftRecursion_direct")
def test_indirect_left_recursion(self): def test_indirect_left_recursion(self):
pass minilang = ARITHMETIC2_EBNF
snippet = "5 + 3 * 4"
parser = parser_factory(minilang)()
assert parser
syntax_tree = parser(snippet)
assert not syntax_tree.collect_errors()
assert snippet == str(syntax_tree)
if is_logging():
syntax_tree.log("test_LeftRecursion_indirect.cst")
def test_inifinite_loops(self): def test_inifinite_loops(self):
minilang = """not_forever = { // } \n""" minilang = """not_forever = { // } \n"""
...@@ -218,6 +247,22 @@ class TestPopRetrieve: ...@@ -218,6 +247,22 @@ class TestPopRetrieve:
syntax_tree = self.minilang_parser3(proper) syntax_tree = self.minilang_parser3(proper)
assert not syntax_tree.error_flag, str(syntax_tree.collect_errors()) assert not syntax_tree.error_flag, str(syntax_tree.collect_errors())
def test_cache_neutrality(self):
"""Test that packrat-caching does not interfere with
Capture-Retrieve-Stack."""
lang = """
text = opening closing
opening = (unmarked_package | marked_package)
closing = ::variable
unmarked_package = package "."
marked_package = package "*" "."
package = "(" variable ")"
variable = /\w+/~
"""
case = "(secret)*. secret"
gr = parser_factory(lang)()
st = gr(case)
assert not st.error_flag, str(st.collect_errors())
def test_single_line(self): def test_single_line(self):
teststr = "Anfang ```code block `` <- keine Ende-Zeichen ! ``` Ende" teststr = "Anfang ```code block `` <- keine Ende-Zeichen ! ``` Ende"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment