parser.py: disabled caching for variable manipulating parsers (and

containing parsers), because caching does not keep track of the state
of variables.

DHParser/parsers.py

@@ -194,8 +194,10 @@ def add_parser_guard(parser_func):
 
             if node is not None:
                 # in case of a recursive call saves the result of the first
-                # (or left-most) call that matches
-                parser.visited[location] = (node, rest)
+                # (or left-most) call that matches; but not for variable manipulating parsers,
+                # because caching would interfere with changes of variable state
+                if not (grammar.rollback__ and grammar.rollback__[-1][0] <= location):
+                    parser.visited[location] = (node, rest)
                 grammar.last_node__ = node   # store last node for Lookbehind operator
             elif location in parser.visited:
                 # if parser did non match but a saved result exits, assume
@@ -248,9 +250,9 @@ class Parser(ParserBase, metaclass=ParserMetaClass):
         return self.__class__(self.name)
 
     def reset(self):
-        self.visited = dict()  # type: Dict[int, Tuple[Node, str]]
+        self.visited = dict()            # type: Dict[int, Tuple[Node, str]]
         self.recursion_counter = dict()  # type: Dict[int, int]
-        self.cycle_detection = set()  # type: Set[Callable]
+        self.cycle_detection = set()     # type: Set[Callable]
         return self
 
     def __call__(self, text: str) -> Tuple[Node, str]:
@@ -372,16 +374,16 @@ class Grammar:
             raise KeyError('Unknown parser "%s" !' % key)
 
     def _reset(self):
-        self.document__ = ""       # type: str
+        self.document__ = ""          # type: str
         # variables stored and recalled by Capture and Retrieve parsers
-        self.variables__ = dict()  # type: Dict[str, List[str]]
-        self.rollback__ = []       # type: List[Tuple[int, Callable]]
+        self.variables__ = dict()     # type: Dict[str, List[str]]
+        self.rollback__ = []          # type: List[Tuple[int, Callable]]
         # previously parsed node, needed by Lookbehind parser
-        self.last_node__ = None    # type: Node
+        self.last_node__ = None       # type: Node
         # support for call stack tracing
-        self.call_stack__ = []     # type: List[Parser]
+        self.call_stack__ = []        # type: List[Parser]
         # snapshots of call stacks
-        self.history__ = []        # type: List[HistoryRecord]
+        self.history__ = []           # type: List[HistoryRecord]
         # also needed for call stack tracing
         self.moving_forward__ = True  # type: bool
 
@@ -1089,6 +1091,7 @@ class Capture(UnaryOperator):
             stack = self.grammar.variables__.setdefault(self.name, [])
             stack.append(str(node))
             self.grammar.rollback__.append((len(text), lambda : stack.pop()))
+            # block caching, because it would prevent recapturing of rolled back captures
             return Node(self, node), text_
         else:
             return None, text

test/test_parsers.py

@@ -41,18 +41,39 @@ ARITHMETIC_EBNF = """
     # example:  "5 + 3 * 4"
     """
 
-
-ARITHMETIC_EBNF_transformation_table = {
-    # AST Transformations for the DSL-grammar
-    "formula": [remove_expendables],
-    "term, expr": [replace_by_single_child, flatten],
-    "factor": [remove_expendables, reduce_single_child],
-    (TOKEN_PTYPE): [remove_expendables, reduce_single_child],
-    "*": [remove_expendables, replace_by_single_child]
-}
+ARITHMETIC2_EBNF = """
+    @ whitespace = linefeed
+    formula = [ //~ ] expr
+    expr = ex
+    ex   = expr ("+"|"-") term | term
+    term = term ("*"|"/") factor | factor
+    factor = /[0-9]+/~
+    # example:  "5 + 3 * 4"
+    """
 
 
-ARITHMETIC_EBNFTransform = partial(traverse, processing_table=ARITHMETIC_EBNF_transformation_table)
+# ARITHMETIC_EBNF_transformation_table = {
+#     # AST Transformations for the DSL-grammar
+#     "formula": [remove_expendables],
+#     "term, expr": [replace_by_single_child, flatten],
+#     "factor": [remove_expendables, reduce_single_child],
+#     (TOKEN_PTYPE): [remove_expendables, reduce_single_child],
+#     "*": [remove_expendables, replace_by_single_child]
+# }
+#
+#
+# ARITHMETIC2_EBNF_transformation_table = {
+#     # AST Transformations for the DSL-grammar
+#     "formula": [remove_expendables],
+#     "term, ex": [replace_by_single_child, flatten],
+#     "factor": [remove_expendables, reduce_single_child],
+#     (TOKEN_PTYPE): [remove_expendables, reduce_single_child],
+#     "*": [remove_expendables, replace_by_single_child]
+# }
+#
+#
+# ARITHMETIC_EBNFTransform = partial(traverse, processing_table=ARITHMETIC_EBNF_transformation_table)
+# ARITHMETIC2_EBNFTransform = partial(traverse, processing_table=ARITHMETIC2_EBNF_transformation_table)
 
 
 class TestInfiLoopsAndRecursion:
@@ -69,7 +90,15 @@ class TestInfiLoopsAndRecursion:
             # self.minilang_parser1.log_parsing_history("test_LeftRecursion_direct")
 
     def test_indirect_left_recursion(self):
-        pass
+        minilang = ARITHMETIC2_EBNF
+        snippet = "5 + 3 * 4"
+        parser = parser_factory(minilang)()
+        assert parser
+        syntax_tree = parser(snippet)
+        assert not syntax_tree.collect_errors()
+        assert snippet == str(syntax_tree)
+        if is_logging():
+            syntax_tree.log("test_LeftRecursion_indirect.cst")
 
     def test_inifinite_loops(self):
         minilang = """not_forever = { // } \n"""
@@ -218,6 +247,22 @@ class TestPopRetrieve:
         syntax_tree = self.minilang_parser3(proper)
         assert not syntax_tree.error_flag, str(syntax_tree.collect_errors())
 
+    def test_cache_neutrality(self):
+        """Test that packrat-caching does not interfere with
+        Capture-Retrieve-Stack."""
+        lang = """
+            text = opening closing
+            opening = (unmarked_package | marked_package)
+            closing = ::variable
+            unmarked_package = package "."
+            marked_package = package "*" "."
+            package = "(" variable ")"
+            variable = /\w+/~
+            """
+        case = "(secret)*. secret"
+        gr = parser_factory(lang)()
+        st = gr(case)
+        assert not st.error_flag, str(st.collect_errors())
 
     def test_single_line(self):
         teststr = "Anfang ```code block `` <- keine Ende-Zeichen ! ``` Ende"