- added test for indirect left recursion: ilr works with DHParser but stresses...

- added test for indirect left recursion: ilr works with DHParser but stresses python's recursion depth limit

DHParser/parsers.py

@@ -51,14 +51,19 @@ https://bitbucket.org/apalala/grako
 
 import abc
 import copy
-import os
 from functools import partial
+import os
+import platform
 try:
     import regex as re
 except ImportError:
     import re
 try:
-    from typing import Any, Callable, Collection, cast, Dict, Iterator, List, Set, Tuple, Union
+    from typing import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union
+    try:
+        from typing import Collection
+    except ImportError:
+        pass
 except ImportError:
     from .typing34 import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union
 
@@ -116,10 +121,11 @@ __all__ = ['ScannerFunc',
 ScannerFunc = Union[Callable[[str], str], partial]
 
 
-LEFT_RECURSION_DEPTH = 10  # because of pythons recursion depth limit, this
-                           # value ought not to be set too high
-MAX_DROPOUTS = 25  # stop trying to recover parsing after so many errors
-
+LEFT_RECURSION_DEPTH = 20 if platform.python_implementation() == "PyPy" \
+                          else 8 # type: int
+# because of python's recursion depth limit, this value ought not to be set too high
+MAX_DROPOUTS = 25  # type: int
+# stop trying to recover parsing after so many errors
 
 class HistoryRecord:
     """
@@ -258,10 +264,10 @@ class Parser(ParserBase, metaclass=ParserMetaClass):
     def __call__(self, text: str) -> Tuple[Node, str]:
         return None, text  # default behaviour: don't match
 
-    def __add__(self, other):
+    def __add__(self, other: 'Parser') -> 'Series':
         return Series(self, other)
 
-    def __or__(self, other):
+    def __or__(self, other: 'Parser') -> 'Alternative':
         return Alternative(self, other)
 
     @property
@@ -620,14 +626,14 @@ class ScannerToken(Parser):
             elif end == 0:
                 node = Node(self, '').add_error(
                     'Scanner token cannot have zero length. '
-                    '(Most likely due to a scanner bug!)')  # type: Node
+                    '(Most likely due to a scanner bug!)')
                 return node, text[2:]
             elif text.find(BEGIN_SCANNER_TOKEN, 1, end) >= 0:
                 node = Node(self, text[len(self.name) + 1:end])
                 node.add_error(
                     'Scanner tokens must not be nested or contain '
                     'BEGIN_SCANNER_TOKEN delimiter as part of their argument. '
-                    '(Most likely due to a scanner bug!)')  # type: Node
+                    '(Most likely due to a scanner bug!)')
                 return node, text[end:]
             if text[1:len(self.name) + 1] == self.name:
                 return Node(self, text[len(self.name) + 1:end]), \
@@ -921,15 +927,18 @@ class Series(NaryOperator):
         return " ".join(parser.repr for parser in self.parsers)
 
     def __add__(self, other: Parser) -> 'Series':
-        other_parsers = cast('Series', other).parsers if isinstance(other, Series) else (other,)
+        other_parsers = cast('Series', other).parsers if isinstance(other, Series) \
+                        else cast(Tuple[Parser, ...], (other,))  # type: Tuple[Parser, ...]
         return Series(*(self.parsers + other_parsers))
 
     def __radd__(self, other: Parser) -> 'Series':
-        other_parsers = cast('Series', other).parsers if isinstance(other, Series) else (other,)
+        other_parsers = cast('Series', other).parsers if isinstance(other, Series) \
+                        else cast(Tuple[Parser, ...], (other,))  # type: Tuple[Parser, ...]
         return Series(*(other_parsers + self.parsers))
 
-    def __iadd__(self, other):
-        other_parsers = cast('Series', other).parsers if isinstance(other, Series) else (other,)
+    def __iadd__(self, other: Parser) -> 'Series':
+        other_parsers = cast('Series', other).parsers if isinstance(other, Series) \
+                        else cast(Tuple[Parser, ...], (other,))  # type: Tuple[Parser, ...]
         self.parsers += other_parsers
         return self
 
@@ -961,34 +970,38 @@ class Alternative(NaryOperator):
 
     def __call__(self, text: str) -> Tuple[Node, str]:
         location = len(text)
-        pindex = self.been_here.get(location, 0)
+        pindex = self.been_here.setdefault(location, 0)
         for parser in self.parsers[pindex:]:
             node, text_ = parser(text)
             if node:
                 return Node(self, node), text_
-            pindex += 1
-        # self.been_here[location] = pindex
+            # self.been_here[location] += 1
         return None, text
 
     def __repr__(self):
         return '(' + ' | '.join(parser.repr for parser in self.parsers) + ')'
 
     def __or__(self, other: Parser) -> 'Alternative':
-        other_parsers = cast('Alternative', other).parsers \
-                        if isinstance(other, Alternative) else (other,)
+        other_parsers = cast('Alternative', other).parsers if isinstance(other, Alternative) \
+                        else cast(Tuple[Parser, ...], (other,))  # type: Tuple[Parser, ...]
         return Alternative(*(self.parsers + other_parsers))
 
     def __ror__(self, other: Parser) -> 'Alternative':
-        other_parsers = cast('Alternative', other).parsers \
-                        if isinstance(other, Alternative) else (other,)
+        other_parsers = cast('Alternative', other).parsers if isinstance(other, Alternative) \
+                        else cast(Tuple[Parser, ...], (other,))  # type: Tuple[Parser, ...]
         return Alternative(*(other_parsers + self.parsers))
 
-    def __ior__(self, other):
-        other_parsers = cast('Alternative', other.parsers) \
-                        if isinstance(other, Alternative) else (other,)
+    def __ior__(self, other: Parser) -> 'Alternative':
+        other_parsers = cast('Alternative', other).parsers if isinstance(other, Alternative) \
+                        else cast(Tuple[Parser, ...], (other,))  # type: Tuple[Parser, ...]
         self.parsers += other_parsers
         return self
 
+    def reset(self):
+        super(Alternative, self).reset()
+        self.been_here = {}
+        return self
+
 
 ########################################################################
 #

test/run.py

 if __name__ == "__main__":
     import os
 
-    os.chdir('..')
+    if os.getcwd().endswith('test'):
+        os.chdir('..')
     print("Running nosetests:")
     os.system("nosetests")

test/test_parsers.py

@@ -50,7 +50,7 @@ class TestInfiLoopsAndRecursion:
             syntax_tree.log("test_LeftRecursion_direct.cst")
             # self.minilang_parser1.log_parsing_history__("test_LeftRecursion_direct")
 
-    def test_direct_left_recursion2(self):
+    def test_indirect_left_recursion1(self):
         minilang = """
             @ whitespace = linefeed
             formula = [ //~ ] expr
@@ -67,26 +67,28 @@ class TestInfiLoopsAndRecursion:
         assert not syntax_tree.collect_errors()
         assert snippet == str(syntax_tree)
         if is_logging():
-            syntax_tree.log("test_LeftRecursion_direct2.cst")
+            syntax_tree.log("test_LeftRecursion_indirect1.cst")
 
-    def test_indirect_left_recursion(self):
-        return
+    def test_indirect_left_recursion2(self):
         minilang = """
             Expr    = //~ (Product | Sum | Value)
-            Product = Expr { ('*' | '/') Expr }
-            Sum     = Expr { ('+' | '-') Expr }
+            Product = Expr { ('*' | '/') Expr }+
+            Sum     = Expr { ('+' | '-') Expr }+
             Value   = /[0-9.]+/~ | '(' Expr ')'
             """
         parser = parser_factory(minilang)()
         assert parser
-        snippet = "7 * 8"
+        snippet = "8 * 4"
         syntax_tree = parser(snippet)
-        print(syntax_tree.as_sxpr())
         assert not syntax_tree.error_flag
+
+        snippet = "7 + 8 * 4"
+        syntax_tree = parser(snippet)
+        assert not syntax_tree.error_flag
+
         snippet = "9 + 8 * (4 + 3)"
         syntax_tree = parser(snippet)
-        print(syntax_tree.as_sxpr())
-        assert not syntax_tree.error_flag, print(syntax_tree.collect_errors())
+        assert not syntax_tree.error_flag, syntax_tree.collect_errors()
         assert snippet == str(syntax_tree)
         if is_logging():
             syntax_tree.log("test_LeftRecursion_indirect2.cst")