Commit c8bde767 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

- added test for indirect left recursion: ilr works with DHParser but stresses...

- added test for indirect left recursion: ilr works with DHParser but stresses python's recursion depth limit
parent a50ca9ad
...@@ -51,14 +51,19 @@ https://bitbucket.org/apalala/grako ...@@ -51,14 +51,19 @@ https://bitbucket.org/apalala/grako
import abc import abc
import copy import copy
import os
from functools import partial from functools import partial
import os
import platform
try: try:
import regex as re import regex as re
except ImportError: except ImportError:
import re import re
try: try:
from typing import Any, Callable, Collection, cast, Dict, Iterator, List, Set, Tuple, Union from typing import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union
try:
from typing import Collection
except ImportError:
pass
except ImportError: except ImportError:
from .typing34 import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union from .typing34 import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union
...@@ -116,10 +121,11 @@ __all__ = ['ScannerFunc', ...@@ -116,10 +121,11 @@ __all__ = ['ScannerFunc',
ScannerFunc = Union[Callable[[str], str], partial] ScannerFunc = Union[Callable[[str], str], partial]
LEFT_RECURSION_DEPTH = 10 # because of pythons recursion depth limit, this LEFT_RECURSION_DEPTH = 20 if platform.python_implementation() == "PyPy" \
# value ought not to be set too high else 8 # type: int
MAX_DROPOUTS = 25 # stop trying to recover parsing after so many errors # because of python's recursion depth limit, this value ought not to be set too high
MAX_DROPOUTS = 25 # type: int
# stop trying to recover parsing after so many errors
class HistoryRecord: class HistoryRecord:
""" """
...@@ -258,10 +264,10 @@ class Parser(ParserBase, metaclass=ParserMetaClass): ...@@ -258,10 +264,10 @@ class Parser(ParserBase, metaclass=ParserMetaClass):
def __call__(self, text: str) -> Tuple[Node, str]: def __call__(self, text: str) -> Tuple[Node, str]:
return None, text # default behaviour: don't match return None, text # default behaviour: don't match
def __add__(self, other): def __add__(self, other: 'Parser') -> 'Series':
return Series(self, other) return Series(self, other)
def __or__(self, other): def __or__(self, other: 'Parser') -> 'Alternative':
return Alternative(self, other) return Alternative(self, other)
@property @property
...@@ -620,14 +626,14 @@ class ScannerToken(Parser): ...@@ -620,14 +626,14 @@ class ScannerToken(Parser):
elif end == 0: elif end == 0:
node = Node(self, '').add_error( node = Node(self, '').add_error(
'Scanner token cannot have zero length. ' 'Scanner token cannot have zero length. '
'(Most likely due to a scanner bug!)') # type: Node '(Most likely due to a scanner bug!)')
return node, text[2:] return node, text[2:]
elif text.find(BEGIN_SCANNER_TOKEN, 1, end) >= 0: elif text.find(BEGIN_SCANNER_TOKEN, 1, end) >= 0:
node = Node(self, text[len(self.name) + 1:end]) node = Node(self, text[len(self.name) + 1:end])
node.add_error( node.add_error(
'Scanner tokens must not be nested or contain ' 'Scanner tokens must not be nested or contain '
'BEGIN_SCANNER_TOKEN delimiter as part of their argument. ' 'BEGIN_SCANNER_TOKEN delimiter as part of their argument. '
'(Most likely due to a scanner bug!)') # type: Node '(Most likely due to a scanner bug!)')
return node, text[end:] return node, text[end:]
if text[1:len(self.name) + 1] == self.name: if text[1:len(self.name) + 1] == self.name:
return Node(self, text[len(self.name) + 1:end]), \ return Node(self, text[len(self.name) + 1:end]), \
...@@ -921,15 +927,18 @@ class Series(NaryOperator): ...@@ -921,15 +927,18 @@ class Series(NaryOperator):
return " ".join(parser.repr for parser in self.parsers) return " ".join(parser.repr for parser in self.parsers)
def __add__(self, other: Parser) -> 'Series': def __add__(self, other: Parser) -> 'Series':
other_parsers = cast('Series', other).parsers if isinstance(other, Series) else (other,) other_parsers = cast('Series', other).parsers if isinstance(other, Series) \
else cast(Tuple[Parser, ...], (other,)) # type: Tuple[Parser, ...]
return Series(*(self.parsers + other_parsers)) return Series(*(self.parsers + other_parsers))
def __radd__(self, other: Parser) -> 'Series': def __radd__(self, other: Parser) -> 'Series':
other_parsers = cast('Series', other).parsers if isinstance(other, Series) else (other,) other_parsers = cast('Series', other).parsers if isinstance(other, Series) \
else cast(Tuple[Parser, ...], (other,)) # type: Tuple[Parser, ...]
return Series(*(other_parsers + self.parsers)) return Series(*(other_parsers + self.parsers))
def __iadd__(self, other): def __iadd__(self, other: Parser) -> 'Series':
other_parsers = cast('Series', other).parsers if isinstance(other, Series) else (other,) other_parsers = cast('Series', other).parsers if isinstance(other, Series) \
else cast(Tuple[Parser, ...], (other,)) # type: Tuple[Parser, ...]
self.parsers += other_parsers self.parsers += other_parsers
return self return self
...@@ -961,34 +970,38 @@ class Alternative(NaryOperator): ...@@ -961,34 +970,38 @@ class Alternative(NaryOperator):
def __call__(self, text: str) -> Tuple[Node, str]: def __call__(self, text: str) -> Tuple[Node, str]:
location = len(text) location = len(text)
pindex = self.been_here.get(location, 0) pindex = self.been_here.setdefault(location, 0)
for parser in self.parsers[pindex:]: for parser in self.parsers[pindex:]:
node, text_ = parser(text) node, text_ = parser(text)
if node: if node:
return Node(self, node), text_ return Node(self, node), text_
pindex += 1 # self.been_here[location] += 1
# self.been_here[location] = pindex
return None, text return None, text
def __repr__(self): def __repr__(self):
return '(' + ' | '.join(parser.repr for parser in self.parsers) + ')' return '(' + ' | '.join(parser.repr for parser in self.parsers) + ')'
def __or__(self, other: Parser) -> 'Alternative': def __or__(self, other: Parser) -> 'Alternative':
other_parsers = cast('Alternative', other).parsers \ other_parsers = cast('Alternative', other).parsers if isinstance(other, Alternative) \
if isinstance(other, Alternative) else (other,) else cast(Tuple[Parser, ...], (other,)) # type: Tuple[Parser, ...]
return Alternative(*(self.parsers + other_parsers)) return Alternative(*(self.parsers + other_parsers))
def __ror__(self, other: Parser) -> 'Alternative': def __ror__(self, other: Parser) -> 'Alternative':
other_parsers = cast('Alternative', other).parsers \ other_parsers = cast('Alternative', other).parsers if isinstance(other, Alternative) \
if isinstance(other, Alternative) else (other,) else cast(Tuple[Parser, ...], (other,)) # type: Tuple[Parser, ...]
return Alternative(*(other_parsers + self.parsers)) return Alternative(*(other_parsers + self.parsers))
def __ior__(self, other): def __ior__(self, other: Parser) -> 'Alternative':
other_parsers = cast('Alternative', other.parsers) \ other_parsers = cast('Alternative', other).parsers if isinstance(other, Alternative) \
if isinstance(other, Alternative) else (other,) else cast(Tuple[Parser, ...], (other,)) # type: Tuple[Parser, ...]
self.parsers += other_parsers self.parsers += other_parsers
return self return self
def reset(self):
super(Alternative, self).reset()
self.been_here = {}
return self
######################################################################## ########################################################################
# #
......
if __name__ == "__main__": if __name__ == "__main__":
import os import os
os.chdir('..') if os.getcwd().endswith('test'):
os.chdir('..')
print("Running nosetests:") print("Running nosetests:")
os.system("nosetests") os.system("nosetests")
...@@ -50,7 +50,7 @@ class TestInfiLoopsAndRecursion: ...@@ -50,7 +50,7 @@ class TestInfiLoopsAndRecursion:
syntax_tree.log("test_LeftRecursion_direct.cst") syntax_tree.log("test_LeftRecursion_direct.cst")
# self.minilang_parser1.log_parsing_history__("test_LeftRecursion_direct") # self.minilang_parser1.log_parsing_history__("test_LeftRecursion_direct")
def test_direct_left_recursion2(self): def test_indirect_left_recursion1(self):
minilang = """ minilang = """
@ whitespace = linefeed @ whitespace = linefeed
formula = [ //~ ] expr formula = [ //~ ] expr
...@@ -67,26 +67,28 @@ class TestInfiLoopsAndRecursion: ...@@ -67,26 +67,28 @@ class TestInfiLoopsAndRecursion:
assert not syntax_tree.collect_errors() assert not syntax_tree.collect_errors()
assert snippet == str(syntax_tree) assert snippet == str(syntax_tree)
if is_logging(): if is_logging():
syntax_tree.log("test_LeftRecursion_direct2.cst") syntax_tree.log("test_LeftRecursion_indirect1.cst")
def test_indirect_left_recursion(self): def test_indirect_left_recursion2(self):
return
minilang = """ minilang = """
Expr = //~ (Product | Sum | Value) Expr = //~ (Product | Sum | Value)
Product = Expr { ('*' | '/') Expr } Product = Expr { ('*' | '/') Expr }+
Sum = Expr { ('+' | '-') Expr } Sum = Expr { ('+' | '-') Expr }+
Value = /[0-9.]+/~ | '(' Expr ')' Value = /[0-9.]+/~ | '(' Expr ')'
""" """
parser = parser_factory(minilang)() parser = parser_factory(minilang)()
assert parser assert parser
snippet = "7 * 8" snippet = "8 * 4"
syntax_tree = parser(snippet) syntax_tree = parser(snippet)
print(syntax_tree.as_sxpr())
assert not syntax_tree.error_flag assert not syntax_tree.error_flag
snippet = "7 + 8 * 4"
syntax_tree = parser(snippet)
assert not syntax_tree.error_flag
snippet = "9 + 8 * (4 + 3)" snippet = "9 + 8 * (4 + 3)"
syntax_tree = parser(snippet) syntax_tree = parser(snippet)
print(syntax_tree.as_sxpr()) assert not syntax_tree.error_flag, syntax_tree.collect_errors()
assert not syntax_tree.error_flag, print(syntax_tree.collect_errors())
assert snippet == str(syntax_tree) assert snippet == str(syntax_tree)
if is_logging(): if is_logging():
syntax_tree.log("test_LeftRecursion_indirect2.cst") syntax_tree.log("test_LeftRecursion_indirect2.cst")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment