Commit bc2af75d authored by Eckhart Arnold's avatar Eckhart Arnold

Beipiel und kleine Änderungen

parent 322c891d
...@@ -254,25 +254,29 @@ def add_parser_guard(parser_func): ...@@ -254,25 +254,29 @@ def add_parser_guard(parser_func):
if location in parser.visited: if location in parser.visited:
return parser.visited[location] return parser.visited[location]
# break left recursion at the maximum allowed depth
if parser.recursion_counter.setdefault(location, 0) > LEFT_RECURSION_DEPTH:
grammar.recursion_locations__.add(location)
return None, text
if grammar.history_tracking__: if grammar.history_tracking__:
grammar.call_stack__.append(parser) grammar.call_stack__.append(parser)
grammar.moving_forward__ = True grammar.moving_forward__ = True
parser.recursion_counter[location] += 1 # break left recursion at the maximum allowed depth
if grammar.left_recursion_handling__:
if parser.recursion_counter.setdefault(location, 0) > LEFT_RECURSION_DEPTH:
grammar.recursion_locations__.add(location)
return None, text
parser.recursion_counter[location] += 1
# run original __call__ method # run original __call__ method
node, rest = parser_func(parser, text) node, rest = parser_func(parser, text)
if grammar.left_recursion_handling__:
parser.recursion_counter[location] -= 1
if node is None: if node is None:
# retrieve an earlier match result (from left recursion) if it exists # retrieve an earlier match result (from left recursion) if it exists
if location in grammar.recursion_locations__: if location in grammar.recursion_locations__:
if location in parser.visited: if location in parser.visited:
node, rest = parser.visited[location] node, rest = parser.visited[location]
# TODO: add a warning about occurence of left-recursion here
# don't overwrite any positive match (i.e. node not None) in the cache # don't overwrite any positive match (i.e. node not None) in the cache
# and don't add empty entries for parsers returning from left recursive calls! # and don't add empty entries for parsers returning from left recursive calls!
elif grammar.memoization__: elif grammar.memoization__:
...@@ -286,8 +290,6 @@ def add_parser_guard(parser_func): ...@@ -286,8 +290,6 @@ def add_parser_guard(parser_func):
# matches will store its result in the cache # matches will store its result in the cache
parser.visited[location] = (node, rest) parser.visited[location] = (node, rest)
parser.recursion_counter[location] -= 1
if grammar.history_tracking__: if grammar.history_tracking__:
# don't track returning parsers except in case an error has occurred # don't track returning parsers except in case an error has occurred
remaining = len(rest) remaining = len(rest)
...@@ -299,7 +301,7 @@ def add_parser_guard(parser_func): ...@@ -299,7 +301,7 @@ def add_parser_guard(parser_func):
grammar.call_stack__.pop() grammar.call_stack__.pop()
except RecursionError: except RecursionError:
node = Node(None, text[:min(10, max(1, text.find("\n")))] + " ...") node = Node(None, str(text[:min(10, max(1, text.find("\n")))]) + " ...")
node.add_error("maximum recursion depth of parser reached; " node.add_error("maximum recursion depth of parser reached; "
"potentially due to too many errors!") "potentially due to too many errors!")
rest = EMPTY_STRING_VIEW rest = EMPTY_STRING_VIEW
...@@ -640,7 +642,11 @@ class Grammar: ...@@ -640,7 +642,11 @@ class Grammar:
memoization__: Turns full memoization on or off. Turning memoization off memoization__: Turns full memoization on or off. Turning memoization off
results in less memory usage and sometimes reduced parsing time. results in less memory usage and sometimes reduced parsing time.
In some situations it may drastically increase parsing time, so In some situations it may drastically increase parsing time, so
it is safer to leave it on. it is safer to leave it on. (Default: on)
left_recursion_handling__: Turns left recursion handling on or off.
If turned off, a recursion error will result in case of left
recursion.
""" """
root__ = None # type: Union[Parser, None] root__ = None # type: Union[Parser, None]
# root__ must be overwritten with the root-parser by grammar subclass # root__ must be overwritten with the root-parser by grammar subclass
...@@ -694,10 +700,11 @@ class Grammar: ...@@ -694,10 +700,11 @@ class Grammar:
# self.wspL__ = '' # self.wspL__ = ''
# if not hasattr(self.__class__, 'wspR__'): # if not hasattr(self.__class__, 'wspR__'):
# self.wspR__ = '' # self.wspR__ = ''
self.all_parsers__ = set() # type: Set[Parser] self.all_parsers__ = set() # type: Set[Parser]
self._dirty_flag__ = False # type: bool self._dirty_flag__ = False # type: bool
self.history_tracking__ = False # type: bool self.history_tracking__ = False # type: bool
self.memoization__ = True # type: bool self.memoization__ = True # type: bool
self.left_recursion_handling__ = False # type: bool
self._reset__() self._reset__()
# prepare parsers in the class, first # prepare parsers in the class, first
......
...@@ -358,7 +358,7 @@ class Node(collections.abc.Sized): ...@@ -358,7 +358,7 @@ class Node(collections.abc.Sized):
return head + '\n'.join([tab + dataF(s) for s in res.split('\n')]) + tail.lstrip(D) return head + '\n'.join([tab + dataF(s) for s in res.split('\n')]) + tail.lstrip(D)
def as_sxpr(self, src: str=None) -> str: def as_sxpr(self, src: str=None, compact: bool=False) -> str:
""" """
Returns content as S-expression, i.e. in lisp-like form. Returns content as S-expression, i.e. in lisp-like form.
...@@ -368,8 +368,10 @@ class Node(collections.abc.Sized): ...@@ -368,8 +368,10 @@ class Node(collections.abc.Sized):
reported as line and column. reported as line and column.
""" """
lB, rB, D = ('', '', 1) if compact else ('(', '\n)', 0)
def opening(node) -> str: def opening(node) -> str:
s = '(' + node.tag_name s = lB + node.tag_name
# s += " '(pos %i)" % node.pos # s += " '(pos %i)" % node.pos
if src: if src:
s += " '(pos %i " % node.pos + " %i %i)" % line_col(src, node.pos) s += " '(pos %i " % node.pos + " %i %i)" % line_col(src, node.pos)
...@@ -383,7 +385,7 @@ class Node(collections.abc.Sized): ...@@ -383,7 +385,7 @@ class Node(collections.abc.Sized):
else "'%s'" % s if s.find("'") < 0 \ else "'%s'" % s if s.find("'") < 0 \
else '"%s"' % s.replace('"', r'\"') else '"%s"' % s.replace('"', r'\"')
return self._tree_repr(' ', opening, lambda node: '\n)', pretty, density=0) return self._tree_repr(' ', opening, lambda node: rB, pretty, density=D)
def as_xml(self, src: str=None) -> str: def as_xml(self, src: str=None) -> str:
......
...@@ -14,8 +14,8 @@ ...@@ -14,8 +14,8 @@
like the comment above like the comment above
% or like thist comment. % or like thist comment.
Comment lines do not break paragraphs. Comment lines do not break paragraphs.
% There can even be several % There can even be several comment lines,
% comment lines % even indented comment lines,
in sequence. in sequence.
5 : Paragraphs may contain {\em emphasized} or {\bf bold} text. 5 : Paragraphs may contain {\em emphasized} or {\bf bold} text.
...@@ -43,10 +43,17 @@ ...@@ -43,10 +43,17 @@
[fail:paragraph] [fail:paragraph]
1 : \begin{enumerate} 1 : Paragraphs are separated by gaps.
2 : \item
3 : und Vieh; \paragraph Like this one.
4 : Paragraphs will end
2 : \begin{enumerate}
3 : \item
4 : und Vieh; \paragraph
5 : Paragraphs will end
\begin{quotation} \begin{quotation}
at block environments at block environments
\end{quotation} \end{quotation}
...@@ -68,11 +75,9 @@ ...@@ -68,11 +75,9 @@
% sequences of separators % sequences of separators
% and comments % and comments
% or sequences of comment lines % or sequences of comment lines
In the end such a sequence counts In the end such a sequence counts
% merely as one comment % merely as one comment
......
import functools
import sys
sys.path.extend(['../../', '../', './'])
from DHParser import *
arithmetik_ebnf = """
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/~ | "(" expr ")"
"""
ASTTable = {
"+": remove_expendables,
"*": replace_by_single_child,
"factor": [reduce_single_child, remove_brackets]
}
parser = grammar_provider(arithmetik_ebnf)()
transformer = functools.partial(traverse, processing_table=ASTTable.copy())
syntax_tree = parser("5 + 3 * 4")
assert not syntax_tree.error_flag, str(syntax_tree.collect_errors())
transformer(syntax_tree)
print(">>>>> Ausdruck ohne Klammern:\n")
print(syntax_tree.as_sxpr(compact = True))
syntax_tree = parser("(5 + 3) * 4")
assert not syntax_tree.error_flag, str(syntax_tree.collect_errors())
transformer(syntax_tree)
print("\n\n>>>>> Ausdruck mit Klammern:\n")
print(syntax_tree.as_sxpr(compact = True))
...@@ -34,12 +34,9 @@ from DHParser.dsl import grammar_provider, DHPARSER_IMPORTS ...@@ -34,12 +34,9 @@ from DHParser.dsl import grammar_provider, DHPARSER_IMPORTS
class TestInfiLoopsAndRecursion: class TestInfiLoopsAndRecursion:
def test_direct_left_recursion1(self): def test_direct_left_recursion1(self):
minilang =""" minilang ="""
@ whitespace = linefeed
formula = [ //~ ] expr
expr = expr ("+"|"-") term | term expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor term = term ("*"|"/") factor | factor
factor = /[0-9]+/~ factor = /[0-9]+/~
# example: "5 + 3 * 4"
""" """
snippet = "9 + 8 + 7 + 6 + 5 + 3 * 4" snippet = "9 + 8 + 7 + 6 + 5 + 3 * 4"
parser = grammar_provider(minilang)() parser = grammar_provider(minilang)()
...@@ -53,13 +50,10 @@ class TestInfiLoopsAndRecursion: ...@@ -53,13 +50,10 @@ class TestInfiLoopsAndRecursion:
def test_direct_left_recursion2(self): def test_direct_left_recursion2(self):
minilang = """ minilang = """
@ whitespace = linefeed
formula = [ //~ ] expr
expr = ex expr = ex
ex = expr ("+"|"-") term | term ex = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor term = term ("*"|"/") factor | factor
factor = /[0-9]+/~ factor = /[0-9]+/~
# example: "5 + 3 * 4"
""" """
snippet = "9 + 8 + 7 + 6 + 5 + 3 * 4" snippet = "9 + 8 + 7 + 6 + 5 + 3 * 4"
parser = grammar_provider(minilang)() parser = grammar_provider(minilang)()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment