Commit 219a01a8 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

- parsrs.py optimization for guarded_call rollback-tests

parent e4638fde
......@@ -172,10 +172,9 @@ def add_parser_guard(parser_func):
if not grammar.moving_forward__:
# rollback variable changes for discarded branch of parsing tree
while grammar.rollback__ and grammar.rollback__[-1][0] <= location:
grammar.rollback__[-1][1]()
grammar.rollback__.pop()
grammar.moving_forward__ = True
if grammar.last_rb__loc__ <= location:
grammar.rollback_to__(location)
grammar.moving_forward__ = True
if grammar.history_tracking__:
grammar.call_stack__.append(parser)
......@@ -197,7 +196,7 @@ def add_parser_guard(parser_func):
# in case of a recursive call saves the result of the first
# (or left-most) call that matches; but not for variable manipulating parsers,
# because caching would interfere with changes of variable state
if not (grammar.rollback__ and grammar.rollback__[-1][0] <= location):
if grammar.last_rb__loc__ > location:
parser.visited[location] = (node, rest)
grammar.last_node__ = node # store last node for Lookbehind parser
elif location in parser.visited:
......@@ -339,7 +338,7 @@ class Grammar:
self.all_parsers__ = set() # type: Set[Parser]
self.dirty_flag__ = False
self.history_tracking__ = False
self._reset()
self._reset__()
# prepare parsers in the class, first
self._assign_parser_names()
# then deep-copy the parser tree from class to instance;
......@@ -359,7 +358,7 @@ class Grammar:
self.all_parsers__.add(self.wsp_right_parser__) # don't you forget about me...
else:
self.wsp_right_parser__ = ZOMBIE_PARSER
self.root__.apply(self._add_parser)
self.root__.apply(self._add_parser__)
def __getitem__(self, key):
try:
......@@ -369,16 +368,17 @@ class Grammar:
if parser_template:
# add parser to grammar object on the fly...
parser = copy.deepcopy(parser_template)
parser.apply(self._add_parser)
parser.apply(self._add_parser__)
# assert self[key] == parser
return self[key]
raise KeyError('Unknown parser "%s" !' % key)
def _reset(self):
def _reset__(self):
self.document__ = "" # type: str
# variables stored and recalled by Capture and Retrieve parsers
self.variables__ = dict() # type: Dict[str, List[str]]
self.rollback__ = [] # type: List[Tuple[int, Callable]]
self.last_rb__loc__ = -1 # type: int
# previously parsed node, needed by Lookbehind parser
self.last_node__ = None # type: Node
# support for call stack tracing
......@@ -388,7 +388,7 @@ class Grammar:
# also needed for call stack tracing
self.moving_forward__ = True # type: bool
def _add_parser(self, parser: Parser) -> None:
def _add_parser__(self, parser: Parser) -> None:
"""Adds the particular copy of the parser object to this
particular instance of Grammar.
"""
......@@ -417,13 +417,14 @@ class Grammar:
if self.root__ is None:
raise NotImplementedError()
if self.dirty_flag__:
self._reset()
self._reset__()
for parser in self.all_parsers__:
parser.reset()
else:
self.dirty_flag__ = True
self.history_tracking__ = is_logging()
self.document__ = document
self.last_rb__loc__ = len(document) + 1 # rollback location
parser = self[start_parser] if isinstance(start_parser, str) else start_parser
assert parser.grammar == self, "Cannot run parsers from a different grammar object!" \
" %s vs. %s" % (str(self), str(parser.grammar))
......@@ -469,7 +470,19 @@ class Grammar:
result.pos = 0 # calculate all positions
return result
def log_parsing_history(self, log_file_name: str='') -> None:
def push_rollback__(self, location, func):
self.rollback__.append((location, func))
self.last_rb__loc__ = location
def rollback_to__(self, location):
while self.rollback__ and self.rollback__[-1][0] <= location:
loc, rollback_func = self.rollback__.pop()
assert not loc > self.last_rb__loc__
rollback_func()
self.last_rb__loc__ == self.rollback__[-1][0] if self.rollback__ \
else (len(self.document__) + 1)
def log_parsing_history__(self, log_file_name: str = '') -> None:
"""Writes a log of the parsing history of the most recently parsed
document.
"""
......@@ -1085,7 +1098,8 @@ class Capture(UnaryOperator):
if node:
stack = self.grammar.variables__.setdefault(self.name, [])
stack.append(str(node))
self.grammar.rollback__.append((len(text), lambda : stack.pop()))
self.grammar.push_rollback__(len(text), lambda: stack.pop())
# self.grammar.rollback__.append((len(text), lambda : stack.pop()))
# block caching, because it would prevent recapturing of rolled back captures
return Node(self, node), text_
else:
......@@ -1148,7 +1162,8 @@ class Pop(Retrieve):
if nd and not nd.error_flag:
stack = self.grammar.variables__[self.symbol.name]
value = stack.pop()
self.grammar.rollback__.append((len(text), lambda : stack.append(value)))
self.grammar.push_rollback__(len(text), lambda: stack.append(value))
# self.grammar.rollback__.append((len(text), lambda : stack.append(value)))
return nd, txt
def __repr__(self):
......@@ -1306,7 +1321,7 @@ def compile_source(source: str,
syntax_tree = parser(source_text)
if is_logging():
syntax_tree.log(log_file_name + '.cst')
parser.log_parsing_history(log_file_name)
parser.log_parsing_history__(log_file_name)
assert syntax_tree.error_flag or str(syntax_tree) == source_text, str(syntax_tree)
# only compile if there were no syntax errors, for otherwise it is
......
......@@ -232,7 +232,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
cst = parser(test_code, parser_name)
if is_logging():
cst.log("match_%s_%s.cst" % (parser_name, test_name))
parser.log_parsing_history("match_%s_%s.log" % (parser_name, test_name))
parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
tests.setdefault('__cst__', {})[test_name] = cst
if "ast" in tests or report:
ast = copy.deepcopy(cst)
......@@ -270,7 +270,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
cst = parser(test_code, parser_name)
if is_logging():
cst.log("fail_%s_%s.cst" % (parser_name, test_name))
parser.log_parsing_history("fail_%s_%s.log" % (parser_name, test_name))
parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
if not cst.error_flag:
errata.append('Fail test "%s" for parser "%s" yields match instead of '
'expected failure!' % (test_name, parser_name))
......
......@@ -1406,7 +1406,7 @@ def full_compilation(source, grammar_base, AST_transformations, compiler):
syntax_tree = grammar_base(source)
syntax_tree.log(grammar_base.log_file_name, ext='.cst')
grammar_base.log_parsing_history()
grammar_base.log_parsing_history__()
assert syntax_tree.error_flag or str(syntax_tree) == source, str(syntax_tree)
# only compile if there were no syntax errors, for otherwise it is
......
......@@ -86,5 +86,5 @@ if __name__ == "__main__":
else:
# run self test
# selftest('EBNF/EBNF.ebnf')
with logging(True):
with logging(False):
profile(partial(selftest, file_name='EBNF/EBNF.ebnf'))
......@@ -25,8 +25,6 @@ from functools import partial
sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging, logging, compile_python_object
from DHParser.syntaxtree import traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten, TOKEN_PTYPE
from DHParser.parsers import compile_source, Retrieve
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import parser_factory, DHPARSER_IMPORTS
......@@ -86,7 +84,7 @@ class TestInfiLoopsAndRecursion:
assert snippet == str(syntax_tree)
if is_logging():
syntax_tree.log("test_LeftRecursion_direct.cst")
# self.minilang_parser1.log_parsing_history("test_LeftRecursion_direct")
# self.minilang_parser1.log_parsing_history__("test_LeftRecursion_direct")
def test_indirect_left_recursion(self):
minilang = ARITHMETIC2_EBNF
......@@ -143,7 +141,7 @@ class TestRegex:
assert not messages
parser = compile_python_object(DHPARSER_IMPORTS + result, '\w+Grammar$')()
result = parser(testdoc)
# parser.log_parsing_history("test.log")
# parser.log_parsing_history__("test.log")
assert not result.error_flag
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment