Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 20dd34fb authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

Ticket #232

parent 622f2348
...@@ -454,11 +454,11 @@ class TreeProcessor(Compiler): ...@@ -454,11 +454,11 @@ class TreeProcessor(Compiler):
functions which makes sure that a tree-processor is only invoked if no functions which makes sure that a tree-processor is only invoked if no
fatal errors have occurred in any of the earlier stages. fatal errors have occurred in any of the earlier stages.
""" """
def __call__(self, root: RootNode) -> None: def __call__(self, root: RootNode) -> Optional[RootNode]:
assert isinstance(root, RootNode) assert isinstance(root, RootNode)
result = super().__call__(root) result = super().__call__(root)
assert isinstance(result, RootNode), str(result) assert result is None or isinstance(result, RootNode), str(result)
return result
TreeProcessorCallable = Union[TreeProcessor, Callable[[RootNode], RootNode], functools.partial] TreeProcessorCallable = Union[TreeProcessor, Callable[[RootNode], RootNode], functools.partial]
......
...@@ -1621,11 +1621,11 @@ class Grammar: ...@@ -1621,11 +1621,11 @@ class Grammar:
i = self.ff_pos__ or tail_pos(stitches) i = self.ff_pos__ or tail_pos(stitches)
fs = self.document__[i:i + 10] fs = self.document__[i:i + 10]
if i + 10 < len(self.document__) - 1: fs += ' ...' if i + 10 < len(self.document__) - 1: fs += ' ...'
root_name = self.root_parser__.pname \ root_name = self.start_parser__.pname \
or self.associated_symbol__(self.root_parser__).pname or self.associated_symbol__(self.start_parser__).pname
error_msg = f'Parser "{root_name}" ' \ error_msg = f'Parser "{root_name}" ' \
"stopped before end, at: " + fs + \ "stopped before end, at: »" + fs + \
((" Trying to recover" + (("« Trying to recover" +
(" but stopping history recording at this point." (" but stopping history recording at this point."
if self.history_tracking__ else "...")) if self.history_tracking__ else "..."))
if len(stitches) < self.max_parser_dropouts__ if len(stitches) < self.max_parser_dropouts__
......
...@@ -505,9 +505,9 @@ the other end of the tree rooted in `node`:: ...@@ -505,9 +505,9 @@ the other end of the tree rooted in `node`::
>>> t = parse_sxpr('(A (B 1) (C (D (E 2) (F 3))) (G 4) (H (I 5) (J 6)) (K 7))') >>> t = parse_sxpr('(A (B 1) (C (D (E 2) (F 3))) (G 4) (H (I 5) (J 6)) (K 7))')
>>> pointer = t.pick_context('G') >>> pointer = t.pick_context('G')
>>> [serialize_context(ctx, with_content=1) for ctx in select_context(pointer, ALL_CONTEXTS)] >>> [serialize_context(ctx, with_content=1) for ctx in select_context(pointer, ANY_CONTEXT)]
['A <- G:4', 'A <- H <- I:5', 'A <- H <- J:6', 'A <- H:56', 'A <- K:7', 'A:1234567'] ['A <- G:4', 'A <- H <- I:5', 'A <- H <- J:6', 'A <- H:56', 'A <- K:7', 'A:1234567']
>>> [serialize_context(ctx, with_content=1) for ctx in select_context(pointer, ALL_CONTEXTS, reverse=True)] >>> [serialize_context(ctx, with_content=1) for ctx in select_context(pointer, ANY_CONTEXT, reverse=True)]
['A <- G:4', 'A <- C <- D <- F:3', 'A <- C <- D <- E:2', 'A <- C <- D:23', 'A <- C:23', 'A <- B:1', 'A:1234567'] ['A <- G:4', 'A <- C <- D <- F:3', 'A <- C <- D <- E:2', 'A <- C <- D:23', 'A <- C:23', 'A <- B:1', 'A:1234567']
Another important difference, besides the starting point is then the Another important difference, besides the starting point is then the
...@@ -515,10 +515,10 @@ Another important difference, besides the starting point is then the ...@@ -515,10 +515,10 @@ Another important difference, besides the starting point is then the
post-order (or "depth first"), while the respective methods ot the post-order (or "depth first"), while the respective methods ot the
Node-class traverse the tree pre-order. See the difference:: Node-class traverse the tree pre-order. See the difference::
>>> l = [serialize_context(ctx, with_content=1) for ctx in t.select_context(ALL_CONTEXTS, include_root=True)] >>> l = [serialize_context(ctx, with_content=1) for ctx in t.select_context(ANY_CONTEXT, include_root=True)]
>>> l[l.index('A <- G:4'):] >>> l[l.index('A <- G:4'):]
['A <- G:4', 'A <- H:56', 'A <- H <- I:5', 'A <- H <- J:6', 'A <- K:7'] ['A <- G:4', 'A <- H:56', 'A <- H <- I:5', 'A <- H <- J:6', 'A <- K:7']
>>> l = [serialize_context(ctx, with_content=1) for ctx in t.select_context(ALL_CONTEXTS, include_root=True, reverse=True)] >>> l = [serialize_context(ctx, with_content=1) for ctx in t.select_context(ANY_CONTEXT, include_root=True, reverse=True)]
>>> l[l.index('A <- G:4'):] >>> l[l.index('A <- G:4'):]
['A <- G:4', 'A <- C:23', 'A <- C <- D:23', 'A <- C <- D <- F:3', 'A <- C <- D <- E:2', 'A <- B:1'] ['A <- G:4', 'A <- C:23', 'A <- C <- D:23', 'A <- C <- D <- F:3', 'A <- C <- D <- E:2', 'A <- B:1']
...@@ -696,12 +696,12 @@ __all__ = ('WHITESPACE_PTYPE', ...@@ -696,12 +696,12 @@ __all__ = ('WHITESPACE_PTYPE',
'CriteriaType', 'CriteriaType',
'NodeMatchFunction', 'NodeMatchFunction',
'ContextMatchFunction', 'ContextMatchFunction',
'ALL_NODES', 'ANY_NODE',
'NO_NODES', 'NO_NODE',
'LEAF_NODES', 'LEAF_NODE',
'ALL_CONTEXTS', 'ANY_CONTEXT',
'NO_CONTEXTS', 'NO_CONTEXT',
'LEAF_CONTEXTS', 'LEAF_CONTEXT',
'Node', 'Node',
'content', 'content',
'validate_token_sequence', 'validate_token_sequence',
...@@ -788,13 +788,13 @@ TreeContext = List['Node'] ...@@ -788,13 +788,13 @@ TreeContext = List['Node']
NodeMatchFunction = Callable[['Node'], bool] NodeMatchFunction = Callable[['Node'], bool]
ContextMatchFunction = Callable[[TreeContext], bool] ContextMatchFunction = Callable[[TreeContext], bool]
ALL_NODES = lambda nd: True ANY_NODE = lambda nd: True
NO_NODES = lambda nd: False NO_NODE = lambda nd: False
LEAF_NODES = lambda nd: not nd._children LEAF_NODE = lambda nd: not nd._children
ALL_CONTEXTS = lambda ctx: True ANY_CONTEXT = lambda ctx: True
NO_CONTEXTS = lambda ctx: False NO_CONTEXT = lambda ctx: False
LEAF_CONTEXTS = lambda ctx: not ctx[-1].children LEAF_CONTEXT = lambda ctx: not ctx[-1].children
def create_match_function(criterion: CriteriaType) -> NodeMatchFunction: def create_match_function(criterion: CriteriaType) -> NodeMatchFunction:
...@@ -1582,7 +1582,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -1582,7 +1582,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def select_if(self, match_function: NodeMatchFunction, def select_if(self, match_function: NodeMatchFunction,
include_root: bool = False, reverse: bool = False, include_root: bool = False, reverse: bool = False,
skip_subtree: NodeMatchFunction = NO_NODES) -> Iterator['Node']: skip_subtree: NodeMatchFunction = NO_NODE) -> Iterator['Node']:
""" """
Generates an iterator over all nodes in the tree for which Generates an iterator over all nodes in the tree for which
`match_function()` returns True. See the more general function `match_function()` returns True. See the more general function
...@@ -1600,7 +1600,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -1600,7 +1600,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def select(self, criterion: CriteriaType, def select(self, criterion: CriteriaType,
include_root: bool = False, reverse: bool = False, include_root: bool = False, reverse: bool = False,
skip_subtree: CriteriaType = NO_NODES) -> Iterator['Node']: skip_subtree: CriteriaType = NO_NODE) -> Iterator['Node']:
""" """
Generates an iterator over all nodes in the tree that fulfill the Generates an iterator over all nodes in the tree that fulfill the
given criterion. See :py:func:`create_match_function()` for a given criterion. See :py:func:`create_match_function()` for a
...@@ -1651,7 +1651,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -1651,7 +1651,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def pick(self, criterion: CriteriaType, def pick(self, criterion: CriteriaType,
include_root: bool = False, include_root: bool = False,
reverse: bool = False, reverse: bool = False,
skip_subtree: CriteriaType = NO_NODES) -> Optional['Node']: skip_subtree: CriteriaType = NO_NODE) -> Optional['Node']:
""" """
Picks the first (or last if run in reverse mode) descendant that Picks the first (or last if run in reverse mode) descendant that
fulfils the given criterion. See :py:func:`create_match_function()` fulfils the given criterion. See :py:func:`create_match_function()`
...@@ -1714,7 +1714,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -1714,7 +1714,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def select_context_if(self, match_function: ContextMatchFunction, def select_context_if(self, match_function: ContextMatchFunction,
include_root: bool = False, include_root: bool = False,
reverse: bool = False, reverse: bool = False,
skip_subtree: ContextMatchFunction = NO_CONTEXTS) -> Iterator[TreeContext]: skip_subtree: ContextMatchFunction = NO_CONTEXT) -> Iterator[TreeContext]:
""" """
Like :py:func:`Node.select_if()` but yields the entire context (i.e. list Like :py:func:`Node.select_if()` but yields the entire context (i.e. list
of descendants, the last one being the matching node) instead of just of descendants, the last one being the matching node) instead of just
...@@ -1738,7 +1738,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -1738,7 +1738,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def select_context(self, criterion: CriteriaType, def select_context(self, criterion: CriteriaType,
include_root: bool = False, include_root: bool = False,
reverse: bool = False, reverse: bool = False,
skip_subtree: CriteriaType = NO_CONTEXTS) -> Iterator[TreeContext]: skip_subtree: CriteriaType = NO_CONTEXT) -> Iterator[TreeContext]:
""" """
Like :py:meth:`Node.select()` but yields the entire context (i.e. list of Like :py:meth:`Node.select()` but yields the entire context (i.e. list of
descendants, the last one being the matching node) instead of just descendants, the last one being the matching node) instead of just
...@@ -1751,7 +1751,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -1751,7 +1751,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def pick_context(self, criterion: CriteriaType, def pick_context(self, criterion: CriteriaType,
include_root: bool = False, include_root: bool = False,
reverse: bool = False, reverse: bool = False,
skip_subtree: CriteriaType = NO_CONTEXTS) -> TreeContext: skip_subtree: CriteriaType = NO_CONTEXT) -> TreeContext:
""" """
Like :py:meth:`Node.pick()`, only that the entire context (i.e. Like :py:meth:`Node.pick()`, only that the entire context (i.e.
chain of descendants) relative to `self` is returned. chain of descendants) relative to `self` is returned.
...@@ -2471,17 +2471,31 @@ def ensuing_str(context: TreeContext, length: int = -1) -> str: ...@@ -2471,17 +2471,31 @@ def ensuing_str(context: TreeContext, length: int = -1) -> str:
def select_context_if(context: TreeContext, def select_context_if(context: TreeContext,
match_function: ContextMatchFunction, match_function: ContextMatchFunction,
reverse: bool = False) -> Iterator[TreeContext]: include_root: bool = False,
reverse: bool = False,
skip_subtree: CriteriaType = NO_CONTEXT) -> Iterator[TreeContext]:
""" """
Creates an Iterator yielding all `contexts` for which the Creates an Iterator yielding all `contexts` for which the
`match_function` is true, starting from `context`. `match_function` is true, starting from `context`.
""" """
def recursive(ctx, include_root):
nonlocal match_function, reverse, skip_subtree
if include_root and match_function(ctx):
yield ctx
top = ctx[-1]
child_iterator = reversed(top._children) if reverse else top._children
for child in child_iterator:
child_ctx = ctx + [child]
if match_function(child_ctx):
yield child_ctx
if child._children and not skip_subtree(child_ctx):
yield from recursive(child_ctx, include_root=False)
context = context.copy() context = context.copy()
while context: while context:
if match_function(context): yield from recursive(context, include_root)
yield context
node = context.pop() node = context.pop()
edge, delta = (0, -1) if reverse else (-1, 1) edge, delta = (0, -1) if reverse else (-1, 1)
while context and node is context[-1]._children[edge]: while context and node is context[-1]._children[edge]:
if match_function(context): if match_function(context):
...@@ -2491,25 +2505,48 @@ def select_context_if(context: TreeContext, ...@@ -2491,25 +2505,48 @@ def select_context_if(context: TreeContext,
parent = context[-1] parent = context[-1]
i = parent.index(node) i = parent.index(node)
nearest_sibling = parent._children[i + delta] nearest_sibling = parent._children[i + delta]
innermost_ctx = nearest_sibling.pick_context( context.append(nearest_sibling)
LEAF_CONTEXTS, include_root=True, reverse=reverse) include_root = True
context.extend(innermost_ctx)
# context = context.copy()
# while context:
# if match_function(context):
# yield context
# node = context.pop()
#
# edge, delta = (0, -1) if reverse else (-1, 1)
# while context and node is context[-1]._children[edge]:
# if match_function(context):
# yield context
# node = context.pop()
# if context:
# parent = context[-1]
# i = parent.index(node)
# nearest_sibling = parent._children[i + delta]
# innermost_ctx = nearest_sibling.pick_context(
# LEAF_CONTEXT, include_root=True, reverse=reverse)
# context.extend(innermost_ctx)
def select_context(context: TreeContext, def select_context(context: TreeContext,
criterion: CriteriaType, criterion: CriteriaType,
reverse: bool = False) -> Iterator[TreeContext]: include_root: bool = False,
reverse: bool = False,
skip_subtree: CriteriaType = NO_CONTEXT) -> Iterator[TreeContext]:
""" """
Like `select_context_if()` but yields the entire context (i.e. list of Like `select_context_if()` but yields the entire context (i.e. list of
descendants, the last one being the matching node) instead of just descendants, the last one being the matching node) instead of just
the matching nodes. the matching nodes.
""" """
return select_context_if(context, create_context_match_function(criterion), reverse) return select_context_if(context, create_context_match_function(criterion),
include_root, reverse, skip_subtree)
def pick_context(context: TreeContext, def pick_context(context: TreeContext,
criterion: CriteriaType, criterion: CriteriaType,
reverse: bool = False) -> Optional[TreeContext]: include_root: bool = False,
reverse: bool = False,
skip_subtree: CriteriaType = NO_CONTEXT) -> Optional[TreeContext]:
""" """
Like `Node.pick()`, only that the entire context (i.e. chain of descendants) Like `Node.pick()`, only that the entire context (i.e. chain of descendants)
relative to `self` is returned. relative to `self` is returned.
...@@ -2595,7 +2632,7 @@ def generate_context_mapping(node: Node) -> ContextMapping: ...@@ -2595,7 +2632,7 @@ def generate_context_mapping(node: Node) -> ContextMapping:
""" """
pos = 0 pos = 0
pos_list, ctx_list = [], [] pos_list, ctx_list = [], []
for ctx in node.select_context_if(LEAF_CONTEXTS, include_root=True): for ctx in node.select_context_if(LEAF_CONTEXT, include_root=True):
pos_list.append(pos) pos_list.append(pos)
ctx_list.append(ctx) ctx_list.append(ctx)
pos += len(ctx[-1]) pos += len(ctx[-1])
......
...@@ -29,9 +29,10 @@ sys.path.append(os.path.abspath(os.path.join(scriptpath, '..'))) ...@@ -29,9 +29,10 @@ sys.path.append(os.path.abspath(os.path.join(scriptpath, '..')))
from DHParser.configuration import get_config_value, set_config_value from DHParser.configuration import get_config_value, set_config_value
from DHParser.syntaxtree import Node, RootNode, parse_sxpr, parse_xml, flatten_sxpr, \ from DHParser.syntaxtree import Node, RootNode, parse_sxpr, parse_xml, flatten_sxpr, \
flatten_xml, parse_json, ZOMBIE_TAG, EMPTY_NODE, ALL_NODES, next_context, \ flatten_xml, parse_json, ZOMBIE_TAG, EMPTY_NODE, ANY_NODE, next_context, \
prev_context, serialize_context, generate_context_mapping, map_pos_to_context, \ prev_context, serialize_context, generate_context_mapping, map_pos_to_context, \
select_context_if, select_context, create_context_match_function select_context_if, select_context, create_context_match_function, pick_context, \
LEAF_CONTEXT
from DHParser.transform import traverse, reduce_single_child, \ from DHParser.transform import traverse, reduce_single_child, \
replace_by_single_child, flatten, remove_empty, remove_whitespace replace_by_single_child, flatten, remove_empty, remove_whitespace
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
...@@ -307,7 +308,7 @@ class TestNode: ...@@ -307,7 +308,7 @@ class TestNode:
for node in self.unique_tree.select_if(lambda nd: True, include_root=True)] for node in self.unique_tree.select_if(lambda nd: True, include_root=True)]
assert ''.join(tags) == "abdfg", ''.join(tags) assert ''.join(tags) == "abdfg", ''.join(tags)
tags = [node.tag_name tags = [node.tag_name
for node in self.unique_tree.select(ALL_NODES, include_root=True, skip_subtree='f')] for node in self.unique_tree.select(ANY_NODE, include_root=True, skip_subtree='f')]
assert ''.join(tags) == "abdf", ''.join(tags) assert ''.join(tags) == "abdf", ''.join(tags)
def test_tree_select_context_if(self): def test_tree_select_context_if(self):
...@@ -349,7 +350,7 @@ class TestNode: ...@@ -349,7 +350,7 @@ class TestNode:
def test_select_children(self): def test_select_children(self):
tree = parse_sxpr('(A (B 1) (C (X 1) (Y 1)) (B 2))') tree = parse_sxpr('(A (B 1) (C (X 1) (Y 1)) (B 2))')
children = list(nd.tag_name for nd in tree.select_children(ALL_NODES)) children = list(nd.tag_name for nd in tree.select_children(ANY_NODE))
assert children == ['B', 'C', 'B'] assert children == ['B', 'C', 'B']
B_values = list(nd.content for nd in tree.select_children('B', reverse=True)) B_values = list(nd.content for nd in tree.select_children('B', reverse=True))
assert B_values == ['2', '1'] assert B_values == ['2', '1']
...@@ -845,7 +846,7 @@ class TestContextNavigation: ...@@ -845,7 +846,7 @@ class TestContextNavigation:
save = start.copy() save = start.copy()
sequence = [] sequence = []
for ctx in select_context_if( for ctx in select_context_if(
start, lambda c: True, reverse=True): start, lambda c: True, include_root=True, reverse=True):
sequence.append(''.join(n.tag_name for n in ctx)) sequence.append(''.join(n.tag_name for n in ctx))
assert sequence == ['ACE', 'ACD', 'AC', 'AB', 'A'] assert sequence == ['ACE', 'ACD', 'AC', 'AB', 'A']
assert save == start # context passed should not be changed by select_context assert save == start # context passed should not be changed by select_context
...@@ -853,10 +854,17 @@ class TestContextNavigation: ...@@ -853,10 +854,17 @@ class TestContextNavigation:
start = self.tree.pick_context('D') start = self.tree.pick_context('D')
sequence = [] sequence = []
for ctx in select_context_if( for ctx in select_context_if(
start, lambda c: True, reverse=False): start, lambda c: True, include_root=True, reverse=False):
sequence.append(''.join(n.tag_name for n in ctx)) sequence.append(''.join(n.tag_name for n in ctx))
assert sequence == ['ACD', 'ACE', 'AC', 'AF', 'A'] assert sequence == ['ACD', 'ACE', 'AC', 'AF', 'A']
def test_standalone_pick_context(self):
start = self.tree.pick_context('A', include_root=True)
anfang = pick_context(start, LEAF_CONTEXT)
ende = pick_context(start, LEAF_CONTEXT, reverse=True)
assert anfang[-1].tag_name == 'B'
assert ende[-1].tag_name == 'F'
if __name__ == "__main__": if __name__ == "__main__":
from DHParser.testing import runner from DHParser.testing import runner
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment