- bug fixes

DHParser/error.py

@@ -41,7 +41,7 @@ import bisect
 from DHParser.preprocess import SourceMapFunc
 from DHParser.stringview import StringView
 from DHParser.toolkit import typing
-from typing import Iterable, Iterator, Union, Tuple, List
+from typing import Iterable, Iterator, Union, Tuple, List, Optional
 
 __all__ = ('Error',
            'is_error',
@@ -54,7 +54,7 @@ __all__ = ('Error',
 
 
 class Error:
-    __slots__ = ['message', 'level', 'code', 'pos', 'orig_pos', 'line', 'column']
+    __slots__ = ['message', 'level', 'code', '_pos', 'orig_pos', 'line', 'column', '_node']
 
     # error levels
 
@@ -75,14 +75,19 @@ class Error:
     MANDATORY_CONTINUATION = 1001
 
     def __init__(self, message: str, code: int = ERROR, pos: int = -1,
-                 orig_pos: int = -1, line: int = -1, column: int = -1) -> None:
+                 orig_pos: int = -1, line: int = -1, column: int = -1,
+                 node: Optional['Node'] = None) -> None:
         self.message = message
         assert code >= 0
         self.code = code
-        self.pos = pos
+        self._pos = pos
         self.orig_pos = orig_pos
         self.line = line
         self.column = column
+        self._node = node
+        if node is not None:
+            assert self._pos < 0 or self._pos == node._pos
+            self._pos = node._pos
 
     def __str__(self):
         prefix = ''
@@ -94,6 +99,15 @@ class Error:
         return 'Error("%s", %s, %i, %i, %i, %i)' \
                % (self.message, repr(self.code), self.pos, self.orig_pos, self.line, self.column)
 
+    @property
+    def pos(self):
+        if self._pos < 0:
+            assert self._node and self._node.pos >= 0
+            self._pos = self._node.pos   # lazy evaluation of position
+        self._node = None  # forget node to allow GC to free memory
+        return self._pos
+
+
     @property
     def severity(self):
         """Returns a string representation of the error level, e.g. "warning"."""

DHParser/syntaxtree.py

@@ -432,7 +432,7 @@ class Node(collections.abc.Sized):
         return self._pos
 
 
-    def init_pos(self, pos: int, overwrite: bool = True) -> 'Node':
+    def init_pos(self, pos: int) -> 'Node':
         """
         (Re-)initialize position value. Usually, the parser guard
         (`parsers.add_parser_guard()`) takes care of assigning the
@@ -442,16 +442,12 @@ class Node(collections.abc.Sized):
         This function recursively reassigns the position values
         of the child nodes, too.
         """
-        if overwrite or self._pos < 0:
-            self._pos = pos
-            for err in self.errors:
-                err.pos = pos
-        else:
-            assert self._pos == pos, str("%i != %i" % (self._pos, pos))
+        assert self._pos < 0 or self.pos == pos, str("pos mismatch %i != %i" % (self._pos, pos))
+        self._pos = pos
         # recursively adjust pos-values of all children
         offset = self.pos
         for child in self.children:
-            child.init_pos(offset, overwrite)
+            child.init_pos(offset)
             offset = child.pos + len(child)
         return self
 
@@ -534,6 +530,7 @@ class Node(collections.abc.Sized):
         """
 
         left_bracket, right_bracket, density = ('', '', 1) if compact else ('(', '\n)', 0)
+        lbreaks = linebreaks(src) if src else []  # type: List[int]
 
         def opening(node) -> str:
             """Returns the opening string for the representation of `node`."""
@@ -542,7 +539,7 @@ class Node(collections.abc.Sized):
             if hasattr(node, '_xml_attr'):
                 txt.extend(' `(%s "%s")' % (k, v) for k, v in node.attributes.items())
             if src:
-                txt.append(" `(pos %i %i %i)" % (node.pos, *line_col(src, node.add_pos)))
+                txt.append(" `(pos %i %i %i)" % (node.pos, *line_col(lbreaks, node.pos)))
             # if node.error_flag:   # just for debugging error collecting
             #     txt += " HAS ERRORS"
             if showerrors and node.errors:
@@ -686,12 +683,13 @@ class RootNode(Node):
         error_flag (int):  the highest warning or error level of all errors
                 that occurred.
     """
-    def __init__(self):
+    def __init__(self, node: Optional[Node] = None) -> 'RootNode':
         super().__init__(ZOMBIE_PARSER, '')
         self.all_errors = []
         self.err_nodes = []
         self.error_flag = 0
-        self.error_propagation = False
+        if node is not None:
+            self.swallow(node)
 
     # def _propagate_errors(self):
     #     if not self.all_errors or not self.error_propagation:
@@ -738,7 +736,7 @@ class RootNode(Node):
             message(str): A string with the error message.abs
             code(int):    An error code to identify the kind of error
         """
-        error = Error(message, code)
+        error = Error(message, code, node=node)
         self.all_errors.append(error)
         self.error_flag = max(self.error_flag, code)
         node._errors.append(error)
@@ -748,9 +746,9 @@ class RootNode(Node):
     def collect_errors(self, clear_errors=False) -> List[Error]:
         """Returns the list of errors, ordered bv their position.
         """
-        for node in self.err_nodes:  # lazy evaluation of positions
-            for err in node.errors:
-                err.pos = node.pos
+        # for node in self.err_nodes:  # lazy evaluation of positions
+        #     for err in node.errors:  # moved to error.Error.pos
+        #         err.pos = node.pos
         self.all_errors.sort(key=lambda e: e.pos)
         errors = self.all_errors
         if clear_errors:
@@ -817,13 +815,9 @@ def parse_sxpr(sxpr: str) -> Node:
         tagname = sxpr[:end]
         name, class_name = (tagname.split(':') + [''])[:2]
         sxpr = sxpr[end:].strip()
-        pos = 0
         attributes = OrderedDict()
         if sxpr[0] == '(':
             result = tuple(inner_parser(block) for block in next_block(sxpr))
-            for node in result:
-                node._pos = pos
-                pos += len(node)
         else:
             lines = []
             while sxpr and sxpr[0:1] != ')':
@@ -864,7 +858,6 @@ def parse_sxpr(sxpr: str) -> Node:
         node = Node(mock_parsers.setdefault(tagname, MockParser(name, ':' + class_name)), result)
         if attributes:
             node.attributes.update(attributes)
-        node._pos = pos
         return node
 
     return inner_parser(sxpr)

DHParser/testing.py

@@ -39,7 +39,7 @@ import sys
 from DHParser.error import is_error, adjust_error_locations
 from DHParser.log import is_logging, clear_logs, log_ST, log_parsing_history
 from DHParser.parse import UnknownParserError
-from DHParser.syntaxtree import Node, parse_sxpr, flatten_sxpr, ZOMBIE_PARSER
+from DHParser.syntaxtree import Node, RootNode, parse_sxpr, flatten_sxpr, ZOMBIE_PARSER
 from DHParser.toolkit import re, typing
 
 from typing import Tuple
@@ -354,7 +354,8 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
             try:
                 cst = parser(test_code, parser_name)
             except UnknownParserError as upe:
-                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
+                node = Node(ZOMBIE_PARSER, "").init_pos(0)
+                cst = RootNode().swallow(node).add_error(node, str(upe))
             if not is_error(cst.error_flag):
                 errata.append('Fail test "%s" for parser "%s" yields match instead of '
                               'expected failure!' % (test_name, parser_name))

DHParser/transform.py

@@ -888,8 +888,8 @@ assert_has_children = error_on(lambda nd: nd.children, 'Element "%s" has no chil
 def assert_content(context: List[Node], regexp: str):
     node = context[-1]
     if not has_content(context, regexp):
-        node.add_error('Element "%s" violates %s on %s' %
-                       (node.parser.name, str(regexp), node.content))
+        context[0].add_error(node, 'Element "%s" violates %s on %s' %
+                             (node.parser.name, str(regexp), node.content))
 
 
 @transformation_factory
@@ -897,8 +897,8 @@ def require(context: List[Node], child_tags: AbstractSet[str]):
     node = context[-1]
     for child in node.children:
         if child.tag_name not in child_tags:
-            node.add_error('Element "%s" is not allowed inside "%s".' %
-                           (child.parser.name, node.parser.name))
+            context[0].add_error(node, 'Element "%s" is not allowed inside "%s".' %
+                                 (child.parser.name, node.parser.name))
 
 
 @transformation_factory
@@ -906,6 +906,6 @@ def forbid(context: List[Node], child_tags: AbstractSet[str]):
     node = context[-1]
     for child in node.children:
         if child.tag_name in child_tags:
-            node.add_error('Element "%s" cannot be nested inside "%s".' %
-                           (child.parser.name, node.parser.name))
+            context[0].add_error(node, 'Element "%s" cannot be nested inside "%s".' %
+                                 (child.parser.name, node.parser.name))
 

examples/BibTeX/BibTeXCompiler.py

@@ -23,7 +23,7 @@ from DHParser import is_filename, load_if_file, \
     Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
     last_value, counterpart, accumulate, PreprocessorFunc, \
-    Node, TransformationDict, TRUE_CONDITION, \
+    Node, TransformationDict, Whitespace, \
     traverse, remove_children_if, merge_children, is_anonymous, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
     remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \
@@ -106,13 +106,14 @@ class BibTeXGrammar(Grammar):
     CONTENT_STRING = { /[^{}%]+/ | /(?=%)/~ }+
     """
     text = Forward()
-    source_hash__ = "9c5e3c56e3d420e2ea5885612c07de46"
+    source_hash__ = "5ce8838ebbb255548cf3e14cd90bae6d"
     parser_initialization__ = "upon instantiation"
     COMMENT__ = r'(?i)%.*(?:\n|$)'
     WHITESPACE__ = r'\s*'
     WSP__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
     wspL__ = ''
     wspR__ = WSP__
+    whitespace__ = Whitespace(WSP__)
     CONTENT_STRING = OneOrMore(Alternative(RegExp('(?i)[^{}%]+'), RE('(?i)(?=%)')))
     COMMA_TERMINATED_STRING = ZeroOrMore(Alternative(RegExp('(?i)[^,%]+'), RE('(?i)(?=%)')))
     NO_BLANK_STRING = RE('(?i)[^ \\t\\n,%]+')

examples/EBNF/EBNFCompiler.py

@@ -105,6 +105,7 @@ class EBNFGrammar(Grammar):
     WSP__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
     wspL__ = ''
     wspR__ = WSP__
+    whitespace__ = Whitespace(WSP__)
     EOF = NegativeLookahead(RegExp('.'))
     list_ = Series(RE('\\w+'), ZeroOrMore(Series(Token(","), RE('\\w+'))))
     whitespace = RE('~')

examples/Tutorial/Lyrik.ebnf

+@ whitespace = horizontal
+
 gedicht           = bibliographisches { LEERZEILE }+ [serie] §titel text /\s*/ ENDE
 
 bibliographisches = autor §"," [NZ] werk "," [NZ] ort "," [NZ] jahr "."

test/test_compile.py

@@ -23,20 +23,20 @@ limitations under the License.
 from DHParser import parse_sxpr, Compiler
 
 
-class TestCompilerClass:
-    def test_error_propagations(self):
-        tree = parse_sxpr('(A (B 1) (C (D (E 2) (F 3))))')
-        A = tree
-        B = next(tree.select(lambda node: str(node) == "1"))
-        D = next(tree.select(lambda node: node.parser.name == "D"))
-        F = next(tree.select(lambda node: str(node) == "3"))
-        B.add_error("Error in child node")
-        F.add_error("Error in child's child node")
-        Compiler.propagate_error_flags(tree, lazy=True)
-        assert A.error_flag
-        assert not D.error_flag
-        Compiler.propagate_error_flags(tree, lazy=False)
-        assert D.error_flag
+# class TestCompilerClass:
+#     def test_error_propagations(self):
+#         tree = parse_sxpr('(A (B 1) (C (D (E 2) (F 3))))')
+#         A = tree
+#         B = next(tree.select(lambda node: str(node) == "1"))
+#         D = next(tree.select(lambda node: node.parser.name == "D"))
+#         F = next(tree.select(lambda node: str(node) == "3"))
+#         B.add_error("Error in child node")
+#         F.add_error("Error in child's child node")
+#         Compiler.propagate_error_flags(tree, lazy=True)
+#         assert A.error_flag
+#         assert not D.error_flag
+#         Compiler.propagate_error_flags(tree, lazy=False)
+#         assert D.error_flag
 
 
 if __name__ == "__main__":