Merge remote-tracking branch 'origin/development' into development

DHParser/dsl.py

@@ -470,7 +470,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
                 f = None
 
         if RX_WHITESPACE.fullmatch(intro):
-            intro = '#!/usr/bin/python'
+            intro = '#!/usr/bin/python3'
         if RX_WHITESPACE.fullmatch(outro):
             outro = DHPARSER_MAIN.format(NAME=compiler_name)
         if RX_WHITESPACE.fullmatch(imports):

DHParser/ebnf.py

@@ -936,7 +936,7 @@ class EBNFCompiler(Compiler):
         rule = node.children[0].content
         if rule in self.rules:
             first = self.rules[rule][0]
-            if not first.errors:
+            if not id(first) in self.tree.error_nodes:
                 self.tree.new_error(first, 'First definition of rule "%s" '
                                     'followed by illegal redefinitions.' % rule)
             self.tree.new_error(node, 'A rule "%s" has already been defined earlier.' % rule)

DHParser/error.py

@@ -73,7 +73,7 @@ class Error:
 
     REDECLARED_TOKEN_WARNING                 = ErrorCode(120)
     UNUSED_ERROR_HANDLING_WARNING            = ErrorCode(130)
-    LEFT_RECURSION_WARING                    = ErrorCode(140)
+    LEFT_RECURSION_WARNING                   = ErrorCode(140)
 
     UNDEFINED_SYMBOL_IN_TRANSTABLE_WARNING   = ErrorCode(610)
 

DHParser/log.py

@@ -52,12 +52,12 @@ import collections
 import contextlib
 import html
 import os
+from typing import List, Tuple, Union, Optional
 
 from DHParser.error import Error
 from DHParser.stringview import StringView
 from DHParser.syntaxtree import Node
-from DHParser.toolkit import is_filename, escape_control_characters, GLOBALS, typing
-from typing import List, Tuple, Union
+from DHParser.toolkit import is_filename, escape_control_characters, GLOBALS
 
 __all__ = ('log_dir',
            'logging',
@@ -222,13 +222,13 @@ class HistoryRecord:
     HTML_LEAD_OUT = '\n</body>\n</html>\n'
 
     def __init__(self, call_stack: List[str],
-                 node: Node,
+                 node: Optional[Node],
                  text: StringView,
                  line_col: Tuple[int, int],
                  errors: List[Error] = []) -> None:
         # copy call stack, dropping uninformative Forward-Parsers
         self.call_stack = [tn for tn in call_stack if tn != ":Forward"]  # type: List[str]
-        self.node = node                # type: Node
+        self.node = node                # type: Optional[Node]
         self.text = text                # type: StringView
         self.line_col = line_col        # type: Tuple[int, int]
         self.errors = errors            # type: List[Error]

DHParser/parse.py

@@ -340,10 +340,11 @@ class Parser:
                 if location in grammar.recursion_locations__:
                     if location in self.visited:
                         node, rest = self.visited[location]
-                        if location != grammar.last_recursion_location__:
-                            grammar.tree__.add_error(node, Error("Left recursion encountered. "
-                                                                 "Refactor grammar to avoid slow parsing.",
-                                                                 node.pos, Error.LEFT_RECURSION_WARING))
+                        if node and location != grammar.last_recursion_location__:
+                            grammar.tree__.add_error(
+                                node, Error("Left recursion encountered. "
+                                            "Refactor grammar to avoid slow parsing.",
+                                            node.pos, Error.LEFT_RECURSION_WARNING))
                             grammar.last_recursion_location__ = location
                     # don't overwrite any positive match (i.e. node not None) in the cache
                     # and don't add empty entries for parsers returning from left recursive calls!
@@ -515,7 +516,7 @@ class UnknownParserError(KeyError):
     is referred to that does not exist."""
 
 
-GrammarErrorType = List[Tuple[str, Parser, Error]]      # TODO: replace with a named tuple?
+GrammarErrorType = Tuple[str, Parser, Error]      # TODO: replace with a named tuple?
 
 
 class GrammarError(Exception):
@@ -915,9 +916,9 @@ class Grammar:
             """
             last_record = self.history__[-2] if len(self.history__) > 1 else None  # type: Optional[HistoryRecord]
             return last_record and parser != self.root_parser__ \
-                    and any(self.history__[i].status == HistoryRecord.MATCH \
-                            and self.history__[i].node.pos \
-                            + len(self.history__[i].node) >= len(self.document__) \
+                    and any(self.history__[i].status == HistoryRecord.MATCH
+                            and ((self.history__[i].node.pos + len(self.history__[i].node))
+                                 if self.history__[i].node else 0) >= len(self.document__)
                             and any(tn in self and isinstance(self[tn], Lookahead)
                                     or tn[0] == ':' and issubclass(eval(tn[1:]), Lookahead)
                                     for tn in self.history__[i].call_stack)
@@ -1376,7 +1377,7 @@ class MetaParser(Parser):
                     return Node(self.tag_name, node)
                 return node
             elif self.pname:
-                return Node(self.tag_name, ())  # type: Node
+                return Node(self.tag_name, ())
             return EMPTY_NODE  # avoid creation of a node object for anonymous empty nodes
         return Node(self.tag_name, node or ())  # unoptimized code
 
@@ -1392,7 +1393,7 @@ class MetaParser(Parser):
         N = len(results)
         if N > 1:
             if self.grammar.flatten_tree__:
-                nr = []
+                nr = []  # type: List[Node]
                 for child in results:
                     if child.children and child.tag_name[0] == ':':  # faster than c.is_anonymous():
                         nr.extend(child.children)

DHParser/stringview.py

@@ -42,7 +42,7 @@ try:
     cython_optimized = cython.compiled  # type: bool
 except ImportError:
     # import DHParser.Shadow as cython
-    cython_optimized = False            # type: bool
+    cython_optimized = False
     import DHParser.shadow_cython as cython
 
 

DHParser/syntaxtree.py

@@ -25,11 +25,11 @@ parser classes are defined in the ``parse`` module.
 
 from collections import OrderedDict
 import copy
+from typing import Callable, cast, Iterator, List, AbstractSet, Set, Union, Tuple, Optional, Dict
 
 from DHParser.error import Error, ErrorCode, linebreaks, line_col
 from DHParser.stringview import StringView
 from DHParser.toolkit import get_config_value, re
-from typing import Callable, cast, Iterator, List, AbstractSet, Set, Union, Tuple, Optional
 
 
 __all__ = ('WHITESPACE_PTYPE',
@@ -544,9 +544,9 @@ class Node:  # (collections.abc.Sized): Base class omitted for cython-compatibil
 
         left_bracket, right_bracket, density = ('', '', 1) if compact else ('(', '\n)', 0)
         lbreaks = linebreaks(src) if src else []  # type: List[int]
-        root = cast(RootNode, self) if isinstance(self, RootNode) else None  # type: Optional[Node]
+        root = cast(RootNode, self) if isinstance(self, RootNode) else None  # type: Optional[RootNode]
 
-        def opening(node) -> str:
+        def opening(node: Node) -> str:
             """Returns the opening string for the representation of `node`."""
             txt = [left_bracket, node.tag_name]
             # s += " '(pos %i)" % node.add_pos
@@ -560,11 +560,11 @@ class Node:  # (collections.abc.Sized): Base class omitted for cython-compatibil
                 txt.append(" `(err `%s)" % ' '.join(str(err) for err in root.get_errors(node)))
             return "".join(txt) + '\n'
 
-        def closing(node) -> str:
+        def closing(node: Node) -> str:
             """Returns the closing string for the representation of `node`."""
             return right_bracket
 
-        def pretty(strg):
+        def pretty(strg: str) -> str:
             """Encloses `strg` with the right kind of quotation marks."""
             return '"%s"' % strg if strg.find('"') < 0 \
                 else "'%s'" % strg if strg.find("'") < 0 \
@@ -596,9 +596,9 @@ class Node:  # (collections.abc.Sized): Base class omitted for cython-compatibil
             empty_tags:  A set of tags which shall be rendered as empty elements, e.g.
                 "<empty/>" instead of "<empty><empty>".
         """
-        root = cast(RootNode, self) if isinstance(self, RootNode) else None  # type: Optional[Node]
+        root = cast(RootNode, self) if isinstance(self, RootNode) else None  # type: Optional[RootNode]
 
-        def opening(node) -> str:
+        def opening(node: Node) -> str:
             """Returns the opening string for the representation of `node`."""
             if node.tag_name in omit_tags:
                 return ''
@@ -620,7 +620,7 @@ class Node:  # (collections.abc.Sized): Base class omitted for cython-compatibil
                 ending = ">\n"
             return "".join(txt + [ending])
 
-        def closing(node):
+        def closing(node: Node):
             """Returns the closing string for the representation of `node`."""
             if node.tag_name in omit_tags or node.tag_name in empty_tags:
                 return ''
@@ -632,8 +632,7 @@ class Node:  # (collections.abc.Sized): Base class omitted for cython-compatibil
             content = content.replace('<', '&lt;').replace('>', '&gt;')
             return content
 
-
-        def inlining(node):
+        def inlining(node: Node):
             """Returns True, if `node`'s tag name is contained in `inline_tags`,
             thereby signalling that the children of this node shall not be
             printed on several lines to avoid unwanted gaps in the output.
@@ -808,7 +807,7 @@ def tree_sanity_check(tree: Node) -> bool:
     :param tree: the root of the tree to be checked
     :return: True, if the tree is `sane`, False otherwise.
     """
-    node_set = set()
+    node_set = set()  # type: Set[Node]
     for node in tree.select(lambda nd: True, include_root=True):
         if node in node_set or isinstance(Node, FrozenNode):
             return False
@@ -832,6 +831,12 @@ class RootNode(Node):
                 processing (i.e. parsing, AST-transformation, compiling)
                 of this tree.
 
+        error_nodes (dict): A mapping of node-ids to a list of errors that
+                occurred on the node with the respective id.
+
+        error_positions (dict): A mapping of locations to a set of ids of
+                nodes that contain an error at that particular location
+
         error_flag (int):  the highest warning or error level of all errors
                 that occurred.
 
@@ -844,7 +849,7 @@ class RootNode(Node):
 
     def __init__(self, node: Optional[Node] = None):
         super().__init__('__not_yet_ready__', '')
-        self.errors = []           # type: List[Error]
+        self.errors = []               # type: List[Error]
         self.error_nodes = dict()      # type: Dict[int, List[Error]]  # id(node) -> error list
         self.error_positions = dict()  # type: Dict[int, Set[int]]  # pos -> set of id(node)
         self.error_flag = 0
@@ -852,8 +857,8 @@ class RootNode(Node):
             self.swallow(node)
         # customization for XML-Representation
         self.inline_tags = set()  # type: Set[str]
-        self.omit_tags = set()  # type: Set[str]
-        self.empty_tags = set()  # type: Set[str]
+        self.omit_tags = set()    # type: Set[str]
+        self.empty_tags = set()   # type: Set[str]
 
     def __deepcopy__(self, memodict={}):
         duplicate = self.__class__(None)

DHParser/toolkit.py

@@ -42,7 +42,7 @@ try:
     if cython_optimized:
         import DHParser.shadow_cython as cython
 except ImportError:
-    cython_optimized = False            # type: bool
+    cython_optimized = False
     import DHParser.shadow_cython as cython
 
 from DHParser.configuration import CONFIG_PRESET

DHParser/transform.py

@@ -591,7 +591,7 @@ def flatten_anonymous_nodes(context: List[Node]):
     """
     node = context[-1]
     if node.children:
-        new_result = []
+        new_result = []  # type: List[Node]
         for child in node.children:
             if child.is_anonymous():
                 if child.children:
@@ -819,22 +819,35 @@ def merge_whitespace(context):
 
 
 @transformation_factory(collections.abc.Callable)
-def move_adjacent(context, condition: Callable = is_insignificant_whitespace):
+def move_adjacent(context: List[Node], condition: Callable = is_insignificant_whitespace):
     """
     Moves adjacent nodes that fulfill the given condition to the parent node.
     """
+    def join_results(a: Node, b: Node, c: Node) -> bool:
+        """Joins the results of node `a` and `b` and write them to the result
+        of `c` type-safely, if possible. Return True, if join was possible
+        and done, False otherwise."""
+        if a.children and b.children:
+            c.result = cast(Tuple[Node, ...], a.result) + cast(Tuple[Node, ...], b.result)
+            return True
+        elif not a.children and not b.children:
+            c.result = cast(str, a.result) + cast(str, b.result)
+            return True
+        return False
+
+
     node = context[-1]
     if len(context) <= 1 or not node.children:
         return
     parent = context[-2]
     children = node.children
     if condition([children[0]]):
-        before = (children[0],)
+        before = (children[0],)   # type: Tuple[Node, ...]
         children = children[1:]
     else:
         before = ()
     if children and condition([children[-1]]):
-        after = (children[-1],)
+        after = (children[-1],)   # type: Tuple[Node, ...]
         children = children[:-1]
     else:
         after = tuple()
@@ -849,11 +862,15 @@ def move_adjacent(context, condition: Callable = is_insignificant_whitespace):
         prevN = parent.children[i - 1] if i > 0 else None
         nextN = parent.children[i + 1] if i < len(parent.children) - 1 else None
         if before and prevN and condition([prevN]):
-            prevN.result = prevN.result + before[0].result
-            before = ()
+            # prevN.result = prevN.result + before[0].result
+            # before = ()
+            if join_results(prevN, before[0], prevN):
+                before = ()
         if after and nextN and condition([nextN]):
-            nextN.result = after[0].result + nextN.result
-            after = ()
+            # nextN.result = after[0].result + nextN.result
+            # after = ()
+            if join_results(after[0], nextN, nextN):
+                after = ()
 
         parent.result = parent.children[:i] + before + (node,) + after + parent.children[i+1:]
 

setup.py

@@ -52,7 +52,7 @@ setup(
     ],
     entry_points={
         'console_scripts': [
-            'dhparser = dhparser'
+            'dhparser = dhparser.py'
         ]
     }
 )