- syntaxtree.py: internal amendments to pretty-printing as S-expression or XML

DHParser/syntaxtree.py

@@ -31,7 +31,7 @@ except ImportError:
     from .typing34 import AbstractSet, Any, ByteString, Callable, cast, Container, Dict, \
         Iterator, List, NamedTuple, Sequence, Union, Text, Tuple
 
-from DHParser.toolkit import log_dir, line_col
+from DHParser.toolkit import log_dir, line_col, identity
 
 __all__ = ('WHITESPACE_PTYPE',
            'TOKEN_PTYPE',
@@ -267,7 +267,7 @@ class Node:
             else str(self.result)
         return (' <<< Error on "%s" | %s >>> ' % (s, '; '.join(self._errors))) if self._errors else s
 
-    def _tree_repr(self, tab, openF, closeF, dataF=lambda s: s) -> str:
+    def _tree_repr(self, tab, openF, closeF, dataF=identity, density=0) -> str:
         """
         Generates a tree representation of this node and its children
         in string from.
@@ -293,24 +293,24 @@ class Node:
         tail = closeF(self)
 
         if not self.result:
-            return head + tail
+            return head.rstrip() + tail.lstrip()
 
-        head = head + '\n'  # place the head, tail and content
-        tail = '\n' + tail  # of the node on different lines
+        D = None if density & 2 else ''
 
         if self.children:
             content = []
             for child in self.children:
-                subtree = child._tree_repr(tab, openF, closeF, dataF).split('\n')
+                subtree = child._tree_repr(tab, openF, closeF, dataF, density).split('\n')
                 content.append('\n'.join((tab + s) for s in subtree))
-            return head + '\n'.join(content) + tail
+            return head + '\n'.join(content) + tail.lstrip(D)
 
         res = cast(str, self.result)  # safe, because if there are no children, result is a string
-        if head[0] == "<" and res.find('\n') < 0:
-            # for XML: place tags for leaf-nodes on one line if possible
-            return head[:-1] + self.result + tail[1:]
+        if density & 1 and res.find('\n') < 0:  # and head[0] == "<":
+            # except for XML, add a gap between opening statement and content
+            gap = ' ' if head.rstrip()[-1] != '>' else ''
+            return head.rstrip() + gap + dataF(self.result) + tail.lstrip()
         else:
-            return head + '\n'.join([tab + dataF(s) for s in res.split('\n')]) + tail
+            return head + '\n'.join([tab + dataF(s) for s in res.split('\n')]) + tail.lstrip(D)
 
     def as_sxpr(self, src: str=None) -> str:
         """
@@ -330,14 +330,14 @@ class Node:
             if node.errors:
                 s += " '(err '(%s))" % ' '.join(str(err).replace('"', r'\"')
                                                 for err in node.errors)
-            return s
+            return s + '\n'
 
         def pretty(s):
             return '"%s"' % s if s.find('"') < 0 \
                 else "'%s'" % s if s.find("'") < 0 \
                 else '"%s"' % s.replace('"', r'\"')
 
-        return self._tree_repr('    ', opening, lambda node: ')', pretty)
+        return self._tree_repr('    ', opening, lambda node: '\n)', pretty, density=0)
 
     def as_xml(self, src: str=None) -> str:
         """
@@ -356,14 +356,12 @@ class Node:
                 s += ' line="%i" col="%i"' % line_col(src, node.pos)
             if node.errors:
                 s += ' err="%s"' % ''.join(str(err).replace('"', r'\"') for err in node.errors)
-            s += ">"
-            return s
+            return s + ">\n"
 
         def closing(node):
-            s = '</' + node.tag_name + '>'
-            return s
+            return '\n</' + node.tag_name + '>'
 
-        return self._tree_repr('    ', opening, closing)
+        return self._tree_repr('    ', opening, closing, density=1)
 
     def add_error(self, error_str) -> 'Node':
         self._errors.append(error_str)
@@ -461,7 +459,7 @@ class Node:
     #     return nav(path.split('/'))
 
 
-def mock_syntax_tree(sexpr):
+def mock_syntax_tree(sxpr):
     """
     Generates a tree of nodes from an S-expression.
 
@@ -486,31 +484,47 @@ def mock_syntax_tree(sexpr):
             yield s[:i]
             s = s[i:].strip()
 
-    sexpr = sexpr.strip()
-    if sexpr[0] != '(': raise ValueError('"(" expected, not ' + sexpr[:10])
-    # assert sexpr[0] == '(', sexpr
-    sexpr = sexpr[1:].strip()
-    m = re.match('[\w:]+', sexpr)
-    name, class_name = (sexpr[:m.end()].split(':') + [''])[:2]
-    sexpr = sexpr[m.end():].strip()
-    if sexpr[0] == '(':
-        result = tuple(mock_syntax_tree(block) for block in next_block(sexpr))
+    sxpr = sxpr.strip()
+    if sxpr[0] != '(': raise ValueError('"(" expected, not ' + sxpr[:10])
+    # assert sxpr[0] == '(', sxpr
+    sxpr = sxpr[1:].strip()
+    m = re.match('[\w:]+', sxpr)
+    name, class_name = (sxpr[:m.end()].split(':') + [''])[:2]
+    sxpr = sxpr[m.end():].strip()
+    if sxpr[0] == '(':
+        result = tuple(mock_syntax_tree(block) for block in next_block(sxpr))
     else:
         lines = []
-        while sexpr and sexpr[0] != ')':
+        while sxpr and sxpr[0] != ')':
             for qm in ['"""', "'''", '"', "'"]:
-                m = re.match(qm + r'.*?' + qm, sexpr)
+                m = re.match(qm + r'.*?' + qm, sxpr, re.DOTALL)
                 if m:
                     i = len(qm)
-                    lines.append(sexpr[i:m.end() - i])
-                    sexpr = sexpr[m.end():].strip()
+                    lines.append(sxpr[i:m.end() - i])
+                    sxpr = sxpr[m.end():].strip()
                     break
             else:
-                m = re.match(r'(?:(?!\)).)*', sexpr)
-                lines.append(sexpr[:m.end()])
-                sexpr = sexpr[m.end():]
+                m = re.match(r'(?:(?!\)).)*', sxpr, re.DOTALL)
+                lines.append(sxpr[:m.end()])
+                sxpr = sxpr[m.end():]
         result = "\n".join(lines)
     return Node(MockParser(name, ':' + class_name), result)
 
 
+def compact_sxpr(s) -> str:
+    """Returns S-expression ``s`` as a one liner without unnecessary
+    whitespace.
+
+    Example:
+    >>> compact_sxpr('(a\\n    (b\\n        c\\n    )\\n)\\n')
+    '(a (b c))'
+    """
+    return re.sub('\s(?=\))', '', re.sub('\s+', ' ', s)).strip()
+
+
 TransformationFunc = Union[Callable[[Node], Any], partial]
+
+if __name__ == "__main__":
+    st = mock_syntax_tree("(alpha (beta (gamma i\nj\nk) (delta y)) (epsilon z))")
+    print(st.as_sxpr())
+    print(st.as_xml())

DHParser/testing.py

@@ -27,8 +27,8 @@ except ImportError:
     import re
 
 from DHParser import error_messages
-from DHParser.toolkit import compact_sexpr, is_logging
-from DHParser.syntaxtree import mock_syntax_tree
+from DHParser.toolkit import is_logging
+from DHParser.syntaxtree import mock_syntax_tree, compact_sxpr
 
 __all__ = ('unit_from_configfile',
            'unit_from_json',
@@ -171,8 +171,8 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
                     errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                   '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                   % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
-                                     compact_sexpr(compare.as_sxpr()),
-                                     compact_sexpr(ast.as_sxpr())))
+                                     compact_sxpr(compare.as_sxpr()),
+                                     compact_sxpr(ast.as_sxpr())))
                     tests.setdefault('__err__', {})[test_name] = errata[-1]
             if verbose:
                 print(infostr + ("OK" if len(errata) == errflag else "FAIL"))

DHParser/toolkit.py

@@ -35,11 +35,13 @@ import collections
 import contextlib
 import hashlib
 import os
+
 try:
     import regex as re
 except ImportError:
     import re
 import sys
+
 try:
     from typing import Any, List, Tuple
 except ImportError:
@@ -51,10 +53,9 @@ __all__ = ('logging',
            'logfile_basename',
            # 'supress_warnings',
            # 'warnings',
-           'repr_call',
+           # 'repr_call',
            'line_col',
            'error_messages',
-           'compact_sexpr',
            'escape_re',
            'is_filename',
            'load_if_file',
@@ -79,7 +80,7 @@ def log_dir() -> str:
     # the try-except clauses in the following are precautions for multiprocessing
     global LOGGING
     try:
-        dirname = LOGGING    # raises a name error if LOGGING is not defined
+        dirname = LOGGING  # raises a name error if LOGGING is not defined
         if not dirname:
             raise NameError  # raise a name error if LOGGING evaluates to False
     except NameError:
@@ -104,7 +105,7 @@ def log_dir() -> str:
 
 
 @contextlib.contextmanager
-def logging(dirname: str = "LOGS"):
+def logging(dirname="LOGS"):
     """Context manager. Log files within this context will be stored in
     directory ``dirname``. Logging is turned off if name is empty.
     
@@ -113,7 +114,7 @@ def logging(dirname: str = "LOGS"):
             turn logging of
     """
     global LOGGING
-    if dirname == True:  dirname = "LOGS"  # be fail tolerant here...
+    if dirname and not isinstance(dirname, str):  dirname = "LOGS"  # be fail tolerant here...
     try:
         save = LOGGING
     except NameError:
@@ -132,46 +133,25 @@ def is_logging() -> bool:
         return False
 
 
-# @contextlib.contextmanager
-# def supress_warnings(supress: bool = True):
-#     global SUPRESS_WARNINGS
-#     try:
-#         save = SUPRESS_WARNINGS
-#     except NameError:
-#         save = False  # global default for warning supression is False
-#     SUPRESS_WARNINGS = supress
-#     yield
-#     SUPRESS_WARNINGS = save
-#
-#
-# def warnings() -> bool:
-#     global SUPRESS_WARNINGS
-#     try:
-#         return not SUPRESS_WARNINGS
-#     except NameError:
-#         return True
-
-
-def repr_call(f, parameter_list) -> str:
-    """Turns a list of items into a string resembling the parameter
-    list of a function call by omitting default values at the end:
-    >>> def f(a, b=1):    print(a, b)
-    >>> repr_call(f, (5,1))
-    'f(5)'
-    >>> repr_call(f, (5,2))
-    'f(5, 2)'
-    """
-    i = 0
-    defaults = f.__defaults__ if f.__defaults__ is not None else []
-    for parameter, default in zip(reversed(parameter_list), reversed(defaults)):
-        if parameter != default:
-            break
-        i -= 1
-    if i < 0:
-        parameter_list = parameter_list[:i]
-    name = f.__self__.__class__.__name__ if f.__name__ == '__init__' else f.__name__
-    return "%s(%s)" % (name, ", ".join(repr(item) for item in parameter_list))
-
+# def repr_call(f, parameter_list) -> str:
+#     """Turns a list of items into a string resembling the parameter
+#     list of a function call by omitting default values at the end:
+#     >>> def f(a, b=1):    print(a, b)
+#     >>> repr_call(f, (5,1))
+#     'f(5)'
+#     >>> repr_call(f, (5,2))
+#     'f(5, 2)'
+#     """
+#     i = 0
+#     defaults = f.__defaults__ if f.__defaults__ is not None else []
+#     for parameter, default in zip(reversed(parameter_list), reversed(defaults)):
+#         if parameter != default:
+#             break
+#         i -= 1
+#     if i < 0:
+#         parameter_list = parameter_list[:i]
+#     name = f.__self__.__class__.__name__ if f.__name__ == '__init__' else f.__name__
+#     return "%s(%s)" % (name, ", ".join(repr(item) for item in parameter_list))
 
 
 def line_col(text: str, pos: int) -> Tuple[int, int]:
@@ -200,29 +180,6 @@ def error_messages(source_text, errors) -> List[str]:
             for err in sorted(list(errors))]
 
 
-def compact_sexpr(s) -> str:
-    """Returns S-expression ``s`` as a one liner without unnecessary
-    whitespace.
-
-    Example:
-    >>> compact_sexpr('(a\\n    (b\\n        c\\n    )\\n)\\n')
-    '(a (b c))'
-    """
-    return re.sub('\s(?=\))', '', re.sub('\s+', ' ', s)).strip()
-
-
-# def quick_report(parsing_result) -> str:
-#     """Returns short report (compact s-expression + errors messages)
-#     of the parsing results by either a call to a grammar or to a parser
-#     directly."""
-#     err = ''
-#     if isinstance(parsing_result, collections.Collection):
-#         result = parsing_result[0]
-#         err = ('\nUnmatched sequence: ' + parsing_result[1]) if parsing_result[1] else ''
-#     sexpr = compact_sexpr(result.as_sxpr())
-#     return sexpr + err
-
-
 def escape_re(s) -> str:
     """Returns `s` with all regular expression special characters escaped.
     """
@@ -257,7 +214,7 @@ def logfile_basename(filename_or_text, function_or_class_or_instance) -> str:
 def load_if_file(text_or_file) -> str:
     """Reads and returns content of a text-file if parameter
     `text_or_file` is a file name (i.e. a single line string),
-    otherwise (i.e. if `text_or_file` is a multiline string)
+    otherwise (i.e. if `text_or_file` is a multi-line string)
     `text_or_file` is returned.
     """
     if is_filename(text_or_file):
@@ -265,7 +222,7 @@ def load_if_file(text_or_file) -> str:
             with open(text_or_file, encoding="utf-8") as f:
                 content = f.read()
             return content
-        except FileNotFoundError as error:
+        except FileNotFoundError:
             if re.fullmatch(r'[\w/:. \\]+', text_or_file):
                 raise FileNotFoundError('Not a valid file: ' + text_or_file + '!\n(Add "\\n" '
                                         'to distinguish source data from a file name.)')
@@ -364,7 +321,7 @@ def expand_table(compact_table):
 
 def sane_parser_name(name) -> bool:
     """Checks whether given name is an acceptable parser name. Parser names
-    must not be preceeded or succeeded by a double underscore '__'!
+    must not be preceded or succeeded by a double underscore '__'!
     """
     return name and name[:2] != '__' and name[-2:] != '__'
 
@@ -385,13 +342,17 @@ def compile_python_object(python_src, catch_obj_regex=""):
             raise ValueError("No object matching /%s/ defined in source code." %
                              catch_obj_regex.pattern)
         elif len(matches) > 1:
-            raise ValueError("Ambigous matches for %s : %s" %
+            raise ValueError("Ambiguous matches for %s : %s" %
                              (str(catch_obj_regex), str(matches)))
         return namespace[matches[0]] if matches else None
     else:
         return namespace
 
 
+def identity(anything: Any) -> Any:
+    return anything
+
+
 try:
     if sys.stdout.encoding.upper() != "UTF-8":
         # make sure that `print()` does not raise an error on 

test/test_syntaxtree.py

@@ -30,6 +30,18 @@ from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compi
 from DHParser.dsl import grammar_provider
 
 
+class TestMockSyntaxTree:
+    def test_mock_syntax_tree(self):
+        tree = mock_syntax_tree('(a (b c))')
+        tree = mock_syntax_tree('(a i\nj\nk)')
+        try:
+            tree = mock_syntax_tree('a b c')
+            assert False, "mock_syntax_tree() should raise a ValueError " \
+                          "if argument is not a tree!"
+        except ValueError:
+            pass
+
+
 class TestNode:
     """
     Tests for class Node 

test/test_testing.py

@@ -24,8 +24,7 @@ from functools import partial
 
 sys.path.extend(['../', './'])
 
-from DHParser.toolkit import compact_sexpr
-from DHParser.syntaxtree import TOKEN_PTYPE, mock_syntax_tree
+from DHParser.syntaxtree import TOKEN_PTYPE, mock_syntax_tree, compact_sxpr
 from DHParser.transform import traverse, remove_expendables, \
     replace_by_single_child, reduce_single_child, flatten
 from DHParser.dsl import grammar_provider
@@ -119,25 +118,25 @@ class TestSExpr:
     Tests for S-expression handling.
     """
     def test_compact_sexpr(self):
-        assert compact_sexpr("(a\n    (b\n        c\n    )\n)\n") == "(a (b c))"
+        assert compact_sxpr("(a\n    (b\n        c\n    )\n)\n") == "(a (b c))"
 
     def test_mock_syntax_tree(self):
         sexpr = '(a (b c) (d e) (f (g h)))'
         tree = mock_syntax_tree(sexpr)
-        assert compact_sexpr(tree.as_sxpr().replace('"', '')) == sexpr
+        assert compact_sxpr(tree.as_sxpr().replace('"', '')) == sexpr
 
         # test different quotation marks
         sexpr = '''(a (b """c""" 'k' "l") (d e) (f (g h)))'''
         sexpr_stripped = '(a (b c k l) (d e) (f (g h)))'
         tree = mock_syntax_tree(sexpr)
-        assert compact_sexpr(tree.as_sxpr().replace('"', '')) == sexpr_stripped
+        assert compact_sxpr(tree.as_sxpr().replace('"', '')) == sexpr_stripped
 
         sexpr_clean = '(a (b "c" "k" "l") (d "e") (f (g "h")))'
         tree = mock_syntax_tree(sexpr_clean)
-        assert compact_sexpr(tree.as_sxpr()) == sexpr_clean
+        assert compact_sxpr(tree.as_sxpr()) == sexpr_clean
 
         tree = mock_syntax_tree(sexpr_stripped)
-        assert compact_sexpr(tree.as_sxpr()) == '(a (b "c k l") (d "e") (f (g "h")))'
+        assert compact_sxpr(tree.as_sxpr()) == '(a (b "c k l") (d "e") (f (g "h")))'
 
     def test_mock_syntax_tree_with_classes(self):
         sexpr = '(a:class1 (b:class2 x) (:class3 y) (c z))'