Skip to content

GitLab

D
DHParser
Commit 2e5f466d authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files
Options

- syntaxtree.py: internal amendments to pretty-printing as S-expression or XML

parent 90f77098
Hide whitespace changes
Inline Side-by-side
Showing with 104 additions and 118 deletions
DHParser/syntaxtree.py
View file @ 2e5f466d
......@@ -31,7 +31,7 @@ except ImportError:
from .typing34 import AbstractSet, Any, ByteString, Callable, cast, Container, Dict, \
Iterator, List, NamedTuple, Sequence, Union, Text, Tuple
from DHParser.toolkit import log_dir, line_col
from DHParser.toolkit import log_dir, line_col, identity
__all__ = ('WHITESPACE_PTYPE',
'TOKEN_PTYPE',
......@@ -267,7 +267,7 @@ class Node:
else str(self.result)
return (' <<< Error on "%s" | %s >>> ' % (s, '; '.join(self._errors))) if self._errors else s
def _tree_repr(self, tab, openF, closeF, dataF=lambda s: s) -> str:
def _tree_repr(self, tab, openF, closeF, dataF=identity, density=0) -> str:
"""
Generates a tree representation of this node and its children
in string from.
......@@ -293,24 +293,24 @@ class Node:
tail = closeF(self)
if not self.result:
return head + tail
return head.rstrip() + tail.lstrip()
head = head + '\n' # place the head, tail and content
tail = '\n' + tail # of the node on different lines
D = None if density & 2 else ''
if self.children:
content = []
for child in self.children:
subtree = child._tree_repr(tab, openF, closeF, dataF).split('\n')
subtree = child._tree_repr(tab, openF, closeF, dataF, density).split('\n')
content.append('\n'.join((tab + s) for s in subtree))
return head + '\n'.join(content) + tail
return head + '\n'.join(content) + tail.lstrip(D)
res = cast(str, self.result) # safe, because if there are no children, result is a string
if head[0] == "<" and res.find('\n') < 0:
# for XML: place tags for leaf-nodes on one line if possible
return head[:-1] + self.result + tail[1:]
if density & 1 and res.find('\n') < 0: # and head[0] == "<":
# except for XML, add a gap between opening statement and content
gap = ' ' if head.rstrip()[-1] != '>' else ''
return head.rstrip() + gap + dataF(self.result) + tail.lstrip()
else:
return head + '\n'.join([tab + dataF(s) for s in res.split('\n')]) + tail
return head + '\n'.join([tab + dataF(s) for s in res.split('\n')]) + tail.lstrip(D)
def as_sxpr(self, src: str=None) -> str:
"""
......@@ -330,14 +330,14 @@ class Node:
if node.errors:
s += " '(err '(%s))" % ' '.join(str(err).replace('"', r'\"')
for err in node.errors)
return s
return s + '\n'
def pretty(s):
return '"%s"' % s if s.find('"') < 0 \
else "'%s'" % s if s.find("'") < 0 \
else '"%s"' % s.replace('"', r'\"')
return self._tree_repr(' ', opening, lambda node: ')', pretty)
return self._tree_repr(' ', opening, lambda node: '\n)', pretty, density=0)
def as_xml(self, src: str=None) -> str:
"""
......@@ -356,14 +356,12 @@ class Node:
s += ' line="%i" col="%i"' % line_col(src, node.pos)
if node.errors:
s += ' err="%s"' % ''.join(str(err).replace('"', r'\"') for err in node.errors)
s += ">"
return s
return s + ">\n"
def closing(node):
s = '</' + node.tag_name + '>'
return s
return '\n</' + node.tag_name + '>'
return self._tree_repr(' ', opening, closing)
return self._tree_repr(' ', opening, closing, density=1)
def add_error(self, error_str) -> 'Node':
self._errors.append(error_str)
......@@ -461,7 +459,7 @@ class Node:
# return nav(path.split('/'))
def mock_syntax_tree(sexpr):
def mock_syntax_tree(sxpr):
"""
Generates a tree of nodes from an S-expression.
......@@ -486,31 +484,47 @@ def mock_syntax_tree(sexpr):
yield s[:i]
s = s[i:].strip()
sexpr = sexpr.strip()
if sexpr[0] != '(': raise ValueError('"(" expected, not ' + sexpr[:10])
# assert sexpr[0] == '(', sexpr
sexpr = sexpr[1:].strip()
m = re.match('[\w:]+', sexpr)
name, class_name = (sexpr[:m.end()].split(':') + [''])[:2]
sexpr = sexpr[m.end():].strip()
if sexpr[0] == '(':
result = tuple(mock_syntax_tree(block) for block in next_block(sexpr))
sxpr = sxpr.strip()
if sxpr[0] != '(': raise ValueError('"(" expected, not ' + sxpr[:10])
# assert sxpr[0] == '(', sxpr
sxpr = sxpr[1:].strip()
m = re.match('[\w:]+', sxpr)
name, class_name = (sxpr[:m.end()].split(':') + [''])[:2]
sxpr = sxpr[m.end():].strip()
if sxpr[0] == '(':
result = tuple(mock_syntax_tree(block) for block in next_block(sxpr))
else:
lines = []
while sexpr and sexpr[0] != ')':
while sxpr and sxpr[0] != ')':
for qm in ['"""', "'''", '"', "'"]:
m = re.match(qm + r'.*?' + qm, sexpr)
m = re.match(qm + r'.*?' + qm, sxpr, re.DOTALL)
if m:
i = len(qm)
lines.append(sexpr[i:m.end() - i])
sexpr = sexpr[m.end():].strip()
lines.append(sxpr[i:m.end() - i])
sxpr = sxpr[m.end():].strip()
break
else:
m = re.match(r'(?:(?!\)).)*', sexpr)
lines.append(sexpr[:m.end()])
sexpr = sexpr[m.end():]
m = re.match(r'(?:(?!\)).)*', sxpr, re.DOTALL)
lines.append(sxpr[:m.end()])
sxpr = sxpr[m.end():]
result = "\n".join(lines)
return Node(MockParser(name, ':' + class_name), result)
def compact_sxpr(s) -> str:
"""Returns S-expression ``s`` as a one liner without unnecessary
whitespace.
Example:
>>> compact_sxpr('(a\\n (b\\n c\\n )\\n)\\n')
'(a (b c))'
"""
return re.sub('\s(?=\))', '', re.sub('\s+', ' ', s)).strip()
TransformationFunc = Union[Callable[[Node], Any], partial]
if __name__ == "__main__":
st = mock_syntax_tree("(alpha (beta (gamma i\nj\nk) (delta y)) (epsilon z))")
print(st.as_sxpr())
print(st.as_xml())
DHParser/testing.py
View file @ 2e5f466d
......@@ -27,8 +27,8 @@ except ImportError:
import re
from DHParser import error_messages
from DHParser.toolkit import compact_sexpr, is_logging
from DHParser.syntaxtree import mock_syntax_tree
from DHParser.toolkit import is_logging
from DHParser.syntaxtree import mock_syntax_tree, compact_sxpr
__all__ = ('unit_from_configfile',
'unit_from_json',
......@@ -171,8 +171,8 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
'\n\tExpr.: %s\n\tExpected: %s\n\tReceived: %s'
% (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
compact_sexpr(compare.as_sxpr()),
compact_sexpr(ast.as_sxpr())))
compact_sxpr(compare.as_sxpr()),
compact_sxpr(ast.as_sxpr())))
tests.setdefault('__err__', {})[test_name] = errata[-1]
if verbose:
print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
......
DHParser/toolkit.py
View file @ 2e5f466d
......@@ -35,11 +35,13 @@ import collections
import contextlib
import hashlib
import os
try:
import regex as re
except ImportError:
import re
import sys
try:
from typing import Any, List, Tuple
except ImportError:
......@@ -51,10 +53,9 @@ __all__ = ('logging',
'logfile_basename',
# 'supress_warnings',
# 'warnings',
'repr_call',
# 'repr_call',
'line_col',
'error_messages',
'compact_sexpr',
'escape_re',
'is_filename',
'load_if_file',
......@@ -79,7 +80,7 @@ def log_dir() -> str:
# the try-except clauses in the following are precautions for multiprocessing
global LOGGING
try:
dirname = LOGGING # raises a name error if LOGGING is not defined
dirname = LOGGING # raises a name error if LOGGING is not defined
if not dirname:
raise NameError # raise a name error if LOGGING evaluates to False
except NameError:
......@@ -104,7 +105,7 @@ def log_dir() -> str:
@contextlib.contextmanager
def logging(dirname: str = "LOGS"):
def logging(dirname="LOGS"):
"""Context manager. Log files within this context will be stored in
directory ``dirname``. Logging is turned off if name is empty.
......@@ -113,7 +114,7 @@ def logging(dirname: str = "LOGS"):
turn logging of
"""
global LOGGING
if dirname == True: dirname = "LOGS" # be fail tolerant here...
if dirname and not isinstance(dirname, str): dirname = "LOGS" # be fail tolerant here...
try:
save = LOGGING
except NameError:
......@@ -132,46 +133,25 @@ def is_logging() -> bool:
return False
# @contextlib.contextmanager
# def supress_warnings(supress: bool = True):
# global SUPRESS_WARNINGS
# try:
# save = SUPRESS_WARNINGS
# except NameError:
# save = False # global default for warning supression is False
# SUPRESS_WARNINGS = supress
# yield
# SUPRESS_WARNINGS = save
#
#
# def warnings() -> bool:
# global SUPRESS_WARNINGS
# try:
# return not SUPRESS_WARNINGS
# except NameError:
# return True
def repr_call(f, parameter_list) -> str:
"""Turns a list of items into a string resembling the parameter
list of a function call by omitting default values at the end:
>>> def f(a, b=1): print(a, b)
>>> repr_call(f, (5,1))
'f(5)'
>>> repr_call(f, (5,2))
'f(5, 2)'
"""
i = 0
defaults = f.__defaults__ if f.__defaults__ is not None else []
for parameter, default in zip(reversed(parameter_list), reversed(defaults)):
if parameter != default:
break
i -= 1
if i < 0:
parameter_list = parameter_list[:i]
name = f.__self__.__class__.__name__ if f.__name__ == '__init__' else f.__name__
return "%s(%s)" % (name, ", ".join(repr(item) for item in parameter_list))
# def repr_call(f, parameter_list) -> str:
# """Turns a list of items into a string resembling the parameter
# list of a function call by omitting default values at the end:
# >>> def f(a, b=1): print(a, b)
# >>> repr_call(f, (5,1))
# 'f(5)'
# >>> repr_call(f, (5,2))
# 'f(5, 2)'
# """
# i = 0
# defaults = f.__defaults__ if f.__defaults__ is not None else []
# for parameter, default in zip(reversed(parameter_list), reversed(defaults)):
# if parameter != default:
# break
# i -= 1
# if i < 0:
# parameter_list = parameter_list[:i]
# name = f.__self__.__class__.__name__ if f.__name__ == '__init__' else f.__name__
# return "%s(%s)" % (name, ", ".join(repr(item) for item in parameter_list))
def line_col(text: str, pos: int) -> Tuple[int, int]:
......@@ -200,29 +180,6 @@ def error_messages(source_text, errors) -> List[str]:
for err in sorted(list(errors))]
def compact_sexpr(s) -> str:
"""Returns S-expression ``s`` as a one liner without unnecessary
whitespace.
Example:
>>> compact_sexpr('(a\\n (b\\n c\\n )\\n)\\n')
'(a (b c))'
"""
return re.sub('\s(?=\))', '', re.sub('\s+', ' ', s)).strip()
# def quick_report(parsing_result) -> str:
# """Returns short report (compact s-expression + errors messages)
# of the parsing results by either a call to a grammar or to a parser
# directly."""
# err = ''
# if isinstance(parsing_result, collections.Collection):
# result = parsing_result[0]
# err = ('\nUnmatched sequence: ' + parsing_result[1]) if parsing_result[1] else ''
# sexpr = compact_sexpr(result.as_sxpr())
# return sexpr + err
def escape_re(s) -> str:
"""Returns `s` with all regular expression special characters escaped.
"""
......@@ -257,7 +214,7 @@ def logfile_basename(filename_or_text, function_or_class_or_instance) -> str:
def load_if_file(text_or_file) -> str:
"""Reads and returns content of a text-file if parameter
`text_or_file` is a file name (i.e. a single line string),
otherwise (i.e. if `text_or_file` is a multiline string)
otherwise (i.e. if `text_or_file` is a multi-line string)
`text_or_file` is returned.
"""
if is_filename(text_or_file):
......@@ -265,7 +222,7 @@ def load_if_file(text_or_file) -> str:
with open(text_or_file, encoding="utf-8") as f:
content = f.read()
return content
except FileNotFoundError as error:
except FileNotFoundError:
if re.fullmatch(r'[\w/:. \\]+', text_or_file):
raise FileNotFoundError('Not a valid file: ' + text_or_file + '!\n(Add "\\n" '
'to distinguish source data from a file name.)')
......@@ -364,7 +321,7 @@ def expand_table(compact_table):
def sane_parser_name(name) -> bool:
"""Checks whether given name is an acceptable parser name. Parser names
must not be preceeded or succeeded by a double underscore '__'!
must not be preceded or succeeded by a double underscore '__'!
"""
return name and name[:2] != '__' and name[-2:] != '__'
......@@ -385,13 +342,17 @@ def compile_python_object(python_src, catch_obj_regex=""):
raise ValueError("No object matching /%s/ defined in source code." %
catch_obj_regex.pattern)
elif len(matches) > 1:
raise ValueError("Ambigous matches for %s : %s" %
raise ValueError("Ambiguous matches for %s : %s" %
(str(catch_obj_regex), str(matches)))
return namespace[matches[0]] if matches else None
else:
return namespace
def identity(anything: Any) -> Any:
return anything
try:
if sys.stdout.encoding.upper() != "UTF-8":
# make sure that `print()` does not raise an error on
......
test/test_syntaxtree.py
View file @ 2e5f466d
......@@ -30,6 +30,18 @@ from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compi
from DHParser.dsl import grammar_provider
class TestMockSyntaxTree:
def test_mock_syntax_tree(self):
tree = mock_syntax_tree('(a (b c))')
tree = mock_syntax_tree('(a i\nj\nk)')
try:
tree = mock_syntax_tree('a b c')
assert False, "mock_syntax_tree() should raise a ValueError " \
"if argument is not a tree!"
except ValueError:
pass
class TestNode:
"""
Tests for class Node
......
test/test_testing.py
View file @ 2e5f466d
......@@ -24,8 +24,7 @@ from functools import partial
sys.path.extend(['../', './'])
from DHParser.toolkit import compact_sexpr
from DHParser.syntaxtree import TOKEN_PTYPE, mock_syntax_tree
from DHParser.syntaxtree import TOKEN_PTYPE, mock_syntax_tree, compact_sxpr
from DHParser.transform import traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten
from DHParser.dsl import grammar_provider
......@@ -119,25 +118,25 @@ class TestSExpr:
Tests for S-expression handling.
"""
def test_compact_sexpr(self):
assert compact_sexpr("(a\n (b\n c\n )\n)\n") == "(a (b c))"
assert compact_sxpr("(a\n (b\n c\n )\n)\n") == "(a (b c))"
def test_mock_syntax_tree(self):
sexpr = '(a (b c) (d e) (f (g h)))'
tree = mock_syntax_tree(sexpr)
assert compact_sexpr(tree.as_sxpr().replace('"', '')) == sexpr
assert compact_sxpr(tree.as_sxpr().replace('"', '')) == sexpr
# test different quotation marks
sexpr = '''(a (b """c""" 'k' "l") (d e) (f (g h)))'''
sexpr_stripped = '(a (b c k l) (d e) (f (g h)))'
tree = mock_syntax_tree(sexpr)
assert compact_sexpr(tree.as_sxpr().replace('"', '')) == sexpr_stripped
assert compact_sxpr(tree.as_sxpr().replace('"', '')) == sexpr_stripped
sexpr_clean = '(a (b "c" "k" "l") (d "e") (f (g "h")))'
tree = mock_syntax_tree(sexpr_clean)
assert compact_sexpr(tree.as_sxpr()) == sexpr_clean
assert compact_sxpr(tree.as_sxpr()) == sexpr_clean
tree = mock_syntax_tree(sexpr_stripped)
assert compact_sexpr(tree.as_sxpr()) == '(a (b "c k l") (d "e") (f (g "h")))'
assert compact_sxpr(tree.as_sxpr()) == '(a (b "c k l") (d "e") (f (g "h")))'
def test_mock_syntax_tree_with_classes(self):
sexpr = '(a:class1 (b:class2 x) (:class3 y) (c z))'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment