Commit 0b24bd82 authored by di68kap's avatar di68kap
Browse files

syntaxtree.py: parse_sxpr(): now raises an error if not the complete string...

syntaxtree.py: parse_sxpr(): now raises an error if not the complete string has been parsed; flatten_sxpr(): does not shorten multicharacter blanks in quoted strings, e.g. "   ", any more.
parent bd926ede
......@@ -122,7 +122,7 @@ def flatten_sxpr(sxpr: str, threshold: int = -1) -> str:
assert RX_IS_SXPR.match(sxpr)
if threshold == 0:
return sxpr
flat = re.sub(r'\s(?=\))', '', re.sub(r'\s+', ' ', sxpr)).strip()
flat = re.sub(r'\s(?=\))', '', re.sub(r'(?<!")\s+', ' ', sxpr).replace('\n', '')).strip()
if len(flat) > threshold >= 0:
return sxpr.strip()
return flat
......@@ -1868,6 +1868,8 @@ def parse_sxpr(sxpr: Union[str, StringView]) -> Node:
>>> tree['C'].pos
1
"""
remaining = sxpr # type: StringView
@cython.locals(level=cython.int, k=cython.int)
def next_block(s: StringView) -> Iterator[StringView]:
"""Generator that yields all characters until the next closing bracket
......@@ -1898,6 +1900,9 @@ def parse_sxpr(sxpr: Union[str, StringView]) -> Node:
errmsg = ('Malformed S-expression. Unprocessed part: "%s"' % s) if s \
else 'Malformed S-expression. Closing bracket(s) ")" missing.'
raise AssertionError(errmsg)
nonlocal remaining
remaining = s
@cython.locals(pos=cython.int, i=cython.int, k=cython.int, end=cython.int)
def inner_parser(sxpr: StringView) -> Node:
......@@ -1961,6 +1966,8 @@ def parse_sxpr(sxpr: Union[str, StringView]) -> Node:
lines.append(str(sxpr[:end]))
sxpr = sxpr[end:]
result = "\n".join(lines) # # type: Union[Tuple[Node, ...], str]
nonlocal remaining
remaining = sxpr
node = Node(str(name or ':' + class_name), result)
node._pos = pos
if attributes:
......@@ -1968,7 +1975,10 @@ def parse_sxpr(sxpr: Union[str, StringView]) -> Node:
return node
xpr = StringView(sxpr).strip() if isinstance(sxpr, str) else sxpr.strip() # type: StringView
return inner_parser(xpr)
tree = inner_parser(xpr)
if remaining != ')':
raise ValueError('Malformed S-expression. Superfluous characters: ' + remaining[1:])
return tree
RX_WHITESPACE_TAIL = re.compile(r'\s*$')
......
......@@ -30,6 +30,7 @@ import asyncio
import collections
import concurrent.futures
import copy
# import difflib
import fnmatch
import inspect
import json
......@@ -457,7 +458,11 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report='REPORT'
write(infostr + ("OK" if len(errata) == errflag else "FAIL"))
if "cst" in tests and len(errata) == errflag:
compare = parse_tree(get(tests, "cst", test_name))
try:
compare = parse_tree(get(tests, "cst", test_name))
except ValueError as e:
raise SyntaxError('CST-TEST "%s" of parser "%s" failed with:\n%s'
% (test_name, parser_name, str(e)))
if compare:
if not compare.equals(cst):
errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
......@@ -467,15 +472,22 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report='REPORT'
write(infostr + ("OK" if len(errata) == errflag else "FAIL"))
if "ast" in tests and len(errata) == errflag:
compare = parse_tree(get(tests, "ast", test_name))
try:
compare = parse_tree(get(tests, "ast", test_name))
except ValueError as e:
raise SyntaxError('AST-TEST "%s" of parser "%s" failed with:\n%s'
% (test_name, parser_name, str(e)))
if compare:
traverse(compare, {'*': remove_children({'__TESTING_ARTIFACT__'})})
if not compare.equals(ast): # no worry: ast is defined if "ast" in tests
ast_str = flatten_sxpr(ast.as_sxpr())
compare_str = flatten_sxpr(compare.as_sxpr())
# differ = difflib.Differ()
# difference = ''.join(differ.compare([compare_str + '\n'], [ast_str + '\n']))
errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
'\n\tExpr.: %s\n\tExpected: %s\n\tReceived: %s'
% (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
flatten_sxpr(compare.as_sxpr()),
flatten_sxpr(ast.as_sxpr())))
compare_str, ast_str))
if verbose:
infostr = ' ast-test "' + test_name + '" ... '
write(infostr + ("OK" if len(errata) == errflag else "FAIL"))
......
......@@ -689,7 +689,7 @@ class TestErrorRecovery:
assert 'Skipping' in str(st.errors_sorted[1])
def test_series_skip2(self):
grammar = """
grammar = r"""
@whitespace = vertical
@literalws = right
@document_skip = /\s+|(?=$)/
......
......@@ -52,6 +52,11 @@ class TestParseSxpression:
"if argument is not a tree!"
except ValueError:
pass
try:
tree = parse_sxpr('(a (b c)))')
assert False, "parse_sxpr() should raise a ValueError for too many matching brackets."
except ValueError:
pass
def test_parse_s_expression_w_attributes(self):
s = '(A `(attr "1") (B "X"))'
......@@ -70,6 +75,12 @@ class TestParseSxpression:
tree = parse_sxpr(r'(LINEFEED "\\")')
assert tree
def test_flatten_sxpr(self):
tree = parse_sxpr('(a (b " ") (d (e f) (h i)))')
sxpr = tree.as_sxpr()
flat = flatten_sxpr(sxpr)
assert flat == '(a (b " ") (d (e "f") (h "i")))'
class TestParseXML:
def test_roundtrip(self):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment