2.12.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit 71d05783 authored by eckhart's avatar eckhart
Browse files

- superflous print message from tests removed

parent fbd7bd4e
...@@ -78,6 +78,13 @@ CONFIG_PRESET['cst_serialization'] = "compact" ...@@ -78,6 +78,13 @@ CONFIG_PRESET['cst_serialization'] = "compact"
CONFIG_PRESET['ast_serialization'] = "XML" CONFIG_PRESET['ast_serialization'] = "XML"
CONFIG_PRESET['default_serialization'] = "S-expression" CONFIG_PRESET['default_serialization'] = "S-expression"
# Defines the maximum line length for flattened S-expressions.
# Below this threshold S-expressions will be returned in flattened
# form by DhParser.syntaxtree.serialize() and other functions
# that use serialize(), like, for example, the reporting functions
# in DHParser.testing.
CONFIG_PRESET['flatten_sxpr_threshold'] = 120
# Allows (coarse-grained) parallelization for running tests via the # Allows (coarse-grained) parallelization for running tests via the
# Python multiprocessing module # Python multiprocessing module
# Default value: True # Default value: True
......
...@@ -297,6 +297,10 @@ class Parser: ...@@ -297,6 +297,10 @@ class Parser:
try: try:
# PARSER CALL: run _parse() method # PARSER CALL: run _parse() method
node, rest = self._parse(text) node, rest = self._parse(text)
# TODO: infinite loop protection. Definition "infinite loop":
# 1. same parser, 2. same postion, 3. same recursion depth
# if is_logging() and self.pname:
# print(len(text), len(grammar.call_stack__), bool(node), location in self.visited, self.pname, text)
except ParserError as error: except ParserError as error:
# does this play well with variable setting? add rollback clause here? tests needed... # does this play well with variable setting? add rollback clause here? tests needed...
gap = len(text) - len(error.rest) gap = len(text) - len(error.rest)
...@@ -340,11 +344,12 @@ class Parser: ...@@ -340,11 +344,12 @@ class Parser:
if location in grammar.recursion_locations__: if location in grammar.recursion_locations__:
if location in self.visited: if location in self.visited:
node, rest = self.visited[location] node, rest = self.visited[location]
if node and location != grammar.last_recursion_location__: if location != grammar.last_recursion_location__:
grammar.tree__.add_error( grammar.tree__.add_error(
node, Error("Left recursion encountered. " node, Error("Left recursion encountered. "
"Refactor grammar to avoid slow parsing.", "Refactor grammar to avoid slow parsing.",
node.pos, Error.LEFT_RECURSION_WARNING)) node.pos if node else location,
Error.LEFT_RECURSION_WARNING))
grammar.last_recursion_location__ = location grammar.last_recursion_location__ = location
# don't overwrite any positive match (i.e. node not None) in the cache # don't overwrite any positive match (i.e. node not None) in the cache
# and don't add empty entries for parsers returning from left recursive calls! # and don't add empty entries for parsers returning from left recursive calls!
......
...@@ -76,17 +76,27 @@ StrictResultType = Union[ChildrenType, StringView, str] ...@@ -76,17 +76,27 @@ StrictResultType = Union[ChildrenType, StringView, str]
ResultType = Union[ChildrenType, 'Node', StringView, str, None] ResultType = Union[ChildrenType, 'Node', StringView, str, None]
def flatten_sxpr(sxpr: str) -> str: def flatten_sxpr(sxpr: str, threshold: int = -1) -> str:
""" """
Returns S-expression ``sxpr`` as a one-liner without unnecessary Returns S-expression ``sxpr`` as a one-liner without unnecessary
whitespace. whitespace.
The ``threshold`` value is a maximum number of
characters allowed in the flattened expression. If this number
is exceeded the the unflattened S-expression is returned. A
negative number means that the S-expression will always be
flattened. Zero or (any postive integer <= 3) essentially means
that the expression will not be flattened.
Example: Example:
>>> flatten_sxpr('(a\\n (b\\n c\\n )\\n)\\n') >>> flatten_sxpr('(a\\n (b\\n c\\n )\\n)\\n')
'(a (b c))' '(a (b c))'
""" """
return re.sub(r'\s(?=\))', '', re.sub(r'\s+', ' ', sxpr)).strip() flat = re.sub(r'\s(?=\))', '', re.sub(r'\s+', ' ', sxpr)).strip()
if threshold >= 0 and len(flat) > threshold:
return sxpr.strip()
return flat
def flatten_xml(xml: str) -> str: def flatten_xml(xml: str) -> str:
...@@ -537,7 +547,8 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -537,7 +547,8 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def as_sxpr(self, src: str = None, def as_sxpr(self, src: str = None,
indentation: int = 2, indentation: int = 2,
compact: bool = False) -> str: compact: bool = False,
flatten_threshold: int = 0) -> str:
""" """
Returns content as S-expression, i.e. in lisp-like form. If this Returns content as S-expression, i.e. in lisp-like form. If this
method is callad on a RootNode-object, method is callad on a RootNode-object,
...@@ -550,6 +561,9 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -550,6 +561,9 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
compact: If True, a compact representation is returned where compact: If True, a compact representation is returned where
brackets are omitted and only the indentation indicates the brackets are omitted and only the indentation indicates the
tree structure. tree structure.
flatten_threshold: Return the S-expression in flattened form if
the flattened expression does not exceed the threshold length.
A negative number means that it will always be flattened.
""" """
left_bracket, right_bracket, density = ('', '', 1) if compact else ('(', '\n)', 0) left_bracket, right_bracket, density = ('', '', 1) if compact else ('(', '\n)', 0)
...@@ -580,7 +594,8 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -580,7 +594,8 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
else "'%s'" % strg if strg.find("'") < 0 \ else "'%s'" % strg if strg.find("'") < 0 \
else '"%s"' % strg.replace('"', r'\"') else '"%s"' % strg.replace('"', r'\"')
return self._tree_repr(' ' * indentation, opening, closing, pretty, density=density) sxpr = self._tree_repr(' ' * indentation, opening, closing, pretty, density=density)
return flatten_sxpr(sxpr, flatten_threshold)
def as_xml(self, src: str = None, def as_xml(self, src: str = None,
...@@ -759,7 +774,7 @@ def serialize(node: Node, how: str='default') -> str: ...@@ -759,7 +774,7 @@ def serialize(node: Node, how: str='default') -> str:
switch = get_config_value('default_serialization').lower() switch = get_config_value('default_serialization').lower()
if switch == 's-expression': if switch == 's-expression':
return node.as_sxpr() return node.as_sxpr(flatten_threshold=get_config_value('flatten_sxpr_threshold'))
elif switch == 'xml': elif switch == 'xml':
return node.as_xml() return node.as_xml()
elif switch == 'compact': elif switch == 'compact':
...@@ -918,15 +933,17 @@ class RootNode(Node): ...@@ -918,15 +933,17 @@ class RootNode(Node):
self.error_nodes[id(self)] = self.error_nodes[id(node)] self.error_nodes[id(self)] = self.error_nodes[id(node)]
return self return self
def add_error(self, node: Node, error: Error) -> 'RootNode': def add_error(self, node: Optional[Node], error: Error) -> 'RootNode':
""" """
Adds an Error object to the tree, locating it at a specific node. Adds an Error object to the tree, locating it at a specific node.
""" """
if not node:
node = Node(ZOMBIE_TAG, '').with_pos(error.pos)
assert node.pos == error.pos or isinstance(node, FrozenNode) assert node.pos == error.pos or isinstance(node, FrozenNode)
self.errors.append(error)
self.error_flag = max(self.error_flag, error.code)
self.error_nodes.setdefault(id(node), []).append(error) self.error_nodes.setdefault(id(node), []).append(error)
self.error_positions.setdefault(error.pos, set()).add(id(node)) self.error_positions.setdefault(error.pos, set()).add(id(node))
self.errors.append(error)
self.error_flag = max(self.error_flag, error.code)
return self return self
def new_error(self, def new_error(self,
......
...@@ -249,11 +249,6 @@ def get_report(test_unit): ...@@ -249,11 +249,6 @@ def get_report(test_unit):
lines[0] = ' ' + lines[0] lines[0] = ' ' + lines[0]
return "\n ".join(lines) return "\n ".join(lines)
def flatten(serialization):
if serialization.lstrip().startswith('(') and serialization.count('\n') <= 16:
return flatten_sxpr(serialization)
return serialization
report = [] report = []
for parser_name, tests in test_unit.items(): for parser_name, tests in test_unit.items():
heading = 'Test of parser: "%s"' % parser_name heading = 'Test of parser: "%s"' % parser_name
...@@ -271,10 +266,10 @@ def get_report(test_unit): ...@@ -271,10 +266,10 @@ def get_report(test_unit):
cst = tests.get('__cst__', {}).get(test_name, None) cst = tests.get('__cst__', {}).get(test_name, None)
if cst and (not ast or str(test_name).endswith('*')): if cst and (not ast or str(test_name).endswith('*')):
report.append('\n### CST') report.append('\n### CST')
report.append(indent(flatten(serialize(cst, 'cst')))) report.append(indent(serialize(cst, 'cst')))
if ast: if ast:
report.append('\n### AST') report.append('\n### AST')
report.append(indent(flatten(serialize(ast, 'ast')))) report.append(indent(serialize(ast, 'ast')))
for test_name, test_code in tests.get('fail', dict()).items(): for test_name, test_code in tests.get('fail', dict()).items():
heading = 'Fail-test "%s"' % test_name heading = 'Fail-test "%s"' % test_name
report.append('\n%s\n%s\n' % (heading, '-' * len(heading))) report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
...@@ -360,6 +355,9 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve ...@@ -360,6 +355,9 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
and raw_errors[-1].code == Error.MANDATORY_CONTINUATION_AT_EOF)) and raw_errors[-1].code == Error.MANDATORY_CONTINUATION_AT_EOF))
for parser_name, tests in test_unit.items(): for parser_name, tests in test_unit.items():
if not get_config_value('test_parallelization'):
print(' ' + parser_name)
assert parser_name, "Missing parser name in test %s!" % unit_name assert parser_name, "Missing parser name in test %s!" % unit_name
assert not any(test_type in RESULT_STAGES for test_type in tests), \ assert not any(test_type in RESULT_STAGES for test_type in tests), \
("Test %s in %s already has results. Use reset_unit() before running again!" ("Test %s in %s already has results. Use reset_unit() before running again!"
...@@ -384,6 +382,9 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve ...@@ -384,6 +382,9 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
# run match tests # run match tests
for test_name, test_code in tests.get('match', dict()).items(): for test_name, test_code in tests.get('match', dict()).items():
if not get_config_value('test_parallelization'):
print(' ' + test_name)
errflag = len(errata) errflag = len(errata)
try: try:
cst = parser(test_code, parser_name, track_history=has_lookahead(parser_name)) cst = parser(test_code, parser_name, track_history=has_lookahead(parser_name))
...@@ -533,6 +534,7 @@ def grammar_suite(directory, parser_factory, transformer_factory, ...@@ -533,6 +534,7 @@ def grammar_suite(directory, parser_factory, transformer_factory,
for filename in sorted(os.listdir('.')): for filename in sorted(os.listdir('.')):
if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns): if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
parameters = filename, parser_factory, transformer_factory, report, verbose parameters = filename, parser_factory, transformer_factory, report, verbose
print(filename)
results.append((filename, grammar_unit(*parameters))) results.append((filename, grammar_unit(*parameters)))
for filename, errata in results: for filename, errata in results:
if errata: if errata:
......
[match:group] [match:group]
M1*: "(2 + x)" M1: "(2 + x)"
M2: "(3)" M2: "(3)"
[fail:group] [fail:group]
...@@ -11,10 +11,10 @@ F2: "y" ...@@ -11,10 +11,10 @@ F2: "y"
M1: "-2" M1: "-2"
M2: "-2.71828" M2: "-2.71828"
M3: "-x" M3: "-x"
M4*: "(2 + x)" M4: "(2 + x)"
M5: "-(a * b)" M5: "-(a * b)"
M6*: "4x" M6: "4x"
M7*: "-2x" M7: "-2x"
[fail:factor] [fail:factor]
F1: "+22" F1: "+22"
...@@ -27,8 +27,8 @@ M1: "2 * 4" ...@@ -27,8 +27,8 @@ M1: "2 * 4"
M2: "3x" M2: "3x"
M3: "5 / 2" M3: "5 / 2"
M4: "5 / 2x" M4: "5 / 2x"
M5*: "5 / -2x" M5: "5 / -2x"
M6*: "-3*2y" M6: "-3*2y"
[fail:term] [fail:term]
F1: "2 + 4" F1: "2 + 4"
...@@ -36,8 +36,9 @@ F2: "4 - 5" ...@@ -36,8 +36,9 @@ F2: "4 - 5"
[match:expression] [match:expression]
M1: "3 + x" M1: "3 + x"
M2*: "-5 - -4x" M2: "-5 - -4x"
M3*: "(a + b)(a - b)" M3: "(a + b)(a - b)"
M4: "a - 5 + b"
[fail:expression] [fail:expression]
F1: "-5 - - 4x" F1: "-5 - - 4x"
......
...@@ -16,14 +16,23 @@ scriptpath = os.path.dirname(__file__) ...@@ -16,14 +16,23 @@ scriptpath = os.path.dirname(__file__)
try: try:
from DHParser import dsl from DHParser import dsl
import DHParser.log import DHParser.log
from DHParser import testing from DHParser import testing, create_test_templates, CONFIG_PRESET
except ModuleNotFoundError: except ModuleNotFoundError:
print('Could not import DHParser. Please adjust sys.path in file ' print('Could not import DHParser. Please adjust sys.path in file '
'"%s" manually' % __file__) '"%s" manually' % __file__)
sys.exit(1) sys.exit(1)
CONFIG_PRESET['ast_serialization'] = "S-expression"
def recompile_grammar(grammar_src, force): def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
if not os.path.exists(grammar_tests_dir) \
or not any(os.path.isfile(os.path.join(grammar_tests_dir, entry))
for entry in os.listdir(grammar_tests_dir)):
print('No grammar-tests found, generating test templates.')
create_test_templates(grammar_src, grammar_tests_dir)
with DHParser.log.logging(LOGGING): with DHParser.log.logging(LOGGING):
# recompiles Grammar only if it has changed # recompiles Grammar only if it has changed
name = os.path.splitext(os.path.basename(grammar_src))[0] name = os.path.splitext(os.path.basename(grammar_src))[0]
......
...@@ -91,6 +91,7 @@ class TestInfiLoopsAndRecursion: ...@@ -91,6 +91,7 @@ class TestInfiLoopsAndRecursion:
snippet = "9 + 8 + 7 + 6 + 5 + 3 * 4" snippet = "9 + 8 + 7 + 6 + 5 + 3 * 4"
parser = grammar_provider(minilang)() parser = grammar_provider(minilang)()
assert parser assert parser
with logging():
syntax_tree = parser(snippet) syntax_tree = parser(snippet)
assert not is_error(syntax_tree.error_flag), syntax_tree.errors_sorted assert not is_error(syntax_tree.error_flag), syntax_tree.errors_sorted
assert snippet == syntax_tree.content assert snippet == syntax_tree.content
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment