Commit d9fbc53c authored by Eckhart Arnold's avatar Eckhart Arnold

- BUG: named nodes might be dropped by Series (fixed), ZeroOrMore and potentially other parsers

parent 6ef3b9fe
......@@ -99,7 +99,7 @@ from DHParser import logging, is_filename, load_if_file, \\
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \\
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \\
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \\
error_on, recompile_grammar, GLOBALS
error_on, recompile_grammar, left_associative, swing_left, GLOBALS
'''.format(dhparserdir=dhparserdir)
......
......@@ -1748,7 +1748,7 @@ class Series(NaryParser):
else:
results += (node,)
break
if node._result or parser.pname: # optimization
if node._result or parser.pname or node.tag_name[0:1] != ':': # optimization
results += (node,)
# assert len(results) <= len(self.parsers) \
# or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG])
......
......@@ -58,6 +58,8 @@ __all__ = ('TransformationDict',
'replace_content_by',
'normalize_whitespace',
'move_adjacent',
'left_associative',
'swing_left',
'apply_if',
'apply_unless',
'traverse_locally',
......@@ -878,20 +880,20 @@ def left_associative(context: List[Node]):
Rearranges a flat node with infix operators into a left associative tree.
"""
node = context[-1]
assert len(node.children) >= 3
assert (len(node.children) + 1) % 2 == 0
rest = list(node.result)
left, rest = rest[0], rest[1:]
while rest:
infix, right, rest = rest[:2], rest[2:]
assert not infix.children
assert infix.tag_name[0:1] != ":"
left = Node(infix.tag_name, (left, right))
node.result = left
if len(node.children) >= 3:
assert (len(node.children) + 1) % 2 == 0
rest = list(node.result)
left, rest = rest[0], rest[1:]
while rest:
infix, right, rest = rest[0], rest[1], rest[2:]
assert not infix.children
assert infix.tag_name[0:1] != ":"
left = Node(infix.tag_name, (left, right))
node.result = left
@transformation_factory(collections.abc.Set)
def left_swing(context: List[Node], operators: AbstractSet[str]):
def swing_left(context: List[Node], operators: AbstractSet[str]):
"""
Rearranges a node that contains a sub-node on the right
with a left-associative operator so that the tree structure
......
......@@ -19,7 +19,7 @@
#######################################################################
expression = term { (PLUS|MINUS) term}
term = factor { (DIV|[MUL]) factor}
term = factor { (DIV|MUL) factor}
factor = [sign] ( NUMBER | VARIABLE | group )
sign = POSITIVE | NEGATIVE
group = "(" expression ")"
......@@ -32,7 +32,7 @@ group = "(" expression ")"
PLUS = "+"
MINUS = "-"
MUL = "*"
MUL = "*" | &factor # TODO: higher precedence of &factor
DIV = "/"
POSITIVE = /[+]/ # no implicit whitespace after signs
......
......@@ -33,7 +33,7 @@ from DHParser import logging, is_filename, load_if_file, \
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \
error_on, recompile_grammar, GLOBALS
error_on, recompile_grammar, left_associative, GLOBALS
#######################################################################
......@@ -59,7 +59,7 @@ class ArithmeticGrammar(Grammar):
r"""Parser for an Arithmetic source file.
"""
expression = Forward()
source_hash__ = "9f06b2623e1d797c32efc3b864fec5bd"
source_hash__ = "a8142ddc723ae56bbdf6c898efc7af45"
static_analysis_pending__ = [True]
parser_initialization__ = ["upon instantiation"]
resume_rules__ = {}
......@@ -79,7 +79,7 @@ class ArithmeticGrammar(Grammar):
group = Series(Series(DropToken("("), dwsp__), expression, Series(DropToken(")"), dwsp__))
sign = Alternative(POSITIVE, NEGATIVE)
factor = Series(Option(sign), Alternative(NUMBER, VARIABLE, group))
term = Series(factor, ZeroOrMore(Series(Alternative(DIV, Option(MUL)), factor)))
term = Series(factor, ZeroOrMore(Series(Alternative(DIV, MUL), factor)))
expression.set(Series(term, ZeroOrMore(Series(Alternative(PLUS, MINUS), term))))
root__ = expression
......@@ -101,17 +101,17 @@ def get_grammar() -> ArithmeticGrammar:
#
#######################################################################
def group_no_asterix_mul(context: List[Node]):
pass
# TODO: Find an algorithm, here
Arithmetic_AST_transformation_table = {
# AST Transformations for the Arithmetic-grammar
"<": flatten,
"expression": [],
"term": [reduce_single_child],
"factor": [reduce_single_child],
"expression": [left_associative, replace_by_single_child],
"term": [left_associative, replace_by_single_child],
"factor": [replace_by_single_child],
"group": [remove_tokens('(', ')'), replace_by_single_child],
"NUMBER": [],
"VARIABLE": [],
":Token": reduce_single_child,
"*": replace_by_single_child
"sign": [replace_by_single_child]
}
......
......@@ -23,11 +23,13 @@ F2: "- 2"
M1: "2 * 4"
M2: "3x"
M3: "5 / 2"
M4: "5 / 2x"
M4*: "5 / 2x"
M5: "5 / -2x"
M6: "-3*2y"
M7: "4x"
M8: "-2x"
M9: "20 / 2 * 2"
M10: "20 / (2 * 2)"
[fail:term]
F1: "2 + 4"
......@@ -39,6 +41,8 @@ M2: "-5 - -4x"
M3: "(a + b)(a - b)"
M4: "a - 5 + b"
M5: "-5 - +4x"
M6: "5 - 4 + 3"
M7: "5 - (4 + 3)"
[fail:expression]
F1: "-5 - - 4x"
......
......@@ -24,7 +24,7 @@ except ModuleNotFoundError:
CONFIG_PRESET['ast_serialization'] = "S-expression"
CONFIG_PRESET['test_parallelization'] = False
def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
......
......@@ -119,26 +119,26 @@ class TestInfiLoopsAndRecursion:
log_ST(syntax_tree, "test_LeftRecursion_indirect.cst")
log_parsing_history(parser, "test_LeftRecursion_indirect")
# BEWARE: EXPERIMENTAL TEST can be long running
def test_indirect_left_recursion2(self):
arithmetic_syntax = """
expression = addition | subtraction
addition = (expression | term) "+" (expression | term)
subtraction = (expression | term) "-" (expression | term)
term = multiplication | division
multiplication = (term | factor) "*" (term | factor)
division = (term | factor) "/" (term | factor)
factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
group = "(" §expression ")"
SIGN = /[+-]/
NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
VARIABLE = /[A-Za-z]/~
"""
arithmetic = grammar_provider(arithmetic_syntax)()
arithmetic.left_recursion_depth__ = 2
assert arithmetic
syntax_tree = arithmetic("(a + b) * (a - b)")
assert syntax_tree.errors
# # BEWARE: EXPERIMENTAL TEST can be long running
# def test_indirect_left_recursion2(self):
# arithmetic_syntax = """
# expression = addition | subtraction
# addition = (expression | term) "+" (expression | term)
# subtraction = (expression | term) "-" (expression | term)
# term = multiplication | division
# multiplication = (term | factor) "*" (term | factor)
# division = (term | factor) "/" (term | factor)
# factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
# group = "(" expression ")"
# SIGN = /[+-]/
# NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
# VARIABLE = /[A-Za-z]/~
# """
# arithmetic = grammar_provider(arithmetic_syntax)()
# arithmetic.left_recursion_depth__ = 2
# assert arithmetic
# syntax_tree = arithmetic("(a + b) * (a - b)")
# assert syntax_tree.errors
def test_break_inifnite_loop_ZeroOrMore(self):
forever = ZeroOrMore(RegExp(''))
......@@ -886,6 +886,18 @@ class TestMetaParser:
rv = self.mp._return_values((Node('tag', 'content'), EMPTY_NODE))
assert rv[-1].tag_name != EMPTY_NODE.tag_name, rv[-1].tag_name
def test_in_context(self):
minilang = """
term = factor { (DIV|MUL) factor}
factor = NUMBER | VARIABLE
MUL = "*" | &factor
DIV = "/"
NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
VARIABLE = /[A-Za-z]/~
"""
gr = grammar_provider(minilang)()
cst = gr("2x")
assert bool(cst.pick('MUL'))
if __name__ == "__main__":
from DHParser.testing import runner
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment