Commit d9fbc53c authored by Eckhart Arnold's avatar Eckhart Arnold

- BUG: named nodes might be dropped by Series (fixed), ZeroOrMore and potentially other parsers

parent 6ef3b9fe
...@@ -99,7 +99,7 @@ from DHParser import logging, is_filename, load_if_file, \\ ...@@ -99,7 +99,7 @@ from DHParser import logging, is_filename, load_if_file, \\
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \\ keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \\
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \\ remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \\
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \\ replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \\
error_on, recompile_grammar, GLOBALS error_on, recompile_grammar, left_associative, swing_left, GLOBALS
'''.format(dhparserdir=dhparserdir) '''.format(dhparserdir=dhparserdir)
......
...@@ -1748,7 +1748,7 @@ class Series(NaryParser): ...@@ -1748,7 +1748,7 @@ class Series(NaryParser):
else: else:
results += (node,) results += (node,)
break break
if node._result or parser.pname: # optimization if node._result or parser.pname or node.tag_name[0:1] != ':': # optimization
results += (node,) results += (node,)
# assert len(results) <= len(self.parsers) \ # assert len(results) <= len(self.parsers) \
# or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG]) # or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG])
......
...@@ -58,6 +58,8 @@ __all__ = ('TransformationDict', ...@@ -58,6 +58,8 @@ __all__ = ('TransformationDict',
'replace_content_by', 'replace_content_by',
'normalize_whitespace', 'normalize_whitespace',
'move_adjacent', 'move_adjacent',
'left_associative',
'swing_left',
'apply_if', 'apply_if',
'apply_unless', 'apply_unless',
'traverse_locally', 'traverse_locally',
...@@ -878,20 +880,20 @@ def left_associative(context: List[Node]): ...@@ -878,20 +880,20 @@ def left_associative(context: List[Node]):
Rearranges a flat node with infix operators into a left associative tree. Rearranges a flat node with infix operators into a left associative tree.
""" """
node = context[-1] node = context[-1]
assert len(node.children) >= 3 if len(node.children) >= 3:
assert (len(node.children) + 1) % 2 == 0 assert (len(node.children) + 1) % 2 == 0
rest = list(node.result) rest = list(node.result)
left, rest = rest[0], rest[1:] left, rest = rest[0], rest[1:]
while rest: while rest:
infix, right, rest = rest[:2], rest[2:] infix, right, rest = rest[0], rest[1], rest[2:]
assert not infix.children assert not infix.children
assert infix.tag_name[0:1] != ":" assert infix.tag_name[0:1] != ":"
left = Node(infix.tag_name, (left, right)) left = Node(infix.tag_name, (left, right))
node.result = left node.result = left
@transformation_factory(collections.abc.Set) @transformation_factory(collections.abc.Set)
def left_swing(context: List[Node], operators: AbstractSet[str]): def swing_left(context: List[Node], operators: AbstractSet[str]):
""" """
Rearranges a node that contains a sub-node on the right Rearranges a node that contains a sub-node on the right
with a left-associative operator so that the tree structure with a left-associative operator so that the tree structure
......
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
####################################################################### #######################################################################
expression = term { (PLUS|MINUS) term} expression = term { (PLUS|MINUS) term}
term = factor { (DIV|[MUL]) factor} term = factor { (DIV|MUL) factor}
factor = [sign] ( NUMBER | VARIABLE | group ) factor = [sign] ( NUMBER | VARIABLE | group )
sign = POSITIVE | NEGATIVE sign = POSITIVE | NEGATIVE
group = "(" expression ")" group = "(" expression ")"
...@@ -32,7 +32,7 @@ group = "(" expression ")" ...@@ -32,7 +32,7 @@ group = "(" expression ")"
PLUS = "+" PLUS = "+"
MINUS = "-" MINUS = "-"
MUL = "*" MUL = "*" | &factor # TODO: higher precedence of &factor
DIV = "/" DIV = "/"
POSITIVE = /[+]/ # no implicit whitespace after signs POSITIVE = /[+]/ # no implicit whitespace after signs
......
...@@ -33,7 +33,7 @@ from DHParser import logging, is_filename, load_if_file, \ ...@@ -33,7 +33,7 @@ from DHParser import logging, is_filename, load_if_file, \
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \ keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \ remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \ replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \
error_on, recompile_grammar, GLOBALS error_on, recompile_grammar, left_associative, GLOBALS
####################################################################### #######################################################################
...@@ -59,7 +59,7 @@ class ArithmeticGrammar(Grammar): ...@@ -59,7 +59,7 @@ class ArithmeticGrammar(Grammar):
r"""Parser for an Arithmetic source file. r"""Parser for an Arithmetic source file.
""" """
expression = Forward() expression = Forward()
source_hash__ = "9f06b2623e1d797c32efc3b864fec5bd" source_hash__ = "a8142ddc723ae56bbdf6c898efc7af45"
static_analysis_pending__ = [True] static_analysis_pending__ = [True]
parser_initialization__ = ["upon instantiation"] parser_initialization__ = ["upon instantiation"]
resume_rules__ = {} resume_rules__ = {}
...@@ -79,7 +79,7 @@ class ArithmeticGrammar(Grammar): ...@@ -79,7 +79,7 @@ class ArithmeticGrammar(Grammar):
group = Series(Series(DropToken("("), dwsp__), expression, Series(DropToken(")"), dwsp__)) group = Series(Series(DropToken("("), dwsp__), expression, Series(DropToken(")"), dwsp__))
sign = Alternative(POSITIVE, NEGATIVE) sign = Alternative(POSITIVE, NEGATIVE)
factor = Series(Option(sign), Alternative(NUMBER, VARIABLE, group)) factor = Series(Option(sign), Alternative(NUMBER, VARIABLE, group))
term = Series(factor, ZeroOrMore(Series(Alternative(DIV, Option(MUL)), factor))) term = Series(factor, ZeroOrMore(Series(Alternative(DIV, MUL), factor)))
expression.set(Series(term, ZeroOrMore(Series(Alternative(PLUS, MINUS), term)))) expression.set(Series(term, ZeroOrMore(Series(Alternative(PLUS, MINUS), term))))
root__ = expression root__ = expression
...@@ -101,17 +101,17 @@ def get_grammar() -> ArithmeticGrammar: ...@@ -101,17 +101,17 @@ def get_grammar() -> ArithmeticGrammar:
# #
####################################################################### #######################################################################
def group_no_asterix_mul(context: List[Node]):
pass
# TODO: Find an algorithm, here
Arithmetic_AST_transformation_table = { Arithmetic_AST_transformation_table = {
# AST Transformations for the Arithmetic-grammar # AST Transformations for the Arithmetic-grammar
"<": flatten, "expression": [left_associative, replace_by_single_child],
"expression": [], "term": [left_associative, replace_by_single_child],
"term": [reduce_single_child], "factor": [replace_by_single_child],
"factor": [reduce_single_child],
"group": [remove_tokens('(', ')'), replace_by_single_child], "group": [remove_tokens('(', ')'), replace_by_single_child],
"NUMBER": [], "sign": [replace_by_single_child]
"VARIABLE": [],
":Token": reduce_single_child,
"*": replace_by_single_child
} }
......
...@@ -23,11 +23,13 @@ F2: "- 2" ...@@ -23,11 +23,13 @@ F2: "- 2"
M1: "2 * 4" M1: "2 * 4"
M2: "3x" M2: "3x"
M3: "5 / 2" M3: "5 / 2"
M4: "5 / 2x" M4*: "5 / 2x"
M5: "5 / -2x" M5: "5 / -2x"
M6: "-3*2y" M6: "-3*2y"
M7: "4x" M7: "4x"
M8: "-2x" M8: "-2x"
M9: "20 / 2 * 2"
M10: "20 / (2 * 2)"
[fail:term] [fail:term]
F1: "2 + 4" F1: "2 + 4"
...@@ -39,6 +41,8 @@ M2: "-5 - -4x" ...@@ -39,6 +41,8 @@ M2: "-5 - -4x"
M3: "(a + b)(a - b)" M3: "(a + b)(a - b)"
M4: "a - 5 + b" M4: "a - 5 + b"
M5: "-5 - +4x" M5: "-5 - +4x"
M6: "5 - 4 + 3"
M7: "5 - (4 + 3)"
[fail:expression] [fail:expression]
F1: "-5 - - 4x" F1: "-5 - - 4x"
......
...@@ -24,7 +24,7 @@ except ModuleNotFoundError: ...@@ -24,7 +24,7 @@ except ModuleNotFoundError:
CONFIG_PRESET['ast_serialization'] = "S-expression" CONFIG_PRESET['ast_serialization'] = "S-expression"
CONFIG_PRESET['test_parallelization'] = False
def recompile_grammar(grammar_src, force): def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests') grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
......
...@@ -119,26 +119,26 @@ class TestInfiLoopsAndRecursion: ...@@ -119,26 +119,26 @@ class TestInfiLoopsAndRecursion:
log_ST(syntax_tree, "test_LeftRecursion_indirect.cst") log_ST(syntax_tree, "test_LeftRecursion_indirect.cst")
log_parsing_history(parser, "test_LeftRecursion_indirect") log_parsing_history(parser, "test_LeftRecursion_indirect")
# BEWARE: EXPERIMENTAL TEST can be long running # # BEWARE: EXPERIMENTAL TEST can be long running
def test_indirect_left_recursion2(self): # def test_indirect_left_recursion2(self):
arithmetic_syntax = """ # arithmetic_syntax = """
expression = addition | subtraction # expression = addition | subtraction
addition = (expression | term) "+" (expression | term) # addition = (expression | term) "+" (expression | term)
subtraction = (expression | term) "-" (expression | term) # subtraction = (expression | term) "-" (expression | term)
term = multiplication | division # term = multiplication | division
multiplication = (term | factor) "*" (term | factor) # multiplication = (term | factor) "*" (term | factor)
division = (term | factor) "/" (term | factor) # division = (term | factor) "/" (term | factor)
factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group } # factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
group = "(" §expression ")" # group = "(" expression ")"
SIGN = /[+-]/ # SIGN = /[+-]/
NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~ # NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
VARIABLE = /[A-Za-z]/~ # VARIABLE = /[A-Za-z]/~
""" # """
arithmetic = grammar_provider(arithmetic_syntax)() # arithmetic = grammar_provider(arithmetic_syntax)()
arithmetic.left_recursion_depth__ = 2 # arithmetic.left_recursion_depth__ = 2
assert arithmetic # assert arithmetic
syntax_tree = arithmetic("(a + b) * (a - b)") # syntax_tree = arithmetic("(a + b) * (a - b)")
assert syntax_tree.errors # assert syntax_tree.errors
def test_break_inifnite_loop_ZeroOrMore(self): def test_break_inifnite_loop_ZeroOrMore(self):
forever = ZeroOrMore(RegExp('')) forever = ZeroOrMore(RegExp(''))
...@@ -886,6 +886,18 @@ class TestMetaParser: ...@@ -886,6 +886,18 @@ class TestMetaParser:
rv = self.mp._return_values((Node('tag', 'content'), EMPTY_NODE)) rv = self.mp._return_values((Node('tag', 'content'), EMPTY_NODE))
assert rv[-1].tag_name != EMPTY_NODE.tag_name, rv[-1].tag_name assert rv[-1].tag_name != EMPTY_NODE.tag_name, rv[-1].tag_name
def test_in_context(self):
minilang = """
term = factor { (DIV|MUL) factor}
factor = NUMBER | VARIABLE
MUL = "*" | &factor
DIV = "/"
NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
VARIABLE = /[A-Za-z]/~
"""
gr = grammar_provider(minilang)()
cst = gr("2x")
assert bool(cst.pick('MUL'))
if __name__ == "__main__": if __name__ == "__main__":
from DHParser.testing import runner from DHParser.testing import runner
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment