- BUG: named nodes might be dropped by Series (fixed), ZeroOrMore and potentially other parsers

DHParser/ebnf.py

@@ -99,7 +99,7 @@ from DHParser import logging, is_filename, load_if_file, \\
     keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \\
     remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \\
     replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \\
-    error_on, recompile_grammar, GLOBALS
+    error_on, recompile_grammar, left_associative, swing_left, GLOBALS
 '''.format(dhparserdir=dhparserdir)
 
 

DHParser/parse.py

@@ -1748,7 +1748,7 @@ class Series(NaryParser):
                     else:
                         results += (node,)
                         break
-            if node._result or parser.pname:  # optimization
+            if node._result or parser.pname or node.tag_name[0:1] != ':':  # optimization
                 results += (node,)
         # assert len(results) <= len(self.parsers) \
         #        or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG])

DHParser/transform.py

@@ -58,6 +58,8 @@ __all__ = ('TransformationDict',
            'replace_content_by',
            'normalize_whitespace',
            'move_adjacent',
+           'left_associative',
+           'swing_left',
            'apply_if',
            'apply_unless',
            'traverse_locally',
@@ -878,12 +880,12 @@ def left_associative(context: List[Node]):
     Rearranges a flat node with infix operators into a left associative tree.
     """
     node = context[-1]
-    assert len(node.children) >= 3
+    if len(node.children) >= 3:
         assert (len(node.children) + 1) % 2 == 0
         rest = list(node.result)
         left, rest = rest[0], rest[1:]
         while rest:
-        infix, right, rest = rest[:2], rest[2:]
+            infix, right, rest = rest[0], rest[1], rest[2:]
             assert not infix.children
             assert infix.tag_name[0:1] != ":"
             left = Node(infix.tag_name, (left, right))
@@ -891,7 +893,7 @@ def left_associative(context: List[Node]):
 
 
 @transformation_factory(collections.abc.Set)
-def left_swing(context: List[Node], operators: AbstractSet[str]):
+def swing_left(context: List[Node], operators: AbstractSet[str]):
     """
     Rearranges a node that contains a sub-node on the right
     with a left-associative operator so that the tree structure

examples/Arithmetic/Arithmetic.ebnf

@@ -19,7 +19,7 @@
 #######################################################################
 
 expression = term  { (PLUS|MINUS) term}
-term       = factor  { (DIV|[MUL]) factor}
+term       = factor { (DIV|MUL) factor}
 factor     = [sign] ( NUMBER | VARIABLE | group )
 sign       = POSITIVE | NEGATIVE
 group      = "(" expression ")"
@@ -32,7 +32,7 @@ group      = "(" expression ")"
 
 PLUS       = "+"
 MINUS      = "-"
-MUL        = "*"
+MUL        = "*" | &factor  # TODO: higher precedence of &factor
 DIV        = "/"
 
 POSITIVE   = /[+]/      # no implicit whitespace after signs

examples/Arithmetic/ArithmeticCompiler.py

@@ -33,7 +33,7 @@ from DHParser import logging, is_filename, load_if_file, \
     keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \
     remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \
     replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \
-    error_on, recompile_grammar, GLOBALS
+    error_on, recompile_grammar, left_associative, GLOBALS
 
 
 #######################################################################
@@ -59,7 +59,7 @@ class ArithmeticGrammar(Grammar):
     r"""Parser for an Arithmetic source file.
     """
     expression = Forward()
-    source_hash__ = "9f06b2623e1d797c32efc3b864fec5bd"
+    source_hash__ = "a8142ddc723ae56bbdf6c898efc7af45"
     static_analysis_pending__ = [True]
     parser_initialization__ = ["upon instantiation"]
     resume_rules__ = {}
@@ -79,7 +79,7 @@ class ArithmeticGrammar(Grammar):
     group = Series(Series(DropToken("("), dwsp__), expression, Series(DropToken(")"), dwsp__))
     sign = Alternative(POSITIVE, NEGATIVE)
     factor = Series(Option(sign), Alternative(NUMBER, VARIABLE, group))
-    term = Series(factor, ZeroOrMore(Series(Alternative(DIV, Option(MUL)), factor)))
+    term = Series(factor, ZeroOrMore(Series(Alternative(DIV, MUL), factor)))
     expression.set(Series(term, ZeroOrMore(Series(Alternative(PLUS, MINUS), term))))
     root__ = expression
     
@@ -101,17 +101,17 @@ def get_grammar() -> ArithmeticGrammar:
 #
 #######################################################################
 
+def group_no_asterix_mul(context: List[Node]):
+    pass
+    # TODO: Find an algorithm, here
+
 Arithmetic_AST_transformation_table = {
     # AST Transformations for the Arithmetic-grammar
-    "<": flatten,
-    "expression": [],
-    "term": [reduce_single_child],
-    "factor": [reduce_single_child],
+    "expression": [left_associative, replace_by_single_child],
+    "term": [left_associative, replace_by_single_child],
+    "factor": [replace_by_single_child],
     "group": [remove_tokens('(', ')'), replace_by_single_child],
-    "NUMBER": [],
-    "VARIABLE": [],
-    ":Token": reduce_single_child,
-    "*": replace_by_single_child
+    "sign": [replace_by_single_child]
 }
 
 

examples/Arithmetic/grammar_tests/02_test_components.ini

@@ -23,11 +23,13 @@ F2: "- 2"
 M1: "2 * 4"
 M2: "3x"
 M3: "5 / 2"
-M4: "5 / 2x"
+M4*: "5 / 2x"
 M5: "5 / -2x"
 M6: "-3*2y"
 M7: "4x"
 M8: "-2x"
+M9: "20 / 2 * 2"
+M10: "20 / (2 * 2)"
 
 [fail:term]
 F1: "2 + 4"
@@ -39,6 +41,8 @@ M2: "-5 - -4x"
 M3: "(a + b)(a - b)"
 M4: "a - 5 + b"
 M5: "-5 - +4x"
+M6: "5 - 4 + 3"
+M7: "5 - (4 + 3)"
 
 [fail:expression]
 F1: "-5 - - 4x"

examples/Arithmetic/tst_Arithmetic_grammar.py

@@ -24,7 +24,7 @@ except ModuleNotFoundError:
 
 
 CONFIG_PRESET['ast_serialization'] = "S-expression"
-
+CONFIG_PRESET['test_parallelization'] = False
 
 def recompile_grammar(grammar_src, force):
     grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')

test/test_parse.py

@@ -119,26 +119,26 @@ class TestInfiLoopsAndRecursion:
             log_ST(syntax_tree, "test_LeftRecursion_indirect.cst")
             log_parsing_history(parser, "test_LeftRecursion_indirect")
 
-    # BEWARE: EXPERIMENTAL TEST can be long running
-    def test_indirect_left_recursion2(self):
-        arithmetic_syntax = """
-            expression     = addition | subtraction
-            addition       = (expression | term) "+" (expression | term)
-            subtraction    = (expression | term) "-" (expression | term)
-            term           = multiplication | division
-            multiplication = (term | factor) "*" (term | factor)
-            division       = (term | factor) "/" (term | factor)
-            factor         = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
-            group          = "(" §expression ")"
-            SIGN           = /[+-]/
-            NUMBER         = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
-            VARIABLE       = /[A-Za-z]/~
-            """
-        arithmetic = grammar_provider(arithmetic_syntax)()
-        arithmetic.left_recursion_depth__ = 2
-        assert arithmetic
-        syntax_tree = arithmetic("(a + b) * (a - b)")
-        assert syntax_tree.errors
+    # # BEWARE: EXPERIMENTAL TEST can be long running
+    # def test_indirect_left_recursion2(self):
+    #     arithmetic_syntax = """
+    #         expression     = addition | subtraction
+    #         addition       = (expression | term) "+" (expression | term)
+    #         subtraction    = (expression | term) "-" (expression | term)
+    #         term           = multiplication | division
+    #         multiplication = (term | factor) "*" (term | factor)
+    #         division       = (term | factor) "/" (term | factor)
+    #         factor         = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
+    #         group          = "(" expression ")"
+    #         SIGN           = /[+-]/
+    #         NUMBER         = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
+    #         VARIABLE       = /[A-Za-z]/~
+    #         """
+    #     arithmetic = grammar_provider(arithmetic_syntax)()
+    #     arithmetic.left_recursion_depth__ = 2
+    #     assert arithmetic
+    #     syntax_tree = arithmetic("(a + b) * (a - b)")
+    #     assert syntax_tree.errors
 
     def test_break_inifnite_loop_ZeroOrMore(self):
         forever = ZeroOrMore(RegExp(''))
@@ -886,6 +886,18 @@ class TestMetaParser:
         rv = self.mp._return_values((Node('tag', 'content'), EMPTY_NODE))
         assert rv[-1].tag_name != EMPTY_NODE.tag_name, rv[-1].tag_name
 
+    def test_in_context(self):
+        minilang = """
+            term       = factor  { (DIV|MUL) factor}
+            factor     = NUMBER | VARIABLE
+            MUL        = "*" | &factor
+            DIV        = "/"            
+            NUMBER     = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
+            VARIABLE   = /[A-Za-z]/~
+            """
+        gr = grammar_provider(minilang)()
+        cst = gr("2x")
+        assert bool(cst.pick('MUL'))
 
 if __name__ == "__main__":
     from DHParser.testing import runner