diff --git a/DHParser/ebnf.py b/DHParser/ebnf.py index 3a51562f608b8be23a2772fcadd84ff77c7eb977..15798b68aed74df540157399ae76ac8b522c70f2 100644 --- a/DHParser/ebnf.py +++ b/DHParser/ebnf.py @@ -99,7 +99,7 @@ from DHParser import logging, is_filename, load_if_file, \\ keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \\ remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \\ replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \\ - error_on, recompile_grammar, GLOBALS + error_on, recompile_grammar, left_associative, swing_left, GLOBALS '''.format(dhparserdir=dhparserdir) diff --git a/DHParser/parse.py b/DHParser/parse.py index 0f5a1d6bd6a364e95c01eb8e753d757088a46c65..d68c2dfa06d81e610daa8b921e3904f06a3a7499 100644 --- a/DHParser/parse.py +++ b/DHParser/parse.py @@ -1748,7 +1748,7 @@ class Series(NaryParser): else: results += (node,) break - if node._result or parser.pname: # optimization + if node._result or parser.pname or node.tag_name[0:1] != ':': # optimization results += (node,) # assert len(results) <= len(self.parsers) \ # or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG]) diff --git a/DHParser/transform.py b/DHParser/transform.py index 7715991b51c3f0cfd86691866fb973d134e71dc4..abca17a7903967dfbed1a472ca22c1cda6c9a281 100644 --- a/DHParser/transform.py +++ b/DHParser/transform.py @@ -58,6 +58,8 @@ __all__ = ('TransformationDict', 'replace_content_by', 'normalize_whitespace', 'move_adjacent', + 'left_associative', + 'swing_left', 'apply_if', 'apply_unless', 'traverse_locally', @@ -878,20 +880,20 @@ def left_associative(context: List[Node]): Rearranges a flat node with infix operators into a left associative tree. """ node = context[-1] - assert len(node.children) >= 3 - assert (len(node.children) + 1) % 2 == 0 - rest = list(node.result) - left, rest = rest[0], rest[1:] - while rest: - infix, right, rest = rest[:2], rest[2:] - assert not infix.children - assert infix.tag_name[0:1] != ":" - left = Node(infix.tag_name, (left, right)) - node.result = left + if len(node.children) >= 3: + assert (len(node.children) + 1) % 2 == 0 + rest = list(node.result) + left, rest = rest[0], rest[1:] + while rest: + infix, right, rest = rest[0], rest[1], rest[2:] + assert not infix.children + assert infix.tag_name[0:1] != ":" + left = Node(infix.tag_name, (left, right)) + node.result = left @transformation_factory(collections.abc.Set) -def left_swing(context: List[Node], operators: AbstractSet[str]): +def swing_left(context: List[Node], operators: AbstractSet[str]): """ Rearranges a node that contains a sub-node on the right with a left-associative operator so that the tree structure diff --git a/examples/Arithmetic/Arithmetic.ebnf b/examples/Arithmetic/Arithmetic.ebnf index 718e9e92a8938144705b52d71eeb6e3ba6aec50a..ce7866361f62b937647fb74eb84b79c2e979d3a0 100644 --- a/examples/Arithmetic/Arithmetic.ebnf +++ b/examples/Arithmetic/Arithmetic.ebnf @@ -19,7 +19,7 @@ ####################################################################### expression = term { (PLUS|MINUS) term} -term = factor { (DIV|[MUL]) factor} +term = factor { (DIV|MUL) factor} factor = [sign] ( NUMBER | VARIABLE | group ) sign = POSITIVE | NEGATIVE group = "(" expression ")" @@ -32,7 +32,7 @@ group = "(" expression ")" PLUS = "+" MINUS = "-" -MUL = "*" +MUL = "*" | &factor # TODO: higher precedence of &factor DIV = "/" POSITIVE = /[+]/ # no implicit whitespace after signs diff --git a/examples/Arithmetic/ArithmeticCompiler.py b/examples/Arithmetic/ArithmeticCompiler.py index 01026ab53b085e7c6448d89f001a87e0080694a6..12e73933e4f048c4e8e2afb359d18402369ea063 100755 --- a/examples/Arithmetic/ArithmeticCompiler.py +++ b/examples/Arithmetic/ArithmeticCompiler.py @@ -33,7 +33,7 @@ from DHParser import logging, is_filename, load_if_file, \ keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \ remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \ replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \ - error_on, recompile_grammar, GLOBALS + error_on, recompile_grammar, left_associative, GLOBALS ####################################################################### @@ -59,7 +59,7 @@ class ArithmeticGrammar(Grammar): r"""Parser for an Arithmetic source file. """ expression = Forward() - source_hash__ = "9f06b2623e1d797c32efc3b864fec5bd" + source_hash__ = "a8142ddc723ae56bbdf6c898efc7af45" static_analysis_pending__ = [True] parser_initialization__ = ["upon instantiation"] resume_rules__ = {} @@ -79,7 +79,7 @@ class ArithmeticGrammar(Grammar): group = Series(Series(DropToken("("), dwsp__), expression, Series(DropToken(")"), dwsp__)) sign = Alternative(POSITIVE, NEGATIVE) factor = Series(Option(sign), Alternative(NUMBER, VARIABLE, group)) - term = Series(factor, ZeroOrMore(Series(Alternative(DIV, Option(MUL)), factor))) + term = Series(factor, ZeroOrMore(Series(Alternative(DIV, MUL), factor))) expression.set(Series(term, ZeroOrMore(Series(Alternative(PLUS, MINUS), term)))) root__ = expression @@ -101,17 +101,17 @@ def get_grammar() -> ArithmeticGrammar: # ####################################################################### +def group_no_asterix_mul(context: List[Node]): + pass + # TODO: Find an algorithm, here + Arithmetic_AST_transformation_table = { # AST Transformations for the Arithmetic-grammar - "<": flatten, - "expression": [], - "term": [reduce_single_child], - "factor": [reduce_single_child], + "expression": [left_associative, replace_by_single_child], + "term": [left_associative, replace_by_single_child], + "factor": [replace_by_single_child], "group": [remove_tokens('(', ')'), replace_by_single_child], - "NUMBER": [], - "VARIABLE": [], - ":Token": reduce_single_child, - "*": replace_by_single_child + "sign": [replace_by_single_child] } diff --git a/examples/Arithmetic/grammar_tests/02_test_components.ini b/examples/Arithmetic/grammar_tests/02_test_components.ini index 4b081ceb1a9253b924c31cdc7d717b392c3f4200..04ce2aeb066e3949e5d934cf6fa77e5038831c07 100644 --- a/examples/Arithmetic/grammar_tests/02_test_components.ini +++ b/examples/Arithmetic/grammar_tests/02_test_components.ini @@ -23,11 +23,13 @@ F2: "- 2" M1: "2 * 4" M2: "3x" M3: "5 / 2" -M4: "5 / 2x" +M4*: "5 / 2x" M5: "5 / -2x" M6: "-3*2y" M7: "4x" M8: "-2x" +M9: "20 / 2 * 2" +M10: "20 / (2 * 2)" [fail:term] F1: "2 + 4" @@ -39,6 +41,8 @@ M2: "-5 - -4x" M3: "(a + b)(a - b)" M4: "a - 5 + b" M5: "-5 - +4x" +M6: "5 - 4 + 3" +M7: "5 - (4 + 3)" [fail:expression] F1: "-5 - - 4x" diff --git a/examples/Arithmetic/tst_Arithmetic_grammar.py b/examples/Arithmetic/tst_Arithmetic_grammar.py index 7ae5dc7b008b2bcb9a7fd5f59bb17c707042b07d..662250d8843dc99275fcbbe14660fb074fe05ccd 100755 --- a/examples/Arithmetic/tst_Arithmetic_grammar.py +++ b/examples/Arithmetic/tst_Arithmetic_grammar.py @@ -24,7 +24,7 @@ except ModuleNotFoundError: CONFIG_PRESET['ast_serialization'] = "S-expression" - +CONFIG_PRESET['test_parallelization'] = False def recompile_grammar(grammar_src, force): grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests') diff --git a/test/test_parse.py b/test/test_parse.py index 9928ca1f1ae1457ab10ea2b69c52c29f212af4ba..7dd9ad12c3fbc52e6fa817ec605415c7036abff1 100644 --- a/test/test_parse.py +++ b/test/test_parse.py @@ -119,26 +119,26 @@ class TestInfiLoopsAndRecursion: log_ST(syntax_tree, "test_LeftRecursion_indirect.cst") log_parsing_history(parser, "test_LeftRecursion_indirect") - # BEWARE: EXPERIMENTAL TEST can be long running - def test_indirect_left_recursion2(self): - arithmetic_syntax = """ - expression = addition | subtraction - addition = (expression | term) "+" (expression | term) - subtraction = (expression | term) "-" (expression | term) - term = multiplication | division - multiplication = (term | factor) "*" (term | factor) - division = (term | factor) "/" (term | factor) - factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group } - group = "(" §expression ")" - SIGN = /[+-]/ - NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~ - VARIABLE = /[A-Za-z]/~ - """ - arithmetic = grammar_provider(arithmetic_syntax)() - arithmetic.left_recursion_depth__ = 2 - assert arithmetic - syntax_tree = arithmetic("(a + b) * (a - b)") - assert syntax_tree.errors + # # BEWARE: EXPERIMENTAL TEST can be long running + # def test_indirect_left_recursion2(self): + # arithmetic_syntax = """ + # expression = addition | subtraction + # addition = (expression | term) "+" (expression | term) + # subtraction = (expression | term) "-" (expression | term) + # term = multiplication | division + # multiplication = (term | factor) "*" (term | factor) + # division = (term | factor) "/" (term | factor) + # factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group } + # group = "(" expression ")" + # SIGN = /[+-]/ + # NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~ + # VARIABLE = /[A-Za-z]/~ + # """ + # arithmetic = grammar_provider(arithmetic_syntax)() + # arithmetic.left_recursion_depth__ = 2 + # assert arithmetic + # syntax_tree = arithmetic("(a + b) * (a - b)") + # assert syntax_tree.errors def test_break_inifnite_loop_ZeroOrMore(self): forever = ZeroOrMore(RegExp('')) @@ -886,6 +886,18 @@ class TestMetaParser: rv = self.mp._return_values((Node('tag', 'content'), EMPTY_NODE)) assert rv[-1].tag_name != EMPTY_NODE.tag_name, rv[-1].tag_name + def test_in_context(self): + minilang = """ + term = factor { (DIV|MUL) factor} + factor = NUMBER | VARIABLE + MUL = "*" | &factor + DIV = "/" + NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~ + VARIABLE = /[A-Za-z]/~ + """ + gr = grammar_provider(minilang)() + cst = gr("2x") + assert bool(cst.pick('MUL')) if __name__ == "__main__": from DHParser.testing import runner