ebnf.py, examples/FixedEBNF, examples/FlexibleEBNF: directive can now contain...

ebnf.py, examples/FixedEBNF, examples/FlexibleEBNF: directive can now contain grammar expressions, next to regular expressions, symbols and literals

DHParser/parse.py

@@ -1654,7 +1654,10 @@ class Grammar:
             if rest:
                 stitches.append(Node(ZOMBIE_TAG, rest))
             result = Node(ZOMBIE_TAG, tuple(stitches)).with_pos(0)
-        if any(self.variables__.values()):
+        if any(self.variables__.values()) \
+                and (start_parser is self.root_parser__ or start_parser == 'root_parser__'):
+                # capture stack not empty will only be reported for root-parsers
+                # to avoid false negatives when testing
             error_msg = "Capture-stack not empty after end of parsing: " \
                 + ', '.join(k for k, i in self.variables__.items() if len(i) >= 1)
             if parser.apply(has_non_autocaptured_symbols):
@@ -2841,8 +2844,8 @@ class MandatoryNary(NaryParser):
                                   location, MALFORMED_ERROR_STRING)
                     grammar.tree__.add_error(err_node, error)
         else:
-            msg = '%s expected by parser %s, »%s« found!' \
-                  % (expected, repr(sym), found)
+            msg = '%s expected by parser %s, but »%s« found instead!' \
+                  % (repr(expected), repr(sym), found)
         error = Error(msg, location,
                       MANDATORY_CONTINUATION_AT_EOF if (failed_on_lookahead and not text_)
                       else MANDATORY_CONTINUATION,
@@ -3968,7 +3971,7 @@ class Forward(UnaryParser):
                         break
                     result = next_result
                     depth += 1
-            # grammar.suspend_memoization__ = recursion_state \
+            # grammar.suspend_memoization__ = memoization_state \
             #     or location <= (grammar.last_rb__loc__ + int(text._len == result[1]._len))
             grammar.suspend_memoization__ = memoization_state
             if not grammar.suspend_memoization__:

examples/FixedEBNF/FixedEBNF.ebnf

@@ -37,7 +37,7 @@ definition = symbol §DEF~ [ OR~ ] expression ENDL~ & FOLLOW_UP  # [OR~] to supp
 
 directive  = "@" §symbol "=" component { "," component } & FOLLOW_UP
   # component  = (regexp | literals | procedure | symbol !DEF)
-  component  = (procedure | expression)
+  component  = literals | procedure | expression
   literals   = { literal }+                       # string chaining, only allowed in directives!
   procedure  = SYM_REGEX "()"                     # procedure name, only allowed in directives!
 

examples/FixedEBNF/FixedEBNFParser.py

@@ -75,7 +75,7 @@ class FixedEBNFGrammar(Grammar):
     countable = Forward()
     element = Forward()
     expression = Forward()
-    source_hash__ = "0da7ed4d4fae3d6fa358ddd7867ccadb"
+    source_hash__ = "d39bd97362e79f1a15bdca37c067d78b"
     disposable__ = re.compile('component$|pure_elem$|countable$|FOLLOW_UP$|SYM_REGEX$|ANY_SUFFIX$|EOF$')
     static_analysis_pending__ = []  # type: List[bool]
     parser_initialization__ = ["upon instantiation"]
@@ -124,9 +124,9 @@ class FixedEBNFGrammar(Grammar):
     retrieveop = Alternative(Series(Text("::"), dwsp__), Series(Text(":?"), dwsp__), Series(Text(":"), dwsp__))
     flowmarker = Alternative(Series(Text("!"), dwsp__), Series(Text("&"), dwsp__), Series(Text("<-!"), dwsp__), Series(Text("<-&"), dwsp__))
     ANY_SUFFIX = RegExp('[?*+]')
-    procedure = Series(SYM_REGEX, Series(Text("()"), dwsp__))
+    literals = OneOrMore(literal)
     pure_elem = Series(element, NegativeLookahead(ANY_SUFFIX), mandatory=1)
-    component = Alternative(procedure, expression)
+    procedure = Series(SYM_REGEX, Series(Text("()"), dwsp__))
     term = Alternative(oneormore, counted, repetition, option, pure_elem)
     difference = Series(term, Option(Series(Series(Text("-"), dwsp__), Alternative(oneormore, pure_elem), mandatory=1)))
     lookaround = Series(flowmarker, Alternative(oneormore, pure_elem), mandatory=1)
@@ -134,8 +134,8 @@ class FixedEBNFGrammar(Grammar):
     sequence = Series(Option(Series(Text("§"), dwsp__)), Alternative(interleave, lookaround), ZeroOrMore(Series(AND, dwsp__, Option(Series(Text("§"), dwsp__)), Alternative(interleave, lookaround))))
     FOLLOW_UP = Alternative(Text("@"), symbol, EOF)
     definition = Series(symbol, DEF, dwsp__, Option(Series(OR, dwsp__)), expression, ENDL, dwsp__, Lookahead(FOLLOW_UP), mandatory=1)
+    component = Alternative(literals, procedure, expression)
     directive = Series(Series(Text("@"), dwsp__), symbol, Series(Text("="), dwsp__), component, ZeroOrMore(Series(Series(Text(","), dwsp__), component)), Lookahead(FOLLOW_UP), mandatory=1)
-    literals = OneOrMore(literal)
     element.set(Alternative(Series(Option(retrieveop), symbol, NegativeLookahead(DEF)), literal, plaintext, regexp, Series(character, dwsp__), any_char, whitespace, group))
     countable.set(Alternative(option, oneormore, element))
     expression.set(Series(sequence, ZeroOrMore(Series(OR, dwsp__, sequence))))

examples/FlexibleEBNF/FlexibleEBNF.ebnf

@@ -61,7 +61,7 @@ FOLLOW_UP  = `@` | symbol | EOF
 
 expression = sequence { :OR~ sequence }
 sequence   = ["§"] ( interleave | lookaround )  # "§" means all following terms mandatory
-             { :AND~ ["§"] ( interleave | lookaround ) }
+             { !`@` !(symbol :DEF) :AND~ ["§"] ( interleave | lookaround ) }
 interleave = difference { "°" ["§"] difference }
 lookaround = flowmarker § (oneormore | pure_elem)
 difference = term ["-" § (oneormore | pure_elem)]

examples/FlexibleEBNF/FlexibleEBNFParser.py

@@ -94,7 +94,7 @@ class FlexibleEBNFGrammar(Grammar):
     countable = Forward()
     element = Forward()
     expression = Forward()
-    source_hash__ = "431992357f565327257002ab0af2018a"
+    source_hash__ = "c76fcc24e5077d4e150b771e6b60f0a1"
     disposable__ = re.compile('component$|pure_elem$|countable$|FOLLOW_UP$|SYM_REGEX$|ANY_SUFFIX$|EOF$')
     static_analysis_pending__ = []  # type: List[bool]
     parser_initialization__ = ["upon instantiation"]
@@ -150,7 +150,7 @@ class FlexibleEBNFGrammar(Grammar):
     difference = Series(term, Option(Series(Series(Text("-"), dwsp__), Alternative(oneormore, pure_elem), mandatory=1)))
     lookaround = Series(flowmarker, Alternative(oneormore, pure_elem), mandatory=1)
     interleave = Series(difference, ZeroOrMore(Series(Series(Text("°"), dwsp__), Option(Series(Text("§"), dwsp__)), difference)))
-    sequence = Series(Option(Series(Text("§"), dwsp__)), Alternative(interleave, lookaround), ZeroOrMore(Series(Retrieve(AND), dwsp__, Option(Series(Text("§"), dwsp__)), Alternative(interleave, lookaround))))
+    sequence = Series(Option(Series(Text("§"), dwsp__)), Alternative(interleave, lookaround), ZeroOrMore(Series(NegativeLookahead(Text("@")), NegativeLookahead(Series(symbol, Retrieve(DEF))), Retrieve(AND), dwsp__, Option(Series(Text("§"), dwsp__)), Alternative(interleave, lookaround))))
     FOLLOW_UP = Alternative(Text("@"), symbol, EOF)
     definition = Series(symbol, Retrieve(DEF), dwsp__, Option(Series(Retrieve(OR), dwsp__)), expression, Retrieve(ENDL), dwsp__, Lookahead(FOLLOW_UP), mandatory=1)
     component = Alternative(literals, procedure, expression)

examples/FlexibleEBNF/tests_grammar/03_test_delimiters.ini

@@ -21,6 +21,7 @@ F1: ' '
 
 
 [match:AND]
+M1: ','
 
 [ast:AND]
 

examples/FlexibleEBNF/tests_grammar/09_test_top-level.ini

@@ -161,6 +161,19 @@ M2: """/* Source: https://www.w3.org/TR/xml/ */
     NotationDecl   ::= '<!NOTATION' S Name S (ExternalID | PublicID) S? '>'	[VC: Unique Notation Name]
     PublicID	   ::= 'PUBLIC' S PubidLiteral"""
 
+M3: '''@ literalws = right
+       @ drop = whitespace   # <- there is no alternative syntax for directives!!!
+       expression ::= term, { ("+" | "-"), term};
+       term       ::= factor, { ("*" | "/"), factor};
+       factor     ::= [sign], (NUMBER | VARIABLE | group), { VARIABLE | group };
+       sign       ::= `+` | `-`;
+       group      ::= "(", expression, ")";
+       NUMBER     ::= /(?:0|(?:[1-9]\\d*))(?:\\.\\d+)?/, ~;
+       VARIABLE   ::= /[A-Za-z]/, ~;'''
+
+M4: '''@ literalws = right
+    @ drop = whitespace  # <- there is no alternative syntax for directives!!!
+    expression = `a`, `b`'''
 
 
 [ast:syntax]
@@ -169,6 +182,8 @@ M2: """/* Source: https://www.w3.org/TR/xml/ */
 
 
 [match:definition]
+M1*: 'expression ::= term, { ("+" | "-"), term};'
+M2: 'expression = `a`, `b`'
 
 [ast:definition]
 

examples/FlexibleEBNF/tst_FlexibleEBNF_grammar.py

@@ -84,7 +84,7 @@ if __name__ == '__main__':
     except ValueError:
         profile = False
     if len(argv) > 1 and argv[1] == "--debug":
-        LOGGING = True
+        LOGGING = "LOGS"
         del argv[1]
     if (len(argv) >= 2 and (argv[1].endswith('.ebnf') or
         os.path.splitext(argv[1])[1].lower() in testing.TEST_READERS.keys())):

tests/test_parse.py

@@ -1050,7 +1050,7 @@ class TestPopRetrieve:
         assert not st.errors
         assert st.equals(st1)
         st = parser('')
-        assert "EOF expected" in str(st.errors)
+        assert "'EOF' expected" in str(st.errors), st.as_sxpr()
 
 
 class TestWhitespaceHandling: