- better testing support for mandatory lookaheads

DHParser/error.py

@@ -71,19 +71,20 @@ class Error:
 
     # warning codes
 
-    REDEFINED_DIRECTIVE_WARNING              = ErrorCode(101)
-    REDECLARED_TOKEN_WARNING                 = ErrorCode(102)
+    REDEFINED_DIRECTIVE_WARNING              = ErrorCode(110)
+    REDECLARED_TOKEN_WARNING                 = ErrorCode(120)
 
-    UNDEFINED_SYMBOL_IN_TRANSFORMATION_TABLE = ErrorCode(601)
+    UNDEFINED_SYMBOL_IN_TRANSFORMATION_TABLE = ErrorCode(610)
 
     # error codes
 
-    MANDATORY_CONTINUATION                   = ErrorCode(1001)
-    PARSER_DID_NOT_MATCH                     = ErrorCode(1002)
-    PARSER_LOOKAHEAD_MATCH_ONLY              = ErrorCode(1003)
-    PARSER_STOPPED_BEFORE_END                = ErrorCode(1004)
-    CAPTURE_STACK_NOT_EMPTY                  = ErrorCode(1005)
-    MALFORMED_ERROR_STRING                   = ErrorCode(1006)
+    MANDATORY_CONTINUATION                   = ErrorCode(1010)
+    MANDATORY_CONTINUATION_AT_EOF            = ErrorCode(1015)
+    PARSER_DID_NOT_MATCH                     = ErrorCode(1020)
+    PARSER_LOOKAHEAD_MATCH_ONLY              = ErrorCode(1030)
+    PARSER_STOPPED_BEFORE_END                = ErrorCode(1040)
+    CAPTURE_STACK_NOT_EMPTY                  = ErrorCode(1050)
+    MALFORMED_ERROR_STRING                   = ErrorCode(1060)
 
     def __init__(self, message: str, pos, code: ErrorCode = ERROR,
                  orig_pos: int = -1, line: int = -1, column: int = -1) -> None:
@@ -102,7 +103,7 @@ class Error:
         prefix = ''
         if self.line > 0:
             prefix = "%i:%i: " % (max(self.line, 0), max(self.column, 0))
-        return prefix + "%s: %s" % (self.severity, self.message)
+        return prefix + "%s (%i): %s" % (self.severity, self.code, self.message)
 
     def __repr__(self):
         return 'Error("%s", %s, %i, %i, %i, %i)' \

DHParser/parse.py

@@ -1434,7 +1434,9 @@ class Series(NaryOperator):
                                 self.grammar.tree__.add_error(node, error)
                     else:
                         msg = '%s expected, "%s" found!' % (parser.repr, found)
-                    mandatory_violation = Error(msg, location, Error.MANDATORY_CONTINUATION)
+                    mandatory_violation = Error(msg, location,
+                                                Error.MANDATORY_CONTINUATION if text_
+                                                else Error.MANDATORY_CONTINUATION_AT_EOF)
                     self.grammar.tree__.add_error(node, mandatory_violation)
                     text_ = text_[i:]
                     results += (node,)

DHParser/testing.py

@@ -334,16 +334,22 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
             return True
         return False
 
-    def lookahead_artifact(raw_errors):
+    def lookahead_artifact(parser, raw_errors):
         """
-        Returns True, if the error merely occured, because the parser
+        Returns True, if the error merely occurred, because the parser
         stopped in front of a sequence that was captured by a lookahead
-        operator. This is required for testing of parsers that put a
-        lookahead operator at the end. See test_testing.TestLookahead.
+        operator or if a mandatory lookahead failed at the end of data.
+        This is required for testing of parsers that put a lookahead
+        operator at the end. See test_testing.TestLookahead.
         """
-        return len(raw_errors) == 2 \
-            and raw_errors[-1].code == Error.PARSER_LOOKAHEAD_MATCH_ONLY \
-            and raw_errors[-2].code == Error.PARSER_STOPPED_BEFORE_END
+        return ((len(raw_errors) == 2  # case 1:  superfluous data for lookahead
+                 and raw_errors[-1].code == Error.PARSER_LOOKAHEAD_MATCH_ONLY
+                 and raw_errors[-2].code == Error.PARSER_STOPPED_BEFORE_END)
+                #  case 2:  mandatory lookahead failure at end of text
+                or (len(raw_errors) == 1
+                    and raw_errors[-1].code == Error.MANDATORY_CONTINUATION_AT_EOF)
+                    and any(isinstance(parser, Lookahead)
+                            for parser in parser.history__[-1].call_stack))
 
     for parser_name, tests in test_unit.items():
         assert parser_name, "Missing parser name in test %s!" % unit_name
@@ -374,7 +380,6 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
                 infostr = '    match-test "' + test_name + '" ... '
                 errflag = len(errata)
             try:
-
                 cst = parser(test_code, parser_name, track_history=has_lookahead(parser_name))
             except UnknownParserError as upe:
                 cst = RootNode()
@@ -388,7 +393,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
                 tests.setdefault('__ast__', {})[test_name] = ast
                 # log_ST(ast, "match_%s_%s.ast" % (parser_name, clean_test_name))
             raw_errors = cst.collect_errors()
-            if is_error(cst.error_flag) and not lookahead_artifact(raw_errors):
+            if is_error(cst.error_flag) and not lookahead_artifact(parser, raw_errors):
                 errors = adjust_error_locations(raw_errors, test_code)
                 errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
                               (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
@@ -430,7 +435,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
                 cst = RootNode(node).new_error(node, str(upe))
                 errata.append('Unknown parser "{}" in fail test "{}"!'.format(parser_name, test_name))
                 tests.setdefault('__err__', {})[test_name] = errata[-1]
-            if not is_error(cst.error_flag) and not lookahead_artifact(cst.collect_errors()):
+            if not is_error(cst.error_flag) and not lookahead_artifact(parser, cst.collect_errors()):
                 errata.append('Fail test "%s" for parser "%s" yields match instead of '
                               'expected failure!' % (test_name, parser_name))
                 tests.setdefault('__err__', {})[test_name] = errata[-1]

test/test_testing.py

@@ -253,7 +253,7 @@ class TestLookahead:
     """
     EBNF = r"""
         document = { category | entry } { LF }
-        category = {LF } sequence_of_letters { /:/ sequence_of_letters } /:/ &(LF sequence_of_letters) 
+        category = {LF } sequence_of_letters { /:/ sequence_of_letters } /:/ §&(LF sequence_of_letters) 
         entry = { LF } sequence_of_letters !/:/
         sequence_of_letters = /[A-Za-z0-9 ]+/
         LF = / *\n/
@@ -263,7 +263,8 @@ class TestLookahead:
         "category": {
             "match": {
                 1: """Mountains: big:
-                          K2"""
+                          K2""",
+                2: """Rivers:"""  # allowed because lookahaead failure occurs at end of file and is mandatory!
             },
             "fail": {
                 6: """Mountains: big:"""