procedural searches and error messages

DHParser/ebnf.py

@@ -900,6 +900,11 @@ class EBNFCompiler(Compiler):
         elif nd.tag_name == 'literal':
             s = nd.content[1:-1]  # remove quotation marks
             return unrepr("re.compile(r'(?=%s)')" % escape_re(s))
+        elif nd.tag_name == 'procedure':
+            return unrepr(nd.content)
+        elif nd.tag_name != 'symbol':
+            self.tree.new_error(nd, 'Only regular expressions, string literals and external '
+                                'procedures are allowed as search rules, but not: ' + nd.tag_name)
         return ''
 
 
@@ -998,12 +1003,16 @@ class EBNFCompiler(Compiler):
                     except IndexError:
                         nd = self.tree
                         refined = ""
+                    except KeyError:
+                        # rule represents a procedure name
+                        refined = rule
                     if refined:
                         refined_rules.append(refined)
                     else:
-                        self.tree.new_error(nd, 'Symbol "%s" cannot be used in resume rule, since'
-                                            ' it represents neither literal nor regexp!',
-                                            INAPPROPRIATE_SYMBOL_FOR_DIRECTIVE)
+                        self.tree.new_error(
+                            nd, 'Symbol "%s" cannot be used in resume rule, since it represents '
+                            'neither literal nor regexp nor procedure!',
+                            INAPPROPRIATE_SYMBOL_FOR_DIRECTIVE)
                 else:
                     refined_rules.append(rule)
             resume_rules[symbol] = refined_rules
@@ -1023,6 +1032,9 @@ class EBNFCompiler(Compiler):
                         search = self._gen_search_rule(nd)
                     except IndexError:
                         search = ''
+                    except KeyError:
+                        # rule represents a procedure name
+                        pass
                 rules.append(search)
             skip_rules[symbol] = rules
         if skip_rules:

DHParser/parse.py

@@ -131,6 +131,7 @@ class ParserError(Exception):
 ResumeList = List[Union[RxPatternType, str, Callable]]  # list of strings or regular expressiones
 ReentryPointAlgorithm = Callable[[StringView, int], Tuple[int, int]]
 # (text, start point) => (reentry point, match length)
+# A return value of (-1, x) means that no reentry point before the end of the document was found
 
 
 @cython.locals(upper_limit=cython.int, closest_match=cython.int, pos=cython.int)

test/test_parse.py

@@ -23,7 +23,7 @@ import copy
 import os
 import sys
 from functools import partial
-from typing import List
+from typing import List, Tuple
 
 scriptpath = os.path.dirname(__file__) or '.'
 sys.path.append(os.path.abspath(os.path.join(scriptpath, '..')))
@@ -39,7 +39,7 @@ from DHParser.parse import ParserError, Parser, Grammar, Forward, TKN, ZeroOrMor
     GrammarError
 from DHParser import compile_source
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
-    parse_ebnf, DHPARSER_IMPORTS
+    parse_ebnf, DHPARSER_IMPORTS, compile_ebnf
 from DHParser.dsl import grammar_provider
 from DHParser.syntaxtree import Node, parse_sxpr
 from DHParser.stringview import StringView
@@ -1009,7 +1009,6 @@ ENDL       = `;` | ``
 EOF = !/./ [:?DEF] [:?OR] [:?AND] [:?ENDL]
 """
 
-
 class TestReentryAfterError:
     testlang = """
     document = alpha [beta] gamma "."
@@ -1106,6 +1105,28 @@ class TestReentryAfterError:
         # there should be only two error messages
         assert len([err for err in cst.errors_sorted if err.code >= 1000]) == 2
 
+    def test_algorithmic_resume(self):
+        lang = r"""
+            document = block_A block_B
+            @ block_A_resume = next_valid_letter()
+            @ block_A_skip = next_valid_letter()
+            block_A = "a" §"b" "c"
+            block_B = "x" "y" "z"
+            """
+        proc = """
+def next_valid_letter(text: StringView, start: int) -> Tuple[int, int]:
+    while start < len(text):
+        if str(text[start]) in 'abcxyz':
+            return start, 1
+    return -1, 0
+"""
+        code, errors, ast = compile_ebnf(lang)
+        code += proc
+
+        tree = gr('ab*xyz')
+        print(tree.as_sxpr())
+
+
     def test_skip_comment_on_resume(self):
         lang = r"""
             @ comment =  /(?:\/\/.*)|(?:\/\*(?:.|\n)*?\*\/)/  # Kommentare im C++-Stil