parse.py: Refactoring of Capture/Pop/Retrieve: plugabble match-functions...

parse.py: Refactoring of Capture/Pop/Retrieve: plugabble match-functions instead of filter functions; optional_last_value() match function added; new EBNF-operator ":?" that matches empty-text but still pops the last captured value

DHParser/ebnf.py

@@ -99,7 +99,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \\
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \\
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \\
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, remove_if, \\
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, remove_if, \\
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \\
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \\
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
@@ -113,7 +113,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     chain, get_config_value, XML_SERIALIZATION, SXPRESSION_SERIALIZATION, \\
     COMPACT_SERIALIZATION, JSON_SERIALIZATION, access_thread_locals, access_presets, \\
     finalize_presets, ErrorCode, RX_NEVER_MATCH, set_tracer, resume_notices_on, \\
-    trace_history, has_descendant, neg, has_parent
+    trace_history, has_descendant, neg, has_parent, optional_last_value
 '''
 
 
@@ -179,7 +179,7 @@ class EBNFGrammar(Grammar):
 
     flowmarker = "!"  | "&"                 # '!' negative lookahead, '&' positive lookahead
                | "-!" | "-&"                # '-' negative lookbehind, '-&' positive lookbehind
-    retrieveop = "::" | ":"                 # '::' pop, ':' retrieve
+    retrieveop = "::" | ":?" | ":"          # '::' pop, ':?' optional pop, ':' retrieve
 
     #: groups
 
@@ -222,7 +222,7 @@ class EBNFGrammar(Grammar):
     unordered = Series(Series(Token("<"), wsp__), expression, Series(Token(">"), wsp__),
                        mandatory=1)
     group = Series(Series(Token("("), wsp__), expression, Series(Token(")"), wsp__), mandatory=1)
-    retrieveop = Alternative(Series(Token("::"), wsp__), Series(Token(":"), wsp__))
+    retrieveop = Alternative(Series(Token("::"), wsp__), Series(Token(":?"), wsp__), Series(Token(":"), wsp__))
     flowmarker = Alternative(Series(Token("!"), wsp__), Series(Token("&"), wsp__),
                              Series(Token("-!"), wsp__), Series(Token("-&"), wsp__))
     term = Alternative(Series(Option(flowmarker), Option(retrieveop), symbol,
@@ -651,7 +651,7 @@ class EBNFDirectives:
 
         tokens:  set of the names of preprocessor tokens
 
-        filter:  mapping of symbols to python filter functions that
+        filter:  mapping of symbols to python match functions that
                 will be called on any retrieve / pop - operations on
                 these symbols
 
@@ -857,7 +857,7 @@ class EBNFCompiler(Compiler):
     PREFIX_TABLE = {'§': 'Required',
                     '&': 'Lookahead', '!': 'NegativeLookahead',
                     '-&': 'Lookbehind', '-!': 'NegativeLookbehind',
-                    '::': 'Pop', ':': 'Retrieve'}
+                    '::': 'Pop', ':?': 'Pop', ':': 'Retrieve'}
     REPEATABLE_DIRECTIVES = {'tokens'}
 
 
@@ -1604,7 +1604,7 @@ class EBNFCompiler(Compiler):
         prefix = node.children[0].content
         custom_args = []  # type: List[str]
 
-        if prefix in {'::', ':'}:
+        if prefix in {'::', ':?', ':'}:
             assert len(node.children) == 2
             arg = node.children[-1]
             if arg.tag_name != 'symbol':
@@ -1616,8 +1616,15 @@ class EBNFCompiler(Compiler):
                     node, ('Retrive operator "%s" does not work with anonymous parsers like %s')
                     % (prefix, arg.content))
                 return arg.content
+
+            match_func = 'last_value'
             if arg.content in self.directives.filter:
-                custom_args = ['rfilter=%s' % self.directives.filter[arg.content]]
+                match_func = self.directives.filter[arg.content]
+            if prefix.endswith('?'):
+                match_func = 'optional_' + match_func
+            if match_func != 'last_value':
+                custom_args = ['match_func=%s' % match_func]
+
             self.variables.add(str(arg))  # cast(str, arg.result)
 
         elif len(node.children) > 2:

DHParser/parse.py

@@ -85,8 +85,8 @@ __all__ = ('ParserError',
            'Lookbehind',
            'NegativeLookbehind',
            'last_value',
-           'counterpart',
-           'accumulate',
+           'optional_last_value',
+           'matching_bracket',
            'Capture',
            'Retrieve',
            'Pop',
@@ -375,7 +375,7 @@ class Parser:
         try:
             # rollback variable changing operation if parser backtracks
             # to a position before the variable changing operation occurred
-            if grammar.last_rb__loc__ >= location:
+            if grammar.last_rb__loc__ > location:
                 grammar.rollback_to__(location)
 
             # if location has already been visited by the current parser, return saved result
@@ -1287,12 +1287,14 @@ class Grammar:
         self.rollback__.append((location, func))
         self.last_rb__loc__ = location
 
+
     @property
     def document_lbreaks__(self) -> List[int]:
         if not self._document_lbreaks__:
             self._document_lbreaks__ = linebreaks(self.document__)
         return self._document_lbreaks__
 
+
     def rollback_to__(self, location):
         """
         Rolls back the variable stacks (`self.variables`) to its
@@ -2547,26 +2549,38 @@ class Capture(UnaryParser):
         return self.parser.repr
 
 
-RetrieveFilter = Callable[[List[str]], str]
+MatchVariableFunc = Callable[[Union[StringView, str], List[str]], Optional[str]]
+# (text, stack) -> value, where:
+# text is the following text for be parsed
+# stack is a stack of stored variables (for a particular symbol)
+# and the return value is the matched text (which can be the empty string) or
+# None, if no match occurred
 
 
-def last_value(stack: List[str]) -> str:
-    """Returns the last value on the cpature stack. This is the default case
-    when retrieving cpatured substrings."""
-    return stack[-1]
+def last_value(text: Union[StringView, str], stack: List[str]) -> str:
+    """Matches `text` with the most recent value on the capture stack.
+    This is the default case when retrieving captured substrings."""
+    value = stack[-1]
+    return value if text.startswith(value) else None
 
 
-def counterpart(stack: List[str]) -> str:
-    """Returns a closing bracket for the opening bracket on the capture stack,
-    i.e. if "[" was captured, "]" will be retrieved."""
+def optional_last_value(text: Union[StringView, str], stack: List[str]) -> str:
+    """Matches `text` with the most recent value on the capture stack or
+    with the empty string, i.e. `optional_match` never returns `None` but
+    either the value on the stack or the empty string.
+    Use case: Implement shorthand notation for matching tags, i.e.:
+        Good Morning, Mrs. <emph>Smith</>!
+    """
     value = stack[-1]
-    return value.replace("(", ")").replace("[", "]").replace("{", "}").replace("<", ">")
+    return value if text.startswith(value) else ""
 
 
-def accumulate(stack: List[str]) -> str:
-    """Returns an accumulation of all values on the stack.
-    By the way: I cannot remember any reasonable use case for this!?"""
-    return "".join(stack) if len(stack) > 1 else stack[-1]  # provoke IndexError if stack empty
+def matching_bracket(text: Union[StringView, str], stack: List[str]) -> str:
+    """Returns a closing bracket for the opening bracket on the capture stack,
+    i.e. if "[" was captured, "]" will be retrieved."""
+    value = stack[-1]
+    value = value.replace("(", ")").replace("[", "]").replace("{", "}").replace("<", ">")
+    return value if text.startswith(value) else None
 
 
 class Retrieve(Parser):
@@ -2582,19 +2596,19 @@ class Retrieve(Parser):
     Attributes:
         symbol: The parser that has stored the value to be retrieved, in
             other words: "the observed parser"
-        rfilter: a procedure that through which the processing to the
+        match_func: a procedure that through which the processing to the
             retrieved symbols is channeled. In the simplest case it merely
             returns the last string stored by the observed parser. This can
             be (mis-)used to execute any kind of semantic action.
     """
 
-    def __init__(self, symbol: Parser, rfilter: RetrieveFilter = None) -> None:
+    def __init__(self, symbol: Parser, match_func: MatchVariableFunc = None) -> None:
         super(Retrieve, self).__init__()
         self.symbol = symbol
-        self.filter = rfilter if rfilter else last_value
+        self.match = match_func if match_func else last_value
 
     def __deepcopy__(self, memo):
-        duplicate = self.__class__(self.symbol, self.filter)
+        duplicate = self.__class__(self.symbol, self.match)
         copy_parser_attrs(self, duplicate)
         return duplicate
 
@@ -2608,25 +2622,24 @@ class Retrieve(Parser):
 
     def retrieve_and_match(self, text: StringView) -> Tuple[Optional[Node], StringView]:
         """
-        Retrieves variable from stack through the filter function passed to
+        Retrieves variable from stack through the match function passed to
         the class' constructor and tries to match the variable's value with
         the following text. Returns a Node containing the value or `None`
         accordingly.
         """
         try:
             stack = self.grammar.variables__[self.symbol.pname]
-            value = self.filter(stack)
+            value = self.match(text, stack)
         except (KeyError, IndexError):
             node = Node(self.tag_name, '').with_pos(self.grammar.document_length__ - text.__len__())
             self.grammar.tree__.new_error(
                 node, dsl_error_msg(self, "'%s' undefined or exhausted." % self.symbol.pname))
             return node, text
-        if text.startswith(value):
-            if self.drop_content:
-                return EMPTY_NODE, text[len(value):]
-            return Node(self.tag_name, value), text[len(value):]
-        else:
+        if value is None:
             return None, text
+        elif self.drop_content:
+            return EMPTY_NODE, text[len(value):]
+        return Node(self.tag_name, value), text[len(value):]
 
 
 class Pop(Retrieve):
@@ -2640,21 +2653,21 @@ class Pop(Retrieve):
     The constructor parameter `symbol` determines which variable is
     used.
     """
-    def __init__(self, symbol: Parser, rfilter: RetrieveFilter = None) -> None:
-        super(Pop, self).__init__(symbol, rfilter)
+    def __init__(self, symbol: Parser, match_func: MatchVariableFunc = None) -> None:
+        super(Pop, self).__init__(symbol, match_func)
 
     def reset(self):
         super(Pop, self).reset()
         self.values = []
 
     def __deepcopy__(self, memo):
-        duplicate = self.__class__(self.symbol, self.filter)
+        duplicate = self.__class__(self.symbol, self.match)
         copy_parser_attrs(self, duplicate)
         duplicate.values = self.values[:]
         return duplicate
 
     def _rollback(self):
-        return self.grammar.variables__[self.symbol.pname].append(self.values.pop())
+        self.grammar.variables__[self.symbol.pname].append(self.values.pop())
 
     def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
         node, txt = self.retrieve_and_match(text)
@@ -2665,7 +2678,7 @@ class Pop(Retrieve):
         return node, txt
 
     def __repr__(self):
-        return '::' + self.symbol.repr
+        return ':?' + self.symbol.repr if self.match.__name__.startswith('optional_') else '::'
 
 
 ########################################################################

examples/Arithmetic/ArithmeticParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, is_filename, load_if_file, \
     Lookbehind, Lookahead, Alternative, Pop, Token, Drop, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, accumulate, PreprocessorFunc, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/ArithmeticRightRecursive/ArithmeticRightRecursiveParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, is_filename, load_if_file, \
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, accumulate, PreprocessorFunc, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/ArithmeticSimple/ArithmeticRightRecursiveParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, is_filename, load_if_file, \
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, accumulate, PreprocessorFunc, is_empty, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/ArithmeticSimple/ArithmeticSimpleParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, is_filename, load_if_file, \
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, accumulate, PreprocessorFunc, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/BibTeX/BibTeXParser.py

@@ -24,7 +24,7 @@ from DHParser import is_filename, load_if_file, \
     Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym, \
     Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
-    last_value, counterpart, accumulate, PreprocessorFunc, \
+    last_value, matching_bracket, PreprocessorFunc, \
     Node, TransformationDict, Whitespace, \
     traverse, remove_children_if, is_anonymous, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/EBNF/ClassicEBNFParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, remove_if, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, remove_if, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
@@ -88,9 +88,9 @@ class ClassicEBNFGrammar(Grammar):
     group = Series(Series(Token("("), dwsp__), expression, Series(Token(")"), dwsp__), mandatory=1)
     retrieveop = Alternative(Series(Token("::"), dwsp__), Series(Token(":"), dwsp__))
     flowmarker = Alternative(Series(Token("!"), dwsp__), Series(Token("&"), dwsp__), Series(Token("-!"), dwsp__), Series(Token("-&"), dwsp__))
-    term = Alternative(Series(Option(flowmarker), Option(retrieveop), symbol, NegativeLookahead(Series(Token("="), dwsp__))), Series(Option(flowmarker), literal), Series(Option(flowmarker), plaintext), Series(Option(flowmarker), regexp), Series(Option(flowmarker), whitespace), Series(Option(flowmarker), oneormore), Series(Option(flowmarker), group), Series(Option(flowmarker), unordered), repetition, option)
-    sequence = OneOrMore(Series(Option(Series(Token("§"), dwsp__)), term))
-    expression.set(Series(sequence, ZeroOrMore(Series(Series(Token("|"), dwsp__), sequence))))
+    factor = Alternative(Series(Option(flowmarker), Option(retrieveop), symbol, NegativeLookahead(Series(Token("="), dwsp__))), Series(Option(flowmarker), literal), Series(Option(flowmarker), plaintext), Series(Option(flowmarker), regexp), Series(Option(flowmarker), whitespace), Series(Option(flowmarker), oneormore), Series(Option(flowmarker), group), Series(Option(flowmarker), unordered), repetition, option)
+    term = OneOrMore(Series(Option(Series(Token("§"), dwsp__)), factor))
+    expression.set(Series(term, ZeroOrMore(Series(Series(Token("|"), dwsp__), term))))
     directive = Series(Series(Token("@"), dwsp__), symbol, Series(Token("="), dwsp__), Alternative(regexp, literal, symbol), ZeroOrMore(Series(Series(Token(","), dwsp__), Alternative(regexp, literal, symbol))), mandatory=1)
     definition = Series(symbol, Series(Token("="), dwsp__), expression, mandatory=1)
     syntax = Series(Option(Series(dwsp__, RegExp(''))), ZeroOrMore(Alternative(definition, directive)), EOF, mandatory=2)

examples/EBNF/EBNF.ebnf

@@ -30,7 +30,7 @@ term       = [flowmarker] [retrieveop] symbol !"="   # negative lookahead to be
 
 flowmarker = "!"  | "&"                         # '!' negative lookahead, '&' positive lookahead
            | "-!" | "-&"                        # '-' negative lookbehind, '-&' positive lookbehind
-retrieveop = "::" | ":"                         # '::' pop, ':' retrieve
+retrieveop = "::" | ":?" | ":"                  # '::' pop, ':?' optional pop, ':' retrieve
 
 #: groups
 

examples/EBNF/EBNFParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     Lookbehind, Lookahead, Alternative, Pop, Token, Drop, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, remove_if, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, remove_if, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
@@ -63,7 +63,7 @@ class EBNFGrammar(Grammar):
     r"""Parser for an EBNF source file.
     """
     expression = Forward()
-    source_hash__ = "af60e00f0f916559f98f041221c8127d"
+    source_hash__ = "4a574f623ce70c32a47e8f742b6948e2"
     anonymous__ = re.compile('..(?<=^)')
     static_analysis_pending__ = [True]
     parser_initialization__ = ["upon instantiation"]
@@ -84,7 +84,7 @@ class EBNFGrammar(Grammar):
     oneormore = Series(Series(Token("{"), dwsp__), expression, Series(Token("}+"), dwsp__))
     unordered = Series(Series(Token("<"), dwsp__), expression, Series(Token(">"), dwsp__), mandatory=1)
     group = Series(Series(Token("("), dwsp__), expression, Series(Token(")"), dwsp__), mandatory=1)
-    retrieveop = Alternative(Series(Token("::"), dwsp__), Series(Token(":"), dwsp__))
+    retrieveop = Alternative(Series(Token("::"), dwsp__), Series(Token(":?"), dwsp__), Series(Token(":"), dwsp__))
     flowmarker = Alternative(Series(Token("!"), dwsp__), Series(Token("&"), dwsp__), Series(Token("-!"), dwsp__), Series(Token("-&"), dwsp__))
     term = Alternative(Series(Option(flowmarker), Option(retrieveop), symbol, NegativeLookahead(Series(Token("="), dwsp__))), Series(Option(flowmarker), literal), Series(Option(flowmarker), plaintext), Series(Option(flowmarker), regexp), Series(Option(flowmarker), whitespace), Series(Option(flowmarker), oneormore), Series(Option(flowmarker), group), Series(Option(flowmarker), unordered), repetition, option)
     sequence = OneOrMore(Series(Option(Series(Token("§"), dwsp__)), term))

examples/EBNF/EBNFServer.py

@@ -177,7 +177,7 @@ class EBNFLanguageServerProtocol:
     #                 'will be emitted instead of the stock message, if a mandatory element '
     #                 'violation occured within the given parser. (DHParser-extension to EBNF)'],
     #                ['_filter', '@ ${1:SYMBOL}_filter = ${2:funcname}', 2,
-    #                 'Name of a Python-filter-function that is applied when retrieving a stored '
+    #                 'Name of a Python-match-function that is applied when retrieving a stored '
     #                 'symbol. (DHParser-extension to EBNF)']]
 
 

examples/EBNF/ModernEBNFParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, remove_if, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, remove_if, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/Tutorial/Lyrik_explicit_whitespaceParser.py

@@ -24,7 +24,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     Lookbehind, Lookahead, Alternative, Pop, Token, DropToken, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, remove_if, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, remove_if, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/XML/XMLParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, remove_if, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, remove_if, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/XMLSnippet/XMLSnippetParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, remove_if, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, remove_if, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/json/jsonParser.py

@@ -24,7 +24,7 @@ from DHParser import start_logging, is_filename, load_if_file, \
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/json/json_fail_tolerantParser.py

@@ -24,7 +24,7 @@ from DHParser import start_logging, suspend_logging, resume_logging, is_filename
     Lookbehind, Lookahead, Alternative, Pop, Token, DropToken, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, remove_if, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, remove_if, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

examples/yaml/yamlParser.py

@@ -25,7 +25,7 @@ from DHParser import start_logging, is_filename, load_if_file, \
     Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
     Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
-    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, \
+    grammar_changed, last_value, matching_bracket, PreprocessorFunc, is_empty, \
     Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
     remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
     reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \

test/test_parse.py

@@ -608,7 +608,7 @@ class TestPopRetrieve:
         text           = /[^`]+/
         """
     mini_lang2 = r"""
-        @braces_filter=counterpart
+        @braces_filter=matching_bracket
         document       = { text | codeblock }
         codeblock      = braces { text | opening_braces | (!:braces closing_braces) } ::braces
         braces         = opening_braces
@@ -628,11 +628,20 @@ class TestPopRetrieve:
         name           = /\w+/~
         text           = /[^<>]+/
         """
+    mini_lang4 = r"""
+        document       = { text | env }
+        env            = opentag document closetag
+        opentag        = "<" name ">"
+        closetag       = "</" :?name ">"
+        name           = /\w+/~
+        text           = /[^<>]+/        
+    """
 
     def setup(self):
         self.minilang_parser = grammar_provider(self.mini_language)()
         self.minilang_parser2 = grammar_provider(self.mini_lang2)()
         self.minilang_parser3 = grammar_provider(self.mini_lang3)()
+        self.minilang_parser4 = grammar_provider(self.mini_lang4)()
 
     @staticmethod
     def opening_delimiter(node, name):
@@ -669,6 +678,34 @@ class TestPopRetrieve:
         syntax_tree = self.minilang_parser3(proper)
         assert not syntax_tree.error_flag, str(syntax_tree.errors_sorted)
 
+    def test_optional_match(self):
+        test1 = '<info>Hey, you</info>'
+        st = self.minilang_parser4(test1)
+        assert not st.error_flag
+        test12 = '<info>Hey, <emph>you</emph></info>'
+        st = self.minilang_parser4(test1)
+        assert not st.error_flag
+        test2 = '<info>Hey, you</>'
+        st = self.minilang_parser4(test2)
+        assert not st.error_flag
+        test3 = '<info>Hey, <emph>you</></>'
+        st = self.minilang_parser4(test3)
+        assert not st.error_flag
+        test4 = '<info>Hey, <emph>you</></info>'
+        st = self.minilang_parser4(test4)
+        assert not st.error_flag
+
+    def test_rollback_behaviour_of_optional_match(self):
+        test1 = '<info>Hey, you</info*>'
+        st = self.minilang_parser4(test1)
+        assert not self.minilang_parser4.variables__['name']
+        assert st.error_flag
+        test2 = '<info>Hey, you</*>'
+        st = self.minilang_parser4(test2)
+        assert not self.minilang_parser4.variables__['name']
+        assert st.error_flag
+
+
     def test_cache_neutrality(self):
         """Test that packrat-caching does not interfere with the variable-
         changing parsers: Capture and Retrieve."""