- parser.py: allOf and someOf-parsers + unit tests implemented

DHParser/parser.py

@@ -91,6 +91,8 @@ __all__ = ('PreprocessorFunc',
            'OneOrMore',
            'Series',
            'Alternative',
+           'AllOf',
+           'SomeOf',
            'FlowOperator',
            'Required',
            'Lookahead',
@@ -1535,7 +1537,7 @@ class Alternative(NaryOperator):
 
 class AllOf(NaryOperator):
     """
-    Matches if all elements of a set of parsers match. Each parser must
+    Matches if all elements of a list of parsers match. Each parser must
     match exactly once. Other than in a sequence, the order in which
     the parsers match is arbitrary, however.
 
@@ -1562,15 +1564,14 @@ class AllOf(NaryOperator):
     def __call__(self, text: StringView) -> Tuple[Node, StringView]:
         results = ()  # type: Tuple[Node, ...]
         text_ = text  # type: StringView
-        pset = set(self.parsers)  # type: Set[Parser]
-        while pset:
-            # TODO: Ordnung berücksichtigen, kein SET verwenden!
-            for parser in pset:
+        parsers = list(self.parsers)  # type: List[Parser]
+        while parsers:
+            for i, parser in enumerate(parsers):
                 node, text__ = parser(text_)
                 if node:
                     results += (node,)
                     text_ = text__
-                    pset.remove(parser)
+                    del parsers[i]
                     break
             else:
                 return None, text
@@ -1583,7 +1584,7 @@ class AllOf(NaryOperator):
 
 class SomeOf(NaryOperator):
     """
-    Matches if at least one element of a set of parsers match. No parser
+    Matches if at least one element of a list of parsers match. No parser
     must match more than once . Other than in a sequence, the order in which
     the parsers match is arbitrary, however.
 
@@ -1611,18 +1612,17 @@ class SomeOf(NaryOperator):
     def __call__(self, text: StringView) -> Tuple[Node, StringView]:
         results = ()  # type: Tuple[Node, ...]
         text_ = text  # type: StringView
-        pset = set(self.parsers)  # type: Set[Parser]
-        while pset:
-            # TODO: Ordnung berücksichtigen, kein Set verwenden!!!
-            for parser in pset:
+        parsers = list(self.parsers)  # type: List[Parser]
+        while parsers:
+            for i, parser in enumerate(parsers):
                 node, text__ = parser(text_)
                 if node:
                     results += (node,)
                     text_ = text__
-                    pset.remove(parser)
+                    del parsers[i]
                     break
             else:
-                pset = set()
+                parsers = []
         assert len(results) <= len(self.parsers)
         if results:
             return Node(self, results), text_

DHParser/syntaxtree.py

@@ -305,8 +305,8 @@ class Node(collections.abc.Sized):
         self.error_flag = max(self.error_flag, self._errors[-1].level)
         return self
 
-    def collect_errors(self, document: Union[StringView, str] = '', clear_errors=False) -> List[
-        Error]:
+    def collect_errors(self, document: Union[StringView, str] = '', clear_errors=False) \
+            -> List[Error]:
         """
         Recursively adds line- and column-numbers to all error objects.
         Returns all errors of this node or any child node in the form

examples/EBNF/EBNF.ebnf

@@ -24,7 +24,7 @@ flowmarker =  "!"  | "&"                         # '!' negative lookahead, '&' p
 retrieveop =  "::" | ":"                         # '::' pop, ':' retrieve
 
 group      =  "(" §expression ")"
-set        =  "<" §expression ">"
+unordered  =  "<" §expression ">"                # elements of expression in arbitrary order
 oneormore  =  "{" expression "}+"
 repetition =  "{" §expression "}"
 option     =  "[" §expression "]"

test/test_parser.py

@@ -28,7 +28,7 @@ from DHParser.toolkit import is_logging, logging, compile_python_object
 from DHParser.stringview import StringView
 from DHParser.error import Error
 from DHParser.parser import compile_source, Retrieve, Grammar, Forward, Token, ZeroOrMore, RE, \
-    RegExp, Lookbehind, NegativeLookahead, OneOrMore, Series, Alternative
+    RegExp, Lookbehind, NegativeLookahead, OneOrMore, Series, Alternative, AllOf, SomeOf
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
 from DHParser.dsl import grammar_provider, DHPARSER_IMPORTS
 
@@ -308,6 +308,51 @@ class TestSeries:
         assert st.collect_errors()[0].code == Error.MANDATORY_CONTINUATION
 
 
+class TestAllOfSomeOf:
+    def test_allOf_order(self):
+        """Test that parsers of an AllOf-List can match in arbitrary order."""
+        prefixes = AllOf(Token("A"), Token("B"))
+        assert Grammar(prefixes)('A B').content() == 'A B'
+        assert Grammar(prefixes)('B A').content() == 'B A'
+        # aternative Form
+        prefixes = AllOf(Series(Token("B"), Token("A")))
+        assert Grammar(prefixes)('A B').content() == 'A B'
+
+    def test_allOf_completeness(self):
+        """Test that an error is raised if not  all parsers of an AllOf-List
+        match."""
+        prefixes = AllOf(Token("A"), Token("B"))
+        assert Grammar(prefixes)('B').error_flag
+
+    def test_allOf_redundance(self):
+        """Test that one and the same parser may be listed several times
+        and must be matched several times accordingly."""
+        prefixes = AllOf(Token("A"), Token("B"), Token("A"))
+        assert Grammar(prefixes)('A A B').content() == 'A A B'
+        assert Grammar(prefixes)('A B A').content() == 'A B A'
+        assert Grammar(prefixes)('B A A').content() == 'B A A'
+        assert Grammar(prefixes)('A B B').error_flag
+
+    def test_someOf_order(self):
+        """Test that parsers of an AllOf-List can match in arbitrary order."""
+        prefixes = SomeOf(Token("A"), Token("B"))
+        assert Grammar(prefixes)('A B').content() == 'A B'
+        assert Grammar(prefixes)('B A').content() == 'B A'
+        # aternative Form
+        prefixes = SomeOf(Alternative(Token("B"), Token("A")))
+        assert Grammar(prefixes)('A B').content() == 'A B'
+        assert Grammar(prefixes)('B').content() == 'B'
+
+    def test_someOf_redundance(self):
+        """Test that one and the same parser may be listed several times
+        and must be matched several times accordingly."""
+        prefixes = SomeOf(Token("A"), Token("B"), Token("A"))
+        assert Grammar(prefixes)('A A B').content() == 'A A B'
+        assert Grammar(prefixes)('A B A').content() == 'A B A'
+        assert Grammar(prefixes)('B A A').content() == 'B A A'
+        assert Grammar(prefixes)('A B B').error_flag
+
+
 class TestPopRetrieve:
     mini_language = """
         document       = { text | codeblock }