- parse.py: DropWhitespace attached to tokens if DropToken is set

DHParser/ebnf.py

@@ -805,7 +805,7 @@ class EBNFCompiler(Compiler):
 
         # add special fields for Grammar class
 
-        if DROP_WSPC in self.directives.drop:
+        if DROP_WSPC in self.directives.drop or DROP_TOKEN in self.directives.drop:
             definitions.append((EBNFCompiler.DROP_WHITESPACE_PARSER_KEYWORD,
                                 'DropRegExp(%s)' % EBNFCompiler.WHITESPACE_KEYWORD))
         definitions.append((EBNFCompiler.WHITESPACE_PARSER_KEYWORD,
@@ -1340,16 +1340,18 @@ class EBNFCompiler(Compiler):
         return 'Token('
 
 
-    def WSPC_PARSER(self):
-        if DROP_WSPC in self.directives.drop and (self.context[-2].tag_name != "definition"
-                                                  or self.context[-1].tag_name == 'literal'):
+    def WSPC_PARSER(self, force_drop=False):
+        if ((force_drop or DROP_WSPC in self.directives.drop)
+                and (self.context[-2].tag_name != "definition"
+                     or self.context[-1].tag_name == 'literal')):
             return 'dwsp__'
         return 'wsp__'
 
     def on_literal(self, node: Node) -> str:
         center = self.TOKEN_PARSER() + node.content.replace('\\', r'\\') + ')'
-        left = self.WSPC_PARSER() if 'left' in self.directives.literalws else ''
-        right = self.WSPC_PARSER() if 'right' in self.directives.literalws else ''
+        force = DROP_TOKEN in self.directives.drop
+        left = self.WSPC_PARSER(force) if 'left' in self.directives.literalws else ''
+        right = self.WSPC_PARSER(force) if 'right' in self.directives.literalws else ''
         if left or right:
             return 'Series(' + ", ".join(item for item in (left, center, right) if item) + ')'
         return center

DHParser/parse.py

@@ -65,6 +65,7 @@ __all__ = ('Parser',
            'MetaParser',
            'UnaryParser',
            'NaryParser',
+           'Drop',
            'Synonym',
            'Option',
            'ZeroOrMore',
@@ -1371,19 +1372,6 @@ class Token(Parser):
         return ("'%s'" if self.text.find("'") <= 0 else '"%s"') % self.text
 
 
-class DropToken(Token):
-    """
-    Parses play text string, but returns EMPTY_NODE rather than the parsed
-    string on a match. Violates the invariant: str(parse(text)) == text !
-    """
-    def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
-        assert self.anonymous, "DropToken must not be used for named parsers!"
-        if text.startswith(self.text):
-            return EMPTY_NODE, text[self.len:]
-            # return Node(self.tag_name, self.text, True), text[self.len:]
-        return None, text
-
-
 class RegExp(Parser):
     r"""
     Regular expression parser.
@@ -2494,12 +2482,60 @@ class Pop(Retrieve):
 
 ########################################################################
 #
-# Aliasing parser classes
+# Simplifying parser classes
 #
 ########################################################################
 
 
-# TODO: Add a generic Drop-Parser (see DropToken, DropWhitespace)
+class Drop(UnaryParser):
+    r"""
+    Drops any content that another parser yields and returns either
+    None if the other parser did not match or EMPTY_NODE, if it did.
+    This allows to simplify the syntax tree at a very early stage.
+    Violates the invariant: str(parse(text)) == text !
+    """
+    def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
+        node, text = self.parser.parse(text)
+        if node:
+            return EMPTY_NODE, text
+        return None, text
+
+
+class DropToken(Token):
+    """
+    Parses play text string, but returns EMPTY_NODE rather than the parsed
+    string on a match. Violates the invariant: str(parse(text)) == text !
+    """
+    def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
+        assert self.anonymous, "DropToken must not be used for named parsers!"
+        if text.startswith(self.text):
+            return EMPTY_NODE, text[self.len:]
+            # return Node(self.tag_name, self.text, True), text[self.len:]
+        return None, text
+
+
+class DropRegExp(Whitespace):
+    """
+    Parses a text with a regular expression but never returns the match.
+    Instead EMPTY_NODE is returned on a match.
+    Violates the invariant: str(parse(text)) == text !
+    """
+
+    def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
+        assert self.anonymous, "DropWhitespace must not be used for named parsers!"
+        match = text.match(self.regexp)
+        if match:
+            # capture = match.group(0)
+            end = text.index(match.end())
+            return EMPTY_NODE, text[end:]
+        return None, text
+
+
+########################################################################
+#
+# Aliasing parser classes
+#
+########################################################################
 
 
 class Synonym(UnaryParser):