- test for whitespace handling added

DHParser/dsl.py

@@ -257,6 +257,7 @@ def parser_factory(ebnf_src: str, branding="DSL") -> Grammar:
     """
     grammar_src = compileDSL(ebnf_src, nil_scanner, get_ebnf_grammar(),
                              get_ebnf_transformer(), get_ebnf_compiler(branding))
+    print(grammar_src)
     return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_(?:\w+_)?grammar$')
 
 

DHParser/parsers.py

@@ -707,6 +707,15 @@ class RE(Parser):
 
 
 class Token(RE):
+    """Class Token parses simple strings. Any regular regular
+    expression commands will be interpreted as simple sequence of
+    characters.
+
+    Other than that class Token is essentially a renamed version of
+    class RE. Because tokens often have a particular semantic different
+    from other REs, parsing them with a separate parser class allows to
+    distinguish them by their parser type.
+    """
     assert TOKEN_PTYPE == ":Token"
 
     def __init__(self, token: str, wL=None, wR=None, name: str = '') -> None:

test/test_parsers.py

@@ -274,6 +274,32 @@ class TestPopRetrieve:
             syntax_tree.log("test_PopRetrieve_multi_line.cst")
 
 
+class TestWhitespaceHandling:
+    minilang = """@testing = True
+        doc = A B
+        A = "A"
+        B = "B"
+        Rdoc = ar br
+        ar = /A/
+        br = /B/
+        """
+
+    def setup(self):
+        self.gr = parser_factory(self.minilang)()
+
+    def test_token_whitespace(self):
+        st = self.gr("AB", 'doc')
+        assert not st.error_flag
+        st = self.gr("A B", 'doc')
+        assert not st.error_flag
+
+    def test_regexp_whitespace(self):
+        st = self.gr("AB", 'Rdoc')
+        assert not st.error_flag
+        st = self.gr("A B", 'Rdoc')
+        assert st.error_flag
+
+
 if __name__ == "__main__":
     from DHParser.testing import runner
     runner("", globals())