better dsl compiling

DHParser/dsl.py

@@ -207,9 +207,17 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
         which conforms to the language defined by ``ebnf_src``.
     """
     grammar = ebnf_grammar_obj or get_ebnf_grammar()
-    grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, get_ebnf_compiler())
-    return grammar_src if source_only else \
-        compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
+    compiler = get_ebnf_compiler("DSL")
+    grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler)
+    if source_only:
+        src = [SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_IMPORTS,
+               SECTION_MARKER.format(marker=SCANNER_SECTION), compiler.gen_scanner_skeleton(),
+               SECTION_MARKER.format(marker=PARSER_SECTION), grammar_src,
+               SECTION_MARKER.format(marker=AST_SECTION), compiler.gen_transformer_skeleton(),
+               SECTION_MARKER.format(marker=COMPILER_SECTION), compiler.gen_compiler_skeleton()]
+        return '\n'.join(src)
+    else:
+        return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$')
 
 
 def load_compiler_suite(compiler_suite):
@@ -345,7 +353,8 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
 
     elif trans == get_ebnf_transformer or trans == EBNFTransformer:  # either an EBNF- or no compiler suite given
         global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
-            AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER
+            AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
+            DHPARSER_MAIN, DHPARSER_IMPORTS
         f = None
         try:
             f = open(rootname + '_compiler.py', 'r', encoding="utf-8")

DHParser/ebnf.py

@@ -192,9 +192,9 @@ EBNF_transformation_table = {
         remove_expendables,
     "directive, definition":
         partial(remove_tokens, tokens={'@', '='}),
-    "expression, chain":
+    "expression":
         [replace_by_single_child, flatten,
-         partial(remove_tokens, tokens={'|', '--'})],
+         partial(remove_tokens, tokens={'|'})],
     "term":
         [replace_by_single_child, flatten],  # supports both idioms:  "{ factor }+" and "factor { factor }"
     "factor, flowmarker, retrieveop":

DHParser/parsers.py

@@ -1081,8 +1081,10 @@ def compile_source(source, scanner, parser, transformer, compiler):
     return result, messages, syntax_tree
 
 
-def test_grammar(test_suite, parser, transform):
+def test_grammar(test_suite, parser_factory, transformer_factory):
     errata = []
+    parser = parser_factory()
+    transform = transformer_factory()
     for parser_name, tests in test_suite.items():
         assert set(tests.keys()).issubset({'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'})
 

test/test_dsl.py

@@ -24,7 +24,22 @@ import os
 import sys
 sys.path.extend(['../', './'])
 
-from DHParser.dsl import compile_on_disk, run_compiler
+from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF
+
+
+ARITHMETIC_EBNF = """
+    @ whitespace = linefeed
+    formula = [ //~ ] expr
+    expr = expr ("+"|"-") term | term
+    term = term ("*"|"/") factor | factor
+    factor = /[0-9]+/~
+    # example:  "5 + 3 * 4"
+    """
+
+class TestCompileFunctions:
+    def test_compileEBNF(self):
+        parser_src = compileEBNF(ARITHMETIC_EBNF, source_only=True)
+        print(parser_src)
 
 class TestCompilerGeneration:
     trivial_lang = """
@@ -81,4 +96,4 @@ class TestCompilerGeneration:
 
 if __name__ == "__main__":
     from run import runner
-    runner("", globals())
\ No newline at end of file
+    runner("TestCompileFunctions", globals())
\ No newline at end of file

test/test_parsers.py

@@ -19,25 +19,41 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+from functools import partial
 import sys
 sys.path.extend(['../', './'])
 
 from DHParser.toolkit import is_logging, compile_python_object
+from DHParser.syntaxtree import no_operation, traverse
 from DHParser.parsers import compile_source
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
 from DHParser.dsl import compileEBNF, DHPARSER_IMPORTS
 
 
+ARITHMETIC_EBNF = """
+    @ whitespace = linefeed
+    formula = [ //~ ] expr
+    expr = expr ("+"|"-") term | term
+    term = term ("*"|"/") factor | factor
+    factor = /[0-9]+/~
+    # example:  "5 + 3 * 4"
+    """
+
+ARITHMETIC_EBNF_transformation_table = {
+    # AST Transformations for the DSL-grammar
+    "formula": no_operation,
+    "expr": no_operation,
+    "term": no_operation,
+    "factor": no_operation,
+    "": no_operation
+}
+
+ARITHMETIC_EBNFTransform = partial(traverse, processing_table=ARITHMETIC_EBNF_transformation_table)
+
+
 class TestInfiLoopsAndRecursion:
     def test_direct_left_recursion(self):
-        minilang = """
-            @ whitespace = linefeed
-            formula = [ //~ ] expr
-            expr = expr ("+"|"-") term | term
-            term = term ("*"|"/") factor | factor
-            factor = /[0-9]+/~
-            # example:  "5 + 3 * 4"
-            """
+        minilang = ARITHMETIC_EBNF
         snippet = "5 + 3 * 4"
         parser = compileEBNF(minilang)()
         assert parser
@@ -60,6 +76,21 @@ class TestInfiLoopsAndRecursion:
         # print(syntax_tree.collect_errors())
 
 
+class TestTestGrammar:
+    cases = {
+        "factor": {
+            "match": {
+                1: "0",
+                2: "314",
+            },
+            "fail": {
+                1: "21F",
+                2: "G123"
+            }
+        }
+    }
+
+
 class TestRegex:
     def test_multilineRegex(self):
         mlregex = r"""