small refactorings

DHParser/dsl.py

@@ -38,6 +38,7 @@ __all__ = ['GrammarError',
            'CompilationError',
            'load_compiler_suite',
            'compileDSL',
+           'raw_compileEBNF',
            'compileEBNF',
            'parser_factory',
            'compile_on_disk']
@@ -186,10 +187,31 @@ def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler)
     return result
 
 
-def compileEBNF(ebnf_src, branding = "DSL"):
-    """Compiles an EBNF source file. Either returns a factory function
-    for the grammar-paraser or the source code of a compiler suite with
-    skeletons for scanner, transformer and compiler.
+def raw_compileEBNF(ebnf_src, branding="DSL"):
+    """Compiles an EBNF grammar file and returns the compiler object
+    that was used and which can now be queried for the result as well
+    as skeleton code for scanner, transformer and compiler objects.
+    
+    Args:
+        ebnf_src(str):  Either the file name of an EBNF grammar or
+            the EBNF grammar itself as a string.
+        branding (str):  Branding name for the compiler suite source
+            code. 
+    Returns:
+        An instance of class ``ebnf.EBNFCompiler``
+    Raises:
+        CompilationError if any errors occurred during compilation      
+    """
+    grammar = get_ebnf_grammar()
+    compiler = get_ebnf_compiler(branding , ebnf_src)
+    compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler)
+    return compiler
+
+
+def compileEBNF(ebnf_src, branding="DSL"):
+    """Compiles an EBNF source file and returns the source code of a
+    compiler suite with skeletons for scanner, transformer and 
+    compiler.
 
     Args:
         ebnf_src(str):  Either the file name of an EBNF grammar or
@@ -201,13 +223,11 @@ def compileEBNF(ebnf_src, branding = "DSL"):
     Raises:
         CompilationError if any errors occurred during compilation        
     """
-    grammar = get_ebnf_grammar()
-    compiler = get_ebnf_compiler(branding , ebnf_src)
-    grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler)
+    compiler = raw_compileEBNF(ebnf_src, branding)
     src = ["#/usr/bin/python\n",
            SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_IMPORTS,
            SECTION_MARKER.format(marker=SCANNER_SECTION), compiler.gen_scanner_skeleton(),
-           SECTION_MARKER.format(marker=PARSER_SECTION), grammar_src,
+           SECTION_MARKER.format(marker=PARSER_SECTION), compiler.result,
            SECTION_MARKER.format(marker=AST_SECTION), compiler.gen_transformer_skeleton(),
            SECTION_MARKER.format(marker=COMPILER_SECTION), compiler.gen_compiler_skeleton(),
            SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_MAIN.format(NAME=branding)]

DHParser/ebnf.py

@@ -303,6 +303,7 @@ class EBNFCompiler(CompilerBase):
         self._reset()
 
     def _reset(self):
+        self._result = None
         self.rules = set()
         self.variables = set()
         self.symbol_nodes = []
@@ -315,6 +316,10 @@ class EBNFCompiler(CompilerBase):
                            'tokens': set(),     # alt. 'scanner_tokens'
                            'counterpart': set()}     # alt. 'retrieve_counterpart'
 
+    @property
+    def result(self):
+        return self._result
+
     def gen_scanner_skeleton(self):
         name = self.grammar_name + "Scanner"
         return "def %s(text):\n    return text\n" % name \
@@ -322,8 +327,8 @@ class EBNFCompiler(CompilerBase):
 
     def gen_transformer_skeleton(self):
         if not self.definition_names:
-            raise EBNFCompilerError('Compiler has not been run before calling '
-                                    '"gen_AST_Skeleton()"!')
+            raise EBNFCompilerError('Compiler must be run before calling '
+                                    '"gen_transformer_Skeleton()"!')
         tt_name = self.grammar_name + '_AST_transformation_table'
         tf_name = self.grammar_name + 'Transform'
         transtable = [tt_name + ' = {',
@@ -360,7 +365,7 @@ class EBNFCompiler(CompilerBase):
         compiler += [COMPILER_FACTORY.format(NAME=self.grammar_name)]
         return '\n'.join(compiler)
 
-    def gen_parser(self, definitions):
+    def assemble_parser(self, definitions):
         # fix capture of variables that have been defined before usage [sic!]
 
         if self.variables:
@@ -424,8 +429,9 @@ class EBNFCompiler(CompilerBase):
         if self.root and 'root__' not in self.rules:
             declarations.append('root__ = ' + self.root)
         declarations.append('')
-        return '\n    '.join(declarations) \
-               + GRAMMAR_FACTORY.format(NAME=self.grammar_name)
+        self._result = '\n    '.join(declarations) \
+                       + GRAMMAR_FACTORY.format(NAME=self.grammar_name)
+        return self._result
 
     def on_syntax(self, node):
         self._reset()
@@ -444,7 +450,7 @@ class EBNFCompiler(CompilerBase):
                 assert nd.parser.name == "directive", nd.as_sexpr()
                 self._compile(nd)
 
-        return self.gen_parser(definitions)
+        return self.assemble_parser(definitions)
 
     def on_definition(self, node):
         rule = node.result[0].result

test/test_parsers.py

@@ -25,7 +25,7 @@ sys.path.extend(['../', './'])
 
 from DHParser.toolkit import is_logging, compile_python_object
 from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \
-    replace_by_single_child, reduce_single_child, flatten
+    replace_by_single_child, reduce_single_child, flatten, TOKEN_KEYWORD
 from DHParser.parsers import compile_source
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
 from DHParser.dsl import parser_factory, DHPARSER_IMPORTS
@@ -46,6 +46,7 @@ ARITHMETIC_EBNF_transformation_table = {
     "formula": [remove_expendables],
     "term, expr": [replace_by_single_child, flatten],
     "factor": [remove_expendables, reduce_single_child],
+    (TOKEN_KEYWORD): [remove_expendables, reduce_single_child],
     "": [remove_expendables, replace_by_single_child]
 }