moved code generation of factory functions from dsl.py to ebnf.py; finished...

moved code generation of factory functions from dsl.py to ebnf.py; finished testing function at the end of parser.py (not yet tested)

DHParser/dsl.py

@@ -27,7 +27,7 @@ try:
 except ImportError:
     import re
 
-from .ebnf import EBNFGrammar, EBNFTransform, EBNFCompiler, grammar_changed, \
+from .ebnf import EBNFGrammar, EBNFTransformer, EBNFCompiler, grammar_changed, \
     get_ebnf_scanner, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
 from .toolkit import logging, load_if_file, is_python_code, compile_python_object
 from .parsers import GrammarBase, CompilerBase, compile_source, nil_scanner
@@ -105,43 +105,6 @@ from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \\
 '''
 
 
-DHPARSER_SCANNER = '''
-def get_{NAME}_scanner():
-    return {NAME}Scanner
-'''
-
-
-DHPARSER_GRAMMAR = '''
-def get_{NAME}_grammar():
-    global thread_local_{NAME}_grammar_singleton
-    try:
-        grammar = thread_local_{NAME}_grammar_singleton
-        return grammar
-    except NameError:
-        thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
-        return thread_local_{NAME}_grammar_singleton
-'''
-
-
-DHPARSER_TRANSFORMER = '''
-def get_{NAME}_transformer():
-    return {NAME}Transform
-'''
-
-
-DHPARSER_COMPILER = '''
-def get_{NAME}_compiler(grammar_name="{NAME}", grammar_source=""):
-    global thread_local_{NAME}_compiler_singleton
-    try:
-        compiler = thread_local_{NAME}_compiler_singleton
-        compiler.set_grammar_name(grammar_name, grammar_source)
-        return compiler
-    except NameError:
-        thread_local_{NAME}_compiler_singleton = {NAME}Compiler(grammar_name, grammar_source)
-        return thread_local_{NAME}_compiler_singleton 
-'''
-
-
 DHPARSER_MAIN = '''
 def compile_{NAME}(source):
     """Compiles ``source`` and returns (result, errors, ast).
@@ -243,8 +206,8 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
         A Grammar class that can be instantiated for parsing a text
         which conforms to the language defined by ``ebnf_src``.
     """
-    grammar = ebnf_grammar_obj or EBNFGrammar()
-    grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransform, EBNFCompiler())
+    grammar = ebnf_grammar_obj or get_ebnf_grammar()
+    grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, get_ebnf_compiler())
     return grammar_src if source_only else \
         compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
 
@@ -380,7 +343,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
     if errors:
         return errors
 
-    elif trans == get_ebnf_transformer or trans == EBNFTransform:  # either an EBNF- or no compiler suite given
+    elif trans == get_ebnf_transformer or trans == EBNFTransformer:  # either an EBNF- or no compiler suite given
         global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
             AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER
         f = None
@@ -406,14 +369,11 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
         if RX_WHITESPACE.fullmatch(imports):
             imports = DHPARSER_IMPORTS
         if RX_WHITESPACE.fullmatch(scanner):
-            scanner = compiler1.gen_scanner_skeleton() + \
-                      DHPARSER_SCANNER.format(NAME=compiler_name)
+            scanner = compiler1.gen_scanner_skeleton()
         if RX_WHITESPACE.fullmatch(ast):
-            ast = compiler1.gen_AST_skeleton() + \
-                  DHPARSER_TRANSFORMER.format(NAME=compiler_name)
+            ast = compiler1.gen_transformer_skeleton()
         if RX_WHITESPACE.fullmatch(compiler):
-            compiler = compiler1.gen_compiler_skeleton() + \
-                       DHPARSER_COMPILER.format(NAME=compiler_name)
+            compiler = compiler1.gen_compiler_skeleton()
 
         try:
             f = open(rootname + '_compiler.py', 'w', encoding="utf-8")
@@ -423,7 +383,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
             f.write(SECTION_MARKER.format(marker=SCANNER_SECTION))
             f.write(scanner)
             f.write(SECTION_MARKER.format(marker=PARSER_SECTION))
-            f.write(result);  f.write(DHPARSER_GRAMMAR.format(NAME=compiler_name))
+            f.write(result)
             f.write(SECTION_MARKER.format(marker=AST_SECTION))
             f.write(ast)
             f.write(SECTION_MARKER.format(marker=COMPILER_SECTION))

DHParser/ebnf.py

@@ -35,7 +35,7 @@ from .versionnumber import __version__
 
 
 __all__ = ['EBNFGrammar',
-           'EBNFTransform',
+           'EBNFTransformer',
            'EBNFCompilerError',
            'EBNFCompiler',
            'grammar_changed']
@@ -222,13 +222,13 @@ EBNF_validation_table = {
 }
 
 
-def EBNFTransform(syntax_tree):
+def EBNFTransformer(syntax_tree):
     for processing_table in [EBNF_transformation_table, EBNF_validation_table]:
         traverse(syntax_tree, processing_table)
 
 
 def get_ebnf_transformer():
-    return EBNFTransform
+    return EBNFTransformer
 
 
 ########################################################################
@@ -237,6 +237,48 @@ def get_ebnf_transformer():
 #
 ########################################################################
 
+SCANNER_FACTORY = '''
+
+def get_{NAME}_scanner():
+    return {NAME}Scanner
+'''
+
+
+GRAMMAR_FACTORY = '''
+
+def get_{NAME}_grammar():
+    global thread_local_{NAME}_grammar_singleton
+    try:
+        grammar = thread_local_{NAME}_grammar_singleton
+        return grammar
+    except NameError:
+        thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
+        return thread_local_{NAME}_grammar_singleton
+'''
+
+
+TRANSFORMER_FACTORY = '''
+
+def get_{NAME}_transformer():
+    return {NAME}Transform
+'''
+
+
+COMPILER_FACTORY = '''
+
+def get_{NAME}_compiler(grammar_name="{NAME}", 
+                        grammar_source=""):
+    global thread_local_{NAME}_compiler_singleton
+    try:
+        compiler = thread_local_{NAME}_compiler_singleton
+        compiler.set_grammar_name(grammar_name, grammar_source)
+        return compiler
+    except NameError:
+        thread_local_{NAME}_compiler_singleton = \\
+            {NAME}Compiler(grammar_name, grammar_source)
+        return thread_local_{NAME}_compiler_singleton 
+'''
+
 
 class EBNFCompilerError(Exception):
     """Error raised by `EBNFCompiler` class. (Not compilation errors
@@ -279,9 +321,10 @@ class EBNFCompiler(CompilerBase):
 
     def gen_scanner_skeleton(self):
         name = self.grammar_name + "Scanner"
-        return "def %s(text):\n    return text\n" % name
+        return "def %s(text):\n    return text\n" % name \
+               + SCANNER_FACTORY.format(NAME=self.grammar_name)
 
-    def gen_AST_skeleton(self):
+    def gen_transformer_skeleton(self):
         if not self.definition_names:
             raise EBNFCompilerError('Compiler has not been run before calling '
                                     '"gen_AST_Skeleton()"!')
@@ -294,6 +337,7 @@ class EBNFCompiler(CompilerBase):
             transtable.append('    "' + name + '": no_operation,')
         transtable += ['    "": no_operation', '}', '',  tf_name +
                        ' = partial(traverse, processing_table=%s)' % tt_name, '']
+        transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
         return '\n'.join(transtable)
 
     def gen_compiler_skeleton(self):
@@ -317,6 +361,7 @@ class EBNFCompiler(CompilerBase):
             else:
                 compiler += ['    def ' + method_name + '(self, node):',
                              '        pass', '']
+        compiler += [COMPILER_FACTORY.format(NAME=self.grammar_name)]
         return '\n'.join(compiler)
 
     def gen_parser(self, definitions):
@@ -379,7 +424,8 @@ class EBNFCompiler(CompilerBase):
         if self.root and 'root__' not in self.rules:
             declarations.append('root__ = ' + self.root)
         declarations.append('')
-        return '\n    '.join(declarations)
+        return '\n    '.join(declarations) \
+               + GRAMMAR_FACTORY.format(NAME=self.grammar_name)
 
     def on_syntax(self, node):
         self._reset()

DHParser/parsers.py

@@ -288,7 +288,6 @@ class GrammarBase:
     def __init__(self):
         self.all_parsers = set()
         self.dirty_flag = False
-        self.history_tracking = is_logging()
         self._reset()
         self._assign_parser_names()
         self.root__ = copy.deepcopy(self.__class__.root__)
@@ -315,6 +314,7 @@ class GrammarBase:
         self.last_node = None
         self.call_stack = []  # support for call stack tracing
         self.history = []  # snapshots of call stacks
+        self.history_tracking = is_logging()
         self.moving_forward = True  # also needed for call stack tracing
 
     def _add_parser(self, parser):
@@ -1081,13 +1081,31 @@ def compile_source(source, scanner, parser, transformer, compiler):
     return result, messages, syntax_tree
 
 
-def test_grammar(test_suite, parse_function, transform):
+def test_grammar(test_suite, parser, transform):
+    errata = []
     for parser_name, tests in test_suite.items():
-        assert set(tests.keys()).issubset({'match', 'fail', 'ast', 'cst'})
+        assert set(tests.keys()).issubset({'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'})
+
         for test_name, test_code in tests['match'].items():
-            cst = parse_function(test_code, parser_name)
+            cst = parser(test_code, parser_name)
+            tests['__cst__'][test_name] = cst
+            if cst.error_flag:
+                errata.append("Test %s for parser %s : didn't match: \n%s" %
+                              (test_name, parser_name,
+                               error_messages(test_code, cst.collect_errors())))
+            elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
+                    errata.append("Test %s for parser %s : wrong CST: \n%s" %
+                                  (test_name, parser_name, cst.as_sexpr()))
+            elif "ast" in tests:
+                ast = transform(cst)
+                tests['__ast__'][test_name] = ast
+                if mock_syntax_tree(tests["ast"][test_name]) != ast:
+                    errata.append("Test %s for parser %s : wrong AST: \n%s" %
+                                  (test_name, parser_name, ast.as_sexpr()))
+
+        for test_name, test_code in tests['fail'].items():
+            cst = parser(test_code, parser_name)
             if not cst.error_flag:
-                yield "Test %s for parser %s did not match" % (test_name, parser_name)
-            if "cst" in tests:
-                if tests["cst"][test_name] != mock_syntax_tree(cst):
-                    pass # TO BE CONTINUED
\ No newline at end of file
+                errata.append("Test %s for parser %s : shouldn't have matched!" %
+                              (test_name, parser_name))
+    return errata

test/test_dsl.py

@@ -46,9 +46,9 @@ class TestCompilerGeneration:
             f.write(self.trivial_text)
 
     def teardown(self):
-        # for name in (self.grammar_name, self.compiler_name, self.text_name, self.result_name):
-        #     if os.path.exists(name):
-        #         os.remove(name)
+        for name in (self.grammar_name, self.compiler_name, self.text_name, self.result_name):
+            if os.path.exists(name):
+                os.remove(name)
         pass
 
     def test_compiling_functions(self):

test/test_ebnf.py

@@ -27,7 +27,7 @@ sys.path.extend(['../', './'])
 
 from DHParser.toolkit import is_logging
 from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner
-from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransform, get_ebnf_compiler
+from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
 from DHParser.dsl import compileEBNF, compileDSL
 
 
@@ -206,7 +206,7 @@ class TestSemanticValidation:
         grammar = get_ebnf_grammar()
         st = grammar(minilang)
         assert not st.collect_errors()
-        EBNFTransform(st)
+        EBNFTransformer(st)
         assert bool_filter(st.collect_errors())
 
     def test_illegal_nesting(self):