more unit tests; API changes in dsl.py

DHParser/dsl.py

@@ -39,7 +39,7 @@ __all__ = ['GrammarError',
            'CompilationError',
            'load_compiler_suite',
            'compileDSL',
-           'run_compiler']
+           'compile_on_disk']
 
 
 SECTION_MARKER = """\n
@@ -98,10 +98,10 @@ from DHParser.parsers import GrammarBase, CompilerBase, nil_scanner, \\
     Lookbehind, Lookahead, Alternative, Pop, Required, Token, \\
     Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Sequence, RE, Capture, \\
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, full_compilation
-from DHParser.syntaxtree import Node, remove_enclosing_delimiters, remove_children_if, \\
-    reduce_single_child, replace_by_single_child, remove_whitespace, TOKEN_KEYWORD, \\
-    no_operation, remove_expendables, remove_tokens, flatten, WHITESPACE_KEYWORD, \\
-    is_whitespace, is_expendable
+from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \\
+    remove_children_if, reduce_single_child, replace_by_single_child, remove_whitespace, \\
+    no_operation, remove_expendables, remove_tokens, flatten, is_whitespace, is_expendable, \\
+    WHITESPACE_KEYWORD, TOKEN_KEYWORD
 '''
 
 
@@ -154,8 +154,7 @@ def get_grammar_instance(grammar):
     return parser_root, grammar_src
 
 
-def compileDSL(text_or_file, dsl_grammar, ast_transformation, compiler,
-               scanner=nil_scanner):
+def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler):
     """Compiles a text in a domain specific language (DSL) with an
     EBNF-specified grammar. Returns the compiled text or raises a
     compilation error.
@@ -164,6 +163,7 @@ def compileDSL(text_or_file, dsl_grammar, ast_transformation, compiler,
         CompilationError if any errors occured during compilation
     """
     assert isinstance(text_or_file, str)
+    assert isinstance(dsl_grammar, GrammarBase)
     assert isinstance(compiler, CompilerBase)
     parser_root, grammar_src = get_grammar_instance(dsl_grammar)
     src = load_if_file(text_or_file)
@@ -196,7 +196,7 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
         which conforms to the language defined by ``ebnf_src``.
     """
     grammar = ebnf_grammar_obj or EBNFGrammar()
-    grammar_src = compileDSL(ebnf_src, grammar, EBNFTransform, EBNFCompiler())
+    grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransform, EBNFCompiler())
     return grammar_src if source_only else \
         compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
 
@@ -216,7 +216,7 @@ def load_compiler_suite(compiler_suite):
             raise AssertionError('File "' + compiler_suite + '" seems to be corrupted. '
                                  'Please delete or repair file manually.')
         scanner = compile_python_object(imports + scanner_py, '\w*Scanner$')
-        ast = compile_python_object(imports + ast_py, '\w*Pipeline$')
+        ast = compile_python_object(imports + ast_py, '\w*Transform$')
         compiler = compile_python_object(imports + compiler_py, '\w*Compiler$')
     else:
         # assume source is an ebnf grammar
@@ -226,8 +226,8 @@ def load_compiler_suite(compiler_suite):
             raise GrammarError('\n\n'.join(errors), source)
         scanner = nil_scanner
         ast = EBNFTransform
-        compiler = EBNFCompiler()
-    parser = compile_python_object(DHPARSER_IMPORTS + parser_py, '\w*Grammar$')()
+        compiler = EBNFCompiler
+    parser = compile_python_object(DHPARSER_IMPORTS + parser_py, '\w*Grammar$')
 
     return scanner, parser, ast, compiler
 
@@ -241,7 +241,7 @@ def suite_outdated(compiler_suite, grammar_source):
     with whitespace order to trigger their recreation. Note: Do not
     delete or overwrite the section marker itself. 
 
-    Parameters:
+    Args:
         compiler_suite:  the parser class representing the grammar
             or the file name of a compiler suite containing the grammar
         grammar_source:  File name or string representation of the
@@ -257,7 +257,30 @@ def suite_outdated(compiler_suite, grammar_source):
         return True
 
 
-def run_compiler(source_file, compiler_suite="", extension=".xml"):
+def run_compiler(text_or_file, compiler_suite):
+    """Compiles a source with a given compiler suite.
+
+    Args:
+        text_or_file (str):  Either the file name of the source code or
+            the source code directly. (Which is determined by 
+            heuristics. If ``text_or_file`` contains at least on
+            linefeed then it is always assumed to be a source text and
+            not a file name.)
+        compiler_suite(str):  File name of the compiler suite to be
+            used.
+        
+    Returns:
+        The result of the compilation, the form and type of which 
+        depends entirely on the compiler.
+        
+    Raises:
+        CompilerError
+    """
+    scanner, parser, ast, compiler = load_compiler_suite(compiler_suite)
+    return compileDSL(text_or_file, scanner, parser(), ast, compiler())
+
+
+def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
     """Compiles the a source file with a given compiler and writes the
     result to a file.
 
@@ -267,9 +290,26 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
     skeletons for a scanner, AST transformation table, and compiler.
     If the Python script already exists only the parser name in the
     script will be updated. (For this to work, the different names
-    need to be delimited section marker blocks.). `run_compiler()`
+    need to be delimited section marker blocks.). `compile_on_disk()`
     returns a list of error messages or an empty list if no errors
     occurred.
+    
+    Parameters:
+        source_file(str):  The file name of the source text to be
+            compiled.
+        compiler_suite(str):  The file name of the compiler suite
+            (usually ending with '_compiler.py'), with which the source
+            file shall be compiled. If this is left empty, the source
+            file is assumed to be an EBNF-Grammar that will be compiled
+            with the internal EBNF-Compiler.
+        extension(str):  The result of the compilation (if successful)
+            is written to a file with the same name but a different
+            extension than the source file. This parameter sets the
+            extension.
+            
+    Returns:
+        A list of error messages or an empty list if there were no 
+        errors. 
     """
     filepath = os.path.normpath(source_file)
     # with open(source_file, encoding="utf-8") as f:
@@ -277,7 +317,8 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
     rootname = os.path.splitext(filepath)[0]
     compiler_name = os.path.basename(rootname)
     if compiler_suite:
-        scanner, parser, trans, cclass = load_compiler_suite(compiler_suite)
+        scanner, pclass, trans, cclass = load_compiler_suite(compiler_suite)
+        parser = pclass()
         compiler1 = cclass()
     else:
         scanner = nil_scanner
@@ -289,10 +330,9 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
         return errors
 
     elif trans == EBNFTransform:  # either an EBNF- or no compiler suite given
-        f = None
-
         global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
             AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER
+        f = None
         try:
             f = open(rootname + '_compiler.py', 'r', encoding="utf-8")
             source = f.read()
@@ -304,7 +344,9 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
             raise ValueError('File "' + rootname + '_compiler.py" seems to be corrupted. '
                                                    'Please delete or repair file manually!')
         finally:
-            if f:  f.close()
+            if f:
+                f.close()
+                f = None
 
         if RX_WHITESPACE.fullmatch(intro):
             intro = '#!/usr/bin/python'

DHParser/parsers.py

@@ -337,6 +337,7 @@ class GrammarBase:
         Returns:
             Node: The root node ot the parse tree.
         """
+        assert isinstance(document, str)
         if self.root__ is None:
             raise NotImplementedError()
         if self.dirty_flag:
@@ -391,7 +392,6 @@ class GrammarBase:
                 os.remove(path)
 
         if IS_LOGGING():
-            assert self.history
             if not log_file_name:
                 name = self.__class__.__name__
                 log_file_name = name[:-7] if name.lower().endswith('grammar') else name
@@ -948,7 +948,18 @@ class Forward(Parser):
 
 
 class CompilerBase:
+    def __init__(self):
+        self.dirty_flag = False
+
+    def _reset(self):
+        pass
+
     def compile__(self, node):
+        # if self.dirty_flag:
+        #     self._reset()
+        # else:
+        #     self.dirty_flag = True
+
         comp, cls = node.parser.name, node.parser.__class__.__name__
         elem = comp or cls
         if not sane_parser_name(elem):
@@ -976,7 +987,7 @@ def full_compilation(source, scanner, parser, transform, compiler):
     Args:
         source (str): The input text for compilation or a the name of a
             file containing the input text.
-        scanner (funciton):  text -> text. A scanner function or None,
+        scanner (function):  text -> text. A scanner function or None,
             if no scanner is needed.
         parser (GrammarBase):  The GrammarBase object
         transform (function):  A transformation function that takes
@@ -991,8 +1002,7 @@ def full_compilation(source, scanner, parser, transform, compiler):
         (result, errors, abstract syntax tree). In detail:
         1. The result as returned by the compiler or ``None`` in case
             of failure,
-        2. A list of error messages, each of which is a tuple
-            (position: int, error: str)
+        2. A list of error messages
         3. The root-node of the abstract syntax treelow
     """
     assert isinstance(compiler, CompilerBase)

DHParser/toolkit.py

@@ -162,15 +162,17 @@ def load_if_file(text_or_file):
     file name (i.e. a single line string), otherwise (i.e. if `text_or_file` is
     a multiline string) `text_or_file` is returned.
     """
-    if text_or_file and text_or_file.find('\n') < 0:
+    if text_or_file.find('\n') < 0:
         try:
             with open(text_or_file, encoding="utf-8") as f:
                 content = f.read()
             return content
         except FileNotFoundError as error:
-            if not re.match(r'\w+', text_or_file):
-                raise FileNotFoundError('Not a valid file: ' + text_or_file +
-                                        '\nAdd "\\n" to distinguish source data from a file name!')
+            if re.fullmatch(r'[\w/:\\]+', text_or_file):
+                raise FileNotFoundError('Not a valid file: ' + text_or_file + '\nAdd "\\n" '
+                                        'to distinguish source data from a file name!')
+            else:
+                return text_or_file
     else:
         return text_or_file
 
@@ -253,6 +255,8 @@ def expand_table(compact_table):
     for key in keys:
         value = compact_table[key]
         for k in smart_list(key):
+            if k in expanded_table:
+                raise KeyError("Key %s used more than once in compact table!" % key)
             expanded_table[k] = value
     return expanded_table
 

OLDSTUFF/ParserCombinators_obsolete.py

@@ -2045,7 +2045,7 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
      skeletons for a scanner, AST transformation table, and compiler.
      If the Python script already exists only the parser name in the
      script will be updated. (For this to work, the different names
-     need to be delimited section marker blocks.). `run_compiler()`
+     need to be delimited section marker blocks.). `compile_on_disk()`
      returns a list of error messages or an empty list if no errors
      occurred.
      """

dhparser.py

@@ -24,9 +24,9 @@ import os
 import sys
 from functools import partial
 
-from DHParser.dsl import compileDSL, run_compiler
+from DHParser.dsl import compileDSL, compile_on_disk
 from DHParser.ebnf import EBNFGrammar, EBNFTransform, EBNFCompiler
-from DHParser.parsers import full_compilation
+from DHParser.parsers import full_compilation, nil_scanner
 
 
 def selftest(file_name):
@@ -45,7 +45,7 @@ def selftest(file_name):
     else:
         # compile the grammar again using the result of the previous
         # compilation as parser
-        result = compileDSL(grammar, result, EBNFTransform, compiler)
+        result = compileDSL(grammar, nil_scanner, result, EBNFTransform, compiler)
         print(result)
     return result
 
@@ -71,8 +71,8 @@ def profile(func):
 if __name__ == "__main__":
     print(sys.argv)
     if len(sys.argv) > 1:
-        _errors = run_compiler(sys.argv[1],
-                               sys.argv[2] if len(sys.argv) > 2 else "")
+        _errors = compile_on_disk(sys.argv[1],
+                                  sys.argv[2] if len(sys.argv) > 2 else "")
         if _errors:
             print('\n\n'.join(_errors))
             sys.exit(1)

examples/MLW/OLDSTUFF/compile_MLW-grammar.py

@@ -22,8 +22,8 @@ limitations under the License.
 import os
 import sys
 sys.path.append(os.path.abspath('../../'))
-from DHParser.dsl import run_compiler
-errors = run_compiler("MLW.ebnf")
+from DHParser.dsl import compile_on_disk
+errors = compile_on_disk("MLW.ebnf")
 if errors:
     print("\n".join(errors))
     sys.exit(1)

examples/MLW/samples/compile_MLW-entry.py

@@ -24,7 +24,7 @@ import sys
 sys.path.append(os.path.abspath('../../../'))
 import DHParser.toolkit as toolkit
 from DHParser.ebnf import grammar_changed
-from DHParser.dsl import run_compiler
+from DHParser.dsl import compile_on_disk
 
 MLW_ebnf = os.path.join('..', 'MLW.ebnf')
 MLW_compiler = os.path.join('..', 'MLW_compiler.py')
@@ -36,14 +36,14 @@ toolkit.logging_off()
 if (not os.path.exists(MLW_compiler) or
     grammar_changed(MLW_compiler, MLW_ebnf)):
     print("recompiling parser")
-    errors = run_compiler(MLW_ebnf)
+    errors = compile_on_disk(MLW_ebnf)
     if errors:
         print('\n'.join(errors))
         sys.exit(1)
 
 toolkit.logging_on()
 
-errors = run_compiler("fascitergula.mlw", MLW_compiler, ".xml")
+errors = compile_on_disk("fascitergula.mlw", MLW_compiler, ".xml")
 if errors:
     print('\n'.join(errors))
     sys.exit(1)

experimental/ParserCombinators_no_TOKEN_KEYWORD.py

@@ -2045,7 +2045,7 @@ def run_compiler(source_file, compiler_suite="", extension=".dst"):
      script already exists only the parser name in the script will be
      updated. (For this to work, the different names need to be delimited
      by the standard `DELIMITER`-line!).
-     `run_compiler()` returns a list of error messages or an empty list if no
+     `compile_on_disk()` returns a list of error messages or an empty list if no
      errors occured.
      """
     filepath = os.path.normpath(source_file)

tests/no_unit_tests/compile_PopRetrieve_EBNF.py

@@ -23,12 +23,12 @@ limitations under the License.
 import os
 import sys
 sys.path.append(os.path.abspath('../../'))
-from DHParser.dsl import run_compiler, suite_outdated
+from DHParser.dsl import compile_on_disk, suite_outdated
 
 if (not os.path.exists('PopRetrieve_compiler.py') or
     suite_outdated('PopRetrieve_compiler.py', 'PopRetrieve.ebnf')):
     print("recompiling PopRetrieve parser")
-    errors = run_compiler("PopRetrieve.ebnf")
+    errors = compile_on_disk("PopRetrieve.ebnf")
     if errors:
         print('\n\n'.join(errors))
         sys.exit(1)
@@ -54,13 +54,13 @@ if (not os.path.exists('PopRetrieve_compiler.py') or
 
 
 print("PopRetrieveTest 1")
-errors = run_compiler("PopRetrieveTest.txt", 'PopRetrieve_compiler.py')
+errors = compile_on_disk("PopRetrieveTest.txt", 'PopRetrieve_compiler.py')
 if errors:
     print(errors)
     sys.exit(1)
 
 print("PopRetrieveTest 2")
-errors = run_compiler("PopRetrieveTest2.txt", 'PopRetrieve_compiler.py')
+errors = compile_on_disk("PopRetrieveTest2.txt", 'PopRetrieve_compiler.py')
 if errors:
     print(errors)
     sys.exit(1)
@@ -70,7 +70,7 @@ if errors:
 if (not os.path.exists('PopRetrieveComplement_compiler.py') or
         suite_outdated('PopRetrieveComplement_compiler.py', 'PopRetrieveComplement.ebnf')):
     print("recompiling PopRetrieveComplement parser")
-    errors = run_compiler("PopRetrieveComplement.ebnf")
+    errors = compile_on_disk("PopRetrieveComplement.ebnf")
     if errors:
         print('\n\n'.join(errors))
         sys.exit(1)

tests/run.py

@@ -45,20 +45,23 @@ def runner(tests, namespace):
             if name.lower().startswith('test') and inspect.isclass(namespace[name]):
                 tests.append(name)
 
+    obj = None
     for test in tests:
-        if test.find('.') >= 0:
-            cls_name, method_name = test.split('.')
-            obj = instantiate(cls_name)
-            print("Running " + cls_name + "." + method_name)
-            exec('obj.' + method_name + '()')
-        else:
-            obj = instantiate(test)
-            for name in dir(obj):
-                if name.lower().startswith("test"):
-                    print("Running " + test + "." + name)
-                    exec('obj.' + name + '()')
-        if "teardown" in dir(obj):
-            obj.teardown()
+        try:
+            if test.find('.') >= 0:
+                cls_name, method_name = test.split('.')
+                obj = instantiate(cls_name)
+                print("Running " + cls_name + "." + method_name)
+                exec('obj.' + method_name + '()')
+            else:
+                obj = instantiate(test)
+                for name in dir(obj):
+                    if name.lower().startswith("test"):
+                        print("Running " + test + "." + name)
+                        exec('obj.' + name + '()')
+        finally:
+            if "teardown" in dir(obj):
+                obj.teardown()
 
 if __name__ == "__main__":
     # runner("", globals())

tests/test_dsl.py

@@ -22,27 +22,58 @@ limitations under the License.
 
 import os
 import sys
-from DHParser.dsl import run_compiler
+
+from DHParser.dsl import compile_on_disk, run_compiler
 
 class TestCompilerGeneration:
     trivial_lang = """
-        text = { word | WSPC }
+        text = { word | WSPC } "."
         word = /\w+/
-        WSPC = /\s*/
+        WSPC = /\s+/
         """
     trivial_text = """Es war ein König in Thule."""
-
+    grammar_name = "tmp/TestCompilerGeneration.ebnf"
+    compiler_name = "tmp/TestCompilerGeneration_compiler.py"
+    text_name = "tmp/TestCompilerGeneration_text.txt"
+    result_name = "tmp/TestCompilerGeneration_text.xml"
 
     def setup(self):
+        with open(self.grammar_name, "w") as f:
+            f.write(self.trivial_lang)
+        with open(self.text_name, "w") as f:
+            f.write(self.trivial_text)
+
+    def teardown(self):
+        for name in (self.grammar_name, self.compiler_name, self.text_name, self.result_name):
+            if os.path.exists(name):
+                os.remove(name)
+
+    def test_compiling_functions(self):
+        # test if cutting and reassembling of compiler suite works:
+        errors = compile_on_disk(self.grammar_name)
+        assert not errors
+        with open(self.compiler_name, 'r') as f:
+            compiler_suite = f.read()
+        errors = compile_on_disk(self.grammar_name)
+        assert not errors
+        with open(self.compiler_name, 'r') as f:
+            compiler_suite_2nd_run = f.read()
+        assert compiler_suite == compiler_suite_2nd_run
+
+        # test compiling with a generated compiler suite
+        errors = compile_on_disk(self.text_name, self.compiler_name)
+        assert not errors, str(errors)
+        assert os.path.exists(self.result_name)
+        with open(self.result_name, 'r') as f:
+            output = f.read()
 
+        # test compiling in memory
+        result = run_compiler(self.trivial_text, self.compiler_name)
+        assert output == result.as_xml(), str(result)
 
-if (not os.path.exists('PopRetrieve_compiler.py') or
-            suite_outdated('PopRetrieve_compiler.py', 'PopRetrieve.ebnf')):
-        print("recompiling PopRetrieve parser")
-        errors = run_compiler("PopRetrieve.ebnf")
-        if errors:
-            print('\n\n'.join(errors))
-            sys.exit(1)
+        sys.path.append('tmp')
+        from TestCompilerGeneration_compiler import compile_TestCompilerGeneration
+        result, errors, ast = compile_TestCompilerGeneration(self.trivial_text)
 
 
 if __name__ == "__main__":

tests/test_parsers.py

@@ -42,6 +42,7 @@ class TestInfiLoopsAndRecursion:
             # example:  "5 + 3 * 4"
             """
         snippet = "5 + 3 * 4"
+        print(compileEBNF(minilang, source_only=True))
         parser = compileEBNF(minilang)()
         assert parser
         syntax_tree = parser.parse(snippet)

tests/test_toolkit.py

+#!/usr/bin/python3
+
+"""test_tookkit.py - tests of the toolkit-module of DHParser 
+
+
+Author: Eckhart Arnold <arnold@badw.de>
+
+Copyright 2017 Bavarian Academy of Sciences and Humanities
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import sys
+
+from DHParser.toolkit import load_if_file
+
+class TestToolkit:
+    filename = "tmp/test.py"
+    code1 = "x = 46"
+    code2 = "def f():\n    return 46"
+
+    def setup(self):
+        with open(self.filename, 'w') as f:
+            f.write(self.code2)
+
+    def teardown(self):
+        os.remove(self.filename)
+
+    def test_load_if_file(self):
+        # an error should be raised if file expected but not found
+        error_raised = False
+        try:
+            load_if_file('this_is_code_and_not_a_file')
+        except FileNotFoundError:
+            error_raised = True
+        assert error_raised
+
+        # multiline text will never be mistaken for a file
+        assert load_if_file('this_is_code_and_not_a_file\n')
+
+        # neither will text that does not look like a file name
+        s = "this is code and not a file"
+        assert s == load_if_file(s)
+
+        # not a file and not mistaken for a file
+        assert self.code1 == load_if_file(self.code1)
+
+        # not a file and not mistaken for a file either
+        assert self.code2 == load_if_file(self.code2)
+
+        # file correctly loaded
+        assert self.code2 == load_if_file(self.filename)
+
+
+if __name__ == "__main__":
+    from run import runner
+    runner("", globals())
\ No newline at end of file