Commit 88ad74ab authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

more unit tests; API changes in dsl.py

parent 5d9dfd50
......@@ -39,7 +39,7 @@ __all__ = ['GrammarError',
'CompilationError',
'load_compiler_suite',
'compileDSL',
'run_compiler']
'compile_on_disk']
SECTION_MARKER = """\n
......@@ -98,10 +98,10 @@ from DHParser.parsers import GrammarBase, CompilerBase, nil_scanner, \\
Lookbehind, Lookahead, Alternative, Pop, Required, Token, \\
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Sequence, RE, Capture, \\
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, full_compilation
from DHParser.syntaxtree import Node, remove_enclosing_delimiters, remove_children_if, \\
reduce_single_child, replace_by_single_child, remove_whitespace, TOKEN_KEYWORD, \\
no_operation, remove_expendables, remove_tokens, flatten, WHITESPACE_KEYWORD, \\
is_whitespace, is_expendable
from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \\
remove_children_if, reduce_single_child, replace_by_single_child, remove_whitespace, \\
no_operation, remove_expendables, remove_tokens, flatten, is_whitespace, is_expendable, \\
WHITESPACE_KEYWORD, TOKEN_KEYWORD
'''
......@@ -154,8 +154,7 @@ def get_grammar_instance(grammar):
return parser_root, grammar_src
def compileDSL(text_or_file, dsl_grammar, ast_transformation, compiler,
scanner=nil_scanner):
def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler):
"""Compiles a text in a domain specific language (DSL) with an
EBNF-specified grammar. Returns the compiled text or raises a
compilation error.
......@@ -164,6 +163,7 @@ def compileDSL(text_or_file, dsl_grammar, ast_transformation, compiler,
CompilationError if any errors occured during compilation
"""
assert isinstance(text_or_file, str)
assert isinstance(dsl_grammar, GrammarBase)
assert isinstance(compiler, CompilerBase)
parser_root, grammar_src = get_grammar_instance(dsl_grammar)
src = load_if_file(text_or_file)
......@@ -196,7 +196,7 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
which conforms to the language defined by ``ebnf_src``.
"""
grammar = ebnf_grammar_obj or EBNFGrammar()
grammar_src = compileDSL(ebnf_src, grammar, EBNFTransform, EBNFCompiler())
grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransform, EBNFCompiler())
return grammar_src if source_only else \
compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
......@@ -216,7 +216,7 @@ def load_compiler_suite(compiler_suite):
raise AssertionError('File "' + compiler_suite + '" seems to be corrupted. '
'Please delete or repair file manually.')
scanner = compile_python_object(imports + scanner_py, '\w*Scanner$')
ast = compile_python_object(imports + ast_py, '\w*Pipeline$')
ast = compile_python_object(imports + ast_py, '\w*Transform$')
compiler = compile_python_object(imports + compiler_py, '\w*Compiler$')
else:
# assume source is an ebnf grammar
......@@ -226,8 +226,8 @@ def load_compiler_suite(compiler_suite):
raise GrammarError('\n\n'.join(errors), source)
scanner = nil_scanner
ast = EBNFTransform
compiler = EBNFCompiler()
parser = compile_python_object(DHPARSER_IMPORTS + parser_py, '\w*Grammar$')()
compiler = EBNFCompiler
parser = compile_python_object(DHPARSER_IMPORTS + parser_py, '\w*Grammar$')
return scanner, parser, ast, compiler
......@@ -241,7 +241,7 @@ def suite_outdated(compiler_suite, grammar_source):
with whitespace order to trigger their recreation. Note: Do not
delete or overwrite the section marker itself.
Parameters:
Args:
compiler_suite: the parser class representing the grammar
or the file name of a compiler suite containing the grammar
grammar_source: File name or string representation of the
......@@ -257,7 +257,30 @@ def suite_outdated(compiler_suite, grammar_source):
return True
def run_compiler(source_file, compiler_suite="", extension=".xml"):
def run_compiler(text_or_file, compiler_suite):
"""Compiles a source with a given compiler suite.
Args:
text_or_file (str): Either the file name of the source code or
the source code directly. (Which is determined by
heuristics. If ``text_or_file`` contains at least on
linefeed then it is always assumed to be a source text and
not a file name.)
compiler_suite(str): File name of the compiler suite to be
used.
Returns:
The result of the compilation, the form and type of which
depends entirely on the compiler.
Raises:
CompilerError
"""
scanner, parser, ast, compiler = load_compiler_suite(compiler_suite)
return compileDSL(text_or_file, scanner, parser(), ast, compiler())
def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
"""Compiles the a source file with a given compiler and writes the
result to a file.
......@@ -267,9 +290,26 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
skeletons for a scanner, AST transformation table, and compiler.
If the Python script already exists only the parser name in the
script will be updated. (For this to work, the different names
need to be delimited section marker blocks.). `run_compiler()`
need to be delimited section marker blocks.). `compile_on_disk()`
returns a list of error messages or an empty list if no errors
occurred.
Parameters:
source_file(str): The file name of the source text to be
compiled.
compiler_suite(str): The file name of the compiler suite
(usually ending with '_compiler.py'), with which the source
file shall be compiled. If this is left empty, the source
file is assumed to be an EBNF-Grammar that will be compiled
with the internal EBNF-Compiler.
extension(str): The result of the compilation (if successful)
is written to a file with the same name but a different
extension than the source file. This parameter sets the
extension.
Returns:
A list of error messages or an empty list if there were no
errors.
"""
filepath = os.path.normpath(source_file)
# with open(source_file, encoding="utf-8") as f:
......@@ -277,7 +317,8 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
rootname = os.path.splitext(filepath)[0]
compiler_name = os.path.basename(rootname)
if compiler_suite:
scanner, parser, trans, cclass = load_compiler_suite(compiler_suite)
scanner, pclass, trans, cclass = load_compiler_suite(compiler_suite)
parser = pclass()
compiler1 = cclass()
else:
scanner = nil_scanner
......@@ -289,10 +330,9 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
return errors
elif trans == EBNFTransform: # either an EBNF- or no compiler suite given
f = None
global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER
f = None
try:
f = open(rootname + '_compiler.py', 'r', encoding="utf-8")
source = f.read()
......@@ -304,7 +344,9 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
raise ValueError('File "' + rootname + '_compiler.py" seems to be corrupted. '
'Please delete or repair file manually!')
finally:
if f: f.close()
if f:
f.close()
f = None
if RX_WHITESPACE.fullmatch(intro):
intro = '#!/usr/bin/python'
......
......@@ -337,6 +337,7 @@ class GrammarBase:
Returns:
Node: The root node ot the parse tree.
"""
assert isinstance(document, str)
if self.root__ is None:
raise NotImplementedError()
if self.dirty_flag:
......@@ -391,7 +392,6 @@ class GrammarBase:
os.remove(path)
if IS_LOGGING():
assert self.history
if not log_file_name:
name = self.__class__.__name__
log_file_name = name[:-7] if name.lower().endswith('grammar') else name
......@@ -948,7 +948,18 @@ class Forward(Parser):
class CompilerBase:
def __init__(self):
self.dirty_flag = False
def _reset(self):
pass
def compile__(self, node):
# if self.dirty_flag:
# self._reset()
# else:
# self.dirty_flag = True
comp, cls = node.parser.name, node.parser.__class__.__name__
elem = comp or cls
if not sane_parser_name(elem):
......@@ -976,7 +987,7 @@ def full_compilation(source, scanner, parser, transform, compiler):
Args:
source (str): The input text for compilation or a the name of a
file containing the input text.
scanner (funciton): text -> text. A scanner function or None,
scanner (function): text -> text. A scanner function or None,
if no scanner is needed.
parser (GrammarBase): The GrammarBase object
transform (function): A transformation function that takes
......@@ -991,8 +1002,7 @@ def full_compilation(source, scanner, parser, transform, compiler):
(result, errors, abstract syntax tree). In detail:
1. The result as returned by the compiler or ``None`` in case
of failure,
2. A list of error messages, each of which is a tuple
(position: int, error: str)
2. A list of error messages
3. The root-node of the abstract syntax treelow
"""
assert isinstance(compiler, CompilerBase)
......
......@@ -162,15 +162,17 @@ def load_if_file(text_or_file):
file name (i.e. a single line string), otherwise (i.e. if `text_or_file` is
a multiline string) `text_or_file` is returned.
"""
if text_or_file and text_or_file.find('\n') < 0:
if text_or_file.find('\n') < 0:
try:
with open(text_or_file, encoding="utf-8") as f:
content = f.read()
return content
except FileNotFoundError as error:
if not re.match(r'\w+', text_or_file):
raise FileNotFoundError('Not a valid file: ' + text_or_file +
'\nAdd "\\n" to distinguish source data from a file name!')
if re.fullmatch(r'[\w/:\\]+', text_or_file):
raise FileNotFoundError('Not a valid file: ' + text_or_file + '\nAdd "\\n" '
'to distinguish source data from a file name!')
else:
return text_or_file
else:
return text_or_file
......@@ -253,6 +255,8 @@ def expand_table(compact_table):
for key in keys:
value = compact_table[key]
for k in smart_list(key):
if k in expanded_table:
raise KeyError("Key %s used more than once in compact table!" % key)
expanded_table[k] = value
return expanded_table
......
......@@ -2045,7 +2045,7 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"):
skeletons for a scanner, AST transformation table, and compiler.
If the Python script already exists only the parser name in the
script will be updated. (For this to work, the different names
need to be delimited section marker blocks.). `run_compiler()`
need to be delimited section marker blocks.). `compile_on_disk()`
returns a list of error messages or an empty list if no errors
occurred.
"""
......
......@@ -24,9 +24,9 @@ import os
import sys
from functools import partial
from DHParser.dsl import compileDSL, run_compiler
from DHParser.dsl import compileDSL, compile_on_disk
from DHParser.ebnf import EBNFGrammar, EBNFTransform, EBNFCompiler
from DHParser.parsers import full_compilation
from DHParser.parsers import full_compilation, nil_scanner
def selftest(file_name):
......@@ -45,7 +45,7 @@ def selftest(file_name):
else:
# compile the grammar again using the result of the previous
# compilation as parser
result = compileDSL(grammar, result, EBNFTransform, compiler)
result = compileDSL(grammar, nil_scanner, result, EBNFTransform, compiler)
print(result)
return result
......@@ -71,8 +71,8 @@ def profile(func):
if __name__ == "__main__":
print(sys.argv)
if len(sys.argv) > 1:
_errors = run_compiler(sys.argv[1],
sys.argv[2] if len(sys.argv) > 2 else "")
_errors = compile_on_disk(sys.argv[1],
sys.argv[2] if len(sys.argv) > 2 else "")
if _errors:
print('\n\n'.join(_errors))
sys.exit(1)
......
......@@ -22,8 +22,8 @@ limitations under the License.
import os
import sys
sys.path.append(os.path.abspath('../../'))
from DHParser.dsl import run_compiler
errors = run_compiler("MLW.ebnf")
from DHParser.dsl import compile_on_disk
errors = compile_on_disk("MLW.ebnf")
if errors:
print("\n".join(errors))
sys.exit(1)
......
......@@ -24,7 +24,7 @@ import sys
sys.path.append(os.path.abspath('../../../'))
import DHParser.toolkit as toolkit
from DHParser.ebnf import grammar_changed
from DHParser.dsl import run_compiler
from DHParser.dsl import compile_on_disk
MLW_ebnf = os.path.join('..', 'MLW.ebnf')
MLW_compiler = os.path.join('..', 'MLW_compiler.py')
......@@ -36,14 +36,14 @@ toolkit.logging_off()
if (not os.path.exists(MLW_compiler) or
grammar_changed(MLW_compiler, MLW_ebnf)):
print("recompiling parser")
errors = run_compiler(MLW_ebnf)
errors = compile_on_disk(MLW_ebnf)
if errors:
print('\n'.join(errors))
sys.exit(1)
toolkit.logging_on()
errors = run_compiler("fascitergula.mlw", MLW_compiler, ".xml")
errors = compile_on_disk("fascitergula.mlw", MLW_compiler, ".xml")
if errors:
print('\n'.join(errors))
sys.exit(1)
......@@ -2045,7 +2045,7 @@ def run_compiler(source_file, compiler_suite="", extension=".dst"):
script already exists only the parser name in the script will be
updated. (For this to work, the different names need to be delimited
by the standard `DELIMITER`-line!).
`run_compiler()` returns a list of error messages or an empty list if no
`compile_on_disk()` returns a list of error messages or an empty list if no
errors occured.
"""
filepath = os.path.normpath(source_file)
......
......@@ -23,12 +23,12 @@ limitations under the License.
import os
import sys
sys.path.append(os.path.abspath('../../'))
from DHParser.dsl import run_compiler, suite_outdated
from DHParser.dsl import compile_on_disk, suite_outdated
if (not os.path.exists('PopRetrieve_compiler.py') or
suite_outdated('PopRetrieve_compiler.py', 'PopRetrieve.ebnf')):
print("recompiling PopRetrieve parser")
errors = run_compiler("PopRetrieve.ebnf")
errors = compile_on_disk("PopRetrieve.ebnf")
if errors:
print('\n\n'.join(errors))
sys.exit(1)
......@@ -54,13 +54,13 @@ if (not os.path.exists('PopRetrieve_compiler.py') or
print("PopRetrieveTest 1")
errors = run_compiler("PopRetrieveTest.txt", 'PopRetrieve_compiler.py')
errors = compile_on_disk("PopRetrieveTest.txt", 'PopRetrieve_compiler.py')
if errors:
print(errors)
sys.exit(1)
print("PopRetrieveTest 2")
errors = run_compiler("PopRetrieveTest2.txt", 'PopRetrieve_compiler.py')
errors = compile_on_disk("PopRetrieveTest2.txt", 'PopRetrieve_compiler.py')
if errors:
print(errors)
sys.exit(1)
......@@ -70,7 +70,7 @@ if errors:
if (not os.path.exists('PopRetrieveComplement_compiler.py') or
suite_outdated('PopRetrieveComplement_compiler.py', 'PopRetrieveComplement.ebnf')):
print("recompiling PopRetrieveComplement parser")
errors = run_compiler("PopRetrieveComplement.ebnf")
errors = compile_on_disk("PopRetrieveComplement.ebnf")
if errors:
print('\n\n'.join(errors))
sys.exit(1)
......
......@@ -45,20 +45,23 @@ def runner(tests, namespace):
if name.lower().startswith('test') and inspect.isclass(namespace[name]):
tests.append(name)
obj = None
for test in tests:
if test.find('.') >= 0:
cls_name, method_name = test.split('.')
obj = instantiate(cls_name)
print("Running " + cls_name + "." + method_name)
exec('obj.' + method_name + '()')
else:
obj = instantiate(test)
for name in dir(obj):
if name.lower().startswith("test"):
print("Running " + test + "." + name)
exec('obj.' + name + '()')
if "teardown" in dir(obj):
obj.teardown()
try:
if test.find('.') >= 0:
cls_name, method_name = test.split('.')
obj = instantiate(cls_name)
print("Running " + cls_name + "." + method_name)
exec('obj.' + method_name + '()')
else:
obj = instantiate(test)
for name in dir(obj):
if name.lower().startswith("test"):
print("Running " + test + "." + name)
exec('obj.' + name + '()')
finally:
if "teardown" in dir(obj):
obj.teardown()
if __name__ == "__main__":
# runner("", globals())
......
......@@ -22,27 +22,58 @@ limitations under the License.
import os
import sys
from DHParser.dsl import run_compiler
from DHParser.dsl import compile_on_disk, run_compiler
class TestCompilerGeneration:
trivial_lang = """
text = { word | WSPC }
text = { word | WSPC } "."
word = /\w+/
WSPC = /\s*/
WSPC = /\s+/
"""
trivial_text = """Es war ein König in Thule."""
grammar_name = "tmp/TestCompilerGeneration.ebnf"
compiler_name = "tmp/TestCompilerGeneration_compiler.py"
text_name = "tmp/TestCompilerGeneration_text.txt"
result_name = "tmp/TestCompilerGeneration_text.xml"
def setup(self):
with open(self.grammar_name, "w") as f:
f.write(self.trivial_lang)
with open(self.text_name, "w") as f:
f.write(self.trivial_text)
def teardown(self):
for name in (self.grammar_name, self.compiler_name, self.text_name, self.result_name):
if os.path.exists(name):
os.remove(name)
def test_compiling_functions(self):
# test if cutting and reassembling of compiler suite works:
errors = compile_on_disk(self.grammar_name)
assert not errors
with open(self.compiler_name, 'r') as f:
compiler_suite = f.read()
errors = compile_on_disk(self.grammar_name)
assert not errors
with open(self.compiler_name, 'r') as f:
compiler_suite_2nd_run = f.read()
assert compiler_suite == compiler_suite_2nd_run
# test compiling with a generated compiler suite
errors = compile_on_disk(self.text_name, self.compiler_name)
assert not errors, str(errors)
assert os.path.exists(self.result_name)
with open(self.result_name, 'r') as f:
output = f.read()
# test compiling in memory
result = run_compiler(self.trivial_text, self.compiler_name)
assert output == result.as_xml(), str(result)
if (not os.path.exists('PopRetrieve_compiler.py') or
suite_outdated('PopRetrieve_compiler.py', 'PopRetrieve.ebnf')):
print("recompiling PopRetrieve parser")
errors = run_compiler("PopRetrieve.ebnf")
if errors:
print('\n\n'.join(errors))
sys.exit(1)
sys.path.append('tmp')
from TestCompilerGeneration_compiler import compile_TestCompilerGeneration
result, errors, ast = compile_TestCompilerGeneration(self.trivial_text)
if __name__ == "__main__":
......
......@@ -42,6 +42,7 @@ class TestInfiLoopsAndRecursion:
# example: "5 + 3 * 4"
"""
snippet = "5 + 3 * 4"
print(compileEBNF(minilang, source_only=True))
parser = compileEBNF(minilang)()
assert parser
syntax_tree = parser.parse(snippet)
......
#!/usr/bin/python3
"""test_tookkit.py - tests of the toolkit-module of DHParser
Author: Eckhart Arnold <arnold@badw.de>
Copyright 2017 Bavarian Academy of Sciences and Humanities
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import sys
from DHParser.toolkit import load_if_file
class TestToolkit:
filename = "tmp/test.py"
code1 = "x = 46"
code2 = "def f():\n return 46"
def setup(self):
with open(self.filename, 'w') as f:
f.write(self.code2)
def teardown(self):
os.remove(self.filename)
def test_load_if_file(self):
# an error should be raised if file expected but not found
error_raised = False
try:
load_if_file('this_is_code_and_not_a_file')
except FileNotFoundError:
error_raised = True
assert error_raised
# multiline text will never be mistaken for a file
assert load_if_file('this_is_code_and_not_a_file\n')
# neither will text that does not look like a file name
s = "this is code and not a file"
assert s == load_if_file(s)
# not a file and not mistaken for a file
assert self.code1 == load_if_file(self.code1)
# not a file and not mistaken for a file either
assert self.code2 == load_if_file(self.code2)
# file correctly loaded
assert self.code2 == load_if_file(self.filename)
if __name__ == "__main__":
from run import runner
runner("", globals())
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment