Skip to content

GitLab

Commit def999dd authored by eckhart's avatar eckhart
Browse files

cleanups finished

parent 83a0b5dd
Hide whitespace changes
Inline Side-by-side
Showing with 74 additions and 57 deletions
DHParser/dsl.py
View file @ def999dd
......@@ -169,14 +169,15 @@ def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
# read grammar
grammar_src = load_if_file(grammar_representation)
if is_python_code(grammar_src):
parser_py, messages, AST = grammar_src, [], None # type: str, List[Error], Node
parser_py, messages = grammar_src, [] # type: str, List[Error]
else:
with logging(False):
parser_py, messages, AST = compile_source(grammar_src, None,
parser_py, messages, _ = compile_source(
grammar_src, None,
get_ebnf_grammar(), get_ebnf_transformer(), get_ebnf_compiler())
if has_errors(messages):
raise GrammarError(only_errors(messages), grammar_src)
parser_root = compile_python_object(DHPARSER_IMPORTS + parser_py, '\w+Grammar$')()
parser_root = compile_python_object(DHPARSER_IMPORTS + parser_py, r'\w+Grammar$')()
else:
# assume that dsl_grammar is a ParserHQ-object or Grammar class
grammar_src = ''
......@@ -197,7 +198,7 @@ def compileDSL(text_or_file: str,
Compiles a text in a domain specific language (DSL) with an
EBNF-specified grammar. Returns the compiled text or raises a
compilation error.
Raises:
CompilationError if any errors occurred during compilation
"""
......@@ -218,16 +219,16 @@ def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
Compiles an EBNF grammar file and returns the compiler object
that was used and which can now be queried for the result as well
as skeleton code for preprocessor, transformer and compiler objects.
Args:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str): Branding name for the compiler suite source
code.
code.
Returns:
An instance of class ``ebnf.EBNFCompiler``
Raises:
CompilationError if any errors occurred during compilation
CompilationError if any errors occurred during compilation
"""
grammar = get_ebnf_grammar()
compiler = get_ebnf_compiler(branding, ebnf_src)
......@@ -246,11 +247,11 @@ def compileEBNF(ebnf_src: str, branding="DSL") -> str:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str): Branding name for the compiler suite source
code.
code.
Returns:
The complete compiler suite skeleton as Python source code.
Raises:
CompilationError if any errors occurred during compilation
CompilationError if any errors occurred during compilation
"""
compiler = raw_compileEBNF(ebnf_src, branding)
src = ["#/usr/bin/python\n",
......@@ -272,23 +273,23 @@ def grammar_provider(ebnf_src: str, branding="DSL") -> Grammar:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str or bool): Branding name for the compiler
suite source code.
suite source code.
Returns:
A provider function for a grammar object for texts in the
language defined by ``ebnf_src``.
"""
grammar_src = compileDSL(ebnf_src, nil_preprocessor, get_ebnf_grammar(),
get_ebnf_transformer(), get_ebnf_compiler(branding))
return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_(?:\w+_)?grammar$')
return compile_python_object(DHPARSER_IMPORTS + grammar_src, r'get_(?:\w+_)?grammar$')
def load_compiler_suite(compiler_suite: str) -> \
Tuple[PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc]:
Tuple[PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc]:
"""
Extracts a compiler suite from file or string ``compiler suite``
and returns it as a tuple (preprocessor, parser, ast, compiler).
Returns:
4-tuple (preprocessor function, parser class, ast transformer function, compiler class)
"""
......@@ -298,26 +299,27 @@ def load_compiler_suite(compiler_suite: str) -> \
imports = DHPARSER_IMPORTS
if is_python_code(compiler_suite):
try:
intro, imports, preprocessor_py, parser_py, ast_py, compiler_py, outro = \
_, imports, preprocessor_py, parser_py, ast_py, compiler_py, _ = \
RX_SECTION_MARKER.split(source)
except ValueError as error:
except ValueError:
raise AssertionError('File "' + compiler_suite + '" seems to be corrupted. '
'Please delete or repair file manually.')
# TODO: Compile in one step and pick parts from namespace later ?
preprocessor = compile_python_object(imports + preprocessor_py, 'get_(?:\w+_)?preprocessor$')
parser = compile_python_object(imports + parser_py, 'get_(?:\w+_)?grammar$')
ast = compile_python_object(imports + ast_py, 'get_(?:\w+_)?transformer$')
preprocessor = compile_python_object(imports + preprocessor_py,
r'get_(?:\w+_)?preprocessor$')
parser = compile_python_object(imports + parser_py, r'get_(?:\w+_)?grammar$')
ast = compile_python_object(imports + ast_py, r'get_(?:\w+_)?transformer$')
else:
# assume source is an ebnf grammar. Is there really any reasonable application case for this?
with logging(False):
compile_py, messages, AST = compile_source(source, None, get_ebnf_grammar(),
get_ebnf_transformer(), get_ebnf_compiler())
compiler_py, messages, _ = compile_source(source, None, get_ebnf_grammar(),
get_ebnf_transformer(), get_ebnf_compiler())
if has_errors(messages):
raise GrammarError(only_errors(messages), source)
preprocessor = get_ebnf_preprocessor
parser = get_ebnf_grammar
ast = get_ebnf_transformer
compiler = compile_python_object(imports + compiler_py, 'get_(?:\w+_)?compiler$')
compiler = compile_python_object(imports + compiler_py, r'get_(?:\w+_)?compiler$')
return preprocessor, parser, ast, compiler
......@@ -325,12 +327,12 @@ def load_compiler_suite(compiler_suite: str) -> \
def is_outdated(compiler_suite: str, grammar_source: str) -> bool:
"""
Returns ``True`` if the ``compile_suite`` needs to be updated.
An update is needed, if either the grammar in the compieler suite
does not reflect the latest changes of ``grammar_source`` or if
An update is needed, if either the grammar in the compieler suite
does not reflect the latest changes of ``grammar_source`` or if
sections from the compiler suite have diligently been overwritten
with whitespace order to trigger their recreation. Note: Do not
delete or overwrite the section marker itself.
delete or overwrite the section marker itself.
Args:
compiler_suite: the parser class representing the grammar
......@@ -342,7 +344,7 @@ def is_outdated(compiler_suite: str, grammar_source: str) -> bool:
True, if ``compiler_suite`` seems to be out of date.
"""
try:
preprocessor, grammar, ast, compiler = load_compiler_suite(compiler_suite)
_, grammar, _, _ = load_compiler_suite(compiler_suite)
return grammar_changed(grammar(), grammar_source)
except ValueError:
return True
......@@ -353,17 +355,17 @@ def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
Args:
text_or_file (str): Either the file name of the source code or
the source code directly. (Which is determined by
the source code directly. (Which is determined by
heuristics. If ``text_or_file`` contains at least on
linefeed then it is always assumed to be a source text and
not a file name.)
compiler_suite(str): File name of the compiler suite to be
used.
Returns:
The result of the compilation, the form and type of which
The result of the compilation, the form and type of which
depends entirely on the compiler.
Raises:
CompilerError
"""
......@@ -398,7 +400,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
is written to a file with the same name but a different
extension than the source file. This parameter sets the
extension.
Returns:
A (potentially empty) list of error or warning messages.
"""
......
DHParser/ebnf.py
View file @ def999dd
......@@ -430,13 +430,13 @@ class EBNFCompiler(Compiler):
' # AST Transformations for the ' + self.grammar_name + '-grammar']
transtable.append(' "+": remove_empty,')
for name in self.rules:
tf = '[]'
transformations = '[]'
rule = self.definitions[name]
if rule.startswith('Alternative'):
tf = '[replace_or_reduce]'
transformations = '[replace_or_reduce]'
elif rule.startswith('Synonym'):
tf = '[replace_by_single_child]'
transtable.append(' "' + name + '": %s,' % tf)
transformations = '[replace_by_single_child]'
transtable.append(' "' + name + '": %s,' % transformations)
transtable.append(' ":Token, :RE": reduce_single_child,')
transtable += [' "*": replace_by_single_child', '}', '']
transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
......@@ -459,7 +459,7 @@ class EBNFCompiler(Compiler):
self.grammar_name + '", grammar_source=""):',
' super(' + self.grammar_name +
'Compiler, self).__init__(grammar_name, grammar_source)',
" assert re.match('\w+\Z', grammar_name)", '']
r" assert re.match('\w+\Z', grammar_name)", '']
for name in self.rules:
method_name = Compiler.method_name(name)
if name == self.root_symbol:
......@@ -548,6 +548,8 @@ class EBNFCompiler(Compiler):
defined_symbols.difference_update(self.RESERVED_SYMBOLS)
def remove_connections(symbol):
"""Recursiviely removes all symbols which appear in the
definiens of a particular symbol."""
if symbol in defined_symbols:
defined_symbols.remove(symbol)
for related in self.rules[symbol][1:]:
......@@ -555,8 +557,9 @@ class EBNFCompiler(Compiler):
remove_connections(self.root_symbol)
for leftover in defined_symbols:
self.rules[leftover][0].add_error(('Rule "%s" is not connected to '
'parser root "%s" !') % (leftover, self.root_symbol), Error.WARNING)
self.rules[leftover][0].add_error(
('Rule "%s" is not connected to parser root "%s" !') %
(leftover, self.root_symbol), Error.WARNING)
# set root_symbol parser and assemble python grammar definition
......@@ -573,6 +576,12 @@ class EBNFCompiler(Compiler):
def on_syntax(self, node: Node) -> str:
definitions = [] # type: List[Tuple[str, str]]
# transfer directives to re_flags where needed
if self.directives['ignorecase']:
self.re_flags.add('i')
elif 'i' in self.re_flags:
self.re_flags.remove('i')
# drop the wrapping sequence node
if len(node.children) == 1 and not node.children[0].parser.name:
node = node.children[0]
......@@ -678,6 +687,8 @@ class EBNFCompiler(Compiler):
self.directives['ignorecase'] == value
if value:
self.re_flags.add('i')
elif 'i' in self.re_flags:
self.re_flags.remove('i')
# elif key == 'testing':
# value = str(node.children[1])
......@@ -685,14 +696,14 @@ class EBNFCompiler(Compiler):
elif key == 'literalws':
value = {item.lower() for item in self.compile(node.children[1])}
if (len(value - {'left', 'right', 'both', 'none'}) > 0
if ((value - {'left', 'right', 'both', 'none'})
or ('none' in value and len(value) > 1)):
node.add_error('Directive "literalws" allows the values '
'`left`, `right`, `both` or `none`, '
'but not `%s`' % ", ".join(value))
ws = {'left', 'right'} if 'both' in value \
wsp = {'left', 'right'} if 'both' in value \
else {} if 'none' in value else value
self.directives[key] = list(ws)
self.directives[key] = list(wsp)
elif key in {'tokens', 'preprocessor_tokens'}:
self.directives['tokens'] |= self.compile(node.children[1])
......@@ -747,7 +758,7 @@ class EBNFCompiler(Compiler):
i += 1
saved_result = node.result
node.result = tuple(filtered_children)
custom_args = ['mandatory=%i' % mandatory_marker[0]] if mandatory_marker else []
custom_args = ['mandatory=%i' % mandatory_marker[0]] if mandatory_marker else []
compiled = self.non_terminal(node, 'Series', custom_args)
node.result = saved_result
return compiled
......@@ -832,10 +843,10 @@ class EBNFCompiler(Compiler):
# return self.non_terminal(node, 'Unordered')
assert len(node.children) == 1
nd = node.children[0]
for r in nd.children:
if r.parser.ptype == TOKEN_PTYPE and str(nd) == "§":
for child in nd.children:
if child.parser.ptype == TOKEN_PTYPE and str(nd) == "§":
node.add_error("Unordered parser lists cannot contain mandatory (§) items.")
args = ', '.join(self.compile(r) for r in nd.children)
args = ', '.join(self.compile(child) for child in nd.children)
if nd.parser.name == "term":
return "AllOf(" + args + ")"
elif nd.parser.name == "expression":
......
DHParser/testing.py
View file @ def999dd
......@@ -189,8 +189,8 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
# write parsing-history log only in case of failure!
parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
(test_name, parser_name, cst.as_sxpr()))
errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
(test_name, parser_name, cst.as_sxpr()))
elif "ast" in tests:
compare = mock_syntax_tree(tests["ast"][test_name])
if compare != ast:
......@@ -245,7 +245,8 @@ def grammar_suite(directory, parser_factory, transformer_factory,
print("\nScanning test-directory: " + directory)
save_cwd = os.getcwd()
os.chdir(directory)
if is_logging(): clear_logs()
if is_logging():
clear_logs()
for filename in sorted(os.listdir()):
if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
try:
......@@ -259,7 +260,8 @@ def grammar_suite(directory, parser_factory, transformer_factory,
if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
raise e
os.chdir(save_cwd)
error_report = []; err_N = 0
error_report = []
err_N = 0
if all_errors:
for filename in all_errors:
error_report.append('Errors found by unit test "%s":' % filename)
......@@ -267,16 +269,18 @@ def grammar_suite(directory, parser_factory, transformer_factory,
for error in all_errors[filename]:
error_report.append('\t' + '\n\t'.join(error.split('\n')))
if error_report:
if verbose: print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
if verbose:
print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
return ('Test suite "%s" revealed some errors:\n\n' % directory) + '\n'.join(error_report)
if verbose: print("\nSUCCESS! All tests passed :-)\n")
if verbose:
print("\nSUCCESS! All tests passed :-)\n")
return ''
def runner(tests, namespace):
"""
Runs all or some selected Python unit tests found in the
namespace. To run all tests in a module, call
namespace. To run all tests in a module, call
``runner("", globals())`` from within that module.
Args:
......@@ -284,9 +288,9 @@ def runner(tests, namespace):
names of test or test classes. Each test and, in the case
of a test class, all tests within the test class will be
run.
namespace: The namespace for running the test, usually
namespace: The namespace for running the test, usually
``globals()`` should be used.
Example:
class TestSomething()
def setup(self):
......@@ -295,10 +299,10 @@ def runner(tests, namespace):
pass
def test_something(self):
pass
if __name__ == "__main__":
from DHParser.testing import runner
runner("", globals())
runner("", globals())
"""
def instantiate(cls_name):
exec("obj = " + cls_name + "()", namespace)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment