Commit 991c1065 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

moved code generation of factory functions from dsl.py to ebnf.py; finished...

moved code generation of factory functions from dsl.py to ebnf.py; finished testing function at the end of parser.py (not yet tested)
parent 60c1491d
......@@ -27,7 +27,7 @@ try:
except ImportError:
import re
from .ebnf import EBNFGrammar, EBNFTransform, EBNFCompiler, grammar_changed, \
from .ebnf import EBNFGrammar, EBNFTransformer, EBNFCompiler, grammar_changed, \
get_ebnf_scanner, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from .toolkit import logging, load_if_file, is_python_code, compile_python_object
from .parsers import GrammarBase, CompilerBase, compile_source, nil_scanner
......@@ -105,43 +105,6 @@ from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \\
'''
DHPARSER_SCANNER = '''
def get_{NAME}_scanner():
return {NAME}Scanner
'''
DHPARSER_GRAMMAR = '''
def get_{NAME}_grammar():
global thread_local_{NAME}_grammar_singleton
try:
grammar = thread_local_{NAME}_grammar_singleton
return grammar
except NameError:
thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
return thread_local_{NAME}_grammar_singleton
'''
DHPARSER_TRANSFORMER = '''
def get_{NAME}_transformer():
return {NAME}Transform
'''
DHPARSER_COMPILER = '''
def get_{NAME}_compiler(grammar_name="{NAME}", grammar_source=""):
global thread_local_{NAME}_compiler_singleton
try:
compiler = thread_local_{NAME}_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except NameError:
thread_local_{NAME}_compiler_singleton = {NAME}Compiler(grammar_name, grammar_source)
return thread_local_{NAME}_compiler_singleton
'''
DHPARSER_MAIN = '''
def compile_{NAME}(source):
"""Compiles ``source`` and returns (result, errors, ast).
......@@ -243,8 +206,8 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
A Grammar class that can be instantiated for parsing a text
which conforms to the language defined by ``ebnf_src``.
"""
grammar = ebnf_grammar_obj or EBNFGrammar()
grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransform, EBNFCompiler())
grammar = ebnf_grammar_obj or get_ebnf_grammar()
grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, get_ebnf_compiler())
return grammar_src if source_only else \
compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
......@@ -380,7 +343,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
if errors:
return errors
elif trans == get_ebnf_transformer or trans == EBNFTransform: # either an EBNF- or no compiler suite given
elif trans == get_ebnf_transformer or trans == EBNFTransformer: # either an EBNF- or no compiler suite given
global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER
f = None
......@@ -406,14 +369,11 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
if RX_WHITESPACE.fullmatch(imports):
imports = DHPARSER_IMPORTS
if RX_WHITESPACE.fullmatch(scanner):
scanner = compiler1.gen_scanner_skeleton() + \
DHPARSER_SCANNER.format(NAME=compiler_name)
scanner = compiler1.gen_scanner_skeleton()
if RX_WHITESPACE.fullmatch(ast):
ast = compiler1.gen_AST_skeleton() + \
DHPARSER_TRANSFORMER.format(NAME=compiler_name)
ast = compiler1.gen_transformer_skeleton()
if RX_WHITESPACE.fullmatch(compiler):
compiler = compiler1.gen_compiler_skeleton() + \
DHPARSER_COMPILER.format(NAME=compiler_name)
compiler = compiler1.gen_compiler_skeleton()
try:
f = open(rootname + '_compiler.py', 'w', encoding="utf-8")
......@@ -423,7 +383,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
f.write(SECTION_MARKER.format(marker=SCANNER_SECTION))
f.write(scanner)
f.write(SECTION_MARKER.format(marker=PARSER_SECTION))
f.write(result); f.write(DHPARSER_GRAMMAR.format(NAME=compiler_name))
f.write(result)
f.write(SECTION_MARKER.format(marker=AST_SECTION))
f.write(ast)
f.write(SECTION_MARKER.format(marker=COMPILER_SECTION))
......
......@@ -35,7 +35,7 @@ from .versionnumber import __version__
__all__ = ['EBNFGrammar',
'EBNFTransform',
'EBNFTransformer',
'EBNFCompilerError',
'EBNFCompiler',
'grammar_changed']
......@@ -222,13 +222,13 @@ EBNF_validation_table = {
}
def EBNFTransform(syntax_tree):
def EBNFTransformer(syntax_tree):
for processing_table in [EBNF_transformation_table, EBNF_validation_table]:
traverse(syntax_tree, processing_table)
def get_ebnf_transformer():
return EBNFTransform
return EBNFTransformer
########################################################################
......@@ -237,6 +237,48 @@ def get_ebnf_transformer():
#
########################################################################
SCANNER_FACTORY = '''
def get_{NAME}_scanner():
return {NAME}Scanner
'''
GRAMMAR_FACTORY = '''
def get_{NAME}_grammar():
global thread_local_{NAME}_grammar_singleton
try:
grammar = thread_local_{NAME}_grammar_singleton
return grammar
except NameError:
thread_local_{NAME}_grammar_singleton = {NAME}Grammar()
return thread_local_{NAME}_grammar_singleton
'''
TRANSFORMER_FACTORY = '''
def get_{NAME}_transformer():
return {NAME}Transform
'''
COMPILER_FACTORY = '''
def get_{NAME}_compiler(grammar_name="{NAME}",
grammar_source=""):
global thread_local_{NAME}_compiler_singleton
try:
compiler = thread_local_{NAME}_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except NameError:
thread_local_{NAME}_compiler_singleton = \\
{NAME}Compiler(grammar_name, grammar_source)
return thread_local_{NAME}_compiler_singleton
'''
class EBNFCompilerError(Exception):
"""Error raised by `EBNFCompiler` class. (Not compilation errors
......@@ -279,9 +321,10 @@ class EBNFCompiler(CompilerBase):
def gen_scanner_skeleton(self):
name = self.grammar_name + "Scanner"
return "def %s(text):\n return text\n" % name
return "def %s(text):\n return text\n" % name \
+ SCANNER_FACTORY.format(NAME=self.grammar_name)
def gen_AST_skeleton(self):
def gen_transformer_skeleton(self):
if not self.definition_names:
raise EBNFCompilerError('Compiler has not been run before calling '
'"gen_AST_Skeleton()"!')
......@@ -294,6 +337,7 @@ class EBNFCompiler(CompilerBase):
transtable.append(' "' + name + '": no_operation,')
transtable += [' "": no_operation', '}', '', tf_name +
' = partial(traverse, processing_table=%s)' % tt_name, '']
transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
return '\n'.join(transtable)
def gen_compiler_skeleton(self):
......@@ -317,6 +361,7 @@ class EBNFCompiler(CompilerBase):
else:
compiler += [' def ' + method_name + '(self, node):',
' pass', '']
compiler += [COMPILER_FACTORY.format(NAME=self.grammar_name)]
return '\n'.join(compiler)
def gen_parser(self, definitions):
......@@ -379,7 +424,8 @@ class EBNFCompiler(CompilerBase):
if self.root and 'root__' not in self.rules:
declarations.append('root__ = ' + self.root)
declarations.append('')
return '\n '.join(declarations)
return '\n '.join(declarations) \
+ GRAMMAR_FACTORY.format(NAME=self.grammar_name)
def on_syntax(self, node):
self._reset()
......
......@@ -288,7 +288,6 @@ class GrammarBase:
def __init__(self):
self.all_parsers = set()
self.dirty_flag = False
self.history_tracking = is_logging()
self._reset()
self._assign_parser_names()
self.root__ = copy.deepcopy(self.__class__.root__)
......@@ -315,6 +314,7 @@ class GrammarBase:
self.last_node = None
self.call_stack = [] # support for call stack tracing
self.history = [] # snapshots of call stacks
self.history_tracking = is_logging()
self.moving_forward = True # also needed for call stack tracing
def _add_parser(self, parser):
......@@ -1081,13 +1081,31 @@ def compile_source(source, scanner, parser, transformer, compiler):
return result, messages, syntax_tree
def test_grammar(test_suite, parse_function, transform):
def test_grammar(test_suite, parser, transform):
errata = []
for parser_name, tests in test_suite.items():
assert set(tests.keys()).issubset({'match', 'fail', 'ast', 'cst'})
assert set(tests.keys()).issubset({'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'})
for test_name, test_code in tests['match'].items():
cst = parse_function(test_code, parser_name)
cst = parser(test_code, parser_name)
tests['__cst__'][test_name] = cst
if cst.error_flag:
errata.append("Test %s for parser %s : didn't match: \n%s" %
(test_name, parser_name,
error_messages(test_code, cst.collect_errors())))
elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
errata.append("Test %s for parser %s : wrong CST: \n%s" %
(test_name, parser_name, cst.as_sexpr()))
elif "ast" in tests:
ast = transform(cst)
tests['__ast__'][test_name] = ast
if mock_syntax_tree(tests["ast"][test_name]) != ast:
errata.append("Test %s for parser %s : wrong AST: \n%s" %
(test_name, parser_name, ast.as_sexpr()))
for test_name, test_code in tests['fail'].items():
cst = parser(test_code, parser_name)
if not cst.error_flag:
yield "Test %s for parser %s did not match" % (test_name, parser_name)
if "cst" in tests:
if tests["cst"][test_name] != mock_syntax_tree(cst):
pass # TO BE CONTINUED
\ No newline at end of file
errata.append("Test %s for parser %s : shouldn't have matched!" %
(test_name, parser_name))
return errata
......@@ -46,9 +46,9 @@ class TestCompilerGeneration:
f.write(self.trivial_text)
def teardown(self):
# for name in (self.grammar_name, self.compiler_name, self.text_name, self.result_name):
# if os.path.exists(name):
# os.remove(name)
for name in (self.grammar_name, self.compiler_name, self.text_name, self.result_name):
if os.path.exists(name):
os.remove(name)
pass
def test_compiling_functions(self):
......
......@@ -27,7 +27,7 @@ sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging
from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransform, get_ebnf_compiler
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, compileDSL
......@@ -206,7 +206,7 @@ class TestSemanticValidation:
grammar = get_ebnf_grammar()
st = grammar(minilang)
assert not st.collect_errors()
EBNFTransform(st)
EBNFTransformer(st)
assert bool_filter(st.collect_errors())
def test_illegal_nesting(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment