Commit c97e4d48 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

better dsl compiling

parent 991c1065
......@@ -207,9 +207,17 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
which conforms to the language defined by ``ebnf_src``.
"""
grammar = ebnf_grammar_obj or get_ebnf_grammar()
grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, get_ebnf_compiler())
return grammar_src if source_only else \
compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
compiler = get_ebnf_compiler("DSL")
grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler)
if source_only:
src = [SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_IMPORTS,
SECTION_MARKER.format(marker=SCANNER_SECTION), compiler.gen_scanner_skeleton(),
SECTION_MARKER.format(marker=PARSER_SECTION), grammar_src,
SECTION_MARKER.format(marker=AST_SECTION), compiler.gen_transformer_skeleton(),
SECTION_MARKER.format(marker=COMPILER_SECTION), compiler.gen_compiler_skeleton()]
return '\n'.join(src)
else:
return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$')
def load_compiler_suite(compiler_suite):
......@@ -345,7 +353,8 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
elif trans == get_ebnf_transformer or trans == EBNFTransformer: # either an EBNF- or no compiler suite given
global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER
AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
DHPARSER_MAIN, DHPARSER_IMPORTS
f = None
try:
f = open(rootname + '_compiler.py', 'r', encoding="utf-8")
......
......@@ -192,9 +192,9 @@ EBNF_transformation_table = {
remove_expendables,
"directive, definition":
partial(remove_tokens, tokens={'@', '='}),
"expression, chain":
"expression":
[replace_by_single_child, flatten,
partial(remove_tokens, tokens={'|', '--'})],
partial(remove_tokens, tokens={'|'})],
"term":
[replace_by_single_child, flatten], # supports both idioms: "{ factor }+" and "factor { factor }"
"factor, flowmarker, retrieveop":
......
......@@ -1081,8 +1081,10 @@ def compile_source(source, scanner, parser, transformer, compiler):
return result, messages, syntax_tree
def test_grammar(test_suite, parser, transform):
def test_grammar(test_suite, parser_factory, transformer_factory):
errata = []
parser = parser_factory()
transform = transformer_factory()
for parser_name, tests in test_suite.items():
assert set(tests.keys()).issubset({'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'})
......
......@@ -24,7 +24,22 @@ import os
import sys
sys.path.extend(['../', './'])
from DHParser.dsl import compile_on_disk, run_compiler
from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF
ARITHMETIC_EBNF = """
@ whitespace = linefeed
formula = [ //~ ] expr
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/~
# example: "5 + 3 * 4"
"""
class TestCompileFunctions:
def test_compileEBNF(self):
parser_src = compileEBNF(ARITHMETIC_EBNF, source_only=True)
print(parser_src)
class TestCompilerGeneration:
trivial_lang = """
......@@ -81,4 +96,4 @@ class TestCompilerGeneration:
if __name__ == "__main__":
from run import runner
runner("", globals())
\ No newline at end of file
runner("TestCompileFunctions", globals())
\ No newline at end of file
......@@ -19,25 +19,41 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
from functools import partial
import sys
sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging, compile_python_object
from DHParser.syntaxtree import no_operation, traverse
from DHParser.parsers import compile_source
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, DHPARSER_IMPORTS
ARITHMETIC_EBNF = """
@ whitespace = linefeed
formula = [ //~ ] expr
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/~
# example: "5 + 3 * 4"
"""
ARITHMETIC_EBNF_transformation_table = {
# AST Transformations for the DSL-grammar
"formula": no_operation,
"expr": no_operation,
"term": no_operation,
"factor": no_operation,
"": no_operation
}
ARITHMETIC_EBNFTransform = partial(traverse, processing_table=ARITHMETIC_EBNF_transformation_table)
class TestInfiLoopsAndRecursion:
def test_direct_left_recursion(self):
minilang = """
@ whitespace = linefeed
formula = [ //~ ] expr
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/~
# example: "5 + 3 * 4"
"""
minilang = ARITHMETIC_EBNF
snippet = "5 + 3 * 4"
parser = compileEBNF(minilang)()
assert parser
......@@ -60,6 +76,21 @@ class TestInfiLoopsAndRecursion:
# print(syntax_tree.collect_errors())
class TestTestGrammar:
cases = {
"factor": {
"match": {
1: "0",
2: "314",
},
"fail": {
1: "21F",
2: "G123"
}
}
}
class TestRegex:
def test_multilineRegex(self):
mlregex = r"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment