In January 2021 we will introduce a 10 GB quota for project repositories. Higher limits for individual projects will be available on request. Please see https://doku.lrz.de/display/PUBLIC/GitLab for more information.

Commit c721e133 authored by Eckhart Arnold's avatar Eckhart Arnold

split compileEBNF into two functions: compileEBNF and compile_parser;...

split compileEBNF into two functions: compileEBNF and compile_parser; adjustments for compatibility with pypy3 / python 3.5
parent 6d4c1c4a
......@@ -38,6 +38,8 @@ __all__ = ['GrammarError',
'CompilationError',
'load_compiler_suite',
'compileDSL',
'compileEBNF',
'compile_parser',
'compile_on_disk']
......@@ -60,31 +62,6 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved.
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
class GrammarError(Exception):
"""Raised when (already) the grammar of a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, grammar_src):
self.error_messages = error_messages
self.grammar_src = grammar_src
class CompilationError(Exception):
"""Raised when a string or file in a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, dsl_text, dsl_grammar, AST):
self.error_messages = error_messages
self.dsl_text = dsl_text
self.dsl_grammar = dsl_grammar
self.AST = AST
def __str__(self):
return '\n'.join(self.error_messages)
DHPARSER_IMPORTS = '''
from functools import partial
import os
......@@ -134,6 +111,31 @@ if __name__ == "__main__":
'''
class GrammarError(Exception):
"""Raised when (already) the grammar of a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, grammar_src):
self.error_messages = error_messages
self.grammar_src = grammar_src
class CompilationError(Exception):
"""Raised when a string or file in a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, dsl_text, dsl_grammar, AST):
self.error_messages = error_messages
self.dsl_text = dsl_text
self.dsl_grammar = dsl_grammar
self.AST = AST
def __str__(self):
return '\n'.join(self.error_messages)
def grammar_instance(grammar_representation):
"""Returns a grammar object and the source code of the grammar, from
the given `grammar`-data which can be either a file name, ebnf-code,
......@@ -184,7 +186,7 @@ def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler)
return result
def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
def compileEBNF(ebnf_src, branding = "DSL"):
"""Compiles an EBNF source file. Either returns a factory function
for the grammar-paraser or the source code of a compiler suite with
skeletons for scanner, transformer and compiler.
......@@ -192,31 +194,41 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
Args:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
ebnf_grammar_obj: An existing instance of the
DHParser.EBNFcompiler.EBNFGrammar object. This can speed
up compilation, because no new EBNFGrammar object needs to
be instantiated.
source_only (str or bool): Branding name for the compiler
suite source code. If ``True`` the default branding "DSL"
will be used. If False (default), no source but a factory
function for grammar-parser callables will be returned.
branding (str or bool): Branding name for the compiler
suite source code.
Returns:
A factory function for a grammar-parser for texts in the
language defined by ``ebnf_src``. With the ``source_only``
a complete compiler suite skeleton will be returned instead.
The complete compiler suite skeleton as Python source code.
"""
grammar = ebnf_grammar_obj or get_ebnf_grammar()
if source_only == True: source_only = "DSL"
compiler = get_ebnf_compiler(source_only or "DSL")
grammar = get_ebnf_grammar()
if branding == True: branding = "DSL"
compiler = get_ebnf_compiler(branding or "DSL", ebnf_src)
grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler)
if source_only:
src = [DHPARSER_IMPORTS, compiler.gen_scanner_skeleton(), grammar_src,
compiler.gen_transformer_skeleton(), compiler.gen_compiler_skeleton(),
DHPARSER_MAIN.format(NAME=source_only)]
return '\n'.join(src)
else:
return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$')
src = [DHPARSER_IMPORTS,
compiler.gen_scanner_skeleton(),
grammar_src,
compiler.gen_transformer_skeleton(),
compiler.gen_compiler_skeleton(),
DHPARSER_MAIN.format(NAME=branding)]
return '\n'.join(src)
def compile_parser(ebnf_src, branding="DSL"):
"""Compiles an EBNF grammar and returns a grammar-parser factory
function for that grammar.
Args:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str or bool): Branding name for the compiler
suite source code.
Returns:
A factory function for a grammar-parser for texts in the
language defined by ``ebnf_src``.
"""
grammar_src = compileDSL(ebnf_src, nil_scanner, get_ebnf_grammar(),
get_ebnf_transformer(), get_ebnf_compiler(branding))
return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$')
def load_compiler_suite(compiler_suite):
......@@ -350,7 +362,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
if errors:
return errors
elif trans == get_ebnf_transformer or trans == EBNFTransformer: # either an EBNF- or no compiler suite given
elif cfactory == get_ebnf_compiler: # trans == get_ebnf_transformer or trans == EBNFTransformer: # either an EBNF- or no compiler suite given
global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
DHPARSER_MAIN, DHPARSER_IMPORTS
......
......@@ -105,9 +105,11 @@ class ZombieParser(MockParser):
ZOMBIE_PARSER = ZombieParser()
class Error(NamedTuple):
pos: int
msg: str
# # Python 3.6:
# class Error(NamedTuple):
# pos: int
# msg: str
Error = NamedTuple('Error', [('pos', int), ('msg', str)])
class Node:
......
......@@ -267,7 +267,7 @@ def smart_list(arg):
if len(lst) > 1:
return (s.strip() for s in lst)
return (s.strip() for s in arg.strip().split(' '))
elif isinstance(arg, collections.abc.Collection):
elif isinstance(arg, collections.abc.Sequence): # python 3.6: collections.abc.Collection
return arg
elif isinstance(arg, collections.abc.Iterable):
return list(arg)
......@@ -301,20 +301,24 @@ def sane_parser_name(name):
return name and name[:2] != '__' and name[-2:] != '__'
def compile_python_object(python_src, catch_obj_regex):
"""Compiles the python source code and returns the object the name of which
ends is matched by ``catch_obj_regex``.
def compile_python_object(python_src, catch_obj_regex=""):
"""Compiles the python source code and returns the (first) object
the name of which is matched by ``catch_obj_regex``. If catch_obj
is the empty string, the namespace dictionary will be returned.
"""
if isinstance(catch_obj_regex, str):
catch_obj_regex = re.compile(catch_obj_regex)
code = compile(python_src, '<string>', 'exec')
namespace = {}
exec(code, namespace) # safety risk?
matches = [key for key in namespace.keys() if catch_obj_regex.match(key)]
if len(matches) == 0:
raise ValueError("No object matching /%s/ defined in source code." %
catch_obj_regex.pattern)
elif len(matches) > 1:
raise ValueError("Ambigous matches for %s : %s" %
(str(catch_obj_regex), str(matches)))
return namespace[matches[0]] if matches else None
if catch_obj_regex:
matches = [key for key in namespace.keys() if catch_obj_regex.match(key)]
if len(matches) == 0:
raise ValueError("No object matching /%s/ defined in source code." %
catch_obj_regex.pattern)
elif len(matches) > 1:
raise ValueError("Ambigous matches for %s : %s" %
(str(catch_obj_regex), str(matches)))
return namespace[matches[0]] if matches else None
else:
return namespace
......@@ -24,7 +24,7 @@ import os
import sys
sys.path.extend(['../', './'])
from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF
from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF, compile_parser
ARITHMETIC_EBNF = """
......@@ -38,13 +38,13 @@ ARITHMETIC_EBNF = """
class TestCompileFunctions:
def test_compileEBNF(self):
parser_src = compileEBNF(ARITHMETIC_EBNF, source_only=True)
parser_src = compileEBNF(ARITHMETIC_EBNF)
assert isinstance(parser_src, str), str(type(parser_src))
assert parser_src.find('get_DSL') >= 0
parser_src = compileEBNF(ARITHMETIC_EBNF, source_only="CustomDSL")
parser_src = compileEBNF(ARITHMETIC_EBNF, branding="CustomDSL")
assert isinstance(parser_src, str), str(type(parser_src))
assert parser_src.find('get_CustomDSL') >= 0
parser_factory = compileEBNF(ARITHMETIC_EBNF, source_only=False)
parser_factory = compile_parser(ARITHMETIC_EBNF, branding="TestDSL")
assert callable(parser_factory)
parser = parser_factory()
result = parser("5 + 3 * 4")
......
......@@ -28,7 +28,7 @@ sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging
from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, compileDSL
from DHParser.dsl import compileEBNF, compileDSL, compile_parser
class TestDirectives:
......@@ -42,7 +42,7 @@ class TestDirectives:
def test_whitespace_linefeed(self):
lang = "@ whitespace = linefeed\n" + self.mini_language
MinilangParser = compileEBNF(lang)
MinilangParser = compile_parser(lang)
parser = MinilangParser()
assert parser
syntax_tree = parser("3 + 4 * 12")
......@@ -58,7 +58,7 @@ class TestDirectives:
def test_whitespace_vertical(self):
lang = "@ whitespace = vertical\n" + self.mini_language
parser = compileEBNF(lang)()
parser = compile_parser(lang)()
assert parser
syntax_tree = parser("3 + 4 * 12")
assert not syntax_tree.collect_errors()
......@@ -71,7 +71,7 @@ class TestDirectives:
def test_whitespace_horizontal(self):
lang = "@ whitespace = horizontal\n" + self.mini_language
parser = compileEBNF(lang)()
parser = compile_parser(lang)()
assert parser
syntax_tree = parser("3 + 4 * 12")
assert not syntax_tree.collect_errors()
......@@ -131,8 +131,8 @@ class TestPopRetrieve:
"""
def setup(self):
self.minilang_parser = compileEBNF(self.mini_language)()
self.minilang_parser2 = compileEBNF(self.mini_lang2)()
self.minilang_parser = compile_parser(self.mini_language)()
self.minilang_parser2 = compile_parser(self.mini_lang2)()
@staticmethod
def opening_delimiter(node, name):
......
......@@ -28,7 +28,7 @@ from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten
from DHParser.parsers import compile_source
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, DHPARSER_IMPORTS
from DHParser.dsl import compile_parser, DHPARSER_IMPORTS
ARITHMETIC_EBNF = """
......@@ -72,7 +72,7 @@ class TestInfiLoopsAndRecursion:
def test_direct_left_recursion(self):
minilang = ARITHMETIC_EBNF
snippet = "5 + 3 * 4"
parser = compileEBNF(minilang)()
parser = compile_parser(minilang)()
assert parser
syntax_tree = parser(snippet)
assert not syntax_tree.collect_errors()
......@@ -87,7 +87,7 @@ class TestInfiLoopsAndRecursion:
def test_inifinite_loops(self):
minilang = """not_forever = { // } \n"""
snippet = " "
parser = compileEBNF(minilang)()
parser = compile_parser(minilang)()
syntax_tree = parser(snippet)
assert syntax_tree.error_flag
# print(syntax_tree.collect_errors())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment