2.12.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit c721e133 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

split compileEBNF into two functions: compileEBNF and compile_parser;...

split compileEBNF into two functions: compileEBNF and compile_parser; adjustments for compatibility with pypy3 / python 3.5
parent 6d4c1c4a
...@@ -38,6 +38,8 @@ __all__ = ['GrammarError', ...@@ -38,6 +38,8 @@ __all__ = ['GrammarError',
'CompilationError', 'CompilationError',
'load_compiler_suite', 'load_compiler_suite',
'compileDSL', 'compileDSL',
'compileEBNF',
'compile_parser',
'compile_on_disk'] 'compile_on_disk']
...@@ -60,31 +62,6 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved. ...@@ -60,31 +62,6 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved.
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS" END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
class GrammarError(Exception):
"""Raised when (already) the grammar of a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, grammar_src):
self.error_messages = error_messages
self.grammar_src = grammar_src
class CompilationError(Exception):
"""Raised when a string or file in a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, dsl_text, dsl_grammar, AST):
self.error_messages = error_messages
self.dsl_text = dsl_text
self.dsl_grammar = dsl_grammar
self.AST = AST
def __str__(self):
return '\n'.join(self.error_messages)
DHPARSER_IMPORTS = ''' DHPARSER_IMPORTS = '''
from functools import partial from functools import partial
import os import os
...@@ -134,6 +111,31 @@ if __name__ == "__main__": ...@@ -134,6 +111,31 @@ if __name__ == "__main__":
''' '''
class GrammarError(Exception):
"""Raised when (already) the grammar of a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, grammar_src):
self.error_messages = error_messages
self.grammar_src = grammar_src
class CompilationError(Exception):
"""Raised when a string or file in a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, dsl_text, dsl_grammar, AST):
self.error_messages = error_messages
self.dsl_text = dsl_text
self.dsl_grammar = dsl_grammar
self.AST = AST
def __str__(self):
return '\n'.join(self.error_messages)
def grammar_instance(grammar_representation): def grammar_instance(grammar_representation):
"""Returns a grammar object and the source code of the grammar, from """Returns a grammar object and the source code of the grammar, from
the given `grammar`-data which can be either a file name, ebnf-code, the given `grammar`-data which can be either a file name, ebnf-code,
...@@ -184,7 +186,7 @@ def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler) ...@@ -184,7 +186,7 @@ def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler)
return result return result
def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False): def compileEBNF(ebnf_src, branding = "DSL"):
"""Compiles an EBNF source file. Either returns a factory function """Compiles an EBNF source file. Either returns a factory function
for the grammar-paraser or the source code of a compiler suite with for the grammar-paraser or the source code of a compiler suite with
skeletons for scanner, transformer and compiler. skeletons for scanner, transformer and compiler.
...@@ -192,31 +194,41 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False): ...@@ -192,31 +194,41 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
Args: Args:
ebnf_src(str): Either the file name of an EBNF grammar or ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string. the EBNF grammar itself as a string.
ebnf_grammar_obj: An existing instance of the branding (str or bool): Branding name for the compiler
DHParser.EBNFcompiler.EBNFGrammar object. This can speed suite source code.
up compilation, because no new EBNFGrammar object needs to
be instantiated.
source_only (str or bool): Branding name for the compiler
suite source code. If ``True`` the default branding "DSL"
will be used. If False (default), no source but a factory
function for grammar-parser callables will be returned.
Returns: Returns:
A factory function for a grammar-parser for texts in the The complete compiler suite skeleton as Python source code.
language defined by ``ebnf_src``. With the ``source_only``
a complete compiler suite skeleton will be returned instead.
""" """
grammar = ebnf_grammar_obj or get_ebnf_grammar() grammar = get_ebnf_grammar()
if source_only == True: source_only = "DSL" if branding == True: branding = "DSL"
compiler = get_ebnf_compiler(source_only or "DSL") compiler = get_ebnf_compiler(branding or "DSL", ebnf_src)
grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler) grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler)
if source_only: src = [DHPARSER_IMPORTS,
src = [DHPARSER_IMPORTS, compiler.gen_scanner_skeleton(), grammar_src, compiler.gen_scanner_skeleton(),
compiler.gen_transformer_skeleton(), compiler.gen_compiler_skeleton(), grammar_src,
DHPARSER_MAIN.format(NAME=source_only)] compiler.gen_transformer_skeleton(),
return '\n'.join(src) compiler.gen_compiler_skeleton(),
else: DHPARSER_MAIN.format(NAME=branding)]
return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$') return '\n'.join(src)
def compile_parser(ebnf_src, branding="DSL"):
"""Compiles an EBNF grammar and returns a grammar-parser factory
function for that grammar.
Args:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str or bool): Branding name for the compiler
suite source code.
Returns:
A factory function for a grammar-parser for texts in the
language defined by ``ebnf_src``.
"""
grammar_src = compileDSL(ebnf_src, nil_scanner, get_ebnf_grammar(),
get_ebnf_transformer(), get_ebnf_compiler(branding))
return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$')
def load_compiler_suite(compiler_suite): def load_compiler_suite(compiler_suite):
...@@ -350,7 +362,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"): ...@@ -350,7 +362,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
if errors: if errors:
return errors return errors
elif trans == get_ebnf_transformer or trans == EBNFTransformer: # either an EBNF- or no compiler suite given elif cfactory == get_ebnf_compiler: # trans == get_ebnf_transformer or trans == EBNFTransformer: # either an EBNF- or no compiler suite given
global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \ global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \ AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
DHPARSER_MAIN, DHPARSER_IMPORTS DHPARSER_MAIN, DHPARSER_IMPORTS
......
...@@ -105,9 +105,11 @@ class ZombieParser(MockParser): ...@@ -105,9 +105,11 @@ class ZombieParser(MockParser):
ZOMBIE_PARSER = ZombieParser() ZOMBIE_PARSER = ZombieParser()
class Error(NamedTuple): # # Python 3.6:
pos: int # class Error(NamedTuple):
msg: str # pos: int
# msg: str
Error = NamedTuple('Error', [('pos', int), ('msg', str)])
class Node: class Node:
......
...@@ -267,7 +267,7 @@ def smart_list(arg): ...@@ -267,7 +267,7 @@ def smart_list(arg):
if len(lst) > 1: if len(lst) > 1:
return (s.strip() for s in lst) return (s.strip() for s in lst)
return (s.strip() for s in arg.strip().split(' ')) return (s.strip() for s in arg.strip().split(' '))
elif isinstance(arg, collections.abc.Collection): elif isinstance(arg, collections.abc.Sequence): # python 3.6: collections.abc.Collection
return arg return arg
elif isinstance(arg, collections.abc.Iterable): elif isinstance(arg, collections.abc.Iterable):
return list(arg) return list(arg)
...@@ -301,20 +301,24 @@ def sane_parser_name(name): ...@@ -301,20 +301,24 @@ def sane_parser_name(name):
return name and name[:2] != '__' and name[-2:] != '__' return name and name[:2] != '__' and name[-2:] != '__'
def compile_python_object(python_src, catch_obj_regex): def compile_python_object(python_src, catch_obj_regex=""):
"""Compiles the python source code and returns the object the name of which """Compiles the python source code and returns the (first) object
ends is matched by ``catch_obj_regex``. the name of which is matched by ``catch_obj_regex``. If catch_obj
is the empty string, the namespace dictionary will be returned.
""" """
if isinstance(catch_obj_regex, str): if isinstance(catch_obj_regex, str):
catch_obj_regex = re.compile(catch_obj_regex) catch_obj_regex = re.compile(catch_obj_regex)
code = compile(python_src, '<string>', 'exec') code = compile(python_src, '<string>', 'exec')
namespace = {} namespace = {}
exec(code, namespace) # safety risk? exec(code, namespace) # safety risk?
matches = [key for key in namespace.keys() if catch_obj_regex.match(key)] if catch_obj_regex:
if len(matches) == 0: matches = [key for key in namespace.keys() if catch_obj_regex.match(key)]
raise ValueError("No object matching /%s/ defined in source code." % if len(matches) == 0:
catch_obj_regex.pattern) raise ValueError("No object matching /%s/ defined in source code." %
elif len(matches) > 1: catch_obj_regex.pattern)
raise ValueError("Ambigous matches for %s : %s" % elif len(matches) > 1:
(str(catch_obj_regex), str(matches))) raise ValueError("Ambigous matches for %s : %s" %
return namespace[matches[0]] if matches else None (str(catch_obj_regex), str(matches)))
return namespace[matches[0]] if matches else None
else:
return namespace
...@@ -24,7 +24,7 @@ import os ...@@ -24,7 +24,7 @@ import os
import sys import sys
sys.path.extend(['../', './']) sys.path.extend(['../', './'])
from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF, compile_parser
ARITHMETIC_EBNF = """ ARITHMETIC_EBNF = """
...@@ -38,13 +38,13 @@ ARITHMETIC_EBNF = """ ...@@ -38,13 +38,13 @@ ARITHMETIC_EBNF = """
class TestCompileFunctions: class TestCompileFunctions:
def test_compileEBNF(self): def test_compileEBNF(self):
parser_src = compileEBNF(ARITHMETIC_EBNF, source_only=True) parser_src = compileEBNF(ARITHMETIC_EBNF)
assert isinstance(parser_src, str), str(type(parser_src)) assert isinstance(parser_src, str), str(type(parser_src))
assert parser_src.find('get_DSL') >= 0 assert parser_src.find('get_DSL') >= 0
parser_src = compileEBNF(ARITHMETIC_EBNF, source_only="CustomDSL") parser_src = compileEBNF(ARITHMETIC_EBNF, branding="CustomDSL")
assert isinstance(parser_src, str), str(type(parser_src)) assert isinstance(parser_src, str), str(type(parser_src))
assert parser_src.find('get_CustomDSL') >= 0 assert parser_src.find('get_CustomDSL') >= 0
parser_factory = compileEBNF(ARITHMETIC_EBNF, source_only=False) parser_factory = compile_parser(ARITHMETIC_EBNF, branding="TestDSL")
assert callable(parser_factory) assert callable(parser_factory)
parser = parser_factory() parser = parser_factory()
result = parser("5 + 3 * 4") result = parser("5 + 3 * 4")
......
...@@ -28,7 +28,7 @@ sys.path.extend(['../', './']) ...@@ -28,7 +28,7 @@ sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging from DHParser.toolkit import is_logging
from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, compileDSL from DHParser.dsl import compileEBNF, compileDSL, compile_parser
class TestDirectives: class TestDirectives:
...@@ -42,7 +42,7 @@ class TestDirectives: ...@@ -42,7 +42,7 @@ class TestDirectives:
def test_whitespace_linefeed(self): def test_whitespace_linefeed(self):
lang = "@ whitespace = linefeed\n" + self.mini_language lang = "@ whitespace = linefeed\n" + self.mini_language
MinilangParser = compileEBNF(lang) MinilangParser = compile_parser(lang)
parser = MinilangParser() parser = MinilangParser()
assert parser assert parser
syntax_tree = parser("3 + 4 * 12") syntax_tree = parser("3 + 4 * 12")
...@@ -58,7 +58,7 @@ class TestDirectives: ...@@ -58,7 +58,7 @@ class TestDirectives:
def test_whitespace_vertical(self): def test_whitespace_vertical(self):
lang = "@ whitespace = vertical\n" + self.mini_language lang = "@ whitespace = vertical\n" + self.mini_language
parser = compileEBNF(lang)() parser = compile_parser(lang)()
assert parser assert parser
syntax_tree = parser("3 + 4 * 12") syntax_tree = parser("3 + 4 * 12")
assert not syntax_tree.collect_errors() assert not syntax_tree.collect_errors()
...@@ -71,7 +71,7 @@ class TestDirectives: ...@@ -71,7 +71,7 @@ class TestDirectives:
def test_whitespace_horizontal(self): def test_whitespace_horizontal(self):
lang = "@ whitespace = horizontal\n" + self.mini_language lang = "@ whitespace = horizontal\n" + self.mini_language
parser = compileEBNF(lang)() parser = compile_parser(lang)()
assert parser assert parser
syntax_tree = parser("3 + 4 * 12") syntax_tree = parser("3 + 4 * 12")
assert not syntax_tree.collect_errors() assert not syntax_tree.collect_errors()
...@@ -131,8 +131,8 @@ class TestPopRetrieve: ...@@ -131,8 +131,8 @@ class TestPopRetrieve:
""" """
def setup(self): def setup(self):
self.minilang_parser = compileEBNF(self.mini_language)() self.minilang_parser = compile_parser(self.mini_language)()
self.minilang_parser2 = compileEBNF(self.mini_lang2)() self.minilang_parser2 = compile_parser(self.mini_lang2)()
@staticmethod @staticmethod
def opening_delimiter(node, name): def opening_delimiter(node, name):
......
...@@ -28,7 +28,7 @@ from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \ ...@@ -28,7 +28,7 @@ from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten replace_by_single_child, reduce_single_child, flatten
from DHParser.parsers import compile_source from DHParser.parsers import compile_source
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, DHPARSER_IMPORTS from DHParser.dsl import compile_parser, DHPARSER_IMPORTS
ARITHMETIC_EBNF = """ ARITHMETIC_EBNF = """
...@@ -72,7 +72,7 @@ class TestInfiLoopsAndRecursion: ...@@ -72,7 +72,7 @@ class TestInfiLoopsAndRecursion:
def test_direct_left_recursion(self): def test_direct_left_recursion(self):
minilang = ARITHMETIC_EBNF minilang = ARITHMETIC_EBNF
snippet = "5 + 3 * 4" snippet = "5 + 3 * 4"
parser = compileEBNF(minilang)() parser = compile_parser(minilang)()
assert parser assert parser
syntax_tree = parser(snippet) syntax_tree = parser(snippet)
assert not syntax_tree.collect_errors() assert not syntax_tree.collect_errors()
...@@ -87,7 +87,7 @@ class TestInfiLoopsAndRecursion: ...@@ -87,7 +87,7 @@ class TestInfiLoopsAndRecursion:
def test_inifinite_loops(self): def test_inifinite_loops(self):
minilang = """not_forever = { // } \n""" minilang = """not_forever = { // } \n"""
snippet = " " snippet = " "
parser = compileEBNF(minilang)() parser = compile_parser(minilang)()
syntax_tree = parser(snippet) syntax_tree = parser(snippet)
assert syntax_tree.error_flag assert syntax_tree.error_flag
# print(syntax_tree.collect_errors()) # print(syntax_tree.collect_errors())
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment