The expiration time for new job artifacts in CI/CD pipelines is now 30 days (GitLab default). Previously generated artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit c721e133 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

split compileEBNF into two functions: compileEBNF and compile_parser;...

split compileEBNF into two functions: compileEBNF and compile_parser; adjustments for compatibility with pypy3 / python 3.5
parent 6d4c1c4a
...@@ -38,6 +38,8 @@ __all__ = ['GrammarError', ...@@ -38,6 +38,8 @@ __all__ = ['GrammarError',
'CompilationError', 'CompilationError',
'load_compiler_suite', 'load_compiler_suite',
'compileDSL', 'compileDSL',
'compileEBNF',
'compile_parser',
'compile_on_disk'] 'compile_on_disk']
...@@ -60,31 +62,6 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved. ...@@ -60,31 +62,6 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved.
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS" END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
class GrammarError(Exception):
"""Raised when (already) the grammar of a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, grammar_src):
self.error_messages = error_messages
self.grammar_src = grammar_src
class CompilationError(Exception):
"""Raised when a string or file in a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, dsl_text, dsl_grammar, AST):
self.error_messages = error_messages
self.dsl_text = dsl_text
self.dsl_grammar = dsl_grammar
self.AST = AST
def __str__(self):
return '\n'.join(self.error_messages)
DHPARSER_IMPORTS = ''' DHPARSER_IMPORTS = '''
from functools import partial from functools import partial
import os import os
...@@ -134,6 +111,31 @@ if __name__ == "__main__": ...@@ -134,6 +111,31 @@ if __name__ == "__main__":
''' '''
class GrammarError(Exception):
"""Raised when (already) the grammar of a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, grammar_src):
self.error_messages = error_messages
self.grammar_src = grammar_src
class CompilationError(Exception):
"""Raised when a string or file in a domain specific language (DSL)
contains errors.
"""
def __init__(self, error_messages, dsl_text, dsl_grammar, AST):
self.error_messages = error_messages
self.dsl_text = dsl_text
self.dsl_grammar = dsl_grammar
self.AST = AST
def __str__(self):
return '\n'.join(self.error_messages)
def grammar_instance(grammar_representation): def grammar_instance(grammar_representation):
"""Returns a grammar object and the source code of the grammar, from """Returns a grammar object and the source code of the grammar, from
the given `grammar`-data which can be either a file name, ebnf-code, the given `grammar`-data which can be either a file name, ebnf-code,
...@@ -184,7 +186,7 @@ def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler) ...@@ -184,7 +186,7 @@ def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler)
return result return result
def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False): def compileEBNF(ebnf_src, branding = "DSL"):
"""Compiles an EBNF source file. Either returns a factory function """Compiles an EBNF source file. Either returns a factory function
for the grammar-paraser or the source code of a compiler suite with for the grammar-paraser or the source code of a compiler suite with
skeletons for scanner, transformer and compiler. skeletons for scanner, transformer and compiler.
...@@ -192,30 +194,40 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False): ...@@ -192,30 +194,40 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
Args: Args:
ebnf_src(str): Either the file name of an EBNF grammar or ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string. the EBNF grammar itself as a string.
ebnf_grammar_obj: An existing instance of the branding (str or bool): Branding name for the compiler
DHParser.EBNFcompiler.EBNFGrammar object. This can speed suite source code.
up compilation, because no new EBNFGrammar object needs to
be instantiated.
source_only (str or bool): Branding name for the compiler
suite source code. If ``True`` the default branding "DSL"
will be used. If False (default), no source but a factory
function for grammar-parser callables will be returned.
Returns: Returns:
A factory function for a grammar-parser for texts in the The complete compiler suite skeleton as Python source code.
language defined by ``ebnf_src``. With the ``source_only``
a complete compiler suite skeleton will be returned instead.
""" """
grammar = ebnf_grammar_obj or get_ebnf_grammar() grammar = get_ebnf_grammar()
if source_only == True: source_only = "DSL" if branding == True: branding = "DSL"
compiler = get_ebnf_compiler(source_only or "DSL") compiler = get_ebnf_compiler(branding or "DSL", ebnf_src)
grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler) grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler)
if source_only: src = [DHPARSER_IMPORTS,
src = [DHPARSER_IMPORTS, compiler.gen_scanner_skeleton(), grammar_src, compiler.gen_scanner_skeleton(),
compiler.gen_transformer_skeleton(), compiler.gen_compiler_skeleton(), grammar_src,
DHPARSER_MAIN.format(NAME=source_only)] compiler.gen_transformer_skeleton(),
compiler.gen_compiler_skeleton(),
DHPARSER_MAIN.format(NAME=branding)]
return '\n'.join(src) return '\n'.join(src)
else:
def compile_parser(ebnf_src, branding="DSL"):
"""Compiles an EBNF grammar and returns a grammar-parser factory
function for that grammar.
Args:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str or bool): Branding name for the compiler
suite source code.
Returns:
A factory function for a grammar-parser for texts in the
language defined by ``ebnf_src``.
"""
grammar_src = compileDSL(ebnf_src, nil_scanner, get_ebnf_grammar(),
get_ebnf_transformer(), get_ebnf_compiler(branding))
return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$') return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$')
...@@ -350,7 +362,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"): ...@@ -350,7 +362,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
if errors: if errors:
return errors return errors
elif trans == get_ebnf_transformer or trans == EBNFTransformer: # either an EBNF- or no compiler suite given elif cfactory == get_ebnf_compiler: # trans == get_ebnf_transformer or trans == EBNFTransformer: # either an EBNF- or no compiler suite given
global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \ global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \ AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
DHPARSER_MAIN, DHPARSER_IMPORTS DHPARSER_MAIN, DHPARSER_IMPORTS
......
...@@ -105,9 +105,11 @@ class ZombieParser(MockParser): ...@@ -105,9 +105,11 @@ class ZombieParser(MockParser):
ZOMBIE_PARSER = ZombieParser() ZOMBIE_PARSER = ZombieParser()
class Error(NamedTuple): # # Python 3.6:
pos: int # class Error(NamedTuple):
msg: str # pos: int
# msg: str
Error = NamedTuple('Error', [('pos', int), ('msg', str)])
class Node: class Node:
......
...@@ -267,7 +267,7 @@ def smart_list(arg): ...@@ -267,7 +267,7 @@ def smart_list(arg):
if len(lst) > 1: if len(lst) > 1:
return (s.strip() for s in lst) return (s.strip() for s in lst)
return (s.strip() for s in arg.strip().split(' ')) return (s.strip() for s in arg.strip().split(' '))
elif isinstance(arg, collections.abc.Collection): elif isinstance(arg, collections.abc.Sequence): # python 3.6: collections.abc.Collection
return arg return arg
elif isinstance(arg, collections.abc.Iterable): elif isinstance(arg, collections.abc.Iterable):
return list(arg) return list(arg)
...@@ -301,15 +301,17 @@ def sane_parser_name(name): ...@@ -301,15 +301,17 @@ def sane_parser_name(name):
return name and name[:2] != '__' and name[-2:] != '__' return name and name[:2] != '__' and name[-2:] != '__'
def compile_python_object(python_src, catch_obj_regex): def compile_python_object(python_src, catch_obj_regex=""):
"""Compiles the python source code and returns the object the name of which """Compiles the python source code and returns the (first) object
ends is matched by ``catch_obj_regex``. the name of which is matched by ``catch_obj_regex``. If catch_obj
is the empty string, the namespace dictionary will be returned.
""" """
if isinstance(catch_obj_regex, str): if isinstance(catch_obj_regex, str):
catch_obj_regex = re.compile(catch_obj_regex) catch_obj_regex = re.compile(catch_obj_regex)
code = compile(python_src, '<string>', 'exec') code = compile(python_src, '<string>', 'exec')
namespace = {} namespace = {}
exec(code, namespace) # safety risk? exec(code, namespace) # safety risk?
if catch_obj_regex:
matches = [key for key in namespace.keys() if catch_obj_regex.match(key)] matches = [key for key in namespace.keys() if catch_obj_regex.match(key)]
if len(matches) == 0: if len(matches) == 0:
raise ValueError("No object matching /%s/ defined in source code." % raise ValueError("No object matching /%s/ defined in source code." %
...@@ -318,3 +320,5 @@ def compile_python_object(python_src, catch_obj_regex): ...@@ -318,3 +320,5 @@ def compile_python_object(python_src, catch_obj_regex):
raise ValueError("Ambigous matches for %s : %s" % raise ValueError("Ambigous matches for %s : %s" %
(str(catch_obj_regex), str(matches))) (str(catch_obj_regex), str(matches)))
return namespace[matches[0]] if matches else None return namespace[matches[0]] if matches else None
else:
return namespace
...@@ -24,7 +24,7 @@ import os ...@@ -24,7 +24,7 @@ import os
import sys import sys
sys.path.extend(['../', './']) sys.path.extend(['../', './'])
from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF, compile_parser
ARITHMETIC_EBNF = """ ARITHMETIC_EBNF = """
...@@ -38,13 +38,13 @@ ARITHMETIC_EBNF = """ ...@@ -38,13 +38,13 @@ ARITHMETIC_EBNF = """
class TestCompileFunctions: class TestCompileFunctions:
def test_compileEBNF(self): def test_compileEBNF(self):
parser_src = compileEBNF(ARITHMETIC_EBNF, source_only=True) parser_src = compileEBNF(ARITHMETIC_EBNF)
assert isinstance(parser_src, str), str(type(parser_src)) assert isinstance(parser_src, str), str(type(parser_src))
assert parser_src.find('get_DSL') >= 0 assert parser_src.find('get_DSL') >= 0
parser_src = compileEBNF(ARITHMETIC_EBNF, source_only="CustomDSL") parser_src = compileEBNF(ARITHMETIC_EBNF, branding="CustomDSL")
assert isinstance(parser_src, str), str(type(parser_src)) assert isinstance(parser_src, str), str(type(parser_src))
assert parser_src.find('get_CustomDSL') >= 0 assert parser_src.find('get_CustomDSL') >= 0
parser_factory = compileEBNF(ARITHMETIC_EBNF, source_only=False) parser_factory = compile_parser(ARITHMETIC_EBNF, branding="TestDSL")
assert callable(parser_factory) assert callable(parser_factory)
parser = parser_factory() parser = parser_factory()
result = parser("5 + 3 * 4") result = parser("5 + 3 * 4")
......
...@@ -28,7 +28,7 @@ sys.path.extend(['../', './']) ...@@ -28,7 +28,7 @@ sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging from DHParser.toolkit import is_logging
from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, compileDSL from DHParser.dsl import compileEBNF, compileDSL, compile_parser
class TestDirectives: class TestDirectives:
...@@ -42,7 +42,7 @@ class TestDirectives: ...@@ -42,7 +42,7 @@ class TestDirectives:
def test_whitespace_linefeed(self): def test_whitespace_linefeed(self):
lang = "@ whitespace = linefeed\n" + self.mini_language lang = "@ whitespace = linefeed\n" + self.mini_language
MinilangParser = compileEBNF(lang) MinilangParser = compile_parser(lang)
parser = MinilangParser() parser = MinilangParser()
assert parser assert parser
syntax_tree = parser("3 + 4 * 12") syntax_tree = parser("3 + 4 * 12")
...@@ -58,7 +58,7 @@ class TestDirectives: ...@@ -58,7 +58,7 @@ class TestDirectives:
def test_whitespace_vertical(self): def test_whitespace_vertical(self):
lang = "@ whitespace = vertical\n" + self.mini_language lang = "@ whitespace = vertical\n" + self.mini_language
parser = compileEBNF(lang)() parser = compile_parser(lang)()
assert parser assert parser
syntax_tree = parser("3 + 4 * 12") syntax_tree = parser("3 + 4 * 12")
assert not syntax_tree.collect_errors() assert not syntax_tree.collect_errors()
...@@ -71,7 +71,7 @@ class TestDirectives: ...@@ -71,7 +71,7 @@ class TestDirectives:
def test_whitespace_horizontal(self): def test_whitespace_horizontal(self):
lang = "@ whitespace = horizontal\n" + self.mini_language lang = "@ whitespace = horizontal\n" + self.mini_language
parser = compileEBNF(lang)() parser = compile_parser(lang)()
assert parser assert parser
syntax_tree = parser("3 + 4 * 12") syntax_tree = parser("3 + 4 * 12")
assert not syntax_tree.collect_errors() assert not syntax_tree.collect_errors()
...@@ -131,8 +131,8 @@ class TestPopRetrieve: ...@@ -131,8 +131,8 @@ class TestPopRetrieve:
""" """
def setup(self): def setup(self):
self.minilang_parser = compileEBNF(self.mini_language)() self.minilang_parser = compile_parser(self.mini_language)()
self.minilang_parser2 = compileEBNF(self.mini_lang2)() self.minilang_parser2 = compile_parser(self.mini_lang2)()
@staticmethod @staticmethod
def opening_delimiter(node, name): def opening_delimiter(node, name):
......
...@@ -28,7 +28,7 @@ from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \ ...@@ -28,7 +28,7 @@ from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten replace_by_single_child, reduce_single_child, flatten
from DHParser.parsers import compile_source from DHParser.parsers import compile_source
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, DHPARSER_IMPORTS from DHParser.dsl import compile_parser, DHPARSER_IMPORTS
ARITHMETIC_EBNF = """ ARITHMETIC_EBNF = """
...@@ -72,7 +72,7 @@ class TestInfiLoopsAndRecursion: ...@@ -72,7 +72,7 @@ class TestInfiLoopsAndRecursion:
def test_direct_left_recursion(self): def test_direct_left_recursion(self):
minilang = ARITHMETIC_EBNF minilang = ARITHMETIC_EBNF
snippet = "5 + 3 * 4" snippet = "5 + 3 * 4"
parser = compileEBNF(minilang)() parser = compile_parser(minilang)()
assert parser assert parser
syntax_tree = parser(snippet) syntax_tree = parser(snippet)
assert not syntax_tree.collect_errors() assert not syntax_tree.collect_errors()
...@@ -87,7 +87,7 @@ class TestInfiLoopsAndRecursion: ...@@ -87,7 +87,7 @@ class TestInfiLoopsAndRecursion:
def test_inifinite_loops(self): def test_inifinite_loops(self):
minilang = """not_forever = { // } \n""" minilang = """not_forever = { // } \n"""
snippet = " " snippet = " "
parser = compileEBNF(minilang)() parser = compile_parser(minilang)()
syntax_tree = parser(snippet) syntax_tree = parser(snippet)
assert syntax_tree.error_flag assert syntax_tree.error_flag
# print(syntax_tree.collect_errors()) # print(syntax_tree.collect_errors())
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment