split compileEBNF into two functions: compileEBNF and compile_parser;...

split compileEBNF into two functions: compileEBNF and compile_parser; adjustments for compatibility with pypy3 / python 3.5

DHParser/dsl.py

@@ -38,6 +38,8 @@ __all__ = ['GrammarError',
            'CompilationError',
            'load_compiler_suite',
            'compileDSL',
+           'compileEBNF',
+           'compile_parser',
            'compile_on_disk']
 
 
@@ -60,31 +62,6 @@ COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved.
 END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
 
 
-class GrammarError(Exception):
-    """Raised when (already) the grammar of a domain specific language (DSL)
-    contains errors.
-    """
-
-    def __init__(self, error_messages, grammar_src):
-        self.error_messages = error_messages
-        self.grammar_src = grammar_src
-
-
-class CompilationError(Exception):
-    """Raised when a string or file in a domain specific language (DSL)
-    contains errors.
-    """
-
-    def __init__(self, error_messages, dsl_text, dsl_grammar, AST):
-        self.error_messages = error_messages
-        self.dsl_text = dsl_text
-        self.dsl_grammar = dsl_grammar
-        self.AST = AST
-
-    def __str__(self):
-        return '\n'.join(self.error_messages)
-
-
 DHPARSER_IMPORTS = '''
 from functools import partial
 import os
@@ -134,6 +111,31 @@ if __name__ == "__main__":
 '''
 
 
+class GrammarError(Exception):
+    """Raised when (already) the grammar of a domain specific language (DSL)
+    contains errors.
+    """
+
+    def __init__(self, error_messages, grammar_src):
+        self.error_messages = error_messages
+        self.grammar_src = grammar_src
+
+
+class CompilationError(Exception):
+    """Raised when a string or file in a domain specific language (DSL)
+    contains errors.
+    """
+
+    def __init__(self, error_messages, dsl_text, dsl_grammar, AST):
+        self.error_messages = error_messages
+        self.dsl_text = dsl_text
+        self.dsl_grammar = dsl_grammar
+        self.AST = AST
+
+    def __str__(self):
+        return '\n'.join(self.error_messages)
+
+
 def grammar_instance(grammar_representation):
     """Returns a grammar object and the source code of the grammar, from
     the given `grammar`-data which can be either a file name, ebnf-code,
@@ -184,7 +186,7 @@ def compileDSL(text_or_file, scanner, dsl_grammar, ast_transformation, compiler)
     return result
 
 
-def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
+def compileEBNF(ebnf_src, branding = "DSL"):
     """Compiles an EBNF source file. Either returns a factory function
     for the grammar-paraser or the source code of a compiler suite with
     skeletons for scanner, transformer and compiler.
@@ -192,31 +194,41 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
     Args:
         ebnf_src(str):  Either the file name of an EBNF grammar or
             the EBNF grammar itself as a string.
-        ebnf_grammar_obj:  An existing instance of the 
-            DHParser.EBNFcompiler.EBNFGrammar object. This can speed
-            up compilation, because no new EBNFGrammar object needs to
-            be instantiated.
-        source_only (str or bool):  Branding name for the compiler
-            suite source code. If ``True`` the default branding "DSL"
-            will be used. If False (default), no source but a factory
-            function for grammar-parser callables will be returned.
-        
+        branding (str or bool):  Branding name for the compiler
+            suite source code. 
     Returns:
-        A factory function for a grammar-parser for texts in the
-        language defined by ``ebnf_src``. With the ``source_only``
-        a complete compiler suite skeleton will be returned instead.
+        The complete compiler suite skeleton as Python source code.
     """
-    grammar = ebnf_grammar_obj or get_ebnf_grammar()
-    if source_only == True:  source_only = "DSL"
-    compiler = get_ebnf_compiler(source_only or "DSL")
+    grammar = get_ebnf_grammar()
+    if branding == True:  branding = "DSL"
+    compiler = get_ebnf_compiler(branding or "DSL", ebnf_src)
     grammar_src = compileDSL(ebnf_src, nil_scanner, grammar, EBNFTransformer, compiler)
-    if source_only:
-        src = [DHPARSER_IMPORTS, compiler.gen_scanner_skeleton(), grammar_src,
-               compiler.gen_transformer_skeleton(), compiler.gen_compiler_skeleton(),
-               DHPARSER_MAIN.format(NAME=source_only)]
-        return '\n'.join(src)
-    else:
-        return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$')
+    src = [DHPARSER_IMPORTS,
+           compiler.gen_scanner_skeleton(),
+           grammar_src,
+           compiler.gen_transformer_skeleton(),
+           compiler.gen_compiler_skeleton(),
+           DHPARSER_MAIN.format(NAME=branding)]
+    return '\n'.join(src)
+
+
+def compile_parser(ebnf_src, branding="DSL"):
+    """Compiles an EBNF grammar and returns a grammar-parser factory
+    function for that grammar.
+
+    Args:
+        ebnf_src(str):  Either the file name of an EBNF grammar or
+            the EBNF grammar itself as a string.
+        branding (str or bool):  Branding name for the compiler
+            suite source code. 
+    
+    Returns:
+        A factory function for a grammar-parser for texts in the
+        language defined by ``ebnf_src``.
+    """
+    grammar_src = compileDSL(ebnf_src, nil_scanner, get_ebnf_grammar(),
+                             get_ebnf_transformer(), get_ebnf_compiler(branding))
+    return compile_python_object(DHPARSER_IMPORTS + grammar_src, 'get_\w*_grammar$')
 
 
 def load_compiler_suite(compiler_suite):
@@ -350,7 +362,7 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
     if errors:
         return errors
 
-    elif trans == get_ebnf_transformer or trans == EBNFTransformer:  # either an EBNF- or no compiler suite given
+    elif cfactory == get_ebnf_compiler:  # trans == get_ebnf_transformer or trans == EBNFTransformer:  # either an EBNF- or no compiler suite given
         global SECTION_MARKER, RX_SECTION_MARKER, SCANNER_SECTION, PARSER_SECTION, \
             AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
             DHPARSER_MAIN, DHPARSER_IMPORTS

DHParser/syntaxtree.py

@@ -105,9 +105,11 @@ class ZombieParser(MockParser):
 ZOMBIE_PARSER = ZombieParser()
 
 
-class Error(NamedTuple):
-    pos: int
-    msg: str
+# # Python 3.6:
+# class Error(NamedTuple):
+#     pos: int
+#     msg: str
+Error = NamedTuple('Error', [('pos', int), ('msg', str)])
 
 
 class Node:

DHParser/toolkit.py

@@ -267,7 +267,7 @@ def smart_list(arg):
             if len(lst) > 1:
                 return (s.strip() for s in lst)
         return (s.strip() for s in arg.strip().split(' '))
-    elif isinstance(arg, collections.abc.Collection):
+    elif isinstance(arg, collections.abc.Sequence):  # python 3.6: collections.abc.Collection
         return arg
     elif isinstance(arg, collections.abc.Iterable):
         return list(arg)
@@ -301,20 +301,24 @@ def sane_parser_name(name):
     return name and name[:2] != '__' and name[-2:] != '__'
 
 
-def compile_python_object(python_src, catch_obj_regex):
-    """Compiles the python source code and returns the object the name of which
-    ends is matched by ``catch_obj_regex``.
+def compile_python_object(python_src, catch_obj_regex=""):
+    """Compiles the python source code and returns the (first) object 
+    the name of which is matched by ``catch_obj_regex``. If catch_obj
+    is the empty string, the namespace dictionary will be returned.
     """
     if isinstance(catch_obj_regex, str):
         catch_obj_regex = re.compile(catch_obj_regex)
     code = compile(python_src, '<string>', 'exec')
     namespace = {}
     exec(code, namespace)  # safety risk?
-    matches = [key for key in namespace.keys() if catch_obj_regex.match(key)]
-    if len(matches) == 0:
-        raise ValueError("No object matching /%s/ defined in source code." %
-                         catch_obj_regex.pattern)
-    elif len(matches) > 1:
-        raise ValueError("Ambigous matches for %s : %s" %
-                         (str(catch_obj_regex), str(matches)))
-    return namespace[matches[0]] if matches else None
+    if catch_obj_regex:
+        matches = [key for key in namespace.keys() if catch_obj_regex.match(key)]
+        if len(matches) == 0:
+            raise ValueError("No object matching /%s/ defined in source code." %
+                             catch_obj_regex.pattern)
+        elif len(matches) > 1:
+            raise ValueError("Ambigous matches for %s : %s" %
+                             (str(catch_obj_regex), str(matches)))
+        return namespace[matches[0]] if matches else None
+    else:
+        return namespace

test/test_dsl.py

@@ -24,7 +24,7 @@ import os
 import sys
 sys.path.extend(['../', './'])
 
-from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF
+from DHParser.dsl import compile_on_disk, run_compiler, compileDSL, compileEBNF, compile_parser
 
 
 ARITHMETIC_EBNF = """
@@ -38,13 +38,13 @@ ARITHMETIC_EBNF = """
 
 class TestCompileFunctions:
     def test_compileEBNF(self):
-        parser_src = compileEBNF(ARITHMETIC_EBNF, source_only=True)
+        parser_src = compileEBNF(ARITHMETIC_EBNF)
         assert isinstance(parser_src, str), str(type(parser_src))
         assert parser_src.find('get_DSL') >= 0
-        parser_src = compileEBNF(ARITHMETIC_EBNF, source_only="CustomDSL")
+        parser_src = compileEBNF(ARITHMETIC_EBNF, branding="CustomDSL")
         assert isinstance(parser_src, str), str(type(parser_src))
         assert parser_src.find('get_CustomDSL') >= 0
-        parser_factory = compileEBNF(ARITHMETIC_EBNF, source_only=False)
+        parser_factory = compile_parser(ARITHMETIC_EBNF, branding="TestDSL")
         assert callable(parser_factory)
         parser = parser_factory()
         result = parser("5 + 3 * 4")

test/test_ebnf.py

@@ -28,7 +28,7 @@ sys.path.extend(['../', './'])
 from DHParser.toolkit import is_logging
 from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
-from DHParser.dsl import compileEBNF, compileDSL
+from DHParser.dsl import compileEBNF, compileDSL, compile_parser
 
 
 class TestDirectives:
@@ -42,7 +42,7 @@ class TestDirectives:
 
     def test_whitespace_linefeed(self):
         lang = "@ whitespace = linefeed\n" + self.mini_language
-        MinilangParser = compileEBNF(lang)
+        MinilangParser = compile_parser(lang)
         parser = MinilangParser()
         assert parser
         syntax_tree = parser("3 + 4 * 12")
@@ -58,7 +58,7 @@ class TestDirectives:
 
     def test_whitespace_vertical(self):
         lang = "@ whitespace = vertical\n" + self.mini_language
-        parser = compileEBNF(lang)()
+        parser = compile_parser(lang)()
         assert parser
         syntax_tree = parser("3 + 4 * 12")
         assert not syntax_tree.collect_errors()
@@ -71,7 +71,7 @@ class TestDirectives:
 
     def test_whitespace_horizontal(self):
         lang = "@ whitespace = horizontal\n" + self.mini_language
-        parser = compileEBNF(lang)()
+        parser = compile_parser(lang)()
         assert parser
         syntax_tree = parser("3 + 4 * 12")
         assert not syntax_tree.collect_errors()
@@ -131,8 +131,8 @@ class TestPopRetrieve:
         """
 
     def setup(self):
-        self.minilang_parser = compileEBNF(self.mini_language)()
-        self.minilang_parser2 = compileEBNF(self.mini_lang2)()
+        self.minilang_parser = compile_parser(self.mini_language)()
+        self.minilang_parser2 = compile_parser(self.mini_lang2)()
 
     @staticmethod
     def opening_delimiter(node, name):

test/test_parsers.py

@@ -28,7 +28,7 @@ from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \
     replace_by_single_child, reduce_single_child, flatten
 from DHParser.parsers import compile_source
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
-from DHParser.dsl import compileEBNF, DHPARSER_IMPORTS
+from DHParser.dsl import compile_parser, DHPARSER_IMPORTS
 
 
 ARITHMETIC_EBNF = """
@@ -72,7 +72,7 @@ class TestInfiLoopsAndRecursion:
     def test_direct_left_recursion(self):
         minilang = ARITHMETIC_EBNF
         snippet = "5 + 3 * 4"
-        parser = compileEBNF(minilang)()
+        parser = compile_parser(minilang)()
         assert parser
         syntax_tree = parser(snippet)
         assert not syntax_tree.collect_errors()
@@ -87,7 +87,7 @@ class TestInfiLoopsAndRecursion:
     def test_inifinite_loops(self):
         minilang = """not_forever = { // } \n"""
         snippet = " "
-        parser = compileEBNF(minilang)()
+        parser = compile_parser(minilang)()
         syntax_tree = parser(snippet)
         assert syntax_tree.error_flag
         # print(syntax_tree.collect_errors())