- DevScripts/create_standalone.py now tested and working properly

DHParser/ebnf.py

@@ -38,7 +38,7 @@ from DHParser.transform import traverse, remove_brackets, \
     remove_tokens, flatten, forbid, assert_content, key_tag_name
 from DHParser.versionnumber import __version__
 
-__all__ = ['get_ebnf_preprocessor',
+__all__ = ('get_ebnf_preprocessor',
            'get_ebnf_grammar',
            'get_ebnf_transformer',
            'get_ebnf_compiler',
@@ -50,7 +50,7 @@ __all__ = ['get_ebnf_preprocessor',
            'PreprocessorFactoryFunc',
            'ParserFactoryFunc',
            'TransformerFactoryFunc',
-           'CompilerFactoryFunc']
+           'CompilerFactoryFunc')
 
 
 ########################################################################
@@ -90,7 +90,6 @@ class EBNFGrammar(Grammar):
                 | [flowmarker] literal
                 | [flowmarker] regexp
                 | [flowmarker] group
-                | [flowmarker] regexchain
                 | [flowmarker] oneormore
                 | repetition
                 | option

DHParser/parser.py

@@ -245,10 +245,10 @@ def add_parser_guard(parser_func):
 
 class ParserMetaClass(abc.ABCMeta):
     def __init__(cls, name, bases, attrs):
-        # The following condition is necessary for classes that don't override
+        guarded_parser_call = add_parser_guard(cls.__call__)
+        # The following check is necessary for classes that don't override
         # the __call__() method, because in these cases the non-overridden
         # __call__()-method would be substituted a second time!
-        guarded_parser_call = add_parser_guard(cls.__call__)
         if cls.__call__.__code__ != guarded_parser_call.__code__:
             cls.__call__ = guarded_parser_call
         super(ParserMetaClass, cls).__init__(name, bases, attrs)

DHParser/versionnumber.py

@@ -16,5 +16,5 @@ implied.  See the License for the specific language governing
 permissions and limitations under the License.
 """
 
-__all__ = ('__version__')
+__all__ = ('__version__',)
 __version__ = '0.7.6' # + '_dev' + str(os.stat(__file__).st_mtime)

DevScripts/Readme-DevScripts.md

@@ -4,4 +4,7 @@ Folder "DevScripts"
 This folder contains helper scripts for the
 development of DHParser.
 
-collect_symbols.py - Lists all exported symbols from DHParser modules
+* collect_symbols.py - Lists all exported symbols from DHParser modules
+
+* create_standalone.py - merges the DHParser modules into a standalone
+                        DHParser.py module for easier deployment.

DevScripts/create_standalone.py

-"""create_standalone.py - merges the DHParser modules into a standalone
-        DHParser.py module for easier deployment.
+#!/usr/bin/python3
+
+"""create_standalone.py - merges the DHParser modules into a single
+        standalone DHParser.py module for easier deployment.
 
 Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                 Bavarian Academy of Sciences an Humanities (badw.de)
@@ -19,14 +21,25 @@ permissions and limitations under the License.
 
 import functools
 import operator
+import os
 import sys
 
 sys.path.append('../')
+sys.path.append('./')
 try:
     import regex as re
 except ImportError:
     import re
 
+from DHParser import toolkit
+from DHParser import syntaxtree
+from DHParser import parser
+from DHParser import transform
+from DHParser import ebnf
+from DHParser import dsl
+from DHParser import testing
+from DHParser import versionnumber
+
 modules = ('toolkit', 'syntaxtree', 'parser', 'transform', 'ebnf', 'dsl', 'testing', 'versionnumber')
 
 all_symbols = list(functools.reduce(operator.or_, (set(eval(m + '.__all__')) for m in modules)))
@@ -40,7 +53,7 @@ def start(module):
     return i
 
 
-doc = "DHParser.py - Packrat-parser and parser-generator\n\n" + __doc__[__doc__.find('Copyright'):]
+doc = '"""DHParser.py - Packrat-parser and parser-generator\n\n' + __doc__[__doc__.find('Copyright'):] + '\n"""'
 
 imports = """
 import abc
@@ -50,10 +63,11 @@ from collections import OrderedDict
 import configparser
 import contextlib
 import copy
-from functools import partial
+from functools import partial, reduce, singledispatch
 import hashlib
 import inspect
 import json
+import keyword
 import os
 import platform
 try:
@@ -65,6 +79,8 @@ from typing import AbstractSet, Any, ByteString, Callable, cast, Container, Dict
         Iterator, List, NamedTuple, Sequence, Set, Union, Text, Tuple
 """
 
+symbols = "\n__all__ = (" + ",\n           ".join("'%s'" % sym for sym in all_symbols) + ")\n"
+
 heading = """
 #######################################################################
 #######################################################################
@@ -75,9 +91,52 @@ heading = """
 #######################################################################
 """
 
+main = r"""
+#######################################################################
+#######################################################################
+#
+# main / selftest
+#
+#######################################################################
+#######################################################################
+
+
+def selftest() -> bool:
+    print("DHParser selftest...")
+    print("\nSTAGE I:  Trying to compile EBNF-Grammar:\n")
+    builtin_ebnf_parser = get_ebnf_grammar()
+    ebnf_src = builtin_ebnf_parser.__doc__[builtin_ebnf_parser.__doc__.find('#'):]
+    ebnf_transformer = get_ebnf_transformer()
+    ebnf_compiler = get_ebnf_compiler('EBNF')
+    generated_ebnf_parser, errors, ast = compile_source(ebnf_src, None,
+        builtin_ebnf_parser, ebnf_transformer, ebnf_compiler)
+
+    if errors:
+        print("Selftest FAILED :-(")
+        print("\n\n".join(errors))
+        return False
+    print(generated_ebnf_parser)
+    print("\n\nSTAGE 2: Selfhosting-test: Trying to compile EBNF-Grammar with generated parser...\n")
+    selfhosted_ebnf_parser = compileDSL(ebnf_src, None, generated_ebnf_parser,
+                                        ebnf_transformer, ebnf_compiler)
+    print(selfhosted_ebnf_parser)
+    print("\n\n Selftest SUCCEEDED :-)\n\n")
+    return True
+
+
+if __name__ == "__main__":
+    if not selftest():  sys.exit(1)
+
+"""
 
 def merge_modules(module_names, dhp_path='../DHParser/'):
-    components = [doc, imports]
+    paths = [dhp_path, '../DHParser/', 'DHParser/', '']
+    for pth in paths:
+        if os.path.exists(pth + 'ebnf.py'):
+            dhp_path = pth
+            break
+
+    components = [doc, imports, symbols]
 
     for name in module_names:
         with open(dhp_path + '%s.py' % name) as f:
@@ -85,8 +144,18 @@ def merge_modules(module_names, dhp_path='../DHParser/'):
             content = module[start(module):]
             components.append(heading % name)
             components.append(content)
+    components.append(main)
 
     return "\n".join(components)
 
 
-print(merge_modules(modules))
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python create_standalone.py [FILENAME]")
+    elif os.path.exists(sys.argv[1]):
+        print("File '%s' already exits. Please delete file first." % (sys.argv[1]))
+    else:
+
+        with open(sys.argv[1], 'w') as f:
+            f.write(merge_modules(modules))

dhparser.py

@@ -30,42 +30,71 @@ from DHParser.parser import compile_source, nil_preprocessor
 from DHParser.toolkit import logging
 
 
-def selftest(file_name):
-    print(file_name)
-    with open('examples/' + file_name, encoding="utf-8") as f:
-        grammar = f.read()
-    compiler_name = os.path.basename(os.path.splitext(file_name)[0])
-    parser = get_ebnf_grammar()
-    print("\nAlphabetical List of Parsers:\n")
-    parser_list = sorted([p for p in parser.all_parsers__ if p.name], key=lambda p: p.name)
-    for p in parser_list:
-        print(p)
-    print('\n\n')
-    transformer = get_ebnf_transformer()
-    compiler = get_ebnf_compiler(compiler_name, grammar)
-    result, errors, syntax_tree = compile_source(grammar, None, parser,
-                                                 transformer, compiler)
-    print(result)
+# def selftest(file_name):
+#     print(file_name)
+#     with open('examples/' + file_name, encoding="utf-8") as f:
+#         grammar = f.read()
+#     compiler_name = os.path.basename(os.path.splitext(file_name)[0])
+#     parser = get_ebnf_grammar()
+#     print("\nAlphabetical List of Parsers:\n")
+#     parser_list = sorted([p for p in parser.all_parsers__ if p.name], key=lambda p: p.name)
+#     for p in parser_list:
+#         print(p)
+#     print('\n\n')
+#     transformer = get_ebnf_transformer()
+#     compiler = get_ebnf_compiler(compiler_name, grammar)
+#     result, errors, syntax_tree = compile_source(grammar, None, parser,
+#                                                  transformer, compiler)
+#     print(result)
+#     if errors:
+#         print('\n\n'.join(errors))
+#         sys.exit(1)
+#     else:
+#         # compile the grammar again using the result of the previous
+#         # compilation as parser
+#         for i in range(1):
+#             result = compileDSL(grammar, nil_preprocessor, result, transformer, compiler)
+#         print(result)
+#     return result
+
+
+
+def selftest() -> bool:
+    print("DHParser selftest...")
+    print("\nSTAGE I:  Trying to compile EBNF-Grammar:\n")
+    builtin_ebnf_parser = get_ebnf_grammar()
+    ebnf_src = builtin_ebnf_parser.__doc__[builtin_ebnf_parser.__doc__.find('#'):]
+    ebnf_transformer = get_ebnf_transformer()
+    ebnf_compiler = get_ebnf_compiler('EBNF')
+    generated_ebnf_parser, errors, ast = compile_source(ebnf_src, None,
+        builtin_ebnf_parser, ebnf_transformer, ebnf_compiler)
+
     if errors:
-        print('\n\n'.join(errors))
-        sys.exit(1)
-    else:
-        # compile the grammar again using the result of the previous
-        # compilation as parser
-        for i in range(1):
-            result = compileDSL(grammar, nil_preprocessor, result, transformer, compiler)
-        print(result)
-    return result
+        print("Selftest FAILED :-(")
+        print("\n\n".join(errors))
+        return False
+    print(generated_ebnf_parser)
+    print("\n\nSTAGE 2: Selfhosting-test: Trying to compile EBNF-Grammar with generated parser...\n")
+    selfhosted_ebnf_parser = compileDSL(ebnf_src, None, generated_ebnf_parser,
+                                        ebnf_transformer, ebnf_compiler)
+    print(selfhosted_ebnf_parser)
+    print("\n\n Selftest SUCCEEDED :-)\n\n")
+    return True
+
 
 
 def profile(func):
     import cProfile
     pr = cProfile.Profile()
     pr.enable()
-    func()
+    for i in range(1):
+        success = func()
+        if not success:
+            break
     pr.disable()
     # after your program ends
     pr.print_stats(sort="tottime")
+    return success
 
 
 # # Changes in the EBNF source that are not reflected in this file could be
@@ -88,4 +117,6 @@ if __name__ == "__main__":
         # run self test
         # selftest('EBNF/EBNF.ebnf')
         with logging(False):
-            profile(partial(selftest, file_name='EBNF/EBNF.ebnf'))
+            if not profile(selftest):
+                sys.exit(1)
+