- script for creating single-module ("stand-alone") version of DHParser

DHParser/__init__.py

@@ -24,6 +24,7 @@ from .parser import *
 from .syntaxtree import *
 # Flat namespace for the DHParser Package. Is this a good idea...?
 from .toolkit import *
+from .transform import *
 # from .testing import *
 from .versionnumber import __version__
 

DHParser/dsl.py

@@ -20,6 +20,7 @@ compilation of domain specific languages based on an EBNF-grammar.
 """
 
 import os
+
 try:
     import regex as re
 except ImportError:
@@ -36,15 +37,14 @@ from DHParser.toolkit import logging, load_if_file, is_python_code, compile_pyth
 from DHParser.parser import Grammar, Compiler, compile_source, nil_preprocessor, PreprocessorFunc
 from DHParser.syntaxtree import Node, TransformationFunc
 
-
-__all__ = ['GrammarError',
+__all__ = ('GrammarError',
            'CompilationError',
            'load_compiler_suite',
            'compileDSL',
            'raw_compileEBNF',
            'compileEBNF',
            'parser_factory',
-           'compile_on_disk']
+           'compile_on_disk')
 
 
 SECTION_MARKER = """\n
@@ -74,14 +74,14 @@ try:
     import regex as re
 except ImportError:
     import re
-from DHParser.toolkit import logging, is_filename, load_if_file    
-from DHParser.parser import Grammar, Compiler, nil_preprocessor, \\
+from DHParser import logging, is_filename, load_if_file, \\
+    Grammar, Compiler, nil_preprocessor, \\
     Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym, \\
     Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
-    last_value, counterpart, accumulate, PreprocessorFunc
-from DHParser.syntaxtree import Node, TransformationFunc
-from DHParser.transform import traverse, remove_children_if, \\
+    last_value, counterpart, accumulate, PreprocessorFunc, \\
+    Node, TransformationFunc, \\
+    traverse, remove_children_if, \\
     reduce_single_child, replace_by_single_child, remove_whitespace, \\
     remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
     is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \\
@@ -483,3 +483,45 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml"):
             if f:  f.close()
 
     return []
+
+
+def recompile_grammar(ebnf_filename, force=False) -> bool:
+    """
+    Recompiles an ebnf-grammar if necessary, that is, if either no
+    corresponding 'XXXXCompiler.py'-file exists or if that file is
+    outdated.
+
+    Parameters:
+        ebnf_filename(str):  The filename of the ebnf-source of the
+            grammar. In case this is a directory and not a file, all
+            files within this directory ending with .ebnf will be
+            compiled.
+        force(bool):  If False (default), the grammar will only be
+            recompiled if it has been changed.
+    """
+    if os.path.isdir(ebnf_filename):
+        success = True
+        for entry in os.listdir(ebnf_filename):
+            if entry.lower().endswith('.ebnf') and os.path.isfile(entry):
+                success = success and recompile_grammar(entry, force)
+        return success
+
+    base, ext = os.path.splitext(ebnf_filename)
+    compiler_name = base + 'Compiler.py'
+    error_file_name = base + '_ebnf_ERRORS.txt'
+    errors = []
+    if (not os.path.exists(compiler_name) or force or
+            grammar_changed(compiler_name, ebnf_filename)):
+        # print("recompiling parser for: " + ebnf_filename)
+        errors = compile_on_disk(ebnf_filename)
+        if errors:
+            # print("Errors while compiling: " + ebnf_filename + '!')
+            with open(error_file_name, 'w') as f:
+                for e in errors:
+                    f.write(e)
+                    f.write('\n')
+            return False
+
+    if not errors and os.path.exists(error_file_name):
+        os.remove(error_file_name)
+    return True

DHParser/parser.py

@@ -61,10 +61,10 @@ except ImportError:
     import re
 try:
     from typing import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union
-    try:
-        from typing import Collection
-    except ImportError:
-        pass
+    # try:
+    #     from typing import Collection
+    # except ImportError:
+    #     pass
 except ImportError:
     from .typing34 import Any, Callable, cast, Dict, Iterator, List, Set, Tuple, Union
 
@@ -73,7 +73,7 @@ from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, ZOMBIE_PARSER, Pa
     Node, TransformationFunc
 from DHParser.toolkit import load_if_file, error_messages
 
-__all__ = ['PreprocessorFunc',
+__all__ = ('PreprocessorFunc',
            'HistoryRecord',
            'Parser',
            'Grammar',
@@ -109,7 +109,7 @@ __all__ = ['PreprocessorFunc',
            'Pop',
            'Forward',
            'Compiler',
-           'compile_source']
+           'compile_source')
 
 
 

DHParser/syntaxtree.py

@@ -33,14 +33,14 @@ except ImportError:
 
 from DHParser.toolkit import log_dir, line_col
 
-
-__all__ = ['WHITESPACE_PTYPE',
+__all__ = ('WHITESPACE_PTYPE',
            'TOKEN_PTYPE',
            'ZOMBIE_PARSER',
            'ParserBase',
            'Error',
            'Node',
-           'TransformationFunc']
+           'mock_syntax_tree',
+           'TransformationFunc')
 
 
 class ParserBase:
@@ -461,4 +461,56 @@ class Node:
     #     return nav(path.split('/'))
 
 
+def mock_syntax_tree(sexpr):
+    """
+    Generates a tree of nodes from an S-expression.
+
+    Example:
+    >>> mock_syntax_tree("(a (b c))").as_sxpr()
+    '(a\\n    (b\\n        "c"\\n    )\\n)'
+    """
+
+    def next_block(s):
+        s = s.strip()
+        while s[0] != ')':
+            if s[0] != '(': raise ValueError('"(" expected, not ' + s[:10])
+            # assert s[0] == '(', s
+            level = 1
+            i = 1
+            while level > 0:
+                if s[i] == '(':
+                    level += 1
+                elif s[i] == ')':
+                    level -= 1
+                i += 1
+            yield s[:i]
+            s = s[i:].strip()
+
+    sexpr = sexpr.strip()
+    if sexpr[0] != '(': raise ValueError('"(" expected, not ' + sexpr[:10])
+    # assert sexpr[0] == '(', sexpr
+    sexpr = sexpr[1:].strip()
+    m = re.match('[\w:]+', sexpr)
+    name, class_name = (sexpr[:m.end()].split(':') + [''])[:2]
+    sexpr = sexpr[m.end():].strip()
+    if sexpr[0] == '(':
+        result = tuple(mock_syntax_tree(block) for block in next_block(sexpr))
+    else:
+        lines = []
+        while sexpr and sexpr[0] != ')':
+            for qm in ['"""', "'''", '"', "'"]:
+                m = re.match(qm + r'.*?' + qm, sexpr)
+                if m:
+                    i = len(qm)
+                    lines.append(sexpr[i:m.end() - i])
+                    sexpr = sexpr[m.end():].strip()
+                    break
+            else:
+                m = re.match(r'(?:(?!\)).)*', sexpr)
+                lines.append(sexpr[:m.end()])
+                sexpr = sexpr[m.end():]
+        result = "\n".join(lines)
+    return Node(MockParser(name, ':' + class_name), result)
+
+
 TransformationFunc = Union[Callable[[Node], Any], partial]

DHParser/testing.py

@@ -26,15 +26,11 @@ try:
 except ImportError:
     import re
 
-from DHParser import Node, error_messages
+from DHParser import error_messages
 from DHParser.toolkit import compact_sexpr, is_logging
-from DHParser.syntaxtree import MockParser
-from DHParser.ebnf import grammar_changed
-from DHParser.dsl import compile_on_disk
+from DHParser.syntaxtree import mock_syntax_tree
 
-__all__ = ('mock_syntax_tree',
-           'recompile_grammar',
-           'unit_from_configfile',
+__all__ = ('unit_from_configfile',
            'unit_from_json',
            'unit_from_file',
            'get_report',
@@ -42,100 +38,6 @@ __all__ = ('mock_syntax_tree',
            'grammar_suite',
            'runner')
 
-
-def mock_syntax_tree(sexpr):
-    """
-    Generates a tree of nodes from an S-expression.
-
-    Example: 
-    >>> mock_syntax_tree("(a (b c))").as_sxpr()
-    '(a\\n    (b\\n        "c"\\n    )\\n)'
-    """
-    def next_block(s):
-        s = s.strip()
-        while s[0] != ')':
-            if s[0] != '(': raise ValueError('"(" expected, not ' + s[:10])
-            # assert s[0] == '(', s
-            level = 1
-            i = 1
-            while level > 0:
-                if s[i] == '(':
-                    level += 1
-                elif s[i] == ')':
-                    level -= 1
-                i += 1
-            yield s[:i]
-            s = s[i:].strip()
-
-    sexpr = sexpr.strip()
-    if sexpr[0] != '(': raise ValueError('"(" expected, not ' + sexpr[:10])
-    # assert sexpr[0] == '(', sexpr
-    sexpr = sexpr[1:].strip()
-    m = re.match('[\w:]+', sexpr)
-    name, class_name = (sexpr[:m.end()].split(':') + [''])[:2]
-    sexpr = sexpr[m.end():].strip()
-    if sexpr[0] == '(':
-        result = tuple(mock_syntax_tree(block) for block in next_block(sexpr))
-    else:
-        lines = []
-        while sexpr and sexpr[0] != ')':
-            for qm in ['"""', "'''", '"', "'"]:
-                m = re.match(qm + r'.*?' + qm, sexpr)
-                if m:
-                    i = len(qm)
-                    lines.append(sexpr[i:m.end() - i])
-                    sexpr = sexpr[m.end():].strip()
-                    break
-            else:
-                m = re.match(r'(?:(?!\)).)*', sexpr)
-                lines.append(sexpr[:m.end()])
-                sexpr = sexpr[m.end():]
-        result = "\n".join(lines)
-    return Node(MockParser(name, ':' + class_name), result)
-
-
-def recompile_grammar(ebnf_filename, force=False) -> bool:
-    """
-    Recompiles an ebnf-grammar if necessary, that is if either no
-    corresponding 'XXXXCompiler.py'-file exists or if that file is
-    outdated.
-    
-    Parameters:
-        ebnf_filename(str):  The filename of the ebnf-source of the 
-            grammar. In case this is a directory and not a file all
-            files within this directory ending with .ebnf will be 
-            compiled.
-        force(bool):  If False (default), the grammar will only be
-            recompiled if it has been changed.
-    """
-    if os.path.isdir(ebnf_filename):
-        success = True
-        for entry in os.listdir(ebnf_filename):
-            if entry.lower().endswith('.ebnf') and os.path.isfile(entry):
-                success = success and recompile_grammar(entry, force)
-        return success
-
-    base, ext = os.path.splitext(ebnf_filename)
-    compiler_name = base + 'Compiler.py'
-    error_file_name = base + '_ebnf_ERRORS.txt'
-    errors = []
-    if (not os.path.exists(compiler_name) or force or
-        grammar_changed(compiler_name, ebnf_filename)):
-        # print("recompiling parser for: " + ebnf_filename)
-        errors = compile_on_disk(ebnf_filename)
-        if errors:
-            # print("Errors while compiling: " + ebnf_filename + '!')
-            with open(error_file_name, 'w') as f:
-                for e in errors:
-                    f.write(e)
-                    f.write('\n')
-            return False
-
-    if not errors and os.path.exists(error_file_name):
-        os.remove(error_file_name)
-    return True
-
-
 UNIT_STAGES = {'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'}
 
 

DHParser/toolkit.py

@@ -45,8 +45,7 @@ try:
 except ImportError:
     from .typing34 import Any, List, Tuple
 
-
-__all__ = ['logging',
+__all__ = ('logging',
            'is_logging',
            'log_dir',
            'logfile_basename',
@@ -63,7 +62,7 @@ __all__ = ['logging',
            'md5',
            'expand_table',
            'smart_list',
-           'sane_parser_name']
+           'sane_parser_name')
 
 
 def log_dir() -> str:

DHParser/transform.py

@@ -52,6 +52,7 @@ __all__ = ('transformation_factory',
            'is_token',
            'has_name',
            'has_content',
+           'remove_children_if',
            'remove_parser',
            'remove_content',
            'remove_first',
@@ -61,6 +62,7 @@ __all__ = ('transformation_factory',
            'remove_expendables',
            'remove_brackets',
            'remove_tokens',
+           'keep_children',
            'flatten',
            'forbid',
            'require',

DHParser/versionnumber.py

@@ -16,5 +16,5 @@ implied.  See the License for the specific language governing
 permissions and limitations under the License.
 """
 
+__all__ = ('__version__')
 __version__ = '0.7.6' # + '_dev' + str(os.stat(__file__).st_mtime)
-__all__ = ['__version__']

DevScripts/Readme-DevScripts.md

@@ -3,3 +3,5 @@ Folder "DevScripts"
 
 This folder contains helper scripts for the
 development of DHParser.
+
+collect_symbols.py - Lists all exported symbols from DHParser modules

DevScripts/collect_symbols.py

-"""collect_symbols.py - EBNF -> Lists all exported symbols from DHParser
+"""collect_symbols.py - Lists all exported symbols from DHParser modules
 
 Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                 Bavarian Academy of Sciences an Humanities (badw.de)

DevScripts/create_standalone.py

+"""create_standalone.py - merges the DHParser modules into a standalone
+        DHParser.py module for easier deployment.
+
+Copyright 2016  by Eckhart Arnold (arnold@badw.de)
+                Bavarian Academy of Sciences an Humanities (badw.de)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.
+"""
+
+import functools
+import operator
+import sys
+
+sys.path.append('../')
+try:
+    import regex as re
+except ImportError:
+    import re
+
+modules = ('toolkit', 'syntaxtree', 'parser', 'transform', 'ebnf', 'dsl', 'testing', 'versionnumber')
+
+all_symbols = list(functools.reduce(operator.or_, (set(eval(m + '.__all__')) for m in modules)))
+all_symbols.sort()
+
+
+def start(module):
+    i = module.find('__all__')
+    i = module.find(')', i)
+    i = module.find('\n', i) + 1
+    return i
+
+
+doc = "DHParser.py - Packrat-parser and parser-generator\n\n" + __doc__[__doc__.find('Copyright'):]
+
+imports = """
+import abc
+import codecs
+import collections
+from collections import OrderedDict
+import configparser
+import contextlib
+import copy
+from functools import partial
+import hashlib
+import inspect
+import json
+import os
+import platform
+try:
+    import regex as re
+except ImportError:
+    import re
+import sys
+from typing import AbstractSet, Any, ByteString, Callable, cast, Container, Dict, \\
+        Iterator, List, NamedTuple, Sequence, Set, Union, Text, Tuple
+"""
+
+heading = """
+#######################################################################
+#######################################################################
+#
+# %s
+#
+#######################################################################
+#######################################################################
+"""
+
+
+def merge_modules(module_names, dhp_path='../DHParser/'):
+    components = [doc, imports]
+
+    for name in module_names:
+        with open(dhp_path + '%s.py' % name) as f:
+            module = f.read()
+            content = module[start(module):]
+            components.append(heading % name)
+            components.append(content)
+
+    return "\n".join(components)
+
+
+print(merge_modules(modules))

examples/LaTeX/recompile_grammar_obsolote.py

@@ -20,14 +20,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import cProfile
 import functools
 import sys
 
 sys.path.extend(['../../', '../', './'])
 
 from DHParser.toolkit import logging
-from DHParser.testing import recompile_grammar
+from DHParser.dsl import recompile_grammar
 
 
 def profile(func):

examples/LaTeX/tst_grammar.py

@@ -20,10 +20,14 @@ limitations under the License.
 """
 
 import sys
+
+import DHParser.dsl
+
 sys.path.extend(['../../', '../', './'])
 
 from DHParser import testing
-if not testing.recompile_grammar('LaTeX.ebnf', force=True):  # recompiles Grammar only if it has changed
+
+if not DHParser.dsl.recompile_grammar('LaTeX.ebnf', force=True):  # recompiles Grammar only if it has changed
     with open('LaTeX_ebnf_ERRORS.txt') as f:
         print(f.read())
     sys.exit(1)

examples/MLW/recompile_grammar.py

@@ -20,7 +20,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-from DHParser.testing import recompile_grammar
+from DHParser.dsl import recompile_grammar
+
 recompile_grammar('.')
 
 # import os

examples/Tutorial/recompile_grammar.py

@@ -26,7 +26,7 @@ import sys
 sys.path.extend(['../../', '../', './'])
 flag = os.path.exists('LyrikCompiler.py')
 
-from DHParser.testing import recompile_grammar
+from DHParser.dsl import recompile_grammar
 
 if not recompile_grammar('.', force=True):
     with open('Lyrik_ebnf_ERRORS.txt') as f:

test/test_syntaxtree.py

@@ -23,10 +23,9 @@ import copy
 import sys
 sys.path.extend(['../', './'])
 
-from DHParser.syntaxtree import Node, TOKEN_PTYPE
+from DHParser.syntaxtree import Node, TOKEN_PTYPE, mock_syntax_tree
 from DHParser.transform import traverse, reduce_single_child, \
     replace_by_single_child, flatten, remove_expendables
-from DHParser.testing import mock_syntax_tree
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
 from DHParser.dsl import parser_factory
 

test/test_testing.py

@@ -25,11 +25,11 @@ from functools import partial
 sys.path.extend(['../', './'])
 
 from DHParser.toolkit import compact_sexpr
-from DHParser.syntaxtree import TOKEN_PTYPE
+from DHParser.syntaxtree import TOKEN_PTYPE, mock_syntax_tree
 from DHParser.transform import traverse, remove_expendables, \
     replace_by_single_child, reduce_single_child, flatten
 from DHParser.dsl import parser_factory
-from DHParser.testing import grammar_unit, mock_syntax_tree
+from DHParser.testing import grammar_unit
 
 ARITHMETIC_EBNF = """
     @ whitespace = linefeed