refactoring: split syntaxtree into syntaxtree and transform

DHParser/__init__.py

@@ -18,15 +18,15 @@ implied.  See the License for the specific language governing
 permissions and limitations under the License.
 """
 
+from .dsl import *
+from .ebnf import *
+from .parser import *
+from .syntaxtree import *
 # Flat namespace for the DHParser Package. Is this a good idea...?
 from .toolkit import *
-from .syntaxtree import *
-from .parsers import *
-from .ebnf import *
-from .dsl import *
 # from .testing import *
 from .versionnumber import __version__
 
 __author__ = "Eckhart Arnold <arnold@badw.de>"
 __copyright__ = "http://www.apache.org/licenses/LICENSE-2.0"
-# __all__ = ['toolkit', 'syntaxtree', 'parsers', 'ebnf', 'dsl', 'testing', 'versionnumber']  # flat namespace
+# __all__ = ['toolkit', 'syntaxtree', 'parser', 'transform', 'ebnf', 'dsl', 'testing', 'versionnumber']  # flat namespace

DHParser/dsl.py

@@ -33,7 +33,7 @@ from DHParser.ebnf import EBNFTransformer, EBNFCompiler, grammar_changed, \
     get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
     PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
 from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object
-from DHParser.parsers import Grammar, Compiler, compile_source, nil_preprocessor, PreprocessorFunc
+from DHParser.parser import Grammar, Compiler, compile_source, nil_preprocessor, PreprocessorFunc
 from DHParser.syntaxtree import Node, TransformationFunc
 
 
@@ -75,16 +75,17 @@ try:
 except ImportError:
     import re
 from DHParser.toolkit import logging, is_filename, load_if_file    
-from DHParser.parsers import Grammar, Compiler, nil_preprocessor, \\
+from DHParser.parser import Grammar, Compiler, nil_preprocessor, \\
     Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym, \\
     Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
     last_value, counterpart, accumulate, PreprocessorFunc
-from DHParser.syntaxtree import Node, traverse, remove_children_if, \\
+from DHParser.syntaxtree import Node, TransformationFunc
+from DHParser.transform import traverse, remove_children_if, \\
     reduce_single_child, replace_by_single_child, remove_whitespace, \\
     remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
     is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \\
-    TransformationFunc, remove_parser, remove_content, remove_brackets, \\
+    remove_parser, remove_content, remove_brackets, \\
     keep_children, has_name, has_content, apply_if, remove_first, remove_last
 '''
 

DHParser/ebnf.py

@@ -29,13 +29,13 @@ except ImportError:
     from .typing34 import Callable, Dict, List, Set, Tuple
 
 from DHParser.toolkit import load_if_file, escape_re, md5, sane_parser_name
-from DHParser.parsers import Grammar, mixin_comment, nil_preprocessor, Forward, RE, NegativeLookahead, \
+from DHParser.parser import Grammar, mixin_comment, nil_preprocessor, Forward, RE, NegativeLookahead, \
     Alternative, Series, Optional, Required, OneOrMore, ZeroOrMore, Token, Compiler, \
     PreprocessorFunc
-from DHParser.syntaxtree import Node, traverse, remove_brackets, \
-    reduce_single_child, replace_by_single_child, TOKEN_PTYPE, remove_expendables, \
-    remove_tokens, flatten, forbid, assert_content, WHITESPACE_PTYPE, key_tag_name, \
-    TransformationFunc
+from DHParser.syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, Node, TransformationFunc
+from DHParser.transform import traverse, remove_brackets, \
+    reduce_single_child, replace_by_single_child, remove_expendables, \
+    remove_tokens, flatten, forbid, assert_content, key_tag_name
 from DHParser.versionnumber import __version__
 
 __all__ = ['get_ebnf_preprocessor',

DHParser/testing.py

@@ -32,6 +32,16 @@ from DHParser.syntaxtree import MockParser
 from DHParser.ebnf import grammar_changed
 from DHParser.dsl import compile_on_disk
 
+__all__ = ('mock_syntax_tree',
+           'recompile_grammar',
+           'unit_from_configfile',
+           'unit_from_json',
+           'unit_from_file',
+           'get_report',
+           'grammar_unit',
+           'grammar_suite',
+           'runner')
+
 
 def mock_syntax_tree(sexpr):
     """

DHParser/versionnumber.py

@@ -16,6 +16,5 @@ implied.  See the License for the specific language governing
 permissions and limitations under the License.
 """
 
-import os
-
 __version__ = '0.7.6' # + '_dev' + str(os.stat(__file__).st_mtime)
+__all__ = ['__version__']

DevScripts/Readme-DevScripts.md

+Folder "DevScripts"
+===================
+
+This folder contains helper scripts for the
+development of DHParser.

DevScripts/collect_symbols.py

+"""collect_symbols.py - EBNF -> Lists all exported symbols from DHParser
+
+Copyright 2016  by Eckhart Arnold (arnold@badw.de)
+                Bavarian Academy of Sciences an Humanities (badw.de)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.
+"""
+
+import sys
+import textwrap
+
+sys.path.append('../')
+
+from DHParser import toolkit
+from DHParser import syntaxtree
+from DHParser import parser
+from DHParser import transform
+from DHParser import ebnf
+from DHParser import dsl
+from DHParser import testing
+from DHParser import versionnumber
+
+symbols_table = {
+    'toolkit': list(toolkit.__all__),
+    'syntaxtree': list(syntaxtree.__all__),
+    'parser': list(parser.__all__),
+    'transform': list(transform.__all__),
+    'ebnf': list(ebnf.__all__),
+    'dsl': list(dsl.__all__),
+    'testing': list(testing.__all__),
+    'versionnumber': list(versionnumber.__all__)
+}
+
+DSL_imports = {'parser', 'syntaxtree', 'transform'}
+
+all_symbols = []
+for module, symbols in symbols_table.items():
+    assert len(set(symbols)) == len(symbols), "Double symbols in field '__all__' of module " + module
+    for sym in symbols:
+        exec("from DHParser.%s import %s" % (module, sym))
+    symbols_copy = tuple(symbols)
+    symbols.sort()
+    if symbols_copy != tuple(symbols):
+        print()
+        print(module)
+        all = "__all__ = (" + ",\n           ".join("'%s'" % s for s in symbols) + ")"
+        print(all)
+        print()
+        all = "\n    ".join(textwrap.wrap("from DHParser.%s import " % module + ", ".join(symbols), 99))
+        print(all)
+    if module in DSL_imports:
+        all_symbols.extend(symbols)
+
+assert len(set(all_symbols)) == len(all_symbols), "Double symbols in module " + module
+all_symbols.extend(['logging', 'is_filename', 'load_if_file'])
+all_symbols.sort()
+print()
+print("DSL-imports")
+all = "\n   ".join(textwrap.wrap("from DHParser import " + ", ".join(all_symbols), 99))
+print(all)

dhparser.py

@@ -26,7 +26,7 @@ from functools import partial
 
 from DHParser.dsl import compileDSL, compile_on_disk
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
-from DHParser.parsers import compile_source, nil_preprocessor
+from DHParser.parser import compile_source, nil_preprocessor
 from DHParser.toolkit import logging
 
 

examples/LaTeX/LaTeXCompiler.py

@@ -16,7 +16,7 @@ try:
 except ImportError:
     import re
 from DHParser.toolkit import logging, is_filename
-from DHParser.parsers import Grammar, Compiler, Alternative, Pop, Required, Token, Synonym, \
+from DHParser.parser import Grammar, Compiler, Alternative, Pop, Required, Token, Synonym, \
     Optional, OneOrMore, Series, RE, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
     PreprocessorFunc

examples/MLW/VERALTET/MLW_compiler.py

@@ -14,7 +14,7 @@ try:
     import regex as re
 except ImportError:
     import re
-from DHParser.parsers import Grammar, Compiler, Alternative, Required, Token, \
+from DHParser.parser import Grammar, Compiler, Alternative, Required, Token, \
     Optional, OneOrMore, Series, RE, ZeroOrMore, NegativeLookahead, mixin_comment, compile_source
 from DHParser.syntaxtree import traverse, reduce_single_child, replace_by_single_child, no_transformation, \
     remove_expendables, remove_tokens, flatten, \

examples/Tutorial/LyrikCompiler_example.py

@@ -17,17 +17,14 @@ try:
     import regex as re
 except ImportError:
     import re
-from DHParser.toolkit import logging, is_filename, load_if_file
-from DHParser.parsers import Grammar, Compiler, nil_preprocessor, \
-    Lookbehind, Lookahead, Alternative, Pop, Required, Token, \
-    Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \
-    ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
+from DHParser.toolkit import logging, is_filename
+from DHParser.parser import Grammar, Compiler, Required, Token, \
+    Optional, OneOrMore, Series, RE, ZeroOrMore, NegativeLookahead, mixin_comment, compile_source, \
     PreprocessorFunc, Synonym
-from DHParser.syntaxtree import Node, traverse, remove_first, remove_last, \
-    remove_children_if, reduce_single_child, replace_by_single_child, remove_whitespace, \
-    remove_expendables, remove_tokens, flatten, is_whitespace, is_expendable, \
-    collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, TransformationFunc, \
-    remove_parser, remove_empty, has_content, has_name
+from DHParser.syntaxtree import Node, traverse, remove_last, \
+    reduce_single_child, replace_by_single_child, remove_tokens, flatten, is_whitespace, collapse, replace_content, \
+    TransformationFunc, \
+    remove_parser, remove_empty, has_name
 
 
 #######################################################################

test/test_DHParser.py

@@ -23,13 +23,6 @@ limitations under the License.
 import sys
 sys.path.extend(['../', './'])
 
-from DHParser.toolkit import *
-from DHParser.syntaxtree import *
-from DHParser.parsers import *
-from DHParser.ebnf import *
-from DHParser.dsl import *
-
-
 if __name__ == "__main__":
     from DHParser.testing import runner
 

test/test_dsl.py

@@ -24,11 +24,9 @@ import os
 import sys
 sys.path.extend(['../', './'])
 
-from DHParser.parsers import Grammar, Compiler
+from DHParser.parser import Grammar, Compiler
 from DHParser.dsl import compile_on_disk, run_compiler, compileEBNF, parser_factory, \
     load_compiler_suite
-from DHParser.toolkit import is_filename
-
 
 ARITHMETIC_EBNF = """
     @ whitespace = linefeed

test/test_ebnf.py

@@ -25,13 +25,12 @@ try:
 except ImportError:
     import re
 import sys
-from functools import partial
 from multiprocessing import Pool
 
 sys.path.extend(['../', './'])
 
-from DHParser.toolkit import is_logging, compile_python_object
-from DHParser.parsers import compile_source, Retrieve, WHITESPACE_PTYPE, nil_preprocessor
+from DHParser.toolkit import compile_python_object
+from DHParser.parser import compile_source, WHITESPACE_PTYPE, nil_preprocessor
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
 from DHParser.dsl import CompilationError, compileDSL, DHPARSER_IMPORTS, parser_factory
 

test/test_parsers.py

@@ -25,7 +25,7 @@ from functools import partial
 sys.path.extend(['../', './'])
 
 from DHParser.toolkit import is_logging, logging, compile_python_object
-from DHParser.parsers import compile_source, Retrieve, Grammar, Forward, Token, ZeroOrMore, RE
+from DHParser.parser import compile_source, Retrieve, Grammar, Forward, Token, ZeroOrMore, RE
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
 from DHParser.dsl import parser_factory, DHPARSER_IMPORTS
 

test/test_syntaxtree.py

@@ -23,8 +23,9 @@ import copy
 import sys
 sys.path.extend(['../', './'])
 
-from DHParser.syntaxtree import Node, traverse, reduce_single_child, \
-    replace_by_single_child, flatten, remove_expendables, TOKEN_PTYPE
+from DHParser.syntaxtree import Node, TOKEN_PTYPE
+from DHParser.transform import traverse, reduce_single_child, \
+    replace_by_single_child, flatten, remove_expendables
 from DHParser.testing import mock_syntax_tree
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
 from DHParser.dsl import parser_factory

test/test_testing.py

@@ -25,8 +25,9 @@ from functools import partial
 sys.path.extend(['../', './'])
 
 from DHParser.toolkit import compact_sexpr
-from DHParser.syntaxtree import traverse, remove_expendables, \
-    replace_by_single_child, reduce_single_child, flatten, TOKEN_PTYPE
+from DHParser.syntaxtree import TOKEN_PTYPE
+from DHParser.transform import traverse, remove_expendables, \
+    replace_by_single_child, reduce_single_child, flatten
 from DHParser.dsl import parser_factory
 from DHParser.testing import grammar_unit, mock_syntax_tree