From 0cf9ea37982d49916b09bed794c2744cd4c6b424 Mon Sep 17 00:00:00 2001 From: Eckhart Arnold Date: Wed, 12 Apr 2017 22:27:12 +0200 Subject: [PATCH] - DSLsupport.run_compiler now produces callable DSL compiler script stubs; code reorganization all DHParser modules moved to a subdirectory containing the DHParser package; dhparser.py script remains in the main directory --- DSLsupport.py => DHParser/DSLsupport.py | 57 ++++++++++++++----- EBNFcompiler.py => DHParser/EBNFcompiler.py | 12 ++-- __init__.py => DHParser/__init__.py | 2 + .../parsercombinators.py | 4 +- syntaxtree.py => DHParser/syntaxtree.py | 2 +- toolkit.py => DHParser/toolkit.py | 0 dhparser.py | 11 ++-- tests/no_unit_tests/PopRetrieve_compiler.py | 35 +++++++++--- .../no_unit_tests/compile_PopRetrieve_EBNF.py | 23 +++++++- tests/test_EBNFcompiler.py | 2 +- tests/test_syntaxtree.py | 2 +- 11 files changed, 109 insertions(+), 41 deletions(-) rename DSLsupport.py => DHParser/DSLsupport.py (85%) rename EBNFcompiler.py => DHParser/EBNFcompiler.py (98%) rename __init__.py => DHParser/__init__.py (88%) rename parsercombinators.py => DHParser/parsercombinators.py (99%) rename syntaxtree.py => DHParser/syntaxtree.py (99%) rename toolkit.py => DHParser/toolkit.py (100%) diff --git a/DSLsupport.py b/DHParser/DSLsupport.py similarity index 85% rename from DSLsupport.py rename to DHParser/DSLsupport.py index fa89a8a1..6df64a27 100644 --- a/DSLsupport.py +++ b/DHParser/DSLsupport.py @@ -29,10 +29,11 @@ try: except ImportError: import re -from EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler -from toolkit import IS_LOGGING, load_if_file, is_python_code, md5, compile_python_object -from parsercombinators import GrammarBase, CompilerBase, full_compilation, nil_scanner -from syntaxtree import Node +from .__init__ import __version__ +from .EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler +from .toolkit import IS_LOGGING, load_if_file, is_python_code, md5, compile_python_object +from .parsercombinators import GrammarBase, CompilerBase, full_compilation, nil_scanner +from .syntaxtree import Node __all__ = ['GrammarError', @@ -88,21 +89,44 @@ class CompilationError(Exception): DHPARSER_IMPORTS = """ from functools import partial +import sys try: import regex as re except ImportError: import re -from parsercombinators import GrammarBase, CompilerBase, nil_scanner, \\ +from DHParser.toolkit import load_if_file +from DHParser.parsercombinators import GrammarBase, CompilerBase, nil_scanner, \\ Lookbehind, Lookahead, Alternative, Pop, Required, Token, \\ Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Sequence, RE, Capture, \\ - ZeroOrMore, Forward, NegativeLookahead, mixin_comment -from syntaxtree import Node, remove_enclosing_delimiters, remove_children_if, \\ + ZeroOrMore, Forward, NegativeLookahead, mixin_comment, full_compilation +from DHParser.syntaxtree import Node, remove_enclosing_delimiters, remove_children_if, \\ reduce_single_child, replace_by_single_child, remove_whitespace, TOKEN_KEYWORD, \\ no_operation, remove_expendables, remove_tokens, flatten, WHITESPACE_KEYWORD, \\ is_whitespace, is_expendable """ +DHPARSER_COMPILER = ''' +def compile_{NAME}(source): + """Compiles ``source`` and returns (result, errors, ast). + """ + source_text = load_if_file(source) + return full_compilation({NAME}Scanner(source_text), + {NAME}Grammar(), {NAME}_ASTPipeline, {NAME}Compiler()) + +if __name__ == "__main__": + if len(sys.argv) > 1: + result, errors, ast = compile_{NAME}(sys.argv[1]) + if errors: + for error in errors: + print(error) + sys.exit(1) + else: + print(result) + else: + print("Usage: {NAME}_compiler.py [FILENAME]") +''' + def get_grammar_instance(grammar): """Returns a grammar object and the source code of the grammar, from @@ -141,14 +165,14 @@ def load_compiler_suite(compiler_suite): source = load_if_file(compiler_suite) if is_python_code(compiler_suite): try: - intro, syms, scanner_py, parser_py, ast_py, compiler_py, outro = \ + intro, imports, scanner_py, parser_py, ast_py, compiler_py, outro = \ RX_SECTION_MARKER.split(source) except ValueError as error: raise ValueError('File "' + compiler_suite + '" seems to be corrupted. ' 'Please delete or repair file manually.') - scanner = compile_python_object(DHPARSER_IMPORTS + scanner_py, '\w*Scanner$') - ast = compile_python_object(DHPARSER_IMPORTS + ast_py, '\w*Pipeline$') - compiler = compile_python_object(DHPARSER_IMPORTS + compiler_py, '\w*Compiler$') + scanner = compile_python_object(imports + scanner_py, '\w*Scanner$') + ast = compile_python_object(imports + ast_py, '\w*Pipeline$') + compiler = compile_python_object(imports + compiler_py, '\w*Compiler$') else: # assume source is an ebnf grammar parser_py, errors, AST = full_compilation( @@ -215,6 +239,7 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"): with open(source_file, encoding="utf-8") as f: source = f.read() rootname = os.path.splitext(filepath)[0] + compiler_name = os.path.basename(rootname) if compiler_suite: scanner, parser, trans, cclass = load_compiler_suite(compiler_suite) compiler = cclass() @@ -222,7 +247,7 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"): scanner = nil_scanner parser = EBNFGrammar() trans = EBNF_ASTPipeline - compiler = EBNFCompiler(os.path.basename(rootname), source) + compiler = EBNFCompiler(compiler_name, source) result, errors, ast = full_compilation(scanner(source), parser, trans, compiler) if errors: @@ -236,10 +261,10 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"): try: f = open(rootname + '_compiler.py', 'r', encoding="utf-8") source = f.read() - intro, syms, scanner, parser, ast, compiler, outro = RX_SECTION_MARKER.split(source) + intro, imports, scanner, parser, ast, compiler, outro = RX_SECTION_MARKER.split(source) except (PermissionError, FileNotFoundError, IOError) as error: intro, outro = '', '' - syms = DHPARSER_IMPORTS + imports = DHPARSER_IMPORTS scanner = compiler.gen_scanner_skeleton() ast = compiler.gen_AST_skeleton() compiler = compiler.gen_compiler_skeleton() @@ -251,9 +276,10 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"): try: f = open(rootname + '_compiler.py', 'w', encoding="utf-8") + f.write("#!/usr/bin/python") f.write(intro) f.write(SECTION_MARKER.format(marker=SYMBOLS_SECTION)) - f.write(syms) + f.write(imports) f.write(SECTION_MARKER.format(marker=SCANNER_SECTION)) f.write(scanner) f.write(SECTION_MARKER.format(marker=PARSER_SECTION)) @@ -264,6 +290,7 @@ def run_compiler(source_file, compiler_suite="", extension=".xml"): f.write(compiler) f.write(SECTION_MARKER.format(marker=END_SECTIONS_MARKER)) f.write(outro) + f.write(DHPARSER_COMPILER.format(NAME=compiler_name)) except (PermissionError, FileNotFoundError, IOError) as error: print('# Could not write file "' + rootname + '_compiler.py" because of: ' + "\n# ".join(str(error).split('\n)'))) diff --git a/EBNFcompiler.py b/DHParser/EBNFcompiler.py similarity index 98% rename from EBNFcompiler.py rename to DHParser/EBNFcompiler.py index f8bd1b65..03bb1ad3 100644 --- a/EBNFcompiler.py +++ b/DHParser/EBNFcompiler.py @@ -26,13 +26,13 @@ try: except ImportError: import re -from toolkit import load_if_file, escape_re, md5, sane_parser_name -from parsercombinators import GrammarBase, mixin_comment, Forward, RE, NegativeLookahead, \ +from .__init__ import __version__ +from .toolkit import load_if_file, escape_re, md5, sane_parser_name +from .parsercombinators import GrammarBase, mixin_comment, Forward, RE, NegativeLookahead, \ Alternative, Sequence, Optional, Required, OneOrMore, ZeroOrMore, Token, CompilerBase -from syntaxtree import Node, remove_enclosing_delimiters, reduce_single_child, \ +from .syntaxtree import Node, remove_enclosing_delimiters, reduce_single_child, \ replace_by_single_child, TOKEN_KEYWORD, remove_expendables, remove_tokens, flatten, \ WHITESPACE_KEYWORD -from __init__ import __version__ __all__ = ['EBNFGrammar', @@ -202,7 +202,7 @@ class EBNFCompiler(CompilerBase): self.grammar_name + '-grammar'] for name in self.definition_names: transtable.append(' "' + name + '": no_operation,') - transtable += [' "": no_operation', '}', '', pl_name + ' = [%s]' % tt_name] + transtable += [' "": no_operation', '}', '', pl_name + ' = [%s]' % tt_name, ''] return '\n'.join(transtable) def gen_compiler_skeleton(self): @@ -225,7 +225,7 @@ class EBNFCompiler(CompilerBase): else: compiler += [' def ' + name + '(self, node):', ' pass', ''] - return '\n'.join(compiler + ['']) + return '\n'.join(compiler) def gen_parser(self, definitions): # fix capture of variables that have been defined before usage [sic!] diff --git a/__init__.py b/DHParser/__init__.py similarity index 88% rename from __init__.py rename to DHParser/__init__.py index ee430f21..7029c93f 100644 --- a/__init__.py +++ b/DHParser/__init__.py @@ -22,3 +22,5 @@ import os __version__ = '0.5.4' + '_dev' + str(os.stat(__file__).st_mtime) __all__ = ['toolkit', 'syntaxtree', 'parsercombinators', 'EBNFcompiler', 'DSLsupport'] +__author__ = "Eckhart Arnold " +__copyright__ = "http://www.apache.org/licenses/LICENSE-2.0" diff --git a/parsercombinators.py b/DHParser/parsercombinators.py similarity index 99% rename from parsercombinators.py rename to DHParser/parsercombinators.py index 0f2beec2..cfbc2ead 100644 --- a/parsercombinators.py +++ b/DHParser/parsercombinators.py @@ -59,8 +59,8 @@ try: except ImportError: import re -from toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name, sequence -from syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \ +from .toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name, sequence +from .syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \ error_messages, traverse diff --git a/syntaxtree.py b/DHParser/syntaxtree.py similarity index 99% rename from syntaxtree.py rename to DHParser/syntaxtree.py index cbece1d0..76cef227 100644 --- a/syntaxtree.py +++ b/DHParser/syntaxtree.py @@ -29,7 +29,7 @@ except ImportError: import re from typing import NamedTuple -from toolkit import IS_LOGGING, LOGS_DIR, expand_table, line_col, sequence +from .toolkit import IS_LOGGING, LOGS_DIR, expand_table, line_col, sequence __all__ = ['WHITESPACE_KEYWORD', diff --git a/toolkit.py b/DHParser/toolkit.py similarity index 100% rename from toolkit.py rename to DHParser/toolkit.py diff --git a/dhparser.py b/dhparser.py index 2eea624b..bba35acc 100644 --- a/dhparser.py +++ b/dhparser.py @@ -18,13 +18,15 @@ implied. See the License for the specific language governing permissions and limitations under the License. """ +#TODO: This is still a stub... + import os import sys from functools import partial -from DSLsupport import compileDSL, run_compiler -from EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler -from parsercombinators import full_compilation +from DHParser.DSLsupport import compileDSL, run_compiler +from DHParser.EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler +from DHParser.parsercombinators import full_compilation def selftest(file_name): @@ -34,8 +36,7 @@ def selftest(file_name): compiler_name = os.path.basename(os.path.splitext(file_name)[0]) compiler = EBNFCompiler(compiler_name, grammar) parser = EBNFGrammar() - result, errors, syntax_tree = full_compilation(grammar, - parser, EBNF_ASTPipeline, compiler) + result, errors, syntax_tree = full_compilation(grammar, parser, EBNF_ASTPipeline, compiler) print(result) if errors: print(errors) diff --git a/tests/no_unit_tests/PopRetrieve_compiler.py b/tests/no_unit_tests/PopRetrieve_compiler.py index a067094c..b7c9873c 100644 --- a/tests/no_unit_tests/PopRetrieve_compiler.py +++ b/tests/no_unit_tests/PopRetrieve_compiler.py @@ -1,4 +1,4 @@ - +#!/usr/bin/python ####################################################################### # @@ -8,15 +8,17 @@ from functools import partial +import sys try: import regex as re except ImportError: import re -from parsercombinators import GrammarBase, CompilerBase, nil_scanner, \ +from DHParser.toolkit import load_if_file +from DHParser.parsercombinators import GrammarBase, CompilerBase, nil_scanner, \ Lookbehind, Lookahead, Alternative, Pop, Required, Token, \ Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Sequence, RE, Capture, \ - ZeroOrMore, Forward, NegativeLookahead, mixin_comment -from syntaxtree import Node, remove_enclosing_delimiters, remove_children_if, \ + ZeroOrMore, Forward, NegativeLookahead, mixin_comment, full_compilation +from DHParser.syntaxtree import Node, remove_enclosing_delimiters, remove_children_if, \ reduce_single_child, replace_by_single_child, remove_whitespace, TOKEN_KEYWORD, \ no_operation, remove_expendables, remove_tokens, flatten, WHITESPACE_KEYWORD, \ is_whitespace, is_expendable @@ -47,7 +49,7 @@ class PopRetrieveGrammar(GrammarBase): delimiter_sign = /`+/ text = /[^`]+/ """ - source_hash__ = "48a3fd5a35aeaa7ce1729e09c65594b0" + source_hash__ = "a418b812a36733a4713eb4e06322e1b5" parser_initialization__ = "upon instatiation" COMMENT__ = r'' WSP__ = mixin_comment(whitespace=r'[ ]*', comment=r'') @@ -79,6 +81,7 @@ PopRetrieve_ASTTransform = { PopRetrieve_ASTPipeline = [PopRetrieve_ASTTransform] + ####################################################################### # # COMPILER SECTION - Can be edited. Changes will be preserved. @@ -109,10 +112,28 @@ class PopRetrieveCompiler(CompilerBase): pass - ####################################################################### # -# END OF PYDSL-SECTIONS +# END OF DHPARSER-SECTIONS # ####################################################################### + +def compile_PopRetrieve(source): + """Compiles ``source`` and returns (result, errors, ast). + """ + source_text = load_if_file(source) + return full_compilation(PopRetrieveScanner(source_text), + PopRetrieveGrammar(), PopRetrieve_ASTPipeline, PopRetrieveCompiler()) + +if __name__ == "__main__": + if len(sys.argv) > 1: + result, errors, ast = compile_PopRetrieve(sys.argv[1]) + if errors: + for error in errors: + print(error) + sys.exit(1) + else: + print(result) + else: + print("Usage: PopRetrieve_compiler.py [FILENAME]") diff --git a/tests/no_unit_tests/compile_PopRetrieve_EBNF.py b/tests/no_unit_tests/compile_PopRetrieve_EBNF.py index 7ed3c485..77732759 100644 --- a/tests/no_unit_tests/compile_PopRetrieve_EBNF.py +++ b/tests/no_unit_tests/compile_PopRetrieve_EBNF.py @@ -23,7 +23,7 @@ limitations under the License. import os import sys sys.path.append(os.path.abspath('../../')) -from DSLsupport import run_compiler, source_changed +from DHParser.DSLsupport import run_compiler, source_changed if (not os.path.exists('PopRetrieve_compiler.py') or source_changed('PopRetrieve.ebnf', 'PopRetrieve_compiler.py')): @@ -33,12 +33,29 @@ if (not os.path.exists('PopRetrieve_compiler.py') or print(errors) sys.exit(1) -errors = run_compiler("PopRetrieveTest.txt", 'PopRetrieve_compiler.py') +from PopRetrieve_compiler import compile_PopRetrieve + +result, errors, ast = compile_PopRetrieve("PopRetrieveTest.txt") if errors: print(errors) sys.exit(1) +else: + print(result) -errors = run_compiler("PopRetrieveTest2.txt", 'PopRetrieve_compiler.py') +result, errors, ast = compile_PopRetrieve("PopRetrieveTest2.txt") if errors: print(errors) sys.exit(1) +else: + print(result) + + +# errors = run_compiler("PopRetrieveTest.txt", 'PopRetrieve_compiler.py') +# if errors: +# print(errors) +# sys.exit(1) +# +# errors = run_compiler("PopRetrieveTest2.txt", 'PopRetrieve_compiler.py') +# if errors: +# print(errors) +# sys.exit(1) diff --git a/tests/test_EBNFcompiler.py b/tests/test_EBNFcompiler.py index 5405cc7d..0d405594 100644 --- a/tests/test_EBNFcompiler.py +++ b/tests/test_EBNFcompiler.py @@ -23,7 +23,7 @@ limitations under the License. import os import sys sys.path.append(os.path.abspath('../../')) -from DSLsupport import compileEBNF, run_compiler, source_changed +from DHParser.DSLsupport import compileEBNF, run_compiler, source_changed WRITE_LOGS = True diff --git a/tests/test_syntaxtree.py b/tests/test_syntaxtree.py index eae954f2..5eae7d3d 100644 --- a/tests/test_syntaxtree.py +++ b/tests/test_syntaxtree.py @@ -24,7 +24,7 @@ import os import re import sys sys.path.append(os.path.abspath('../../')) -from syntaxtree import Node, compact_sexpr +from DHParser.syntaxtree import Node, compact_sexpr class DummyParser: def __init__(self, name=''): -- GitLab