Skip to content

GitLab

D
DHParser
Commit c0877f53 authored by eckhart's avatar eckhart
Browse files
Options

- scripts/dhparser.py, dsl.py: code-templates are now read from template files... 

- scripts/dhparser.py, dsl.py: code-templates are now read from template files in templates-subdirectory
parent 3de3ea94
Hide whitespace changes
Inline Side-by-side
Showing with 488 additions and 204 deletions
DHParser/dsl.py
View file @ c0877f53
......@@ -37,8 +37,7 @@ from DHParser.parse import Grammar
from DHParser.preprocess import nil_preprocessor, PreprocessorFunc
from DHParser.syntaxtree import Node
from DHParser.transform import TransformationFunc
from DHParser.toolkit import load_if_file, is_python_code, compile_python_object, \
re
from DHParser.toolkit import DHPARSER_DIR, load_if_file, is_python_code, compile_python_object, re
from typing import Any, cast, List, Tuple, Union, Iterator, Iterable, Optional, \
Callable, Generator
......@@ -54,6 +53,15 @@ __all__ = ('DefinitionError',
'recompile_grammar')
def read_template(template_name: str) -> str:
"""
Reads a script-template from a template file named `template_name`
in the template-directory and returns it as a string.
"""
with open(os.path.join(DHPARSER_DIR, 'templates', template_name), 'r') as f:
return f.read()
SECTION_MARKER = """\n
#######################################################################
#
......@@ -72,50 +80,7 @@ AST_SECTION = "AST SECTION - Can be edited. Changes will be preserved."
COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved."
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
DHPARSER_MAIN = '''
def compile_src(source, log_dir=''):
"""Compiles ``source`` and returns (result, errors, ast).
"""
with logging(log_dir):
compiler = get_compiler()
result_tuple = compile_source(source, get_preprocessor(),
get_grammar(),
get_transformer(), compiler)
return result_tuple
if __name__ == "__main__":
# recompile grammar if needed
grammar_path = os.path.abspath(__file__).replace('Compiler.py', '.ebnf')
if os.path.exists(grammar_path):
if not recompile_grammar(grammar_path, force=False,
notify=lambda:print('recompiling ' + grammar_path)):
error_file = os.path.basename(__file__).replace('Compiler.py', '_ebnf_ERRORS.txt')
with open(error_file, encoding="utf-8") as f:
print(f.read())
sys.exit(1)
else:
print('Could not check whether grammar requires recompiling, '
'because grammar was not found at: ' + grammar_path)
if len(sys.argv) > 1:
# compile file
file_name, log_dir = sys.argv[1], ''
if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
file_name, log_dir = sys.argv[2], 'LOGS'
result, errors, _ = compile_src(file_name, log_dir)
if errors:
cwd = os.getcwd()
rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
for error in errors:
print(rel_path + ':' + str(error))
sys.exit(1)
else:
print(result.as_xml() if isinstance(result, Node) else result)
else:
print("Usage: {NAME}Compiler.py [FILENAME]")
'''
# TODO: Add support for spawning a compilation server via supprocess.Popen() to DHParser main
DHPARSER_MAIN = read_template('DSLCompiler.pyi')
class DSLException(Exception):
......
DHParser/ebnf.py
View file @ c0877f53
......@@ -38,7 +38,7 @@ from DHParser.parse import Grammar, mixin_comment, Forward, RegExp, DropWhitespa
from DHParser.preprocess import nil_preprocessor, PreprocessorFunc
from DHParser.syntaxtree import Node, WHITESPACE_PTYPE, TOKEN_PTYPE
from DHParser.toolkit import load_if_file, escape_re, md5, sane_parser_name, re, expand_table, \
GLOBALS, get_config_value, unrepr, compile_python_object
GLOBALS, get_config_value, unrepr, compile_python_object, DHPARSER_DIR
from DHParser.transform import TransformationFunc, traverse, remove_brackets, \
reduce_single_child, replace_by_single_child, remove_whitespace, remove_empty, \
remove_tokens, flatten, forbid, assert_content
......@@ -68,7 +68,7 @@ __all__ = ('get_ebnf_preprocessor',
########################################################################
dhparserdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
dhparser_parentdir = os.path.dirname(DHPARSER_DIR)
DHPARSER_IMPORTS = '''
......@@ -77,7 +77,7 @@ from functools import partial
import os
import sys
sys.path.append(r'{dhparserdir}')
sys.path.append(r'{dhparser_parentdir}')
try:
import regex as re
......@@ -101,7 +101,7 @@ from DHParser import logging, is_filename, load_if_file, \\
error_on, recompile_grammar, left_associative, lean_left, set_config_value, \\
get_config_value, XML_SERIALIZATION, SXPRESSION_SERIALIZATION, COMPACT_SERIALIZATION, \\
JSON_SERIALIZATION, CONFIG_PRESET, GLOBALS
'''.format(dhparserdir=dhparserdir)
'''.format(dhparser_parentdir=dhparser_parentdir)
########################################################################
......
DHParser/scripts/dhparser.py
View file @ c0877f53
......@@ -31,6 +31,8 @@ if i >= 0:
else:
dhparserdir = ''
templatedir = os.path.join(os.path.dirname(scriptdir.rstrip('/')), 'templates')
from DHParser.compile import compile_source
from DHParser.dsl import compileDSL, compile_on_disk # , recompile_grammar
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
......@@ -40,159 +42,14 @@ from typing import cast
LOGGING = False
EBNF_TEMPLATE = r"""-grammar
#######################################################################
#
# EBNF-Directives
#
#######################################################################
@ whitespace = vertical # implicit whitespace, includes any number of line feeds
@ literalws = right # literals have implicit whitespace on the right hand side
@ comment = /#.*/ # comments range from a '#'-character to the end of the line
@ ignorecase = False # literals and regular expressions are case-sensitive
#######################################################################
#
#: Structure and Components
#
#######################################################################
document = ~ { WORD } §EOF # root parser: a sequence of words preceded
# by whitespace until the end of file
#######################################################################
#
#: Regular Expressions
#
#######################################################################
WORD = /\w+/~ # a sequence of letters, optional trailing whitespace
EOF = !/./ # no more characters ahead, end of file reached
"""
TEST_WORD_TEMPLATE = r'''[match:WORD]
M1: word
M2: one_word_with_underscores
[fail:WORD]
F1: two words
'''
TEST_DOCUMENT_TEMPLATE = r'''[match:document]
M1: """This is a sequence of words
extending over several lines"""
M2: """ This sequence contains leading whitespace"""
[fail:document]
F1: """This test should fail, because neither
comma nor full have been defined anywhere."""
'''
README_TEMPLATE = """# {name}
PLACE A SHORT DESCRIPTION HERE
Author: AUTHOR'S NAME <EMAIL>, AFFILIATION
## License
{name} is open source software under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0)
Copyright YEAR AUTHOR'S NAME <EMAIL>, AFFILIATION
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
GRAMMAR_TEST_TEMPLATE = r'''#!/usr/bin/python3
"""tst_{name}_grammar.py - runs the unit tests for the {name}-grammar
"""
import os
import sys
LOGGING = False
scriptpath = os.path.dirname(__file__)
dhparserdir = os.path.abspath(os.path.join(scriptpath, '{reldhparserdir}'))
if dhparserdir not in sys.path:
sys.path.append(dhparserdir)
try:
from DHParser import dsl
import DHParser.log
from DHParser import testing
except ModuleNotFoundError:
print('Could not import DHParser. Please adjust sys.path in file '
'"%s" manually' % __file__)
sys.exit(1)
def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
testing.create_test_templates(grammar_src, grammar_tests_dir)
with DHParser.log.logging(False):
# recompiles Grammar only if it has changed
if not dsl.recompile_grammar(grammar_src, force=force,
notify=lambda: print('recompiling ' + grammar_src)):
print('\nErrors while recompiling "%s":' % grammar_src +
'\n--------------------------------------\n\n')
with open('{name}_ebnf_ERRORS.txt') as f:
print(f.read())
sys.exit(1)
def run_grammar_tests(glob_pattern, get_grammar, get_transformer):
with DHParser.log.logging(LOGGING):
error_report = testing.grammar_suite(
os.path.join(scriptpath, 'grammar_tests'),
get_grammar, get_transformer,
fn_patterns=[glob_pattern], report=True, verbose=True)
return error_report
if __name__ == '__main__':
argv = sys.argv[:]
if len(argv) > 1 and sys.argv[1] == "--debug":
LOGGING = True
del argv[1]
if (len(argv) >= 2 and (argv[1].endswith('.ebnf') or
os.path.splitext(argv[1])[1].lower() in testing.TEST_READERS.keys())):
# if called with a single filename that is either an EBNF file or a known
# test file type then use the given argument
arg = argv[1]
else:
# otherwise run all tests in the test directory
arg = '*_test_*.ini'
if arg.endswith('.ebnf'):
recompile_grammar(arg, force=True)
else:
recompile_grammar(os.path.join(scriptpath, '{name}.ebnf'),
force=False)
sys.path.append('.')
from {name}Compiler import get_grammar, get_transformer
error_report = run_grammar_tests(arg, get_grammar, get_transformer)
if error_report:
print('\n')
print(error_report)
sys.exit(1)
print('ready.\n')
'''
def read_template(template_name: str) -> str:
"""
Reads a script-template from a template file named `template_name`
in the template-directory and returns it as a string.
"""
with open(os.path.join(templatedir, template_name), 'r') as f:
return f.read()
def create_project(path: str):
......@@ -207,6 +64,12 @@ def create_project(path: str):
else:
print('"%s" already exists! Not overwritten.' % name)
EBNF_TEMPLATE = read_template('example_DSL.ebnf')
TEST_WORD_TEMPLATE = read_template('example_01_test_Regular_Expressions.ini')
TEST_DOCUMENT_TEMPLATE = read_template('example_02_test_Structure_and_Components.ini')
README_TEMPLATE = read_template('readme_template.md')
GRAMMAR_TEST_TEMPLATE = read_template('tst_DSL_grammar.pyi')
name = os.path.basename(path)
if not re.match(r'(?!\d)\w+', name):
print('Project name "%s" is not a valid identifier! Aborting.' % name)
......@@ -230,7 +93,7 @@ def create_project(path: str):
TEST_WORD_TEMPLATE)
create_file(os.path.join('grammar_tests', '02_test_Structure_and_Components.ini'),
TEST_DOCUMENT_TEMPLATE)
create_file(name + '.ebnf', '# ' + name + EBNF_TEMPLATE)
create_file(name + '.ebnf', EBNF_TEMPLATE.replace('GRAMMAR_NAME', name, 1))
create_file('README.md', README_TEMPLATE.format(name=name))
create_file('tst_%s_grammar.py' % name, GRAMMAR_TEST_TEMPLATE.format(
name=name, reldhparserdir=os.path.relpath(dhparserdir, os.path.abspath('.'))))
......
DHParser/templates/DSLCompiler.pyi 0 → 100644
View file @ c0877f53
def compile_src(source, log_dir=''):
"""Compiles ``source`` and returns (result, errors, ast).
"""
with logging(log_dir):
compiler = get_compiler()
result_tuple = compile_source(source, get_preprocessor(),
get_grammar(),
get_transformer(), compiler)
return result_tuple
if __name__ == "__main__":
# recompile grammar if needed
grammar_path = os.path.abspath(__file__).replace('Compiler.py', '.ebnf')
if os.path.exists(grammar_path):
if not recompile_grammar(grammar_path, force=False,
notify=lambda:print('recompiling ' + grammar_path)):
error_file = os.path.basename(__file__).replace('Compiler.py', '_ebnf_ERRORS.txt')
with open(error_file, encoding="utf-8") as f:
print(f.read())
sys.exit(1)
else:
print('Could not check whether grammar requires recompiling, '
'because grammar was not found at: ' + grammar_path)
if len(sys.argv) > 1:
# compile file
file_name, log_dir = sys.argv[1], ''
if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
file_name, log_dir = sys.argv[2], 'LOGS'
result, errors, _ = compile_src(file_name, log_dir)
if errors:
cwd = os.getcwd()
rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
for error in errors:
print(rel_path + ':' + str(error))
sys.exit(1)
else:
print(result.as_xml() if isinstance(result, Node) else result)
else:
print("Usage: {NAME}Compiler.py [FILENAME]")
DHParser/templates/DSLServer.pyi 0 → 100644
View file @ c0877f53
#!/usr/bin/python3
"""MLWServer.py - starts a server (if not already running) for the
compilation of the MLW (medieval latin dictionary)
Author: Eckhart Arnold <arnold@badw.de>
Copyright 2017 Bavarian Academy of Sciences and Humanities
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import asyncio
import os
import sys
scriptdir = os.path.dirname(os.path.realpath(__file__))
sys.path.extend([os.path.join(scriptdir, 'DHParser-submodule')])
STOP_SERVER_REQUEST = b"__STOP_SERVER__" # hardcoded in order to avoid import from DHParser.server
IDENTIFY_REQUEST = "identify()"
config_filename_cache = ''
def get_config_filename() -> str:
"""
Returns the file name of a temporary config file that stores
the host and port of the currently running server.
"""
global config_filename_cache
if config_filename_cache:
return config_filename_cache
def probe(dir_list) -> str:
for tmpdir in []:
if os.path.exists(tmpdir) and os.path.isdir(tmpdir):
return tmpdir
return ''
if sys.platform.find('win') >= 0:
tmpdir = probe([r'C:\TEMP', r'C:\TMP', r'\TEMP', r'\TMP'])
else:
tmpdir = probe(['~/tmp', '/tmp', '/var/tmp', 'usr/tmp'])
config_filename_cache = os.path.join(tmpdir, os.path.basename(__file__)) + '.cfg'
return config_filename_cache
def retrieve_host_and_port():
"""
Retrieve host and port from temporary config file or return default values
for host and port, in case the temporary config file does not exist.
"""
host = '127.0.0.1' # default host
port = 8888
cfg_filename = get_config_filename()
try:
with open(cfg_filename) as f:
print('Reading host and port from file: ' + cfg_filename)
host, ports = f.read().strip(' \n').split(' ')
port = int(ports)
except FileNotFoundError:
pass
except ValueError:
print('removing invalid config file: ' + cfg_filename)
os.remove(cfg_filename)
return host, port
def asyncio_run(coroutine):
"""Backward compatible version of Pyhon 3.7's `asyncio.run()`"""
if sys.version_info >= (3, 7):
return asyncio.run(coroutine)
else:
loop = asyncio.get_event_loop()
try:
return loop.run_until_complete(coroutine)
finally:
loop.close()
def json_rpc(func, params=[], ID=None) -> str:
"""Generates a JSON-RPC-call for `func` with parameters `params`"""
return str({"jsonrpc": "2.0", "method": func.__name__, "params": params, "id": ID})
def mlw_compiler(dateiname):
from MLWCompiler import verarbeite_mlw_artikel
print("Generiere HTML " + dateiname)
ergebnis = verarbeite_mlw_artikel(dateiname, '', {})
return ergebnis
def run_server(host, port):
from DHParser.server import LanguageServer
config_filename = get_config_filename()
try:
with open(config_filename, 'w') as f:
f.write(host + ' ' + str(port))
except PermissionError:
print('PermissionError: Could not write temporary config file: ' + config_filename)
print('Starting server on %s:%i' % (host, port))
mlw_server = LanguageServer({'mlw_compiler': mlw_compiler})
mlw_server.run_server(host, port)
async def send_request(request, host, port):
reader, writer = await asyncio.open_connection(host, port)
writer.write(request.encode() if isinstance(request, str) else request)
data = await reader.read(500)
writer.close()
return data.decode()
def start_server_daemon(host, port):
import subprocess, time
try:
subprocess.Popen([__file__, '--startserver', host, str(port)])
except OSError:
try:
subprocess.Popen(['python3', __file__, '--startserver', host, str(port)])
except FileNotFoundError:
subprocess.Popen(['python', __file__, '--startserver', host, str(port)])
countdown = 20
delay = 0.05
result = None
while countdown > 0:
try:
result = asyncio_run(send_request(IDENTIFY_REQUEST, host, port))
countdown = 0
except ConnectionRefusedError:
time.sleep(delay)
delay += 0.0
countdown -= 1
if result is None:
print('Could not start server or establish connection in time :-(')
sys.exit(1)
print(result)
def print_usage_and_exit():
print('Usages:\n'
+ ' python MLWServer.py --startserver [host] [port]\n'
+ ' python MLWServer.py --stopserver\n'
+ ' python MLWServer.py --status\n'
+ ' python MLWServer.py FILENAME.mlw [--host host] [--port port]')
sys.exit(1)
def assert_if(cond: bool, message: str):
if not cond:
if message:
print(message)
print_usage_and_exit()
if __name__ == "__main__":
host, port = '', -1
# read and remove "--host ..." and "--port ..." parameters from sys.argv
argv = []
i = 0
while i < len(sys.argv):
if sys.argv[i] in ('--host', '-h'):
assert_if(i < len(sys.argv) - 1, 'host missing!')
host = sys.argv[i+1]
i += 1
elif sys.argv[i] in ('--port', '-p'):
assert_if(i < len(sys.argv) - 1, 'port number missing!')
try:
port = int(sys.argv[i+1])
except ValueError:
assert_if(False, 'invalid port number: ' + sys.argv[i+1])
i += 1
else:
argv.append(sys.argv[i])
i += 1
if len(argv) < 2:
print_usage_and_exit()
if port < 0 or not host:
# if host and port have not been specified explicitly on the command
# line, try to retrieve them from (temporary) config file or use
# hard coded default values
host, port = retrieve_host_and_port()
if sys.argv[1] == "--status":
try:
result = asyncio_run(send_request(IDENTIFY_REQUEST, host, port))
print('Server ' + str(result) + ' running on ' + host + ' ' + str(port))
except ConnectionRefusedError:
print('No server running on: ' + host + ' ' + str(port))
elif argv[1] == "--startserver":
if len(argv) == 2:
argv.append(host)
if len(argv) == 3:
argv.append(str(port))
sys.exit(run_server(argv[2], int(argv[3])))
elif argv[1] in ("--stopserver", "--killserver"):
try:
result = asyncio_run(send_request(STOP_SERVER_REQUEST, host, port))
cfg_filename = get_config_filename()
try:
os.remove(cfg_filename)
print('removing temporary config file: ' + cfg_filename)
except FileNotFoundError:
pass
except ConnectionRefusedError as e:
print(e)
sys.exit(1)
print(result)
elif argv[1].startswith('-'):
print_usage_and_exit()
elif argv[1]:
if not argv[1].endswith(')'):
# argv does not seem to be a command (e.g. "identify()") but a file name or path
argv[1] = os.path.abspath(argv[1])
print(argv[1])
try:
result = asyncio_run(send_request(argv[1], host, port))
except ConnectionRefusedError:
start_server_daemon(host, port) # start server first
result = asyncio_run(send_request(argv[1], host, port))
print(result)
else:
print_usage_and_exit()
DHParser/templates/README.txt 0 → 100644
View file @ c0877f53
DHParser script templates
Copyright 2016 by Eckhart Arnold (arnold@badw.de)
Bavarian Academy of Sciences an Humanities (badw.de)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License.
The "DHParser/templates" sub-directory contains templates for python scripts
that are generated by DHParser when creating a new project. Usually, this is
done by substituting placeholders withing the template like "{name}" via
the format()-method of Python's string type.
Scripts are being generated within these modules:
- DHParser/srcipts/dhparser.py
- DHParser/dsl.py
- DHParser/ebnf.py
DHParser/templates/example_01_test_Regular_Expressions.ini 0 → 100644
View file @ c0877f53
[match:WORD]
M1: word
M2: one_word_with_underscores
[fail:WORD]
F1: two words
DHParser/templates/example_02_test_Structure_and_Components.ini 0 → 100644
View file @ c0877f53
[match:document]
M1: """This is a sequence of words
extending over several lines"""
M2: """ This sequence contains leading whitespace"""
[fail:document]
F1: """This test should fail, because neither
comma nor full have been defined anywhere."""
DHParser/templates/example_DSL.ebnf 0 → 100644
View file @ c0877f53
# GRAMMAR_NAME-grammar
#######################################################################
#
# EBNF-Directives
#
#######################################################################
@ whitespace = vertical # implicit whitespace, includes any number of line feeds