Commit a4af9d28 authored by eckhart's avatar eckhart

- new subdirectory examples/demos

parent c0877f53
......@@ -36,6 +36,7 @@ __all__ = ('CONFIG_PRESET',
'XML_SERIALIZATION',
'SXPRESSION_SERIALIZATION',
'COMPACT_SERIALIZATION',
'SMART_SERIALIZATION',
'JSON_SERIALIZATION',
'SERIALIZATIONS')
......@@ -85,6 +86,9 @@ CONFIG_PRESET['max_parser_dropouts'] = 3
# 'compact' - compact tree output, i.e. children a represented on
# indented lines with no opening or closing tags, brackets
# etc.
# 'smart' - serialize as S-expression if the S-expression fits on
# one line (see 'flatten_sxpr_threshold'), otherwise
# serialize as compact tree output
# 'json' - output in JSON-format. This is probably the least
# readable representation, but useful for serialization, for
# example, to return syntax trees from remote procedure calls.
......@@ -93,6 +97,7 @@ CONFIG_PRESET['max_parser_dropouts'] = 3
XML_SERIALIZATION = "XML"
SXPRESSION_SERIALIZATION = "S-expression"
COMPACT_SERIALIZATION = "compact"
SMART_SERIALIZATION = "smart"
JSON_SERIALIZATION = "json"
SERIALIZATIONS = frozenset({XML_SERIALIZATION,
......@@ -100,9 +105,9 @@ SERIALIZATIONS = frozenset({XML_SERIALIZATION,
COMPACT_SERIALIZATION,
JSON_SERIALIZATION})
CONFIG_PRESET['cst_serialization'] = COMPACT_SERIALIZATION
CONFIG_PRESET['ast_serialization'] = XML_SERIALIZATION
CONFIG_PRESET['default_serialization'] = SXPRESSION_SERIALIZATION
CONFIG_PRESET['cst_serialization'] = SMART_SERIALIZATION
CONFIG_PRESET['ast_serialization'] = SMART_SERIALIZATION
CONFIG_PRESET['default_serialization'] = SMART_SERIALIZATION
# Defines the maximum line length for flattened S-expressions.
# Below this threshold S-expressions will be returned in flattened
......
......@@ -38,7 +38,7 @@ from DHParser.parse import Grammar, mixin_comment, Forward, RegExp, DropWhitespa
from DHParser.preprocess import nil_preprocessor, PreprocessorFunc
from DHParser.syntaxtree import Node, WHITESPACE_PTYPE, TOKEN_PTYPE
from DHParser.toolkit import load_if_file, escape_re, md5, sane_parser_name, re, expand_table, \
GLOBALS, get_config_value, unrepr, compile_python_object, DHPARSER_DIR
GLOBALS, get_config_value, unrepr, compile_python_object, DHPARSER_PARENTDIR
from DHParser.transform import TransformationFunc, traverse, remove_brackets, \
reduce_single_child, replace_by_single_child, remove_whitespace, remove_empty, \
remove_tokens, flatten, forbid, assert_content
......@@ -68,9 +68,6 @@ __all__ = ('get_ebnf_preprocessor',
########################################################################
dhparser_parentdir = os.path.dirname(DHPARSER_DIR)
DHPARSER_IMPORTS = '''
import collections
from functools import partial
......@@ -101,7 +98,7 @@ from DHParser import logging, is_filename, load_if_file, \\
error_on, recompile_grammar, left_associative, lean_left, set_config_value, \\
get_config_value, XML_SERIALIZATION, SXPRESSION_SERIALIZATION, COMPACT_SERIALIZATION, \\
JSON_SERIALIZATION, CONFIG_PRESET, GLOBALS
'''.format(dhparser_parentdir=dhparser_parentdir)
'''.format(dhparser_parentdir=DHPARSER_PARENTDIR)
########################################################################
......
......@@ -31,7 +31,7 @@ from typing import Callable, cast, Iterator, Sequence, List, AbstractSet, Set, U
Container, Optional, Dict
from DHParser.configuration import SERIALIZATIONS, XML_SERIALIZATION, SXPRESSION_SERIALIZATION, \
COMPACT_SERIALIZATION, JSON_SERIALIZATION
COMPACT_SERIALIZATION, JSON_SERIALIZATION, SMART_SERIALIZATION
from DHParser.error import Error, ErrorCode, linebreaks, line_col
from DHParser.stringview import StringView
from DHParser.toolkit import get_config_value, re
......@@ -824,7 +824,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
# serialization meta-method ###
def serialize_as(self: 'Node', how: str = 'default') -> str:
def serialize(self: 'Node', how: str = 'default') -> str:
"""
Serializes the tree starting with `node` either as S-expression, XML, JSON,
or in compact form. Possible values for `how` are 'S-expression',
......@@ -849,6 +849,17 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
return self.as_json()
elif switch == COMPACT_SERIALIZATION.lower():
return self.as_sxpr(compact=True)
elif switch == SMART_SERIALIZATION.lower():
threshold = get_config_value('flatten_sxpr_threshold')
if threshold <= 0:
return self.as_sxpr(compact=True)
sxpr = self.as_sxpr(flatten_threshold=threshold)
if sxpr.find('\n') >= 0:
sxpr = re.sub(r'\n(\s*)\(', r'\n\1', sxpr)
sxpr = re.sub(r'\n\s*\)', r'', sxpr)
sxpr = re.sub(r'\)', r'', sxpr)
sxpr = re.sub(r'^\(', r'', sxpr)
return sxpr
else:
raise ValueError('Unknown serialization %s. Allowed values are either: %s or : %s'
% (how, "'ast', 'cst', 'default'", ", ".join(list(SERIALIZATIONS))))
......
#!/usr/bin/python3
"""MLWServer.py - starts a server (if not already running) for the
compilation of the MLW (medieval latin dictionary)
"""DSLServer.py - starts a server (if not already running) for the
compilation of DSL
Author: Eckhart Arnold <arnold@badw.de>
......@@ -24,8 +24,6 @@ import asyncio
import os
import sys
scriptdir = os.path.dirname(os.path.realpath(__file__))
sys.path.extend([os.path.join(scriptdir, 'DHParser-submodule')])
STOP_SERVER_REQUEST = b"__STOP_SERVER__" # hardcoded in order to avoid import from DHParser.server
IDENTIFY_REQUEST = "identify()"
......@@ -94,11 +92,9 @@ def json_rpc(func, params=[], ID=None) -> str:
return str({"jsonrpc": "2.0", "method": func.__name__, "params": params, "id": ID})
def mlw_compiler(dateiname):
from MLWCompiler import verarbeite_mlw_artikel
print("Generiere HTML " + dateiname)
ergebnis = verarbeite_mlw_artikel(dateiname, '', {})
return ergebnis
def DSL_compiler(dateiname):
from DSLCompiler import compile_source
return compile_source(dateiname)
def run_server(host, port):
......@@ -111,8 +107,8 @@ def run_server(host, port):
print('PermissionError: Could not write temporary config file: ' + config_filename)
print('Starting server on %s:%i' % (host, port))
mlw_server = LanguageServer({'mlw_compiler': mlw_compiler})
mlw_server.run_server(host, port)
DSL_server = LanguageServer({'DSL_compiler': DSL_compiler})
DSL_server.run_server(host, port)
async def send_request(request, host, port):
......@@ -151,10 +147,10 @@ def start_server_daemon(host, port):
def print_usage_and_exit():
print('Usages:\n'
+ ' python MLWServer.py --startserver [host] [port]\n'
+ ' python MLWServer.py --stopserver\n'
+ ' python MLWServer.py --status\n'
+ ' python MLWServer.py FILENAME.mlw [--host host] [--port port]')
+ ' python DSLServer.py --startserver [host] [port]\n'
+ ' python DSLServer.py --stopserver\n'
+ ' python DSLServer.py --status\n'
+ ' python DSLServer.py FILENAME.dsl [--host host] [--port port]')
sys.exit(1)
......
......@@ -269,10 +269,10 @@ def get_report(test_unit):
cst = tests.get('__cst__', {}).get(test_name, None)
if cst and (not ast or str(test_name).endswith('*')):
report.append('\n### CST')
report.append(indent(cst.serialize_as('cst')))
report.append(indent(cst.serialize('cst')))
if ast:
report.append('\n### AST')
report.append(indent(ast.serialize_as('ast')))
report.append(indent(ast.serialize('ast')))
for test_name, test_code in tests.get('fail', dict()).items():
heading = 'Fail-test "%s"' % test_name
report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
......@@ -436,7 +436,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
if compare:
if not compare.equals(cst):
errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
(test_name, parser_name, cst.serialize_as('cst')))
(test_name, parser_name, cst.serialize('cst')))
if verbose:
infostr = ' cst-test "' + test_name + '" ... '
write(infostr + ("OK" if len(errata) == errflag else "FAIL"))
......
......@@ -70,6 +70,7 @@ __all__ = ('typing',
'smart_list',
'sane_parser_name',
'DHPARSER_DIR',
'DHPARSER_PARENTDIR',
'GLOBALS',
'get_config_value',
'set_config_value')
......@@ -83,6 +84,7 @@ __all__ = ('typing',
DHPARSER_DIR = os.path.dirname(os.path.abspath(__file__))
DHPARSER_PARENTDIR = os.path.dirname(DHPARSER_DIR.rstrip('/'))
GLOBALS = threading.local()
......
......@@ -16,4 +16,4 @@
# permissions and limitations under the License.
__all__ = ('__version__',)
__version__ = '0.8.9' # + '_dev' + str(os.stat(__file__).st_mtime)
__version__ = '0.9.0' # + '_dev' + str(os.stat(__file__).st_mtime)
DHParser
========
DHParser - The domian specific language (DSL) construction kit for the Digit Humanities
DHParser - A parser generator and domain specific language (DSL) construction
kit for the Digit Humanities
Features
......@@ -22,6 +23,55 @@ Features
message for users that do not habitually deal with formal notations!
Ease of use
-----------
key_value_store.py:
# A mini-DSL for a key value store
from DHParser import *
# specify the grammar of your DSL in EBNF-notation
grammar = '''@ drop = whitespace, token
key_store = ~ { entry }
entry = key "=" value
key = /\w+/~ # Scannerless parsing, use regular
value = /\"[^"\n]*\"/~ # expressions wherever you like'''
# generating a parser is almost as simple as compiling a regular expression
parser_factory = grammar_provider(grammar)
parser = parser_factory() # parser factory for thread-safety
Now, parse some text and extract the data from the Python-shell:
>>> from key_value_store import parser
>>> text = '''
title = "Odysee 2001"
director = "Stanley Kubrick"
'''
>>> data = parser(text)
>>> for entry in data.select('entry'):
print(entry['key'], entry['value'])
title "Odysee 2001"
director "Stanley Kubrick"
Or, serialize as XML:
>>> print(data.as_xml())
<key_store>
<entry>
<key>title</key>
<value>"Odysee 2001"</value>
</entry>
<entry>
<key>director</key>
<value>"Stanley Kubrick"</value>
</entry>
</key_store>
License
-------
......
# A mini-DSL for a key value store
from DHParser import *
# specify the grammar of your DSL in EBNF-notation
grammar = '''
@ drop = whitespace, token
key_store = ~ { entry }
entry = key "=" value
key = /\w+/~ # Scannerless parsing, use regular
value = /\"[^"\n]*\"/~ # expressions wherever you like'''
parser = grammar_provider(grammar)()
if __name__ == '__main__':
text = '''
title = "Odysee 2001"
director = "Stanley Kubrick"
'''
data = parser(text)
for entry in data.select('entry'):
print(entry['key'], entry['value'])
......@@ -31,9 +31,9 @@ setup(
license='[Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0)',
author='Eckhart Arnold',
author_email='arnold@badw.de',
description='DHParser - Domain specific languages for the Digital Humanities',
description='DHParser - Parser generator and DSL-construction-kit',
long_description = read_me,
keywords='Digital Humanities, domain specific languages, parser combinators, EBNF',
keywords='parser generator, domain specific languages, Digital Humanities, parser combinators, EBNF',
classifiers = [
'Development Status :: 4 - Beta',
'Intended Audience :: Developers',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment