Notice: If you are member of any public project or group, please make sure that your GitLab username is not the same as the LRZ identifier/Kennung (see https://gitlab.lrz.de/profile/account). Please change your username if necessary. For more information see the section "Public projects / Öffentliche Projekte" at https://doku.lrz.de/display/PUBLIC/GitLab . Thank you!

Commit 6438ee95 authored by eckhart's avatar eckhart

- server: better server termination

parent 03404315
......@@ -33,7 +33,22 @@ from .syntaxtree import *
from .testing import *
from .toolkit import *
from .transform import *
from .versionnumber import __version__
from .versionnumber import *
__all__ = (compile.__all__ +
configuration.__all__ +
dsl.__all__ +
ebnf.__all__ +
error.__all__ +
log.__all__ +
parse.__all__ +
preprocess.__all__ +
stringview.__all__ +
syntaxtree.__all__ +
testing.__all__ +
toolkit.__all__ +
transform.__all__ +
versionnumber.__all__)
name = "DHParser"
__author__ = "Eckhart Arnold <arnold@badw.de>"
......
......@@ -35,16 +35,15 @@ compiler object.
"""
import copy
import re
from typing import Any, Optional, Tuple, List
from DHParser.preprocess import strip_tokens, with_source_mapping, PreprocessorFunc
from DHParser.preprocess import with_source_mapping, PreprocessorFunc, SourceMapFunc
from DHParser.syntaxtree import Node, RootNode, ZOMBIE_TAG, StrictResultType
from DHParser.transform import TransformationFunc
from DHParser.parse import Grammar
from DHParser.error import adjust_error_locations, is_error, Error
from DHParser.log import log_parsing_history, log_ST, is_logging, logfile_basename
from DHParser.toolkit import typing, sane_parser_name, load_if_file
from typing import Any, Optional, Tuple, List, Callable
from DHParser.toolkit import load_if_file
__all__ = ('CompilerError', 'Compiler', 'compile_source', 'visitor_name')
......@@ -114,7 +113,7 @@ class Compiler:
self.context = [] # type: List[Node]
self._dirty_flag = False
def __call__(self, root: RootNode, source: str='') -> Any:
def __call__(self, root: RootNode, source: str = '') -> Any:
"""
Compiles the abstract syntax tree with the root node `node` and
returns the compiled code. It is up to subclasses implementing
......@@ -126,7 +125,7 @@ class Compiler:
if self._dirty_flag:
self._reset()
self._dirty_flag = True
self.tree = root # type: Node
self.tree = root # type: RootNode
self.source = source # type: str
result = self.compile(root)
return result
......@@ -216,12 +215,12 @@ def compile_source(source: str,
3. The root-node of the abstract syntax tree if `preserve_ast` is True
or `None` otherwise.
"""
ast = None
original_text = load_if_file(source)
log_file_name = logfile_basename(source, compiler)
ast = None # type: Optional[Node]
original_text = load_if_file(source) # type: str
log_file_name = logfile_basename(source, compiler) # type: str
if preprocessor is None:
source_text = original_text
source_mapping = lambda i: i
source_text = original_text # type: str
source_mapping = lambda i: i # type: SourceMapFunc
else:
source_text, source_mapping = with_source_mapping(preprocessor(original_text))
syntax_tree = parser(source_text) # type: RootNode
......@@ -251,7 +250,7 @@ def compile_source(source: str,
# messages.extend(syntax_tree.errors())
# syntax_tree.error_flag = max(syntax_tree.error_flag, efl)
messages = syntax_tree.errors_sorted
messages = syntax_tree.errors_sorted # type: List[Error]
adjust_error_locations(messages, original_text, source_mapping)
return result, messages, ast
......
......@@ -45,7 +45,6 @@ from DHParser.transform import TransformationFunc, traverse, remove_brackets, \
from DHParser.versionnumber import __version__
__all__ = ('get_ebnf_preprocessor',
'get_ebnf_grammar',
'get_ebnf_transformer',
......@@ -89,7 +88,7 @@ from DHParser import logging, is_filename, load_if_file, \\
Lookbehind, Lookahead, Alternative, Pop, Token, DropToken, Synonym, AllOf, SomeOf, \\
Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \\
ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \\
grammar_changed, last_value, counterpart, accumulate, PreprocessorFunc, is_empty, \\
grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, \\
Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \\
remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \\
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
......@@ -111,6 +110,11 @@ from DHParser import logging, is_filename, load_if_file, \\
def get_ebnf_preprocessor() -> PreprocessorFunc:
"""
Returns the preprocessor function for the EBNF compiler.
As of now, no preprocessing is needed for EBNF-sources. Therefore,
just a dummy function is returned.
"""
return nil_preprocessor
......
......@@ -238,8 +238,8 @@ def line_col(lbreaks: List[int], pos: int) -> Tuple[int, int]:
def adjust_error_locations(errors: List[Error],
original_text: Union[StringView, str],
source_mapping: SourceMapFunc = lambda i: i) -> List[Error]:
"""Adds (or adjusts) line and column numbers of error messages in place.
source_mapping: SourceMapFunc = lambda i: i):
"""Adds (or adjusts) line and column numbers of error messages inplace.
Args:
errors: The list of errors as returned by the method
......@@ -248,15 +248,9 @@ def adjust_error_locations(errors: List[Error],
(Needed in order to determine the line and column numbers.)
source_mapping: A function that maps error positions to their
positions in the original source file.
Returns:
The list of errors. (Returning the list of errors is just syntactical
sugar. Be aware that the line, col and orig_pos attr have been
changed in place.)
"""
line_breaks = linebreaks(original_text)
for err in errors:
assert err.pos >= 0
err.orig_pos = source_mapping(err.pos)
err.line, err.column = line_col(line_breaks, err.orig_pos)
return errors
......@@ -32,6 +32,7 @@ for an example.
from collections import defaultdict
import copy
from typing import Callable, cast, List, Tuple, Set, Dict, DefaultDict, Union, Optional, Any
from DHParser.error import Error, linebreaks, line_col
from DHParser.log import is_logging, HistoryRecord
......@@ -40,9 +41,7 @@ from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, FrozenNode, RootNode, WHITESPACE_PTYPE, \
TOKEN_PTYPE, ZOMBIE_TAG, ResultType
from DHParser.toolkit import sane_parser_name, escape_control_characters, get_config_value, \
re, typing, cython
from DHParser.configuration import CONFIG_PRESET
from typing import Callable, cast, List, Tuple, Set, Dict, DefaultDict, Union, Optional, Any
re, cython
__all__ = ('Parser',
......@@ -1009,12 +1008,7 @@ class Grammar:
if stitches:
if rest:
stitches.append(Node(ZOMBIE_TAG, rest))
#try:
result = Node(ZOMBIE_TAG, tuple(stitches)).with_pos(0)
# except AssertionError as error:
# # some debugging output
# print(Node(ZOMBIE_TAG, tuple(stitches)).as_sxpr())
# raise error
if any(self.variables__.values()):
error_msg = "Capture-retrieve-stack not empty after end of parsing: " \
+ str(self.variables__)
......@@ -1029,8 +1023,6 @@ class Grammar:
result.result = result.children + (error_node,)
else:
self.tree__.new_error(result, error_msg, error_code)
# result.pos = 0 # calculate all positions
# result.errors(self.document__)
if result:
self.tree__.swallow(result)
self.start_parser__ = None
......@@ -1338,7 +1330,7 @@ class DropWhitespace(Whitespace):
assert not self.pname, "DropWhitespace must not be used for named parsers!"
match = text.match(self.regexp)
if match:
capture = match.group(0)
# capture = match.group(0)
end = text.index(match.end())
return EMPTY_NODE, text[end:]
return None, text
......@@ -1673,7 +1665,7 @@ def mandatory_violation(grammar: Grammar,
else:
msg = '%s expected, "%s" found!' % (expected, found)
error = Error(msg, location, Error.MANDATORY_CONTINUATION_AT_EOF
if (failed_on_lookahead and not text_) else Error.MANDATORY_CONTINUATION)
if (failed_on_lookahead and not text_) else Error.MANDATORY_CONTINUATION)
grammar.tree__.add_error(err_node, error)
return error, err_node, text_[i:]
......@@ -2214,15 +2206,21 @@ RetrieveFilter = Callable[[List[str]], str]
def last_value(stack: List[str]) -> str:
"""Returns the last value on the cpature stack. This is the default case
when retrieving cpatured substrings."""
return stack[-1]
def counterpart(stack: List[str]) -> str:
"""Returns a closing bracket for the opening bracket on the capture stack,
i.e. if "[" was captured, "]" will be retrieved."""
value = stack[-1]
return value.replace("(", ")").replace("[", "]").replace("{", "}").replace("<", ">")
def accumulate(stack: List[str]) -> str:
"""Returns an accumulation of all values on the stack.
By the way: I cannot remember any reasonable use case for this!?"""
return "".join(stack) if len(stack) > 1 else stack[-1] # provoke IndexError if stack empty
......@@ -2240,7 +2238,7 @@ class Retrieve(Parser):
symbol: The parser that has stored the value to be retrieved, in
other words: "the observed parser"
rfilter: a procedure that through which the processing to the
retrieved symbols is channeld. In the simplemost case it merely
retrieved symbols is channeled. In the simplest case it merely
returns the last string stored by the observed parser. This can
be (mis-)used to execute any kind of semantic action.
"""
......
......@@ -31,10 +31,9 @@ cannot completely be described with context-free grammars.
import bisect
import collections
import functools
from DHParser.toolkit import re, typing
from typing import Union, Callable, Tuple, List
from DHParser.toolkit import re
__all__ = ('RX_TOKEN_NAME',
'BEGIN_TOKEN',
......
......@@ -45,9 +45,10 @@ For JSON see:
import asyncio
import json
from multiprocessing import Process, Value, Queue
from typing import Callable, Optional, Union, Dict, List, Sequence, cast
import sys
from typing import Callable, Optional, Union, Dict, List, Tuple, NamedTuple, Sequence, cast
from DHParser.toolkit import get_config_value, re
from DHParser.toolkit import get_config_value, is_filename, load_if_file, re
__all__ = ('RPC_Table',
'RPC_Type',
......@@ -73,7 +74,7 @@ SERVER_STARTING = 1
SERVER_ONLINE = 2
SERVER_TERMINATE = 3
response_test = b'''HTTP/1.1 200 OK
test_response_test = b'''HTTP/1.1 200 OK
Date: Sun, 18 Oct 2009 08:56:53 GMT
Server: Apache/2.2.14 (Win32)
Last-Modified: Sat, 20 Nov 2004 07:16:26 GMT
......@@ -93,6 +94,26 @@ X-Pad: avoid browser bug
</html>
'''
test_get = b'''GET /method/object HTTP/1.1
Host: 127.0.0.1:8888
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:66.0) Gecko/20100101 Firefox/66.0
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language: de,en-US;q=0.7,en;q=0.3
Accept-Encoding: gzip, deflate
DNT: 1
Connection: keep-alive
Upgrade-Insecure-Requests: 1
'''
# CompilationItem = NamedTuple('CompilationItem',
# [('uri', str),
# ('hash', int),
# ('errors', str),
# ('preview', str)])
class Server:
def __init__(self, rpc_functions: RPC_Type):
......@@ -113,9 +134,11 @@ class Server:
self.server_messages = Queue() # type: Queue
self.server_process = None # type: Optional[Process]
async def handle_compilation_request(self,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
# self.registry = {} # type: Dict[str, ]
async def handle_request(self,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
rpc_error = None # type: Optional[Tuple[int, str]]
json_id = 'null' # type: Tuple[int, str]
obj = {} # type: Dict
......@@ -132,7 +155,7 @@ class Server:
try:
raw = json.loads(data)
except json.decoder.JSONDecodeError as e:
rpc_error = -32700, "JSONDecodeError: " + str(e)
rpc_error = -32700, "JSONDecodeError: " + str(e) + str(data)
if rpc_error is None:
if isinstance(raw, Dict):
......@@ -169,12 +192,12 @@ class Server:
await writer.drain()
writer.close()
# TODO: add these lines in case a terminate signal is received, i.e. exit server coroutine
# gracefully. Is this needed?
# gracefully. Is this needed? Does it work?
# self.server.cancel()
async def serve(self, address: str = '127.0.0.1', port: int = 8888):
self.server = cast(asyncio.base_events.Server,
await asyncio.start_server(self.handle_compilation_request, address, port))
await asyncio.start_server(self.handle_request, address, port))
async with self.server:
self.stage.value = SERVER_ONLINE
self.server_messages.put(SERVER_ONLINE)
......@@ -183,14 +206,16 @@ class Server:
def run_server(self, address: str = '127.0.0.1', port: int = 8888):
self.stage.value = SERVER_STARTING
# loop = asyncio.get_event_loop()
# try:
# loop.run_until_complete(self.serve(address, port))
# finally:
# print(type(self.server))
# # self.server.cancel()
# loop.close()
asyncio.run(self.serve(address, port))
if sys.version_info >= (3, 7):
asyncio.run(self.serve(address, port))
else:
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(self.serve(address, port))
finally:
# self.server.cancel()
loop.close()
def wait_until_server_online(self):
if self.stage.value != SERVER_ONLINE:
......@@ -204,12 +229,16 @@ class Server:
self.wait_until_server_online()
def terminate_server_process(self):
self.server_process.terminate()
if self.server_process:
self.stage.value = SERVER_TERMINATE
self.server_process.terminate()
self.server_process = None
self.stage.value = SERVER_OFFLINE
def wait_for_termination_request(self):
assert self.server_process
# self.wait_until_server_online()
while self.server_messages.get() != SERVER_TERMINATE:
pass
self.terminate_server_process()
self.server_process = None
if self.server_process:
if self.stage.value in (SERVER_STARTING, SERVER_ONLINE):
while self.server_messages.get() != SERVER_TERMINATE:
pass
if self.stage.value == SERVER_TERMINATE:
self.terminate_server_process()
......@@ -32,7 +32,6 @@ speedup. The modules comes with a ``stringview.pxd`` that contains some type
declarations to fully exploit the potential of the Cython-compiler.
"""
import collections
from typing import Optional, Union, Iterable, Tuple
try:
......
......@@ -114,6 +114,7 @@ def flatten_xml(xml: str) -> str:
assert RX_IS_XML.match(xml)
def tag_only(m):
"""Return only the tag, drop the whitespace."""
return m.groupdict()['closing_tag']
return re.sub(r'\s+(?=<[\w:])', '', re.sub(r'(?P<closing_tag></:?\w+>)\s+', tag_only, xml))
......@@ -321,6 +322,14 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def is_anonymous(self) -> bool:
"""Returns True, if the Node is an "anonymous" Node, i.e. a node that
has not been created by a named parser.
The tag name of anonymous node is a colon followed by the class name
of the parser that created the node, i.e. ":Series". It is recommended
practice to remove (or name) all anonymous nodes during the
AST-transformation.
"""
return not self.tag_name or self.tag_name[0] == ':'
......@@ -679,9 +688,9 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def as_xml(self, src: str = None,
indentation: int = 2,
inline_tags: Set[str] = set(),
omit_tags: Set[str] = set(),
empty_tags: Set[str] = set()) -> str:
inline_tags: Set[str] = frozenset(),
omit_tags: Set[str] = frozenset(),
empty_tags: Set[str] = frozenset()) -> str:
"""
Returns content as XML-tree.
......@@ -761,7 +770,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
data.append(self._pos)
if has_attr:
data.append(dict(self._xml_attr))
return { '__class__': 'DHParser.Node', 'data': data }
return {'__class__': 'DHParser.Node', 'data': data}
@staticmethod
......@@ -1153,7 +1162,7 @@ def parse_sxpr(sxpr: Union[str, StringView]) -> Node:
RX_WHITESPACE_TAIL = re.compile(r'\s*$')
def parse_xml(xml: Union[str, StringView], ignore_pos: bool=False) -> Node:
def parse_xml(xml: Union[str, StringView], ignore_pos: bool = False) -> Node:
"""
Generates a tree of nodes from a (Pseudo-)XML-source.
......@@ -1274,13 +1283,13 @@ def parse_tree(xml_sxpr_json: str) -> Optional[Node]:
return parse_xml(xml_sxpr_json)
elif RX_IS_SXPR.match(xml_sxpr_json):
return parse_sxpr(xml_sxpr_json)
elif re.match('\s*', xml_sxpr_json):
elif re.match(r'\s*', xml_sxpr_json):
return None
else:
try:
return parse_json_syntaxtree(xml_sxpr_json)
except json.decoder.JSONDecodeError:
m = re.match('\s*(.*)\n?', xml_sxpr_json)
m = re.match(r'\s*(.*)\n?', xml_sxpr_json)
snippet = m.group(1) if m else ''
raise ValueError('Snippet seems to be neither S-expression nor XML: ' + snippet + ' ...')
......
......@@ -405,7 +405,8 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
# log_ST(cst, "match_%s_%s.cst" % (parser_name, clean_test_name))
tests.setdefault('__cst__', {})[test_name] = cst
if is_error(cst.error_flag) and not lookahead_artifact(cst):
errors = adjust_error_locations(cst.errors_sorted, test_code)
errors = cst.errors_sorted
adjust_error_locations(errors, test_code)
errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.: %s\n\n\t%s\n\n' %
(test_name, parser_name, '\n\t'.join(test_code.split('\n')),
'\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
......
......@@ -6,4 +6,4 @@ universal = 0
[pep8]
max-line-length = 100
ignore = E303
ignore = E303, E731
......@@ -52,6 +52,7 @@ class TestServer:
asyncio.run(compile('Test', ''))
cs.terminate_server_process()
cs.wait_for_termination_request()
if __name__ == "__main__":
from DHParser.testing import runner
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment