Commit 63a54b5d authored by eckhart's avatar eckhart

Merge branch 'development' of gitlab.lrz.de:badw-it/DHParser into development

parents 99e401cc d3009b08
......@@ -28,7 +28,6 @@ DevScripts/DHParser.py
DHParser/cstringview.c
*.so
.mypy_cache
.vscode/
DHParser.egg-info
.noseids/
*.build
......@@ -42,7 +41,6 @@ _build
_build
_static
_templates
.vs
OLDSTUFF
.pytest_cache
*.c
......
......@@ -102,7 +102,8 @@ class Compiler:
context: A list of parent nodes that ends with the currently
compiled node.
tree: The root of the abstract syntax tree.
source: The source code.
finalizers: A stack of tuples (function, parameters) that will be
called in reverse order after compilation.
_dirty_flag: A flag indicating that the compiler has already been
called at least once and that therefore all compilation
......@@ -120,12 +121,28 @@ class Compiler:
def reset(self):
# self.source = ''
self.finlizers = [] # type: List[Callable, Tuple]
self.tree = ROOTNODE_PLACEHOLDER # type: RootNode
self.context = [] # type: List[Node]
self._None_check = True # type: bool
self._dirty_flag = False
self._debug = get_config_value('debug') # type: bool
self._debug_already_compiled = set() # type: Set[Node]
self.finalizers = [] # type: List[Callable, Tuple]
def prepare(self) -> None:
"""
A preparation method that will be called after everything else has
been initialized and immediately before compilation starts. This method
can be overwritten in order to implement preparation tasks.
"""
pass
def finalize(self) -> None:
"""
A finalization method that is called after compilation has finished and
after all tasks from the finalizers stack have been executed
"""
def __call__(self, root: RootNode) -> Any:
"""
......@@ -141,7 +158,11 @@ class Compiler:
self._dirty_flag = True
self.tree = root # type: RootNode
# self.source = source # type: str
self.prepare()
result = self.compile(root)
while self.finalizers:
task, parameters = self.finalizers.pop()
task(*parameters)
return result
# Obsolete, because never used...
......
......@@ -86,6 +86,7 @@ class Error:
PARSER_DID_NOT_MATCH = ErrorCode(1020)
PARSER_LOOKAHEAD_MATCH_ONLY = ErrorCode(1030)
PARSER_STOPPED_BEFORE_END = ErrorCode(1040)
PARSER_STOPPED_EXCEPT_FOR_LOOKAHEAD = ErrorCode(1045)
CAPTURE_STACK_NOT_EMPTY = ErrorCode(1050)
MALFORMED_ERROR_STRING = ErrorCode(1060)
AMBIGUOUS_ERROR_HANDLING = ErrorCode(1070)
......
......@@ -974,7 +974,9 @@ class Grammar:
result, 'Parser "%s" did not match empty document.' % str(parser),
Error.PARSER_DID_NOT_MATCH)
while rest and len(stitches) < self.max_parser_dropouts__:
# copy to local variable, so break condition can be triggered manually
max_parser_dropouts = self.max_parser_dropouts__
while rest and len(stitches) < max_parser_dropouts:
result, rest = parser(rest)
if rest:
fwd = rest.find("\n") + 1 or len(rest)
......@@ -994,13 +996,21 @@ class Grammar:
error_code = Error.PARSER_DID_NOT_MATCH
else:
stitches.append(result)
error_msg = "Parser stopped before end" \
+ (("! trying to recover"
+ (" but stopping history recording at this point."
if self.history_tracking__ else "..."))
if len(stitches) < self.max_parser_dropouts__
else " too often! Terminating parser.")
error_code = Error.PARSER_STOPPED_BEFORE_END
h = self.history__[-1] if self.history__ else \
HistoryRecord([], None, StringView(''), (0, 0))
if h.status == h.MATCH and (h.node.pos + len(h.node) == len(self.document__)):
# TODO: this case still needs unit-tests and support in testing.py
error_msg = "Parser stopped before end, but matched with lookahead."
error_code = Error.PARSER_STOPPED_EXCEPT_FOR_LOOKAHEAD
max_parser_dropouts = -1 # no further retries!
else:
error_msg = "Parser stopped before end" \
+ (("! trying to recover"
+ (" but stopping history recording at this point."
if self.history_tracking__ else "..."))
if len(stitches) < self.max_parser_dropouts__
else " too often! Terminating parser.")
error_code = Error.PARSER_STOPPED_BEFORE_END
stitches.append(Node(ZOMBIE_TAG, skip).with_pos(tail_pos(stitches)))
self.tree__.new_error(stitches[-1], error_msg, error_code)
if self.history_tracking__:
......
......@@ -44,12 +44,14 @@ For JSON see:
# TODO: Test with python 3.5
import asyncio
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, CancelledError
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, CancelledError, \
BrokenExecutor
import json
from multiprocessing import Process, Queue, Value, Array
import sys
import time
from typing import Callable, Coroutine, Optional, Union, Dict, List, Tuple, Sequence, Set, Any, cast
from typing import Callable, Coroutine, Optional, Union, Dict, List, Tuple, Sequence, Set, Any, \
cast
from DHParser.syntaxtree import DHParser_JSONEncoder
from DHParser.toolkit import get_config_value, re
......@@ -255,8 +257,10 @@ class Server:
rpc_error = -32602, "Invalid Params: " + str(e)
except NameError as e:
rpc_error = -32601, "Method not found: " + str(e)
except BrokenExecutor as e:
rpc_error = -32000, "Broken Executor: " + str(e)
except Exception as e:
rpc_error = -32000, "Server Error: " + str(e)
rpc_error = -32000, "Server Error " + str(type(e)) + ': ' + str(e)
if data.startswith(b'GET'):
# HTTP request
......
This diff is collapsed.
......@@ -342,7 +342,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
parser[parser_name].apply(find_lookahead)
return lookahead_found
def lookahead_artifact(st):
def lookahead_artifact(syntax_tree: Node):
"""
Returns True, if the error merely occurred, because the parser
stopped in front of a sequence that was captured by a lookahead
......@@ -350,19 +350,22 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
This is required for testing of parsers that put a lookahead
operator at the end. See test_testing.TestLookahead.
"""
raw_errors = st.errors_sorted
is_artifact = ((len(raw_errors) == 2 # case 1: superfluous data for lookahead
and raw_errors[-1].code == Error.PARSER_LOOKAHEAD_MATCH_ONLY
and raw_errors[-2].code == Error.PARSER_STOPPED_BEFORE_END)
# case 2: mandatory lookahead failure at end of text
raw_errors = syntax_tree.errors_sorted
is_artifact = ((2 <= len(raw_errors) == 3 # case 1: superfluous data for lookahead
and {e.code for e in raw_errors} <=
{Error.PARSER_LOOKAHEAD_MATCH_ONLY,
# Error.PARSER_STOPPED_BEFORE_END,
Error.PARSER_STOPPED_EXCEPT_FOR_LOOKAHEAD})
or (len(raw_errors) == 1
and raw_errors[-1].code == Error.MANDATORY_CONTINUATION_AT_EOF))
and (raw_errors[-1].code == Error.PARSER_STOPPED_EXCEPT_FOR_LOOKAHEAD
# case 2: mandatory lookahead failure at end of text
or raw_errors[-1].code == Error.MANDATORY_CONTINUATION_AT_EOF)))
if is_artifact:
# don't remove zombie node with error message at the end
# but change it's tag_name to indicate that it is an artifact!
for parent in st.select_if(lambda node: any(child.tag_name == ZOMBIE_TAG
for child in node.children),
include_root=True, reverse=True):
for parent in syntax_tree.select_if(lambda node: any(child.tag_name == ZOMBIE_TAG
for child in node.children),
include_root=True, reverse=True):
zombie = parent[ZOMBIE_TAG]
zombie.tag_name = '__TESTING_ARTIFACT__'
zombie.result = 'Artifact can be ignored. Be aware, though, that also the' \
......@@ -472,7 +475,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
cst = RootNode(node).new_error(node, str(upe))
errata.append('Unknown parser "{}" in fail test "{}"!'.format(parser_name, test_name))
tests.setdefault('__err__', {})[test_name] = errata[-1]
if not is_error(cst.error_flag) and not lookahead_artifact(cst):
if not (is_error(cst.error_flag) and not lookahead_artifact(cst)):
errata.append('Fail test "%s" for parser "%s" yields match instead of '
'expected failure!' % (test_name, parser_name))
tests.setdefault('__err__', {})[test_name] = errata[-1]
......@@ -850,6 +853,7 @@ def run_path(path):
def clean_report():
"""Deletes any test-report-files in the REPORT sub-directory and removes
the REPORT sub-directory, if it is empty after deleting the files."""
# TODO: make this thread safe, if possible!!!!
if os.path.exists('REPORT'):
files = os.listdir('REPORT')
flag = False
......
#!/usr/bin/python3
"""Lädt das MLW-Installationsskript vonb gitlab herunter und startet es.
Nach dem Starten wird das Installationsskript wieder gelöscht, d.h. um
es wieder zu starten, muss erst dieses Skript wieder ausgeführt werden,
welches das jeweils aktuelle Installationsskript herunterlädt.
"""
import os
import ssl
import sys
import urllib.request
cwd = os.path.abspath(os.getcwd())
if cwd.endswith('MLW-DSL'):
print('Der Pfad "%s" scheint das MLW-Git-Repositorium zu enthalten.' % cwd)
print('Dieses Skript sollte nicht innerhalb des MLW-Git-Repositoriums ausgeführt werden!')
sys.exit(1)
PFAD = "https://gitlab.lrz.de/badw-it/MLW-DSL/raw/master/Installiere-MLW.py"
SKRIPT_NAME = os.path.basename(PFAD)
# Lade Installationsskript herunter
with urllib.request.urlopen(PFAD, context=ssl.create_default_context()) as https:
skript = https.read()
# Speichere Installationsskript im aktuellen Ordner (Desktop)
with open(SKRIPT_NAME, 'wb') as datei:
datei.write(skript)
# Führe Installationsskript aus
os.system('python.exe ' + SKRIPT_NAME)
# Lösche Installationsskript nach der Ausführung, damit nicht versehentlich
# ein veraltetes Installationsskript gestartet wird.
os.remove(SKRIPT_NAME)
This diff is collapsed.
......@@ -41,7 +41,7 @@ class TestDHParserCommandLineTool:
if not os.path.exists('testdata'):
os.mkdir('testdata')
self.nulldevice = " >/dev/null" if platform.system() != "Windows" else " > NUL"
self.python = 'python3 ' if os.system('python3 -V' + self.nulldevice) == 0 else 'python '
self.python = 'python3 ' if system('python3 -V' + self.nulldevice) == 0 else 'python '
def teardown(self):
if os.path.exists('testdata/neu/neuServer.py'):
......
......@@ -257,10 +257,11 @@ class TestLookahead:
"match": {
1: """Mountains: big:
K2""", # case 1: matches only with lookahead (but should not fail in a test)
2: """Rivers:""" # case 2: lookahaead failure occurs at end of file and is mandatory. (should not fail as a test)
2: """Rivers:""", # case 2: lookahaead failure occurs at end of file and is mandatory. (should not fail as a test)
3: """Mountains: big:""" # same here
},
"fail": {
6: """Mountains: big:"""
6: """Mountains: big: """
}
}
}
......@@ -311,11 +312,15 @@ class TestLookahead:
gr = self.grammar_fac()
# Case 1: Lookahead string is part of the test case; parser fails but for the lookahead
result = gr(self.cases['category']['match'][1], 'category', True)
assert any(e.code == Error.PARSER_LOOKAHEAD_MATCH_ONLY for e in result.errors)
assert any(e.code in (Error.PARSER_LOOKAHEAD_MATCH_ONLY,
Error.PARSER_STOPPED_EXCEPT_FOR_LOOKAHEAD)
for e in result.errors)
# Case 2: Lookahead string is not part of the test case; parser matches but for the mandatory continuation
result = gr(self.cases['category']['match'][2], 'category', True)
assert any(e.code == Error.MANDATORY_CONTINUATION_AT_EOF for e in result.errors)
errata = grammar_unit(self.cases, self.grammar_fac, self.trans_fac)
for e in errata:
print (e)
assert not errata, str(errata)
errata = grammar_unit(self.fail_cases, self.grammar_fac, self.trans_fac)
assert errata
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment