Skip to content

GitLab

D
DHParser

In January 2021 we will introduce a 10 GB quota for project repositories. Higher limits for individual projects will be available on request. Please see https://doku.lrz.de/display/PUBLIC/GitLab for more information.

Commit 9b9293f1 authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files
Options

- testing.py: LOGS now written in testsuite directory; History-Logs only writen for tests that fail

parent d8beaadd
Hide whitespace changes
Inline Side-by-side
Showing with 46 additions and 39 deletions
DHParser/parser.py
View file @ 9b9293f1
......@@ -722,20 +722,23 @@ class Grammar:
elif os.path.exists(path):
os.remove(path)
if not log_file_name:
name = self.__class__.__name__
log_file_name = name[:-7] if name.lower().endswith('grammar') else name
full_history, match_history, errors_only = [], [], []
for record in self.history__:
line = "; ".join(prepare_line(record))
full_history.append(line)
if record.node and record.node.parser.ptype != WHITESPACE_PTYPE:
match_history.append(line)
if record.node.error_flag:
errors_only.append(line)
write_log(full_history, log_file_name + '_full')
write_log(match_history, log_file_name + '_match')
write_log(errors_only, log_file_name + '_errors')
if is_logging():
assert self.history__, \
"Parser did not yet run or logging was turned off when running parser!"
if not log_file_name:
name = self.__class__.__name__
log_file_name = name[:-7] if name.lower().endswith('grammar') else name
full_history, match_history, errors_only = [], [], []
for record in self.history__:
line = "; ".join(prepare_line(record))
full_history.append(line)
if record.node and record.node.parser.ptype != WHITESPACE_PTYPE:
match_history.append(line)
if record.node.error_flag:
errors_only.append(line)
write_log(full_history, log_file_name + '_full')
write_log(match_history, log_file_name + '_match')
write_log(errors_only, log_file_name + '_errors')
def dsl_error_msg(parser: Parser, error_str: str) -> str:
......
DHParser/syntaxtree.py
View file @ 9b9293f1
......@@ -31,7 +31,7 @@ except ImportError:
from .typing34 import AbstractSet, Any, ByteString, Callable, cast, Container, Dict, \
Iterator, List, NamedTuple, Sequence, Union, Text, Tuple
from DHParser.toolkit import log_dir, line_col, identity
from DHParser.toolkit import is_logging, log_dir, line_col, identity
__all__ = ('WHITESPACE_PTYPE',
'TOKEN_PTYPE',
......@@ -472,9 +472,10 @@ class Node:
# return nav(path.split('/'))
def log(self, log_file_name):
st_file_name = log_file_name
with open(os.path.join(log_dir(), st_file_name), "w", encoding="utf-8") as f:
f.write(self.as_sxpr())
if is_logging():
st_file_name = log_file_name
with open(os.path.join(log_dir(), st_file_name), "w", encoding="utf-8") as f:
f.write(self.as_sxpr())
def mock_syntax_tree(sxpr):
......
DHParser/testing.py
View file @ 9b9293f1
......@@ -146,22 +146,21 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
infostr = ' match-test "' + test_name + '" ... '
errflag = len(errata)
cst = parser(test_code, parser_name)
if is_logging():
cst.log("match_%s_%s.cst" % (parser_name, test_name))
parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
cst.log("match_%s_%s.cst" % (parser_name, test_name))
tests.setdefault('__cst__', {})[test_name] = cst
if "ast" in tests or report:
ast = copy.deepcopy(cst)
transform(ast)
tests.setdefault('__ast__', {})[test_name] = ast
if is_logging():
ast.log("match_%s_%s.ast" % (parser_name, test_name))
ast.log("match_%s_%s.ast" % (parser_name, test_name))
if cst.error_flag:
errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.: %s\n\n\t%s' %
(test_name, parser_name, '\n\t'.join(test_code.split('\n')),
'\n\t'.join(m.replace('\n', '\n\t\t') for m in
error_messages(test_code, cst.collect_errors()))))
tests.setdefault('__err__', {})[test_name] = errata[-1]
# write parsing-history log only in case of failure!
parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
(test_name, parser_name, cst.as_sxpr()))
......@@ -184,19 +183,20 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
infostr = ' fail-test "' + test_name + '" ... '
errflag = len(errata)
cst = parser(test_code, parser_name)
if is_logging():
cst.log("fail_%s_%s.cst" % (parser_name, test_name))
parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
# doesn't make sense to write cst for fail-tests
# cst.log("fail_%s_%s.cst" % (parser_name, test_name))
if not cst.error_flag:
errata.append('Fail test "%s" for parser "%s" yields match instead of '
'expected failure!' % (test_name, parser_name))
tests.setdefault('__err__', {})[test_name] = errata[-1]
# write parsing-history log only in case of test-failure
parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
if verbose:
print(infostr + "OK" if len(errata) == errflag else "FAIL")
# write test-report
if report:
report_dir = os.path.join(unit_dir, "REPORT")
report_dir = "REPORT"
if not os.path.exists(report_dir):
os.mkdir(report_dir)
with open(os.path.join(report_dir, unit_name + '.report'), 'w') as f:
......@@ -214,18 +214,21 @@ def grammar_suite(directory, parser_factory, transformer_factory, ignore_unknown
all_errors = collections.OrderedDict()
if verbose:
print("\nScanning test-directory: " + directory)
for filename in sorted(os.listdir(directory)):
save_cwd = os.getcwd()
os.chdir(directory)
for filename in sorted(os.listdir()):
if filename.lower().find("test") >= 0:
try:
if verbose:
print("\nRunning grammar tests from: " + filename)
errata = grammar_unit(os.path.join(directory, filename),
parser_factory, transformer_factory, report, verbose)
errata = grammar_unit(filename, parser_factory,
transformer_factory, report, verbose)
if errata:
all_errors[filename] = errata
except ValueError as e:
if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
raise e
os.chdir(save_cwd)
error_report = []
if all_errors:
for filename in all_errors:
......
examples/LaTeX/LaTeX.ebnf
View file @ 9b9293f1
......@@ -57,8 +57,8 @@ block_environment = known_environment | generic_block
known_environment = itemize | enumerate | figure | table | quotation
| verbatim
generic_block = begin_generic_block sequence §end_generic_block
begin_generic_block = -&LB begin_environment -&LB
end_generic_block = -&LB end_environment -&LB
begin_generic_block = -&LB begin_environment (EOF | -&LB)
end_generic_block = -&LB end_environment (EOF | -&LB)
itemize = "\begin{itemize}" [PARSEP] { item } §"\end{itemize}"
enumerate = "\begin{enumerate}" [PARSEP] {item } §"\end{enumerate}"
......@@ -87,7 +87,7 @@ inline_environment = known_inline_env | generic_inline_env
known_inline_env = inline_math
generic_inline_env = begin_inline_env { text_elements }+ §end_inline_env
begin_inline_env = (-!LB begin_environment) | (begin_environment -!LB)
end_inline_env = (-!LB end_environment) | (end_environment -!LB)
end_inline_env = (-!LB end_environment) | (end_environment -!LB) # ambiguity with genric_block when EOF
begin_environment = "\begin{" §NAME §"}"
end_environment = "\end{" §::NAME §"}"
......
examples/LaTeX/LaTeXCompiler.py
View file @ 9b9293f1
......@@ -108,8 +108,8 @@ class LaTeXGrammar(Grammar):
known_environment = itemize | enumerate | figure | table | quotation
| verbatim
generic_block = begin_generic_block sequence §end_generic_block
begin_generic_block = -&LB begin_environment -&LB
end_generic_block = -&LB end_environment -&LB
begin_generic_block = -&LB begin_environment (EOF | -&LB)
end_generic_block = -&LB end_environment (EOF | -&LB)
itemize = "\begin{itemize}" [PARSEP] { item } §"\end{itemize}"
enumerate = "\begin{enumerate}" [PARSEP] {item } §"\end{enumerate}"
......@@ -138,7 +138,7 @@ class LaTeXGrammar(Grammar):
known_inline_env = inline_math
generic_inline_env = begin_inline_env { text_elements }+ §end_inline_env
begin_inline_env = (-!LB begin_environment) | (begin_environment -!LB)
end_inline_env = (-!LB end_environment) | (end_environment -!LB)
end_inline_env = (-!LB end_environment) | (end_environment -!LB) # ambiguity with genric_block when EOF
begin_environment = "\begin{" §NAME §"}"
end_environment = "\end{" §::NAME §"}"
......@@ -205,7 +205,7 @@ class LaTeXGrammar(Grammar):
block_of_paragraphs = Forward()
end_generic_block = Forward()
text_elements = Forward()
source_hash__ = "cf722f798fd396d8094fcc28831d9dcd"
source_hash__ = "f941997b8aca0a8aa2d2f38cb52818eb"
parser_initialization__ = "upon instantiation"
COMMENT__ = r'%.*(?:\n|$)'
WSP__ = mixin_comment(whitespace=r'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?', comment=r'%.*(?:\n|$)')
......@@ -256,8 +256,8 @@ class LaTeXGrammar(Grammar):
item = Series(Token("\\item"), Optional(PARSEP), sequence)
enumerate = Series(Token("\\begin{enumerate}"), Optional(PARSEP), ZeroOrMore(item), Required(Token("\\end{enumerate}")))
itemize = Series(Token("\\begin{itemize}"), Optional(PARSEP), ZeroOrMore(item), Required(Token("\\end{itemize}")))
end_generic_block.set(Series(Lookbehind(LB), end_environment, Lookbehind(LB)))
begin_generic_block.set(Series(Lookbehind(LB), begin_environment, Lookbehind(LB)))
end_generic_block.set(Series(Lookbehind(LB), end_environment, Alternative(EOF, Lookbehind(LB))))
begin_generic_block.set(Series(Lookbehind(LB), begin_environment, Alternative(EOF, Lookbehind(LB))))
generic_block = Series(begin_generic_block, sequence, Required(end_generic_block))
known_environment = Alternative(itemize, enumerate, figure, table, quotation, verbatim)
block_environment.set(Alternative(known_environment, generic_block))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment