Skip to content
GitLab
Commit abc8d58a authored by eckhart's avatar eckhart
Browse files

bug-fixes: ammendments to history tracing...

parent 3e183f80
Hide whitespace changes
Inline Side-by-side
DHParser/ebnf.py
View file @ abc8d58a
......@@ -834,7 +834,7 @@ class EBNFCompiler(Compiler):
if DROP_WSPC in self.directives.drop or DROP_TOKEN in self.directives.drop:
definitions.append((EBNFCompiler.DROP_WHITESPACE_PARSER_KEYWORD,
'Drop(RegExp(%s))' % EBNFCompiler.WHITESPACE_KEYWORD))
'Drop(Whitespace(%s))' % EBNFCompiler.WHITESPACE_KEYWORD))
definitions.append((EBNFCompiler.WHITESPACE_PARSER_KEYWORD,
'Whitespace(%s)' % EBNFCompiler.WHITESPACE_KEYWORD))
definitions.append((EBNFCompiler.WHITESPACE_KEYWORD,
......
DHParser/log.py
View file @ abc8d58a
......@@ -61,7 +61,7 @@ from DHParser.configuration import access_presets, finalize_presets, get_config_
set_config_value
from DHParser.error import Error
from DHParser.stringview import StringView
from DHParser.syntaxtree import Node, ZOMBIE_TAG
from DHParser.syntaxtree import Node, ZOMBIE_TAG, EMPTY_PTYPE
from DHParser.toolkit import escape_control_characters
__all__ = ('start_logging',
......@@ -207,7 +207,8 @@ def append_log(log_name: str, *strings, echo: bool=False) -> None:
ldir = log_dir()
if ldir and log_name:
log_path = os.path.join(ldir, log_name)
assert os.path.exists(log_path)
if not os.path.exists(log_path):
assert create_log(log_path), 'Could not create log file: "{}"'.format(log_path)
with open(log_path, 'a', encoding='utf-8') as f:
for text in strings:
f.write(text)
......@@ -257,6 +258,7 @@ class HistoryRecord:
__slots__ = ('call_stack', 'node', 'text', 'pos', 'line_col', 'errors')
MATCH = "MATCH"
DROP = "DROP"
ERROR = "ERROR"
FAIL = "FAIL"
Snapshot_Fields = ('line', 'column', 'stack', 'status', 'text')
......@@ -269,16 +271,17 @@ class HistoryRecord:
HTML_LEAD_IN = (
'<!DOCTYPE html>\n'
'<html>\n<head>\n<meta charset="utf-8"/>\n<style>\n'
'table {border-spacing: 0px; border: thin solid darkgrey; width:100%}\n'
'table {border-spacing: 0px; border: thin solid grey; width:100%}\n'
'td,th {font-family:monospace; '
'border-right: thin solid grey; border-bottom: thin solid grey}\n'
'td.line, td.column {color:darkgrey}\n'
'.text{color:darkblue}\n'
'.failtext {font-weight:normal; color:darkgrey}\n'
'.unmatched {font-weight:normal; color:darkgrey}\n'
'td.line, td.column {color:grey}\n'
'.text{color:blue}\n'
'.failtext {font-weight:normal; color:grey}\n'
'.unmatched {font-weight:normal; color:lightgrey}\n'
'.fail {font-weight:bold; color:darkgrey}\n'
'.error {font-weight:bold; color:red}\n'
'.match {font-weight:bold; color:darkgreen}\n'
'.match {font-weight:bold; color:green}\n'
'.drop {font-weight:bold; color:darkslategrey}\n'
'.matchstack {font-weight:bold;color:darkred}\n'
'span {color:darkgrey}\n'
'</style>\n</head>\n<body>\n')
......@@ -329,7 +332,7 @@ class HistoryRecord:
classes = list(HistoryRecord.Snapshot_Fields)
idx = {field_name: i for i, field_name in enumerate(classes)}
classes[idx['status']] = status.lower()
if status == self.MATCH:
if status in (self.MATCH, self.DROP):
n = max(40 - len(excerpt), 0)
dots = '...' if len(self.text) > n else ''
excerpt = excerpt + '<span class="unmatched">' + self.text[:n] + dots + '</span>'
......@@ -346,9 +349,11 @@ class HistoryRecord:
else:
stack = stack[:i] + '<span class="matchstack">' + stack[i:k] \
+ '</span>' + stack[k:]
else:
stack = '<span class="matchstack">{}</span>'.format(stack)
elif status == self.FAIL:
classes[idx['text']] = 'failtext'
else: # status == self.ERROR:
else: # ERROR
stack += '<br/>\n' + status
tpl = self.Snapshot(str(self.line_col[0]), str(self.line_col[1]), stack, status, excerpt)
return ''.join(['<tr>'] + [('<td class="%s">%s</td>' % (cls, item))
......@@ -373,8 +378,16 @@ class HistoryRecord:
@property
def status(self) -> str:
return self.FAIL if self.node is None or self.node.tag_name == ZOMBIE_TAG else \
('"%s"' % self.err_msg()) if self.errors else self.MATCH
if self.node is None or self.node.tag_name == ZOMBIE_TAG:
return self.FAIL
elif self.node.tag_name == EMPTY_PTYPE:
return self.DROP
elif self.errors:
return '"%s"' % self.err_msg()
else:
return self.MATCH
# return self.FAIL if self.node is None or self.node.tag_name == ZOMBIE_TAG else \
# ('"%s"' % self.err_msg()) if self.errors else self.MATCH
@property
def excerpt(self):
......
DHParser/parse.py
View file @ abc8d58a
......@@ -41,7 +41,7 @@ from DHParser.log import HistoryRecord
from DHParser.preprocess import BEGIN_TOKEN, END_TOKEN, RX_TOKEN_NAME
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, FrozenNode, RootNode, WHITESPACE_PTYPE, \
TOKEN_PTYPE, ZOMBIE_TAG, ResultType
TOKEN_PTYPE, ZOMBIE_TAG, EMPTY_NODE, ResultType
from DHParser.toolkit import sane_parser_name, escape_control_characters, re, cython, \
RX_NEVER_MATCH, RxPatternType
......@@ -52,7 +52,6 @@ __all__ = ('ParserError',
'GrammarErrorType',
'GrammarError',
'Grammar',
'EMPTY_NODE',
'PreprocessorToken',
'Token',
'DropToken',
......@@ -208,9 +207,6 @@ def reentry_point(rest: StringView,
########################################################################
EMPTY_NODE = FrozenNode(':EMPTY__', '')
ApplyFunc = Callable[['Parser'], None]
FlagFunc = Callable[[ApplyFunc, Set[ApplyFunc]], bool]
ParseFunc = Callable[['Parser', StringView], Tuple[Optional[Node], StringView]]
......@@ -1806,7 +1802,7 @@ class ZeroOrMore(Option):
'.'
>>> forever = ZeroOrMore(RegExp(''))
>>> Grammar(forever)('') # infinite loops will automatically be broken
Node(:EMPTY__, )
Node(:EMPTY, )
EBNF-Notation: ``{ ... }``
......@@ -1851,7 +1847,7 @@ class OneOrMore(UnaryParser):
' <<< Error on "." | Parser "{/\\w+,?/ ~}+ \'.\' ~" did not match! >>> '
>>> forever = OneOrMore(RegExp(''))
>>> Grammar(forever)('') # infinite loops will automatically be broken
Node(:EMPTY__, )
Node(:EMPTY, )
EBNF-Notation: ``{ ... }+``
......
DHParser/syntaxtree.py
View file @ abc8d58a
......@@ -40,6 +40,7 @@ from DHParser.toolkit import re, cython
__all__ = ('WHITESPACE_PTYPE',
'TOKEN_PTYPE',
'REGEXP_PTYPE',
'EMPTY_PTYPE',
'LEAF_PTYPES',
'ZOMBIE_TAG',
'PLACEHOLDER',
......@@ -48,6 +49,7 @@ __all__ = ('WHITESPACE_PTYPE',
'ChildrenType',
'Node',
'FrozenNode',
'EMPTY_NODE',
'tree_sanity_check',
'RootNode',
'DHParser_JSONEncoder',
......@@ -69,6 +71,7 @@ __all__ = ('WHITESPACE_PTYPE',
WHITESPACE_PTYPE = ':Whitespace'
TOKEN_PTYPE = ':Token'
REGEXP_PTYPE = ':RegExp'
EMPTY_PTYPE = ':EMPTY'
LEAF_PTYPES = {WHITESPACE_PTYPE, TOKEN_PTYPE, REGEXP_PTYPE}
ZOMBIE_TAG = "ZOMBIE__"
......@@ -1254,6 +1257,7 @@ class FrozenNode(Node):
PLACEHOLDER = FrozenNode('__PLACEHOLDER__', '')
EMPTY_NODE = FrozenNode(EMPTY_PTYPE, '')
def tree_sanity_check(tree: Node) -> bool:
......
DHParser/trace.py
View file @ abc8d58a
......@@ -32,9 +32,10 @@ be superceded by tracing.
from typing import Tuple, Optional, List, Collection, Union
from DHParser.stringview import StringView
from DHParser.syntaxtree import Node, REGEXP_PTYPE, TOKEN_PTYPE, WHITESPACE_PTYPE
from DHParser.syntaxtree import Node, REGEXP_PTYPE, TOKEN_PTYPE, WHITESPACE_PTYPE, \
EMPTY_PTYPE, EMPTY_NODE
from DHParser.log import HistoryRecord
from DHParser.parse import Parser, ParserError, ParseFunc, EMPTY_NODE
from DHParser.parse import Parser, ParserError, ParseFunc
__all__ = ('trace_history', 'with_all_descendants', 'with_unnamed_descendants', 'set_tracer')
......@@ -57,12 +58,14 @@ def trace_history(self, text: StringView) -> Tuple[Optional[Node], StringView]:
# Mind that memoized parser calls will not appear in the history record!
# Don't track returning parsers except in case an error has occurred!
# TODO: Try recording all named parsers on the way back?
delta = text._len - rest._len
if ((grammar.moving_forward__ or grammar.most_recent_error__ or (node and not self.anonymous))
and (node != EMPTY_NODE or self.tag_name != WHITESPACE_PTYPE)):
and (self.tag_name != WHITESPACE_PTYPE)): # TODO: Make dropping insignificant whitespace form history configurable
errors = [grammar.most_recent_error__] if grammar.most_recent_error__ else []
grammar.most_recent_error__ = None
line_col = grammar.line_col__(text)
record = HistoryRecord(grammar.call_stack__, node, rest, line_col, errors)
nd = Node(node.tag_name, text[:delta]) if node else None
record = HistoryRecord(grammar.call_stack__, nd, rest, line_col, errors)
if (not grammar.history__ or line_col != grammar.history__[-1].line_col
or record.call_stack != grammar.history__[-1].call_stack[:len(record.call_stack)]):
grammar.history__.append(record)
......
test/test_trace.py
View file @ abc8d58a
......@@ -26,27 +26,45 @@ scriptpath = os.path.dirname(__file__) or '.'
sys.path.append(os.path.abspath(os.path.join(scriptpath, '..')))
from DHParser import grammar_provider, with_all_descendants, with_unnamed_descendants, \
set_tracer, trace_history, log_parsing_history, start_logging
set_tracer, trace_history, log_parsing_history, start_logging, set_config_value
class TestTrace:
def setup(self):
minilang = """
start_logging()
def test_trace_simple(self):
lang = """
expr = term { ("+"|"-") term }
term = factor { ("*"|"/") factor }
factor = /[0-9]+/~ | "(" expr ")"
"""
self.gr = grammar_provider(minilang)()
# def tear_down(self):
# os.remove('trace.log')
gr = grammar_provider(lang)()
all_desc = with_all_descendants(gr.root_parser__)
set_tracer(all_desc, trace_history)
st = gr('2*(3+4)')
log_parsing_history(gr, 'trace_simple')
print(st.serialize())
def test_trace(self):
all_desc = with_all_descendants(self.gr.root_parser__)
def test_trace_drop(self):
lang = r"""
@ drop = token, whitespace
expression = term { ("+" | "-") term}
term = factor { ("*"|"/") factor}
factor = number | variable | "(" expression ")"
| constant | fixed
variable = /[a-z]/~
number = /\d+/~
constant = "A" | "B"
fixed = "X"
"""
set_config_value('compiled_EBNF_log', 'test_trace_parser.py')
gr = grammar_provider(lang)()
all_desc = with_all_descendants(gr.root_parser__)
set_tracer(all_desc, trace_history)
st = self.gr('2*(3+4)')
start_logging()
log_parsing_history(self.gr, 'trace.log')
# st = gr('2*(3+4)')
st = gr('2*(3 + 4*(5 + 6*(7 + 8 + 9*2 - 1/5*1000) + 2) + 5000 + 4000)')
log_parsing_history(gr, 'trace_drop')
print(st.serialize())
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment