Commit b1656cf6 authored by Eckhart Arnold's avatar Eckhart Arnold

- Token nodes and Whitespace nodes are now identified by Parser.ptype rather than by special names

parent cf262cfd
......@@ -79,7 +79,7 @@ from DHParser.parsers import GrammarBase, CompilerBase, nil_scanner, \\
from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \\
remove_children_if, reduce_single_child, replace_by_single_child, remove_whitespace, \\
no_operation, remove_expendables, remove_tokens, flatten, is_whitespace, is_expendable, \\
WHITESPACE_KEYWORD, TOKEN_KEYWORD
WHITESPACE_PTYPE, TOKEN_PTYPE
'''
......
......@@ -29,8 +29,8 @@ from .parsers import GrammarBase, mixin_comment, nil_scanner, Forward, RE, Negat
Alternative, Sequence, Optional, Required, OneOrMore, ZeroOrMore, Token, CompilerBase, \
Capture, Retrieve
from .syntaxtree import Node, traverse, remove_enclosing_delimiters, reduce_single_child, \
replace_by_single_child, TOKEN_KEYWORD, remove_expendables, remove_tokens, flatten, \
forbid, assert_content, WHITESPACE_KEYWORD, key_parser_name, key_tag_name
replace_by_single_child, TOKEN_PTYPE, remove_expendables, remove_tokens, flatten, \
forbid, assert_content, WHITESPACE_PTYPE, key_parser_name, key_tag_name
from .versionnumber import __version__
......@@ -207,7 +207,7 @@ EBNF_transformation_table = {
[reduce_single_child, remove_enclosing_delimiters],
"symbol, literal, regexp":
[remove_expendables, reduce_single_child],
(TOKEN_KEYWORD, WHITESPACE_KEYWORD):
(TOKEN_PTYPE, WHITESPACE_PTYPE):
[remove_expendables, reduce_single_child],
"list_":
[flatten, partial(remove_tokens, tokens={','})],
......@@ -225,7 +225,7 @@ EBNF_validation_table = {
def EBNFTransformer(syntax_tree):
for processing_table, key_func in [(EBNF_transformation_table, key_parser_name),
for processing_table, key_func in [(EBNF_transformation_table, key_tag_name),
(EBNF_validation_table, key_tag_name)]:
traverse(syntax_tree, processing_table, key_func)
......@@ -290,7 +290,8 @@ class EBNFCompiler(CompilerBase):
in EBNF-Notation.
"""
COMMENT_KEYWORD = "COMMENT__"
RESERVED_SYMBOLS = {TOKEN_KEYWORD, WHITESPACE_KEYWORD, COMMENT_KEYWORD}
WHITESPACE_KEYWORD = "WSP__"
RESERVED_SYMBOLS = {WHITESPACE_KEYWORD, COMMENT_KEYWORD}
AST_ERROR = "Badly structured syntax tree. " \
"Potentially due to erroneuos AST transformation."
PREFIX_TABLE = {'§': 'Required',
......@@ -377,11 +378,11 @@ class EBNFCompiler(CompilerBase):
definitions[i] = (definitions[i][0], 'Capture(%s)' % definitions[1])
self.definition_names = [defn[0] for defn in definitions]
definitions.append(('wspR__', WHITESPACE_KEYWORD
definitions.append(('wspR__', self.WHITESPACE_KEYWORD
if 'right' in self.directives['literalws'] else "''"))
definitions.append(('wspL__', WHITESPACE_KEYWORD
definitions.append(('wspL__', self.WHITESPACE_KEYWORD
if 'left' in self.directives['literalws'] else "''"))
definitions.append((WHITESPACE_KEYWORD,
definitions.append((self.WHITESPACE_KEYWORD,
("mixin_comment(whitespace="
"r'{whitespace}', comment=r'{comment}')").
format(**self.directives)))
......@@ -623,13 +624,13 @@ class EBNFCompiler(CompilerBase):
name = []
if rx[:2] == '~/':
if not 'left' in self.directives['literalws']:
name = ['wL=' + WHITESPACE_KEYWORD] + name
name = ['wL=' + self.WHITESPACE_KEYWORD] + name
rx = rx[1:]
elif 'left' in self.directives['literalws']:
name = ["wL=''"] + name
if rx[-2:] == '/~':
if 'right' not in self.directives['literalws']:
name = ['wR=' + WHITESPACE_KEYWORD] + name
name = ['wR=' + self.WHITESPACE_KEYWORD] + name
rx = rx[:-1]
elif 'right' in self.directives['literalws']:
name = ["wR=''"] + name
......
......@@ -58,7 +58,7 @@ except ImportError:
from .toolkit import is_logging, log_dir, logfile_basename, escape_re, sane_parser_name, \
compact_sexpr
from .syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \
from .syntaxtree import WHITESPACE_PTYPE, TOKEN_PTYPE, ZOMBIE_PARSER, Node, \
mock_syntax_tree
from DHParser.toolkit import load_if_file, error_messages
......@@ -205,16 +205,21 @@ class ParserMetaClass(type):
class Parser(metaclass=ParserMetaClass):
def __init__(self, name=None):
assert name is None or isinstance(name, str), str(name)
self.name = name or ''
self.ptype = self.__class__.__name__
def __init__(self, name=''):
assert isinstance(name, str), str(name)
i = name.find(':')
if i >= 0:
self.name = name[:i]
self.ptype = name[i:]
else:
self.name = name
self.ptype = ':' + self.__class__.__name__
# self.pbases = {cls.__name__ for cls in inspect.getmro(self.__class__)}
self._grammar = None # center for global variables etc.
self.reset()
def __deepcopy__(self, memo):
return self.__class__(self.name)
return self.__class__(self.name + self.ptype)
def reset(self):
self.visited = dict()
......@@ -281,27 +286,27 @@ class GrammarBase:
cdict = cls.__dict__
for entry, parser in cdict.items():
if isinstance(parser, Parser) and sane_parser_name(entry):
if not parser.name or parser.name == TOKEN_KEYWORD:
if not parser.name:
parser.name = entry
if (isinstance(parser, Forward) and (not parser.parser.name
or parser.parser.name == TOKEN_KEYWORD)):
if (isinstance(parser, Forward) and (not parser.parser.name)):
parser.parser.name = entry
cls.parser_initialization__ = "done"
def __init__(self):
self.all_parsers = set()
self.dirty_flag = False
self.history_tracking = False
self._reset()
self._assign_parser_names()
self.root__ = copy.deepcopy(self.__class__.root__)
if self.wspL__:
self.wsp_left_parser__ = RegExp(self.wspL__, WHITESPACE_KEYWORD)
self.wsp_left_parser__ = Whitespace(self.wspL__)
self.wsp_left_parser__.grammar = self
self.all_parsers.add(self.wsp_left_parser__) # don't you forget about me...
else:
self.wsp_left_parser__ = ZOMBIE_PARSER
if self.wspR__:
self.wsp_right_parser__ = RegExp(self.wspR__, WHITESPACE_KEYWORD)
self.wsp_right_parser__ = Whitespace(self.wspR__)
self.wsp_right_parser__.grammar = self
self.all_parsers.add(self.wsp_right_parser__) # don't you forget about me...
else:
......@@ -317,7 +322,6 @@ class GrammarBase:
self.last_node = None
self.call_stack = [] # support for call stack tracing
self.history = [] # snapshots of call stacks
self.history_tracking = is_logging()
self.moving_forward = True # also needed for call stack tracing
def _add_parser(self, parser):
......@@ -349,6 +353,7 @@ class GrammarBase:
parser.reset()
else:
self.dirty_flag = True
self.history_tracking = is_logging()
self.document = document
parser = self[start_parser]
stitches = []
......@@ -402,7 +407,7 @@ class GrammarBase:
for record in self.history:
line = "; ".join(prepare_line(record))
full_history.append(line)
if record.node and record.node.parser.name != WHITESPACE_KEYWORD:
if record.node and record.node.parser.ptype != WHITESPACE_PTYPE:
match_history.append(line)
if record.node.errors:
errors_only.append(line)
......@@ -514,7 +519,7 @@ class RegExp(Parser):
other parsers delegate part of the parsing job to other parsers,
but do not match text directly.
"""
def __init__(self, regexp, name=None):
def __init__(self, regexp, name=''):
super(RegExp, self).__init__(name)
self.regexp = re.compile(regexp) if isinstance(regexp, str) else regexp
......@@ -524,7 +529,7 @@ class RegExp(Parser):
regexp = copy.deepcopy(self.regexp, memo)
except TypeError:
regexp = self.regexp.pattern
return RegExp(regexp, self.name)
return RegExp(regexp, self.name + self.ptype)
def __call__(self, text):
match = text[0:1] != BEGIN_SCANNER_TOKEN and self.regexp.match(text) # ESC starts a scanner token.
......@@ -537,6 +542,12 @@ class RegExp(Parser):
return self.name or self.ptype + ' /%s/' % self.regexp.pattern
def Whitespace(re_wsp='\s*'):
rx_parser = RegExp(re_wsp)
rx_parser.ptype = WHITESPACE_PTYPE
return rx_parser
class RE(Parser):
"""Regular Expressions with optional leading or trailing whitespace.
......@@ -550,7 +561,7 @@ class RE(Parser):
respective parameters in the constructor are set to ``None`` the
default whitespace expression from the Grammar object will be used.
"""
def __init__(self, regexp, wL=None, wR=None, name=None):
def __init__(self, regexp, wL=None, wR=None, name=''):
"""Constructor for class RE.
Args:
......@@ -569,8 +580,8 @@ class RE(Parser):
super(RE, self).__init__(name)
self.wL = wL
self.wR = wR
self.wspLeft = RegExp(wL, WHITESPACE_KEYWORD) if wL else ZOMBIE_PARSER
self.wspRight = RegExp(wR, WHITESPACE_KEYWORD) if wR else ZOMBIE_PARSER
self.wspLeft = Whitespace(wL) if wL else ZOMBIE_PARSER
self.wspRight = Whitespace(wR) if wR else ZOMBIE_PARSER
self.main = RegExp(regexp)
def __deepcopy__(self, memo={}):
......@@ -578,7 +589,7 @@ class RE(Parser):
regexp = copy.deepcopy(self.main.regexp, memo)
except TypeError:
regexp = self.main.regexp.pattern
return self.__class__(regexp, self.wL, self.wR, self.name)
return self.__class__(regexp, self.wL, self.wR, self.name + self.ptype)
def __call__(self, text):
# assert self.main.regexp.pattern != "@"
......@@ -593,7 +604,7 @@ class RE(Parser):
return None, text
def __str__(self):
if self.name == TOKEN_KEYWORD:
if self.ptype == TOKEN_PTYPE:
return 'Token "%s"' % self.main.regexp.pattern.replace('\\', '')
return self.name or ('RE ' + ('~' if self.wL else '')
+ '/%s/' % self.main.regexp.pattern + ('~' if self.wR else ''))
......@@ -615,16 +626,16 @@ class RE(Parser):
self.main.apply(func)
def Token(token, wL=None, wR=None, name=None):
def Token(token, wL=None, wR=None, name=''):
"""Returns an RE-parser that matches plain strings that are
considered as 'tokens'.
If the ``name``-parameter is empty, the parser's name will be set
to the TOKEN_KEYWORD, making it easy to identify tokens in the
abstract syntax tree transformation and compilation stage.
The parser's name will be set to the TOKEN_PTYPE, making it easy to
identify tokens in the abstract syntax tree transformation and
compilation stage.
"""
parser = RE(escape_re(token), wL, wR, name or TOKEN_KEYWORD)
parser.ptype = "Token"
parser = RE(escape_re(token), wL, wR, name)
parser.ptype = TOKEN_PTYPE
return parser
......@@ -649,14 +660,14 @@ def mixin_comment(whitespace, comment):
class UnaryOperator(Parser):
def __init__(self, parser, name=None):
def __init__(self, parser, name=''):
super(UnaryOperator, self).__init__(name)
assert isinstance(parser, Parser)
self.parser = parser
def __deepcopy__(self, memo):
parser = copy.deepcopy(self.parser, memo)
return self.__class__(parser, self.name)
return self.__class__(parser, self.name + self.ptype)
def apply(self, func):
if super(UnaryOperator, self).apply(func):
......@@ -664,14 +675,14 @@ class UnaryOperator(Parser):
class NaryOperator(Parser):
def __init__(self, *parsers, name=None):
def __init__(self, *parsers, name=''):
super(NaryOperator, self).__init__(name)
assert all([isinstance(parser, Parser) for parser in parsers]), str(parsers)
self.parsers = parsers
def __deepcopy__(self, memo):
parsers = copy.deepcopy(self.parsers, memo)
return self.__class__(*parsers, name=self.name)
return self.__class__(*parsers, name=self.name + self.ptype)
def apply(self, func):
if super(NaryOperator, self).apply(func):
......@@ -680,7 +691,7 @@ class NaryOperator(Parser):
class Optional(UnaryOperator):
def __init__(self, parser, name=None):
def __init__(self, parser, name=''):
super(Optional, self).__init__(parser, name)
assert isinstance(parser, Parser)
assert not isinstance(parser, Optional), \
......@@ -713,7 +724,7 @@ class ZeroOrMore(Optional):
class OneOrMore(UnaryOperator):
def __init__(self, parser, name=None):
def __init__(self, parser, name=''):
super(OneOrMore, self).__init__(parser, name)
assert not isinstance(parser, Optional), \
"Use ZeroOrMore instead of nesting OneOrMore and Optional: " \
......@@ -737,7 +748,7 @@ class OneOrMore(UnaryOperator):
class Sequence(NaryOperator):
def __init__(self, *parsers, name=None):
def __init__(self, *parsers, name=''):
super(Sequence, self).__init__(*parsers, name=name)
assert len(self.parsers) >= 1
......@@ -757,7 +768,7 @@ class Sequence(NaryOperator):
class Alternative(NaryOperator):
def __init__(self, *parsers, name=None):
def __init__(self, *parsers, name=''):
super(Alternative, self).__init__(*parsers, name=name)
assert len(self.parsers) >= 1
assert all(not isinstance(p, Optional) for p in self.parsers)
......@@ -778,7 +789,7 @@ class Alternative(NaryOperator):
class FlowOperator(UnaryOperator):
def __init__(self, parser, name=None):
def __init__(self, parser, name=''):
super(FlowOperator, self).__init__(parser, name)
......@@ -798,7 +809,7 @@ class Required(FlowOperator):
class Lookahead(FlowOperator):
def __init__(self, parser, name=None):
def __init__(self, parser, name=''):
super(Lookahead, self).__init__(parser, name)
def __call__(self, text):
......@@ -830,7 +841,7 @@ def iter_right_branch(node):
class Lookbehind(FlowOperator):
def __init__(self, parser, name=None):
def __init__(self, parser, name=''):
super(Lookbehind, self).__init__(parser, name)
print("WARNING: Lookbehind Operator is experimental!")
......@@ -870,7 +881,7 @@ class NegativeLookbehind(Lookbehind):
class Capture(UnaryOperator):
def __init__(self, parser, name=None):
def __init__(self, parser, name=''):
super(Capture, self).__init__(parser, name)
print("WARNING: Capture operator is experimental")
......@@ -885,7 +896,7 @@ class Capture(UnaryOperator):
class Retrieve(Parser):
def __init__(self, symbol, counterpart=None, name=None):
def __init__(self, symbol, counterpart=None, name=''):
if not name:
name = symbol.name
super(Retrieve, self).__init__(name)
......@@ -894,7 +905,7 @@ class Retrieve(Parser):
print("WARNING: Retrieve operator is experimental")
def __deepcopy__(self, memo):
return self.__class__(self.symbol, self.counterpart, self.name)
return self.__class__(self.symbol, self.counterpart, self.name + self.ptype)
def __call__(self, text):
stack = self.grammar.variables[self.symbol.name]
......@@ -1012,7 +1023,7 @@ class CompilerBase:
for the parsers of the sub nodes by itself. Rather, this should
be done within the compilation methods.
"""
elem = str(node.parser)
elem = node.parser.name or node.parser.ptype[1:]
if not sane_parser_name(elem):
node.add_error("Reserved name '%s' not allowed as parser "
"name! " % elem + "(Any name starting with "
......@@ -1114,8 +1125,8 @@ def test_grammar(test_suite, parser_factory, transformer_factory):
errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
'\n\tExpr.: %s\n\tExpected: %s\n\tReceived: %s'
% (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
compact_sexpr(ast.as_sexpr()),
compact_sexpr(compare.as_sexpr())))
compact_sexpr(compare.as_sexpr()),
compact_sexpr(ast.as_sexpr())))
for test_name, test_code in tests['fail'].items():
cst = parser(test_code, parser_name)
......
......@@ -30,8 +30,8 @@ from typing import NamedTuple
from .toolkit import is_logging, log_dir, expand_table, line_col, smart_list
__all__ = ['WHITESPACE_KEYWORD',
'TOKEN_KEYWORD',
__all__ = ['WHITESPACE_PTYPE',
'TOKEN_PTYPE',
'ZOMBIE_PARSER',
'Error',
'Node',
......@@ -70,8 +70,9 @@ class MockParser:
object substitute is needed, chose the singleton ZOMBIE_PARSER.
"""
def __init__(self, name='', ptype='', pbases=frozenset()):
assert not ptype or ptype[0] == ':'
self.name = name
self.ptype = ptype or self.__class__.__name__
self.ptype = ptype or ':' + self.__class__.__name__
# self.pbases = pbases or {cls.__name__ for cls in inspect.getmro(self.__class__)}
def __str__(self):
......@@ -469,7 +470,7 @@ def mock_syntax_tree(sexpr):
lines.append(sexpr[:m.end()])
sexpr = sexpr[m.end():]
result = "\n".join(lines)
return Node(MockParser(name, class_name), result)
return Node(MockParser(name, ':' + class_name), result)
########################################################################
......@@ -479,8 +480,8 @@ def mock_syntax_tree(sexpr):
########################################################################
WHITESPACE_KEYWORD = 'WSP__'
TOKEN_KEYWORD = 'TOKEN__'
WHITESPACE_PTYPE = ':Whitespace'
TOKEN_PTYPE = ':Token'
def key_parser_name(node):
......@@ -491,7 +492,7 @@ def key_tag_name(node):
return node.tag_name
def traverse(root_node, processing_table, key_func=key_parser_name):
def traverse(root_node, processing_table, key_func=key_tag_name):
"""Traverses the snytax tree starting with the given ``node`` depth
first and applies the sequences of callback functions registered
in the ``calltable``-dictionary.
......@@ -524,7 +525,7 @@ def traverse(root_node, processing_table, key_func=key_parser_name):
traverse_recursive(child)
node.error_flag |= child.error_flag # propagate error flag
sequence = table.get('*', []) + \
table.get(key_func(node), table.get('?', [])) + \
table.get(key_func(node), table.get('', [])) + \
table.get('~', [])
# '*' always called (before any other processing function)
# '?' called for those nodes for which no (other) processing functions is in the table
......@@ -593,7 +594,7 @@ def replace_parser(node, parser_name, parser_type=''):
def is_whitespace(node):
"""Removes whitespace and comments defined with the
``@comment``-directive."""
return node.parser.name == WHITESPACE_KEYWORD
return node.parser.ptype == WHITESPACE_PTYPE
def is_empty(node):
......@@ -605,7 +606,7 @@ def is_expendable(node):
def is_token(node, token_set=frozenset()):
return node.parser.name == TOKEN_KEYWORD and (not token_set or node.result in token_set)
return node.parser.ptype == TOKEN_PTYPE and (not token_set or node.result in token_set)
def remove_children_if(node, condition):
......
......@@ -43,6 +43,7 @@ except ImportError:
__all__ = ['logging',
'is_logging',
'log_dir',
'logfile_basename',
'line_col',
'error_messages',
'escape_re',
......
......@@ -26,7 +26,7 @@ import sys
sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging
from DHParser.parsers import compile_source, Retrieve, WHITESPACE_KEYWORD, nil_scanner
from DHParser.parsers import compile_source, Retrieve, WHITESPACE_PTYPE, nil_scanner
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, EBNFTransformer, get_ebnf_compiler
from DHParser.dsl import compileEBNF, compileDSL, parser_factory
......@@ -104,7 +104,7 @@ class TestEBNFParser:
result = self.EBNF(snippet, 'literal')
assert not result.error_flag
assert str(result) == snippet
assert result.find(lambda node: str(node) == WHITESPACE_KEYWORD)
assert result.find(lambda node: node.parser.ptype == WHITESPACE_PTYPE)
result = self.EBNF(' "literal"', 'literal')
assert result.error_flag # literals catch following, but not leading whitespace
......
......@@ -26,7 +26,7 @@ sys.path.extend(['../', './'])
from DHParser import parsers
from DHParser.toolkit import is_logging, compile_python_object
from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten, TOKEN_KEYWORD
replace_by_single_child, reduce_single_child, flatten, TOKEN_PTYPE
from DHParser.parsers import compile_source
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import parser_factory, DHPARSER_IMPORTS
......@@ -47,7 +47,7 @@ ARITHMETIC_EBNF_transformation_table = {
"formula": [remove_expendables],
"term, expr": [replace_by_single_child, flatten],
"factor": [remove_expendables, reduce_single_child],
(TOKEN_KEYWORD): [remove_expendables, reduce_single_child],
(TOKEN_PTYPE): [remove_expendables, reduce_single_child],
"": [remove_expendables, replace_by_single_child]
}
......@@ -74,9 +74,9 @@ class TestGrammarTest:
3: "20 / 4 * 3"
},
"ast": {
1: "(term (factor 4) (TOKEN__ *) (factor 5))",
2: "(term (factor 20) (TOKEN__ /) (factor 4))",
3: "(term (term (factor 20) (TOKEN__ /) (factor 4)) (TOKEN__ *) (factor 3))"
1: "(term (factor 4) (:Token *) (factor 5))",
2: "(term (factor 20) (:Token /) (factor 4))",
3: "(term (term (factor 20) (:Token /) (factor 4)) (:Token *) (factor 3))"
},
"fail": {
4: "4 + 5",
......@@ -93,9 +93,9 @@ class TestGrammarTest:
3: "20 / 4 * 3"
},
"ast": {
1: "(term (factor 4) (TOKEN__ *) (factor 5))",
2: "(term (factor 20) (TOKEN__ /) (factor 4))",
3: "(term (term (factor 19) (TOKEN__ /) (factor 4)) (TOKEN__ *) (factor 3))" # error 19 != 20
1: "(term (factor 4) (:Token *) (factor 5))",
2: "(term (factor 20) (:Token /) (factor 4))",
3: "(term (term (factor 19) (:Token /) (factor 4)) (:Token *) (factor 3))" # error 19 != 20
},
"fail": {
4: "4 * 5", # error: this should match
......@@ -108,7 +108,7 @@ class TestGrammarTest:
parser_fac = parser_factory(ARITHMETIC_EBNF)
trans_fac = lambda : ARITHMETIC_EBNFTransform
errata = parsers.test_grammar(self.cases, parser_fac, trans_fac)
assert not errata
assert not errata, str(errata)
errata = parsers.test_grammar(self.failure_cases, parser_fac, trans_fac)
# for e in errata:
# print(e)
......
......@@ -23,9 +23,9 @@ import copy
import sys
sys.path.extend(['../', './'])
from DHParser.toolkit import compact_sexpr
from DHParser.toolkit import compact_sexpr, logging
from DHParser.syntaxtree import traverse, mock_syntax_tree, reduce_single_child, \
replace_by_single_child, flatten, remove_expendables, TOKEN_KEYWORD
replace_by_single_child, flatten, remove_expendables, TOKEN_PTYPE
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import parser_factory
......@@ -71,7 +71,7 @@ class TestSExpr:
tree = mock_syntax_tree(sexpr)
assert tree.tag_name == 'a'
assert tree.result[0].tag_name == 'b'
assert tree.result[1].tag_name == 'class3'
assert tree.result[1].tag_name == ':class3'
assert tree.result[2].tag_name == 'c'
class TestNode:
......@@ -106,12 +106,12 @@ class TestNode:
ebnf = 'term = term ("*"|"/") factor | factor\nfactor = /[0-9]+/~'
att = {"term": [replace_by_single_child, flatten],
"factor": [remove_expendables, reduce_single_child],
(TOKEN_KEYWORD): [remove_expendables, reduce_single_child],
"": [remove_expendables, replace_by_single_child]}
(TOKEN_PTYPE): [remove_expendables, reduce_single_child],
"?": [remove_expendables, replace_by_single_child]}
parser = parser_factory(ebnf)()
tree = parser("20 / 4 * 3")
traverse(tree, att)
compare_tree = mock_syntax_tree("(term (term (factor 20) (TOKEN__ /) (factor 4)) (TOKEN__ *) (factor 3))")
compare_tree = mock_syntax_tree("(term (term (factor 20) (:Token /) (factor 4)) (:Token *) (factor 3))")
assert tree == compare_tree
def test_copy(self):
......@@ -128,7 +128,9 @@ class TestNode:
parser = get_ebnf_grammar()
transform = get_ebnf_transformer()
compiler = get_ebnf_compiler()
tree = parser(ebnf)
with logging():
tree = parser(ebnf)
parser.log_parsing_history()
tree_copy = copy.deepcopy(tree)
transform(tree_copy)
res1 = compiler(tree_copy)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment