Skip to content

GitLab

Commit 9405f741 authored by eckhart's avatar eckhart
Browse files

- setup.py # only cythonize stringview (better for development...)

parent 38b5c32a
Hide whitespace changes
Inline Side-by-side
Showing with 80 additions and 55 deletions
DHParser/compile.py
View file @ 9405f741
......@@ -168,7 +168,9 @@ class Compiler:
for the parsers of the sub nodes by itself. Rather, this should
be done within the compilation methods.
"""
elem = node.parser.name or node.parser.ptype[1:]
elem = node.tag_name
if elem.startswith(':'):
elem = elem[1:]
if not sane_parser_name(elem):
self.tree.new_error(node, "Reserved name '%s' not allowed as parser "
"name! " % elem + "(Any name starting with "
......
DHParser/ebnf.py
View file @ 9405f741
......@@ -660,9 +660,9 @@ class EBNFCompiler(Compiler):
string search or a regular expression from the nodes content. Returns
an empty string in case the node is neither regexp nor literal.
"""
if nd.parser.name == 'regexp':
if nd.tag_name == 'regexp':
return unrepr("re.compile(r'%s')" % self._extract_regex(nd))
elif nd.parser.name == 'literal':
elif nd.tag_name == 'literal':
s = nd.content.strip()
return s.strip('"') if s[0] == '"' else s.strip("'")
return ''
......@@ -847,15 +847,15 @@ class EBNFCompiler(Compiler):
definitions = [] # type: List[Tuple[str, str]]
# drop the wrapping sequence node
if len(node.children) == 1 and not node.children[0].parser.name:
if len(node.children) == 1 and node.children[0].is_anonymous():
node = node.children[0]
# compile definitions and directives and collect definitions
for nd in node.children:
if nd.parser.name == "definition":
if nd.tag_name == "definition":
definitions.append(self.compile(nd))
else:
assert nd.parser.name == "directive", nd.as_sxpr()
assert nd.tag_name == "directive", nd.as_sxpr()
self.compile(nd)
# node.error_flag = max(node.error_flag, nd.error_flag)
self.definitions.update(definitions)
......@@ -920,7 +920,7 @@ class EBNFCompiler(Compiler):
if key in {'comment', 'whitespace'}:
check_argnum()
if node.children[1].parser.name == "symbol":
if node.children[1].tag_name == "symbol":
value = node.children[1].content
if key == 'whitespace' and value in WHITESPACE_TYPES:
value = WHITESPACE_TYPES[value] # replace whitespace-name by regex
......@@ -970,7 +970,7 @@ class EBNFCompiler(Compiler):
if symbol in self.rules:
self.tree.new_error(node, 'Custom error message for symbol "%s"' % symbol
+ 'must be defined before the symbol!')
if node.children[1 if len(node.children) == 2 else 2].parser.name != 'literal':
if node.children[1 if len(node.children) == 2 else 2].tag_name != 'literal':
self.tree.new_error(
node, 'Directive "%s" requires message string or a a pair ' % key +
'(regular expression or search string, message string) as argument!')
......@@ -1035,7 +1035,7 @@ class EBNFCompiler(Compiler):
mandatory_marker = []
filtered_children = [] # type: List[Node]
for nd in node.children:
if nd.parser.ptype == TOKEN_PTYPE and nd.content == "§":
if nd.tag_name == TOKEN_PTYPE and nd.content == "§":
mandatory_marker.append(len(filtered_children))
if len(mandatory_marker) > 1:
self.tree.new_error(nd, 'One mandatory marker (§) sufficient to declare '
......@@ -1088,7 +1088,7 @@ class EBNFCompiler(Compiler):
if prefix in {'::', ':'}:
assert len(node.children) == 2
arg = node.children[-1]
if arg.parser.name != 'symbol':
if arg.tag_name != 'symbol':
self.tree.new_error(node, ('Retrieve Operator "%s" requires a symbol, '
'and not a %s.') % (prefix, str(arg.parser)))
return str(arg.result)
......@@ -1101,7 +1101,7 @@ class EBNFCompiler(Compiler):
# node.result[1].result = shift + node.result[2:]
node.children[1].result = (Node(node.children[1].parser, node.children[1].result),) \
+ node.children[2:]
node.children[1].parser = node.parser
node.children[1].tag_name = node.tag_name
node.result = (node.children[0], node.children[1])
node.result = node.children[1:]
......@@ -1113,7 +1113,7 @@ class EBNFCompiler(Compiler):
nd = node
if len(nd.children) >= 1:
nd = nd.children[0]
while nd.parser.name == "symbol":
while nd.tag_name == "symbol":
symlist = self.rules.get(nd.content, [])
if len(symlist) == 2:
nd = symlist[1]
......@@ -1121,10 +1121,10 @@ class EBNFCompiler(Compiler):
if len(symlist) == 1:
nd = symlist[0].children[1]
break
if (nd.parser.name != "regexp" or nd.content[:1] != '/'
if (nd.tag_name != "regexp" or nd.content[:1] != '/'
or nd.content[-1:] != '/'):
self.tree.new_error(node, "Lookbehind-parser can only be used with RegExp"
"-parsers, not: " + nd.parser.name + nd.parser.ptype)
"-parsers, not: " + nd.tag_name)
if not result.startswith('RegExp('):
self.deferred_tasks.append(lambda: check(node))
......@@ -1154,12 +1154,12 @@ class EBNFCompiler(Compiler):
# return self.non_terminal(node, 'Unordered')
assert len(node.children) == 1
nd = node.children[0]
if nd.parser.name == "term":
if nd.tag_name == "term":
filtered_result, custom_args = self._error_customization(nd)
mock_node = Node(nd.parser, filtered_result)
return self.non_terminal(mock_node, 'AllOf', custom_args)
elif nd.parser.name == "expression":
if any(c.parser.ptype == TOKEN_PTYPE and nd.content == '§' for c in nd.children):
elif nd.tag_name == "expression":
if any(c.tag_name == TOKEN_PTYPE and nd.content == '§' for c in nd.children):
self.tree.new_error(node, "No mandatory items § allowed in SomeOf-operator!")
args = ', '.join(self.compile(child) for child in nd.children)
return "SomeOf(" + args + ")"
......
DHParser/parse.py
View file @ 9405f741
......@@ -38,7 +38,7 @@ from DHParser.log import is_logging, HistoryRecord
from DHParser.preprocess import BEGIN_TOKEN, END_TOKEN, RX_TOKEN_NAME
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, RootNode, ParserBase, WHITESPACE_PTYPE, \
TOKEN_PTYPE, ZOMBIE_PARSER, ResultType
TOKEN_PTYPE, ZOMBIE_PARSER, ZOMBIE, ResultType
from DHParser.toolkit import sane_parser_name, escape_control_characters, re, typing
from typing import Callable, cast, List, Tuple, Set, Dict, DefaultDict, Union, Optional, Any
......@@ -1458,7 +1458,7 @@ class Series(NaryOperator):
break
results += (node,)
assert len(results) <= len(self.parsers) \
or len(self.parsers) >= len([p for p in results if p.parser != ZOMBIE_PARSER])
or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE])
node = Node(self, results)
if error:
raise ParserError(node, text, first_throw=True)
......@@ -1669,7 +1669,7 @@ class AllOf(NaryOperator):
if reloc < 0:
parsers = []
assert len(results) <= len(self.parsers) \
or len(self.parsers) >= len([p for p in results if p.parser != ZOMBIE_PARSER])
or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE])
node = Node(self, results)
if error:
raise ParserError(node, text, first_throw=True)
......
DHParser/syntaxtree.py
View file @ 9405f741
......@@ -39,6 +39,7 @@ __all__ = ('ParserBase',
'TOKEN_PTYPE',
'MockParser',
'ZombieParser',
'ZOMBIE',
'ZOMBIE_PARSER',
'ZOMBIE_NODE',
'ResultType',
......@@ -128,6 +129,9 @@ class MockParser(ParserBase):
self.ptype = ptype # or ':' + self.__class__.__name__
ZOMBIE = "__ZOMBIE__"
class ZombieParser(MockParser):
"""
Serves as a substitute for a Parser instance.
......@@ -146,7 +150,7 @@ class ZombieParser(MockParser):
super(ZombieParser, self).__init__()
assert not self.__class__.alive, "There can be only one!"
assert self.__class__ == ZombieParser, "No derivatives, please!"
self.name = "__ZOMBIE__"
self.name = ZOMBIE
self.__class__.alive = True
def __copy__(self):
......@@ -263,7 +267,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
S-Expression-output.
"""
__slots__ = '_result', 'children', '_len', '_pos', 'parser', 'errors', '_xml_attr', '_content'
__slots__ = '_result', 'children', '_len', '_pos', '_tag_name', 'errors', '_xml_attr', '_content'
def __init__(self, parser, result: ResultType, leafhint: bool = False) -> None:
"""
......@@ -281,7 +285,10 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
self._len = -1 # type: int # lazy evaluation
else:
self.result = result
self.parser = parser or ZOMBIE_PARSER
if parser is None:
self._tag_name = ZOMBIE
else:
self._tag_name = parser.name or parser.ptype
def __deepcopy__(self, memo):
if self.children:
......@@ -304,8 +311,9 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
def __repr__(self):
mpargs = {'name': self.parser.name, 'ptype': self.parser.ptype}
parg = "MockParser({name}, {ptype})".format(**mpargs)
# mpargs = {'name': self.parser.name, 'ptype': self.parser.ptype}
name, ptype = (self._tag_name.split(':') + [''])[:2]
parg = "MockParser({name}, {ptype})".format(name=name, ptype=ptype)
rarg = str(self) if not self.children else \
"(" + ", ".join(repr(child) for child in self.children) + ")"
return "Node(%s, %s)" % (parg, rarg)
......@@ -408,7 +416,22 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
name of the node's parser or, if the node's parser is unnamed, the
node's parser's `ptype`.
"""
return self.parser.name or self.parser.ptype
return self._tag_name
@tag_name.setter
def tag_name(self, tag_name: str):
assert tag_name
self._tag_name = tag_name
def is_anonymous(self):
return self._tag_name[0] == ':'
@property
def parser(self) -> MockParser:
name, ptype = (self.tag_name.split(':') + [''])[:2]
return MockParser(name, ':' + ptype)
@property
......@@ -824,7 +847,7 @@ class RootNode(Node):
duplicate.inline_tags = self.inline_tags
duplicate.omit_tags = self.omit_tags
duplicate.empty_tags = self.empty_tags
duplicate.parser = self.parser
duplicate.tag_name = self.tag_name
return duplicate
......@@ -844,7 +867,7 @@ class RootNode(Node):
self.children = node.children
self._len = node._len
self._pos = node._pos
self.parser = node.parser
self.tag_name = node.tag_name
if hasattr(node, '_xml_attr'):
self._xml_attr = node._xml_attr
self._content = node._content
......@@ -1079,7 +1102,7 @@ def parse_xml(xml: Union[str, StringView]) -> Node:
res.append(child)
s, closing_tagname = parse_closing_tag(s)
assert tagname == closing_tagname
if len(res) == 1 and res[0].parser.ptype == TOKEN_PTYPE:
if len(res) == 1 and res[0].tag_name == TOKEN_PTYPE:
result = res[0].result
else:
result = tuple(res)
......
DHParser/transform.py
View file @ 9405f741
......@@ -44,7 +44,6 @@ __all__ = ('TransformationDict',
'ConditionFunc',
'KeyFunc',
'transformation_factory',
'key_parser_name',
'key_tag_name',
'traverse',
'is_named',
......@@ -231,8 +230,8 @@ def transformation_factory(t1=None, t2=None, t3=None, t4=None, t5=None):
return decorator
def key_parser_name(node: Node) -> str:
return node.parser.name
# def key_parser_name(node: Node) -> str:
# return node.parser.name
def key_tag_name(node: Node) -> str:
......@@ -268,7 +267,7 @@ def traverse(root_node: Node,
is interpreted as a ``compact_table``. See
:func:`expand_table` or :func:`EBNFCompiler.EBNFTransTable`
key_func (function): A mapping key_func(node) -> keystr. The default
key_func yields node.parser.name.
key_func yields node.tag_name.
Example::
......@@ -392,18 +391,18 @@ def is_single_child(context: List[Node]) -> bool:
def is_named(context: List[Node]) -> bool:
"""Returns ``True`` if the current node's parser is a named parser."""
return bool(context[-1].parser.name)
return not context[-1].is_anonymous()
def is_anonymous(context: List[Node]) -> bool:
"""Returns ``True`` if the current node's parser is an anonymous parser."""
return not context[-1].parser.name
return context[-1].is_anonymous()
def is_whitespace(context: List[Node]) -> bool:
"""Returns ``True`` for whitespace and comments defined with the
``@comment``-directive."""
return context[-1].parser.ptype == WHITESPACE_PTYPE
return context[-1].tag_name == WHITESPACE_PTYPE
def is_empty(context: List[Node]) -> bool:
......@@ -426,7 +425,7 @@ def is_token(context: List[Node], tokens: AbstractSet[str] = frozenset()) -> boo
any token is a match.
"""
node = context[-1]
return node.parser.ptype == TOKEN_PTYPE and (not tokens or node.content in tokens)
return node.tag_name == TOKEN_PTYPE and (not tokens or node.content in tokens)
@transformation_factory(collections.abc.Set)
......@@ -487,10 +486,11 @@ def has_parent(context: List[Node], tag_name_set: AbstractSet[str]) -> bool:
def _replace_by(node: Node, child: Node):
if not child.parser.name:
child.parser = MockParser(node.parser.name, child.parser.ptype)
if node.is_anonymous() or not child.is_anonymous():
node.tag_name = child.tag_name
# name, ptype = (node.tag_name.split(':') + [''])[:2]
# child.parser = MockParser(name, ptype)
# parser names must not be overwritten, else: child.parser.name = node.parser.name
node.parser = child.parser
node.result = child.result
if hasattr(child, '_xml_attr'):
node.attr.update(child.attr)
......@@ -599,8 +599,7 @@ def replace_parser(context: List[Node], name: str):
name: "NAME:PTYPE" of the surrogate. The ptype is optional
"""
node = context[-1]
name, ptype = (name.split(':') + [''])[:2]
node.parser = MockParser(name, ':' + ptype)
node.tage_name = name
@transformation_factory(collections.abc.Callable)
......@@ -741,9 +740,9 @@ def merge_whitespace(context):
i = 0
L = len(children)
while i < L:
if children[i].parser.pytpe == WHITESPACE_PTYPE:
if children[i].tag_name == WHITESPACE_PTYPE:
k = i
while i < L and children[k].parser.ptype == WHITESPACE_PTYPE:
while i < L and children[k].tag_name == WHITESPACE_PTYPE:
i += 1
if i > k:
children[k].result = sum(children[n].result for n in range(k, i + 1))
......@@ -761,12 +760,12 @@ def move_whitespace(context):
return
parent = context[-2]
children = node.children
if children[0].parser.ptype == WHITESPACE_PTYPE:
if children[0].tag_name == WHITESPACE_PTYPE:
before = (children[0],)
children = children[1:]
else:
before = ()
if children and children[-1].parser.ptype == WHITESPACE_PTYPE:
if children and children[-1].tag_name == WHITESPACE_PTYPE:
after = (children[-1],)
children = children[:-1]
else:
......@@ -781,10 +780,10 @@ def move_whitespace(context):
# merge adjacent whitespace
prevN = parent.children[i - 1] if i > 0 else None
nextN = parent.children[i + 1] if i < len(parent.children) - 1 else None
if before and prevN and prevN.parser.ptype == WHITESPACE_PTYPE:
if before and prevN and prevN.tag_name == WHITESPACE_PTYPE:
prevN.result = prevN.result + before[0].result
before = ()
if after and nextN and nextN.parser.ptype == WHITESPACE_PTYPE:
if after and nextN and nextN.tag_name == WHITESPACE_PTYPE:
nextN.result = after[0].result + nextN.result
after = ()
......@@ -926,7 +925,7 @@ def remove_first(context: List[Node]):
node = context[-1]
if node.children:
for i, child in enumerate(node.children):
if child.parser.ptype != WHITESPACE_PTYPE:
if child.tag_name != WHITESPACE_PTYPE:
break
else:
return
......@@ -938,7 +937,7 @@ def remove_last(context: List[Node]):
node = context[-1]
if node.children:
for i, child in enumerate(reversed(node.children)):
if child.parser.ptype != WHITESPACE_PTYPE:
if child.tag_name != WHITESPACE_PTYPE:
break
else:
return
......@@ -1026,7 +1025,7 @@ def assert_content(context: List[Node], regexp: str):
node = context[-1]
if not has_content(context, regexp):
cast(RootNode, context[0]).new_error(node, 'Element "%s" violates %s on %s'
% (node.parser.name, str(regexp), node.content))
% (node.tag_name, str(regexp), node.content))
@transformation_factory(collections.abc.Set)
......@@ -1035,7 +1034,7 @@ def require(context: List[Node], child_tags: AbstractSet[str]):
for child in node.children:
if child.tag_name not in child_tags:
cast(RootNode, context[0]).new_error(node, 'Element "%s" is not allowed inside "%s".'
% (child.parser.name, node.parser.name))
% (child.tag_name, node.tag_name))
@transformation_factory(collections.abc.Set)
......@@ -1044,7 +1043,7 @@ def forbid(context: List[Node], child_tags: AbstractSet[str]):
for child in node.children:
if child.tag_name in child_tags:
cast(RootNode, context[0]).new_error(node, 'Element "%s" cannot be nested inside "%s".'
% (child.parser.name, node.parser.name))
% (child.tag_name, node.tag_name))
def peek(context: List[Node]):
......
test/test_parse.py
View file @ 9405f741
......@@ -446,11 +446,12 @@ class TestPopRetrieve:
@staticmethod
def opening_delimiter(node, name):
return node.tag_name == name and not isinstance(node.parser, Retrieve)
return node.tag_name == name # and not isinstance(node.parser, Retrieve)
@staticmethod
def closing_delimiter(node):
return isinstance(node.parser, Retrieve)
return node.tag_name in {':Pop', ':Retrieve'}
# return isinstance(node.parser, Retrieve)
def test_compile_mini_language(self):
assert self.minilang_parser
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment