Commit f2c9a5fc authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

- parse.py: DropWhitespace attached to tokens if DropToken is set

parent 8c7866c8
......@@ -805,7 +805,7 @@ class EBNFCompiler(Compiler):
# add special fields for Grammar class
if DROP_WSPC in self.directives.drop:
if DROP_WSPC in self.directives.drop or DROP_TOKEN in self.directives.drop:
definitions.append((EBNFCompiler.DROP_WHITESPACE_PARSER_KEYWORD,
'DropRegExp(%s)' % EBNFCompiler.WHITESPACE_KEYWORD))
definitions.append((EBNFCompiler.WHITESPACE_PARSER_KEYWORD,
......@@ -1340,16 +1340,18 @@ class EBNFCompiler(Compiler):
return 'Token('
def WSPC_PARSER(self):
if DROP_WSPC in self.directives.drop and (self.context[-2].tag_name != "definition"
or self.context[-1].tag_name == 'literal'):
def WSPC_PARSER(self, force_drop=False):
if ((force_drop or DROP_WSPC in self.directives.drop)
and (self.context[-2].tag_name != "definition"
or self.context[-1].tag_name == 'literal')):
return 'dwsp__'
return 'wsp__'
def on_literal(self, node: Node) -> str:
center = self.TOKEN_PARSER() + node.content.replace('\\', r'\\') + ')'
left = self.WSPC_PARSER() if 'left' in self.directives.literalws else ''
right = self.WSPC_PARSER() if 'right' in self.directives.literalws else ''
force = DROP_TOKEN in self.directives.drop
left = self.WSPC_PARSER(force) if 'left' in self.directives.literalws else ''
right = self.WSPC_PARSER(force) if 'right' in self.directives.literalws else ''
if left or right:
return 'Series(' + ", ".join(item for item in (left, center, right) if item) + ')'
return center
......
......@@ -65,6 +65,7 @@ __all__ = ('Parser',
'MetaParser',
'UnaryParser',
'NaryParser',
'Drop',
'Synonym',
'Option',
'ZeroOrMore',
......@@ -1371,19 +1372,6 @@ class Token(Parser):
return ("'%s'" if self.text.find("'") <= 0 else '"%s"') % self.text
class DropToken(Token):
"""
Parses play text string, but returns EMPTY_NODE rather than the parsed
string on a match. Violates the invariant: str(parse(text)) == text !
"""
def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
assert self.anonymous, "DropToken must not be used for named parsers!"
if text.startswith(self.text):
return EMPTY_NODE, text[self.len:]
# return Node(self.tag_name, self.text, True), text[self.len:]
return None, text
class RegExp(Parser):
r"""
Regular expression parser.
......@@ -2494,12 +2482,60 @@ class Pop(Retrieve):
########################################################################
#
# Aliasing parser classes
# Simplifying parser classes
#
########################################################################
# TODO: Add a generic Drop-Parser (see DropToken, DropWhitespace)
class Drop(UnaryParser):
r"""
Drops any content that another parser yields and returns either
None if the other parser did not match or EMPTY_NODE, if it did.
This allows to simplify the syntax tree at a very early stage.
Violates the invariant: str(parse(text)) == text !
"""
def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
node, text = self.parser.parse(text)
if node:
return EMPTY_NODE, text
return None, text
class DropToken(Token):
"""
Parses play text string, but returns EMPTY_NODE rather than the parsed
string on a match. Violates the invariant: str(parse(text)) == text !
"""
def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
assert self.anonymous, "DropToken must not be used for named parsers!"
if text.startswith(self.text):
return EMPTY_NODE, text[self.len:]
# return Node(self.tag_name, self.text, True), text[self.len:]
return None, text
class DropRegExp(Whitespace):
"""
Parses a text with a regular expression but never returns the match.
Instead EMPTY_NODE is returned on a match.
Violates the invariant: str(parse(text)) == text !
"""
def _parse(self, text: StringView) -> Tuple[Optional[Node], StringView]:
assert self.anonymous, "DropWhitespace must not be used for named parsers!"
match = text.match(self.regexp)
if match:
# capture = match.group(0)
end = text.index(match.end())
return EMPTY_NODE, text[end:]
return None, text
########################################################################
#
# Aliasing parser classes
#
########################################################################
class Synonym(UnaryParser):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment