Skip to content

GitLab

Commit 99ab9629 authored by eckhart's avatar eckhart
Browse files
Options

Merge branch 'transcrypt'

parents 59facd5c 31da9e96
Hide whitespace changes
Inline Side-by-side
Showing with 227 additions and 167 deletions
DHParser/parse.py
View file @ 99ab9629
......@@ -72,6 +72,7 @@ from typing import Any, Callable, cast, Dict, List, Set, Tuple, Union, Optional
__all__ = ('HistoryRecord',
'Parser',
'UnknownParserError',
'Grammar',
'PreprocessorToken',
'RegExp',
......@@ -526,6 +527,12 @@ def mixin_comment(whitespace: str, comment: str) -> str:
return wspc
class UnknownParserError(KeyError):
"""UnknownParserError is raised if a Grammer object is called with a
parser that does not exist or if in the course of parsing a parser
is reffered to that does not exist."""
class Grammar:
r"""
Class Grammar directs the parsing process and stores global state
......@@ -801,7 +808,7 @@ class Grammar:
parser.apply(self._add_parser__)
# assert self[key] == parser
return self[key]
raise KeyError('Unknown parser "%s" !' % key)
raise UnknownParserError('Unknown parser "%s" !' % key)
def _reset__(self):
......@@ -1004,8 +1011,8 @@ class Grammar:
if not is_logging():
raise AssertionError("Cannot log history when logging is turned off!")
assert self.history__, \
"Parser did not yet run or logging was turned off when running parser!"
# assert self.history__, \
# "Parser did not yet run or logging was turned off when running parser!"
if not log_file_name:
name = self.__class__.__name__
log_file_name = name[:-7] if name.lower().endswith('grammar') else name
......
DHParser/syntaxtree.py
View file @ 99ab9629
......@@ -35,6 +35,7 @@ __all__ = ('ParserBase',
'ZOMBIE_PARSER',
'Node',
'mock_syntax_tree',
'flatten_sxpr',
'TransformationFunc')
......
DHParser/testing.py
View file @ 99ab9629
......@@ -24,7 +24,8 @@ import json
import os
from DHParser.toolkit import is_logging, clear_logs, re
from DHParser.syntaxtree import mock_syntax_tree, flatten_sxpr
from DHParser.syntaxtree import Node, mock_syntax_tree, flatten_sxpr, ZOMBIE_PARSER
from DHParser.parse import UnknownParserError
from DHParser.error import is_error, adjust_error_locations
__all__ = ('unit_from_configfile',
......@@ -172,7 +173,10 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
if verbose:
infostr = ' match-test "' + test_name + '" ... '
errflag = len(errata)
cst = parser(test_code, parser_name)
try:
cst = parser(test_code, parser_name)
except UnknownParserError as upe:
cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
cst.log("match_%s_%s.cst" % (parser_name, test_name))
tests.setdefault('__cst__', {})[test_name] = cst
if "ast" in tests or report:
......@@ -210,7 +214,11 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
if verbose:
infostr = ' fail-test "' + test_name + '" ... '
errflag = len(errata)
cst = parser(test_code, parser_name)
# cst = parser(test_code, parser_name)
try:
cst = parser(test_code, parser_name)
except UnknownParserError as upe:
cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
if not is_error(cst.error_flag):
errata.append('Fail test "%s" for parser "%s" yields match instead of '
'expected failure!' % (test_name, parser_name))
......
DHParser/transform.py
View file @ 99ab9629
......@@ -266,18 +266,138 @@ def traverse(root_node: Node,
# assert processing_table['__cache__']
# ------------------------------------------------
#######################################################################
#
# rearranging transformations:
# - tree may be rearranged (e.g.flattened)
# - nodes that are not leaves may be dropped
# - order is preserved
# - leave content is preserved (though not necessarily the leaves themselves)
# meta transformations, i.e. transformations that call other
# transformations
#
# ------------------------------------------------
#######################################################################
def replace_by(node: Node, child: Node):
@transformation_factory(Dict)
def traverse_locally(context: List[Node],
processing_table: Dict, # actually: ProcessingTableType
key_func: Callable=key_tag_name): # actually: KeyFunc
"""Transforms the syntax tree starting from the last node in the context
according to the given processing table. The purpose of this function is
to apply certain transformations locally, i.e. only for those nodes that
have the last node in the context as their parent node.
"""
traverse(context[-1], processing_table, key_func)
# @transformation_factory(List[Callable])
# def apply_to_child(context: List[Node], transformations: List[Callable], condition: Callable):
# """Applies a list of transformations to those children that meet a specifc condition."""
# node = context[-1]
# for child in node.children:
# context.append(child)
# if condition(context):
# for transform in transformations:
# transform(context)
# context.pop()
@transformation_factory(Callable)
def apply_if(context: List[Node], transformation: Callable, condition: Callable):
"""Applies a transformation only if a certain condition is met."""
if condition(context):
transformation(context)
#######################################################################
#
# conditionals that determine whether the context (or the last node in
# the context for that matter) fulfill a specific condition.
# ---------------------------------------------------------------------
#
# The context of a node is understood as a list of all parent nodes
# leading up to and including the node itself. If represented as list,
# the last element of the list is the node itself.
#
#######################################################################
def is_single_child(context: List[Node]) -> bool:
return len(context[-2].children) == 1
def is_named(context: List[Node]) -> bool:
return bool(context[-1].parser.name)
def is_anonymous(context: List[Node]) -> bool:
return not context[-1].parser.name
def is_whitespace(context: List[Node]) -> bool:
"""Removes whitespace and comments defined with the
``@comment``-directive."""
return context[-1].parser.ptype == WHITESPACE_PTYPE
def is_empty(context: List[Node]) -> bool:
return not context[-1].result
def is_expendable(context: List[Node]) -> bool:
return is_empty(context) or is_whitespace(context)
@transformation_factory(AbstractSet[str])
def is_token(context: List[Node], tokens: AbstractSet[str] = frozenset()) -> bool:
"""Checks whether the last node in the context has `ptype == TOKEN_PTYPE`
and it's content matches one of the given tokens. Leading and trailing
whitespace-tokens will be ignored. In case an empty set of tokens is passed,
any token is a match. If only ":" is given all anonymous tokens but no other
tokens are a match.
"""
def stripped(nd: Node) -> str:
# assert node.parser.ptype == TOKEN_PTYPE
if nd.children:
i, k = 0, len(nd.children)
while i < len(nd.children) and nd.children[i].parser.ptype == WHITESPACE_PTYPE:
i += 1
while k > 0 and nd.children[k-1].parser.ptype == WHITESPACE_PTYPE:
k -= 1
return "".join(child.content for child in node.children[i:k])
return nd.content
node = context[-1]
return (node.parser.ptype == TOKEN_PTYPE
and ((not tokens or stripped(node) in tokens)
or (not node.parser.name and len(tokens) == 1 and ":" in tokens)))
@transformation_factory(AbstractSet[str])
def is_one_of(context: List[Node], tag_name_set: AbstractSet[str]) -> bool:
"""Returns true, if the node's tag_name is one of the given tag names."""
return context[-1].tag_name in tag_name_set
@transformation_factory(str)
def has_content(context: List[Node], regexp: str) -> bool:
"""Checks a node's content against a regular expression."""
return bool(re.match(regexp, context[-1].content))
@transformation_factory(AbstractSet[str])
def has_parent(context: List[Node], tag_name_set: AbstractSet[str]) -> bool:
"""Checks whether a node with one of the given tag names appears somewhere
in the context before the last node in the context."""
for i in range(2, len(context)):
if context[-i].tag_name in tag_name_set:
return True
return False
#######################################################################
#
# utility functions (private)
#
#######################################################################
def _replace_by(node: Node, child: Node):
if not child.parser.name:
child.parser = MockParser(node.parser.name, child.parser.ptype)
# parser names must not be overwritten, else: child.parser.name = node.parser.name
......@@ -286,12 +406,12 @@ def replace_by(node: Node, child: Node):
node.result = child.result
def reduce_child(node: Node, child: Node):
def _reduce_child(node: Node, child: Node):
node._errors.extend(child._errors)
node.result = child.result
def pick_child(context: List[Node], criteria: CriteriaType):
def _pick_child(context: List[Node], criteria: CriteriaType):
"""Returns the first child that meets the criteria."""
if isinstance(criteria, int):
try:
......@@ -313,44 +433,52 @@ def pick_child(context: List[Node], criteria: CriteriaType):
return None
def single_child(context: List[Node]) -> bool:
return len(context[-2].children) == 1
@transformation_factory(int, str, Callable)
def replace_by_child(context: List[Node], criteria: CriteriaType=single_child):
"""
Replaces a node by the first of its immediate descendants
that meets the `criteria`. The criteria can either be the
index of the child (counting from zero), or the tag name or
a boolean-valued function on the context of the child.
If no child matching the criteria is found, the node will
not be replaced.
With the default value for `criteria` the same semantics is
the same that of `replace_by_single_child`.
"""
child = pick_child(context, criteria)
if child:
replace_by(context[-1], child)
@transformation_factory(int, str, Callable)
def content_from_child(context: List[Node], criteria: CriteriaType = single_child):
"""
Reduces a node, by transferring the result of the first of its
immediate descendants that meets the `criteria` to this node,
but keeping this node's parser entry. The criteria can either
be the index of the child (counting from zero), or the tag
name or a boolean-valued function on the context of the child.
If no child matching the criteria is found, the node will
not be replaced.
With the default value for `criteria` this has the same semantics
as `content_from_single_child`.
"""
child = pick_child(context, criteria)
if child:
reduce_child(context[-1], child)
#######################################################################
#
# rearranging transformations
#
# - tree may be rearranged (e.g.flattened)
# - nodes that are not leaves may be dropped
# - order is preserved
# - leave content is preserved (though not necessarily the leaves
# themselves)
#
#######################################################################
# @transformation_factory(int, str, Callable)
# def replace_by_child(context: List[Node], criteria: CriteriaType=is_single_child):
# """
# Replaces a node by the first of its immediate descendants
# that meets the `criteria`. The criteria can either be the
# index of the child (counting from zero), or the tag name or
# a boolean-valued function on the context of the child.
# If no child matching the criteria is found, the node will
# not be replaced.
# With the default value for `criteria` the same semantics is
# the same that of `replace_by_single_child`.
# """
# child = _pick_child(context, criteria)
# if child:
# _replace_by(context[-1], child)
#
#
# @transformation_factory(int, str, Callable)
# def content_from_child(context: List[Node], criteria: CriteriaType = is_single_child):
# """
# Reduces a node, by transferring the result of the first of its
# immediate descendants that meets the `criteria` to this node,
# but keeping this node's parser entry. The criteria can either
# be the index of the child (counting from zero), or the tag
# name or a boolean-valued function on the context of the child.
# If no child matching the criteria is found, the node will
# not be replaced.
# With the default value for `criteria` this has the same semantics
# as `content_from_single_child`.
# """
# child = _pick_child(context, criteria)
# if child:
# _reduce_child(context[-1], child)
def replace_by_single_child(context: List[Node]):
......@@ -361,7 +489,7 @@ def replace_by_single_child(context: List[Node]):
"""
node = context[-1]
if len(node.children) == 1:
replace_by(node, node.children[0])
_replace_by(node, node.children[0])
def reduce_single_child(context: List[Node]):
......@@ -373,15 +501,7 @@ def reduce_single_child(context: List[Node]):
"""
node = context[-1]
if len(node.children) == 1:
reduce_child(node, node.children[0])
def is_named(context: List[Node]) -> bool:
return bool(context[-1].parser.name)
def is_anonymous(context: List[Node]) -> bool:
return not context[-1].parser.name
_reduce_child(node, node.children[0])
@transformation_factory(Callable)
......@@ -394,9 +514,9 @@ def replace_or_reduce(context: List[Node], condition: Callable=is_named):
if len(node.children) == 1:
child = node.children[0]
if condition(context):
replace_by(node, child)
_replace_by(node, child)
else:
reduce_child(node, child)
_reduce_child(node, child)
@transformation_factory
......@@ -457,9 +577,9 @@ def collapse(context: List[Node]):
@transformation_factory
def merge_children(context: List[Node], tag_names: List[str]):
"""
Joins all children next to each other and with particular tag-
names into a single child node with a mock-parser with the name of
the first tag-name in the list.
Joins all children next to each other and with particular tag-names
into a single child node with a mock-parser with the name of the
first tag-name in the list.
"""
node = context[-1]
result = []
......@@ -483,17 +603,6 @@ def merge_children(context: List[Node], tag_names: List[str]):
node.result = tuple(result)
# ------------------------------------------------
#
# destructive transformations:
# - tree may be rearranged (flattened),
# - order is preserved
# - but (irrelevant) leaves may be dropped
# - errors of dropped leaves will be lost
#
# ------------------------------------------------
@transformation_factory
def replace_content(context: List[Node], func: Callable): # Callable[[Node], ResultType]
"""Replaces the content of the node. ``func`` takes the node
......@@ -503,18 +612,15 @@ def replace_content(context: List[Node], func: Callable): # Callable[[Node], Re
node.result = func(node.result)
def is_whitespace(context: List[Node]) -> bool:
"""Removes whitespace and comments defined with the
``@comment``-directive."""
return context[-1].parser.ptype == WHITESPACE_PTYPE
def is_empty(context: List[Node]) -> bool:
return not context[-1].result
def is_expendable(context: List[Node]) -> bool:
return is_empty(context) or is_whitespace(context)
#######################################################################
#
# destructive transformations:
#
# - leaves may be dropped (e.g. if deemed irrelevant)
# - errors of dropped leaves will be lost
# - no promise that order will be preserved
#
#######################################################################
@transformation_factory(Callable)
......@@ -553,80 +659,6 @@ def strip(context: List[Node], condition: Callable = is_expendable) -> str:
rstrip(context, condition)
@transformation_factory(AbstractSet[str])
def is_token(context: List[Node], tokens: AbstractSet[str] = frozenset()) -> bool:
"""Checks whether the last node in the context is has `ptype == TOKEN_PTYPE`
and it's content without leading or trailing whitespace child-nodes
matches one of the given tokens. If no tokens are given, any token is a match.
If only ":" is given all anonymous tokens but no other tokens are a match.
"""
def stripped(nd: Node) -> str:
# assert node.parser.ptype == TOKEN_PTYPE
if nd.children:
i, k = 0, len(nd.children)
while i < len(nd.children) and nd.children[i].parser.ptype == WHITESPACE_PTYPE:
i += 1
while k > 0 and nd.children[k-1].parser.ptype == WHITESPACE_PTYPE:
k -= 1
return "".join(child.content for child in node.children[i:k])
return nd.content
node = context[-1]
return (node.parser.ptype == TOKEN_PTYPE
and ((not tokens or stripped(node) in tokens)
or (not node.parser.name and len(tokens) == 1 and ":" in tokens)))
@transformation_factory(AbstractSet[str])
def is_one_of(context: List[Node], tag_name_set: AbstractSet[str]) -> bool:
"""Returns true, if the node's tag_name is on of the
given tag names."""
return context[-1].tag_name in tag_name_set
@transformation_factory(str)
def has_content(context: List[Node], regexp: str) -> bool:
"""Checks a node's content against a regular expression."""
return bool(re.match(regexp, context[-1].content))
@transformation_factory(AbstractSet[str])
def has_parent(context: List[Node], tag_name_set: AbstractSet[str]) -> bool:
"""Checks whether a node with one of the given tag names appears somewhere
in the context before the last node in the context."""
for i in range(2, len(context)):
if context[-i].tag_name in tag_name_set:
return True
return False
@transformation_factory(Callable)
def apply_if(context: List[Node], transformation: Callable, condition: Callable):
"""Applies a transformation only if a certain condition is met."""
if condition(context):
transformation(context)
# @transformation_factory(List[Callable])
# def apply_to_child(context: List[Node], transformations: List[Callable], condition: Callable):
# """Applies a list of transformations to those children that meet a specifc condition."""
# node = context[-1]
# for child in node.children:
# context.append(child)
# if condition(context):
# for transform in transformations:
# transform(context)
# context.pop()
@transformation_factory(Dict)
def traverse_locally(context: List[Node],
processing_table: Dict, # actually: ProcessingTableType
key_func: Callable=key_tag_name): # actually: KeyFunc
"""Transforms the syntax tree starting from the last node in the context
according to the given processing table. The purpose of this function is
to apply certain transformations locally, i.e. only for those nodes that
have the last node in the context as their parent node.
"""
traverse(context[-1], processing_table, key_func)
@transformation_factory(slice)
def keep_children(context: List[Node], section: slice = slice(None)):
"""Keeps only child-nodes which fall into a slice of the result field."""
......@@ -698,6 +730,7 @@ remove_whitespace = remove_children_if(is_whitespace) # partial(remove_children
remove_empty = remove_children_if(is_empty)
remove_anonymous_empty = remove_children_if(lambda ctx: is_empty(ctx) and is_anonymous(ctx))
remove_expendables = remove_children_if(is_expendable) # partial(remove_children_if, condition=is_expendable)
remove_anonymous_expendables = remove_children_if(lambda ctx: is_anonymous(ctx) and is_expendable(ctx))
remove_first = apply_if(keep_children(slice(1, None)), lambda ctx: len(ctx[-1].children) > 1)
remove_last = apply_if(keep_children(slice(None, -1)), lambda ctx: len(ctx[-1].children) > 1)
remove_brackets = apply_if(keep_children(slice(1, -1)), lambda ctx: len(ctx[-1].children) >= 2)
......
test/test_parse.py
View file @ 99ab9629
......@@ -29,7 +29,8 @@ from DHParser.stringview import StringView
from DHParser.error import Error
from DHParser.syntaxtree import mock_syntax_tree
from DHParser.parse import compile_source, Retrieve, Grammar, Forward, Token, ZeroOrMore, RE, \
RegExp, Lookbehind, NegativeLookahead, OneOrMore, Series, Alternative, AllOf, SomeOf, Compiler
RegExp, Lookbehind, NegativeLookahead, OneOrMore, Series, Alternative, AllOf, SomeOf, Compiler, \
UnknownParserError
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import grammar_provider, DHPARSER_IMPORTS
......@@ -562,6 +563,16 @@ class TestCompilerClass:
assert D.error_flag
class TestUnknownParserError:
def test_unknown_parser_error(self):
gr = Grammar()
try:
gr("", "NonExistantParser")
assert False, "UnknownParserError expected!"
except UnknownParserError:
pass
if __name__ == "__main__":
from DHParser.testing import runner
with logging(False):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment