Commit 7f34f386 authored by eckhart's avatar eckhart

Bereinigung von Typenfehlern

parent 3478a3af
No preview for this file type
......@@ -41,7 +41,7 @@ from typing import Any, Optional, Tuple, List, Set, Union, Callable, cast
from DHParser.configuration import get_config_value
from DHParser.preprocess import with_source_mapping, PreprocessorFunc, SourceMapFunc
from DHParser.syntaxtree import Node, RootNode, ZOMBIE_TAG, StrictResultType
from DHParser.syntaxtree import Node, RootNode
from DHParser.transform import TransformationFunc
from DHParser.parse import Grammar
from DHParser.error import adjust_error_locations, is_error, is_fatal, Error
......@@ -411,5 +411,6 @@ def process_tree(tp: TreeProcessor, tree: RootNode) -> RootNode:
return tree
# TODO: Verify compiler against grammar, i.e. make sure that for all on_X()-methods, `X` is the name of a parser
# TODO: Verify compiler against grammar,
# i.e. make sure that for all on_X()-methods, `X` is the name of a parser
# TODO: AST validation against an ASDSL-Specification
......@@ -297,7 +297,7 @@ def load_compiler_suite(compiler_suite: str) -> \
RX_SECTION_MARKER.split(source)
except ValueError:
raise ValueError('File "' + compiler_suite + '" seems to be corrupted. '
'Please delete or repair file manually.')
'Please delete or repair file manually.')
# TODO: Compile in one step and pick parts from namespace later ?
preprocessor = compile_python_object(imports + preprocessor_py,
r'get_(?:\w+_)?preprocessor$')
......@@ -412,17 +412,17 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
sfactory, pfactory, tfactory, cfactory = load_compiler_suite(compiler_suite)
compiler1 = cfactory()
else:
sfactory = get_ebnf_preprocessor # type: PreprocessorFactoryFunc
pfactory = get_ebnf_grammar # type: ParserFactoryFunc
tfactory = get_ebnf_transformer # type: TransformerFactoryFunc
cfactory = get_ebnf_compiler # type: CompilerFactoryFunc
compiler1 = cfactory() # type: Compiler
sfactory = get_ebnf_preprocessor # PreprocessorFactoryFunc
pfactory = get_ebnf_grammar # ParserFactoryFunc
tfactory = get_ebnf_transformer # TransformerFactoryFunc
cfactory = get_ebnf_compiler # CompilerFactoryFunc
compiler1 = cfactory() # Compiler
is_ebnf_compiler = False # type: bool
if isinstance(compiler1, EBNFCompiler):
is_ebnf_compiler = True
compiler1.set_grammar_name(compiler_name, source_file)
result, messages, _ = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
if has_errors(messages):
......
......@@ -780,7 +780,8 @@ class EBNFCompiler(Compiler):
expression. Makes sure that multi-line regular expressions are
prepended by the multi-line-flag. Returns the regular expression string.
"""
# TODO: Support atomic grouping: https://stackoverflow.com/questions/13577372/do-python-regular-expressions-have-an-equivalent-to-rubys-atomic-grouping
# TODO: Support atomic grouping: https://stackoverflow.com/questions/13577372/
# do-python-regular-expressions-have-an-equivalent-to-rubys-atomic-grouping
flags = self.re_flags | {'x'} if rx.find('\n') >= 0 else self.re_flags
if flags:
rx = "(?%s)%s" % ("".join(flags), rx)
......
......@@ -63,7 +63,8 @@ from DHParser.stringview import StringView
from DHParser.syntaxtree import Node, FrozenNode, ZOMBIE_TAG, EMPTY_PTYPE
from DHParser.toolkit import escape_control_characters, abbreviate_middle
__all__ = ('start_logging',
__all__ = ('CallItem',
'start_logging',
'suspend_logging',
'resume_logging',
'log_dir',
......@@ -79,6 +80,9 @@ __all__ = ('start_logging',
'log_parsing_history')
CallItem = Tuple[str, int] # call stack item: (tag_name, location)
#######################################################################
#
# basic logging functionality
......@@ -247,9 +251,9 @@ NONE_TAG = ":None"
NONE_NODE = FrozenNode(NONE_TAG, '')
def freeze_callstack(call_stack: List[Tuple[str, int]]) -> Tuple[Tuple[str, int], ...]:
def freeze_callstack(call_stack: List[CallItem]) -> Tuple[CallItem, ...]:
"""Returns a frozen copy of the call stack."""
return tuple((tn, pos) for tn, pos in call_stack if tn != ":Forward")
return tuple((tn, pos) for tn, pos in call_stack if tn != ":Forward")
class HistoryRecord:
......@@ -298,15 +302,15 @@ class HistoryRecord:
'</style>\n</head>\n<body>\n')
HTML_LEAD_OUT = '\n</body>\n</html>\n'
def __init__(self, call_stack: Union[List[Tuple[str, int]], Tuple[Tuple[str, int], ...]],
def __init__(self, call_stack: Union[List[CallItem], Tuple[CallItem, ...]],
node: Optional[Node],
text: StringView,
line_col: Tuple[int, int],
errors: List[Error] = []) -> None:
# copy call stack, dropping uninformative Forward-Parsers
# self.call_stack = call_stack # type: Tuple[Tuple[str, int],...]
# self.call_stack = call_stack # type: Tuple[CallItem,...]
if isinstance(call_stack, tuple):
self.call_stack = call_stack # type: Tuple[Tuple[str, int],...]
self.call_stack = call_stack # type: Tuple[CallItem,...]
else:
self.call_stack = freeze_callstack(call_stack)
self.node = NONE_NODE if node is None else node # type: Node
......
......@@ -37,7 +37,7 @@ from typing import Callable, cast, List, Tuple, Set, Dict, \
from DHParser.configuration import get_config_value
from DHParser.error import Error, linebreaks, line_col
from DHParser.log import HistoryRecord
from DHParser.log import CallItem, HistoryRecord
from DHParser.preprocess import BEGIN_TOKEN, END_TOKEN, RX_TOKEN_NAME
from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import ChildrenType, Node, RootNode, WHITESPACE_PTYPE, \
......@@ -117,6 +117,7 @@ class ParserError(Exception):
self.rest = rest # type: StringView
self.error = error # type: Error
self.first_throw = first_throw # type: bool
self.frozen_callstack = tuple() # type: Tuple[CallItem, ...] # tag_name, location
def __str__(self):
return "%i: %s %s" % (self.node.pos, str(self.rest[:25]), repr(self.node))
......@@ -511,8 +512,8 @@ class Parser:
if proxy is None:
self._parse_proxy = self._parse
else:
if type(proxy) != type(self._parse):
# assume that proxy is a function
if not isinstance(proxy, type(self._parse)):
# assume that proxy is a function and bind it to self
proxy = proxy.__get__(self, type(self))
else:
# if proxy is a method it must be a method of self
......@@ -536,8 +537,8 @@ class Parser:
self._grammar = grammar
# self._grammar_assigned_notifier()
elif self._grammar != grammar:
raise AssertionError("Parser has already been assigned"
"to a different Grammar object!")
raise AssertionError("Parser has already been assigned"
"to a different Grammar object!")
except AttributeError:
pass # ignore setting of grammar attribute for placeholder parser
except NameError: # Cython: No access to GRAMMA_PLACEHOLDER, yet :-(
......@@ -549,8 +550,8 @@ class Parser:
"""
return tuple()
def _apply(self, func: Callable[['Parser'], None],
flip: Callable[[Callable, Set[Callable]], bool]) -> bool:
def _apply(self, func: Callable[['Parser'], None],
flip: Callable[[Callable, Set[Callable]], bool]) -> bool:
"""
Applies function `func(parser)` recursively to this parser and all
descendant parsers, if any exist.
......@@ -622,6 +623,11 @@ def Drop(parser: Parser) -> Parser:
PARSER_PLACEHOLDER = Parser()
def is_parser_placeholder(parser: Optional[Parser]) -> bool:
"""Returns True, if `parser` is `None` or merely a placeholder for a parser."""
return not parser or parser.ptype == ":Parser"
########################################################################
#
# Grammar class, central administration of all parser of a grammar
......@@ -1070,7 +1076,7 @@ class Grammar:
self.rollback__ = [] # type: List[Tuple[int, Callable]]
self.last_rb__loc__ = -1 # type: int
# support for call stack tracing
self.call_stack__ = [] # type: List[Tuple[str, int]] # tag_name, location
self.call_stack__ = [] # type: List[CallItem] # tag_name, location
# snapshots of call stacks
self.history__ = [] # type: List[HistoryRecord]
# also needed for call stack tracing
......@@ -1990,7 +1996,6 @@ class Series(MandatoryNary):
text_, isinstance(parser, Lookahead), parser.repr, reloc)
# check if parsing of the series can be resumed somewhere
if reloc >= 0:
rest = text_
nd, text_ = parser(text_) # try current parser again
if nd is not None:
results += (node,)
......@@ -2017,7 +2022,7 @@ class Series(MandatoryNary):
# `RE('\d+') + Optional(RE('\.\d+)` instead of `Series(RE('\d+'), Optional(RE('\.\d+))`
@staticmethod
def combined_mandatory(left: 'Series', right: 'Series'):
def combined_mandatory(left: Parser, right: Parser) -> int:
"""
Returns the position of the first mandatory element (if any) when
parsers `left` and `right` are joined to a sequence.
......@@ -2630,7 +2635,7 @@ class Forward(Parser):
def __init__(self):
super(Forward, self).__init__()
self.parser = None # type: Optional[Parser]
self.parser = PARSER_PLACEHOLDER # type: Parser
self.cycle_reached = False
def __deepcopy__(self, memo):
......@@ -2641,8 +2646,7 @@ class Forward(Parser):
memo[id(self)] = duplicate
parser = copy.deepcopy(self.parser, memo)
duplicate.parser = parser
if parser is not None:
duplicate.drop_content = parser.drop_content
duplicate.drop_content = parser.drop_content
return duplicate
def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]:
......@@ -2696,6 +2700,6 @@ class Forward(Parser):
self.drop_content = parser.drop_content
def sub_parsers(self) -> Tuple[Parser, ...]:
if self.parser is not None:
return (self.parser,)
return tuple()
if is_parser_placeholder(self.parser):
return tuple()
return (self.parser,)
This diff is collapsed.
......@@ -32,7 +32,7 @@ speedup. The modules comes with a ``stringview.pxd`` that contains some type
declarations to more fully exploit the benefits of the Cython-compiler.
"""
from typing import Optional, Union, Iterable, Tuple, List
from typing import Optional, Union, Iterable, Tuple, List, Sequence
try:
import cython
......@@ -340,7 +340,7 @@ class StringView: # collections.abc.Sized
return self if end == self._len else self[:end]
@cython.locals(length=cython.int, i=cython.int, k=cython.int)
def split(self, sep=None) -> List[Union['StringView', str]]:
def split(self, sep=None) -> Sequence[Union['StringView', str]]:
"""Returns a list of the words in `self`, using `sep` as the
delimiter string. If `sep` is not specified or is None, any
whitespace string is a separator and empty strings are
......
......@@ -258,8 +258,8 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
__slots__ = '_result', 'children', '_pos', 'tag_name', '_xml_attr'
def __init__(self, tag_name: str,
result: Union[Tuple['Node', ...], 'Node', StringView, str],
def __init__(self, tag_name: str,
result: Union[Tuple['Node', ...], 'Node', StringView, str],
leafhint: bool = False) -> None:
"""
Initializes the ``Node``-object with the ``Parser``-Instance
......@@ -367,7 +367,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
:param result: the new result of the note
"""
if isinstance(result, Node):
self.children = (result,) # type: Tuple[Node, ...]
self.children = (result,)
self._result = self.children
else:
if isinstance(result, tuple):
......@@ -1569,7 +1569,8 @@ def parse_sxpr(sxpr: Union[str, StringView]) -> Node:
attributes[attr] = str(value)
sxpr = sxpr[k + 1:].strip()
if sxpr[0] == '(':
result = tuple(inner_parser(block) for block in next_block(sxpr)) # type: Union[Tuple[Node, ...], str]
result = tuple(inner_parser(block)
for block in next_block(sxpr)) # type: Union[Tuple[Node, ...], str]
else:
lines = []
while sxpr and sxpr[0:1] != ')':
......@@ -1688,7 +1689,7 @@ def parse_xml(xml: Union[str, StringView], ignore_pos: bool = False) -> Node:
if len(res) == 1 and res[0].tag_name == TOKEN_PTYPE:
result = res[0].result # type: Union[Tuple[Node, ...], StringView, str]
else:
result = tuple(res) # type:Union[Tuple[Node, ...], StringView, str]
result = tuple(res)
node = Node(name or ':' + class_name, result)
if not ignore_pos and '_pos' in attrs:
......
......@@ -370,7 +370,7 @@ def compile_python_object(python_src: str, catch_obj_regex="") -> Any:
if isinstance(catch_obj_regex, str):
catch_obj_regex = re.compile(catch_obj_regex)
code = compile(python_src, '<string>', 'exec')
namespace = {}
namespace = {} # type: Dict[str, Any]
exec(code, namespace) # safety risk?
if catch_obj_regex:
matches = [key for key in namespace if catch_obj_regex.match(key)]
......
......@@ -48,9 +48,9 @@ def trace_history(self: Parser, text: StringView) -> Tuple[Optional[Node], Strin
errors = [pe.error]
# ignore inflated length due to gap jumping (see parse.Parser.__call__)
l = sum(len(nd) for nd in pe.node.select_if(lambda n: True, include_root=True)
if not nd.children and nd.tag_name != ZOMBIE_TAG)
text_ = pe.rest[l:]
loc = sum(len(nd) for nd in pe.node.select_if(lambda n: True, include_root=True)
if not nd.children and nd.tag_name != ZOMBIE_TAG)
text_ = pe.rest[loc:]
lc = line_col(grammar.document_lbreaks__, pe.error.pos)
target = text
if len(target) >= 10:
......@@ -83,8 +83,8 @@ def trace_history(self: Parser, text: StringView) -> Tuple[Optional[Node], Strin
if pe.first_throw:
pe.frozen_callstack = freeze_callstack(grammar.call_stack__)
grammar.most_recent_error__ = pe
if self == grammar.start_parser__:
fe = grammar.most_recent_error__
if self == grammar.start_parser__ and grammar.most_recent_error__:
fe = grammar.most_recent_error__ # type: ParserError
lc = line_col(grammar.document_lbreaks__, fe.error.pos)
# TODO: get the call stack from when the error occured, here
nd = fe.node
......@@ -101,9 +101,9 @@ def trace_history(self: Parser, text: StringView) -> Tuple[Optional[Node], Strin
# record history
# TODO: Make dropping insignificant whitespace from history configurable
delta = text._len - rest._len
nd = Node(node.tag_name, text[:delta]).with_pos(location) if node else None
hnd = Node(node.tag_name, text[:delta]).with_pos(location) if node else None
lc = line_col(grammar.document_lbreaks__, location)
record = HistoryRecord(grammar.call_stack__, nd, rest, lc, [])
record = HistoryRecord(grammar.call_stack__, hnd, rest, lc, [])
cs_len = len(record.call_stack)
if (not grammar.history__ or lc != grammar.history__[-1].line_col
or record.call_stack != grammar.history__[-1].call_stack[:cs_len]
......
......@@ -27,7 +27,6 @@ for CST -> AST transformations.
"""
from collections import OrderedDict
import collections.abc
from functools import partial, singledispatch, reduce
import inspect
......@@ -37,7 +36,7 @@ from typing import AbstractSet, Any, ByteString, Callable, cast, Container, Dict
from DHParser.error import Error, ErrorCode
from DHParser.syntaxtree import Node, WHITESPACE_PTYPE, TOKEN_PTYPE, LEAF_PTYPES, PLACEHOLDER, \
RootNode, parse_sxpr, flatten_sxpr
RootNode
from DHParser.toolkit import issubtype, isgenerictype, expand_table, smart_list, re, cython
......
......@@ -93,8 +93,7 @@ class TestTrace:
gr = grammar_provider(lang)()
all_desc = all_descendants(gr.root_parser__)
set_tracer(all_desc, trace_history)
st = gr('2*(3+4)xxx')
# print(st.as_sxpr(compact=True))
_ = gr('2*(3+4)xxx')
log_parsing_history(gr, 'trace_stopped_early')
history = get_history('trace_stopped_early')
assert history.count('<tr>') == 26
......@@ -236,6 +235,7 @@ class TestErrorReporting:
block_A = "a" §"b" "c"
block_B = "x" "y" "z"
"""
def mini_suite(grammar):
tree = grammar('abc/*x*/xyz')
assert not tree.errors
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment