Commit d702fc24 authored by eckhart's avatar eckhart

Corrections of mypy-TypeErrors

parent 4b248f94
...@@ -39,7 +39,7 @@ import os ...@@ -39,7 +39,7 @@ import os
import re import re
from DHParser.preprocess import strip_tokens, with_source_mapping, PreprocessorFunc from DHParser.preprocess import strip_tokens, with_source_mapping, PreprocessorFunc
from DHParser.syntaxtree import Node, RootNode, StrictResultType from DHParser.syntaxtree import Node, RootNode, ZOMBIE_ROOTNODE, StrictResultType
from DHParser.transform import TransformationFunc from DHParser.transform import TransformationFunc
from DHParser.parse import Grammar from DHParser.parse import Grammar
from DHParser.error import adjust_error_locations, is_error, Error from DHParser.error import adjust_error_locations, is_error, Error
...@@ -97,7 +97,7 @@ class Compiler: ...@@ -97,7 +97,7 @@ class Compiler:
self._reset() self._reset()
def _reset(self): def _reset(self):
self.tree = None # type: Optional[RootNode] self.tree = ZOMBIE_ROOTNODE # type: RootNode
self.context = [] # type: List[Node] self.context = [] # type: List[Node]
self._dirty_flag = False self._dirty_flag = False
...@@ -116,7 +116,7 @@ class Compiler: ...@@ -116,7 +116,7 @@ class Compiler:
result = self.compile(root) result = self.compile(root)
return result return result
def set_grammar_name(self, grammar_name: str="", grammar_source: str=""): def set_grammar_name(self, grammar_name: str = "", grammar_source: str = ""):
""" """
Changes the grammar's name and the grammar's source. Changes the grammar's name and the grammar's source.
...@@ -219,7 +219,7 @@ def compile_source(source: str, ...@@ -219,7 +219,7 @@ def compile_source(source: str,
parser: Grammar, # str -> Node (concrete syntax tree (CST)) parser: Grammar, # str -> Node (concrete syntax tree (CST))
transformer: TransformationFunc, # Node (CST) -> Node (abstract syntax tree (AST)) transformer: TransformationFunc, # Node (CST) -> Node (abstract syntax tree (AST))
compiler: Compiler, # Node (AST) -> Any compiler: Compiler, # Node (AST) -> Any
preserve_ast: bool = False) -> Tuple[Any, List[Error], Node]: preserve_ast: bool = False) -> Tuple[Optional[Any], List[Error], Optional[Node]]:
""" """
Compiles a source in four stages: Compiles a source in four stages:
1. Pre-Processing (if needed) 1. Pre-Processing (if needed)
...@@ -259,7 +259,7 @@ def compile_source(source: str, ...@@ -259,7 +259,7 @@ def compile_source(source: str,
source_mapping = lambda i: i source_mapping = lambda i: i
else: else:
source_text, source_mapping = with_source_mapping(preprocessor(original_text)) source_text, source_mapping = with_source_mapping(preprocessor(original_text))
syntax_tree = parser(source_text) syntax_tree = parser(source_text) # type: RootNode
if is_logging(): if is_logging():
log_ST(syntax_tree, log_file_name + '.cst') log_ST(syntax_tree, log_file_name + '.cst')
log_parsing_history(parser, log_file_name) log_parsing_history(parser, log_file_name)
......
...@@ -247,6 +247,7 @@ EBNF_AST_transformation_table = { ...@@ -247,6 +247,7 @@ EBNF_AST_transformation_table = {
def EBNFTransform() -> TransformationFunc: def EBNFTransform() -> TransformationFunc:
return partial(traverse, processing_table=EBNF_AST_transformation_table.copy()) return partial(traverse, processing_table=EBNF_AST_transformation_table.copy())
def get_ebnf_transformer() -> TransformationFunc: def get_ebnf_transformer() -> TransformationFunc:
global thread_local_EBNF_transformer_singleton global thread_local_EBNF_transformer_singleton
try: try:
...@@ -550,11 +551,11 @@ class EBNFCompiler(Compiler): ...@@ -550,11 +551,11 @@ class EBNFCompiler(Compiler):
# add EBNF grammar to the doc string of the parser class # add EBNF grammar to the doc string of the parser class
article = 'an ' if self.grammar_name[0:1] in "AaEeIiOoUu" else 'a ' # what about 'hour', 'universe' etc.? article = 'an ' if self.grammar_name[0:1] in "AaEeIiOoUu" else 'a ' # what about 'hour', 'universe' etc.?
declarations = ['class ' + self.grammar_name + declarations = ['class ' + self.grammar_name
'Grammar(Grammar):', + 'Grammar(Grammar):',
'r"""Parser for ' + article + self.grammar_name + 'r"""Parser for ' + article + self.grammar_name
' source file' + + ' source file'
(', with this grammar:' if self.grammar_source else '.')] + (', with this grammar:' if self.grammar_source else '.')]
definitions.append(('parser_initialization__', '"upon instantiation"')) definitions.append(('parser_initialization__', '"upon instantiation"'))
if self.grammar_source: if self.grammar_source:
definitions.append(('source_hash__', definitions.append(('source_hash__',
...@@ -833,7 +834,7 @@ class EBNFCompiler(Compiler): ...@@ -833,7 +834,7 @@ class EBNFCompiler(Compiler):
# shift = (Node(node.parser, node.result[1].result),) # shift = (Node(node.parser, node.result[1].result),)
# node.result[1].result = shift + node.result[2:] # node.result[1].result = shift + node.result[2:]
node.children[1].result = (Node(node.children[1].parser, node.children[1].result),) \ node.children[1].result = (Node(node.children[1].parser, node.children[1].result),) \
+ node.children[2:] + node.children[2:]
node.children[1].parser = node.parser node.children[1].parser = node.parser
node.result = (node.children[0], node.children[1]) node.result = (node.children[0], node.children[1])
......
...@@ -55,7 +55,7 @@ import os ...@@ -55,7 +55,7 @@ import os
from DHParser.error import line_col from DHParser.error import line_col
from DHParser.stringview import StringView from DHParser.stringview import StringView
from DHParser.syntaxtree import Node, WHITESPACE_PTYPE from DHParser.syntaxtree import Node
from DHParser.toolkit import is_filename, escape_control_characters, typing from DHParser.toolkit import is_filename, escape_control_characters, typing
from typing import List, Tuple, Union from typing import List, Tuple, Union
...@@ -210,7 +210,7 @@ class HistoryRecord: ...@@ -210,7 +210,7 @@ class HistoryRecord:
HTML_LEAD_IN = ('<!DOCTYPE html>\n' HTML_LEAD_IN = ('<!DOCTYPE html>\n'
'<html>\n<head>\n<meta charset="utf-8"/>\n<style>\n' '<html>\n<head>\n<meta charset="utf-8"/>\n<style>\n'
'td,th {font-family:monospace; ' 'td,th {font-family:monospace; '
'border-right: thin solid grey; border-bottom: thin solid grey}\n' 'border-right: thin solid grey; border-bottom: thin solid grey}\n'
'td.line, td.column {color:darkgrey}\n' # 'td.stack {}\n' 'td.line, td.column {color:darkgrey}\n' # 'td.stack {}\n'
'td.status {font-weight:bold}\n' 'td.status {font-weight:bold}\n'
'td.text {color:darkblue}\n' 'td.text {color:darkblue}\n'
...@@ -236,7 +236,7 @@ class HistoryRecord: ...@@ -236,7 +236,7 @@ class HistoryRecord:
def __str__(self): def __str__(self):
return '%4i, %2i: %s; %s; "%s"' % self.as_tuple() return '%4i, %2i: %s; %s; "%s"' % self.as_tuple()
def as_tuple(self) -> Snapshot: def as_tuple(self) -> 'Snapshot':
""" """
Returns history record formatted as a snapshot tuple. Returns history record formatted as a snapshot tuple.
""" """
...@@ -294,7 +294,6 @@ class HistoryRecord: ...@@ -294,7 +294,6 @@ class HistoryRecord:
def status(self) -> str: def status(self) -> str:
return self.FAIL if self.node is None else \ return self.FAIL if self.node is None else \
('"%s"' % self.err_msg()) if self.node.errors else self.MATCH ('"%s"' % self.err_msg()) if self.node.errors else self.MATCH
# has_errors(self.node._errors)
@property @property
def excerpt(self): def excerpt(self):
...@@ -344,8 +343,8 @@ class HistoryRecord: ...@@ -344,8 +343,8 @@ class HistoryRecord:
remaining = -1 remaining = -1
result = None result = None
for record in history: for record in history:
if (record.status == HistoryRecord.MATCH and if (record.status == HistoryRecord.MATCH
(record.remaining < remaining or remaining < 0)): and (record.remaining < remaining or remaining < 0)):
result = record result = record
remaining = record.remaining remaining = record.remaining
return result return result
...@@ -376,7 +375,7 @@ LOG_SIZE_THRESHOLD = 10000 # maximum number of history records to log ...@@ -376,7 +375,7 @@ LOG_SIZE_THRESHOLD = 10000 # maximum number of history records to log
LOG_TAIL_THRESHOLD = 500 # maximum number of history recors for "tail log" LOG_TAIL_THRESHOLD = 500 # maximum number of history recors for "tail log"
def log_parsing_history(grammar, log_file_name: str = '', html: bool=True) -> None: def log_parsing_history(grammar, log_file_name: str = '', html: bool = True) -> None:
""" """
Writes a log of the parsing history of the most recently parsed document. Writes a log of the parsing history of the most recently parsed document.
...@@ -415,8 +414,7 @@ def log_parsing_history(grammar, log_file_name: str = '', html: bool=True) -> No ...@@ -415,8 +414,7 @@ def log_parsing_history(grammar, log_file_name: str = '', html: bool=True) -> No
if not is_logging(): if not is_logging():
raise AssertionError("Cannot log history when logging is turned off!") raise AssertionError("Cannot log history when logging is turned off!")
# assert self.history__, \
# "Parser did not yet run or logging was turned off when running parser!"
if not log_file_name: if not log_file_name:
name = grammar.__class__.__name__ name = grammar.__class__.__name__
log_file_name = name[:-7] if name.lower().endswith('grammar') else name log_file_name = name[:-7] if name.lower().endswith('grammar') else name
...@@ -424,35 +422,23 @@ def log_parsing_history(grammar, log_file_name: str = '', html: bool=True) -> No ...@@ -424,35 +422,23 @@ def log_parsing_history(grammar, log_file_name: str = '', html: bool=True) -> No
log_file_name = log_file_name[:-4] log_file_name = log_file_name[:-4]
full_history = ['<h1>Full parsing history of "%s"</h1>' % log_file_name] # type: List[str] full_history = ['<h1>Full parsing history of "%s"</h1>' % log_file_name] # type: List[str]
# match_history = ['<h1>Match history of parsing "%s"</h1>' % log_file_name] # type: List[str]
# errors_only = ['<h1>Errors when parsing "%s"</h1>' % log_file_name] # type: List[str]
if len(grammar.history__) > LOG_SIZE_THRESHOLD: if len(grammar.history__) > LOG_SIZE_THRESHOLD:
warning =('Sorry, man, %iK history records is just too many! ' warning =('Sorry, man, %iK history records is just too many! '
'Only looking at the last %iK records.' 'Only looking at the last %iK records.'
% (len(grammar.history__)//1000, LOG_SIZE_THRESHOLD//1000)) % (len(grammar.history__) // 1000, LOG_SIZE_THRESHOLD // 1000))
html_warning = '<p><strong>' + warning + '</strong></p>' html_warning = '<p><strong>' + warning + '</strong></p>'
full_history.append(html_warning) full_history.append(html_warning)
# match_history.append(html_warning)
# errors_only.append(html_warning)
lead_in = '\n'. join(['<table>', HistoryRecord.COLGROUP, HistoryRecord.HEADINGS]) lead_in = '\n'. join(['<table>', HistoryRecord.COLGROUP, HistoryRecord.HEADINGS])
full_history.append(lead_in) full_history.append(lead_in)
# match_history.append(lead_in)
# errors_only.append(lead_in)
for record in grammar.history__[-LOG_SIZE_THRESHOLD:]: for record in grammar.history__[-LOG_SIZE_THRESHOLD:]:
line = record.as_html_tr() if html else str(record) line = record.as_html_tr() if html else str(record)
append_line(full_history, line) append_line(full_history, line)
# if record.node and record.node.parser.ptype != WHITESPACE_PTYPE:
# append_line(match_history, line)
# if record.node.errors:
# append_line(errors_only, line)
write_log(full_history, log_file_name + '_full') write_log(full_history, log_file_name + '_full')
if len(full_history) > LOG_TAIL_THRESHOLD + 10: if len(full_history) > LOG_TAIL_THRESHOLD + 10:
heading = '<h1>Last 500 records of parsing history of "%s"</h1>' % log_file_name + lead_in heading = '<h1>Last 500 records of parsing history of "%s"</h1>' % log_file_name + lead_in
write_log([heading] + full_history[-LOG_TAIL_THRESHOLD:], log_file_name + '_full.tail') write_log([heading] + full_history[-LOG_TAIL_THRESHOLD:], log_file_name + '_full.tail')
# write_log(match_history, log_file_name + '_match')
# if (len(errors_only) > 3 or (len(grammar.history__) <= LOG_SIZE_THRESHOLD
# and len(errors_only) > 2)):
# write_log(errors_only, log_file_name + '_errors')
...@@ -40,7 +40,7 @@ from DHParser.stringview import StringView, EMPTY_STRING_VIEW ...@@ -40,7 +40,7 @@ from DHParser.stringview import StringView, EMPTY_STRING_VIEW
from DHParser.syntaxtree import Node, RootNode, ParserBase, WHITESPACE_PTYPE, \ from DHParser.syntaxtree import Node, RootNode, ParserBase, WHITESPACE_PTYPE, \
TOKEN_PTYPE, ZOMBIE_PARSER TOKEN_PTYPE, ZOMBIE_PARSER
from DHParser.toolkit import sane_parser_name, escape_control_characters, re, typing from DHParser.toolkit import sane_parser_name, escape_control_characters, re, typing
from typing import Callable, cast, Dict, DefaultDict, List, Set, Tuple, Union, Optional from typing import Callable, cast, List, Tuple, Set, Dict, DefaultDict, Union, Optional
__all__ = ('Parser', __all__ = ('Parser',
...@@ -263,7 +263,7 @@ class Parser(ParserBase): ...@@ -263,7 +263,7 @@ class Parser(ParserBase):
""" """
duplicate = self.__class__() duplicate = self.__class__()
duplicate.name = self.name duplicate.name = self.name
duplicate.ptype = self.ptype duplicate.ptype = self.ptype
return duplicate return duplicate
def reset(self): def reset(self):
...@@ -271,7 +271,7 @@ class Parser(ParserBase): ...@@ -271,7 +271,7 @@ class Parser(ParserBase):
the `reset()`-method of the parent class must be called from the the `reset()`-method of the parent class must be called from the
`reset()`-method of the derived class.""" `reset()`-method of the derived class."""
self.visited = dict() # type: Dict[int, Tuple[Optional[Node], StringView]] self.visited = dict() # type: Dict[int, Tuple[Optional[Node], StringView]]
self.recursion_counter = defaultdict(lambda :0) # type: DefaultDict[int, int] self.recursion_counter = defaultdict(lambda: 0) # type: DefaultDict[int, int]
self.cycle_detection = set() # type: Set[Callable] self.cycle_detection = set() # type: Set[Callable]
def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]: def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]:
...@@ -293,7 +293,10 @@ class Parser(ParserBase): ...@@ -293,7 +293,10 @@ class Parser(ParserBase):
@property @property
def grammar(self) -> 'Grammar': def grammar(self) -> 'Grammar':
return self._grammar if self._grammar:
return self._grammar
else:
raise AssertionError('Grammar has not yet been set!')
@grammar.setter @grammar.setter
def grammar(self, grammar: 'Grammar'): def grammar(self, grammar: 'Grammar'):
...@@ -301,8 +304,9 @@ class Parser(ParserBase): ...@@ -301,8 +304,9 @@ class Parser(ParserBase):
self._grammar = grammar self._grammar = grammar
self._grammar_assigned_notifier() self._grammar_assigned_notifier()
else: else:
assert self._grammar == grammar, \ if self._grammar != grammar:
"Parser has already been assigned to a different Grammar object!" raise AssertionError("Parser has already been assigned"
"to a different Grammar object!")
def _grammar_assigned_notifier(self): def _grammar_assigned_notifier(self):
"""A function that notifies the parser object that it has been """A function that notifies the parser object that it has been
...@@ -564,12 +568,6 @@ class Grammar: ...@@ -564,12 +568,6 @@ class Grammar:
def __init__(self, root: Parser = None) -> None: def __init__(self, root: Parser = None) -> None:
# if not hasattr(self.__class__, 'parser_initialization__'):
# self.__class__.parser_initialization__ = "pending"
# if not hasattr(self.__class__, 'wspL__'):
# self.wspL__ = ''
# if not hasattr(self.__class__, 'wspR__'):
# self.wspR__ = ''
self.all_parsers__ = set() # type: Set[ParserBase] self.all_parsers__ = set() # type: Set[ParserBase]
self._dirty_flag__ = False # type: bool self._dirty_flag__ = False # type: bool
self.history_tracking__ = False # type: bool self.history_tracking__ = False # type: bool
...@@ -650,7 +648,7 @@ class Grammar: ...@@ -650,7 +648,7 @@ class Grammar:
parser.grammar = self parser.grammar = self
def __call__(self, document: str, start_parser="root__", track_history=False) -> Node: def __call__(self, document: str, start_parser="root__", track_history=False) -> RootNode:
""" """
Parses a document with with parser-combinators. Parses a document with with parser-combinators.
......
...@@ -288,12 +288,12 @@ class StringView(collections.abc.Sized): ...@@ -288,12 +288,12 @@ class StringView(collections.abc.Sized):
return self.fullstring.split(sep) return self.fullstring.split(sep)
else: else:
pieces = [] pieces = []
l = len(sep) length = len(sep)
k = 0 k = 0
i = self.find(sep, k) i = self.find(sep, k)
while i >= 0: while i >= 0:
pieces.append(self.text[self.begin + k: self.begin + i]) pieces.append(self.text[self.begin + k: self.begin + i])
k = i + l k = i + length
i = self.find(sep, k) i = self.find(sep, k)
pieces.append(self.text[self.begin + k: self.end]) pieces.append(self.text[self.begin + k: self.end])
return pieces return pieces
......
This diff is collapsed.
...@@ -166,7 +166,7 @@ if __name__ == '__main__': ...@@ -166,7 +166,7 @@ if __name__ == '__main__':
# if called with a single filename that is either an EBNF file or a known # if called with a single filename that is either an EBNF file or a known
# test file type then use the given argument # test file type then use the given argument
arg = argv[1] arg = argv[1]
else: else:
# otherwise run all tests in the test directory # otherwise run all tests in the test directory
arg = '*_test_*.ini' arg = '*_test_*.ini'
if arg.endswith('.ebnf'): if arg.endswith('.ebnf'):
...@@ -328,8 +328,8 @@ def main(): ...@@ -328,8 +328,8 @@ def main():
file_path = input('Please enter a file path for compilation > ') file_path = input('Please enter a file path for compilation > ')
if os.path.exists(file_path) and os.path.isfile(file_path): if os.path.exists(file_path) and os.path.isfile(file_path):
compiler_suite = input('Compiler suite or ENTER (for ebnf) > ') compiler_suite = input('Compiler suite or ENTER (for ebnf) > ')
if (not compiler_suite or (os.path.exists(compiler_suite) if not compiler_suite or (os.path.exists(compiler_suite)
and os.path.isfile(compiler_suite))): and os.path.isfile(compiler_suite)):
_errors = compile_on_disk(file_path, compiler_suite) _errors = compile_on_disk(file_path, compiler_suite)
if _errors: if _errors:
print('\n\n'.join(str(err) for err in _errors)) print('\n\n'.join(str(err) for err in _errors))
......
...@@ -11,7 +11,7 @@ import time ...@@ -11,7 +11,7 @@ import time
def run_tests(command): def run_tests(command):
testtype = 'DOCTEST' if command.find('doctest') >= 0 else 'UNITTEST' testtype = 'DOCTEST' if command.find('doctest') >= 0 else 'UNITTEST'
filename = command[command.rfind(' ')+1:] filename = command[command.rfind(' ') + 1:]
print('\n' + testtype + ' ' + filename) print('\n' + testtype + ' ' + filename)
os.system(command) os.system(command)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment