16.12.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit f60c89e1 authored by eckhart's avatar eckhart
Browse files

- flake warnings corrected

parent 0bb99a60
...@@ -463,6 +463,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It ...@@ -463,6 +463,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
compiler1 = cfactory() compiler1 = cfactory()
compiler1.set_grammar_name(compiler_name, source_file) compiler1.set_grammar_name(compiler_name, source_file)
result, messages, _ = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1) result, messages, _ = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
if has_errors(messages): if has_errors(messages):
return messages return messages
...@@ -487,8 +488,8 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It ...@@ -487,8 +488,8 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
intro, imports, preprocessor, _, ast, compiler, outro = '', '', '', '', '', '', '' intro, imports, preprocessor, _, ast, compiler, outro = '', '', '', '', '', '', ''
except ValueError: except ValueError:
name = '"' + rootname + 'Compiler.py"' name = '"' + rootname + 'Compiler.py"'
raise ValueError('Could not identify all required sections in ' + name + raise ValueError('Could not identify all required sections in ' + name
'. Please delete or repair ' + name + ' manually!') + '. Please delete or repair ' + name + ' manually!')
finally: finally:
if f: if f:
f.close() f.close()
...@@ -516,7 +517,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It ...@@ -516,7 +517,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
f.write(SECTION_MARKER.format(marker=PREPROCESSOR_SECTION)) f.write(SECTION_MARKER.format(marker=PREPROCESSOR_SECTION))
f.write(preprocessor) f.write(preprocessor)
f.write(SECTION_MARKER.format(marker=PARSER_SECTION)) f.write(SECTION_MARKER.format(marker=PARSER_SECTION))
f.write(result) f.write(cast(str, result))
f.write(SECTION_MARKER.format(marker=AST_SECTION)) f.write(SECTION_MARKER.format(marker=AST_SECTION))
f.write(ast) f.write(ast)
f.write(SECTION_MARKER.format(marker=COMPILER_SECTION)) f.write(SECTION_MARKER.format(marker=COMPILER_SECTION))
...@@ -559,7 +560,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It ...@@ -559,7 +560,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
def recompile_grammar(ebnf_filename, force=False, def recompile_grammar(ebnf_filename, force=False,
notify: Callable=lambda: None) -> bool: notify: Callable = lambda: None) -> bool:
""" """
Re-compiles an EBNF-grammar if necessary, that is, if either no Re-compiles an EBNF-grammar if necessary, that is, if either no
corresponding 'XXXXCompiler.py'-file exists or if that file is corresponding 'XXXXCompiler.py'-file exists or if that file is
......
...@@ -183,7 +183,7 @@ def grammar_changed(grammar_class, grammar_source: str) -> bool: ...@@ -183,7 +183,7 @@ def grammar_changed(grammar_class, grammar_source: str) -> bool:
# grammar_class = load_compiler_suite(grammar_class)[1] # grammar_class = load_compiler_suite(grammar_class)[1]
with open(grammar_class, 'r', encoding='utf8') as f: with open(grammar_class, 'r', encoding='utf8') as f:
pycode = f.read() pycode = f.read()
m = re.search('class \w*\(Grammar\)', pycode) m = re.search(r'class \w*\(Grammar\)', pycode)
if m: if m:
m = re.search(' source_hash__ *= *"([a-z0-9]*)"', m = re.search(' source_hash__ *= *"([a-z0-9]*)"',
pycode[m.span()[1]:]) pycode[m.span()[1]:])
...@@ -476,13 +476,13 @@ class EBNFCompiler(Compiler): ...@@ -476,13 +476,13 @@ class EBNFCompiler(Compiler):
raise EBNFCompilerError('Compiler has not been run before calling ' raise EBNFCompilerError('Compiler has not been run before calling '
'"gen_Compiler_Skeleton()"!') '"gen_Compiler_Skeleton()"!')
compiler = ['class ' + self.grammar_name + 'Compiler(Compiler):', compiler = ['class ' + self.grammar_name + 'Compiler(Compiler):',
' """Compiler for the abstract-syntax-tree of a ' + ' """Compiler for the abstract-syntax-tree of a '
self.grammar_name + ' source file.', + self.grammar_name + ' source file.',
' """', '', ' """', '',
' def __init__(self, grammar_name="' + ' def __init__(self, grammar_name="'
self.grammar_name + '", grammar_source=""):', + self.grammar_name + '", grammar_source=""):',
' super(' + self.grammar_name + ' super(' + self.grammar_name
'Compiler, self).__init__(grammar_name, grammar_source)', + 'Compiler, self).__init__(grammar_name, grammar_source)',
r" assert re.match('\w+\Z', grammar_name)", '', r" assert re.match('\w+\Z', grammar_name)", '',
' def _reset(self):', ' def _reset(self):',
' super()._reset()', ' super()._reset()',
...@@ -542,8 +542,8 @@ class EBNFCompiler(Compiler): ...@@ -542,8 +542,8 @@ class EBNFCompiler(Compiler):
definitions.append((self.WHITESPACE_PARSER_KEYWORD, definitions.append((self.WHITESPACE_PARSER_KEYWORD,
'Whitespace(%s)' % self.WHITESPACE_KEYWORD)) 'Whitespace(%s)' % self.WHITESPACE_KEYWORD))
definitions.append((self.WHITESPACE_KEYWORD, definitions.append((self.WHITESPACE_KEYWORD,
("mixin_comment(whitespace=" + self.RAW_WS_KEYWORD + ("mixin_comment(whitespace=" + self.RAW_WS_KEYWORD
", comment=" + self.COMMENT_KEYWORD + ")"))) + ", comment=" + self.COMMENT_KEYWORD + ")")))
definitions.append((self.RAW_WS_KEYWORD, "r'{whitespace}'".format(**self.directives))) definitions.append((self.RAW_WS_KEYWORD, "r'{whitespace}'".format(**self.directives)))
definitions.append((self.COMMENT_KEYWORD, "r'{comment}'".format(**self.directives))) definitions.append((self.COMMENT_KEYWORD, "r'{comment}'".format(**self.directives)))
...@@ -584,7 +584,7 @@ class EBNFCompiler(Compiler): ...@@ -584,7 +584,7 @@ class EBNFCompiler(Compiler):
for symbol in self.symbols: for symbol in self.symbols:
if symbol not in defined_symbols: if symbol not in defined_symbols:
self.tree.new_error(self.symbols[symbol], self.tree.new_error(self.symbols[symbol],
"Missing definition for symbol '%s'" % symbol) "Missing definition for symbol '%s'" % symbol)
# root_node.error_flag = True # root_node.error_flag = True
# check for unconnected rules # check for unconnected rules
...@@ -643,7 +643,7 @@ class EBNFCompiler(Compiler): ...@@ -643,7 +643,7 @@ class EBNFCompiler(Compiler):
first = self.rules[rule][0] first = self.rules[rule][0]
if not first.errors: if not first.errors:
self.tree.new_error(first, 'First definition of rule "%s" ' self.tree.new_error(first, 'First definition of rule "%s" '
'followed by illegal redefinitions.' % rule) 'followed by illegal redefinitions.' % rule)
self.tree.new_error(node, 'A rule "%s" has already been defined earlier.' % rule) self.tree.new_error(node, 'A rule "%s" has already been defined earlier.' % rule)
elif rule in EBNFCompiler.RESERVED_SYMBOLS: elif rule in EBNFCompiler.RESERVED_SYMBOLS:
self.tree.new_error(node, 'Symbol "%s" is a reserved symbol.' % rule) self.tree.new_error(node, 'Symbol "%s" is a reserved symbol.' % rule)
...@@ -683,7 +683,8 @@ class EBNFCompiler(Compiler): ...@@ -683,7 +683,8 @@ class EBNFCompiler(Compiler):
prepended by the multiline-flag. Returns the regular expression string. prepended by the multiline-flag. Returns the regular expression string.
""" """
flags = self.re_flags | {'x'} if rx.find('\n') >= 0 else self.re_flags flags = self.re_flags | {'x'} if rx.find('\n') >= 0 else self.re_flags
if flags: rx = "(?%s)%s" % ("".join(flags), rx) if flags:
rx = "(?%s)%s" % ("".join(flags), rx)
try: try:
re.compile(rx) re.compile(rx)
except Exception as re_error: except Exception as re_error:
...@@ -770,7 +771,7 @@ class EBNFCompiler(Compiler): ...@@ -770,7 +771,7 @@ class EBNFCompiler(Compiler):
return "" return ""
def non_terminal(self, node: Node, parser_class: str, custom_args: List[str]=[]) -> str: def non_terminal(self, node: Node, parser_class: str, custom_args: List[str] = []) -> str:
""" """
Compiles any non-terminal, where `parser_class` indicates the Parser class Compiles any non-terminal, where `parser_class` indicates the Parser class
name for the particular non-terminal. name for the particular non-terminal.
...@@ -944,7 +945,7 @@ class EBNFCompiler(Compiler): ...@@ -944,7 +945,7 @@ class EBNFCompiler(Compiler):
else: else:
parser = '_RE(' parser = '_RE('
if rx[:2] == '~/': if rx[:2] == '~/':
if not 'left' in self.directives['literalws']: if 'left' not in self.directives['literalws']:
name = ['wL=' + self.WHITESPACE_KEYWORD] + name name = ['wL=' + self.WHITESPACE_KEYWORD] + name
rx = rx[1:] rx = rx[1:]
elif 'left' in self.directives['literalws']: elif 'left' in self.directives['literalws']:
......
...@@ -41,7 +41,7 @@ import bisect ...@@ -41,7 +41,7 @@ import bisect
from DHParser.preprocess import SourceMapFunc from DHParser.preprocess import SourceMapFunc
from DHParser.stringview import StringView from DHParser.stringview import StringView
from DHParser.toolkit import typing from DHParser.toolkit import typing
from typing import Iterable, Iterator, Union, Tuple, List, NewType from typing import Iterable, Iterator, Union, Tuple, List
__all__ = ('ErrorCode', __all__ = ('ErrorCode',
'Error', 'Error',
...@@ -200,7 +200,7 @@ def line_col(lbreaks: List[int], pos: int) -> Tuple[int, int]: ...@@ -200,7 +200,7 @@ def line_col(lbreaks: List[int], pos: int) -> Tuple[int, int]:
def adjust_error_locations(errors: List[Error], def adjust_error_locations(errors: List[Error],
original_text: Union[StringView, str], original_text: Union[StringView, str],
source_mapping: SourceMapFunc=lambda i: i) -> List[Error]: source_mapping: SourceMapFunc = lambda i: i) -> List[Error]:
"""Adds (or adjusts) line and column numbers of error messages in place. """Adds (or adjusts) line and column numbers of error messages in place.
Args: Args:
......
...@@ -206,12 +206,13 @@ class HistoryRecord: ...@@ -206,12 +206,13 @@ class HistoryRecord:
COLGROUP = '<colgroup>\n<col style="width:2%"/><col style="width:2%"/><col ' \ COLGROUP = '<colgroup>\n<col style="width:2%"/><col style="width:2%"/><col ' \
'style="width:75%"/><col style="width:6%"/><col style="width:15%"/>\n</colgroup>' 'style="width:75%"/><col style="width:6%"/><col style="width:15%"/>\n</colgroup>'
HEADINGS = ('<tr><th>L</th><th>C</th><th>parser call sequence</th>' HEADINGS = ('<tr><th>L</th><th>C</th><th>parser call sequence</th>'
'<th>success</th><th>text matched or failed</th></tr>') '<th>success</th><th>text matched or failed</th></tr>')
HTML_LEAD_IN = ('<!DOCTYPE html>\n' HTML_LEAD_IN = (
'<!DOCTYPE html>\n'
'<html>\n<head>\n<meta charset="utf-8"/>\n<style>\n' '<html>\n<head>\n<meta charset="utf-8"/>\n<style>\n'
'td,th {font-family:monospace; ' 'td,th {font-family:monospace; '
'border-right: thin solid grey; border-bottom: thin solid grey}\n' 'border-right: thin solid grey; border-bottom: thin solid grey}\n'
'td.line, td.column {color:darkgrey}\n' # 'td.stack {}\n' 'td.line, td.column {color:darkgrey}\n' # 'td.stack {}\n'
'td.status {font-weight:bold}\n' 'td.status {font-weight:bold}\n'
'td.text {color:darkblue}\n' 'td.text {color:darkblue}\n'
'table {border-spacing: 0px; border: thin solid darkgrey; width:100%}\n' 'table {border-spacing: 0px; border: thin solid darkgrey; width:100%}\n'
...@@ -260,10 +261,10 @@ class HistoryRecord: ...@@ -260,10 +261,10 @@ class HistoryRecord:
if status == self.MATCH: if status == self.MATCH:
status = '<span class="match">' + status + '</span>' status = '<span class="match">' + status + '</span>'
i = stack.rfind('-&gt;') i = stack.rfind('-&gt;')
chr = stack[i+12:i+13] chr = stack[i + 12:i + 13]
while not chr.isidentifier() and i >= 0: while not chr.isidentifier() and i >= 0:
i = stack.rfind('-&gt;', 0, i) i = stack.rfind('-&gt;', 0, i)
chr = stack[i+12:i+13] chr = stack[i + 12:i + 13]
if i >= 0: if i >= 0:
i += 12 i += 12
k = stack.find('<', i) k = stack.find('<', i)
...@@ -424,9 +425,9 @@ def log_parsing_history(grammar, log_file_name: str = '', html: bool = True) -> ...@@ -424,9 +425,9 @@ def log_parsing_history(grammar, log_file_name: str = '', html: bool = True) ->
full_history = ['<h1>Full parsing history of "%s"</h1>' % log_file_name] # type: List[str] full_history = ['<h1>Full parsing history of "%s"</h1>' % log_file_name] # type: List[str]
if len(grammar.history__) > LOG_SIZE_THRESHOLD: if len(grammar.history__) > LOG_SIZE_THRESHOLD:
warning =('Sorry, man, %iK history records is just too many! ' warning = ('Sorry, man, %iK history records is just too many! '
'Only looking at the last %iK records.' 'Only looking at the last %iK records.'
% (len(grammar.history__) // 1000, LOG_SIZE_THRESHOLD // 1000)) % (len(grammar.history__) // 1000, LOG_SIZE_THRESHOLD // 1000))
html_warning = '<p><strong>' + warning + '</strong></p>' html_warning = '<p><strong>' + warning + '</strong></p>'
full_history.append(html_warning) full_history.append(html_warning)
...@@ -441,4 +442,3 @@ def log_parsing_history(grammar, log_file_name: str = '', html: bool = True) -> ...@@ -441,4 +442,3 @@ def log_parsing_history(grammar, log_file_name: str = '', html: bool = True) ->
if len(full_history) > LOG_TAIL_THRESHOLD + 10: if len(full_history) > LOG_TAIL_THRESHOLD + 10:
heading = '<h1>Last 500 records of parsing history of "%s"</h1>' % log_file_name + lead_in heading = '<h1>Last 500 records of parsing history of "%s"</h1>' % log_file_name + lead_in
write_log([heading] + full_history[-LOG_TAIL_THRESHOLD:], log_file_name + '_full.tail') write_log([heading] + full_history[-LOG_TAIL_THRESHOLD:], log_file_name + '_full.tail')
...@@ -607,7 +607,7 @@ class Grammar: ...@@ -607,7 +607,7 @@ class Grammar:
self.document_length__ = 0 # type: int self.document_length__ = 0 # type: int
self.document_lbreaks__ = [] # type: List[int] self.document_lbreaks__ = [] # type: List[int]
# variables stored and recalled by Capture and Retrieve parsers # variables stored and recalled by Capture and Retrieve parsers
self.variables__ = defaultdict(lambda :[]) # type: DefaultDict[str, List[str]] self.variables__ = defaultdict(lambda: []) # type: DefaultDict[str, List[str]]
self.rollback__ = [] # type: List[Tuple[int, Callable]] self.rollback__ = [] # type: List[Tuple[int, Callable]]
self.last_rb__loc__ = -1 # type: int self.last_rb__loc__ = -1 # type: int
# support for call stack tracing # support for call stack tracing
...@@ -666,7 +666,7 @@ class Grammar: ...@@ -666,7 +666,7 @@ class Grammar:
Node: The root node to the parse tree. Node: The root node to the parse tree.
""" """
def tail_pos(predecessors: Union[List[Node], Tuple[Node, ...]]) -> int: def tail_pos(predecessors: Union[List[Node], Tuple[Node, ...], None]) -> int:
"""Adds the position after the last node in the list of """Adds the position after the last node in the list of
predecessors to the node.""" predecessors to the node."""
return predecessors[-1].pos + len(predecessors[-1]) if predecessors else 0 return predecessors[-1].pos + len(predecessors[-1]) if predecessors else 0
...@@ -713,10 +713,11 @@ class Grammar: ...@@ -713,10 +713,11 @@ class Grammar:
str(HistoryRecord.last_match(self.history__))) str(HistoryRecord.last_match(self.history__)))
# Check if a Lookahead-Parser did match. Needed for testing, because # Check if a Lookahead-Parser did match. Needed for testing, because
# in a test case this is not necessarily an error. # in a test case this is not necessarily an error.
last_record = self.history__[-2] if len(self.history__) > 1 else [] last_record = self.history__[-2] if len(self.history__) > 1 else None # type: Optional[HistoryRecord]
if last_record and parser != self.root__ \ if last_record and parser != self.root__ \
and last_record.status == HistoryRecord.MATCH \ and last_record.status == HistoryRecord.MATCH \
and last_record.node.pos + len(last_record.node) >= len(self.document__) \ and last_record.node.pos \
+ len(last_record.node) >= len(self.document__) \
and any(isinstance(parser, Lookahead) and any(isinstance(parser, Lookahead)
for parser in last_record.call_stack): for parser in last_record.call_stack):
error_msg = 'Parser did not match except for lookahead! ' + err_info error_msg = 'Parser did not match except for lookahead! ' + err_info
...@@ -726,14 +727,14 @@ class Grammar: ...@@ -726,14 +727,14 @@ class Grammar:
error_code = Error.PARSER_DID_NOT_MATCH error_code = Error.PARSER_DID_NOT_MATCH
else: else:
stitches.append(result) stitches.append(result)
error_msg = "Parser stopped before end" + \ error_msg = "Parser stopped before end" \
(("! trying to recover" + + (("! trying to recover"
(" but stopping history recording at this point." + (" but stopping history recording at this point."
if self.history_tracking__ else "...")) if self.history_tracking__ else "..."))
if len(stitches) < MAX_DROPOUTS if len(stitches) < MAX_DROPOUTS
else " too often! Terminating parser.") else " too often! Terminating parser.")
error_code = Error.PARSER_STOPPED_BEFORE_END error_code = Error.PARSER_STOPPED_BEFORE_END
stitches.append(Node(None, skip).init_pos(tail_pos(stitches or result))) stitches.append(Node(None, skip).init_pos(tail_pos(stitches)))
self.tree__.new_error(stitches[-1], error_msg, error_code) self.tree__.new_error(stitches[-1], error_msg, error_code)
if self.history_tracking__: if self.history_tracking__:
# # some parsers may have matched and left history records with nodes != None. # # some parsers may have matched and left history records with nodes != None.
...@@ -767,7 +768,8 @@ class Grammar: ...@@ -767,7 +768,8 @@ class Grammar:
self.tree__.new_error(result, error_msg, error_code) self.tree__.new_error(result, error_msg, error_code)
# result.pos = 0 # calculate all positions # result.pos = 0 # calculate all positions
# result.collect_errors(self.document__) # result.collect_errors(self.document__)
self.tree__.swallow(result) if result:
self.tree__.swallow(result)
return self.tree__ return self.tree__
...@@ -847,7 +849,7 @@ class PreprocessorToken(Parser): ...@@ -847,7 +849,7 @@ class PreprocessorToken(Parser):
def __deepcopy__(self, memo): def __deepcopy__(self, memo):
duplicate = self.__class__(self.name) duplicate = self.__class__(self.name)
duplicate.name = self.name duplicate.name = self.name
duplicate.ptype = self.ptype duplicate.ptype = self.ptype
return duplicate return duplicate
def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]: def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]:
...@@ -855,19 +857,22 @@ class PreprocessorToken(Parser): ...@@ -855,19 +857,22 @@ class PreprocessorToken(Parser):
end = text.find(END_TOKEN, 1) end = text.find(END_TOKEN, 1)
if end < 0: if end < 0:
node = Node(self, '') node = Node(self, '')
self.grammar.tree__.new_error(node, self.grammar.tree__.new_error(
node,
'END_TOKEN delimiter missing from preprocessor token. ' 'END_TOKEN delimiter missing from preprocessor token. '
'(Most likely due to a preprocessor bug!)') # type: Node '(Most likely due to a preprocessor bug!)') # type: Node
return node, text[1:] return node, text[1:]
elif end == 0: elif end == 0:
node = Node(self, '') node = Node(self, '')
self.grammar.tree__.new_error(node, self.grammar.tree__.new_error(
node,
'Preprocessor-token cannot have zero length. ' 'Preprocessor-token cannot have zero length. '
'(Most likely due to a preprocessor bug!)') '(Most likely due to a preprocessor bug!)')
return node, text[2:] return node, text[2:]
elif text.find(BEGIN_TOKEN, 1, end) >= 0: elif text.find(BEGIN_TOKEN, 1, end) >= 0:
node = Node(self, text[len(self.name) + 1:end]) node = Node(self, text[len(self.name) + 1:end])
self.grammar.tree__.new_error(node, self.grammar.tree__.new_error(
node,
'Preprocessor-tokens must not be nested or contain ' 'Preprocessor-tokens must not be nested or contain '
'BEGIN_TOKEN delimiter as part of their argument. ' 'BEGIN_TOKEN delimiter as part of their argument. '
'(Most likely due to a preprocessor bug!)') '(Most likely due to a preprocessor bug!)')
...@@ -941,7 +946,7 @@ class RegExp(Parser): ...@@ -941,7 +946,7 @@ class RegExp(Parser):
regexp = self.regexp.pattern regexp = self.regexp.pattern
duplicate = self.__class__(regexp) duplicate = self.__class__(regexp)
duplicate.name = self.name duplicate.name = self.name
duplicate.ptype = self.ptype duplicate.ptype = self.ptype
return duplicate return duplicate
def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]: def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]:
...@@ -962,7 +967,7 @@ class RegExp(Parser): ...@@ -962,7 +967,7 @@ class RegExp(Parser):
return escape_control_characters('/%s/' % self.regexp.pattern) return escape_control_characters('/%s/' % self.regexp.pattern)
def withWS(parser_factory, wsL='', wsR='\s*'): def withWS(parser_factory, wsL='', wsR=r'\s*'):
"""Syntactic Sugar for 'Series(Whitespace(wsL), parser_factory(), Whitespace(wsR))'. """Syntactic Sugar for 'Series(Whitespace(wsL), parser_factory(), Whitespace(wsR))'.
""" """
if wsL and isinstance(wsL, str): if wsL and isinstance(wsL, str):
...@@ -979,12 +984,12 @@ def withWS(parser_factory, wsL='', wsR='\s*'): ...@@ -979,12 +984,12 @@ def withWS(parser_factory, wsL='', wsR='\s*'):
return parser_factory() return parser_factory()
def RE(regexp, wsL='', wsR='\s*'): def RE(regexp, wsL='', wsR=r'\s*'):
"""Syntactic Sugar for 'Series(Whitespace(wsL), RegExp(regexp), Whitespace(wsR))'""" """Syntactic Sugar for 'Series(Whitespace(wsL), RegExp(regexp), Whitespace(wsR))'"""
return withWS(lambda : RegExp(regexp), wsL, wsR) return withWS(lambda: RegExp(regexp), wsL, wsR)
def TKN(token, wsL='', wsR='\s*'): def TKN(token, wsL='', wsR=r'\s*'):
"""Syntactic Sugar for 'Series(Whitespace(wsL), Token(token), Whitespace(wsR))'""" """Syntactic Sugar for 'Series(Whitespace(wsL), Token(token), Whitespace(wsR))'"""
return withWS(lambda: Token(token), wsL, wsR) return withWS(lambda: Token(token), wsL, wsR)
...@@ -1026,7 +1031,7 @@ class UnaryOperator(Parser): ...@@ -1026,7 +1031,7 @@ class UnaryOperator(Parser):
parser = copy.deepcopy(self.parser, memo) parser = copy.deepcopy(self.parser, memo)
duplicate = self.__class__(parser) duplicate = self.__class__(parser)
duplicate.name = self.name duplicate.name = self.name
duplicate.ptype = self.ptype duplicate.ptype = self.ptype
return duplicate return duplicate
def apply(self, func: Parser.ApplyFunc) -> bool: def apply(self, func: Parser.ApplyFunc) -> bool:
...@@ -1057,7 +1062,7 @@ class NaryOperator(Parser): ...@@ -1057,7 +1062,7 @@ class NaryOperator(Parser):
parsers = copy.deepcopy(self.parsers, memo) parsers = copy.deepcopy(self.parsers, memo)
duplicate = self.__class__(*parsers) duplicate = self.__class__(*parsers)
duplicate.name = self.name duplicate.name = self.name
duplicate.ptype = self.ptype duplicate.ptype = self.ptype
return duplicate return duplicate
def apply(self, func: Parser.ApplyFunc) -> bool: def apply(self, func: Parser.ApplyFunc) -> bool:
...@@ -1100,7 +1105,7 @@ class Option(UnaryOperator): ...@@ -1100,7 +1105,7 @@ class Option(UnaryOperator):
super().__init__(parser) super().__init__(parser)
# assert isinstance(parser, Parser) # assert isinstance(parser, Parser)
assert not isinstance(parser, Option), \ assert not isinstance(parser, Option), \
"Redundant nesting of options: %s" % (str(self.ptype), str(parser.name)) "Redundant nesting of options: %s%s" % (str(parser.name), str(self.ptype))
def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]: def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]:
node, text = self.parser(text) node, text = self.parser(text)
...@@ -1245,7 +1250,7 @@ class Series(NaryOperator): ...@@ -1245,7 +1250,7 @@ class Series(NaryOperator):
parsers = copy.deepcopy(self.parsers, memo) parsers = copy.deepcopy(self.parsers, memo)
duplicate = self.__class__(*parsers, mandatory=self.mandatory) duplicate = self.__class__(*parsers, mandatory=self.mandatory)
duplicate.name = self.name duplicate.name = self.name
duplicate.ptype = self.ptype duplicate.ptype = self.ptype
return duplicate return duplicate
def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]: def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]:
...@@ -1589,7 +1594,7 @@ class Lookbehind(FlowOperator): ...@@ -1589,7 +1594,7 @@ class Lookbehind(FlowOperator):
p = p.parser p = p.parser
assert isinstance(p, RegExp) or isinstance(p, Token) assert isinstance(p, RegExp) or isinstance(p, Token)
self.regexp = None self.regexp = None
self.text = None self.text = '' # type: str
if isinstance(p, RegExp): if isinstance(p, RegExp):
self.regexp = cast(RegExp, p).regexp self.regexp = cast(RegExp, p).regexp
else: # p is of type PlainText else: # p is of type PlainText
...@@ -1686,7 +1691,7 @@ class Retrieve(Parser): ...@@ -1686,7 +1691,7 @@ class Retrieve(Parser):
def __deepcopy__(self, memo): def __deepcopy__(self, memo):
duplicate = self.__class__(self.symbol, self.filter) duplicate = self.__class__(self.symbol, self.filter)
duplicate.name = self.name duplicate.name = self.name
duplicate.ptype = self.ptype duplicate.ptype = self.ptype
return duplicate return duplicate
def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]: def __call__(self, text: StringView) -> Tuple[Optional[Node], StringView]:
...@@ -1808,7 +1813,7 @@ class Forward(Parser): ...@@ -1808,7 +1813,7 @@ class Forward(Parser):
def __deepcopy__(self, memo): def __deepcopy__(self, memo):
duplicate = self.__class__() duplicate = self.__class__()
# duplicate.name = self.name # Forward-Parsers should never have a name! # duplicate.name = self.name # Forward-Parsers should never have a name!
duplicate.ptype = self.ptype duplicate.ptype = self.ptype
memo[id(self)] = duplicate memo[id(self)] = duplicate
parser = copy.deepcopy(self.parser, memo) parser = copy.deepcopy(self.parser, memo)
duplicate.set(parser) duplicate.set(parser)
...@@ -1854,4 +1859,3 @@ class Forward(Parser): ...@@ -1854,4 +1859,3 @@ class Forward(Parser):
self.parser.apply(func) self.parser.apply(func)
return True return True
return False return False
...@@ -156,7 +156,6 @@ def strip_tokens(tokenized: str) -> str: ...@@ -156,7 +156,6 @@ def strip_tokens(tokenized: str) -> str:
return ''.join(result) return ''.join(result)
####################################################################### #######################################################################
# #
# Source Maps - mapping source code positions between different # Source Maps - mapping source code positions between different
......
...@@ -54,7 +54,7 @@ def last_char(text, begin: int, end: int) -> int: ...@@ -54,7 +54,7 @@ def last_char(text, begin: int, end: int) -> int:
"""Returns the index of the first non-whitespace character in string """Returns the index of the first non-whitespace character in string
`text` within the bounds [begin, end]. `text` within the bounds [begin, end].
""" """
while end > begin and text[end-1] in ' \n\t': while end > begin and text[end - 1] in ' \n\t':
end -= 1 end -= 1
return end return end
......
...@@ -304,7 +304,7 @@ class Node(collections.abc.Sized): ...@@ -304,7 +304,7 @@ class Node(collections.abc.Sized):
mpargs = {'name': self.parser.name, 'ptype': self.parser.ptype} mpargs = {'name': self.parser.name, 'ptype': self.parser.ptype}
parg = "MockParser({name}, {ptype})".format(**mpargs) parg = "MockParser({name}, {ptype})".format(**mpargs)
rarg = str(self) if not self.children else \ rarg = str(self) if not self.children else \
"(" + ", ".join(repr(child) for child in self.children) + ")" "(" + ", ".join(repr(child) for child in self.children) + ")"
return "Node(%s, %s)" % (parg, rarg) return "Node(%s, %s)" % (parg, rarg)
...@@ -650,7 +650,7 @@ class Node(collections.abc.Sized): ...@@ -650,7 +650,7 @@ class Node(collections.abc.Sized):