Currently job artifacts in CI/CD pipelines on LRZ GitLab never expire. Starting from Wed 26.1.2022 the default expiration time will be 30 days (GitLab default). Currently existing artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 625ad39d authored by di68kap's avatar di68kap
Browse files

- parser.py: bug: pos values not initialized in nodes in history record, in...

- parser.py: bug: pos values not initialized in nodes in history record, in case root parser did not match
parent 90515440
......@@ -76,11 +76,11 @@ from DHParser.parsers import Grammar, Compiler, nil_scanner, \\
Lookbehind, Lookahead, Alternative, Pop, Required, Token, \\
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Sequence, RE, Capture, \\
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
nop_filter, counterpart_filter, accumulating_filter
nop_filter, counterpart_filter, accumulating_filter, ScannerFunc
from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \\
remove_children_if, reduce_single_child, replace_by_single_child, remove_whitespace, \\
no_operation, remove_expendables, remove_tokens, flatten, is_whitespace, is_expendable, \\
collapse, map_content, WHITESPACE_PTYPE, TOKEN_PTYPE
collapse, map_content, WHITESPACE_PTYPE, TOKEN_PTYPE, TransformerFunc
'''
......
......@@ -252,13 +252,13 @@ CompilerFactoryFunc = Callable[[], Compiler]
SCANNER_FACTORY = '''
def get_scanner():
def get_scanner() -> ScannerFunc:
return {NAME}Scanner
'''
GRAMMAR_FACTORY = '''
def get_grammar():
def get_grammar() -> {NAME}Grammar:
global thread_local_{NAME}_grammar_singleton
try:
grammar = thread_local_{NAME}_grammar_singleton
......@@ -270,13 +270,13 @@ def get_grammar():
TRANSFORMER_FACTORY = '''
def get_transformer():
def get_transformer() -> TransformerFunc:
return {NAME}Transform
'''
COMPILER_FACTORY = '''
def get_compiler(grammar_name="{NAME}", grammar_source=""):
def get_compiler(grammar_name="{NAME}", grammar_source="") -> {NAME}Compiler:
global thread_local_{NAME}_compiler_singleton
try:
compiler = thread_local_{NAME}_compiler_singleton
......
......@@ -412,6 +412,13 @@ class Grammar:
stitches.append(Node(None, skip))
stitches[-1].add_error(error_msg)
if self.history_tracking:
# some parsers may have matched and left history records with nodes != None.
# Because these are not connected to the stiched root node, their pos
# properties will not be initialized by setting the root node's pos property
# to zero. Therefore, their pos properties need to be initialized here
for record in self.history:
if record.node and record.node._pos < 0:
record.node.pos = 0
record = HistoryRecord(self.call_stack.copy(), stitches[-1], len(rest))
self.history.append(record)
self.history_tracking = False
......
......@@ -204,7 +204,7 @@ def load_if_file(text_or_file) -> str:
content = f.read()
return content
except FileNotFoundError as error:
if re.fullmatch(r'[\w/:\\]+', text_or_file):
if re.fullmatch(r'[\w/:. \\]+', text_or_file):
raise FileNotFoundError('Not a valid file: ' + text_or_file + '\nAdd "\\n" '
'to distinguish source data from a file name!')
else:
......
......@@ -19,11 +19,11 @@ from DHParser.parsers import Grammar, Compiler, nil_scanner, \
Lookbehind, Lookahead, Alternative, Pop, Required, Token, \
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Sequence, RE, Capture, \
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
nop_filter, counterpart_filter, accumulating_filter
nop_filter, counterpart_filter, accumulating_filter, ScannerFunc
from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \
remove_children_if, reduce_single_child, replace_by_single_child, remove_whitespace, \
no_operation, remove_expendables, remove_tokens, flatten, is_whitespace, is_expendable, \
collapse, map_content, WHITESPACE_PTYPE, TOKEN_PTYPE
collapse, map_content, WHITESPACE_PTYPE, TOKEN_PTYPE, TransformerFunc
#######################################################################
......@@ -35,7 +35,7 @@ from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \
def LyrikScanner(text):
return text
def get_scanner():
def get_scanner() -> ScannerFunc:
return LyrikScanner
......@@ -111,7 +111,7 @@ class LyrikGrammar(Grammar):
gedicht = Sequence(bibliographisches, OneOrMore(LEERZEILE), Optional(serie), Required(titel), Required(text), RE('\\s*', wR=''), Required(ENDE))
root__ = gedicht
def get_grammar():
def get_grammar() -> LyrikGrammar:
global thread_local_Lyrik_grammar_singleton
try:
grammar = thread_local_Lyrik_grammar_singleton
......@@ -159,7 +159,7 @@ Lyrik_AST_transformation_table = {
LyrikTransform = partial(traverse, processing_table=Lyrik_AST_transformation_table)
def get_transformer():
def get_transformer() -> TransformerFunc:
return LyrikTransform
......@@ -250,7 +250,7 @@ class LyrikCompiler(Compiler):
pass
def get_compiler(grammar_name="Lyrik", grammar_source=""):
def get_compiler(grammar_name="Lyrik", grammar_source="") -> LyrikCompiler:
global thread_local_Lyrik_compiler_singleton
try:
compiler = thread_local_Lyrik_compiler_singleton
......@@ -285,7 +285,7 @@ def compile_src(source):
if __name__ == "__main__":
if len(sys.argv) == 1:
sys.argv.append('Lyrisches_Intermezzo_iV.txt')
sys.argv.append("Lyrisches_Intermezzo_IV.txt")
if len(sys.argv) > 1:
result, errors, ast = compile_src(sys.argv[1])
if errors:
......
......@@ -27,6 +27,7 @@ sys.path.extend(['../', './'])
from DHParser.parsers import Grammar, Compiler
from DHParser.dsl import compile_on_disk, run_compiler, compileEBNF, parser_factory, \
load_compiler_suite
from DHParser.toolkit import is_filename
ARITHMETIC_EBNF = """
......@@ -57,12 +58,12 @@ class TestCompileFunctions:
class TestCompilerGeneration:
trivial_lang = """
text = { word | WSPC } "."
text = { word | WSPC } "." [/\s/]
word = /\w+/
WSPC = /\s+/
"""
tmp = 'tmp/' if os.path.isdir('tmp') else ('test/tmp/')
trivial_text = u"""Es war ein Koenig in Thule."""
trivial_text = u"""Es war ein Koenig in Thule.\n"""
grammar_name = tmp + "TestCompilerGeneration.ebnf"
compiler_name = tmp + "TestCompilerGenerationCompiler.py"
text_name = tmp + "TestCompilerGeneration_text.txt"
......@@ -105,6 +106,7 @@ class TestCompilerGeneration:
assert compiler_suite == compiler_suite_2nd_run
# test compiling with a generated compiler suite
# assert is_filename(self.text_name)
errors = compile_on_disk(self.text_name, self.compiler_name)
assert not errors, str(errors)
assert os.path.exists(self.result_name)
......
......@@ -25,7 +25,7 @@ import sys
sys.path.extend(['../', './'])
from DHParser import parsers
from DHParser.toolkit import is_logging, compile_python_object
from DHParser.toolkit import is_logging, logging, compile_python_object
from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten, TOKEN_PTYPE
from DHParser.parsers import compile_source
......@@ -120,6 +120,27 @@ class TestRegex:
assert not result.error_flag
class TestGrammar:
def test_pos_values_initialized(self):
# checks whether pos values in the parsing result and in the
# history record have been initialized
grammar = r"""@whitespace = horizontal
haupt = textzeile LEERZEILE
textzeile = { WORT }+
WORT = /[^ \t]+/~
LEERZEILE = /\n[ \t]*(?=\n)/~
"""
result, messages, syntax_tree = compile_source(grammar, None, get_ebnf_grammar(),
get_ebnf_transformer(), get_ebnf_compiler("PosTest"))
assert result
assert not messages
with logging("LOGS"):
parser = compile_python_object(DHPARSER_IMPORTS + result, '\w+Grammar$')()
result = parser("no_file_name*")
for record in parser.history:
assert not record.node or record.node.pos >= 0
if __name__ == "__main__":
from DHParser.testing import runner
runner("", globals())
......@@ -56,7 +56,7 @@ class TestToolkit:
assert load_if_file('this_is_code_and_not_a_file\n')
# neither will text that does not look like a file name
s = "this is code and not a file"
s = "this is code * and not a file"
assert s == load_if_file(s)
# not a file and not mistaken for a file
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment