Commit a21adc6a authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

Tutorial updated to latest changes

parent b95c2423
...@@ -226,7 +226,7 @@ def compile_source(source: str, ...@@ -226,7 +226,7 @@ def compile_source(source: str,
log_parsing_history(parser, log_file_name) log_parsing_history(parser, log_file_name)
assert is_error(syntax_tree.error_flag) or str(syntax_tree) == strip_tokens(source_text), \ assert is_error(syntax_tree.error_flag) or str(syntax_tree) == strip_tokens(source_text), \
str(syntax_tree) str(syntax_tree) # TODO: Ony valid if neither tokens or whitespace are dropped early!s
# only compile if there were no syntax errors, for otherwise it is # only compile if there were no syntax errors, for otherwise it is
# likely that error list gets littered with compile error messages # likely that error list gets littered with compile error messages
result = None result = None
......
...@@ -382,7 +382,10 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil ...@@ -382,7 +382,10 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
This function recursively reassigns the position values This function recursively reassigns the position values
of the child nodes, too. of the child nodes, too.
""" """
assert self._pos < 0 or self.pos == pos, str("pos mismatch %i != %i" % (self._pos, pos)) # condition self.pos == pos cannot be assumed when tokens or whitespace
# are dropped early!
# assert self._pos < 0 or self.pos == pos, ("pos mismatch %i != %i at Node: %s"
# % (self._pos, pos, repr(self)))
self._pos = pos self._pos = pos
# recursively adjust pos-values of all children # recursively adjust pos-values of all children
offset = self.pos offset = self.pos
......
...@@ -369,29 +369,53 @@ up, how the AST-transformation is specified with DHParser. For this ...@@ -369,29 +369,53 @@ up, how the AST-transformation is specified with DHParser. For this
purpose, you can have a look in file `LyrikCompiler_example.py`. If you purpose, you can have a look in file `LyrikCompiler_example.py`. If you
scroll down to the AST section, you'll see something like this: scroll down to the AST section, you'll see something like this:
Lyrik_AST_transformation_table = { Lyrik_AST_transformation_table = {
# AST Transformations for the Lyrik-grammar # AST Transformations for the Lyrik-grammar
"+": remove_empty, "<": remove_empty,
"bibliographisches": [remove_nodes('NZ'), remove_tokens], "bibliographisches":
"autor, werk, untertitel, ort": [], [flatten, remove_nodes('NZ'), remove_whitespace, remove_tokens],
"jahr": [content_from_sinlge_child], "autor": [],
"wortfolge": [flatten(is_one_of('WORT'), recursive=False), remove_last(is_whitespace), collapse], "werk": [],
"namenfolge": [flatten(is_one_of('NAME'), recursive=False), remove_last(is_whitespace), collapse], "untertitel": [],
"verknüpfung": [remove_tokens('<', '>'), content_from_sinlge_child], "ort": [],
"ziel": content_from_sinlge_child, "jahr":
"gedicht, strophe, text": [flatten, remove_nodes('LEERZEILE'), remove_nodes('NZ')], [reduce_single_child, remove_whitespace, reduce_single_child],
"titel, serie": [flatten, remove_nodes('LEERZEILE'), remove_nodes('NZ'), collapse], "wortfolge":
"vers": collapse, [flatten(is_one_of('WORT'), recursive=False), peek, rstrip,
"zeile": [], collapse],
"ZEICHENFOLGE, NZ, JAHRESZAHL": content_from_sinlge_child, "namenfolge":
"WORT, NAME, LEERZEILE, ENDE": [], [flatten(is_one_of('NAME'), recursive=False), peek, rstrip,
":Whitespace": replace_content(lambda node : " "), collapse],
":Token": content_from_sinlge_child, "verknüpfung":
"*": replace_by_single_child [flatten, remove_tokens('<', '>'), remove_whitespace,
} reduce_single_child],
"ziel":
As you can see, AST-transformations are specified declaratively (with the [reduce_single_child, remove_whitespace, reduce_single_child],
option to add your own Python-programmed transformation rules). This "gedicht, strophe, text":
[flatten, remove_nodes('LEERZEILE'), remove_nodes('NZ')],
"titel, serie":
[flatten, remove_nodes('LEERZEILE'), remove_nodes('NZ'),
collapse],
"zeile": [strip],
"vers":
[strip, collapse],
"WORT": [],
"NAME": [],
"ZEICHENFOLGE":
reduce_single_child,
"NZ":
reduce_single_child,
"LEERZEILE": [],
"JAHRESZAHL":
[reduce_single_child],
"ENDE": [],
":Whitespace":
replace_content(lambda node: " "),
"*": replace_by_single_child
}
As you can see, AST-transformations are specified declaratively (with
the option to add your own Python-programmed transformation rules). This
keeps the specification of the AST-transformation simple and concise. At keeps the specification of the AST-transformation simple and concise. At
the same, we avoid adding hints for the AST-transformation in the the same, we avoid adding hints for the AST-transformation in the
grammar specification, which would render the grammar less readable. grammar specification, which would render the grammar less readable.
......
@ whitespace = horizontal
@ drop = whitespace
gedicht = bibliographisches { LEERZEILE }+ [serie] §titel text /\s*/ ENDE
bibliographisches = autor §"," [NZ~] werk "," [NZ~] ort "," [NZ~] jahr "."
autor = namenfolge [verknüpfung]
werk = wortfolge ["." §untertitel] [verknüpfung]
untertitel = wortfolge [verknüpfung]
ort = wortfolge [verknüpfung]
jahr = JAHRESZAHL ~
wortfolge = { WORT [L] }+
namenfolge = { NAME [L] }+
verknüpfung = "<" ziel ">"
ziel = ZEICHENFOLGE ~
serie = !(titel vers NZ vers) { NZ zeile }+ { LEERZEILE }+
titel = { NZ [L] zeile { LEERZEILE }+ }+
zeile = { ZEICHENFOLGE [L] }+
text = { strophe {LEERZEILE} }+
strophe = { NZ vers }+
vers = { ZEICHENFOLGE [L] }+
WORT = /\w+/
NAME = /\w+\.?/
ZEICHENFOLGE = /[^ \n<>]+/
NZ = /\n/
LEERZEILE = /\n[ \t]*(?=\n)/~
JAHRESZAHL = /\d\d\d\d/
ENDE = !/./
L = /[ \t]+/~ # signifikanter leerraum
\ No newline at end of file
...@@ -25,7 +25,7 @@ from DHParser import is_filename, Grammar, Compiler, Lookbehind, \ ...@@ -25,7 +25,7 @@ from DHParser import is_filename, Grammar, Compiler, Lookbehind, \
reduce_single_child, replace_by_single_child, remove_whitespace, \ reduce_single_child, replace_by_single_child, remove_whitespace, \
flatten, is_empty, collapse, replace_content, remove_brackets, \ flatten, is_empty, collapse, replace_content, remove_brackets, \
is_one_of, rstrip, strip, remove_tokens, remove_nodes, peek, \ is_one_of, rstrip, strip, remove_tokens, remove_nodes, peek, \
is_whitespace, TOKEN_PTYPE is_whitespace, TOKEN_PTYPE, GLOBALS
from DHParser.log import logging from DHParser.log import logging
...@@ -83,7 +83,7 @@ class LyrikGrammar(Grammar): ...@@ -83,7 +83,7 @@ class LyrikGrammar(Grammar):
ENDE = !/./ ENDE = !/./
""" """
source_hash__ = "6602d99972ef2883e28bd735e1fe0401" source_hash__ = "6602d99972ef2883e28bd735e1fe0401"
parser_initialization__ = "upon instantiation" parser_initialization__ = ["upon instantiation"]
COMMENT__ = r'' COMMENT__ = r''
WHITESPACE__ = r'[\t ]*' WHITESPACE__ = r'[\t ]*'
WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__) WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
...@@ -116,12 +116,12 @@ class LyrikGrammar(Grammar): ...@@ -116,12 +116,12 @@ class LyrikGrammar(Grammar):
root__ = gedicht root__ = gedicht
def get_grammar() -> LyrikGrammar: def get_grammar() -> LyrikGrammar:
global thread_local_Lyrik_grammar_singleton global GLOBALS
try: try:
grammar = thread_local_Lyrik_grammar_singleton grammar = GLOBALS.Lyrik_grammar_singleton
except NameError: except AttributeError:
thread_local_Lyrik_grammar_singleton = LyrikGrammar() GLOBALS.Lyrik_grammar_singleton = LyrikGrammar()
grammar = thread_local_Lyrik_grammar_singleton grammar = GLOBALS.Lyrik_grammar_singleton
return grammar return grammar
...@@ -171,15 +171,22 @@ Lyrik_AST_transformation_table = { ...@@ -171,15 +171,22 @@ Lyrik_AST_transformation_table = {
[reduce_single_child], [reduce_single_child],
"ENDE": [], "ENDE": [],
":Whitespace": ":Whitespace":
replace_content(lambda node : " "), replace_content(lambda node: " "),
"*": replace_by_single_child "*": replace_by_single_child
} }
LyrikTransform = partial(traverse, processing_table=Lyrik_AST_transformation_table) def LyrikTransform() -> TransformationDict:
return partial(traverse, processing_table=Lyrik_AST_transformation_table)
def get_transformer() -> TransformationFunc: def get_transformer() -> TransformationFunc:
return LyrikTransform global GLOBALS
try:
transform = GLOBALS.Lyrik_transformer_singleton
except AttributeError:
GLOBALS.Lyrik_transformer_singleton = LyrikTransform()
transform = GLOBALS.Lyrik_transformer_singleton
return transform
####################################################################### #######################################################################
...@@ -266,13 +273,13 @@ class LyrikCompiler(Compiler): ...@@ -266,13 +273,13 @@ class LyrikCompiler(Compiler):
def get_compiler() -> LyrikCompiler: def get_compiler() -> LyrikCompiler:
global thread_local_Lyrik_compiler_singleton global GLOBALS
try: try:
compiler = thread_local_Lyrik_compiler_singleton compiler = GLOBALS.Lyrik_compiler_singleton
return compiler except AttributeError:
except NameError: GLOBALS.Lyrik_compiler_singleton = LyrikCompiler()
thread_local_Lyrik_compiler_singleton = LyrikCompiler() compiler = GLOBALS.Lyrik_compiler_singleton
return thread_local_Lyrik_compiler_singleton return compiler
####################################################################### #######################################################################
......
...@@ -290,7 +290,6 @@ XML_AST_transformation_table = { ...@@ -290,7 +290,6 @@ XML_AST_transformation_table = {
"Char": [], "Char": [],
"S": [], "S": [],
"EOF": [], "EOF": [],
":_Token, :_RE": reduce_single_child,
"*": replace_by_single_child "*": replace_by_single_child
} }
......
...@@ -19,6 +19,13 @@ if __name__ == "__main__": ...@@ -19,6 +19,13 @@ if __name__ == "__main__":
example_dirs = os.listdir(os.path.join(rootdir, 'examples')) example_dirs = os.listdir(os.path.join(rootdir, 'examples'))
run = 0 run = 0
failures = 0 failures = 0
def check(ret):
global run, failures
run += 1
if ret > 0:
failures += 1
for example in example_dirs: for example in example_dirs:
example_path = os.path.join(rootdir, 'examples', example) example_path = os.path.join(rootdir, 'examples', example)
if os.path.isdir(example_path): if os.path.isdir(example_path):
...@@ -28,10 +35,14 @@ if __name__ == "__main__": ...@@ -28,10 +35,14 @@ if __name__ == "__main__":
if os.path.isfile(name) \ if os.path.isfile(name) \
and (name == "recompile_grammar.py" or fnmatch.fnmatch(name, 'tst_*.py')): and (name == "recompile_grammar.py" or fnmatch.fnmatch(name, 'tst_*.py')):
print(os.path.join(example_path, name)) print(os.path.join(example_path, name))
ret = os.system(interpreter + name) check(os.system(interpreter + name))
run += 1
if ret > 0:
failures += 1
os.chdir(save) os.chdir(save)
save = os.getcwd()
os.chdir(os.path.join(scriptdir, 'Tutorial'))
check(os.system('python LyrikCompiler.py Lyrisches_Intermezzo_IV.txt'))
check(os.system('python LyrikCompiler_example.py Lyrisches_Intermezzo_IV.txt'))
os.chdir(save)
print() print()
print("{} tests run, {} tests failed".format(run, failures)) print("{} tests run, {} tests failed".format(run, failures))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment