Commit 703308bd authored by di68kap's avatar di68kap

- changes to MLW example

parent 7ea1fb30
......@@ -40,6 +40,7 @@ __all__ = ['WHITESPACE_KEYWORD',
'no_operation',
'replace_by_single_child',
'reduce_single_child',
'change_parser',
'is_whitespace',
'is_empty',
'is_expendable',
......@@ -522,7 +523,8 @@ def no_operation(node):
# ------------------------------------------------
#
# rearranging transformations:
# - tree may be rearranged (flattened)
# - tree may be rearranged (e.g.flattened)
# - nodes that are not leaves may be dropped
# - order is preserved
# - all leaves are kept
#
......@@ -551,6 +553,13 @@ def reduce_single_child(node):
node.result = node.result[0].result
def change_parser(node, new_parser_name):
"""Changes the parser of a Node to a mock parser with the given
name.
"""
node.parser = MockParser(new_parser_name)
# ------------------------------------------------
#
# destructive transformations:
......@@ -641,6 +650,7 @@ def remove_enclosing_delimiters(node):
########################################################################
#
# syntax tree validation functions
# EXPERIMENTAL!
#
########################################################################
......
......@@ -8,12 +8,13 @@
from functools import partial
import os
import sys
try:
import regex as re
except ImportError:
import re
from DHParser.toolkit import load_if_file
from DHParser.toolkit import logging, is_filename, load_if_file
from DHParser.parsers import GrammarBase, CompilerBase, nil_scanner, \
Lookbehind, Lookahead, Alternative, Pop, Required, Token, \
Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Sequence, RE, Capture, \
......@@ -21,7 +22,8 @@ from DHParser.parsers import GrammarBase, CompilerBase, nil_scanner, \
from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \
remove_children_if, reduce_single_child, replace_by_single_child, remove_whitespace, \
no_operation, remove_expendables, remove_tokens, flatten, is_whitespace, is_expendable, \
WHITESPACE_KEYWORD, TOKEN_KEYWORD
WHITESPACE_KEYWORD, TOKEN_KEYWORD, change_parser
#######################################################################
......@@ -33,6 +35,9 @@ from DHParser.syntaxtree import Node, traverse, remove_enclosing_delimiters, \
def MLWScanner(text):
return text
def get_MLW_scanner():
return MLWScanner
#######################################################################
#
......@@ -143,7 +148,7 @@ class MLWGrammar(GrammarBase):
DATEI_ENDE = !/./
NIEMALS = /(?!.)/
"""
source_hash__ = "9fce888d1b21b2d11a6228e0b97f9291"
source_hash__ = "ce9155e0248ac27756283d067342182e"
parser_initialization__ = "upon instatiation"
COMMENT__ = r'#.*(?:\n|$)'
WSP__ = mixin_comment(whitespace=r'[\t ]*', comment=r'#.*(?:\n|$)')
......@@ -195,6 +200,15 @@ class MLWGrammar(GrammarBase):
Artikel = Sequence(Optional(LEER), Required(LemmaPosition), Optional(ArtikelKopf), Required(BedeutungsPosition), Required(Autorinfo), Optional(LEER), DATEI_ENDE)
root__ = Artikel
def get_MLW_grammar():
global thread_local_MLW_grammar_singleton
try:
grammar = thread_local_MLW_grammar_singleton
return grammar
except NameError:
thread_local_MLW_grammar_singleton = MLWGrammar()
return thread_local_MLW_grammar_singleton
#######################################################################
#
......@@ -220,7 +234,6 @@ def join_strings(node, delimiter='\n'):
new_result.append(nd)
node.result = tuple(new_result)
MLW_AST_transformation_table = {
# AST Transformations for the MLW-grammar
"Artikel": no_operation,
......@@ -269,9 +282,8 @@ MLW_AST_transformation_table = {
"Autorinfo":
[partial(remove_tokens, tokens={'AUTORIN', 'AUTOR'})],
"WORT, WORT_KLEIN, WORT_GROSS, GROSSSCHRIFT":
# test,
[remove_expendables, reduce_single_child],
"LEER": no_operation,
"LEER, TRENNER, ZSPRUNG": partial(change_parser, new_parser_name=WHITESPACE_KEYWORD),
"DATEI_ENDE": no_operation,
"NIEMALS": no_operation,
(TOKEN_KEYWORD, WHITESPACE_KEYWORD):
......@@ -284,9 +296,14 @@ MLW_AST_transformation_table = {
[remove_expendables, replace_by_single_child]
}
MLWTransform = partial(traverse, processing_table=MLW_AST_transformation_table)
def get_MLW_transformer():
return MLWTransform
#######################################################################
#
# COMPILER SECTION - Can be edited. Changes will be preserved.
......@@ -297,8 +314,8 @@ class MLWCompiler(CompilerBase):
"""Compiler for the abstract-syntax-tree of a MLW source file.
"""
def __init__(self, grammar_name="MLW"):
super(MLWCompiler, self).__init__()
def __init__(self, grammar_name="MLW", grammar_source=""):
super(MLWCompiler, self).__init__(grammar_name, grammar_source)
assert re.match('\w+\Z', grammar_name)
def on_Artikel(self, node):
......@@ -434,6 +451,19 @@ class MLWCompiler(CompilerBase):
pass
def get_MLW_compiler(grammar_name="MLW",
grammar_source=""):
global thread_local_MLW_compiler_singleton
try:
compiler = thread_local_MLW_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except NameError:
thread_local_MLW_compiler_singleton = \
MLWCompiler(grammar_name, grammar_source)
return thread_local_MLW_compiler_singleton
#######################################################################
#
# END OF DHPARSER-SECTIONS
......@@ -444,8 +474,16 @@ class MLWCompiler(CompilerBase):
def compile_MLW(source):
"""Compiles ``source`` and returns (result, errors, ast).
"""
return compile_source(source, MLWScanner,
MLWGrammar(), MLWTransform, MLWCompiler())
with logging("LOGS"):
compiler = get_MLW_compiler()
cname = compiler.__class__.__name__
log_file_name = os.path.basename(os.path.splitext(source)[0]) \
if is_filename(source) < 0 else cname[:cname.find('.')] + '_out'
result = compile_source(source, get_MLW_scanner(),
get_MLW_grammar(),
get_MLW_transformer(), compiler)
return result
if __name__ == "__main__":
if len(sys.argv) > 1:
......
This diff is collapsed.
......@@ -31,8 +31,6 @@ MLW_compiler = os.path.join('..', 'MLW_compiler.py')
# print(grammar_changed(MLW_ebnf, MLW_compiler))
toolkit.logging_off()
if (not os.path.exists(MLW_compiler) or
grammar_changed(MLW_compiler, MLW_ebnf)):
print("recompiling parser")
......
This diff is collapsed.
......@@ -23,10 +23,11 @@ from functools import partial
import sys
sys.path.extend(['../', './'])
from DHParser import parsers
from DHParser.toolkit import is_logging, compile_python_object
from DHParser.syntaxtree import no_operation, traverse, remove_expendables, \
replace_by_single_child, reduce_single_child, flatten, TOKEN_KEYWORD
from DHParser.parsers import compile_source, test_grammar
from DHParser.parsers import compile_source
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.dsl import parser_factory, DHPARSER_IMPORTS
......@@ -103,12 +104,12 @@ class TestGrammarTest:
}
}
def test_test_grammar(self):
def test_testing_grammar(self):
parser_fac = parser_factory(ARITHMETIC_EBNF)
trans_fac = lambda : ARITHMETIC_EBNFTransform
errata = test_grammar(self.cases, parser_fac, trans_fac)
errata = parsers.test_grammar(self.cases, parser_fac, trans_fac)
assert not errata
errata = test_grammar(self.failure_cases, parser_fac, trans_fac)
errata = parsers.test_grammar(self.failure_cases, parser_fac, trans_fac)
# for e in errata:
# print(e)
assert len(errata) == 3
......@@ -179,4 +180,4 @@ class TestRegex:
if __name__ == "__main__":
from run import runner
runner("TestGrammarTest", globals())
runner("", globals())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment