Commit c322ad0a authored by eckhart's avatar eckhart
Browse files

fixed/updated examples

parent a2f60b18
......@@ -84,16 +84,13 @@ class Compiler:
Attributes:
context: A list of parent nodes that ends with the currently
compiled node.
grammar_name: The name of the grammar this compiler is related to
grammar_source: The source code of the grammar this compiler is
related to.
_dirty_flag: A flag indicating that the compiler has already been
called at least once and that therefore all compilation
variables must be reset when it is called again.
"""
def __init__(self, grammar_name="", grammar_source=""):
self.set_grammar_name(grammar_name, grammar_source)
def __init__(self):
self._reset()
def _reset(self):
......@@ -116,22 +113,6 @@ class Compiler:
result = self.compile(root)
return result
def set_grammar_name(self, grammar_name: str = "", grammar_source: str = ""):
"""
Changes the grammar's name and the grammar's source.
The grammar name and the source text of the grammar are
metadata about the grammar that do not affect the compilation
process. Classes inheriting from `Compiler` can use this
information to name and annotate its output. Returns `self`.
"""
assert grammar_name == "" or re.match(r'\w+\Z', grammar_name)
if not grammar_name and re.fullmatch(r'[\w/:\\]+', grammar_source):
grammar_name = os.path.splitext(os.path.basename(grammar_source))[0]
self.grammar_name = grammar_name
self.grammar_source = load_if_file(grammar_source)
return self
# @staticmethod
# def propagate_error_flags(node: Node, lazy: bool = True) -> None:
# # See test_parser.TestCompilerClass.test_propagate_error()..
......
......@@ -101,7 +101,8 @@ from DHParser import logging, is_filename, load_if_file, MockParser, \\
remove_nodes, remove_content, remove_brackets, replace_parser, remove_anonymous_tokens, \\
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \\
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \\
replace_content, replace_content_by, error_on, recompile_grammar, GLOBALS
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \\
error_on, recompile_grammar, GLOBALS
'''.format(dhparserdir=dhparserdir)
......@@ -455,13 +456,14 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
compiler_name = os.path.basename(rootname)
if compiler_suite:
sfactory, pfactory, tfactory, cfactory = load_compiler_suite(compiler_suite)
compiler1 = cfactory()
else:
sfactory = get_ebnf_preprocessor
pfactory = get_ebnf_grammar
tfactory = get_ebnf_transformer
cfactory = get_ebnf_compiler
compiler1 = cfactory()
compiler1.set_grammar_name(compiler_name, source_file)
compiler1 = cfactory()
compiler1.set_grammar_name(compiler_name, source_file)
result, messages, _ = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
if has_errors(messages):
......
......@@ -300,13 +300,11 @@ def get_transformer() -> TransformationFunc:
COMPILER_FACTORY = '''
def get_compiler(grammar_name="{NAME}", grammar_source="") -> {NAME}Compiler:
def get_compiler() -> {NAME}Compiler:
try:
compiler = GLOBALS.{NAME}_{ID}_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
except AttributeError:
GLOBALS.{NAME}_{ID}_compiler_singleton = \\
{NAME}Compiler(grammar_name, grammar_source)
GLOBALS.{NAME}_{ID}_compiler_singleton = {NAME}Compiler()
compiler = GLOBALS.{NAME}_{ID}_compiler_singleton
return compiler
'''
......@@ -378,6 +376,10 @@ class EBNFCompiler(Compiler):
re_flags: A set of regular expression flags to be added to all
regular expressions found in the current parsing process
grammar_name: The name of the grammar to be compiled
grammar_source: The source code of the grammar to be compiled.
grammar_id: a unique id for every compiled grammar. (Required for
disambiguation of of thread local variables storing
compiled texts.)
......@@ -399,10 +401,10 @@ class EBNFCompiler(Compiler):
REPEATABLE_DIRECTIVES = {'tokens'}
def __init__(self, grammar_name="", grammar_source=""):
def __init__(self, grammar_name="DSL", grammar_source=""):
self.grammar_id = 0
super(EBNFCompiler, self).__init__(grammar_name, grammar_source)
# self._reset()
super(EBNFCompiler, self).__init__() # calls the _reset()-method
self.set_grammar_name(grammar_name, grammar_source)
def _reset(self):
......@@ -431,6 +433,23 @@ class EBNFCompiler(Compiler):
def result(self) -> str:
return self._result
def set_grammar_name(self, grammar_name: str = "", grammar_source: str = ""):
"""
Changes the grammar name and source.
The grammar name and the source text are metadata that do not affect the
compilation process. It is used to name and annotate the output.
Returns `self`.
"""
assert grammar_name == "" or re.match(r'\w+\Z', grammar_name)
if not grammar_name and re.fullmatch(r'[\w/:\\]+', grammar_source):
grammar_name = os.path.splitext(os.path.basename(grammar_source))[0]
self.grammar_name = grammar_name
self.grammar_source = load_if_file(grammar_source)
return self
# methods for generating skeleton code for preprocessor, transformer, and compiler
def gen_preprocessor_skeleton(self) -> str:
......@@ -481,11 +500,9 @@ class EBNFCompiler(Compiler):
' """Compiler for the abstract-syntax-tree of a '
+ self.grammar_name + ' source file.',
' """', '',
' def __init__(self, grammar_name="'
+ self.grammar_name + '", grammar_source=""):',
' super(' + self.grammar_name
+ 'Compiler, self).__init__(grammar_name, grammar_source)',
r" assert re.match('\w+\Z', grammar_name)", '',
' def __init__(self):',
' super(' + self.grammar_name + 'Compiler, self).__init__()',
'',
' def _reset(self):',
' super()._reset()',
' # initialize your variables here, not in the constructor!']
......@@ -992,5 +1009,7 @@ def get_ebnf_compiler(grammar_name="", grammar_source="") -> EBNFCompiler:
compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except AttributeError:
GLOBALS.ebnf_compiler_singleton = EBNFCompiler(grammar_name, grammar_source)
return GLOBALS.ebnf_compiler_singleton
compiler = EBNFCompiler(grammar_name, grammar_source)
compiler.set_grammar_name(grammar_name, grammar_source)
GLOBALS.ebnf_compiler_singleton = compiler
return compiler
......@@ -69,6 +69,7 @@ __all__ = ('escape_re',
GLOBALS = threading.local()
GLOBALS.config = {}
def escape_re(strg: str) -> str:
......
......@@ -6,3 +6,4 @@ constant = digit {digit}
digit = "0" | "1" | "..." | "9"
test = digit constant variable
......@@ -10,6 +10,9 @@
from functools import partial
import os
import sys
sys.path.extend(['../../', '../', './'])
try:
import regex as re
except ImportError:
......@@ -21,7 +24,7 @@ from DHParser import logging, is_filename, load_if_file, \
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
last_value, counterpart, accumulate, PreprocessorFunc, \
Node, TransformationFunc, TransformationDict, \
traverse, remove_children_if, merge_children, is_anonymous, Whitespace, \
traverse, remove_children_if, is_anonymous, Whitespace, \
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \
is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
......@@ -64,13 +67,11 @@ class ArithmeticGrammar(Grammar):
digit = Forward()
expression = Forward()
variable = Forward()
source_hash__ = "385a94a70cb629d46a13e15305692667"
source_hash__ = "c4e6e090ef9673b972ba18ef39fe7c8e"
parser_initialization__ = "upon instantiation"
COMMENT__ = r''
WHITESPACE__ = r'\s*'
WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
wspL__ = ''
wspR__ = WSP_RE__
wsp__ = Whitespace(WSP_RE__)
test = Series(digit, constant, variable)
digit.set(Alternative(Series(Token("0"), wsp__), Series(Token("1"), wsp__), Series(Token("..."), wsp__), Series(Token("9"), wsp__)))
......@@ -82,12 +83,11 @@ class ArithmeticGrammar(Grammar):
root__ = expression
def get_grammar() -> ArithmeticGrammar:
global thread_local_Arithmetic_grammar_singleton
try:
grammar = thread_local_Arithmetic_grammar_singleton
except NameError:
thread_local_Arithmetic_grammar_singleton = ArithmeticGrammar()
grammar = thread_local_Arithmetic_grammar_singleton
grammar = GLOBALS.Arithmetic_1_grammar_singleton
except AttributeError:
GLOBALS.Arithmetic_1_grammar_singleton = ArithmeticGrammar()
grammar = GLOBALS.Arithmetic_1_grammar_singleton
return grammar
......@@ -135,10 +135,6 @@ class ArithmeticCompiler(Compiler):
"""Compiler for the abstract-syntax-tree of a Arithmetic source file.
"""
def __init__(self, grammar_name="Arithmetic", grammar_source=""):
super(ArithmeticCompiler, self).__init__(grammar_name, grammar_source)
assert re.match('\w+\Z', grammar_name)
def on_expression(self, node):
return node
......@@ -161,14 +157,12 @@ class ArithmeticCompiler(Compiler):
# return node
def get_compiler(grammar_name="Arithmetic", grammar_source="") -> ArithmeticCompiler:
def get_compiler() -> ArithmeticCompiler:
global thread_local_Arithmetic_compiler_singleton
try:
compiler = thread_local_Arithmetic_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
except NameError:
thread_local_Arithmetic_compiler_singleton = \
ArithmeticCompiler(grammar_name, grammar_source)
thread_local_Arithmetic_compiler_singleton = ArithmeticCompiler()
compiler = thread_local_Arithmetic_compiler_singleton
return compiler
......
File mode changed from 100644 to 100755
......@@ -24,7 +24,7 @@ from DHParser import is_filename, load_if_file, \
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
last_value, counterpart, accumulate, PreprocessorFunc, \
Node, TransformationDict, Whitespace, \
traverse, remove_children_if, merge_children, is_anonymous, \
traverse, remove_children_if, is_anonymous, \
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \
is_empty, is_expendable, collapse, replace_content, remove_nodes, remove_content, remove_brackets, replace_parser, \
......@@ -189,11 +189,6 @@ def get_transformer() -> TransformationFunc:
class BibTeXCompiler(Compiler):
"""Compiler for the abstract-syntax-tree of a BibTeX source file.
"""
def __init__(self, grammar_name="BibTeX", grammar_source=""):
super(BibTeXCompiler, self).__init__(grammar_name, grammar_source)
assert re.match('\w+\Z', grammar_name)
def on_bibliography(self, node):
return node
......@@ -228,14 +223,12 @@ class BibTeXCompiler(Compiler):
pass
def get_compiler(grammar_name="BibTeX", grammar_source="") -> BibTeXCompiler:
def get_compiler() -> BibTeXCompiler:
global thread_local_BibTeX_compiler_singleton
try:
compiler = thread_local_BibTeX_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
except NameError:
thread_local_BibTeX_compiler_singleton = \
BibTeXCompiler(grammar_name, grammar_source)
thread_local_BibTeX_compiler_singleton = BibTeXCompiler()
compiler = thread_local_BibTeX_compiler_singleton
return compiler
......
File mode changed from 100644 to 100755
......@@ -11,7 +11,7 @@ from functools import partial
import os
import sys
sys.path.extend(['..\\', '..\\..\\'])
sys.path.extend(['../', '../../', './'])
try:
import regex as re
......@@ -24,13 +24,13 @@ from DHParser import is_filename, load_if_file, \
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
last_value, counterpart, accumulate, PreprocessorFunc, \
Node, TransformationFunc, TransformationDict, Whitespace, \
traverse, remove_children_if, merge_children, is_anonymous, \
traverse, remove_children_if, is_anonymous, \
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \
is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
remove_nodes, remove_content, remove_brackets, replace_parser, \
keep_children, is_one_of, has_content, apply_if, remove_first, remove_last, \
forbid, assert_content, remove_infix_operator
forbid, assert_content, remove_infix_operator, GLOBALS
from DHParser.log import logging
......@@ -100,13 +100,11 @@ class EBNFGrammar(Grammar):
EOF = !/./
"""
expression = Forward()
source_hash__ = "97b616756462a59e1f5162a95ae84c5f"
source_hash__ = "6099690fa36228d49e6c35ec60750c59"
parser_initialization__ = "upon instantiation"
COMMENT__ = r'#.*(?:\n|$)'
WHITESPACE__ = r'\s*'
WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
wspL__ = ''
wspR__ = WSP_RE__
wsp__ = Whitespace(WSP_RE__)
EOF = NegativeLookahead(RegExp('.'))
list_ = Series(RegExp('\\w+'), wsp__, ZeroOrMore(Series(Series(Token(","), wsp__), RegExp('\\w+'), wsp__)))
......@@ -131,12 +129,11 @@ class EBNFGrammar(Grammar):
root__ = syntax
def get_grammar() -> EBNFGrammar:
global thread_local_EBNF_grammar_singleton
try:
grammar = thread_local_EBNF_grammar_singleton
except NameError:
thread_local_EBNF_grammar_singleton = EBNFGrammar()
grammar = thread_local_EBNF_grammar_singleton
grammar = GLOBALS.EBNF_2_grammar_singleton
except AttributeError:
GLOBALS.EBNF_2_grammar_singleton = EBNFGrammar()
grammar = GLOBALS.EBNF_2_grammar_singleton
return grammar
......@@ -201,11 +198,6 @@ def get_transformer() -> TransformationFunc:
class EBNFCompiler(Compiler):
"""Compiler for the abstract-syntax-tree of a EBNF source file.
"""
def __init__(self, grammar_name="EBNF", grammar_source=""):
super(EBNFCompiler, self).__init__(grammar_name, grammar_source)
assert re.match('\w+\Z', grammar_name)
def on_syntax(self, node):
return node
......@@ -258,14 +250,12 @@ class EBNFCompiler(Compiler):
pass
def get_compiler(grammar_name="EBNF", grammar_source="") -> EBNFCompiler:
def get_compiler() -> EBNFCompiler:
global thread_local_EBNF_compiler_singleton
try:
compiler = thread_local_EBNF_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
except NameError:
thread_local_EBNF_compiler_singleton = \
EBNFCompiler(grammar_name, grammar_source)
thread_local_EBNF_compiler_singleton = EBNFCompiler()
compiler = thread_local_EBNF_compiler_singleton
return compiler
......
......@@ -103,13 +103,11 @@ class EBNF_oldGrammar(Grammar):
EOF = !/./
"""
expression = Forward()
source_hash__ = "876bb760b35a20924a3ee449820f4316"
source_hash__ = "249997be7111ca806939bf18070e136e"
parser_initialization__ = "upon instantiation"
COMMENT__ = r'#.*(?:\n|$)'
WHITESPACE__ = r'\s*'
WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
wspL__ = ''
wspR__ = WSP_RE__
wsp__ = Whitespace(WSP_RE__)
EOF = NegativeLookahead(RegExp('.'))
list_ = Series(RegExp('\\w+'), wsp__, ZeroOrMore(Series(Series(Token(","), wsp__), RegExp('\\w+'), wsp__)))
......@@ -134,12 +132,11 @@ class EBNF_oldGrammar(Grammar):
root__ = syntax
def get_grammar() -> EBNF_oldGrammar:
global thread_local_EBNF_old_grammar_singleton
try:
grammar = thread_local_EBNF_old_grammar_singleton
except NameError:
thread_local_EBNF_old_grammar_singleton = EBNF_oldGrammar()
grammar = thread_local_EBNF_old_grammar_singleton
grammar = GLOBALS.EBNF_old_1_grammar_singleton
except AttributeError:
GLOBALS.EBNF_old_1_grammar_singleton = EBNF_oldGrammar()
grammar = GLOBALS.EBNF_old_1_grammar_singleton
return grammar
......
......@@ -24,7 +24,7 @@ from DHParser import is_filename, Grammar, Compiler, Lookbehind, Alternative, Po
Node, TransformationFunc, traverse, remove_children_if, is_anonymous, \
reduce_single_child, replace_by_single_child, remove_whitespace, \
flatten, is_empty, collapse, replace_content, replace_content_by, remove_brackets, \
is_one_of, traverse_locally, remove_tokens, remove_nodes, TOKEN_PTYPE, Error
is_one_of, traverse_locally, remove_tokens, remove_nodes, TOKEN_PTYPE, Error, GLOBALS
from DHParser.log import logging
......@@ -332,12 +332,11 @@ class LaTeXGrammar(Grammar):
root__ = latexdoc
def get_grammar() -> LaTeXGrammar:
global thread_local_LaTeX_grammar_singleton
try:
grammar = thread_local_LaTeX_grammar_singleton
except NameError:
thread_local_LaTeX_grammar_singleton = LaTeXGrammar()
grammar = thread_local_LaTeX_grammar_singleton
grammar = GLOBALS.LaTeX_1_grammar_singleton
except AttributeError:
GLOBALS.LaTeX_1_grammar_singleton = LaTeXGrammar()
grammar = GLOBALS.LaTeX_1_grammar_singleton
return grammar
......@@ -496,9 +495,8 @@ class LaTeXCompiler(Compiler):
KNOWN_DOCUMENT_CLASSES = {'book', 'article'}
KNOWN_LANGUAGES = {'english', 'german'}
def __init__(self, grammar_name="LaTeX", grammar_source=""):
super(LaTeXCompiler, self).__init__(grammar_name, grammar_source)
assert re.match('\w+\Z', grammar_name)
def __init__(self):
super(LaTeXCompiler, self).__init__()
self.metadata = defaultdict(empty_defaultdict)
# def on_latexdoc(self, node):
......@@ -771,14 +769,12 @@ class LaTeXCompiler(Compiler):
# return node
def get_compiler(grammar_name="LaTeX", grammar_source="") -> LaTeXCompiler:
def get_compiler() -> LaTeXCompiler:
global thread_local_LaTeX_compiler_singleton
try:
compiler = thread_local_LaTeX_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
except NameError:
thread_local_LaTeX_compiler_singleton = \
LaTeXCompiler(grammar_name, grammar_source)
thread_local_LaTeX_compiler_singleton = LaTeXCompiler()
compiler = thread_local_LaTeX_compiler_singleton
return compiler
......
......@@ -192,10 +192,6 @@ class LyrikCompiler(Compiler):
"""Compiler for the abstract-syntax-tree of a Lyrik source file.
"""
def __init__(self, grammar_name="Lyrik", grammar_source=""):
super(LyrikCompiler, self).__init__(grammar_name, grammar_source)
assert re.match('\w+\Z', grammar_name)
def on_gedicht(self, node):
return node
......@@ -269,15 +265,13 @@ class LyrikCompiler(Compiler):
pass
def get_compiler(grammar_name="Lyrik", grammar_source="") -> LyrikCompiler:
def get_compiler() -> LyrikCompiler:
global thread_local_Lyrik_compiler_singleton
try:
compiler = thread_local_Lyrik_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
return compiler
except NameError:
thread_local_Lyrik_compiler_singleton = \
LyrikCompiler(grammar_name, grammar_source)
thread_local_Lyrik_compiler_singleton = LyrikCompiler()
return thread_local_Lyrik_compiler_singleton
......
......@@ -546,9 +546,8 @@ class XMLCompiler(Compiler):
"""Compiler for the abstract-syntax-tree of a XML source file.
"""
def __init__(self, grammar_name="XML", grammar_source=""):
super(XMLCompiler, self).__init__(grammar_name, grammar_source)
assert re.match('\w+\Z', grammar_name)
def __init__(self):
super(XMLCompiler, self).__init__()
self.cleanup_whitespace = True # remove empty CharData from mixed elements
def _reset(self):
......@@ -909,14 +908,12 @@ class XMLCompiler(Compiler):
# return node
def get_compiler(grammar_name="XML", grammar_source="") -> XMLCompiler:
def get_compiler() -> XMLCompiler:
global thread_local_XML_compiler_singleton
try:
compiler = thread_local_XML_compiler_singleton
compiler.set_grammar_name(grammar_name, grammar_source)
except NameError:
thread_local_XML_compiler_singleton = \
XMLCompiler(grammar_name, grammar_source)
thread_local_XML_compiler_singleton = XMLCompiler()
compiler = thread_local_XML_compiler_singleton
return compiler
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment