dsl.py 24.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# dsl.py - Support for domain specific notations for DHParser
#
# Copyright 2016  by Eckhart Arnold (arnold@badw.de)
#                 Bavarian Academy of Sciences an Humanities (badw.de)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
17
18


19
"""
20
Module ``dsl`` contains various functions to support the
21
22
23
compilation of domain specific languages based on an EBNF-grammar.
"""

24

25
import os
eckhart's avatar
eckhart committed
26
27
import platform
import stat
28

eckhart's avatar
eckhart committed
29
from DHParser.compile import Compiler, compile_source
30
from DHParser.ebnf import EBNFCompiler, grammar_changed, \
31
32
    get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
    PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
33
from DHParser.error import Error, is_error, has_errors, only_errors
34
from DHParser.log import logging
35
from DHParser.parse import Grammar
36
from DHParser.preprocess import nil_preprocessor, PreprocessorFunc
37
from DHParser.syntaxtree import Node
eckhart's avatar
eckhart committed
38
from DHParser.transform import TransformationFunc
39
from DHParser.toolkit import load_if_file, is_python_code, compile_python_object, \
eckhart's avatar
eckhart committed
40
    re, typing
41
from typing import Any, cast, List, Tuple, Union, Iterator, Iterable, Optional, Callable
eckhart's avatar
eckhart committed
42

43

44
45
__all__ = ('DHPARSER_IMPORTS',
           'GrammarError',
Eckhart Arnold's avatar
Eckhart Arnold committed
46
47
48
           'CompilationError',
           'load_compiler_suite',
           'compileDSL',
Eckhart Arnold's avatar
Eckhart Arnold committed
49
           'raw_compileEBNF',
50
           'compileEBNF',
51
           'grammar_provider',
52
53
           'compile_on_disk',
           'recompile_grammar')
Eckhart Arnold's avatar
Eckhart Arnold committed
54
55


56
57
58
59
60
61
62
63
64
SECTION_MARKER = """\n
#######################################################################
#
# {marker}
#
#######################################################################
\n"""

RX_SECTION_MARKER = re.compile(SECTION_MARKER.format(marker=r'.*?SECTION.*?'))
65
RX_WHITESPACE = re.compile(r'\s*')
66
67

SYMBOLS_SECTION = "SYMBOLS SECTION - Can be edited. Changes will be preserved."
68
PREPROCESSOR_SECTION = "PREPROCESSOR SECTION - Can be edited. Changes will be preserved."
69
70
71
PARSER_SECTION = "PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!"
AST_SECTION = "AST SECTION - Can be edited. Changes will be preserved."
COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved."
di68kap's avatar
di68kap committed
72
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
73
74


eckhart's avatar
eckhart committed
75
76
77
dhparserdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))


78
DHPARSER_IMPORTS = '''
79
import collections
di68kap's avatar
di68kap committed
80
from functools import partial
Eckhart Arnold's avatar
Eckhart Arnold committed
81
import os
82
import sys
eckhart's avatar
eckhart committed
83

84
sys.path.append(r'{dhparserdir}')
eckhart's avatar
eckhart committed
85

di68kap's avatar
di68kap committed
86
87
88
89
try:
    import regex as re
except ImportError:
    import re
di68kap's avatar
di68kap committed
90
from DHParser import logging, is_filename, load_if_file, MockParser, \\
91
    Grammar, Compiler, nil_preprocessor, PreprocessorToken, Whitespace, \\
92
    Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, Unordered, \\
93
    Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \\
eckhart's avatar
eckhart committed
94
    ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \\
95
    grammar_changed, last_value, counterpart, accumulate, PreprocessorFunc, \\
di68kap's avatar
di68kap committed
96
97
    Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \\
    remove_children_if, move_whitespace, normalize_whitespace, is_anonymous, matches_re, \\
Eckhart Arnold's avatar
Eckhart Arnold committed
98
    reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
di68kap's avatar
di68kap committed
99
100
    remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, is_empty, \\
    is_expendable, collapse, collapse_if, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \\
101
    remove_nodes, remove_content, remove_brackets, replace_parser, remove_anonymous_tokens, \\
di68kap's avatar
di68kap committed
102
    keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \\
103
    remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \\
104
    replace_content, replace_content_by, recompile_grammar
eckhart's avatar
eckhart committed
105
'''.format(dhparserdir=dhparserdir)
di68kap's avatar
di68kap committed
106

107

Eckhart Arnold's avatar
Eckhart Arnold committed
108
DHPARSER_MAIN = '''
109
def compile_src(source, log_dir=''):
110
111
    """Compiles ``source`` and returns (result, errors, ast).
    """
di68kap's avatar
di68kap committed
112
    with logging(log_dir):
113
        compiler = get_compiler()
Eckhart Arnold's avatar
Eckhart Arnold committed
114
        cname = compiler.__class__.__name__
115
        result = compile_source(source, get_preprocessor(),
116
117
                                get_grammar(),
                                get_transformer(), compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
118
119
    return result

120
121

if __name__ == "__main__":
122
123
124
125
126
127
128
129
130
131
132
133
134
    # recompile grammar if needed
    grammar_path = os.path.abspath(__file__).replace('Compiler.py', '.ebnf')
    if os.path.exists(grammar_path):
        if not recompile_grammar(grammar_path, force=False,
                                  notify=lambda:print('recompiling ' + grammar_path)):
            error_file = os.path.basename(__file__).replace('Compiler.py', '_ebnf_ERRORS.txt')
            with open(error_file, encoding="utf-8") as f:
                print(f.read())
            sys.exit(1)
    else:
        print('Could not check whether grammar requires recompiling, '
              'because grammar was not found at: ' + grammar_path)

135
    if len(sys.argv) > 1:
136
        # compile file 
di68kap's avatar
di68kap committed
137
138
139
        file_name, log_dir = sys.argv[1], ''
        if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
            file_name, log_dir = sys.argv[2], 'LOGS'
140
        result, errors, ast = compile_src(file_name, log_dir)
141
        if errors:
di68kap's avatar
di68kap committed
142
143
            cwd = os.getcwd()
            rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
144
            for error in errors:
di68kap's avatar
di68kap committed
145
                print(rel_path + ':' + str(error))
146
            sys.exit(1)
147
        else:
148
            print(result.as_xml() if isinstance(result, Node) else result)
149
    else:
150
        print("Usage: {NAME}Compiler.py [FILENAME]")
151
152
'''

153

154
155
156
157
158
class DSLException(Exception):
    """
    Base class for DSL-exceptions.
    """
    def __init__(self, errors):
Eckhart Arnold's avatar
Eckhart Arnold committed
159
160
        assert isinstance(errors, Iterator) or isinstance(errors, list) \
               or isinstance(errors, tuple)
161
162
163
164
165
166
167
        self.errors = errors

    def __str__(self):
        return '\n'.join(str(err) for err in self.errors)


class GrammarError(DSLException):
168
169
    """
    Raised when (already) the grammar of a domain specific language (DSL)
170
171
    contains errors.
    """
172
173
    def __init__(self, errors, grammar_src):
        super().__init__(errors)
174
175
176
        self.grammar_src = grammar_src


177
class CompilationError(DSLException):
178
179
    """
    Raised when a string or file in a domain specific language (DSL)
180
181
    contains errors.
    """
182
183
    def __init__(self, errors, dsl_text, dsl_grammar, AST, result):
        super().__init__(errors)
184
185
186
        self.dsl_text = dsl_text
        self.dsl_grammar = dsl_grammar
        self.AST = AST
187
        self.result = result
188
189


190
191
192
193
194
def error_str(messages: Iterable[Error]) -> str:
    """
    Returns all true errors (i.e. not just warnings) from the
    `messages` as a concatenated multiline string.
    """
195
    return '\n\n'.join(str(m) for m in messages if is_error(m.code))
196
197


198
def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
199
200
    """
    Returns a grammar object and the source code of the grammar, from
201
    the given `grammar`-data which can be either a file name, ebnf-code,
202
    python-code, a Grammar-derived grammar class or an instance of
203
204
    such a class (i.e. a grammar object already).
    """
205
    if isinstance(grammar_representation, str):
206
        # read grammar
207
        grammar_src = load_if_file(grammar_representation)
208
        if is_python_code(grammar_src):
eckhart's avatar
eckhart committed
209
            parser_py, messages = grammar_src, []  # type: str, List[Error]
210
        else:
Eckhart Arnold's avatar
Eckhart Arnold committed
211
            with logging(False):
eckhart's avatar
eckhart committed
212
213
                parser_py, messages, _ = compile_source(
                    grammar_src, None,
Eckhart Arnold's avatar
Eckhart Arnold committed
214
                    get_ebnf_grammar(), get_ebnf_transformer(), get_ebnf_compiler())
215
        if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
216
            raise GrammarError(only_errors(messages), grammar_src)
eckhart's avatar
eckhart committed
217
        parser_root = compile_python_object(DHPARSER_IMPORTS + parser_py, r'\w+Grammar$')()
218
219
220
    else:
        # assume that dsl_grammar is a ParserHQ-object or Grammar class
        grammar_src = ''
221
        if isinstance(grammar_representation, Grammar):
222
            parser_root = grammar_representation
223
        else:
Eckhart Arnold's avatar
Eckhart Arnold committed
224
            # assume ``grammar_representation`` is a grammar class and get the root object
225
            parser_root = grammar_representation()
226
227
228
    return parser_root, grammar_src


229
def compileDSL(text_or_file: str,
230
               preprocessor: PreprocessorFunc,
231
               dsl_grammar: Union[str, Grammar],
232
               ast_transformation: TransformationFunc,
233
               compiler: Compiler) -> Any:
234
235
    """
    Compiles a text in a domain specific language (DSL) with an
236
237
    EBNF-specified grammar. Returns the compiled text or raises a
    compilation error.
eckhart's avatar
eckhart committed
238

239
    Raises:
240
        CompilationError if any errors occurred during compilation
241
242
    """
    assert isinstance(text_or_file, str)
243
    assert isinstance(compiler, Compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
244

245
    parser, grammar_src = grammar_instance(dsl_grammar)
246
    result, messages, AST = compile_source(text_or_file, preprocessor, parser,
247
                                           ast_transformation, compiler)
248
    if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
249
        src = load_if_file(text_or_file)
Eckhart Arnold's avatar
Eckhart Arnold committed
250
        raise CompilationError(only_errors(messages), src, grammar_src, AST, result)
251
252
253
    return result


254
def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
255
256
    """
    Compiles an EBNF grammar file and returns the compiler object
Eckhart Arnold's avatar
Eckhart Arnold committed
257
    that was used and which can now be queried for the result as well
258
    as skeleton code for preprocessor, transformer and compiler objects.
eckhart's avatar
eckhart committed
259

Eckhart Arnold's avatar
Eckhart Arnold committed
260
261
262
263
    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
        branding (str):  Branding name for the compiler suite source
eckhart's avatar
eckhart committed
264
            code.
Eckhart Arnold's avatar
Eckhart Arnold committed
265
266
267
    Returns:
        An instance of class ``ebnf.EBNFCompiler``
    Raises:
eckhart's avatar
eckhart committed
268
        CompilationError if any errors occurred during compilation
Eckhart Arnold's avatar
Eckhart Arnold committed
269
270
    """
    grammar = get_ebnf_grammar()
271
    compiler = get_ebnf_compiler(branding, ebnf_src)
272
273
    transformer = get_ebnf_transformer()
    compileDSL(ebnf_src, nil_preprocessor, grammar, transformer, compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
274
275
276
    return compiler


277
def compileEBNF(ebnf_src: str, branding="DSL") -> str:
278
279
    """
    Compiles an EBNF source file and returns the source code of a
280
    compiler suite with skeletons for preprocessor, transformer and
Eckhart Arnold's avatar
Eckhart Arnold committed
281
    compiler.
282
283
284
285

    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
286
        branding (str):  Branding name for the compiler suite source
eckhart's avatar
eckhart committed
287
            code.
288
    Returns:
289
        The complete compiler suite skeleton as Python source code.
290
    Raises:
eckhart's avatar
eckhart committed
291
        CompilationError if any errors occurred during compilation
292
    """
Eckhart Arnold's avatar
Eckhart Arnold committed
293
    compiler = raw_compileEBNF(ebnf_src, branding)
294
295
    src = ["#/usr/bin/python\n",
           SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_IMPORTS,
296
           SECTION_MARKER.format(marker=PREPROCESSOR_SECTION), compiler.gen_preprocessor_skeleton(),
Eckhart Arnold's avatar
Eckhart Arnold committed
297
           SECTION_MARKER.format(marker=PARSER_SECTION), compiler.result,
298
299
300
           SECTION_MARKER.format(marker=AST_SECTION), compiler.gen_transformer_skeleton(),
           SECTION_MARKER.format(marker=COMPILER_SECTION), compiler.gen_compiler_skeleton(),
           SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_MAIN.format(NAME=branding)]
301
302
303
    return '\n'.join(src)


304
def grammar_provider(ebnf_src: str, branding="DSL") -> Grammar:
305
    """
306
    Compiles an EBNF grammar and returns a grammar-parser provider
307
308
309
310
311
312
    function for that grammar.

    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
        branding (str or bool):  Branding name for the compiler
eckhart's avatar
eckhart committed
313
314
            suite source code.

315
    Returns:
316
        A provider function for a grammar object for texts in the
317
318
        language defined by ``ebnf_src``.
    """
319
    grammar_src = compileDSL(ebnf_src, nil_preprocessor, get_ebnf_grammar(),
eckhart's avatar
eckhart committed
320
321
322
323
                             get_ebnf_transformer(), get_ebnf_compiler(branding, ebnf_src))
    grammar_obj = compile_python_object(DHPARSER_IMPORTS + grammar_src, r'get_(?:\w+_)?grammar$')
    grammar_obj.python_src__ = grammar_src
    return grammar_obj
324
325


326
def load_compiler_suite(compiler_suite: str) -> \
327
328
        Tuple[PreprocessorFactoryFunc, ParserFactoryFunc,
              TransformerFactoryFunc, CompilerFactoryFunc]:
329
    """
330
    Extracts a compiler suite from file or string `compiler_suite`
331
    and returns it as a tuple (preprocessor, parser, ast, compiler).
eckhart's avatar
eckhart committed
332

Eckhart Arnold's avatar
Eckhart Arnold committed
333
    Returns:
334
335
        4-tuple (preprocessor function, parser class,
                 ast transformer function, compiler class)
336
337
338
339
    """
    global RX_SECTION_MARKER
    assert isinstance(compiler_suite, str)
    source = load_if_file(compiler_suite)
340
    imports = DHPARSER_IMPORTS
341
342
    if is_python_code(compiler_suite):
        try:
eckhart's avatar
eckhart committed
343
            _, imports, preprocessor_py, parser_py, ast_py, compiler_py, _ = \
344
                RX_SECTION_MARKER.split(source)
eckhart's avatar
eckhart committed
345
        except ValueError:
346
347
            raise AssertionError('File "' + compiler_suite + '" seems to be corrupted. '
                                 'Please delete or repair file manually.')
348
        # TODO: Compile in one step and pick parts from namespace later ?
eckhart's avatar
eckhart committed
349
350
351
352
        preprocessor = compile_python_object(imports + preprocessor_py,
                                             r'get_(?:\w+_)?preprocessor$')
        parser = compile_python_object(imports + parser_py, r'get_(?:\w+_)?grammar$')
        ast = compile_python_object(imports + ast_py, r'get_(?:\w+_)?transformer$')
353
    else:
354
355
        # Assume source is an ebnf grammar.
        # Is there really any reasonable application case for this?
Eckhart Arnold's avatar
Eckhart Arnold committed
356
        with logging(False):
eckhart's avatar
eckhart committed
357
            compiler_py, messages, n = compile_source(source, None, get_ebnf_grammar(),
eckhart's avatar
eckhart committed
358
359
                                                      get_ebnf_transformer(),
                                                      get_ebnf_compiler(compiler_suite, source))
360
        if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
361
            raise GrammarError(only_errors(messages), source)
362
        preprocessor = get_ebnf_preprocessor
363
        parser = get_ebnf_grammar
364
        ast = get_ebnf_transformer
eckhart's avatar
eckhart committed
365
    compiler = compile_python_object(imports + compiler_py, r'get_(?:\w+_)?compiler$')
366

367
    return preprocessor, parser, ast, compiler
368
369


370
def is_outdated(compiler_suite: str, grammar_source: str) -> bool:
371
372
    """
    Returns ``True``  if the ``compile_suite`` needs to be updated.
eckhart's avatar
eckhart committed
373
374
375

    An update is needed, if either the grammar in the compieler suite
    does not reflect the latest changes of ``grammar_source`` or if
376
377
    sections from the compiler suite have diligently been overwritten
    with whitespace order to trigger their recreation. Note: Do not
eckhart's avatar
eckhart committed
378
    delete or overwrite the section marker itself.
379

380
    Args:
381
382
383
384
385
386
387
388
389
        compiler_suite:  the parser class representing the grammar
            or the file name of a compiler suite containing the grammar
        grammar_source:  File name or string representation of the
            EBNF code of the grammar

    Returns (bool):
        True, if ``compiler_suite`` seems to be out of date.
    """
    try:
eckhart's avatar
eckhart committed
390
        n1, grammar, n2, n3 = load_compiler_suite(compiler_suite)
391
        return grammar_changed(grammar(), grammar_source)
392
393
394
395
    except ValueError:
        return True


396
def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
397
398
399
400
    """Compiles a source with a given compiler suite.

    Args:
        text_or_file (str):  Either the file name of the source code or
eckhart's avatar
eckhart committed
401
            the source code directly. (Which is determined by
402
403
404
405
406
            heuristics. If ``text_or_file`` contains at least on
            linefeed then it is always assumed to be a source text and
            not a file name.)
        compiler_suite(str):  File name of the compiler suite to be
            used.
eckhart's avatar
eckhart committed
407

408
    Returns:
eckhart's avatar
eckhart committed
409
        The result of the compilation, the form and type of which
410
        depends entirely on the compiler.
eckhart's avatar
eckhart committed
411

412
413
414
    Raises:
        CompilerError
    """
415
416
    preprocessor, parser, ast, compiler = load_compiler_suite(compiler_suite)
    return compileDSL(text_or_file, preprocessor(), parser(), ast(), compiler())
417
418


419
def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> Iterable[Error]:
420
421
    """
    Compiles the a source file with a given compiler and writes the
422
423
    result to a file.

Eckhart Arnold's avatar
Eckhart Arnold committed
424
425
426
    If no ``compiler_suite`` is given it is assumed that the source
    file is an EBNF grammar. In this case the result will be a Python
    script containing a parser for that grammar as well as the
427
    skeletons for a preprocessor, AST transformation table, and compiler.
Eckhart Arnold's avatar
Eckhart Arnold committed
428
429
    If the Python script already exists only the parser name in the
    script will be updated. (For this to work, the different names
430
    need to be delimited section marker blocks.). `compile_on_disk()`
Eckhart Arnold's avatar
Eckhart Arnold committed
431
432
    returns a list of error messages or an empty list if no errors
    occurred.
433

434
435
436
437
    Parameters:
        source_file(str):  The file name of the source text to be
            compiled.
        compiler_suite(str):  The file name of the compiler suite
438
            (usually ending with 'Compiler.py'), with which the source
439
440
441
442
443
444
445
            file shall be compiled. If this is left empty, the source
            file is assumed to be an EBNF-Grammar that will be compiled
            with the internal EBNF-Compiler.
        extension(str):  The result of the compilation (if successful)
            is written to a file with the same name but a different
            extension than the source file. This parameter sets the
            extension.
eckhart's avatar
eckhart committed
446

447
    Returns:
448
        A (potentially empty) list of error or warning messages.
Eckhart Arnold's avatar
Eckhart Arnold committed
449
    """
450
    filepath = os.path.normpath(source_file)
451
452
    with open(source_file, encoding="utf-8") as f:
        source = f.read()
453
    rootname = os.path.splitext(filepath)[0]
454
    compiler_name = os.path.basename(rootname)
455
    if compiler_suite:
456
        sfactory, pfactory, tfactory, cfactory = load_compiler_suite(compiler_suite)
457
    else:
458
        sfactory = get_ebnf_preprocessor
459
460
        pfactory = get_ebnf_grammar
        tfactory = get_ebnf_transformer
461
        cfactory = get_ebnf_compiler
462
463
    compiler1 = cfactory()
    compiler1.set_grammar_name(compiler_name, source_file)
464
    result, messages, AST = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
465
466
    if has_errors(messages):
        return messages
467

468
469
470
    elif cfactory == get_ebnf_compiler:
        # trans == get_ebnf_transformer or trans == EBNFTransformer:
        # either an EBNF- or no compiler suite given
471
        ebnf_compiler = cast(EBNFCompiler, compiler1)
472
        global SECTION_MARKER, RX_SECTION_MARKER, PREPROCESSOR_SECTION, PARSER_SECTION, \
Eckhart Arnold's avatar
Eckhart Arnold committed
473
474
            AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
            DHPARSER_MAIN, DHPARSER_IMPORTS
475
        f = None
476
        try:
477
            f = open(rootname + 'Compiler.py', 'r', encoding="utf-8")
478
            source = f.read()
479
            sections = RX_SECTION_MARKER.split(source)
480
            intro, imports, preprocessor, parser, ast, compiler, outro = sections
481
482
483
484
            # TODO: Verify transformation table
            ast_trans_table = compile_python_object(DHPARSER_IMPORTS + ast,
                                                    r'(?:\w+_)?AST_transformation_table$')
            messages.extend(ebnf_compiler.verify_transformation_table(ast_trans_table))
485
        except (PermissionError, FileNotFoundError, IOError) as error:
486
            intro, imports, preprocessor, parser, ast, compiler, outro = '', '', '', '', '', '', ''
487
        except ValueError as error:
488
489
490
            name = '"' + rootname + 'Compiler.py"'
            raise ValueError('Could not identify all required sections in ' + name +
                             '. Please delete or repair ' + name + ' manually!')
491
        finally:
492
493
494
            if f:
                f.close()
                f = None
495

496
497
498
        if RX_WHITESPACE.fullmatch(intro):
            intro = '#!/usr/bin/python'
        if RX_WHITESPACE.fullmatch(outro):
Eckhart Arnold's avatar
Eckhart Arnold committed
499
            outro = DHPARSER_MAIN.format(NAME=compiler_name)
500
501
        if RX_WHITESPACE.fullmatch(imports):
            imports = DHPARSER_IMPORTS
502
503
        if RX_WHITESPACE.fullmatch(preprocessor):
            preprocessor = ebnf_compiler.gen_preprocessor_skeleton()
504
        if RX_WHITESPACE.fullmatch(ast):
505
            ast = ebnf_compiler.gen_transformer_skeleton()
506
        if RX_WHITESPACE.fullmatch(compiler):
507
            compiler = ebnf_compiler.gen_compiler_skeleton()
508

eckhart's avatar
eckhart committed
509
        compilerscript = rootname + 'Compiler.py'
510
        try:
eckhart's avatar
eckhart committed
511
            f = open(compilerscript, 'w', encoding="utf-8")
512
513
            f.write(intro)
            f.write(SECTION_MARKER.format(marker=SYMBOLS_SECTION))
514
            f.write(imports)
515
516
            f.write(SECTION_MARKER.format(marker=PREPROCESSOR_SECTION))
            f.write(preprocessor)
517
            f.write(SECTION_MARKER.format(marker=PARSER_SECTION))
518
            f.write(result)
519
520
521
522
523
524
525
            f.write(SECTION_MARKER.format(marker=AST_SECTION))
            f.write(ast)
            f.write(SECTION_MARKER.format(marker=COMPILER_SECTION))
            f.write(compiler)
            f.write(SECTION_MARKER.format(marker=END_SECTIONS_MARKER))
            f.write(outro)
        except (PermissionError, FileNotFoundError, IOError) as error:
eckhart's avatar
eckhart committed
526
            print('# Could not write file "' + compilerscript + '" because of: '
527
528
529
                  + "\n# ".join(str(error).split('\n)')))
            print(result)
        finally:
530
531
            if f:
                f.close()
532

eckhart's avatar
eckhart committed
533
534
535
536
537
        if platform.system() != "Windows":
            # set file permissions so that the compilerscript can be executed
            st = os.stat(compilerscript)
            os.chmod(compilerscript, st.st_mode | stat.S_IEXEC)

538
    else:
539
        f = None
540
541
542
        try:
            f = open(rootname + extension, 'w', encoding="utf-8")
            if isinstance(result, Node):
543
544
545
546
                if extension.lower() == '.xml':
                    f.write(result.as_xml())
                else:
                    f.write(result.as_sxpr())
547
548
549
550
551
552
553
            else:
                f.write(result)
        except (PermissionError, FileNotFoundError, IOError) as error:
            print('# Could not write file "' + rootname + '.py" because of: '
                  + "\n# ".join(str(error).split('\n)')))
            print(result)
        finally:
554
555
            if f:
                f.close()
556

557
    return messages
558
559


560
561
def recompile_grammar(ebnf_filename, force=False,
                      notify: Callable=lambda: None) -> bool:
562
    """
563
    Re-compiles an EBNF-grammar if necessary, that is, if either no
564
565
566
567
    corresponding 'XXXXCompiler.py'-file exists or if that file is
    outdated.

    Parameters:
568
569
570
        ebnf_filename(str):  The filename of the ebnf-source of the grammar.
            In case this is a directory and not a file, all files within
            this directory ending with .ebnf will be compiled.
571
572
        force(bool):  If False (default), the grammar will only be
            recompiled if it has been changed.
573
574
575
        notify(Callable):  'notify' is a function without parameters that
            is called when recompilation actually takes place. This can
            be used to inform the user.
576
577
578
579
580
581
582
583
584
585
586
    """
    if os.path.isdir(ebnf_filename):
        success = True
        for entry in os.listdir(ebnf_filename):
            if entry.lower().endswith('.ebnf') and os.path.isfile(entry):
                success = success and recompile_grammar(entry, force)
        return success

    base, ext = os.path.splitext(ebnf_filename)
    compiler_name = base + 'Compiler.py'
    error_file_name = base + '_ebnf_ERRORS.txt'
587
    messages = []  # type: Iterable[Error]
588
589
    if (not os.path.exists(compiler_name) or force or
            grammar_changed(compiler_name, ebnf_filename)):
590
        notify()
591
592
        messages = compile_on_disk(ebnf_filename)
        if messages:
593
            # print("Errors while compiling: " + ebnf_filename + '!')
Eckhart Arnold's avatar
Eckhart Arnold committed
594
            with open(error_file_name, 'w', encoding="utf-8") as f:
595
                for e in messages:
Eckhart Arnold's avatar
Eckhart Arnold committed
596
                    f.write(str(e))
597
                    f.write('\n')
598
599
            if has_errors(messages):
                return False
600

601
    if not messages and os.path.exists(error_file_name):
602
603
        os.remove(error_file_name)
    return True