dsl.py 22.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# dsl.py - Support for domain specific notations for DHParser
#
# Copyright 2016  by Eckhart Arnold (arnold@badw.de)
#                 Bavarian Academy of Sciences an Humanities (badw.de)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
17
18


19
"""
20
Module ``dsl`` contains various functions to support the
21
22
23
compilation of domain specific languages based on an EBNF-grammar.
"""

24

25
import os
26

eckhart's avatar
eckhart committed
27
from DHParser.compile import Compiler, compile_source
28
from DHParser.ebnf import EBNFCompiler, grammar_changed, \
29
30
    get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
    PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
31
from DHParser.error import Error, is_error, has_errors, only_errors
32
from DHParser.log import logging
33
from DHParser.parse import Grammar
34
from DHParser.preprocess import nil_preprocessor, PreprocessorFunc
35
from DHParser.syntaxtree import Node
eckhart's avatar
eckhart committed
36
from DHParser.transform import TransformationFunc
37
from DHParser.toolkit import load_if_file, is_python_code, compile_python_object, \
eckhart's avatar
eckhart committed
38
39
40
    re, typing
from typing import Any, cast, List, Tuple, Union, Iterator, Iterable

41

42
43
__all__ = ('DHPARSER_IMPORTS',
           'GrammarError',
Eckhart Arnold's avatar
Eckhart Arnold committed
44
45
46
           'CompilationError',
           'load_compiler_suite',
           'compileDSL',
Eckhart Arnold's avatar
Eckhart Arnold committed
47
           'raw_compileEBNF',
48
           'compileEBNF',
49
           'grammar_provider',
50
51
           'compile_on_disk',
           'recompile_grammar')
Eckhart Arnold's avatar
Eckhart Arnold committed
52
53


54
55
56
57
58
59
60
61
62
SECTION_MARKER = """\n
#######################################################################
#
# {marker}
#
#######################################################################
\n"""

RX_SECTION_MARKER = re.compile(SECTION_MARKER.format(marker=r'.*?SECTION.*?'))
63
RX_WHITESPACE = re.compile(r'\s*')
64
65

SYMBOLS_SECTION = "SYMBOLS SECTION - Can be edited. Changes will be preserved."
66
PREPROCESSOR_SECTION = "PREPROCESSOR SECTION - Can be edited. Changes will be preserved."
67
68
69
PARSER_SECTION = "PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!"
AST_SECTION = "AST SECTION - Can be edited. Changes will be preserved."
COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved."
di68kap's avatar
di68kap committed
70
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
71
72


73
DHPARSER_IMPORTS = '''
di68kap's avatar
di68kap committed
74
from functools import partial
Eckhart Arnold's avatar
Eckhart Arnold committed
75
import os
76
import sys
di68kap's avatar
di68kap committed
77
78
79
80
try:
    import regex as re
except ImportError:
    import re
81
from DHParser import logging, is_filename, load_if_file, \\
82
    Grammar, Compiler, nil_preprocessor, PreprocessorToken, \\
83
    Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, Unordered, \\
84
    Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\
85
    ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
86
    last_value, counterpart, accumulate, PreprocessorFunc, \\
eckhart's avatar
eckhart committed
87
    Node, TransformationFunc, TransformationDict, \\
88
    traverse, remove_children_if, merge_children, is_anonymous, \\
Eckhart Arnold's avatar
Eckhart Arnold committed
89
    reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
90
    remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
91
    is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \\
eckhart's avatar
eckhart committed
92
    remove_nodes, remove_content, remove_brackets, replace_parser, \\
93
94
    keep_children, is_one_of, has_content, apply_if, remove_first, remove_last, \\
    remove_anonymous_empty, keep_nodes, traverse_locally, strip
95
'''
di68kap's avatar
di68kap committed
96

97

Eckhart Arnold's avatar
Eckhart Arnold committed
98
DHPARSER_MAIN = '''
di68kap's avatar
di68kap committed
99
def compile_src(source, log_dir=''):
100
101
    """Compiles ``source`` and returns (result, errors, ast).
    """
di68kap's avatar
di68kap committed
102
    with logging(log_dir):
103
        compiler = get_compiler()
Eckhart Arnold's avatar
Eckhart Arnold committed
104
105
        cname = compiler.__class__.__name__
        log_file_name = os.path.basename(os.path.splitext(source)[0]) \\
106
107
            if is_filename(source) < 0 else cname[:cname.find('.')] + '_out'
        result = compile_source(source, get_preprocessor(),
108
109
                                get_grammar(),
                                get_transformer(), compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
110
111
    return result

112
113
114

if __name__ == "__main__":
    if len(sys.argv) > 1:
di68kap's avatar
di68kap committed
115
116
117
118
        file_name, log_dir = sys.argv[1], ''
        if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
            file_name, log_dir = sys.argv[2], 'LOGS'
        result, errors, ast = compile_src(file_name, log_dir)
119
        if errors:
di68kap's avatar
di68kap committed
120
121
            cwd = os.getcwd()
            rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
122
            for error in errors:
di68kap's avatar
di68kap committed
123
                print(rel_path + ':' + str(error))
124
            sys.exit(1)
125
        else:
126
            print(result.as_xml() if isinstance(result, Node) else result)
127
    else:
128
        print("Usage: {NAME}Compiler.py [FILENAME]")
129
130
'''

131

132
133
134
135
136
class DSLException(Exception):
    """
    Base class for DSL-exceptions.
    """
    def __init__(self, errors):
Eckhart Arnold's avatar
Eckhart Arnold committed
137
138
        assert isinstance(errors, Iterator) or isinstance(errors, list) \
               or isinstance(errors, tuple)
139
140
141
142
143
144
145
        self.errors = errors

    def __str__(self):
        return '\n'.join(str(err) for err in self.errors)


class GrammarError(DSLException):
146
147
    """
    Raised when (already) the grammar of a domain specific language (DSL)
148
149
    contains errors.
    """
150
151
    def __init__(self, errors, grammar_src):
        super().__init__(errors)
152
153
154
        self.grammar_src = grammar_src


155
class CompilationError(DSLException):
156
157
    """
    Raised when a string or file in a domain specific language (DSL)
158
159
    contains errors.
    """
160
161
    def __init__(self, errors, dsl_text, dsl_grammar, AST, result):
        super().__init__(errors)
162
163
164
        self.dsl_text = dsl_text
        self.dsl_grammar = dsl_grammar
        self.AST = AST
165
        self.result = result
166
167


168
169
170
171
172
def error_str(messages: Iterable[Error]) -> str:
    """
    Returns all true errors (i.e. not just warnings) from the
    `messages` as a concatenated multiline string.
    """
173
    return '\n\n'.join(str(m) for m in messages if is_error(m.code))
174
175


176
def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
177
178
    """
    Returns a grammar object and the source code of the grammar, from
179
    the given `grammar`-data which can be either a file name, ebnf-code,
180
    python-code, a Grammar-derived grammar class or an instance of
181
182
    such a class (i.e. a grammar object already).
    """
183
    if isinstance(grammar_representation, str):
184
        # read grammar
185
        grammar_src = load_if_file(grammar_representation)
186
        if is_python_code(grammar_src):
eckhart's avatar
eckhart committed
187
            parser_py, messages = grammar_src, []  # type: str, List[Error]
188
        else:
Eckhart Arnold's avatar
Eckhart Arnold committed
189
            with logging(False):
eckhart's avatar
eckhart committed
190
191
                parser_py, messages, _ = compile_source(
                    grammar_src, None,
Eckhart Arnold's avatar
Eckhart Arnold committed
192
                    get_ebnf_grammar(), get_ebnf_transformer(), get_ebnf_compiler())
193
        if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
194
            raise GrammarError(only_errors(messages), grammar_src)
eckhart's avatar
eckhart committed
195
        parser_root = compile_python_object(DHPARSER_IMPORTS + parser_py, r'\w+Grammar$')()
196
197
198
    else:
        # assume that dsl_grammar is a ParserHQ-object or Grammar class
        grammar_src = ''
199
        if isinstance(grammar_representation, Grammar):
200
            parser_root = grammar_representation
201
        else:
Eckhart Arnold's avatar
Eckhart Arnold committed
202
            # assume ``grammar_representation`` is a grammar class and get the root object
203
            parser_root = grammar_representation()
204
205
206
    return parser_root, grammar_src


207
def compileDSL(text_or_file: str,
208
               preprocessor: PreprocessorFunc,
209
               dsl_grammar: Union[str, Grammar],
210
               ast_transformation: TransformationFunc,
211
               compiler: Compiler) -> Any:
212
213
    """
    Compiles a text in a domain specific language (DSL) with an
214
215
    EBNF-specified grammar. Returns the compiled text or raises a
    compilation error.
eckhart's avatar
eckhart committed
216

217
    Raises:
218
        CompilationError if any errors occurred during compilation
219
220
    """
    assert isinstance(text_or_file, str)
221
    assert isinstance(compiler, Compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
222

223
    parser, grammar_src = grammar_instance(dsl_grammar)
224
    result, messages, AST = compile_source(text_or_file, preprocessor, parser,
225
                                           ast_transformation, compiler)
226
    if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
227
        src = load_if_file(text_or_file)
Eckhart Arnold's avatar
Eckhart Arnold committed
228
        raise CompilationError(only_errors(messages), src, grammar_src, AST, result)
229
230
231
    return result


232
def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
233
234
    """
    Compiles an EBNF grammar file and returns the compiler object
Eckhart Arnold's avatar
Eckhart Arnold committed
235
    that was used and which can now be queried for the result as well
236
    as skeleton code for preprocessor, transformer and compiler objects.
eckhart's avatar
eckhart committed
237

Eckhart Arnold's avatar
Eckhart Arnold committed
238
239
240
241
    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
        branding (str):  Branding name for the compiler suite source
eckhart's avatar
eckhart committed
242
            code.
Eckhart Arnold's avatar
Eckhart Arnold committed
243
244
245
    Returns:
        An instance of class ``ebnf.EBNFCompiler``
    Raises:
eckhart's avatar
eckhart committed
246
        CompilationError if any errors occurred during compilation
Eckhart Arnold's avatar
Eckhart Arnold committed
247
248
    """
    grammar = get_ebnf_grammar()
249
    compiler = get_ebnf_compiler(branding, ebnf_src)
250
251
    transformer = get_ebnf_transformer()
    compileDSL(ebnf_src, nil_preprocessor, grammar, transformer, compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
252
253
254
    return compiler


255
def compileEBNF(ebnf_src: str, branding="DSL") -> str:
256
257
    """
    Compiles an EBNF source file and returns the source code of a
258
    compiler suite with skeletons for preprocessor, transformer and
Eckhart Arnold's avatar
Eckhart Arnold committed
259
    compiler.
260
261
262
263

    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
264
        branding (str):  Branding name for the compiler suite source
eckhart's avatar
eckhart committed
265
            code.
266
    Returns:
267
        The complete compiler suite skeleton as Python source code.
268
    Raises:
eckhart's avatar
eckhart committed
269
        CompilationError if any errors occurred during compilation
270
    """
Eckhart Arnold's avatar
Eckhart Arnold committed
271
    compiler = raw_compileEBNF(ebnf_src, branding)
272
273
    src = ["#/usr/bin/python\n",
           SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_IMPORTS,
274
           SECTION_MARKER.format(marker=PREPROCESSOR_SECTION), compiler.gen_preprocessor_skeleton(),
Eckhart Arnold's avatar
Eckhart Arnold committed
275
           SECTION_MARKER.format(marker=PARSER_SECTION), compiler.result,
276
277
278
           SECTION_MARKER.format(marker=AST_SECTION), compiler.gen_transformer_skeleton(),
           SECTION_MARKER.format(marker=COMPILER_SECTION), compiler.gen_compiler_skeleton(),
           SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_MAIN.format(NAME=branding)]
279
280
281
    return '\n'.join(src)


282
def grammar_provider(ebnf_src: str, branding="DSL") -> Grammar:
283
    """
284
    Compiles an EBNF grammar and returns a grammar-parser provider
285
286
287
288
289
290
    function for that grammar.

    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
        branding (str or bool):  Branding name for the compiler
eckhart's avatar
eckhart committed
291
292
            suite source code.

293
    Returns:
294
        A provider function for a grammar object for texts in the
295
296
        language defined by ``ebnf_src``.
    """
297
    grammar_src = compileDSL(ebnf_src, nil_preprocessor, get_ebnf_grammar(),
298
                             get_ebnf_transformer(), get_ebnf_compiler(branding))
eckhart's avatar
eckhart committed
299
    return compile_python_object(DHPARSER_IMPORTS + grammar_src, r'get_(?:\w+_)?grammar$')
300
301


302
def load_compiler_suite(compiler_suite: str) -> \
303
304
        Tuple[PreprocessorFactoryFunc, ParserFactoryFunc,
              TransformerFactoryFunc, CompilerFactoryFunc]:
305
    """
306
    Extracts a compiler suite from file or string `compiler_suite`
307
    and returns it as a tuple (preprocessor, parser, ast, compiler).
eckhart's avatar
eckhart committed
308

Eckhart Arnold's avatar
Eckhart Arnold committed
309
    Returns:
310
        4-tuple (preprocessor function, parser class, ast transformer function, compiler class)
311
312
313
314
    """
    global RX_SECTION_MARKER
    assert isinstance(compiler_suite, str)
    source = load_if_file(compiler_suite)
315
    imports = DHPARSER_IMPORTS
316
317
    if is_python_code(compiler_suite):
        try:
eckhart's avatar
eckhart committed
318
            _, imports, preprocessor_py, parser_py, ast_py, compiler_py, _ = \
319
                RX_SECTION_MARKER.split(source)
eckhart's avatar
eckhart committed
320
        except ValueError:
321
322
            raise AssertionError('File "' + compiler_suite + '" seems to be corrupted. '
                                 'Please delete or repair file manually.')
323
        # TODO: Compile in one step and pick parts from namespace later ?
eckhart's avatar
eckhart committed
324
325
326
327
        preprocessor = compile_python_object(imports + preprocessor_py,
                                             r'get_(?:\w+_)?preprocessor$')
        parser = compile_python_object(imports + parser_py, r'get_(?:\w+_)?grammar$')
        ast = compile_python_object(imports + ast_py, r'get_(?:\w+_)?transformer$')
328
    else:
329
330
        # Assume source is an ebnf grammar.
        # Is there really any reasonable application case for this?
Eckhart Arnold's avatar
Eckhart Arnold committed
331
        with logging(False):
eckhart's avatar
eckhart committed
332
            compiler_py, messages, n = compile_source(source, None, get_ebnf_grammar(),
eckhart's avatar
eckhart committed
333
                                                      get_ebnf_transformer(), get_ebnf_compiler())
334
        if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
335
            raise GrammarError(only_errors(messages), source)
336
        preprocessor = get_ebnf_preprocessor
337
        parser = get_ebnf_grammar
338
        ast = get_ebnf_transformer
eckhart's avatar
eckhart committed
339
    compiler = compile_python_object(imports + compiler_py, r'get_(?:\w+_)?compiler$')
340

341
    return preprocessor, parser, ast, compiler
342
343


344
def is_outdated(compiler_suite: str, grammar_source: str) -> bool:
345
346
    """
    Returns ``True``  if the ``compile_suite`` needs to be updated.
eckhart's avatar
eckhart committed
347
348
349

    An update is needed, if either the grammar in the compieler suite
    does not reflect the latest changes of ``grammar_source`` or if
350
351
    sections from the compiler suite have diligently been overwritten
    with whitespace order to trigger their recreation. Note: Do not
eckhart's avatar
eckhart committed
352
    delete or overwrite the section marker itself.
353

354
    Args:
355
356
357
358
359
360
361
362
363
        compiler_suite:  the parser class representing the grammar
            or the file name of a compiler suite containing the grammar
        grammar_source:  File name or string representation of the
            EBNF code of the grammar

    Returns (bool):
        True, if ``compiler_suite`` seems to be out of date.
    """
    try:
eckhart's avatar
eckhart committed
364
        n1, grammar, n2, n3 = load_compiler_suite(compiler_suite)
365
        return grammar_changed(grammar(), grammar_source)
366
367
368
369
    except ValueError:
        return True


370
def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
371
372
373
374
    """Compiles a source with a given compiler suite.

    Args:
        text_or_file (str):  Either the file name of the source code or
eckhart's avatar
eckhart committed
375
            the source code directly. (Which is determined by
376
377
378
379
380
            heuristics. If ``text_or_file`` contains at least on
            linefeed then it is always assumed to be a source text and
            not a file name.)
        compiler_suite(str):  File name of the compiler suite to be
            used.
eckhart's avatar
eckhart committed
381

382
    Returns:
eckhart's avatar
eckhart committed
383
        The result of the compilation, the form and type of which
384
        depends entirely on the compiler.
eckhart's avatar
eckhart committed
385

386
387
388
    Raises:
        CompilerError
    """
389
390
    preprocessor, parser, ast, compiler = load_compiler_suite(compiler_suite)
    return compileDSL(text_or_file, preprocessor(), parser(), ast(), compiler())
391
392


393
def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> Iterable[Error]:
394
395
    """
    Compiles the a source file with a given compiler and writes the
396
397
    result to a file.

Eckhart Arnold's avatar
Eckhart Arnold committed
398
399
400
    If no ``compiler_suite`` is given it is assumed that the source
    file is an EBNF grammar. In this case the result will be a Python
    script containing a parser for that grammar as well as the
401
    skeletons for a preprocessor, AST transformation table, and compiler.
Eckhart Arnold's avatar
Eckhart Arnold committed
402
403
    If the Python script already exists only the parser name in the
    script will be updated. (For this to work, the different names
404
    need to be delimited section marker blocks.). `compile_on_disk()`
Eckhart Arnold's avatar
Eckhart Arnold committed
405
406
    returns a list of error messages or an empty list if no errors
    occurred.
407

408
409
410
411
    Parameters:
        source_file(str):  The file name of the source text to be
            compiled.
        compiler_suite(str):  The file name of the compiler suite
412
            (usually ending with 'Compiler.py'), with which the source
413
414
415
416
417
418
419
            file shall be compiled. If this is left empty, the source
            file is assumed to be an EBNF-Grammar that will be compiled
            with the internal EBNF-Compiler.
        extension(str):  The result of the compilation (if successful)
            is written to a file with the same name but a different
            extension than the source file. This parameter sets the
            extension.
eckhart's avatar
eckhart committed
420

421
    Returns:
422
        A (potentially empty) list of error or warning messages.
Eckhart Arnold's avatar
Eckhart Arnold committed
423
    """
424
    filepath = os.path.normpath(source_file)
425
426
    with open(source_file, encoding="utf-8") as f:
        source = f.read()
427
    rootname = os.path.splitext(filepath)[0]
428
    compiler_name = os.path.basename(rootname)
429
    if compiler_suite:
430
        sfactory, pfactory, tfactory, cfactory = load_compiler_suite(compiler_suite)
431
    else:
432
        sfactory = get_ebnf_preprocessor
433
434
        pfactory = get_ebnf_grammar
        tfactory = get_ebnf_transformer
435
        cfactory = get_ebnf_compiler
436
437
    compiler1 = cfactory()
    compiler1.set_grammar_name(compiler_name, source_file)
438
    result, messages, AST = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
439
440
    if has_errors(messages):
        return messages
441

442
443
444
    elif cfactory == get_ebnf_compiler:
        # trans == get_ebnf_transformer or trans == EBNFTransformer:
        # either an EBNF- or no compiler suite given
445
        ebnf_compiler = cast(EBNFCompiler, compiler1)
446
        global SECTION_MARKER, RX_SECTION_MARKER, PREPROCESSOR_SECTION, PARSER_SECTION, \
Eckhart Arnold's avatar
Eckhart Arnold committed
447
448
            AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
            DHPARSER_MAIN, DHPARSER_IMPORTS
449
        f = None
450
        try:
451
            f = open(rootname + 'Compiler.py', 'r', encoding="utf-8")
452
            source = f.read()
453
            sections = RX_SECTION_MARKER.split(source)
454
            intro, imports, preprocessor, parser, ast, compiler, outro = sections
455
456
457
458
            # TODO: Verify transformation table
            ast_trans_table = compile_python_object(DHPARSER_IMPORTS + ast,
                                                    r'(?:\w+_)?AST_transformation_table$')
            messages.extend(ebnf_compiler.verify_transformation_table(ast_trans_table))
459
        except (PermissionError, FileNotFoundError, IOError) as error:
460
            intro, imports, preprocessor, parser, ast, compiler, outro = '', '', '', '', '', '', ''
461
        except ValueError as error:
462
463
464
            name = '"' + rootname + 'Compiler.py"'
            raise ValueError('Could not identify all required sections in ' + name +
                             '. Please delete or repair ' + name + ' manually!')
465
        finally:
466
467
468
            if f:
                f.close()
                f = None
469

470
471
472
        if RX_WHITESPACE.fullmatch(intro):
            intro = '#!/usr/bin/python'
        if RX_WHITESPACE.fullmatch(outro):
Eckhart Arnold's avatar
Eckhart Arnold committed
473
            outro = DHPARSER_MAIN.format(NAME=compiler_name)
474
475
        if RX_WHITESPACE.fullmatch(imports):
            imports = DHPARSER_IMPORTS
476
477
        if RX_WHITESPACE.fullmatch(preprocessor):
            preprocessor = ebnf_compiler.gen_preprocessor_skeleton()
478
        if RX_WHITESPACE.fullmatch(ast):
479
            ast = ebnf_compiler.gen_transformer_skeleton()
480
        if RX_WHITESPACE.fullmatch(compiler):
481
            compiler = ebnf_compiler.gen_compiler_skeleton()
482

483
        try:
484
            f = open(rootname + 'Compiler.py', 'w', encoding="utf-8")
485
486
            f.write(intro)
            f.write(SECTION_MARKER.format(marker=SYMBOLS_SECTION))
487
            f.write(imports)
488
489
            f.write(SECTION_MARKER.format(marker=PREPROCESSOR_SECTION))
            f.write(preprocessor)
490
            f.write(SECTION_MARKER.format(marker=PARSER_SECTION))
491
            f.write(result)
492
493
494
495
496
497
498
            f.write(SECTION_MARKER.format(marker=AST_SECTION))
            f.write(ast)
            f.write(SECTION_MARKER.format(marker=COMPILER_SECTION))
            f.write(compiler)
            f.write(SECTION_MARKER.format(marker=END_SECTIONS_MARKER))
            f.write(outro)
        except (PermissionError, FileNotFoundError, IOError) as error:
499
            print('# Could not write file "' + rootname + 'Compiler.py" because of: '
500
501
502
                  + "\n# ".join(str(error).split('\n)')))
            print(result)
        finally:
503
504
            if f:
                f.close()
505
506

    else:
507
        f = None
508
509
510
        try:
            f = open(rootname + extension, 'w', encoding="utf-8")
            if isinstance(result, Node):
511
512
513
514
                if extension.lower() == '.xml':
                    f.write(result.as_xml())
                else:
                    f.write(result.as_sxpr())
515
516
517
518
519
520
521
            else:
                f.write(result)
        except (PermissionError, FileNotFoundError, IOError) as error:
            print('# Could not write file "' + rootname + '.py" because of: '
                  + "\n# ".join(str(error).split('\n)')))
            print(result)
        finally:
522
523
            if f:
                f.close()
524

525
    return messages
526
527
528
529


def recompile_grammar(ebnf_filename, force=False) -> bool:
    """
530
    Re-compiles an EBNF-grammar if necessary, that is, if either no
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
    corresponding 'XXXXCompiler.py'-file exists or if that file is
    outdated.

    Parameters:
        ebnf_filename(str):  The filename of the ebnf-source of the
            grammar. In case this is a directory and not a file, all
            files within this directory ending with .ebnf will be
            compiled.
        force(bool):  If False (default), the grammar will only be
            recompiled if it has been changed.
    """
    if os.path.isdir(ebnf_filename):
        success = True
        for entry in os.listdir(ebnf_filename):
            if entry.lower().endswith('.ebnf') and os.path.isfile(entry):
                success = success and recompile_grammar(entry, force)
        return success

    base, ext = os.path.splitext(ebnf_filename)
    compiler_name = base + 'Compiler.py'
    error_file_name = base + '_ebnf_ERRORS.txt'
552
    messages = []  # type: Iterable[Error]
553
554
555
    if (not os.path.exists(compiler_name) or force or
            grammar_changed(compiler_name, ebnf_filename)):
        # print("recompiling parser for: " + ebnf_filename)
556
557
        messages = compile_on_disk(ebnf_filename)
        if messages:
558
            # print("Errors while compiling: " + ebnf_filename + '!')
Eckhart Arnold's avatar
Eckhart Arnold committed
559
            with open(error_file_name, 'w', encoding="utf-8") as f:
560
                for e in messages:
Eckhart Arnold's avatar
Eckhart Arnold committed
561
                    f.write(str(e))
562
                    f.write('\n')
563
564
            if has_errors(messages):
                return False
565

566
    if not messages and os.path.exists(error_file_name):
567
568
        os.remove(error_file_name)
    return True