dsl.py 22.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# dsl.py - Support for domain specific notations for DHParser
#
# Copyright 2016  by Eckhart Arnold (arnold@badw.de)
#                 Bavarian Academy of Sciences an Humanities (badw.de)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
17
18


19
"""
20
Module ``dsl`` contains various functions to support the
21
22
23
compilation of domain specific languages based on an EBNF-grammar.
"""

24

25
import os
26
from typing import Any, cast, List, Tuple, Union, Iterator, Iterable
27

28
from DHParser.ebnf import EBNFCompiler, grammar_changed, \
29
30
    get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler, \
    PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc
31
from DHParser.error import Error, is_error, has_errors, only_errors
32
from DHParser.log import logging
33
34
from DHParser.parse import Grammar
from DHParser import Compiler, compile_source, TransformationFunc
35
from DHParser.preprocess import nil_preprocessor, PreprocessorFunc
36
from DHParser.syntaxtree import Node
37
from DHParser.toolkit import load_if_file, is_python_code, compile_python_object, \
38
    re
39

40
41
__all__ = ('DHPARSER_IMPORTS',
           'GrammarError',
Eckhart Arnold's avatar
Eckhart Arnold committed
42
43
44
           'CompilationError',
           'load_compiler_suite',
           'compileDSL',
Eckhart Arnold's avatar
Eckhart Arnold committed
45
           'raw_compileEBNF',
46
           'compileEBNF',
47
           'grammar_provider',
48
49
           'compile_on_disk',
           'recompile_grammar')
Eckhart Arnold's avatar
Eckhart Arnold committed
50
51


52
53
54
55
56
57
58
59
60
SECTION_MARKER = """\n
#######################################################################
#
# {marker}
#
#######################################################################
\n"""

RX_SECTION_MARKER = re.compile(SECTION_MARKER.format(marker=r'.*?SECTION.*?'))
61
RX_WHITESPACE = re.compile(r'\s*')
62
63

SYMBOLS_SECTION = "SYMBOLS SECTION - Can be edited. Changes will be preserved."
64
PREPROCESSOR_SECTION = "PREPROCESSOR SECTION - Can be edited. Changes will be preserved."
65
66
67
PARSER_SECTION = "PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!"
AST_SECTION = "AST SECTION - Can be edited. Changes will be preserved."
COMPILER_SECTION = "COMPILER SECTION - Can be edited. Changes will be preserved."
di68kap's avatar
di68kap committed
68
END_SECTIONS_MARKER = "END OF DHPARSER-SECTIONS"
69
70


71
DHPARSER_IMPORTS = '''
di68kap's avatar
di68kap committed
72
from functools import partial
Eckhart Arnold's avatar
Eckhart Arnold committed
73
import os
74
import sys
di68kap's avatar
di68kap committed
75
76
77
78
try:
    import regex as re
except ImportError:
    import re
79
from DHParser import logging, is_filename, load_if_file, \\
80
    Grammar, Compiler, nil_preprocessor, PreprocessorToken, \\
81
    Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, Unordered, \\
82
    Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \\
83
    ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \\
84
    last_value, counterpart, accumulate, PreprocessorFunc, \\
eckhart's avatar
eckhart committed
85
    Node, TransformationFunc, TransformationDict, \\
86
    traverse, remove_children_if, merge_children, is_anonymous, \\
Eckhart Arnold's avatar
Eckhart Arnold committed
87
    reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
88
    remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
89
    is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \\
eckhart's avatar
eckhart committed
90
    remove_nodes, remove_content, remove_brackets, replace_parser, \\
91
92
    keep_children, is_one_of, has_content, apply_if, remove_first, remove_last, \\
    remove_anonymous_empty, keep_nodes, traverse_locally, strip
93
'''
di68kap's avatar
di68kap committed
94

95

Eckhart Arnold's avatar
Eckhart Arnold committed
96
DHPARSER_MAIN = '''
di68kap's avatar
di68kap committed
97
def compile_src(source, log_dir=''):
98
99
    """Compiles ``source`` and returns (result, errors, ast).
    """
di68kap's avatar
di68kap committed
100
    with logging(log_dir):
101
        compiler = get_compiler()
Eckhart Arnold's avatar
Eckhart Arnold committed
102
103
        cname = compiler.__class__.__name__
        log_file_name = os.path.basename(os.path.splitext(source)[0]) \\
104
105
            if is_filename(source) < 0 else cname[:cname.find('.')] + '_out'
        result = compile_source(source, get_preprocessor(),
106
107
                                get_grammar(),
                                get_transformer(), compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
108
109
    return result

110
111
112

if __name__ == "__main__":
    if len(sys.argv) > 1:
di68kap's avatar
di68kap committed
113
114
115
116
        file_name, log_dir = sys.argv[1], ''
        if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
            file_name, log_dir = sys.argv[2], 'LOGS'
        result, errors, ast = compile_src(file_name, log_dir)
117
        if errors:
di68kap's avatar
di68kap committed
118
119
            cwd = os.getcwd()
            rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
120
            for error in errors:
di68kap's avatar
di68kap committed
121
                print(rel_path + ':' + str(error))
122
            sys.exit(1)
123
        else:
124
            print(result.as_xml() if isinstance(result, Node) else result)
125
    else:
126
        print("Usage: {NAME}Compiler.py [FILENAME]")
127
128
'''

129

130
131
132
133
134
class DSLException(Exception):
    """
    Base class for DSL-exceptions.
    """
    def __init__(self, errors):
Eckhart Arnold's avatar
Eckhart Arnold committed
135
136
        assert isinstance(errors, Iterator) or isinstance(errors, list) \
               or isinstance(errors, tuple)
137
138
139
140
141
142
143
        self.errors = errors

    def __str__(self):
        return '\n'.join(str(err) for err in self.errors)


class GrammarError(DSLException):
144
145
    """
    Raised when (already) the grammar of a domain specific language (DSL)
146
147
    contains errors.
    """
148
149
    def __init__(self, errors, grammar_src):
        super().__init__(errors)
150
151
152
        self.grammar_src = grammar_src


153
class CompilationError(DSLException):
154
155
    """
    Raised when a string or file in a domain specific language (DSL)
156
157
    contains errors.
    """
158
159
    def __init__(self, errors, dsl_text, dsl_grammar, AST, result):
        super().__init__(errors)
160
161
162
        self.dsl_text = dsl_text
        self.dsl_grammar = dsl_grammar
        self.AST = AST
163
        self.result = result
164
165


166
167
168
169
170
def error_str(messages: Iterable[Error]) -> str:
    """
    Returns all true errors (i.e. not just warnings) from the
    `messages` as a concatenated multiline string.
    """
171
    return '\n\n'.join(str(m) for m in messages if is_error(m.code))
172
173


174
def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
175
176
    """
    Returns a grammar object and the source code of the grammar, from
177
    the given `grammar`-data which can be either a file name, ebnf-code,
178
    python-code, a Grammar-derived grammar class or an instance of
179
180
    such a class (i.e. a grammar object already).
    """
181
    if isinstance(grammar_representation, str):
182
        # read grammar
183
        grammar_src = load_if_file(grammar_representation)
184
        if is_python_code(grammar_src):
eckhart's avatar
eckhart committed
185
            parser_py, messages = grammar_src, []  # type: str, List[Error]
186
        else:
Eckhart Arnold's avatar
Eckhart Arnold committed
187
            with logging(False):
eckhart's avatar
eckhart committed
188
189
                parser_py, messages, _ = compile_source(
                    grammar_src, None,
Eckhart Arnold's avatar
Eckhart Arnold committed
190
                    get_ebnf_grammar(), get_ebnf_transformer(), get_ebnf_compiler())
191
        if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
192
            raise GrammarError(only_errors(messages), grammar_src)
eckhart's avatar
eckhart committed
193
        parser_root = compile_python_object(DHPARSER_IMPORTS + parser_py, r'\w+Grammar$')()
194
195
196
    else:
        # assume that dsl_grammar is a ParserHQ-object or Grammar class
        grammar_src = ''
197
        if isinstance(grammar_representation, Grammar):
198
            parser_root = grammar_representation
199
        else:
Eckhart Arnold's avatar
Eckhart Arnold committed
200
            # assume ``grammar_representation`` is a grammar class and get the root object
201
            parser_root = grammar_representation()
202
203
204
    return parser_root, grammar_src


205
def compileDSL(text_or_file: str,
206
               preprocessor: PreprocessorFunc,
207
               dsl_grammar: Union[str, Grammar],
208
               ast_transformation: TransformationFunc,
209
               compiler: Compiler) -> Any:
210
211
    """
    Compiles a text in a domain specific language (DSL) with an
212
213
    EBNF-specified grammar. Returns the compiled text or raises a
    compilation error.
eckhart's avatar
eckhart committed
214

215
    Raises:
216
        CompilationError if any errors occurred during compilation
217
218
    """
    assert isinstance(text_or_file, str)
219
    assert isinstance(compiler, Compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
220

221
    parser, grammar_src = grammar_instance(dsl_grammar)
222
    result, messages, AST = compile_source(text_or_file, preprocessor, parser,
223
                                           ast_transformation, compiler)
224
    if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
225
        src = load_if_file(text_or_file)
Eckhart Arnold's avatar
Eckhart Arnold committed
226
        raise CompilationError(only_errors(messages), src, grammar_src, AST, result)
227
228
229
    return result


230
def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
231
232
    """
    Compiles an EBNF grammar file and returns the compiler object
Eckhart Arnold's avatar
Eckhart Arnold committed
233
    that was used and which can now be queried for the result as well
234
    as skeleton code for preprocessor, transformer and compiler objects.
eckhart's avatar
eckhart committed
235

Eckhart Arnold's avatar
Eckhart Arnold committed
236
237
238
239
    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
        branding (str):  Branding name for the compiler suite source
eckhart's avatar
eckhart committed
240
            code.
Eckhart Arnold's avatar
Eckhart Arnold committed
241
242
243
    Returns:
        An instance of class ``ebnf.EBNFCompiler``
    Raises:
eckhart's avatar
eckhart committed
244
        CompilationError if any errors occurred during compilation
Eckhart Arnold's avatar
Eckhart Arnold committed
245
246
    """
    grammar = get_ebnf_grammar()
247
    compiler = get_ebnf_compiler(branding, ebnf_src)
248
249
    transformer = get_ebnf_transformer()
    compileDSL(ebnf_src, nil_preprocessor, grammar, transformer, compiler)
Eckhart Arnold's avatar
Eckhart Arnold committed
250
251
252
    return compiler


253
def compileEBNF(ebnf_src: str, branding="DSL") -> str:
254
255
    """
    Compiles an EBNF source file and returns the source code of a
256
    compiler suite with skeletons for preprocessor, transformer and
Eckhart Arnold's avatar
Eckhart Arnold committed
257
    compiler.
258
259
260
261

    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
262
        branding (str):  Branding name for the compiler suite source
eckhart's avatar
eckhart committed
263
            code.
264
    Returns:
265
        The complete compiler suite skeleton as Python source code.
266
    Raises:
eckhart's avatar
eckhart committed
267
        CompilationError if any errors occurred during compilation
268
    """
Eckhart Arnold's avatar
Eckhart Arnold committed
269
    compiler = raw_compileEBNF(ebnf_src, branding)
270
271
    src = ["#/usr/bin/python\n",
           SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_IMPORTS,
272
           SECTION_MARKER.format(marker=PREPROCESSOR_SECTION), compiler.gen_preprocessor_skeleton(),
Eckhart Arnold's avatar
Eckhart Arnold committed
273
           SECTION_MARKER.format(marker=PARSER_SECTION), compiler.result,
274
275
276
           SECTION_MARKER.format(marker=AST_SECTION), compiler.gen_transformer_skeleton(),
           SECTION_MARKER.format(marker=COMPILER_SECTION), compiler.gen_compiler_skeleton(),
           SECTION_MARKER.format(marker=SYMBOLS_SECTION), DHPARSER_MAIN.format(NAME=branding)]
277
278
279
    return '\n'.join(src)


280
def grammar_provider(ebnf_src: str, branding="DSL") -> Grammar:
281
    """
282
    Compiles an EBNF grammar and returns a grammar-parser provider
283
284
285
286
287
288
    function for that grammar.

    Args:
        ebnf_src(str):  Either the file name of an EBNF grammar or
            the EBNF grammar itself as a string.
        branding (str or bool):  Branding name for the compiler
eckhart's avatar
eckhart committed
289
290
            suite source code.

291
    Returns:
292
        A provider function for a grammar object for texts in the
293
294
        language defined by ``ebnf_src``.
    """
295
    grammar_src = compileDSL(ebnf_src, nil_preprocessor, get_ebnf_grammar(),
296
                             get_ebnf_transformer(), get_ebnf_compiler(branding))
eckhart's avatar
eckhart committed
297
    return compile_python_object(DHPARSER_IMPORTS + grammar_src, r'get_(?:\w+_)?grammar$')
298
299


300
def load_compiler_suite(compiler_suite: str) -> \
301
302
        Tuple[PreprocessorFactoryFunc, ParserFactoryFunc,
              TransformerFactoryFunc, CompilerFactoryFunc]:
303
    """
304
    Extracts a compiler suite from file or string `compiler_suite`
305
    and returns it as a tuple (preprocessor, parser, ast, compiler).
eckhart's avatar
eckhart committed
306

Eckhart Arnold's avatar
Eckhart Arnold committed
307
    Returns:
308
        4-tuple (preprocessor function, parser class, ast transformer function, compiler class)
309
310
311
312
    """
    global RX_SECTION_MARKER
    assert isinstance(compiler_suite, str)
    source = load_if_file(compiler_suite)
313
    imports = DHPARSER_IMPORTS
314
315
    if is_python_code(compiler_suite):
        try:
eckhart's avatar
eckhart committed
316
            _, imports, preprocessor_py, parser_py, ast_py, compiler_py, _ = \
317
                RX_SECTION_MARKER.split(source)
eckhart's avatar
eckhart committed
318
        except ValueError:
319
320
            raise AssertionError('File "' + compiler_suite + '" seems to be corrupted. '
                                 'Please delete or repair file manually.')
321
        # TODO: Compile in one step and pick parts from namespace later ?
eckhart's avatar
eckhart committed
322
323
324
325
        preprocessor = compile_python_object(imports + preprocessor_py,
                                             r'get_(?:\w+_)?preprocessor$')
        parser = compile_python_object(imports + parser_py, r'get_(?:\w+_)?grammar$')
        ast = compile_python_object(imports + ast_py, r'get_(?:\w+_)?transformer$')
326
    else:
327
328
        # Assume source is an ebnf grammar.
        # Is there really any reasonable application case for this?
Eckhart Arnold's avatar
Eckhart Arnold committed
329
        with logging(False):
eckhart's avatar
eckhart committed
330
            compiler_py, messages, n = compile_source(source, None, get_ebnf_grammar(),
eckhart's avatar
eckhart committed
331
                                                      get_ebnf_transformer(), get_ebnf_compiler())
332
        if has_errors(messages):
Eckhart Arnold's avatar
Eckhart Arnold committed
333
            raise GrammarError(only_errors(messages), source)
334
        preprocessor = get_ebnf_preprocessor
335
        parser = get_ebnf_grammar
336
        ast = get_ebnf_transformer
eckhart's avatar
eckhart committed
337
    compiler = compile_python_object(imports + compiler_py, r'get_(?:\w+_)?compiler$')
338

339
    return preprocessor, parser, ast, compiler
340
341


342
def is_outdated(compiler_suite: str, grammar_source: str) -> bool:
343
344
    """
    Returns ``True``  if the ``compile_suite`` needs to be updated.
eckhart's avatar
eckhart committed
345
346
347

    An update is needed, if either the grammar in the compieler suite
    does not reflect the latest changes of ``grammar_source`` or if
348
349
    sections from the compiler suite have diligently been overwritten
    with whitespace order to trigger their recreation. Note: Do not
eckhart's avatar
eckhart committed
350
    delete or overwrite the section marker itself.
351

352
    Args:
353
354
355
356
357
358
359
360
361
        compiler_suite:  the parser class representing the grammar
            or the file name of a compiler suite containing the grammar
        grammar_source:  File name or string representation of the
            EBNF code of the grammar

    Returns (bool):
        True, if ``compiler_suite`` seems to be out of date.
    """
    try:
eckhart's avatar
eckhart committed
362
        n1, grammar, n2, n3 = load_compiler_suite(compiler_suite)
363
        return grammar_changed(grammar(), grammar_source)
364
365
366
367
    except ValueError:
        return True


368
def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
369
370
371
372
    """Compiles a source with a given compiler suite.

    Args:
        text_or_file (str):  Either the file name of the source code or
eckhart's avatar
eckhart committed
373
            the source code directly. (Which is determined by
374
375
376
377
378
            heuristics. If ``text_or_file`` contains at least on
            linefeed then it is always assumed to be a source text and
            not a file name.)
        compiler_suite(str):  File name of the compiler suite to be
            used.
eckhart's avatar
eckhart committed
379

380
    Returns:
eckhart's avatar
eckhart committed
381
        The result of the compilation, the form and type of which
382
        depends entirely on the compiler.
eckhart's avatar
eckhart committed
383

384
385
386
    Raises:
        CompilerError
    """
387
388
    preprocessor, parser, ast, compiler = load_compiler_suite(compiler_suite)
    return compileDSL(text_or_file, preprocessor(), parser(), ast(), compiler())
389
390


391
def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> Iterable[Error]:
392
393
    """
    Compiles the a source file with a given compiler and writes the
394
395
    result to a file.

Eckhart Arnold's avatar
Eckhart Arnold committed
396
397
398
    If no ``compiler_suite`` is given it is assumed that the source
    file is an EBNF grammar. In this case the result will be a Python
    script containing a parser for that grammar as well as the
399
    skeletons for a preprocessor, AST transformation table, and compiler.
Eckhart Arnold's avatar
Eckhart Arnold committed
400
401
    If the Python script already exists only the parser name in the
    script will be updated. (For this to work, the different names
402
    need to be delimited section marker blocks.). `compile_on_disk()`
Eckhart Arnold's avatar
Eckhart Arnold committed
403
404
    returns a list of error messages or an empty list if no errors
    occurred.
405

406
407
408
409
    Parameters:
        source_file(str):  The file name of the source text to be
            compiled.
        compiler_suite(str):  The file name of the compiler suite
410
            (usually ending with 'Compiler.py'), with which the source
411
412
413
414
415
416
417
            file shall be compiled. If this is left empty, the source
            file is assumed to be an EBNF-Grammar that will be compiled
            with the internal EBNF-Compiler.
        extension(str):  The result of the compilation (if successful)
            is written to a file with the same name but a different
            extension than the source file. This parameter sets the
            extension.
eckhart's avatar
eckhart committed
418

419
    Returns:
420
        A (potentially empty) list of error or warning messages.
Eckhart Arnold's avatar
Eckhart Arnold committed
421
    """
422
    filepath = os.path.normpath(source_file)
423
424
    with open(source_file, encoding="utf-8") as f:
        source = f.read()
425
    rootname = os.path.splitext(filepath)[0]
426
    compiler_name = os.path.basename(rootname)
427
    if compiler_suite:
428
        sfactory, pfactory, tfactory, cfactory = load_compiler_suite(compiler_suite)
429
    else:
430
        sfactory = get_ebnf_preprocessor
431
432
        pfactory = get_ebnf_grammar
        tfactory = get_ebnf_transformer
433
        cfactory = get_ebnf_compiler
434
435
    compiler1 = cfactory()
    compiler1.set_grammar_name(compiler_name, source_file)
436
    result, messages, AST = compile_source(source, sfactory(), pfactory(), tfactory(), compiler1)
437
438
    if has_errors(messages):
        return messages
439

440
441
442
    elif cfactory == get_ebnf_compiler:
        # trans == get_ebnf_transformer or trans == EBNFTransformer:
        # either an EBNF- or no compiler suite given
443
        ebnf_compiler = cast(EBNFCompiler, compiler1)
444
        global SECTION_MARKER, RX_SECTION_MARKER, PREPROCESSOR_SECTION, PARSER_SECTION, \
Eckhart Arnold's avatar
Eckhart Arnold committed
445
446
            AST_SECTION, COMPILER_SECTION, END_SECTIONS_MARKER, RX_WHITESPACE, \
            DHPARSER_MAIN, DHPARSER_IMPORTS
447
        f = None
448
        try:
449
            f = open(rootname + 'Compiler.py', 'r', encoding="utf-8")
450
            source = f.read()
451
            sections = RX_SECTION_MARKER.split(source)
452
            intro, imports, preprocessor, parser, ast, compiler, outro = sections
453
454
455
456
            # TODO: Verify transformation table
            ast_trans_table = compile_python_object(DHPARSER_IMPORTS + ast,
                                                    r'(?:\w+_)?AST_transformation_table$')
            messages.extend(ebnf_compiler.verify_transformation_table(ast_trans_table))
457
        except (PermissionError, FileNotFoundError, IOError) as error:
458
            intro, imports, preprocessor, parser, ast, compiler, outro = '', '', '', '', '', '', ''
459
        except ValueError as error:
460
461
462
            name = '"' + rootname + 'Compiler.py"'
            raise ValueError('Could not identify all required sections in ' + name +
                             '. Please delete or repair ' + name + ' manually!')
463
        finally:
464
465
466
            if f:
                f.close()
                f = None
467

468
469
470
        if RX_WHITESPACE.fullmatch(intro):
            intro = '#!/usr/bin/python'
        if RX_WHITESPACE.fullmatch(outro):
Eckhart Arnold's avatar
Eckhart Arnold committed
471
            outro = DHPARSER_MAIN.format(NAME=compiler_name)
472
473
        if RX_WHITESPACE.fullmatch(imports):
            imports = DHPARSER_IMPORTS
474
475
        if RX_WHITESPACE.fullmatch(preprocessor):
            preprocessor = ebnf_compiler.gen_preprocessor_skeleton()
476
        if RX_WHITESPACE.fullmatch(ast):
477
            ast = ebnf_compiler.gen_transformer_skeleton()
478
        if RX_WHITESPACE.fullmatch(compiler):
479
            compiler = ebnf_compiler.gen_compiler_skeleton()
480

481
        try:
482
            f = open(rootname + 'Compiler.py', 'w', encoding="utf-8")
483
484
            f.write(intro)
            f.write(SECTION_MARKER.format(marker=SYMBOLS_SECTION))
485
            f.write(imports)
486
487
            f.write(SECTION_MARKER.format(marker=PREPROCESSOR_SECTION))
            f.write(preprocessor)
488
            f.write(SECTION_MARKER.format(marker=PARSER_SECTION))
489
            f.write(result)
490
491
492
493
494
495
496
            f.write(SECTION_MARKER.format(marker=AST_SECTION))
            f.write(ast)
            f.write(SECTION_MARKER.format(marker=COMPILER_SECTION))
            f.write(compiler)
            f.write(SECTION_MARKER.format(marker=END_SECTIONS_MARKER))
            f.write(outro)
        except (PermissionError, FileNotFoundError, IOError) as error:
497
            print('# Could not write file "' + rootname + 'Compiler.py" because of: '
498
499
500
                  + "\n# ".join(str(error).split('\n)')))
            print(result)
        finally:
501
502
            if f:
                f.close()
503
504

    else:
505
        f = None
506
507
508
        try:
            f = open(rootname + extension, 'w', encoding="utf-8")
            if isinstance(result, Node):
509
510
511
512
                if extension.lower() == '.xml':
                    f.write(result.as_xml())
                else:
                    f.write(result.as_sxpr())
513
514
515
516
517
518
519
            else:
                f.write(result)
        except (PermissionError, FileNotFoundError, IOError) as error:
            print('# Could not write file "' + rootname + '.py" because of: '
                  + "\n# ".join(str(error).split('\n)')))
            print(result)
        finally:
520
521
            if f:
                f.close()
522

523
    return messages
524
525
526
527


def recompile_grammar(ebnf_filename, force=False) -> bool:
    """
528
    Re-compiles an EBNF-grammar if necessary, that is, if either no
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
    corresponding 'XXXXCompiler.py'-file exists or if that file is
    outdated.

    Parameters:
        ebnf_filename(str):  The filename of the ebnf-source of the
            grammar. In case this is a directory and not a file, all
            files within this directory ending with .ebnf will be
            compiled.
        force(bool):  If False (default), the grammar will only be
            recompiled if it has been changed.
    """
    if os.path.isdir(ebnf_filename):
        success = True
        for entry in os.listdir(ebnf_filename):
            if entry.lower().endswith('.ebnf') and os.path.isfile(entry):
                success = success and recompile_grammar(entry, force)
        return success

    base, ext = os.path.splitext(ebnf_filename)
    compiler_name = base + 'Compiler.py'
    error_file_name = base + '_ebnf_ERRORS.txt'
550
    messages = []  # type: Iterable[Error]
551
552
553
    if (not os.path.exists(compiler_name) or force or
            grammar_changed(compiler_name, ebnf_filename)):
        # print("recompiling parser for: " + ebnf_filename)
554
555
        messages = compile_on_disk(ebnf_filename)
        if messages:
556
            # print("Errors while compiling: " + ebnf_filename + '!')
Eckhart Arnold's avatar
Eckhart Arnold committed
557
            with open(error_file_name, 'w', encoding="utf-8") as f:
558
                for e in messages:
Eckhart Arnold's avatar
Eckhart Arnold committed
559
                    f.write(str(e))
560
                    f.write('\n')
561
562
            if has_errors(messages):
                return False
563

564
    if not messages and os.path.exists(error_file_name):
565
566
        os.remove(error_file_name)
    return True