22.1.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

dhparser.py 11 KB
Newer Older
1
#!/usr/bin/python
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

"""dhparser.py - command line tool for DHParser

Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                Bavarian Academy of Sciences an Humanities (badw.de)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.
"""

21
#  TODO: This is still a stub...
22

23 24 25
import os
import sys

26
from DHParser.compile import compile_source
27
from DHParser.dsl import compileDSL, compile_on_disk
Eckhart Arnold's avatar
Eckhart Arnold committed
28
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
29
from DHParser.log import logging
30
from DHParser.toolkit import re
31

eckhart's avatar
eckhart committed
32 33
dhparserdir = os.path.dirname(os.path.realpath(__file__))

34
EBNF_TEMPLATE = r"""-grammar
35

36 37 38 39 40 41
#######################################################################
#
#  EBNF-Directives
#
#######################################################################

42
@ whitespace  = vertical        # implicit whitespace, includes any number of line feeds
43
@ literalws   = right           # literals have implicit whitespace on the right hand side
Eckhart Arnold's avatar
Eckhart Arnold committed
44
@ comment     = /#.*/           # comments range from a '#'-character to the end of the line
45 46 47 48 49 50 51 52 53
@ ignorecase  = False           # literals and regular expressions are case-sensitive


#######################################################################
#
#  Structure and Components
#
#######################################################################

54
document = //~ { WORD } §EOF    # root parser: a sequence of words preceded by whitespace
55 56 57 58 59 60 61 62
                                # until the end of file

#######################################################################
#
#  Regular Expressions
#
#######################################################################

63 64
WORD     =  /\w+/~      # a sequence of letters, optional trailing whitespace
EOF      =  !/./        # no more characters ahead, end of file reached
65 66
"""

67
TEST_WORD_TEMPLATE = r'''[match:WORD]
68 69
M1: word
M2: one_word_with_underscores
70 71

[fail:WORD]
72
F1: two words
73 74 75
'''

TEST_DOCUMENT_TEMPLATE = r'''[match:document]
76 77 78
M1: """This is a sequence of words
    extending over several lines"""

79
[fail:document]
80 81
F1: """This test should fail, because neither
    comma nor full have been defined anywhere."""
82
'''
83 84 85 86 87 88 89 90 91 92

README_TEMPLATE = """# {name}

PLACE A SHORT DESCRIPTION HERE

Author: AUTHOR'S NAME <EMAIL>, AFFILIATION


## License

93
{name} is open source software under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0)
94 95 96

Copyright YEAR AUTHOR'S NAME <EMAIL>, AFFILIATION

97 98 99 100 101 102 103 104 105 106 107
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
108 109 110
"""


111
GRAMMAR_TEST_TEMPLATE = r'''#!/usr/bin/python3
112 113 114 115

"""tst_{name}_grammar.py - runs the unit tests for the {name}-grammar
"""

116
import os
117 118
import sys

eckhart's avatar
eckhart committed
119
sys.path.append('{dhparserdir}')
120

121
scriptpath = os.path.dirname(__file__)
122

eckhart's avatar
eckhart committed
123

124 125 126 127 128 129 130
try:
    from DHParser import dsl
    import DHParser.log
    from DHParser import testing
except ModuleNotFoundError:
    print('Could not import DHParser. Please adjust sys.path in file '
          '"%s" manually' % __file__)
131
    sys.exit(1)
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147


def recompile_grammar(grammar_src, force):
    with DHParser.log.logging(False):
        # recompiles Grammar only if it has changed
        if not dsl.recompile_grammar(grammar_src, force=force):
            print('\nErrors while recompiling "%s":' % grammar_src +
                  '\n--------------------------------------\n\n')
            with open('{name}_ebnf_ERRORS.txt') as f:
                print(f.read())
            sys.exit(1)


def run_grammar_tests(glob_pattern):
    with DHParser.log.logging(False):
        error_report = testing.grammar_suite(
eckhart's avatar
eckhart committed
148 149
            os.path.join(scriptpath, 'grammar_tests'),
            get_grammar, get_transformer,
150 151 152 153 154 155 156 157 158
            fn_patterns=[glob_pattern], report=True, verbose=True)
    return error_report


if __name__ == '__main__':
    arg = sys.argv[1] if len(sys.argv) > 1 else '*_test_*.ini'
    if arg.endswith('.ebnf'):
        recompile_grammar(arg, force=True)
    else:
eckhart's avatar
eckhart committed
159
        recompile_grammar(os.path.join(scriptpath, '{name}.ebnf'),
160 161 162 163 164 165 166 167 168
                          force=False)
        sys.path.append('.')
        from {name}Compiler import get_grammar, get_transformer
        error_report = run_grammar_tests(glob_pattern=arg)
        if error_report:
            print('\n')
            print(error_report)
            sys.exit(1)
        print('ready.\n')
169
'''
170 171


172 173 174
def create_project(path: str):
    """Creates the a new DHParser-project in the given `path`.
    """
175
    def create_file(name, content):
176
        """Create a file with `name` and write `content` to file."""
177 178 179 180 181 182 183
        if not os.path.exists(name):
            print('Creating file "%s".' % name)
            with open(name, 'w') as f:
                f.write(content)
        else:
            print('"%s" already exists! Not overwritten.' % name)

184
    name = os.path.basename(path)
eckhart's avatar
eckhart committed
185
    if not re.match(r'(?!\d)\w+', name):
186 187
        print('Project name "%s" is not a valid identifier! Aborting.' % name)
        sys.exit(1)
188
    if os.path.exists(path) and not os.path.isdir(path):
189
        print('Cannot create new project, because a file named "%s" already exists!' % path)
190
        sys.exit(1)
191 192 193 194
    print('Creating new DHParser-project "%s".' % name)
    if not os.path.exists(path):
        os.mkdir(path)
    curr_dir = os.getcwd()
195
    os.chdir(path)
196 197 198 199 200 201 202 203 204 205 206
    if os.path.exists('grammar_tests'):
        if not os.path.isdir('grammar_tests'):
            print('Cannot overwrite existing file "grammar_tests"')
            sys.exit(1)
    else:
        os.mkdir('grammar_tests')

    create_file(os.path.join('grammar_tests', '01_test_word.ini'), TEST_WORD_TEMPLATE)
    create_file(os.path.join('grammar_tests', '02_test_document.ini'), TEST_DOCUMENT_TEMPLATE)
    create_file(name + '.ebnf', '# ' + name + EBNF_TEMPLATE)
    create_file('README.md', README_TEMPLATE.format(name=name))
eckhart's avatar
eckhart committed
207 208
    create_file('tst_%s_grammar.py' % name,
                GRAMMAR_TEST_TEMPLATE.format(name=name, dhparserdir=dhparserdir))
209
    create_file('example.dsl', 'Life is but a walking shadow\n')
210
    os.chmod('tst_%s_grammar.py' % name, 0o755)
211
    os.chdir(curr_dir)
212
    print('ready.\n')
213

214

Eckhart Arnold's avatar
Eckhart Arnold committed
215
def selftest() -> bool:
216 217
    """Run a simple self-text of DHParser.
    """
Eckhart Arnold's avatar
Eckhart Arnold committed
218 219 220 221 222 223
    print("DHParser selftest...")
    print("\nSTAGE I:  Trying to compile EBNF-Grammar:\n")
    builtin_ebnf_parser = get_ebnf_grammar()
    ebnf_src = builtin_ebnf_parser.__doc__[builtin_ebnf_parser.__doc__.find('#'):]
    ebnf_transformer = get_ebnf_transformer()
    ebnf_compiler = get_ebnf_compiler('EBNF')
224 225
    generated_ebnf_parser, errors, _ = compile_source(
        ebnf_src, None,
Eckhart Arnold's avatar
Eckhart Arnold committed
226 227 228 229
        builtin_ebnf_parser, ebnf_transformer, ebnf_compiler)

    if errors:
        print("Selftest FAILED :-(")
230
        print("\n\n".join(str(err) for err in errors))
Eckhart Arnold's avatar
Eckhart Arnold committed
231 232
        return False
    print(generated_ebnf_parser)
233 234
    print("\n\nSTAGE 2: Selfhosting-test: "
          "Trying to compile EBNF-Grammar with generated parser...\n")
Eckhart Arnold's avatar
Eckhart Arnold committed
235 236 237 238 239 240 241
    selfhosted_ebnf_parser = compileDSL(ebnf_src, None, generated_ebnf_parser,
                                        ebnf_transformer, ebnf_compiler)
    ebnf_compiler.gen_transformer_skeleton()
    print(selfhosted_ebnf_parser)
    return True


242
def cpu_profile(func, repetitions=1):
243 244 245 246 247 248
    """Profile the function `func`.
    """
    import cProfile
    import pstats
    profile = cProfile.Profile()
    profile.enable()
249
    success = True
250
    for _ in range(repetitions):
251 252 253
        success = func()
        if not success:
            break
254
    profile.disable()
255
    # after your program ends
256 257 258
    stats = pstats.Stats(profile)
    stats.strip_dirs()
    stats.sort_stats('time').print_stats(40)
259
    return success
260 261


262 263 264
def mem_profile(func):
    """Profile memory usage of `func`.
    """
265 266 267 268 269 270 271 272 273 274 275
    import tracemalloc
    tracemalloc.start()
    success = func()
    snapshot = tracemalloc.take_snapshot()
    top_stats = snapshot.statistics('lineno')
    print("[ Top 20 ]")
    for stat in top_stats[:20]:
        print(stat)
    return success


276 277 278 279
def main():
    """Creates a project (if a project name has been passed as command line
    parameter) or runs a quick self-test.
    """
280
    if len(sys.argv) > 1:
281
        if os.path.exists(sys.argv[1]) and os.path.isfile(sys.argv[1]):
282 283 284
            _errors = compile_on_disk(sys.argv[1],
                                      sys.argv[2] if len(sys.argv) > 2 else "")
            if _errors:
285
                print('\n\n'.join(str(err) for err in _errors))
286 287 288
                sys.exit(1)
        else:
            create_project(sys.argv[1])
289
    else:
290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
        print('Usage: \n'
              '    dhparser.py DSL_FILENAME [COMPILER]  - to compile a file\n'
              '    dhparser.py PROJECTNAME  - to create a new project\n\n')
        choice = input('Would you now like to ...\n'
                       '  [1] create a new project\n'
                       '  [2] compile an ebnf-grammar or a dsl-file\n'
                       '  [3] run a self-test\n'
                       '  [q] to quit\n'
                       'Please chose 1, 2 or 3> ')
        if choice.strip() == '1':
            project_name = input('Please project name or path > ')
            create_project(project_name)
        elif choice.strip() == '2':
            file_path = input('Please enter a file path for compilation > ')
            if os.path.exists(file_path) and os.path.isfile(file_path):
                compiler_suite = input('Compiler suite or ENTER (for ebnf) > ')
                if (not compiler_suite or (os.path.exists(compiler_suite)
                        and os.path.isfile(compiler_suite))):
                    _errors = compile_on_disk(file_path, compiler_suite)
                    if _errors:
                        print('\n\n'.join(str(err) for err in _errors))
                        sys.exit(1)
                else:
                    print('Compiler suite %s not found! Aborting' % compiler_suite)
            else:
                print('File %s not found! Aborting.' % file_path)
316
                sys.exit(1)
317 318 319 320 321 322 323 324
        elif choice.strip() == '3':
            with logging(False):
                if not cpu_profile(selftest, 1):
                    print("Selftest FAILED :-(\n")
                    sys.exit(1)
                print("Selftest SUCCEEDED :-)\n")
        elif choice.strip().lower() not in {'q', 'quit', 'exit'}:
            print('No valid choice. Goodbye!')
325

326

327 328
if __name__ == "__main__":
    main()