11.3.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

dhparser.py 11 KB
Newer Older
1
#!/usr/bin/python
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20

"""dhparser.py - command line tool for DHParser

Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                Bavarian Academy of Sciences an Humanities (badw.de)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.
"""

21
#  TODO: This is still a stub...
22

23 24 25
import os
import sys

26
from DHParser.compile import compile_source
27
from DHParser.dsl import compileDSL, compile_on_disk
Eckhart Arnold's avatar
Eckhart Arnold committed
28
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
29
from DHParser.log import logging
30
from DHParser.toolkit import re
31

32
EBNF_TEMPLATE = r"""-grammar
33

34 35 36 37 38 39
#######################################################################
#
#  EBNF-Directives
#
#######################################################################

40
@ whitespace  = vertical        # implicit whitespace, includes any number of line feeds
41
@ literalws   = right           # literals have implicit whitespace on the right hand side
Eckhart Arnold's avatar
Eckhart Arnold committed
42
@ comment     = /#.*/           # comments range from a '#'-character to the end of the line
43 44 45 46 47 48 49 50 51
@ ignorecase  = False           # literals and regular expressions are case-sensitive


#######################################################################
#
#  Structure and Components
#
#######################################################################

52
document = //~ { WORD } §EOF    # root parser: a sequence of words preceded by whitespace
53 54 55 56 57 58 59 60
                                # until the end of file

#######################################################################
#
#  Regular Expressions
#
#######################################################################

61 62
WORD     =  /\w+/~      # a sequence of letters, optional trailing whitespace
EOF      =  !/./        # no more characters ahead, end of file reached
63 64
"""

65
TEST_WORD_TEMPLATE = r'''[match:WORD]
66 67
M1: word
M2: one_word_with_underscores
68 69

[fail:WORD]
70
F1: two words
71 72 73
'''

TEST_DOCUMENT_TEMPLATE = r'''[match:document]
74 75 76
M1: """This is a sequence of words
    extending over several lines"""

77
[fail:document]
78 79
F1: """This test should fail, because neither
    comma nor full have been defined anywhere."""
80
'''
81 82 83 84 85 86 87 88 89 90

README_TEMPLATE = """# {name}

PLACE A SHORT DESCRIPTION HERE

Author: AUTHOR'S NAME <EMAIL>, AFFILIATION


## License

91
{name} is open source software under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0)
92 93 94

Copyright YEAR AUTHOR'S NAME <EMAIL>, AFFILIATION

95 96 97 98 99 100 101 102 103 104 105
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
106 107 108
"""


109
GRAMMAR_TEST_TEMPLATE = r'''#!/usr/bin/python3
110 111 112 113

"""tst_{name}_grammar.py - runs the unit tests for the {name}-grammar
"""

114
import os
115 116
import sys

117
# sys.path.extend(['../../', '../', './'])  # use for developing DHParser
118

119
scriptpath = os.path.dirname(__file__)
120

121 122 123 124 125 126 127
try:
    from DHParser import dsl
    import DHParser.log
    from DHParser import testing
except ModuleNotFoundError:
    print('Could not import DHParser. Please adjust sys.path in file '
          '"%s" manually' % __file__)
128
    sys.exit(1)
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166


def recompile_grammar(grammar_src, force):
    with DHParser.log.logging(False):
        # recompiles Grammar only if it has changed
        if not dsl.recompile_grammar(grammar_src, force=force):
            print('\nErrors while recompiling "%s":' % grammar_src +
                  '\n--------------------------------------\n\n')
            with open('{name}_ebnf_ERRORS.txt') as f:
                print(f.read())
            sys.exit(1)


def run_grammar_tests(glob_pattern):
    with DHParser.log.logging(False):
        print(glob_pattern)
        error_report = testing.grammar_suite(
            os.path.join(scriptpath, 'grammar_tests'), 
            get_grammar, get_transformer, 
            fn_patterns=[glob_pattern], report=True, verbose=True)
    return error_report


if __name__ == '__main__':
    arg = sys.argv[1] if len(sys.argv) > 1 else '*_test_*.ini'
    if arg.endswith('.ebnf'):
        recompile_grammar(arg, force=True)
    else:
        recompile_grammar(os.path.join(scriptpath, '{name}.ebnf'), 
                          force=False)
        sys.path.append('.')
        from {name}Compiler import get_grammar, get_transformer
        error_report = run_grammar_tests(glob_pattern=arg)
        if error_report:
            print('\n')
            print(error_report)
            sys.exit(1)
        print('ready.\n')
167
'''
168 169


170 171 172
def create_project(path: str):
    """Creates the a new DHParser-project in the given `path`.
    """
173
    def create_file(name, content):
174
        """Create a file with `name` and write `content` to file."""
175 176 177 178 179 180 181
        if not os.path.exists(name):
            print('Creating file "%s".' % name)
            with open(name, 'w') as f:
                f.write(content)
        else:
            print('"%s" already exists! Not overwritten.' % name)

182 183 184 185
    name = os.path.basename(path)
    if not re.match('(?!\d)\w+', name):
        print('Project name "%s" is not a valid identifier! Aborting.' % name)
        sys.exit(1)
186
    if os.path.exists(path) and not os.path.isdir(path):
187
        print('Cannot create new project, because a file named "%s" already exists!' % path)
188
        sys.exit(1)
189 190 191 192
    print('Creating new DHParser-project "%s".' % name)
    if not os.path.exists(path):
        os.mkdir(path)
    curr_dir = os.getcwd()
193
    os.chdir(path)
194 195 196 197 198 199 200 201 202 203 204 205
    if os.path.exists('grammar_tests'):
        if not os.path.isdir('grammar_tests'):
            print('Cannot overwrite existing file "grammar_tests"')
            sys.exit(1)
    else:
        os.mkdir('grammar_tests')

    create_file(os.path.join('grammar_tests', '01_test_word.ini'), TEST_WORD_TEMPLATE)
    create_file(os.path.join('grammar_tests', '02_test_document.ini'), TEST_DOCUMENT_TEMPLATE)
    create_file(name + '.ebnf', '# ' + name + EBNF_TEMPLATE)
    create_file('README.md', README_TEMPLATE.format(name=name))
    create_file('tst_%s_grammar.py' % name, GRAMMAR_TEST_TEMPLATE.format(name=name))
206
    create_file('example.dsl', 'Life is but a walking shadow\n')
207
    os.chmod('tst_%s_grammar.py' % name, 0o755)
208
    os.chdir(curr_dir)
209
    print('ready.\n')
210

211

Eckhart Arnold's avatar
Eckhart Arnold committed
212
def selftest() -> bool:
213 214
    """Run a simple self-text of DHParser.
    """
Eckhart Arnold's avatar
Eckhart Arnold committed
215 216 217 218 219 220
    print("DHParser selftest...")
    print("\nSTAGE I:  Trying to compile EBNF-Grammar:\n")
    builtin_ebnf_parser = get_ebnf_grammar()
    ebnf_src = builtin_ebnf_parser.__doc__[builtin_ebnf_parser.__doc__.find('#'):]
    ebnf_transformer = get_ebnf_transformer()
    ebnf_compiler = get_ebnf_compiler('EBNF')
221 222
    generated_ebnf_parser, errors, _ = compile_source(
        ebnf_src, None,
Eckhart Arnold's avatar
Eckhart Arnold committed
223 224 225 226
        builtin_ebnf_parser, ebnf_transformer, ebnf_compiler)

    if errors:
        print("Selftest FAILED :-(")
227
        print("\n\n".join(str(err) for err in errors))
Eckhart Arnold's avatar
Eckhart Arnold committed
228 229
        return False
    print(generated_ebnf_parser)
230 231
    print("\n\nSTAGE 2: Selfhosting-test: "
          "Trying to compile EBNF-Grammar with generated parser...\n")
Eckhart Arnold's avatar
Eckhart Arnold committed
232 233 234 235 236 237 238
    selfhosted_ebnf_parser = compileDSL(ebnf_src, None, generated_ebnf_parser,
                                        ebnf_transformer, ebnf_compiler)
    ebnf_compiler.gen_transformer_skeleton()
    print(selfhosted_ebnf_parser)
    return True


239
def cpu_profile(func, repetitions=1):
240 241 242 243 244 245
    """Profile the function `func`.
    """
    import cProfile
    import pstats
    profile = cProfile.Profile()
    profile.enable()
246
    success = True
247
    for _ in range(repetitions):
248 249 250
        success = func()
        if not success:
            break
251
    profile.disable()
252
    # after your program ends
253 254 255
    stats = pstats.Stats(profile)
    stats.strip_dirs()
    stats.sort_stats('time').print_stats(40)
256
    return success
257 258


259 260 261
def mem_profile(func):
    """Profile memory usage of `func`.
    """
262 263 264 265 266 267 268 269 270 271 272
    import tracemalloc
    tracemalloc.start()
    success = func()
    snapshot = tracemalloc.take_snapshot()
    top_stats = snapshot.statistics('lineno')
    print("[ Top 20 ]")
    for stat in top_stats[:20]:
        print(stat)
    return success


273 274 275 276
def main():
    """Creates a project (if a project name has been passed as command line
    parameter) or runs a quick self-test.
    """
277
    if len(sys.argv) > 1:
278
        if os.path.exists(sys.argv[1]) and os.path.isfile(sys.argv[1]):
279 280 281
            _errors = compile_on_disk(sys.argv[1],
                                      sys.argv[2] if len(sys.argv) > 2 else "")
            if _errors:
282
                print('\n\n'.join(str(err) for err in _errors))
283 284 285
                sys.exit(1)
        else:
            create_project(sys.argv[1])
286
    else:
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
        print('Usage: \n'
              '    dhparser.py DSL_FILENAME [COMPILER]  - to compile a file\n'
              '    dhparser.py PROJECTNAME  - to create a new project\n\n')
        choice = input('Would you now like to ...\n'
                       '  [1] create a new project\n'
                       '  [2] compile an ebnf-grammar or a dsl-file\n'
                       '  [3] run a self-test\n'
                       '  [q] to quit\n'
                       'Please chose 1, 2 or 3> ')
        if choice.strip() == '1':
            project_name = input('Please project name or path > ')
            create_project(project_name)
        elif choice.strip() == '2':
            file_path = input('Please enter a file path for compilation > ')
            if os.path.exists(file_path) and os.path.isfile(file_path):
                compiler_suite = input('Compiler suite or ENTER (for ebnf) > ')
                if (not compiler_suite or (os.path.exists(compiler_suite)
                        and os.path.isfile(compiler_suite))):
                    _errors = compile_on_disk(file_path, compiler_suite)
                    if _errors:
                        print('\n\n'.join(str(err) for err in _errors))
                        sys.exit(1)
                else:
                    print('Compiler suite %s not found! Aborting' % compiler_suite)
            else:
                print('File %s not found! Aborting.' % file_path)
313
                sys.exit(1)
314 315 316 317 318 319 320 321
        elif choice.strip() == '3':
            with logging(False):
                if not cpu_profile(selftest, 1):
                    print("Selftest FAILED :-(\n")
                    sys.exit(1)
                print("Selftest SUCCEEDED :-)\n")
        elif choice.strip().lower() not in {'q', 'quit', 'exit'}:
            print('No valid choice. Goodbye!')
322

323

324 325
if __name__ == "__main__":
    main()