testing.py 22.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# testing.py - test support for DHParser based grammars and compilers
#
# Copyright 2016  by Eckhart Arnold (arnold@badw.de)
#                 Bavarian Academy of Sciences an Humanities (badw.de)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
17

18
19
20
21
22
23
24
25
26
"""
Module ``testing`` contains support for unit-testing domain specific
languages. Tests for arbitrarily small components of the Grammar can
be written into test files with ini-file syntax in order to test
whether the parser matches or fails as expected. It can also be
tested whether it produces an expected concrete or abstract syntax tree.
Usually, however, unexpected failure to match a certain string is the
main cause of trouble when constructing a context free Grammar.
"""
27
28


29
import collections
30
# import configparser
31
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
32
import fnmatch
di68kap's avatar
di68kap committed
33
import inspect
34
35
import json
import os
36
import sys
37
import threading
38

di68kap's avatar
di68kap committed
39
from DHParser.error import Error, is_error, adjust_error_locations
40
from DHParser.log import is_logging, clear_logs, log_ST, log_parsing_history
41
from DHParser.parse import UnknownParserError
eckhart's avatar
eckhart committed
42
from DHParser.syntaxtree import Node, RootNode, parse_sxpr, flatten_sxpr, ZOMBIE_PARSER
43
44
45
from DHParser.toolkit import re, typing

from typing import Tuple
46

di68kap's avatar
di68kap committed
47
__all__ = ('unit_from_config',
48
           'unit_from_json',
di68kap's avatar
di68kap committed
49
           'TEST_READERS',
50
51
52
53
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
54
           'reset_unit',
55
56
           'runner')

57
58
UNIT_STAGES = {'match*', 'match', 'fail', 'ast', 'cst'}
RESULT_STAGES = {'__cst__', '__ast__', '__err__'}
59

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# def unit_from_configfile(config_filename):
#     """
#     Reads a grammar unit test from a config file.
#     """
#     cfg = configparser.ConfigParser(interpolation=None)
#     cfg.read(config_filename, encoding="utf8")
#     OD = collections.OrderedDict
#     unit = OD()
#     for section in cfg.sections():
#         symbol, stage = section.split(':')
#         if stage not in UNIT_STAGES:
#             if symbol in UNIT_STAGES:
#                 symbol, stage = stage, symbol
#             else:
#                 raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
#         for testkey, testcode in cfg[section].items():
#             if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
#                 testcode = testcode[3:-3]
#                 # testcode = testcode.replace('\\#', '#')
#                 testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
#             elif testcode[:1] + testcode[-1:] in {"''", '""'}:
#                 testcode = testcode[1:-1]
#             unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
#     # print(json.dumps(unit, sort_keys=True, indent=4))
#     return unit

eckhart's avatar
eckhart committed
86
RX_SECTION = re.compile(r'\s*\[(?P<stage>\w+):(?P<symbol>\w+)\]')
87
RE_VALUE = '(?:"""((?:.|\n)*?)""")|' + "(?:'''((?:.|\n)*?)''')|" + \
eckhart's avatar
eckhart committed
88
           r'(?:"(.*?)")|' + "(?:'(.*?)')|" + r'(.*(?:\n(?:\s*\n)*    .*)*)'
89
90
# the following does not work with pypy3, because pypy's re-engine does not
# support local flags, e.g. '(?s: )'
eckhart's avatar
eckhart committed
91
92
93
94
# RE_VALUE = r'(?:"""((?s:.*?))""")|' + "(?:'''((?s:.*?))''')|" + \
#            r'(?:"(.*?)")|' + "(?:'(.*?)')|" + '(.*(?:\n(?:\s*\n)*    .*)*)'
RX_ENTRY = re.compile(r'\s*(\w+\*?)\s*:\s*(?:{value})\s*'.format(value=RE_VALUE))
RX_COMMENT = re.compile(r'\s*#.*\n')
95

96

di68kap's avatar
di68kap committed
97
def unit_from_config(config_str):
98
99
100
101
    """ Reads grammar unit tests contained in a file in config file (.ini)
    syntax.

    Args:
di68kap's avatar
di68kap committed
102
        config_str (str): A string containing a config-file with Grammar unit-tests
103
104
105
106

    Returns:
        A dictionary representing the unit tests.
    """
eckhart's avatar
eckhart committed
107
108
    # TODO: issue a warning if the same match:xxx or fail:xxx block appears more than once

109
110
111
112
113
114
115
    def eat_comments(txt, pos):
        m = RX_COMMENT.match(txt, pos)
        while m:
            pos = m.span()[1]
            m = RX_COMMENT.match(txt, pos)
        return pos

di68kap's avatar
di68kap committed
116
    cfg = config_str.replace('\t', '    ')
117

118
119
    OD = collections.OrderedDict
    unit = OD()
120
121
122
123
124
125

    pos = eat_comments(cfg, 0)
    section_match = RX_SECTION.match(cfg, pos)
    while section_match:
        d = section_match.groupdict()
        stage = d['stage']
126
        if stage not in UNIT_STAGES:
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
            raise KeyError('Unknown stage ' + stage + " ! must be one of: " + str(UNIT_STAGES))
        symbol = d['symbol']
        pos = eat_comments(cfg, section_match.span()[1])

        entry_match = RX_ENTRY.match(cfg, pos)
        if entry_match is None:
            raise SyntaxError('No entries in section [%s:%s]' % (stage, symbol))
        while entry_match:
            testkey, testcode = [group for group in entry_match.groups() if group is not None]
            lines = testcode.split('\n')
            if len(lines) > 1:
                indent = sys.maxsize
                for line in lines[1:]:
                    indent = min(indent, len(line) - len(line.lstrip()))
                for i in range(1, len(lines)):
                    lines[i] = lines[i][indent:]
                testcode = '\n'.join(lines)
144
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
145
146
147
148
149
150
            pos = eat_comments(cfg, entry_match.span()[1])
            entry_match = RX_ENTRY.match(cfg, pos)

        section_match = RX_SECTION.match(cfg, pos)

    if pos != len(cfg):
151
        raise SyntaxError('in line %i' % (cfg[:pos].count('\n') + 1))
152

153
    return unit
154

155

di68kap's avatar
di68kap committed
156
def unit_from_json(json_str):
157
    """
di68kap's avatar
di68kap committed
158
    Reads grammar unit tests from a json string.
159
    """
di68kap's avatar
di68kap committed
160
    unit = json.loads(json_str)
161
162
163
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
164
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
165
166
    return unit

di68kap's avatar
di68kap committed
167

168
# TODO: add support for yaml, cson, toml
169
170


di68kap's avatar
di68kap committed
171
172
173
174
175
176
177
178
179
# A dictionary associating file endings with reader functions that
# transfrom strings containing the file's content to a nested dictionary
# structure of test cases.
TEST_READERS = {
    '.ini': unit_from_config,
    '.json': unit_from_json
}


180
def unit_from_file(filename):
181
182
    """
    Reads a grammar unit test from a file. The format of the file is
183
184
    determined by the ending of its name.
    """
di68kap's avatar
di68kap committed
185
186
187
188
189
190
    try:
        reader = TEST_READERS[os.path.splitext(filename)[1].lower()]
        with open(filename, 'r', encoding='utf8') as f:
            data = f.read()
        test_unit = reader(data)
    except KeyError:
191
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
192

di68kap's avatar
di68kap committed
193
194
195
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
di68kap's avatar
di68kap committed
196
197
198
199
200
201
202
203
        # normalize case for test category names
        keys = list(tests.keys())
        for key in keys:
            new_key = key.lower()
            if new_key != key:
                tests[new_key] = tests[keys]
                del tests[keys]

di68kap's avatar
di68kap committed
204
205
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
206
207
        intersection = list(m_names & f_names)
        intersection.sort()
di68kap's avatar
di68kap committed
208
209
210
211
212
213
214
215
216
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

217

di68kap's avatar
di68kap committed
218
219
220
221
222
223
# def all_match_tests(tests):
#     """Returns all match tests from ``tests``, This includes match tests
#     marked with an asterix for CST-output as well as unmarked match-tests.
#     """
#     return itertools.chain(tests.get('match', dict()).items(),
#                            tests.get('match*', dict()).items())
224
225


226
def get_report(test_unit):
227
    """
228
229
230
231
    Returns a text-report of the results of a grammar unit test. The report
    lists the source of all tests as well as the error messages, if a test
    failed or the abstract-syntax-tree (AST) in case of success.

232
233
    If an asterix has been appended to the test name then the concrete syntax
    tree will also be added to the report in this particular case.
234
235
236
237

    The purpose of the latter is to help constructing and debugging
    of AST-Transformations. It is better to switch the CST-output on and off
    with the asterix marker when needed than to output the CST for all tests
238
    which would unnecessarily bloat the test reports.
239
    """
240
241
242
243
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
244
245
246
247
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
248
        for test_name, test_code in tests.get('match', dict()).items():
249
250
251
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
252
            report.append(indent(test_code))
253
254
255
256
257
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
258
            cst = tests.get('__cst__', {}).get(test_name, None)
259
            if cst and (not ast or str(test_name).endswith('*')):
260
                report.append('\n### CST')
eckhart's avatar
eckhart committed
261
                report.append(indent(cst.as_sxpr(compact=True)))
262
            if ast:
263
                report.append('\n### AST')
di68kap's avatar
di68kap committed
264
                report.append(indent(ast.as_xml()))
di68kap's avatar
di68kap committed
265
266
267
268
269
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
270
271
272
273
            messages = tests.get('__msg__', {}).get(test_name, "")
            if messages:
                report.append('\n### Messages:')
                report.append(messages)
di68kap's avatar
di68kap committed
274
275
276
277
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
278
279
280
    return '\n'.join(report)


281
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
282
283
    """
    Unit tests for a grammar-parser and ast transformations.
284
    """
di68kap's avatar
di68kap committed
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
    def clean_key(k):
        try:
            return k.replace('*', '')
        except AttributeError:
            return k

    def get(tests, category, key):
        try:
            value = tests[category][key] if key in tests[category] \
                else tests[category][clean_key(key)]
        except KeyError:
            raise AssertionError('%s-test %s for parser %s missing !?'
                                 % (category, test_name, parser_name))
        return value

300
    if isinstance(test_unit, str):
301
        _, unit_name = os.path.split(os.path.splitext(test_unit)[0])
302
        test_unit = unit_from_file(test_unit)
303
    else:
304
        unit_name = 'unit_test_' + str(id(test_unit))
305
306
    if verbose:
        print("\nUnit: " + unit_name)
307
308
309
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
310

di68kap's avatar
di68kap committed
311
312
313
    def lookahead_artifact(raw_errors):
        """
        Returns True, if the error merely occured, because the parser
eckhart's avatar
eckhart committed
314
        stopped in front of a sequence that was captured by a lookahead
di68kap's avatar
di68kap committed
315
316
317
318
        operator. This is required for testing of parsers that put a
        lookahead operator at the end. See test_testing.TestLookahead.
        """
        return len(raw_errors) == 2 \
eckhart's avatar
eckhart committed
319
320
            and raw_errors[-1].code == Error.PARSER_LOOKAHEAD_MATCH_ONLY \
            and raw_errors[-2].code == Error.PARSER_STOPPED_BEFORE_END
di68kap's avatar
di68kap committed
321

322
    for parser_name, tests in test_unit.items():
323
        assert parser_name, "Missing parser name in test %s!" % unit_name
eckhart's avatar
eckhart committed
324
        assert not any(test_type in RESULT_STAGES for test_type in tests), \
325
326
327
328
329
            ("Test %s in %s already has results. Use reset_unit() before running again!"
             % (parser_name, unit_name))
        assert set(tests.keys()).issubset(UNIT_STAGES), \
            'Unknown test-types: %s ! Must be one of %s' \
            % (set(tests.keys()) - UNIT_STAGES, UNIT_STAGES)
330
331
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
332
        match_tests = set(tests['match'].keys()) if 'match' in tests else set()
333
334
        if 'ast' in tests:
            ast_tests = set(tests['ast'].keys())
di68kap's avatar
di68kap committed
335
336
337
            if not {clean_key(k) for k in ast_tests} <= {clean_key(k) for k in match_tests}:
                raise AssertionError('AST-Tests %s for parser %s lack corresponding match-tests!'
                                     % (str(ast_tests - match_tests), parser_name))
338
339
        if 'cst' in tests:
            cst_tests = set(tests['cst'].keys())
di68kap's avatar
di68kap committed
340
            if not {clean_key(k) for k in cst_tests} <= {clean_key(k) for k in match_tests}:
341
342
                raise AssertionError('CST-Tests %s lack corresponding match-tests!'
                                     % str(cst_tests - match_tests))
343
344
345

        # run match tests

346
        for test_name, test_code in tests.get('match', dict()).items():
347
348
349
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
350
            try:
di68kap's avatar
di68kap committed
351
                cst = parser(test_code, parser_name, track_history=True)
352
            except UnknownParserError as upe:
353
                cst = RootNode()
354
                cst = cst.new_error(Node(ZOMBIE_PARSER, "").init_pos(0), str(upe))
eckhart's avatar
eckhart committed
355
            clean_test_name = str(test_name).replace('*', '')
eckhart's avatar
eckhart committed
356
            # log_ST(cst, "match_%s_%s.cst" % (parser_name, clean_test_name))
357
            tests.setdefault('__cst__', {})[test_name] = cst
358
            if "ast" in tests or report:
359
360
361
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
eckhart's avatar
eckhart committed
362
                # log_ST(ast, "match_%s_%s.ast" % (parser_name, clean_test_name))
di68kap's avatar
di68kap committed
363
364
365
            raw_errors = cst.collect_errors()
            if is_error(cst.error_flag) and not lookahead_artifact(raw_errors):
                errors = adjust_error_locations(raw_errors, test_code)
Eckhart Arnold's avatar
Eckhart Arnold committed
366
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
367
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
368
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
di68kap's avatar
di68kap committed
369
                # tests.setdefault('__err__', {})[test_name] = errata[-1]
370
                # write parsing-history log only in case of failure!
371
                if is_logging():
di68kap's avatar
di68kap committed
372
                    log_parsing_history(parser, "match_%s_%s.log" % (parser_name, clean_test_name))
di68kap's avatar
di68kap committed
373
            elif "cst" in tests and parse_sxpr(get(tests, "cst", test_name)) != cst:
eckhart's avatar
eckhart committed
374
375
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
376
            elif "ast" in tests:
di68kap's avatar
di68kap committed
377
                compare = parse_sxpr(get(tests, "ast", test_name))
378
379
380
381
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
382
383
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
di68kap's avatar
di68kap committed
384
385
            if errata:
                tests.setdefault('__err__', {})[test_name] = errata[-1]
386
            if verbose:
387
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
388

389
        if verbose and 'fail' in tests:
390
            print('  Fail-Tests for parser "' + parser_name + '"')
391
392
393

        # run fail tests

394
        for test_name, test_code in tests.get('fail', dict()).items():
395
396
397
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
398
399
            # cst = parser(test_code, parser_name)
            try:
di68kap's avatar
di68kap committed
400
                cst = parser(test_code, parser_name, track_history=True)
401
            except UnknownParserError as upe:
eckhart's avatar
eckhart committed
402
                node = Node(ZOMBIE_PARSER, "").init_pos(0)
eckhart's avatar
eckhart committed
403
                cst = RootNode(node).new_error(node, str(upe))
404
                errata.append('Unknown parser "{}" in fail test "{}"!'.format(parser_name, test_name))
405
                tests.setdefault('__err__', {})[test_name] = errata[-1]
di68kap's avatar
di68kap committed
406
            if not is_error(cst.error_flag) and not lookahead_artifact(cst.collect_errors()):
407
408
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
409
                tests.setdefault('__err__', {})[test_name] = errata[-1]
410
                # write parsing-history log only in case of test-failure
411
                if is_logging():
412
                    log_parsing_history(parser, "fail_%s_%s.log" % (parser_name, test_name))
413
414
415
            if cst.error_flag:
                tests.setdefault('__msg__', {})[test_name] = \
                    "\n".join(str(e) for e in cst.collect_errors())
416
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
417
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
418

419
420
    # write test-report
    if report:
421
        report_dir = "REPORT"
422
423
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
424
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
425
            f.write(get_report(test_unit))
426

427
428
429
    return errata


430
def reset_unit(test_unit):
eckhart's avatar
eckhart committed
431
432
433
434
    """
    Resets the tests in ``test_unit`` by removing all results and error
    messages.
    """
435
436
437
438
439
440
441
442
443
    for parser, tests in test_unit.items():
        for key in list(tests.keys()):
            if key not in UNIT_STAGES:
                if key not in RESULT_STAGES:
                    print('Removing unknown component %s from test %s' % (key, parser))
                del tests[key]



Eckhart Arnold's avatar
Eckhart Arnold committed
444
445
446
447
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
448
449
    """
    Runs all grammar unit tests in a directory. A file is considered a test
450
451
    unit, if it has the word "test" in its name.
    """
452
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
453
        fn_patterns = [fn_patterns]
454
    all_errors = collections.OrderedDict()
455
456
    if verbose:
        print("\nScanning test-directory: " + directory)
457
458
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
459
460
    if is_logging():
        clear_logs()
461
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
462
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
463
            try:
464
465
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
466
467
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
468
469
470
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
471
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
472
                    raise e
473
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
474
475
    error_report = []
    err_N = 0
476
477
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
478
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
479
            err_N += len(all_errors[filename])
480
481
482
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
di68kap's avatar
di68kap committed
483
484
485
486
        # if verbose:
        #     print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
        return ('Test suite "%s" revealed %s error%s:\n\n'
                % (directory, err_N, 's' if err_N > 1 else '') + '\n'.join(error_report))
eckhart's avatar
eckhart committed
487
488
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
489
490
491
    return ''


492
def runner(test_classes, namespace):
493
494
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
495
    namespace. To run all tests in a module, call
496
    ``runner("", globals())`` from within that module.
497

498
499
500
501
    Unit-Tests are either classes, the name of which starts with
    "Test" and methods, the name of which starts with "test" contained
    in such classes or functions, the name of which starts with "test".

502
503
504
505
506
    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
507
        namespace: The namespace for running the test, usually
508
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
509

510
511
512
513
514
515
516
517
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
518

519
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
520
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
521
            runner("", globals())
522
523
524
525
526
527
528
529
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

530
531
532
    if test_classes:
        if isinstance(test_classes, str):
            test_classes = test_classes.split(" ")
533
534
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
535
536
        test_classes = []
        test_functions = []
537
        for name in namespace.keys():
538
539
540
541
542
543
            if name.lower().startswith('test'):
                if inspect.isclass(namespace[name]):
                    test_classes.append(name)
                elif inspect.isfunction(namespace[name]):
                    test_functions.append(name)

544
545

    obj = None
546
    for test in test_classes:
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()
562
563
564

    for test in test_functions:
        exec(test + '()', namespace)