testing.py 24 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# testing.py - test support for DHParser based grammars and compilers
#
# Copyright 2016  by Eckhart Arnold (arnold@badw.de)
#                 Bavarian Academy of Sciences an Humanities (badw.de)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
17

18
19
20
21
22
23
24
25
26
"""
Module ``testing`` contains support for unit-testing domain specific
languages. Tests for arbitrarily small components of the Grammar can
be written into test files with ini-file syntax in order to test
whether the parser matches or fails as expected. It can also be
tested whether it produces an expected concrete or abstract syntax tree.
Usually, however, unexpected failure to match a certain string is the
main cause of trouble when constructing a context free Grammar.
"""
27
28


29
import collections
30
# import configparser
31
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
32
import fnmatch
di68kap's avatar
di68kap committed
33
import inspect
34
35
import json
import os
36
import sys
37
import threading
38

di68kap's avatar
di68kap committed
39
from DHParser.error import Error, is_error, adjust_error_locations
40
41
from DHParser.log import is_logging, clear_logs, log_parsing_history
from DHParser.parse import UnknownParserError, Parser, Lookahead
eckhart's avatar
eckhart committed
42
from DHParser.syntaxtree import Node, RootNode, parse_sxpr, flatten_sxpr, ZOMBIE_PARSER
43
44
45
from DHParser.toolkit import re, typing

from typing import Tuple
46

di68kap's avatar
di68kap committed
47
__all__ = ('unit_from_config',
48
           'unit_from_json',
di68kap's avatar
di68kap committed
49
           'TEST_READERS',
50
51
52
53
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
54
           'reset_unit',
55
56
           'runner')

57
58
UNIT_STAGES = {'match*', 'match', 'fail', 'ast', 'cst'}
RESULT_STAGES = {'__cst__', '__ast__', '__err__'}
59

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# def unit_from_configfile(config_filename):
#     """
#     Reads a grammar unit test from a config file.
#     """
#     cfg = configparser.ConfigParser(interpolation=None)
#     cfg.read(config_filename, encoding="utf8")
#     OD = collections.OrderedDict
#     unit = OD()
#     for section in cfg.sections():
#         symbol, stage = section.split(':')
#         if stage not in UNIT_STAGES:
#             if symbol in UNIT_STAGES:
#                 symbol, stage = stage, symbol
#             else:
#                 raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
#         for testkey, testcode in cfg[section].items():
#             if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
#                 testcode = testcode[3:-3]
#                 # testcode = testcode.replace('\\#', '#')
#                 testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
#             elif testcode[:1] + testcode[-1:] in {"''", '""'}:
#                 testcode = testcode[1:-1]
#             unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
#     # print(json.dumps(unit, sort_keys=True, indent=4))
#     return unit

eckhart's avatar
eckhart committed
86
RX_SECTION = re.compile(r'\s*\[(?P<stage>\w+):(?P<symbol>\w+)\]')
87
RE_VALUE = '(?:"""((?:.|\n)*?)""")|' + "(?:'''((?:.|\n)*?)''')|" + \
eckhart's avatar
eckhart committed
88
           r'(?:"(.*?)")|' + "(?:'(.*?)')|" + r'(.*(?:\n(?:\s*\n)*    .*)*)'
89
90
# the following does not work with pypy3, because pypy's re-engine does not
# support local flags, e.g. '(?s: )'
eckhart's avatar
eckhart committed
91
92
93
94
# RE_VALUE = r'(?:"""((?s:.*?))""")|' + "(?:'''((?s:.*?))''')|" + \
#            r'(?:"(.*?)")|' + "(?:'(.*?)')|" + '(.*(?:\n(?:\s*\n)*    .*)*)'
RX_ENTRY = re.compile(r'\s*(\w+\*?)\s*:\s*(?:{value})\s*'.format(value=RE_VALUE))
RX_COMMENT = re.compile(r'\s*#.*\n')
95

96

di68kap's avatar
di68kap committed
97
def unit_from_config(config_str):
98
99
100
101
    """ Reads grammar unit tests contained in a file in config file (.ini)
    syntax.

    Args:
di68kap's avatar
di68kap committed
102
        config_str (str): A string containing a config-file with Grammar unit-tests
103
104
105
106

    Returns:
        A dictionary representing the unit tests.
    """
eckhart's avatar
eckhart committed
107
108
    # TODO: issue a warning if the same match:xxx or fail:xxx block appears more than once

109
110
111
112
113
114
115
    def eat_comments(txt, pos):
        m = RX_COMMENT.match(txt, pos)
        while m:
            pos = m.span()[1]
            m = RX_COMMENT.match(txt, pos)
        return pos

di68kap's avatar
di68kap committed
116
    cfg = config_str.replace('\t', '    ')
117

118
119
    OD = collections.OrderedDict
    unit = OD()
120
121
122
123
124
125

    pos = eat_comments(cfg, 0)
    section_match = RX_SECTION.match(cfg, pos)
    while section_match:
        d = section_match.groupdict()
        stage = d['stage']
126
        if stage not in UNIT_STAGES:
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
            raise KeyError('Unknown stage ' + stage + " ! must be one of: " + str(UNIT_STAGES))
        symbol = d['symbol']
        pos = eat_comments(cfg, section_match.span()[1])

        entry_match = RX_ENTRY.match(cfg, pos)
        if entry_match is None:
            raise SyntaxError('No entries in section [%s:%s]' % (stage, symbol))
        while entry_match:
            testkey, testcode = [group for group in entry_match.groups() if group is not None]
            lines = testcode.split('\n')
            if len(lines) > 1:
                indent = sys.maxsize
                for line in lines[1:]:
                    indent = min(indent, len(line) - len(line.lstrip()))
                for i in range(1, len(lines)):
                    lines[i] = lines[i][indent:]
                testcode = '\n'.join(lines)
144
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
145
146
147
148
149
150
            pos = eat_comments(cfg, entry_match.span()[1])
            entry_match = RX_ENTRY.match(cfg, pos)

        section_match = RX_SECTION.match(cfg, pos)

    if pos != len(cfg):
151
        raise SyntaxError('in line %i' % (cfg[:pos].count('\n') + 1))
152

153
    return unit
154

155

di68kap's avatar
di68kap committed
156
def unit_from_json(json_str):
157
    """
di68kap's avatar
di68kap committed
158
    Reads grammar unit tests from a json string.
159
    """
di68kap's avatar
di68kap committed
160
    unit = json.loads(json_str)
161
162
163
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
164
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
165
166
    return unit

di68kap's avatar
di68kap committed
167

168
# TODO: add support for yaml, cson, toml
169
170


di68kap's avatar
di68kap committed
171
172
173
174
175
176
177
178
179
# A dictionary associating file endings with reader functions that
# transfrom strings containing the file's content to a nested dictionary
# structure of test cases.
TEST_READERS = {
    '.ini': unit_from_config,
    '.json': unit_from_json
}


180
def unit_from_file(filename):
181
182
    """
    Reads a grammar unit test from a file. The format of the file is
183
184
    determined by the ending of its name.
    """
di68kap's avatar
di68kap committed
185
186
187
188
189
190
    try:
        reader = TEST_READERS[os.path.splitext(filename)[1].lower()]
        with open(filename, 'r', encoding='utf8') as f:
            data = f.read()
        test_unit = reader(data)
    except KeyError:
191
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
192

di68kap's avatar
di68kap committed
193
194
195
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
di68kap's avatar
di68kap committed
196
197
198
199
200
201
202
203
        # normalize case for test category names
        keys = list(tests.keys())
        for key in keys:
            new_key = key.lower()
            if new_key != key:
                tests[new_key] = tests[keys]
                del tests[keys]

di68kap's avatar
di68kap committed
204
205
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
206
207
        intersection = list(m_names & f_names)
        intersection.sort()
di68kap's avatar
di68kap committed
208
209
210
211
212
213
214
215
216
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

217

di68kap's avatar
di68kap committed
218
219
220
221
222
223
# def all_match_tests(tests):
#     """Returns all match tests from ``tests``, This includes match tests
#     marked with an asterix for CST-output as well as unmarked match-tests.
#     """
#     return itertools.chain(tests.get('match', dict()).items(),
#                            tests.get('match*', dict()).items())
224
225


226
def get_report(test_unit):
227
    """
228
229
230
231
    Returns a text-report of the results of a grammar unit test. The report
    lists the source of all tests as well as the error messages, if a test
    failed or the abstract-syntax-tree (AST) in case of success.

232
233
    If an asterix has been appended to the test name then the concrete syntax
    tree will also be added to the report in this particular case.
234
235
236
237

    The purpose of the latter is to help constructing and debugging
    of AST-Transformations. It is better to switch the CST-output on and off
    with the asterix marker when needed than to output the CST for all tests
238
    which would unnecessarily bloat the test reports.
239
    """
240
241
242
243
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
244
245
246
247
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
248
        for test_name, test_code in tests.get('match', dict()).items():
249
250
251
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
252
            report.append(indent(test_code))
253
254
255
256
257
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
258
            cst = tests.get('__cst__', {}).get(test_name, None)
259
            if cst and (not ast or str(test_name).endswith('*')):
260
                report.append('\n### CST')
eckhart's avatar
eckhart committed
261
                report.append(indent(cst.as_sxpr(compact=True)))
262
            if ast:
263
                report.append('\n### AST')
di68kap's avatar
di68kap committed
264
                report.append(indent(ast.as_xml()))
di68kap's avatar
di68kap committed
265
266
267
268
269
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
270
271
272
273
            messages = tests.get('__msg__', {}).get(test_name, "")
            if messages:
                report.append('\n### Messages:')
                report.append(messages)
di68kap's avatar
di68kap committed
274
275
276
277
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
278
279
280
    return '\n'.join(report)


281
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
282
283
    """
    Unit tests for a grammar-parser and ast transformations.
284
    """
di68kap's avatar
di68kap committed
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
    def clean_key(k):
        try:
            return k.replace('*', '')
        except AttributeError:
            return k

    def get(tests, category, key):
        try:
            value = tests[category][key] if key in tests[category] \
                else tests[category][clean_key(key)]
        except KeyError:
            raise AssertionError('%s-test %s for parser %s missing !?'
                                 % (category, test_name, parser_name))
        return value

300
    if isinstance(test_unit, str):
301
        _, unit_name = os.path.split(os.path.splitext(test_unit)[0])
302
        test_unit = unit_from_file(test_unit)
303
    else:
304
        unit_name = 'unit_test_' + str(id(test_unit))
305
306
    if verbose:
        print("\nUnit: " + unit_name)
307
308
309
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
310

311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    with_lookahead = set()  # type: Set[Parser]
    visited = set()  # type: Set[Parser]

    def find_lookahead(p: Parser):
        """Raises a StopIterationError if parser `p` is or contains
        a Lookahead-parser."""
        nonlocal with_lookahead, visited
        visited.add(p)
        if p in with_lookahead or isinstance(p, Lookahead):
            raise StopIteration

    def has_lookahead(parser_name: str):
        """Returns `True`, if given parser is or contains a Lookahead-parser."""
        nonlocal with_lookahead, visited, parser
        p = parser[parser_name]
        if p in with_lookahead:
            return True
        try:
            visited = set()
            p.apply(find_lookahead)
        except StopIteration:
            for vp in visited:
                with_lookahead.add(p)
            return True
        return False

337
    def lookahead_artifact(parser, raw_errors):
di68kap's avatar
di68kap committed
338
        """
339
        Returns True, if the error merely occurred, because the parser
eckhart's avatar
eckhart committed
340
        stopped in front of a sequence that was captured by a lookahead
341
342
343
        operator or if a mandatory lookahead failed at the end of data.
        This is required for testing of parsers that put a lookahead
        operator at the end. See test_testing.TestLookahead.
di68kap's avatar
di68kap committed
344
        """
345
346
347
348
349
350
351
352
        return ((len(raw_errors) == 2  # case 1:  superfluous data for lookahead
                 and raw_errors[-1].code == Error.PARSER_LOOKAHEAD_MATCH_ONLY
                 and raw_errors[-2].code == Error.PARSER_STOPPED_BEFORE_END)
                #  case 2:  mandatory lookahead failure at end of text
                or (len(raw_errors) == 1
                    and raw_errors[-1].code == Error.MANDATORY_CONTINUATION_AT_EOF)
                    and any(isinstance(parser, Lookahead)
                            for parser in parser.history__[-1].call_stack))
di68kap's avatar
di68kap committed
353

354
    for parser_name, tests in test_unit.items():
355
        assert parser_name, "Missing parser name in test %s!" % unit_name
eckhart's avatar
eckhart committed
356
        assert not any(test_type in RESULT_STAGES for test_type in tests), \
357
358
359
360
361
            ("Test %s in %s already has results. Use reset_unit() before running again!"
             % (parser_name, unit_name))
        assert set(tests.keys()).issubset(UNIT_STAGES), \
            'Unknown test-types: %s ! Must be one of %s' \
            % (set(tests.keys()) - UNIT_STAGES, UNIT_STAGES)
362
363
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
364
        match_tests = set(tests['match'].keys()) if 'match' in tests else set()
365
366
        if 'ast' in tests:
            ast_tests = set(tests['ast'].keys())
di68kap's avatar
di68kap committed
367
368
369
            if not {clean_key(k) for k in ast_tests} <= {clean_key(k) for k in match_tests}:
                raise AssertionError('AST-Tests %s for parser %s lack corresponding match-tests!'
                                     % (str(ast_tests - match_tests), parser_name))
370
371
        if 'cst' in tests:
            cst_tests = set(tests['cst'].keys())
di68kap's avatar
di68kap committed
372
            if not {clean_key(k) for k in cst_tests} <= {clean_key(k) for k in match_tests}:
373
374
                raise AssertionError('CST-Tests %s lack corresponding match-tests!'
                                     % str(cst_tests - match_tests))
375
376
377

        # run match tests

378
        for test_name, test_code in tests.get('match', dict()).items():
379
380
381
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
382
            try:
383
                cst = parser(test_code, parser_name, track_history=has_lookahead(parser_name))
384
            except UnknownParserError as upe:
385
                cst = RootNode()
386
                cst = cst.new_error(Node(ZOMBIE_PARSER, "").init_pos(0), str(upe))
eckhart's avatar
eckhart committed
387
            clean_test_name = str(test_name).replace('*', '')
eckhart's avatar
eckhart committed
388
            # log_ST(cst, "match_%s_%s.cst" % (parser_name, clean_test_name))
389
            tests.setdefault('__cst__', {})[test_name] = cst
390
            if "ast" in tests or report:
391
392
393
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
eckhart's avatar
eckhart committed
394
                # log_ST(ast, "match_%s_%s.ast" % (parser_name, clean_test_name))
di68kap's avatar
di68kap committed
395
            raw_errors = cst.collect_errors()
396
            if is_error(cst.error_flag) and not lookahead_artifact(parser, raw_errors):
di68kap's avatar
di68kap committed
397
                errors = adjust_error_locations(raw_errors, test_code)
Eckhart Arnold's avatar
Eckhart Arnold committed
398
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
399
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
400
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
di68kap's avatar
di68kap committed
401
                # tests.setdefault('__err__', {})[test_name] = errata[-1]
402
                # write parsing-history log only in case of failure!
403
                if is_logging():
di68kap's avatar
di68kap committed
404
                    log_parsing_history(parser, "match_%s_%s.log" % (parser_name, clean_test_name))
di68kap's avatar
di68kap committed
405
            elif "cst" in tests and parse_sxpr(get(tests, "cst", test_name)) != cst:
eckhart's avatar
eckhart committed
406
407
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
408
            elif "ast" in tests:
di68kap's avatar
di68kap committed
409
                compare = parse_sxpr(get(tests, "ast", test_name))
410
411
412
413
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
414
415
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
di68kap's avatar
di68kap committed
416
417
            if errata:
                tests.setdefault('__err__', {})[test_name] = errata[-1]
418
            if verbose:
419
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
420

421
        if verbose and 'fail' in tests:
422
            print('  Fail-Tests for parser "' + parser_name + '"')
423
424
425

        # run fail tests

426
        for test_name, test_code in tests.get('fail', dict()).items():
427
428
429
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
430
431
            # cst = parser(test_code, parser_name)
            try:
432
                cst = parser(test_code, parser_name, track_history=has_lookahead(parser_name))
433
            except UnknownParserError as upe:
eckhart's avatar
eckhart committed
434
                node = Node(ZOMBIE_PARSER, "").init_pos(0)
eckhart's avatar
eckhart committed
435
                cst = RootNode(node).new_error(node, str(upe))
436
                errata.append('Unknown parser "{}" in fail test "{}"!'.format(parser_name, test_name))
437
                tests.setdefault('__err__', {})[test_name] = errata[-1]
438
            if not is_error(cst.error_flag) and not lookahead_artifact(parser, cst.collect_errors()):
439
440
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
441
                tests.setdefault('__err__', {})[test_name] = errata[-1]
442
                # write parsing-history log only in case of test-failure
443
                if is_logging():
444
                    log_parsing_history(parser, "fail_%s_%s.log" % (parser_name, test_name))
445
446
447
            if cst.error_flag:
                tests.setdefault('__msg__', {})[test_name] = \
                    "\n".join(str(e) for e in cst.collect_errors())
448
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
449
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
450

451
452
    # write test-report
    if report:
453
        report_dir = "REPORT"
454
455
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
456
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
457
            f.write(get_report(test_unit))
458

459
460
461
    return errata


462
def reset_unit(test_unit):
eckhart's avatar
eckhart committed
463
464
465
466
    """
    Resets the tests in ``test_unit`` by removing all results and error
    messages.
    """
467
468
469
470
471
472
473
474
    for parser, tests in test_unit.items():
        for key in list(tests.keys()):
            if key not in UNIT_STAGES:
                if key not in RESULT_STAGES:
                    print('Removing unknown component %s from test %s' % (key, parser))
                del tests[key]


Eckhart Arnold's avatar
Eckhart Arnold committed
475
476
477
478
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
479
480
    """
    Runs all grammar unit tests in a directory. A file is considered a test
481
482
    unit, if it has the word "test" in its name.
    """
483
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
484
        fn_patterns = [fn_patterns]
485
    all_errors = collections.OrderedDict()
486
487
    if verbose:
        print("\nScanning test-directory: " + directory)
488
489
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
490
491
    if is_logging():
        clear_logs()
492
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
493
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
494
            try:
495
496
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
497
498
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
499
500
501
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
502
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
503
                    raise e
504
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
505
506
    error_report = []
    err_N = 0
507
508
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
509
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
510
            err_N += len(all_errors[filename])
511
512
513
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
di68kap's avatar
di68kap committed
514
515
516
517
        # if verbose:
        #     print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
        return ('Test suite "%s" revealed %s error%s:\n\n'
                % (directory, err_N, 's' if err_N > 1 else '') + '\n'.join(error_report))
eckhart's avatar
eckhart committed
518
519
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
520
521
522
    return ''


523
def runner(test_classes, namespace):
524
525
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
526
    namespace. To run all tests in a module, call
527
    ``runner("", globals())`` from within that module.
528

529
530
531
532
    Unit-Tests are either classes, the name of which starts with
    "Test" and methods, the name of which starts with "test" contained
    in such classes or functions, the name of which starts with "test".

533
534
535
536
537
    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
538
        namespace: The namespace for running the test, usually
539
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
540

541
542
543
544
545
546
547
548
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
549

550
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
551
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
552
            runner("", globals())
553
554
555
556
557
558
559
560
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

561
562
563
    if test_classes:
        if isinstance(test_classes, str):
            test_classes = test_classes.split(" ")
564
565
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
566
567
        test_classes = []
        test_functions = []
568
        for name in namespace.keys():
569
570
571
572
573
574
            if name.lower().startswith('test'):
                if inspect.isclass(namespace[name]):
                    test_classes.append(name)
                elif inspect.isfunction(namespace[name]):
                    test_functions.append(name)

575
576

    obj = None
577
    for test in test_classes:
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()
593
594
595

    for test in test_functions:
        exec(test + '()', namespace)