testing.py 24 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# testing.py - test support for DHParser based grammars and compilers
#
# Copyright 2016  by Eckhart Arnold (arnold@badw.de)
#                 Bavarian Academy of Sciences an Humanities (badw.de)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
17

18
19
20
21
22
23
24
25
26
"""
Module ``testing`` contains support for unit-testing domain specific
languages. Tests for arbitrarily small components of the Grammar can
be written into test files with ini-file syntax in order to test
whether the parser matches or fails as expected. It can also be
tested whether it produces an expected concrete or abstract syntax tree.
Usually, however, unexpected failure to match a certain string is the
main cause of trouble when constructing a context free Grammar.
"""
27
28


29
import collections
30
# import configparser
31
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
32
import fnmatch
di68kap's avatar
di68kap committed
33
import inspect
34
35
import json
import os
36
import sys
37
import threading
38

di68kap's avatar
di68kap committed
39
from DHParser.error import Error, is_error, adjust_error_locations
40
41
from DHParser.log import is_logging, clear_logs, log_parsing_history
from DHParser.parse import UnknownParserError, Parser, Lookahead
42
from DHParser.syntaxtree import Node, RootNode, parse_sxpr, flatten_sxpr, ZOMBIE
43
44
45
from DHParser.toolkit import re, typing

from typing import Tuple
46

di68kap's avatar
di68kap committed
47
__all__ = ('unit_from_config',
48
           'unit_from_json',
di68kap's avatar
di68kap committed
49
           'TEST_READERS',
50
51
52
53
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
54
           'reset_unit',
55
56
           'runner')

57
58
UNIT_STAGES = {'match*', 'match', 'fail', 'ast', 'cst'}
RESULT_STAGES = {'__cst__', '__ast__', '__err__'}
59

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# def unit_from_configfile(config_filename):
#     """
#     Reads a grammar unit test from a config file.
#     """
#     cfg = configparser.ConfigParser(interpolation=None)
#     cfg.read(config_filename, encoding="utf8")
#     OD = collections.OrderedDict
#     unit = OD()
#     for section in cfg.sections():
#         symbol, stage = section.split(':')
#         if stage not in UNIT_STAGES:
#             if symbol in UNIT_STAGES:
#                 symbol, stage = stage, symbol
#             else:
#                 raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
#         for testkey, testcode in cfg[section].items():
#             if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
#                 testcode = testcode[3:-3]
#                 # testcode = testcode.replace('\\#', '#')
#                 testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
#             elif testcode[:1] + testcode[-1:] in {"''", '""'}:
#                 testcode = testcode[1:-1]
#             unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
#     # print(json.dumps(unit, sort_keys=True, indent=4))
#     return unit

eckhart's avatar
eckhart committed
86
RX_SECTION = re.compile(r'\s*\[(?P<stage>\w+):(?P<symbol>\w+)\]')
87
RE_VALUE = '(?:"""((?:.|\n)*?)""")|' + "(?:'''((?:.|\n)*?)''')|" + \
eckhart's avatar
eckhart committed
88
           r'(?:"(.*?)")|' + "(?:'(.*?)')|" + r'(.*(?:\n(?:\s*\n)*    .*)*)'
89
90
# the following does not work with pypy3, because pypy's re-engine does not
# support local flags, e.g. '(?s: )'
eckhart's avatar
eckhart committed
91
92
93
94
# RE_VALUE = r'(?:"""((?s:.*?))""")|' + "(?:'''((?s:.*?))''')|" + \
#            r'(?:"(.*?)")|' + "(?:'(.*?)')|" + '(.*(?:\n(?:\s*\n)*    .*)*)'
RX_ENTRY = re.compile(r'\s*(\w+\*?)\s*:\s*(?:{value})\s*'.format(value=RE_VALUE))
RX_COMMENT = re.compile(r'\s*#.*\n')
95

96

di68kap's avatar
di68kap committed
97
def unit_from_config(config_str):
98
99
100
101
    """ Reads grammar unit tests contained in a file in config file (.ini)
    syntax.

    Args:
di68kap's avatar
di68kap committed
102
        config_str (str): A string containing a config-file with Grammar unit-tests
103
104
105
106

    Returns:
        A dictionary representing the unit tests.
    """
eckhart's avatar
eckhart committed
107
108
    # TODO: issue a warning if the same match:xxx or fail:xxx block appears more than once

109
110
111
112
113
114
115
    def eat_comments(txt, pos):
        m = RX_COMMENT.match(txt, pos)
        while m:
            pos = m.span()[1]
            m = RX_COMMENT.match(txt, pos)
        return pos

di68kap's avatar
di68kap committed
116
    cfg = config_str.replace('\t', '    ')
117

118
119
    OD = collections.OrderedDict
    unit = OD()
120
121
122
123
124
125

    pos = eat_comments(cfg, 0)
    section_match = RX_SECTION.match(cfg, pos)
    while section_match:
        d = section_match.groupdict()
        stage = d['stage']
126
        if stage not in UNIT_STAGES:
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
            raise KeyError('Unknown stage ' + stage + " ! must be one of: " + str(UNIT_STAGES))
        symbol = d['symbol']
        pos = eat_comments(cfg, section_match.span()[1])

        entry_match = RX_ENTRY.match(cfg, pos)
        if entry_match is None:
            raise SyntaxError('No entries in section [%s:%s]' % (stage, symbol))
        while entry_match:
            testkey, testcode = [group for group in entry_match.groups() if group is not None]
            lines = testcode.split('\n')
            if len(lines) > 1:
                indent = sys.maxsize
                for line in lines[1:]:
                    indent = min(indent, len(line) - len(line.lstrip()))
                for i in range(1, len(lines)):
                    lines[i] = lines[i][indent:]
                testcode = '\n'.join(lines)
144
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
145
146
147
148
149
150
            pos = eat_comments(cfg, entry_match.span()[1])
            entry_match = RX_ENTRY.match(cfg, pos)

        section_match = RX_SECTION.match(cfg, pos)

    if pos != len(cfg):
151
        raise SyntaxError('in line %i' % (cfg[:pos].count('\n') + 1))
152

153
    return unit
154

155

di68kap's avatar
di68kap committed
156
def unit_from_json(json_str):
157
    """
di68kap's avatar
di68kap committed
158
    Reads grammar unit tests from a json string.
159
    """
di68kap's avatar
di68kap committed
160
    unit = json.loads(json_str)
161
162
163
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
164
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
165
166
    return unit

di68kap's avatar
di68kap committed
167

168
# TODO: add support for yaml, cson, toml
169
170


di68kap's avatar
di68kap committed
171
172
173
174
175
176
177
178
179
# A dictionary associating file endings with reader functions that
# transfrom strings containing the file's content to a nested dictionary
# structure of test cases.
TEST_READERS = {
    '.ini': unit_from_config,
    '.json': unit_from_json
}


180
def unit_from_file(filename):
181
182
    """
    Reads a grammar unit test from a file. The format of the file is
183
184
    determined by the ending of its name.
    """
di68kap's avatar
di68kap committed
185
186
187
188
189
190
    try:
        reader = TEST_READERS[os.path.splitext(filename)[1].lower()]
        with open(filename, 'r', encoding='utf8') as f:
            data = f.read()
        test_unit = reader(data)
    except KeyError:
191
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
192

di68kap's avatar
di68kap committed
193
194
195
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
di68kap's avatar
di68kap committed
196
197
198
199
200
201
202
203
        # normalize case for test category names
        keys = list(tests.keys())
        for key in keys:
            new_key = key.lower()
            if new_key != key:
                tests[new_key] = tests[keys]
                del tests[keys]

di68kap's avatar
di68kap committed
204
205
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
206
207
        intersection = list(m_names & f_names)
        intersection.sort()
di68kap's avatar
di68kap committed
208
209
210
211
212
213
214
215
216
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

217

di68kap's avatar
di68kap committed
218
219
220
221
222
223
# def all_match_tests(tests):
#     """Returns all match tests from ``tests``, This includes match tests
#     marked with an asterix for CST-output as well as unmarked match-tests.
#     """
#     return itertools.chain(tests.get('match', dict()).items(),
#                            tests.get('match*', dict()).items())
224
225


226
def get_report(test_unit):
227
    """
228
229
230
231
    Returns a text-report of the results of a grammar unit test. The report
    lists the source of all tests as well as the error messages, if a test
    failed or the abstract-syntax-tree (AST) in case of success.

232
233
    If an asterix has been appended to the test name then the concrete syntax
    tree will also be added to the report in this particular case.
234
235
236
237

    The purpose of the latter is to help constructing and debugging
    of AST-Transformations. It is better to switch the CST-output on and off
    with the asterix marker when needed than to output the CST for all tests
238
    which would unnecessarily bloat the test reports.
239
    """
240
241
242
243
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
244
245
246
247
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
248
        for test_name, test_code in tests.get('match', dict()).items():
249
250
251
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
252
            report.append(indent(test_code))
253
254
255
256
257
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
258
            cst = tests.get('__cst__', {}).get(test_name, None)
259
            if cst and (not ast or str(test_name).endswith('*')):
260
                report.append('\n### CST')
eckhart's avatar
eckhart committed
261
                report.append(indent(cst.as_sxpr(compact=True)))
262
            if ast:
263
                report.append('\n### AST')
di68kap's avatar
di68kap committed
264
                report.append(indent(ast.as_xml()))
di68kap's avatar
di68kap committed
265
266
267
268
269
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
270
271
272
273
            messages = tests.get('__msg__', {}).get(test_name, "")
            if messages:
                report.append('\n### Messages:')
                report.append(messages)
di68kap's avatar
di68kap committed
274
275
276
277
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
278
279
280
    return '\n'.join(report)


281
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
282
283
    """
    Unit tests for a grammar-parser and ast transformations.
284
    """
di68kap's avatar
di68kap committed
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
    def clean_key(k):
        try:
            return k.replace('*', '')
        except AttributeError:
            return k

    def get(tests, category, key):
        try:
            value = tests[category][key] if key in tests[category] \
                else tests[category][clean_key(key)]
        except KeyError:
            raise AssertionError('%s-test %s for parser %s missing !?'
                                 % (category, test_name, parser_name))
        return value

300
    if isinstance(test_unit, str):
301
        _, unit_name = os.path.split(os.path.splitext(test_unit)[0])
302
        test_unit = unit_from_file(test_unit)
303
    else:
304
        unit_name = 'unit_test_' + str(id(test_unit))
305
306
    if verbose:
        print("\nUnit: " + unit_name)
307
308
309
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
310

311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    with_lookahead = set()  # type: Set[Parser]
    visited = set()  # type: Set[Parser]

    def find_lookahead(p: Parser):
        """Raises a StopIterationError if parser `p` is or contains
        a Lookahead-parser."""
        nonlocal with_lookahead, visited
        visited.add(p)
        if p in with_lookahead or isinstance(p, Lookahead):
            raise StopIteration

    def has_lookahead(parser_name: str):
        """Returns `True`, if given parser is or contains a Lookahead-parser."""
        nonlocal with_lookahead, visited, parser
        p = parser[parser_name]
        if p in with_lookahead:
            return True
        try:
            visited = set()
            p.apply(find_lookahead)
        except StopIteration:
            for vp in visited:
                with_lookahead.add(p)
            return True
        return False

337
    def lookahead_artifact(parser, raw_errors):
di68kap's avatar
di68kap committed
338
        """
339
        Returns True, if the error merely occurred, because the parser
eckhart's avatar
eckhart committed
340
        stopped in front of a sequence that was captured by a lookahead
341
342
343
        operator or if a mandatory lookahead failed at the end of data.
        This is required for testing of parsers that put a lookahead
        operator at the end. See test_testing.TestLookahead.
di68kap's avatar
di68kap committed
344
        """
345
346
347
348
349
350
        return ((len(raw_errors) == 2  # case 1:  superfluous data for lookahead
                 and raw_errors[-1].code == Error.PARSER_LOOKAHEAD_MATCH_ONLY
                 and raw_errors[-2].code == Error.PARSER_STOPPED_BEFORE_END)
                #  case 2:  mandatory lookahead failure at end of text
                or (len(raw_errors) == 1
                    and raw_errors[-1].code == Error.MANDATORY_CONTINUATION_AT_EOF)
Eckhart Arnold's avatar
Eckhart Arnold committed
351
352
                    and any(isinstance(p, Lookahead)
                            for p in parser.history__[-1].call_stack))
di68kap's avatar
di68kap committed
353

354
    for parser_name, tests in test_unit.items():
355
        assert parser_name, "Missing parser name in test %s!" % unit_name
eckhart's avatar
eckhart committed
356
        assert not any(test_type in RESULT_STAGES for test_type in tests), \
357
358
359
360
361
            ("Test %s in %s already has results. Use reset_unit() before running again!"
             % (parser_name, unit_name))
        assert set(tests.keys()).issubset(UNIT_STAGES), \
            'Unknown test-types: %s ! Must be one of %s' \
            % (set(tests.keys()) - UNIT_STAGES, UNIT_STAGES)
362
363
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
364
        match_tests = set(tests['match'].keys()) if 'match' in tests else set()
365
366
        if 'ast' in tests:
            ast_tests = set(tests['ast'].keys())
di68kap's avatar
di68kap committed
367
368
369
            if not {clean_key(k) for k in ast_tests} <= {clean_key(k) for k in match_tests}:
                raise AssertionError('AST-Tests %s for parser %s lack corresponding match-tests!'
                                     % (str(ast_tests - match_tests), parser_name))
370
371
        if 'cst' in tests:
            cst_tests = set(tests['cst'].keys())
di68kap's avatar
di68kap committed
372
            if not {clean_key(k) for k in cst_tests} <= {clean_key(k) for k in match_tests}:
373
374
                raise AssertionError('CST-Tests %s lack corresponding match-tests!'
                                     % str(cst_tests - match_tests))
375
376
377

        # run match tests

378
        for test_name, test_code in tests.get('match', dict()).items():
Eckhart Arnold's avatar
Eckhart Arnold committed
379
            errflag = 0
380
381
382
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
383
            try:
384
                cst = parser(test_code, parser_name, track_history=has_lookahead(parser_name))
385
            except UnknownParserError as upe:
386
                cst = RootNode()
387
                cst = cst.new_error(Node(ZOMBIE, "").init_pos(0), str(upe))
eckhart's avatar
eckhart committed
388
            clean_test_name = str(test_name).replace('*', '')
eckhart's avatar
eckhart committed
389
            # log_ST(cst, "match_%s_%s.cst" % (parser_name, clean_test_name))
390
            tests.setdefault('__cst__', {})[test_name] = cst
391
            if "ast" in tests or report:
392
393
394
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
eckhart's avatar
eckhart committed
395
                # log_ST(ast, "match_%s_%s.ast" % (parser_name, clean_test_name))
di68kap's avatar
di68kap committed
396
            raw_errors = cst.collect_errors()
397
            if is_error(cst.error_flag) and not lookahead_artifact(parser, raw_errors):
di68kap's avatar
di68kap committed
398
                errors = adjust_error_locations(raw_errors, test_code)
Eckhart Arnold's avatar
Eckhart Arnold committed
399
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
400
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
401
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
di68kap's avatar
di68kap committed
402
                # tests.setdefault('__err__', {})[test_name] = errata[-1]
403
                # write parsing-history log only in case of failure!
404
                if is_logging():
di68kap's avatar
di68kap committed
405
                    log_parsing_history(parser, "match_%s_%s.log" % (parser_name, clean_test_name))
di68kap's avatar
di68kap committed
406
            elif "cst" in tests and parse_sxpr(get(tests, "cst", test_name)) != cst:
eckhart's avatar
eckhart committed
407
408
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
409
            elif "ast" in tests:
di68kap's avatar
di68kap committed
410
                compare = parse_sxpr(get(tests, "ast", test_name))
411
412
413
414
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
415
416
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
di68kap's avatar
di68kap committed
417
418
            if errata:
                tests.setdefault('__err__', {})[test_name] = errata[-1]
419
            if verbose:
420
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
421

422
        if verbose and 'fail' in tests:
423
            print('  Fail-Tests for parser "' + parser_name + '"')
424
425
426

        # run fail tests

427
        for test_name, test_code in tests.get('fail', dict()).items():
Eckhart Arnold's avatar
Eckhart Arnold committed
428
            errflag = 0
429
430
431
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
432
433
            # cst = parser(test_code, parser_name)
            try:
434
                cst = parser(test_code, parser_name, track_history=has_lookahead(parser_name))
435
            except UnknownParserError as upe:
436
                node = Node(ZOMBIE, "").init_pos(0)
eckhart's avatar
eckhart committed
437
                cst = RootNode(node).new_error(node, str(upe))
438
                errata.append('Unknown parser "{}" in fail test "{}"!'.format(parser_name, test_name))
439
                tests.setdefault('__err__', {})[test_name] = errata[-1]
440
            if not is_error(cst.error_flag) and not lookahead_artifact(parser, cst.collect_errors()):
441
442
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
443
                tests.setdefault('__err__', {})[test_name] = errata[-1]
444
                # write parsing-history log only in case of test-failure
445
                if is_logging():
446
                    log_parsing_history(parser, "fail_%s_%s.log" % (parser_name, test_name))
447
448
449
            if cst.error_flag:
                tests.setdefault('__msg__', {})[test_name] = \
                    "\n".join(str(e) for e in cst.collect_errors())
450
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
451
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
452

453
454
    # write test-report
    if report:
455
        report_dir = "REPORT"
456
457
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
458
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
459
            f.write(get_report(test_unit))
460

461
462
463
    return errata


464
def reset_unit(test_unit):
eckhart's avatar
eckhart committed
465
466
467
468
    """
    Resets the tests in ``test_unit`` by removing all results and error
    messages.
    """
469
470
471
472
473
474
475
476
    for parser, tests in test_unit.items():
        for key in list(tests.keys()):
            if key not in UNIT_STAGES:
                if key not in RESULT_STAGES:
                    print('Removing unknown component %s from test %s' % (key, parser))
                del tests[key]


Eckhart Arnold's avatar
Eckhart Arnold committed
477
478
479
480
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
481
482
    """
    Runs all grammar unit tests in a directory. A file is considered a test
483
484
    unit, if it has the word "test" in its name.
    """
485
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
486
        fn_patterns = [fn_patterns]
487
    all_errors = collections.OrderedDict()
488
489
    if verbose:
        print("\nScanning test-directory: " + directory)
490
491
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
492
493
    if is_logging():
        clear_logs()
494
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
495
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
496
            try:
497
498
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
499
500
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
501
502
503
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
504
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
505
                    raise e
506
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
507
508
    error_report = []
    err_N = 0
509
510
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
511
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
512
            err_N += len(all_errors[filename])
513
514
515
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
di68kap's avatar
di68kap committed
516
517
518
519
        # if verbose:
        #     print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
        return ('Test suite "%s" revealed %s error%s:\n\n'
                % (directory, err_N, 's' if err_N > 1 else '') + '\n'.join(error_report))
eckhart's avatar
eckhart committed
520
521
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
522
523
524
    return ''


525
def runner(test_classes, namespace):
526
527
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
528
    namespace. To run all tests in a module, call
529
    ``runner("", globals())`` from within that module.
530

531
532
533
534
    Unit-Tests are either classes, the name of which starts with
    "Test" and methods, the name of which starts with "test" contained
    in such classes or functions, the name of which starts with "test".

535
536
537
538
539
    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
540
        namespace: The namespace for running the test, usually
541
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
542

543
544
545
546
547
548
549
550
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
551

552
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
553
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
554
            runner("", globals())
555
556
557
558
559
560
561
562
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

563
564
565
    if test_classes:
        if isinstance(test_classes, str):
            test_classes = test_classes.split(" ")
566
567
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
568
569
        test_classes = []
        test_functions = []
570
        for name in namespace.keys():
571
572
573
574
575
576
            if name.lower().startswith('test'):
                if inspect.isclass(namespace[name]):
                    test_classes.append(name)
                elif inspect.isfunction(namespace[name]):
                    test_functions.append(name)

577
578

    obj = None
579
    for test in test_classes:
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()
595
596
597

    for test in test_functions:
        exec(test + '()', namespace)