testing.py 16.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
"""testing.py - test support for DHParser based grammars and compilers

Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                Bavarian Academy of Sciences an Humanities (badw.de)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.
"""
18
import collections
19
# import configparser
20
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
21
import fnmatch
di68kap's avatar
di68kap committed
22
import inspect
23
24
import json
import os
25
import sys
26

27
from DHParser.error import is_error, adjust_error_locations
28
from DHParser.log import is_logging, clear_logs, log_ST, log_parsing_history
29
from DHParser.parse import UnknownParserError
30
31
from DHParser.syntaxtree import Node, mock_syntax_tree, flatten_sxpr, ZOMBIE_PARSER
from DHParser.toolkit import re
32

33
__all__ = ('unit_from_configfile',
34
35
36
37
38
39
40
           'unit_from_json',
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
           'runner')

41
UNIT_STAGES = {'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'}
42

43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# def unit_from_configfile(config_filename):
#     """
#     Reads a grammar unit test from a config file.
#     """
#     cfg = configparser.ConfigParser(interpolation=None)
#     cfg.read(config_filename, encoding="utf8")
#     OD = collections.OrderedDict
#     unit = OD()
#     for section in cfg.sections():
#         symbol, stage = section.split(':')
#         if stage not in UNIT_STAGES:
#             if symbol in UNIT_STAGES:
#                 symbol, stage = stage, symbol
#             else:
#                 raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
#         for testkey, testcode in cfg[section].items():
#             if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
#                 testcode = testcode[3:-3]
#                 # testcode = testcode.replace('\\#', '#')
#                 testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
#             elif testcode[:1] + testcode[-1:] in {"''", '""'}:
#                 testcode = testcode[1:-1]
#             unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
#     # print(json.dumps(unit, sort_keys=True, indent=4))
#     return unit

RX_SECTION = re.compile('\s*\[(?P<stage>\w+):(?P<symbol>\w+)\]')
RE_VALUE = '(?:"""((?s:.*?))""")|' + "(?:'''((?s:.*?))''')|" + \
           '(?:"(.*?)")|' + "(?:'(.*?)')|" + '(.*(?:\n(?:\s*\n)*    .*)*)'
RX_ENTRY = re.compile('\s*(\w+)\s*:\s*(?:{value})\s*'.format(value=RE_VALUE))
RX_COMMENT = re.compile('\s*#.*\n')

75

76
def unit_from_configfile(config_filename):
77
78
79
80
81
82
83
84
85
86
87
    def eat_comments(txt, pos):
        m = RX_COMMENT.match(txt, pos)
        while m:
            pos = m.span()[1]
            m = RX_COMMENT.match(txt, pos)
        return pos

    with open(config_filename, 'r') as f:
        cfg = f.read()
        cfg = cfg.replace('\t', '    ')

88
89
    OD = collections.OrderedDict
    unit = OD()
90
91
92
93
94
95

    pos = eat_comments(cfg, 0)
    section_match = RX_SECTION.match(cfg, pos)
    while section_match:
        d = section_match.groupdict()
        stage = d['stage']
96
        if stage not in UNIT_STAGES:
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
            raise KeyError('Unknown stage ' + stage + " ! must be one of: " + str(UNIT_STAGES))
        symbol = d['symbol']
        pos = eat_comments(cfg, section_match.span()[1])

        entry_match = RX_ENTRY.match(cfg, pos)
        if entry_match is None:
            raise SyntaxError('No entries in section [%s:%s]' % (stage, symbol))
        while entry_match:
            testkey, testcode = [group for group in entry_match.groups() if group is not None]
            lines = testcode.split('\n')
            if len(lines) > 1:
                indent = sys.maxsize
                for line in lines[1:]:
                    indent = min(indent, len(line) - len(line.lstrip()))
                for i in range(1, len(lines)):
                    lines[i] = lines[i][indent:]
                testcode = '\n'.join(lines)
114
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
115
116
117
118
119
120
121
122
            pos = eat_comments(cfg, entry_match.span()[1])
            entry_match = RX_ENTRY.match(cfg, pos)

        section_match = RX_SECTION.match(cfg, pos)

    if pos != len(cfg):
        raise SyntaxError('in file %s in line %i' % (config_filename, cfg[:pos].count('\n') + 1))

123
    return unit
124

125

126
def unit_from_json(json_filename):
127
128
    """
    Reads a grammar unit test from a json file.
129
    """
di68kap's avatar
di68kap committed
130
    with open(json_filename, 'r', encoding='utf8') as f:
131
132
133
134
        unit = json.load(f)
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
135
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
136
137
    return unit

138
# TODO: add support for yaml, cson, toml
139
140


141
def unit_from_file(filename):
142
143
    """
    Reads a grammar unit test from a file. The format of the file is
144
145
    determined by the ending of its name.
    """
146
    if filename.endswith(".json"):
di68kap's avatar
di68kap committed
147
        test_unit = unit_from_json(filename)
148
    elif filename.endswith(".ini"):
di68kap's avatar
di68kap committed
149
        test_unit = unit_from_configfile(filename)
150
    else:
151
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
152

di68kap's avatar
di68kap committed
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
        intersection = list(m_names & f_names);  intersection.sort()
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

168

169
def get_report(test_unit):
170
171
    """
    Returns a text-report of the results of a grammar unit test.
172
    """
173
174
175
176
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
177
178
179
180
181
182
183
184
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
        for test_name, test_code in tests.get('match', dict()).items():
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
185
            report.append(indent(test_code))
186
187
188
189
190
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
191
192
193
            cst = tests.get('__cst__', {}).get(test_name, None)
            if cst and (not ast or cst == ast):
                report.append('\n### CST')
194
                report.append(indent(cst.as_sxpr()))
195
            elif ast:
196
                report.append('\n### AST')
197
                report.append(indent(ast.as_sxpr()))
di68kap's avatar
di68kap committed
198
199
200
201
202
203
204
205
206
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
207
208
209
    return '\n'.join(report)


210
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
211
212
    """
    Unit tests for a grammar-parser and ast transformations.
213
    """
214
    if isinstance(test_unit, str):
215
        unit_dir, unit_name = os.path.split(os.path.splitext(test_unit)[0])
216
        test_unit = unit_from_file(test_unit)
217
    else:
218
        unit_dir = ""
219
        unit_name = str(id(test_unit))
220
221
    if verbose:
        print("\nUnit: " + unit_name)
222
223
224
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
225
226
    for parser_name, tests in test_unit.items():
        assert set(tests.keys()).issubset(UNIT_STAGES)
227
228
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
229
        for test_name, test_code in tests.get('match', dict()).items():
230
231
232
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
233
234
235
236
            try:
                cst = parser(test_code, parser_name)
            except UnknownParserError as upe:
                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
237
            log_ST(cst, "match_%s_%s.cst" % (parser_name, test_name))
238
            tests.setdefault('__cst__', {})[test_name] = cst
239
            if "ast" in tests or report:
240
241
242
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
243
                log_ST(ast, "match_%s_%s.ast" % (parser_name, test_name))
244
            if is_error(cst.error_flag):
eckhart's avatar
eckhart committed
245
                errors = adjust_error_locations(cst.collect_errors(), test_code)
Eckhart Arnold's avatar
Eckhart Arnold committed
246
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
247
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
248
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
249
                tests.setdefault('__err__', {})[test_name] = errata[-1]
250
                # write parsing-history log only in case of failure!
251
                if is_logging():
252
                    log_parsing_history(parser, "match_%s_%s.log" % (parser_name, test_name))
253
            elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
eckhart's avatar
eckhart committed
254
255
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
256
257
258
259
260
261
            elif "ast" in tests:
                compare = mock_syntax_tree(tests["ast"][test_name])
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
262
263
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
264
                    tests.setdefault('__err__', {})[test_name] = errata[-1]
265
            if verbose:
266
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
267

268
        if verbose and 'fail' in tests:
269
            print('  Fail-Tests for parser "' + parser_name + '"')
270
        for test_name, test_code in tests.get('fail', dict()).items():
271
272
273
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
274
275
276
277
278
            # cst = parser(test_code, parser_name)
            try:
                cst = parser(test_code, parser_name)
            except UnknownParserError as upe:
                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
279
            if not is_error(cst.error_flag):
280
281
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
282
                tests.setdefault('__err__', {})[test_name] = errata[-1]
283
                # write parsing-history log only in case of test-failure
284
                if is_logging():
285
                    log_parsing_history(parser, "fail_%s_%s.log" % (parser_name, test_name))
286
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
287
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
288

289
290
    # write test-report
    if report:
291
        report_dir = "REPORT"
292
293
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
294
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
295
            f.write(get_report(test_unit))
296

297
298
299
    return errata


Eckhart Arnold's avatar
Eckhart Arnold committed
300
301
302
303
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
304
305
    """
    Runs all grammar unit tests in a directory. A file is considered a test
306
307
    unit, if it has the word "test" in its name.
    """
308
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
309
        fn_patterns = [fn_patterns]
310
    all_errors = collections.OrderedDict()
311
312
    if verbose:
        print("\nScanning test-directory: " + directory)
313
314
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
315
316
    if is_logging():
        clear_logs()
317
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
318
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
319
            try:
320
321
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
322
323
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
324
325
326
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
327
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
328
                    raise e
329
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
330
331
    error_report = []
    err_N = 0
332
333
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
334
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
335
            err_N += len(all_errors[filename])
336
337
338
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
di68kap's avatar
di68kap committed
339
340
341
342
        # if verbose:
        #     print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
        return ('Test suite "%s" revealed %s error%s:\n\n'
                % (directory, err_N, 's' if err_N > 1 else '') + '\n'.join(error_report))
eckhart's avatar
eckhart committed
343
344
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
345
346
347
    return ''


348
def runner(tests, namespace):
349
350
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
351
    namespace. To run all tests in a module, call
352
    ``runner("", globals())`` from within that module.
353
354
355
356
357
358

    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
359
        namespace: The namespace for running the test, usually
360
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
361

362
363
364
365
366
367
368
369
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
370

371
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
372
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
373
            runner("", globals())
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

    if tests:
        if isinstance(tests, str):
            tests = tests.split(" ")
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
        tests = []
        for name in namespace.keys():
            if name.lower().startswith('test') and inspect.isclass(namespace[name]):
                tests.append(name)

    obj = None
    for test in tests:
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()