testing.py 17 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# testing.py - test support for DHParser based grammars and compilers
#
# Copyright 2016  by Eckhart Arnold (arnold@badw.de)
#                 Bavarian Academy of Sciences an Humanities (badw.de)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.  See the License for the specific language governing
# permissions and limitations under the License.
17

18
19
20
21
22
23
24
25
26
"""
Module ``testing`` contains support for unit-testing domain specific
languages. Tests for arbitrarily small components of the Grammar can
be written into test files with ini-file syntax in order to test
whether the parser matches or fails as expected. It can also be
tested whether it produces an expected concrete or abstract syntax tree.
Usually, however, unexpected failure to match a certain string is the
main cause of trouble when constructing a context free Grammar.
"""
27
28


29
import collections
30
# import configparser
31
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
32
import fnmatch
di68kap's avatar
di68kap committed
33
import inspect
34
35
import json
import os
36
import sys
37

38
from DHParser.error import is_error, adjust_error_locations
39
from DHParser.log import is_logging, clear_logs, log_ST, log_parsing_history
40
from DHParser.parse import UnknownParserError
41
42
from DHParser.syntaxtree import Node, mock_syntax_tree, flatten_sxpr, ZOMBIE_PARSER
from DHParser.toolkit import re
43

44
__all__ = ('unit_from_configfile',
45
46
47
48
49
50
51
           'unit_from_json',
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
           'runner')

52
UNIT_STAGES = {'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'}
53

54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
# def unit_from_configfile(config_filename):
#     """
#     Reads a grammar unit test from a config file.
#     """
#     cfg = configparser.ConfigParser(interpolation=None)
#     cfg.read(config_filename, encoding="utf8")
#     OD = collections.OrderedDict
#     unit = OD()
#     for section in cfg.sections():
#         symbol, stage = section.split(':')
#         if stage not in UNIT_STAGES:
#             if symbol in UNIT_STAGES:
#                 symbol, stage = stage, symbol
#             else:
#                 raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
#         for testkey, testcode in cfg[section].items():
#             if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
#                 testcode = testcode[3:-3]
#                 # testcode = testcode.replace('\\#', '#')
#                 testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
#             elif testcode[:1] + testcode[-1:] in {"''", '""'}:
#                 testcode = testcode[1:-1]
#             unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
#     # print(json.dumps(unit, sort_keys=True, indent=4))
#     return unit

RX_SECTION = re.compile('\s*\[(?P<stage>\w+):(?P<symbol>\w+)\]')
RE_VALUE = '(?:"""((?s:.*?))""")|' + "(?:'''((?s:.*?))''')|" + \
           '(?:"(.*?)")|' + "(?:'(.*?)')|" + '(.*(?:\n(?:\s*\n)*    .*)*)'
RX_ENTRY = re.compile('\s*(\w+)\s*:\s*(?:{value})\s*'.format(value=RE_VALUE))
RX_COMMENT = re.compile('\s*#.*\n')

86

87
def unit_from_configfile(config_filename):
88
89
90
91
92
93
94
95
96
    """ Reads grammar unit tests contained in a file in config file (.ini)
    syntax.

    Args:
        config_filename (str): A config file containing Grammar unit-tests

    Returns:
        A dictionary representing the unit tests.
    """
97
98
99
100
101
102
103
    def eat_comments(txt, pos):
        m = RX_COMMENT.match(txt, pos)
        while m:
            pos = m.span()[1]
            m = RX_COMMENT.match(txt, pos)
        return pos

Eckhart Arnold's avatar
Eckhart Arnold committed
104
    with open(config_filename, 'r', encoding="utf-8") as f:
105
106
107
        cfg = f.read()
        cfg = cfg.replace('\t', '    ')

108
109
    OD = collections.OrderedDict
    unit = OD()
110
111
112
113
114
115

    pos = eat_comments(cfg, 0)
    section_match = RX_SECTION.match(cfg, pos)
    while section_match:
        d = section_match.groupdict()
        stage = d['stage']
116
        if stage not in UNIT_STAGES:
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
            raise KeyError('Unknown stage ' + stage + " ! must be one of: " + str(UNIT_STAGES))
        symbol = d['symbol']
        pos = eat_comments(cfg, section_match.span()[1])

        entry_match = RX_ENTRY.match(cfg, pos)
        if entry_match is None:
            raise SyntaxError('No entries in section [%s:%s]' % (stage, symbol))
        while entry_match:
            testkey, testcode = [group for group in entry_match.groups() if group is not None]
            lines = testcode.split('\n')
            if len(lines) > 1:
                indent = sys.maxsize
                for line in lines[1:]:
                    indent = min(indent, len(line) - len(line.lstrip()))
                for i in range(1, len(lines)):
                    lines[i] = lines[i][indent:]
                testcode = '\n'.join(lines)
134
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
135
136
137
138
139
140
141
142
            pos = eat_comments(cfg, entry_match.span()[1])
            entry_match = RX_ENTRY.match(cfg, pos)

        section_match = RX_SECTION.match(cfg, pos)

    if pos != len(cfg):
        raise SyntaxError('in file %s in line %i' % (config_filename, cfg[:pos].count('\n') + 1))

143
    return unit
144

145

146
def unit_from_json(json_filename):
147
    """
148
    Reads grammar unit tests from a json file.
149
    """
di68kap's avatar
di68kap committed
150
    with open(json_filename, 'r', encoding='utf8') as f:
151
152
153
154
        unit = json.load(f)
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
155
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
156
157
    return unit

158
# TODO: add support for yaml, cson, toml
159
160


161
def unit_from_file(filename):
162
163
    """
    Reads a grammar unit test from a file. The format of the file is
164
165
    determined by the ending of its name.
    """
166
    if filename.endswith(".json"):
di68kap's avatar
di68kap committed
167
        test_unit = unit_from_json(filename)
168
    elif filename.endswith(".ini"):
di68kap's avatar
di68kap committed
169
        test_unit = unit_from_configfile(filename)
170
    else:
171
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
172

di68kap's avatar
di68kap committed
173
174
175
176
177
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
178
179
        intersection = list(m_names & f_names)
        intersection.sort()
di68kap's avatar
di68kap committed
180
181
182
183
184
185
186
187
188
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

189

190
def get_report(test_unit):
191
192
    """
    Returns a text-report of the results of a grammar unit test.
193
    """
194
195
196
197
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
198
199
200
201
202
203
204
205
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
        for test_name, test_code in tests.get('match', dict()).items():
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
206
            report.append(indent(test_code))
207
208
209
210
211
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
212
213
214
            cst = tests.get('__cst__', {}).get(test_name, None)
            if cst and (not ast or cst == ast):
                report.append('\n### CST')
215
                report.append(indent(cst.as_sxpr()))
216
            elif ast:
217
                report.append('\n### AST')
218
                report.append(indent(ast.as_sxpr()))
di68kap's avatar
di68kap committed
219
220
221
222
223
224
225
226
227
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
228
229
230
    return '\n'.join(report)


231
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
232
233
    """
    Unit tests for a grammar-parser and ast transformations.
234
    """
235
    if isinstance(test_unit, str):
236
        _, unit_name = os.path.split(os.path.splitext(test_unit)[0])
237
        test_unit = unit_from_file(test_unit)
238
239
    else:
        unit_name = str(id(test_unit))
240
241
    if verbose:
        print("\nUnit: " + unit_name)
242
243
244
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
245
246
    for parser_name, tests in test_unit.items():
        assert set(tests.keys()).issubset(UNIT_STAGES)
247
248
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
249
        for test_name, test_code in tests.get('match', dict()).items():
250
251
252
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
253
254
255
256
            try:
                cst = parser(test_code, parser_name)
            except UnknownParserError as upe:
                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
257
            log_ST(cst, "match_%s_%s.cst" % (parser_name, test_name))
258
            tests.setdefault('__cst__', {})[test_name] = cst
259
            if "ast" in tests or report:
260
261
262
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
263
                log_ST(ast, "match_%s_%s.ast" % (parser_name, test_name))
264
            if is_error(cst.error_flag):
eckhart's avatar
eckhart committed
265
                errors = adjust_error_locations(cst.collect_errors(), test_code)
Eckhart Arnold's avatar
Eckhart Arnold committed
266
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
267
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
268
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
269
                tests.setdefault('__err__', {})[test_name] = errata[-1]
270
                # write parsing-history log only in case of failure!
271
                if is_logging():
272
                    log_parsing_history(parser, "match_%s_%s.log" % (parser_name, test_name))
273
            elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
eckhart's avatar
eckhart committed
274
275
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
276
277
278
279
280
281
            elif "ast" in tests:
                compare = mock_syntax_tree(tests["ast"][test_name])
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
282
283
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
284
                    tests.setdefault('__err__', {})[test_name] = errata[-1]
285
            if verbose:
286
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
287

288
        if verbose and 'fail' in tests:
289
            print('  Fail-Tests for parser "' + parser_name + '"')
290
        for test_name, test_code in tests.get('fail', dict()).items():
291
292
293
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
294
295
296
297
298
            # cst = parser(test_code, parser_name)
            try:
                cst = parser(test_code, parser_name)
            except UnknownParserError as upe:
                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
299
            if not is_error(cst.error_flag):
300
301
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
302
                tests.setdefault('__err__', {})[test_name] = errata[-1]
303
                # write parsing-history log only in case of test-failure
304
                if is_logging():
305
                    log_parsing_history(parser, "fail_%s_%s.log" % (parser_name, test_name))
306
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
307
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
308

309
310
    # write test-report
    if report:
311
        report_dir = "REPORT"
312
313
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
314
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
315
            f.write(get_report(test_unit))
316

317
318
319
    return errata


Eckhart Arnold's avatar
Eckhart Arnold committed
320
321
322
323
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
324
325
    """
    Runs all grammar unit tests in a directory. A file is considered a test
326
327
    unit, if it has the word "test" in its name.
    """
328
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
329
        fn_patterns = [fn_patterns]
330
    all_errors = collections.OrderedDict()
331
332
    if verbose:
        print("\nScanning test-directory: " + directory)
333
334
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
335
336
    if is_logging():
        clear_logs()
337
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
338
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
339
            try:
340
341
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
342
343
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
344
345
346
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
347
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
348
                    raise e
349
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
350
351
    error_report = []
    err_N = 0
352
353
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
354
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
355
            err_N += len(all_errors[filename])
356
357
358
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
di68kap's avatar
di68kap committed
359
360
361
362
        # if verbose:
        #     print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
        return ('Test suite "%s" revealed %s error%s:\n\n'
                % (directory, err_N, 's' if err_N > 1 else '') + '\n'.join(error_report))
eckhart's avatar
eckhart committed
363
364
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
365
366
367
    return ''


368
def runner(tests, namespace):
369
370
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
371
    namespace. To run all tests in a module, call
372
    ``runner("", globals())`` from within that module.
373
374
375
376
377
378

    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
379
        namespace: The namespace for running the test, usually
380
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
381

382
383
384
385
386
387
388
389
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
390

391
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
392
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
393
            runner("", globals())
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

    if tests:
        if isinstance(tests, str):
            tests = tests.split(" ")
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
        tests = []
        for name in namespace.keys():
            if name.lower().startswith('test') and inspect.isclass(namespace[name]):
                tests.append(name)

    obj = None
    for test in tests:
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()