testing.py 13.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
"""testing.py - test support for DHParser based grammars and compilers

Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                Bavarian Academy of Sciences an Humanities (badw.de)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.
"""
18
import collections
19
import configparser
20
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
21
import fnmatch
di68kap's avatar
di68kap committed
22
import inspect
23
24
import json
import os
25

26
from DHParser.toolkit import is_logging, clear_logs, re
27
from DHParser.syntaxtree import mock_syntax_tree, flatten_sxpr
28
from DHParser.error import is_error, remap_error_locations
29

30
__all__ = ('unit_from_configfile',
31
32
33
34
35
36
37
           'unit_from_json',
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
           'runner')

38
UNIT_STAGES = {'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'}
39
40


41
def unit_from_configfile(config_filename):
42
43
    """
    Reads a grammar unit test from a config file.
44
    """
45
    cfg = configparser.ConfigParser(interpolation=None)
di68kap's avatar
di68kap committed
46
    cfg.read(config_filename, encoding="utf8")
47
48
    OD = collections.OrderedDict
    unit = OD()
49
50
    for section in cfg.sections():
        symbol, stage = section.split(':')
51
52
        if stage not in UNIT_STAGES:
            if symbol in UNIT_STAGES:
53
54
                symbol, stage = stage, symbol
            else:
55
                raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
56
        for testkey, testcode in cfg[section].items():
57
58
            if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
                testcode = testcode[3:-3]
di68kap's avatar
di68kap committed
59
60
                # testcode = testcode.replace('\\#', '#')
                testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
61
62
            elif testcode[:1] + testcode[-1:] in {"''", '""'}:
                testcode = testcode[1:-1]
63
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
64
65
    # print(json.dumps(unit, sort_keys=True, indent=4))
    return unit
66

67

68
def unit_from_json(json_filename):
69
70
    """
    Reads a grammar unit test from a json file.
71
    """
di68kap's avatar
di68kap committed
72
    with open(json_filename, 'r', encoding='utf8') as f:
73
74
75
76
        unit = json.load(f)
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
77
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
78
79
    return unit

80
# TODO: add support for yaml, cson, toml
81
82


83
def unit_from_file(filename):
84
85
    """
    Reads a grammar unit test from a file. The format of the file is
86
87
    determined by the ending of its name.
    """
88
    if filename.endswith(".json"):
di68kap's avatar
di68kap committed
89
        test_unit = unit_from_json(filename)
90
    elif filename.endswith(".ini"):
di68kap's avatar
di68kap committed
91
        test_unit = unit_from_configfile(filename)
92
    else:
93
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
94

di68kap's avatar
di68kap committed
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
        intersection = list(m_names & f_names);  intersection.sort()
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

110

111
def get_report(test_unit):
112
113
    """
    Returns a text-report of the results of a grammar unit test.
114
    """
115
116
117
118
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
119
120
121
122
123
124
125
126
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
        for test_name, test_code in tests.get('match', dict()).items():
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
127
            report.append(indent(test_code))
128
129
130
131
132
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
133
134
135
            cst = tests.get('__cst__', {}).get(test_name, None)
            if cst and (not ast or cst == ast):
                report.append('\n### CST')
136
                report.append(indent(cst.as_sxpr()))
137
            elif ast:
138
                report.append('\n### AST')
139
                report.append(indent(ast.as_sxpr()))
di68kap's avatar
di68kap committed
140
141
142
143
144
145
146
147
148
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
149
150
151
    return '\n'.join(report)


152
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
153
154
    """
    Unit tests for a grammar-parser and ast transformations.
155
    """
156
    if isinstance(test_unit, str):
157
        unit_dir, unit_name = os.path.split(os.path.splitext(test_unit)[0])
158
        test_unit = unit_from_file(test_unit)
159
    else:
160
        unit_dir = ""
161
        unit_name = str(id(test_unit))
162
163
    if verbose:
        print("\nUnit: " + unit_name)
164
165
166
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
167
168
    for parser_name, tests in test_unit.items():
        assert set(tests.keys()).issubset(UNIT_STAGES)
169
170
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
171
        for test_name, test_code in tests.get('match', dict()).items():
172
173
174
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
175
            cst = parser(test_code, parser_name)
Eckhart Arnold's avatar
Eckhart Arnold committed
176
            cst.log("match_%s_%s.cst" % (parser_name, test_name))
177
            tests.setdefault('__cst__', {})[test_name] = cst
178
            if "ast" in tests or report:
179
180
181
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
Eckhart Arnold's avatar
Eckhart Arnold committed
182
                ast.log("match_%s_%s.ast" % (parser_name, test_name))
183
            if is_error(cst.error_flag):
184
                errors = remap_error_locations(cst.collect_errors(), test_code)
Eckhart Arnold's avatar
Eckhart Arnold committed
185
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
186
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
187
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
188
                tests.setdefault('__err__', {})[test_name] = errata[-1]
189
                # write parsing-history log only in case of failure!
190
191
                if is_logging():
                    parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
192
            elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
eckhart's avatar
eckhart committed
193
194
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
195
196
197
198
199
200
            elif "ast" in tests:
                compare = mock_syntax_tree(tests["ast"][test_name])
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
201
202
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
203
                    tests.setdefault('__err__', {})[test_name] = errata[-1]
204
            if verbose:
205
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
206

207
        if verbose and 'fail' in tests:
208
            print('  Fail-Tests for parser "' + parser_name + '"')
209
        for test_name, test_code in tests.get('fail', dict()).items():
210
211
212
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
213
            cst = parser(test_code, parser_name)
214
            if not is_error(cst.error_flag):
215
216
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
217
                tests.setdefault('__err__', {})[test_name] = errata[-1]
218
                # write parsing-history log only in case of test-failure
219
220
                if is_logging():
                    parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
221
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
222
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
223

224
225
    # write test-report
    if report:
226
        report_dir = "REPORT"
227
228
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
229
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
230
            f.write(get_report(test_unit))
231

232
233
234
    return errata


Eckhart Arnold's avatar
Eckhart Arnold committed
235
236
237
238
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
239
240
    """
    Runs all grammar unit tests in a directory. A file is considered a test
241
242
    unit, if it has the word "test" in its name.
    """
243
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
244
        fn_patterns = [fn_patterns]
245
    all_errors = collections.OrderedDict()
246
247
    if verbose:
        print("\nScanning test-directory: " + directory)
248
249
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
250
251
    if is_logging():
        clear_logs()
252
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
253
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
254
            try:
255
256
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
257
258
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
259
260
261
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
262
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
263
                    raise e
264
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
265
266
    error_report = []
    err_N = 0
267
268
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
269
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
270
            err_N += len(all_errors[filename])
271
272
273
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
di68kap's avatar
di68kap committed
274
275
276
277
        # if verbose:
        #     print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
        return ('Test suite "%s" revealed %s error%s:\n\n'
                % (directory, err_N, 's' if err_N > 1 else '') + '\n'.join(error_report))
eckhart's avatar
eckhart committed
278
279
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
280
281
282
    return ''


283
def runner(tests, namespace):
284
285
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
286
    namespace. To run all tests in a module, call
287
    ``runner("", globals())`` from within that module.
288
289
290
291
292
293

    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
294
        namespace: The namespace for running the test, usually
295
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
296

297
298
299
300
301
302
303
304
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
305

306
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
307
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
308
            runner("", globals())
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

    if tests:
        if isinstance(tests, str):
            tests = tests.split(" ")
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
        tests = []
        for name in namespace.keys():
            if name.lower().startswith('test') and inspect.isclass(namespace[name]):
                tests.append(name)

    obj = None
    for test in tests:
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()