testing.py 13.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
"""testing.py - test support for DHParser based grammars and compilers

Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                Bavarian Academy of Sciences an Humanities (badw.de)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.
"""
18
import collections
19
import configparser
20
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
21
import fnmatch
di68kap's avatar
di68kap committed
22
import inspect
23
24
import json
import os
25

26
from DHParser.toolkit import is_logging, clear_logs, re
27
from DHParser.syntaxtree import mock_syntax_tree, flatten_sxpr
28
from DHParser.error import is_error
29

30
__all__ = ('unit_from_configfile',
31
32
33
34
35
36
37
           'unit_from_json',
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
           'runner')

38
UNIT_STAGES = {'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'}
39
40


41
def unit_from_configfile(config_filename):
42
43
    """
    Reads a grammar unit test from a config file.
44
    """
45
    cfg = configparser.ConfigParser(interpolation=None)
di68kap's avatar
di68kap committed
46
    cfg.read(config_filename, encoding="utf8")
47
48
    OD = collections.OrderedDict
    unit = OD()
49
50
    for section in cfg.sections():
        symbol, stage = section.split(':')
51
52
        if stage not in UNIT_STAGES:
            if symbol in UNIT_STAGES:
53
54
                symbol, stage = stage, symbol
            else:
55
                raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
56
        for testkey, testcode in cfg[section].items():
57
58
            if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
                testcode = testcode[3:-3]
di68kap's avatar
di68kap committed
59
60
                # testcode = testcode.replace('\\#', '#')
                testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
61
62
            elif testcode[:1] + testcode[-1:] in {"''", '""'}:
                testcode = testcode[1:-1]
63
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
64
65
    # print(json.dumps(unit, sort_keys=True, indent=4))
    return unit
66

67

68
def unit_from_json(json_filename):
69
70
    """
    Reads a grammar unit test from a json file.
71
    """
di68kap's avatar
di68kap committed
72
    with open(json_filename, 'r', encoding='utf8') as f:
73
74
75
76
        unit = json.load(f)
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
77
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
78
79
    return unit

80
# TODO: add support for yaml, cson, toml
81
82


83
def unit_from_file(filename):
84
85
    """
    Reads a grammar unit test from a file. The format of the file is
86
87
    determined by the ending of its name.
    """
88
    if filename.endswith(".json"):
di68kap's avatar
di68kap committed
89
        test_unit = unit_from_json(filename)
90
    elif filename.endswith(".ini"):
di68kap's avatar
di68kap committed
91
        test_unit = unit_from_configfile(filename)
92
    else:
93
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
94

di68kap's avatar
di68kap committed
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
        intersection = list(m_names & f_names);  intersection.sort()
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

110

111
def get_report(test_unit):
112
113
    """
    Returns a text-report of the results of a grammar unit test.
114
    """
115
116
117
118
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
119
120
121
122
123
124
125
126
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
        for test_name, test_code in tests.get('match', dict()).items():
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
127
            report.append(indent(test_code))
128
129
130
131
132
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
133
134
135
            cst = tests.get('__cst__', {}).get(test_name, None)
            if cst and (not ast or cst == ast):
                report.append('\n### CST')
136
                report.append(indent(cst.as_sxpr()))
137
            elif ast:
138
                report.append('\n### AST')
139
                report.append(indent(ast.as_sxpr()))
di68kap's avatar
di68kap committed
140
141
142
143
144
145
146
147
148
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
149
150
151
    return '\n'.join(report)


152
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
153
154
    """
    Unit tests for a grammar-parser and ast transformations.
155
    """
156
    if isinstance(test_unit, str):
157
        unit_dir, unit_name = os.path.split(os.path.splitext(test_unit)[0])
158
        test_unit = unit_from_file(test_unit)
159
    else:
160
        unit_dir = ""
161
        unit_name = str(id(test_unit))
162
163
    if verbose:
        print("\nUnit: " + unit_name)
164
165
166
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
167
168
    for parser_name, tests in test_unit.items():
        assert set(tests.keys()).issubset(UNIT_STAGES)
169
170
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
171
        for test_name, test_code in tests.get('match', dict()).items():
172
173
174
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
175
            cst = parser(test_code, parser_name)
Eckhart Arnold's avatar
Eckhart Arnold committed
176
            cst.log("match_%s_%s.cst" % (parser_name, test_name))
177
            tests.setdefault('__cst__', {})[test_name] = cst
178
            if "ast" in tests or report:
179
180
181
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
Eckhart Arnold's avatar
Eckhart Arnold committed
182
                ast.log("match_%s_%s.ast" % (parser_name, test_name))
183
            if is_error(cst.error_flag):
Eckhart Arnold's avatar
Eckhart Arnold committed
184
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
185
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
186
187
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in
                                           cst.collect_errors(test_code))))
188
                tests.setdefault('__err__', {})[test_name] = errata[-1]
189
190
                # write parsing-history log only in case of failure!
                parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
191
            elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
eckhart's avatar
eckhart committed
192
193
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
194
195
196
197
198
199
            elif "ast" in tests:
                compare = mock_syntax_tree(tests["ast"][test_name])
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
200
201
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
202
                    tests.setdefault('__err__', {})[test_name] = errata[-1]
203
            if verbose:
204
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
205

206
        if verbose and 'fail' in tests:
207
            print('  Fail-Tests for parser "' + parser_name + '"')
208
        for test_name, test_code in tests.get('fail', dict()).items():
209
210
211
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
212
            cst = parser(test_code, parser_name)
213
            if not is_error(cst.error_flag):
214
215
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
216
                tests.setdefault('__err__', {})[test_name] = errata[-1]
217
218
                # write parsing-history log only in case of test-failure
                parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
219
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
220
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
221

222
223
    # write test-report
    if report:
224
        report_dir = "REPORT"
225
226
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
227
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
228
            f.write(get_report(test_unit))
229

230
231
232
    return errata


Eckhart Arnold's avatar
Eckhart Arnold committed
233
234
235
236
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
237
238
    """
    Runs all grammar unit tests in a directory. A file is considered a test
239
240
    unit, if it has the word "test" in its name.
    """
241
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
242
        fn_patterns = [fn_patterns]
243
    all_errors = collections.OrderedDict()
244
245
    if verbose:
        print("\nScanning test-directory: " + directory)
246
247
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
248
249
    if is_logging():
        clear_logs()
250
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
251
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
252
            try:
253
254
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
255
256
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
257
258
259
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
260
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
261
                    raise e
262
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
263
264
    error_report = []
    err_N = 0
265
266
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
267
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
268
            err_N += len(all_errors[filename])
269
270
271
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
eckhart's avatar
eckhart committed
272
273
        if verbose:
            print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
274
        return ('Test suite "%s" revealed some errors:\n\n' % directory) + '\n'.join(error_report)
eckhart's avatar
eckhart committed
275
276
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
277
278
279
    return ''


280
def runner(tests, namespace):
281
282
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
283
    namespace. To run all tests in a module, call
284
    ``runner("", globals())`` from within that module.
285
286
287
288
289
290

    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
291
        namespace: The namespace for running the test, usually
292
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
293

294
295
296
297
298
299
300
301
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
302

303
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
304
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
305
            runner("", globals())
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

    if tests:
        if isinstance(tests, str):
            tests = tests.split(" ")
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
        tests = []
        for name in namespace.keys():
            if name.lower().startswith('test') and inspect.isclass(namespace[name]):
                tests.append(name)

    obj = None
    for test in tests:
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()