testing.py 14 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
"""testing.py - test support for DHParser based grammars and compilers

Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                Bavarian Academy of Sciences an Humanities (badw.de)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.
"""
18
import collections
19
import configparser
20
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
21
import fnmatch
di68kap's avatar
di68kap committed
22
import inspect
23
24
import json
import os
25

26
from DHParser.toolkit import is_logging, clear_logs, re
27
28
from DHParser.syntaxtree import Node, mock_syntax_tree, flatten_sxpr, ZOMBIE_PARSER
from DHParser.parse import UnknownParserError
eckhart's avatar
eckhart committed
29
from DHParser.error import is_error, adjust_error_locations
30

31
__all__ = ('unit_from_configfile',
32
33
34
35
36
37
38
           'unit_from_json',
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
           'runner')

39
UNIT_STAGES = {'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'}
40
41


42
def unit_from_configfile(config_filename):
43
44
    """
    Reads a grammar unit test from a config file.
45
    """
46
    cfg = configparser.ConfigParser(interpolation=None)
di68kap's avatar
di68kap committed
47
    cfg.read(config_filename, encoding="utf8")
48
49
    OD = collections.OrderedDict
    unit = OD()
50
51
    for section in cfg.sections():
        symbol, stage = section.split(':')
52
53
        if stage not in UNIT_STAGES:
            if symbol in UNIT_STAGES:
54
55
                symbol, stage = stage, symbol
            else:
56
                raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
57
        for testkey, testcode in cfg[section].items():
58
59
            if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
                testcode = testcode[3:-3]
di68kap's avatar
di68kap committed
60
61
                # testcode = testcode.replace('\\#', '#')
                testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
62
63
            elif testcode[:1] + testcode[-1:] in {"''", '""'}:
                testcode = testcode[1:-1]
64
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
65
66
    # print(json.dumps(unit, sort_keys=True, indent=4))
    return unit
67

68

69
def unit_from_json(json_filename):
70
71
    """
    Reads a grammar unit test from a json file.
72
    """
di68kap's avatar
di68kap committed
73
    with open(json_filename, 'r', encoding='utf8') as f:
74
75
76
77
        unit = json.load(f)
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
78
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
79
80
    return unit

81
# TODO: add support for yaml, cson, toml
82
83


84
def unit_from_file(filename):
85
86
    """
    Reads a grammar unit test from a file. The format of the file is
87
88
    determined by the ending of its name.
    """
89
    if filename.endswith(".json"):
di68kap's avatar
di68kap committed
90
        test_unit = unit_from_json(filename)
91
    elif filename.endswith(".ini"):
di68kap's avatar
di68kap committed
92
        test_unit = unit_from_configfile(filename)
93
    else:
94
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
95

di68kap's avatar
di68kap committed
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
        intersection = list(m_names & f_names);  intersection.sort()
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

111

112
def get_report(test_unit):
113
114
    """
    Returns a text-report of the results of a grammar unit test.
115
    """
116
117
118
119
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
120
121
122
123
124
125
126
127
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
        for test_name, test_code in tests.get('match', dict()).items():
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
128
            report.append(indent(test_code))
129
130
131
132
133
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
134
135
136
            cst = tests.get('__cst__', {}).get(test_name, None)
            if cst and (not ast or cst == ast):
                report.append('\n### CST')
137
                report.append(indent(cst.as_sxpr()))
138
            elif ast:
139
                report.append('\n### AST')
140
                report.append(indent(ast.as_sxpr()))
di68kap's avatar
di68kap committed
141
142
143
144
145
146
147
148
149
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
150
151
152
    return '\n'.join(report)


153
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
154
155
    """
    Unit tests for a grammar-parser and ast transformations.
156
    """
157
    if isinstance(test_unit, str):
158
        unit_dir, unit_name = os.path.split(os.path.splitext(test_unit)[0])
159
        test_unit = unit_from_file(test_unit)
160
    else:
161
        unit_dir = ""
162
        unit_name = str(id(test_unit))
163
164
    if verbose:
        print("\nUnit: " + unit_name)
165
166
167
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
168
169
    for parser_name, tests in test_unit.items():
        assert set(tests.keys()).issubset(UNIT_STAGES)
170
171
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
172
        for test_name, test_code in tests.get('match', dict()).items():
173
174
175
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
176
177
178
179
            try:
                cst = parser(test_code, parser_name)
            except UnknownParserError as upe:
                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
Eckhart Arnold's avatar
Eckhart Arnold committed
180
            cst.log("match_%s_%s.cst" % (parser_name, test_name))
181
            tests.setdefault('__cst__', {})[test_name] = cst
182
            if "ast" in tests or report:
183
184
185
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
Eckhart Arnold's avatar
Eckhart Arnold committed
186
                ast.log("match_%s_%s.ast" % (parser_name, test_name))
187
            if is_error(cst.error_flag):
eckhart's avatar
eckhart committed
188
                errors = adjust_error_locations(cst.collect_errors(), test_code)
Eckhart Arnold's avatar
Eckhart Arnold committed
189
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
190
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
191
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
192
                tests.setdefault('__err__', {})[test_name] = errata[-1]
193
                # write parsing-history log only in case of failure!
194
195
                if is_logging():
                    parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
196
            elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
eckhart's avatar
eckhart committed
197
198
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
199
200
201
202
203
204
            elif "ast" in tests:
                compare = mock_syntax_tree(tests["ast"][test_name])
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
205
206
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
207
                    tests.setdefault('__err__', {})[test_name] = errata[-1]
208
            if verbose:
209
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
210

211
        if verbose and 'fail' in tests:
212
            print('  Fail-Tests for parser "' + parser_name + '"')
213
        for test_name, test_code in tests.get('fail', dict()).items():
214
215
216
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
217
218
219
220
221
            # cst = parser(test_code, parser_name)
            try:
                cst = parser(test_code, parser_name)
            except UnknownParserError as upe:
                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
222
            if not is_error(cst.error_flag):
223
224
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
225
                tests.setdefault('__err__', {})[test_name] = errata[-1]
226
                # write parsing-history log only in case of test-failure
227
228
                if is_logging():
                    parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
229
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
230
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
231

232
233
    # write test-report
    if report:
234
        report_dir = "REPORT"
235
236
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
237
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
238
            f.write(get_report(test_unit))
239

240
241
242
    return errata


Eckhart Arnold's avatar
Eckhart Arnold committed
243
244
245
246
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
247
248
    """
    Runs all grammar unit tests in a directory. A file is considered a test
249
250
    unit, if it has the word "test" in its name.
    """
251
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
252
        fn_patterns = [fn_patterns]
253
    all_errors = collections.OrderedDict()
254
255
    if verbose:
        print("\nScanning test-directory: " + directory)
256
257
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
258
259
    if is_logging():
        clear_logs()
260
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
261
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
262
            try:
263
264
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
265
266
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
267
268
269
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
270
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
271
                    raise e
272
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
273
274
    error_report = []
    err_N = 0
275
276
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
277
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
278
            err_N += len(all_errors[filename])
279
280
281
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
di68kap's avatar
di68kap committed
282
283
284
285
        # if verbose:
        #     print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
        return ('Test suite "%s" revealed %s error%s:\n\n'
                % (directory, err_N, 's' if err_N > 1 else '') + '\n'.join(error_report))
eckhart's avatar
eckhart committed
286
287
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
288
289
290
    return ''


291
def runner(tests, namespace):
292
293
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
294
    namespace. To run all tests in a module, call
295
    ``runner("", globals())`` from within that module.
296
297
298
299
300
301

    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
302
        namespace: The namespace for running the test, usually
303
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
304

305
306
307
308
309
310
311
312
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
313

314
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
315
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
316
            runner("", globals())
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

    if tests:
        if isinstance(tests, str):
            tests = tests.split(" ")
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
        tests = []
        for name in namespace.keys():
            if name.lower().startswith('test') and inspect.isclass(namespace[name]):
                tests.append(name)

    obj = None
    for test in tests:
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()