testing.py 14.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
"""testing.py - test support for DHParser based grammars and compilers

Copyright 2016  by Eckhart Arnold (arnold@badw.de)
                Bavarian Academy of Sciences an Humanities (badw.de)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.
"""
18
import collections
19
import configparser
20
import copy
Eckhart Arnold's avatar
Eckhart Arnold committed
21
import fnmatch
di68kap's avatar
di68kap committed
22
import inspect
23
24
import json
import os
25

26
27
from DHParser.toolkit import re
from DHParser.log import is_logging, clear_logs, log_ST, log_parsing_history
28
29
from DHParser.syntaxtree import Node, mock_syntax_tree, flatten_sxpr, ZOMBIE_PARSER
from DHParser.parse import UnknownParserError
eckhart's avatar
eckhart committed
30
from DHParser.error import is_error, adjust_error_locations
31

32
__all__ = ('unit_from_configfile',
33
34
35
36
37
38
39
           'unit_from_json',
           'unit_from_file',
           'get_report',
           'grammar_unit',
           'grammar_suite',
           'runner')

40
UNIT_STAGES = {'match', 'fail', 'ast', 'cst', '__ast__', '__cst__'}
41
42


43
def unit_from_configfile(config_filename):
44
45
    """
    Reads a grammar unit test from a config file.
46
    """
47
    cfg = configparser.ConfigParser(interpolation=None)
di68kap's avatar
di68kap committed
48
    cfg.read(config_filename, encoding="utf8")
49
50
    OD = collections.OrderedDict
    unit = OD()
51
52
    for section in cfg.sections():
        symbol, stage = section.split(':')
53
54
        if stage not in UNIT_STAGES:
            if symbol in UNIT_STAGES:
55
56
                symbol, stage = stage, symbol
            else:
57
                raise ValueError('Test stage %s not in: ' % (stage, str(UNIT_STAGES)))
58
        for testkey, testcode in cfg[section].items():
59
60
            if testcode[:3] + testcode[-3:] in {"''''''", '""""""'}:
                testcode = testcode[3:-3]
di68kap's avatar
di68kap committed
61
62
                # testcode = testcode.replace('\\#', '#')
                testcode = re.sub(r'(?<!\\)\\#', '#', testcode).replace('\\\\', '\\')
63
64
            elif testcode[:1] + testcode[-1:] in {"''", '""'}:
                testcode = testcode[1:-1]
65
            unit.setdefault(symbol, OD()).setdefault(stage, OD())[testkey] = testcode
66
67
    # print(json.dumps(unit, sort_keys=True, indent=4))
    return unit
68

69

70
def unit_from_json(json_filename):
71
72
    """
    Reads a grammar unit test from a json file.
73
    """
di68kap's avatar
di68kap committed
74
    with open(json_filename, 'r', encoding='utf8') as f:
75
76
77
78
        unit = json.load(f)
    for symbol in unit:
        for stage in unit[symbol]:
            if stage not in UNIT_STAGES:
79
                raise ValueError('Test stage %s not in: %s' % (stage, str(UNIT_STAGES)))
80
81
    return unit

82
# TODO: add support for yaml, cson, toml
83
84


85
def unit_from_file(filename):
86
87
    """
    Reads a grammar unit test from a file. The format of the file is
88
89
    determined by the ending of its name.
    """
90
    if filename.endswith(".json"):
di68kap's avatar
di68kap committed
91
        test_unit = unit_from_json(filename)
92
    elif filename.endswith(".ini"):
di68kap's avatar
di68kap committed
93
        test_unit = unit_from_configfile(filename)
94
    else:
95
        raise ValueError("Unknown unit test file type: " + filename[filename.rfind('.'):])
96

di68kap's avatar
di68kap committed
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
    # Check for ambiguous Test names
    errors = []
    for parser_name, tests in test_unit.items():
        m_names = set(tests.get('match', dict()).keys())
        f_names = set(tests.get('fail', dict()).keys())
        intersection = list(m_names & f_names);  intersection.sort()
        if intersection:
            errors.append("Same names %s assigned to match and fail test "
                          "of parser %s." % (str(intersection), parser_name))
    if errors:
        raise EnvironmentError("Error(s) in Testfile %s :\n" % filename
                               + '\n'.join(errors))

    return test_unit

112

113
def get_report(test_unit):
114
115
    """
    Returns a text-report of the results of a grammar unit test.
116
    """
117
118
119
120
    def indent(txt):
        lines = txt.split('\n')
        lines[0] = '    ' + lines[0]
        return "\n    ".join(lines)
121
122
123
124
125
126
127
128
    report = []
    for parser_name, tests in test_unit.items():
        heading = 'Test of parser: "%s"' % parser_name
        report.append('\n\n%s\n%s\n' % (heading, '=' * len(heading)))
        for test_name, test_code in tests.get('match', dict()).items():
            heading = 'Match-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
129
            report.append(indent(test_code))
130
131
132
133
134
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
            ast = tests.get('__ast__', {}).get(test_name, None)
135
136
137
            cst = tests.get('__cst__', {}).get(test_name, None)
            if cst and (not ast or cst == ast):
                report.append('\n### CST')
138
                report.append(indent(cst.as_sxpr()))
139
            elif ast:
140
                report.append('\n### AST')
141
                report.append(indent(ast.as_sxpr()))
di68kap's avatar
di68kap committed
142
143
144
145
146
147
148
149
150
        for test_name, test_code in tests.get('fail', dict()).items():
            heading = 'Fail-test "%s"' % test_name
            report.append('\n%s\n%s\n' % (heading, '-' * len(heading)))
            report.append('### Test-code:')
            report.append(indent(test_code))
            error = tests.get('__err__', {}).get(test_name, "")
            if error:
                report.append('\n### Error:')
                report.append(error)
151
152
153
    return '\n'.join(report)


154
def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, verbose=False):
155
156
    """
    Unit tests for a grammar-parser and ast transformations.
157
    """
158
    if isinstance(test_unit, str):
159
        unit_dir, unit_name = os.path.split(os.path.splitext(test_unit)[0])
160
        test_unit = unit_from_file(test_unit)
161
    else:
162
        unit_dir = ""
163
        unit_name = str(id(test_unit))
164
165
    if verbose:
        print("\nUnit: " + unit_name)
166
167
168
    errata = []
    parser = parser_factory()
    transform = transformer_factory()
169
170
    for parser_name, tests in test_unit.items():
        assert set(tests.keys()).issubset(UNIT_STAGES)
171
172
        if verbose:
            print('  Match-Tests for parser "' + parser_name + '"')
173
        for test_name, test_code in tests.get('match', dict()).items():
174
175
176
            if verbose:
                infostr = '    match-test "' + test_name + '" ... '
                errflag = len(errata)
177
178
179
180
            try:
                cst = parser(test_code, parser_name)
            except UnknownParserError as upe:
                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
181
            log_ST(cst, "match_%s_%s.cst" % (parser_name, test_name))
182
            tests.setdefault('__cst__', {})[test_name] = cst
183
            if "ast" in tests or report:
184
185
186
                ast = copy.deepcopy(cst)
                transform(ast)
                tests.setdefault('__ast__', {})[test_name] = ast
187
                log_ST(ast, "match_%s_%s.ast" % (parser_name, test_name))
188
            if is_error(cst.error_flag):
eckhart's avatar
eckhart committed
189
                errors = adjust_error_locations(cst.collect_errors(), test_code)
Eckhart Arnold's avatar
Eckhart Arnold committed
190
                errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s\n\n' %
191
                              (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
192
                               '\n\t'.join(str(m).replace('\n', '\n\t\t') for m in errors)))
193
                tests.setdefault('__err__', {})[test_name] = errata[-1]
194
                # write parsing-history log only in case of failure!
195
                if is_logging():
196
                    log_parsing_history(parser, "match_%s_%s.log" % (parser_name, test_name))
197
            elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
eckhart's avatar
eckhart committed
198
199
                errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                              (test_name, parser_name, cst.as_sxpr()))
200
201
202
203
204
205
            elif "ast" in tests:
                compare = mock_syntax_tree(tests["ast"][test_name])
                if compare != ast:
                    errata.append('Abstract syntax tree test "%s" for parser "%s" failed:'
                                  '\n\tExpr.:     %s\n\tExpected:  %s\n\tReceived:  %s'
                                  % (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
Eckhart Arnold's avatar
Eckhart Arnold committed
206
207
                                     flatten_sxpr(compare.as_sxpr()),
                                     flatten_sxpr(ast.as_sxpr())))
208
                    tests.setdefault('__err__', {})[test_name] = errata[-1]
209
            if verbose:
210
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
211

212
        if verbose and 'fail' in tests:
213
            print('  Fail-Tests for parser "' + parser_name + '"')
214
        for test_name, test_code in tests.get('fail', dict()).items():
215
216
217
            if verbose:
                infostr = '    fail-test  "' + test_name + '" ... '
                errflag = len(errata)
218
219
220
221
222
            # cst = parser(test_code, parser_name)
            try:
                cst = parser(test_code, parser_name)
            except UnknownParserError as upe:
                cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
223
            if not is_error(cst.error_flag):
224
225
                errata.append('Fail test "%s" for parser "%s" yields match instead of '
                              'expected failure!' % (test_name, parser_name))
226
                tests.setdefault('__err__', {})[test_name] = errata[-1]
227
                # write parsing-history log only in case of test-failure
228
                if is_logging():
229
                    log_parsing_history(parser, "fail_%s_%s.log" % (parser_name, test_name))
230
            if verbose:
Eckhart Arnold's avatar
Eckhart Arnold committed
231
                print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
232

233
234
    # write test-report
    if report:
235
        report_dir = "REPORT"
236
237
        if not os.path.exists(report_dir):
            os.mkdir(report_dir)
di68kap's avatar
di68kap committed
238
        with open(os.path.join(report_dir, unit_name + '.md'), 'w', encoding='utf8') as f:
239
            f.write(get_report(test_unit))
240

241
242
243
    return errata


Eckhart Arnold's avatar
Eckhart Arnold committed
244
245
246
247
def grammar_suite(directory, parser_factory, transformer_factory,
                  fn_patterns=['*test*'],
                  ignore_unknown_filetypes=False,
                  report=True, verbose=True):
248
249
    """
    Runs all grammar unit tests in a directory. A file is considered a test
250
251
    unit, if it has the word "test" in its name.
    """
252
    if not isinstance(fn_patterns, collections.abc.Iterable):
Eckhart Arnold's avatar
Eckhart Arnold committed
253
        fn_patterns = [fn_patterns]
254
    all_errors = collections.OrderedDict()
255
256
    if verbose:
        print("\nScanning test-directory: " + directory)
257
258
    save_cwd = os.getcwd()
    os.chdir(directory)
eckhart's avatar
eckhart committed
259
260
    if is_logging():
        clear_logs()
261
    for filename in sorted(os.listdir()):
Eckhart Arnold's avatar
Eckhart Arnold committed
262
        if any(fnmatch.fnmatch(filename, pattern) for pattern in fn_patterns):
263
            try:
264
265
                if verbose:
                    print("\nRunning grammar tests from: " + filename)
266
267
                errata = grammar_unit(filename, parser_factory,
                                      transformer_factory, report, verbose)
268
269
270
                if errata:
                    all_errors[filename] = errata
            except ValueError as e:
271
                if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
272
                    raise e
273
    os.chdir(save_cwd)
eckhart's avatar
eckhart committed
274
275
    error_report = []
    err_N = 0
276
277
    if all_errors:
        for filename in all_errors:
di68kap's avatar
di68kap committed
278
            error_report.append('Errors found by unit test "%s":\n' % filename)
di68kap's avatar
di68kap committed
279
            err_N += len(all_errors[filename])
280
281
282
            for error in all_errors[filename]:
                error_report.append('\t' + '\n\t'.join(error.split('\n')))
    if error_report:
di68kap's avatar
di68kap committed
283
284
285
286
        # if verbose:
        #     print("\nFAILURE! %i error%s found!\n" % (err_N, 's' if err_N > 1 else ''))
        return ('Test suite "%s" revealed %s error%s:\n\n'
                % (directory, err_N, 's' if err_N > 1 else '') + '\n'.join(error_report))
eckhart's avatar
eckhart committed
287
288
    if verbose:
        print("\nSUCCESS! All tests passed :-)\n")
289
290
291
    return ''


292
def runner(tests, namespace):
293
294
    """
    Runs all or some selected Python unit tests found in the
eckhart's avatar
eckhart committed
295
    namespace. To run all tests in a module, call
296
    ``runner("", globals())`` from within that module.
297
298
299
300
301
302

    Args:
        tests: Either a string or a list of strings that contains the
            names of test or test classes. Each test and, in the case
            of a test class, all tests within the test class will be
            run.
eckhart's avatar
eckhart committed
303
        namespace: The namespace for running the test, usually
304
            ``globals()`` should be used.
eckhart's avatar
eckhart committed
305

306
307
308
309
310
311
312
313
    Example:
        class TestSomething()
            def setup(self):
                pass
            def teardown(self):
                pass
            def test_something(self):
                pass
eckhart's avatar
eckhart committed
314

315
        if __name__ == "__main__":
di68kap's avatar
di68kap committed
316
            from DHParser.testing import runner
eckhart's avatar
eckhart committed
317
            runner("", globals())
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
    """
    def instantiate(cls_name):
        exec("obj = " + cls_name + "()", namespace)
        obj = namespace["obj"]
        if "setup" in dir(obj):
            obj.setup()
        return obj

    if tests:
        if isinstance(tests, str):
            tests = tests.split(" ")
    else:
        # collect all test classes, in case no methods or classes have been passed explicitly
        tests = []
        for name in namespace.keys():
            if name.lower().startswith('test') and inspect.isclass(namespace[name]):
                tests.append(name)

    obj = None
    for test in tests:
        try:
            if test.find('.') >= 0:
                cls_name, method_name = test.split('.')
                obj = instantiate(cls_name)
                print("Running " + cls_name + "." + method_name)
                exec('obj.' + method_name + '()')
            else:
                obj = instantiate(test)
                for name in dir(obj):
                    if name.lower().startswith("test"):
                        print("Running " + test + "." + name)
                        exec('obj.' + name + '()')
        finally:
            if "teardown" in dir(obj):
                obj.teardown()