test_transform.py 11.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#!/usr/bin/python3

"""test_transform.py - test of tramsform-module of DHParser

Author: Eckhart Arnold <arnold@badw.de>

Copyright 2018 Bavarian Academy of Sciences and Humanities

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

22
import collections.abc
23
24
25
26
import sys

sys.path.extend(['../', './'])

27
from DHParser.syntaxtree import Node, parse_sxpr, flatten_sxpr, parse_xml, PLACEHOLDER, \
28
29
    tree_sanity_check, TOKEN_PTYPE
from DHParser.transform import traverse, reduce_single_child, remove_whitespace, move_adjacent, \
30
    traverse_locally, collapse, collapse_if, lstrip, rstrip, remove_content, remove_tokens, \
31
    transformation_factory, has_parent, contains_only_whitespace, is_insignificant_whitespace
32
33
from DHParser.toolkit import typing
from typing import AbstractSet, List, Sequence, Tuple
34
35
36
37
38


class TestRemoval:
    """Tests removing transformations."""

39
40
41
42
43
    def test_contains_only_whitespace(self):
        assert contains_only_whitespace([Node('test', ' ')])
        assert contains_only_whitespace([Node('test', '')])
        assert contains_only_whitespace([Node('test', '\n')])
        assert not contains_only_whitespace([Node('test', 'Katze')])
44
        assert not contains_only_whitespace([Node('test', ' tag ')])
45

46
    def test_lstrip(self):
47
        cst = parse_sxpr('(_Token (:Whitespace " ") (:Re test))')
48
49
50
51
52
        lstrip([cst])
        assert cst.as_sxpr().find(":Whitespace") < 0
        sxpr1 = cst.as_sxpr()
        lstrip([cst])
        assert sxpr1 == cst.as_sxpr()
53
        cst = parse_sxpr('(_Token)')
54
        lstrip([cst])
55
56
        assert cst.as_sxpr() == '(_Token)'
        cst = parse_sxpr('(_Token (:Whitespace " ") (:Whitespace " ") (:Re test))')
57
58
        lstrip([cst])
        assert cst.as_sxpr().find(":Whitespace") < 0
59
        cst = parse_sxpr('(_Token (:Whitespace " ") (Deeper (:Whitespace " ")) '
60
61
62
                               '(:Whitespace " ") (:Re test))')
        lstrip([cst])
        assert cst.as_sxpr().find(":Whitespace") < 0
63
        cst = parse_sxpr('(_Token (:Re ein) (:Whitespace " ") (:Re test))')
64
65
66
67
        lstrip([cst])
        assert cst.as_sxpr().find(":Whitespace") >= 0

    def test_rstrip(self):
68
        cst = parse_sxpr('(_Token (:Re test) (:Whitespace " "))')
69
70
71
72
73
        rstrip([cst])
        assert cst.as_sxpr().find(":Whitespace") < 0, cst.as_sxpr()
        sxpr1 = cst.as_sxpr()
        rstrip([cst])
        assert sxpr1 == cst.as_sxpr()
74
        cst = parse_sxpr('(_Token)')
75
        rstrip([cst])
76
77
        assert cst.as_sxpr() == '(_Token)'
        cst = parse_sxpr('(_Token  (:Re test) (:Whitespace " ") (:Whitespace " "))')
78
79
        rstrip([cst])
        assert cst.as_sxpr().find(":Whitespace") < 0
80
        cst = parse_sxpr('(_Token  (:Re test) (:Whitespace " ") (Deeper (:Whitespace " ")) '
81
82
83
84
                               '(:Whitespace " "))')
        rstrip([cst])
        assert cst.as_sxpr().find(":Whitespace") < 0, cst.as_sxpr()

85
    def test_remove_content(self):
86
        cst = parse_sxpr('(BelegLemma (:Series (:RegExp "#") (LAT_WORT (:RegExp "facitergula"))))')
87
88
89
90
91
92
        remove_content([cst], '#')
        assert cst.content == "#facitergula", str(cst.content)
        reduce_single_child([cst])
        remove_content([cst], '#')
        assert cst.content == "facitergula"

93
    def test_remove_tokens(self):
94
        cst = parse_sxpr('(wortarten (:Token "ajektiv") (:Token "et") (:Token "praeposition"))')
95
96
97
98
99
        ast_table = {
            "wortarten": [remove_tokens({"et"})],
            "*": []
        }
        traverse(cst, ast_table)
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
        cst1 = cst.as_sxpr()
        assert cst1.find('et') < 0
        ast_table = {
            "wortarten": [remove_tokens("et")],
            "*": []
        }
        traverse(cst, ast_table)
        assert cst1 == cst.as_sxpr()


class TestTransformationFactory:
    def test_mismatching_types(self):
        @transformation_factory(tuple)
        def good_transformation(context: List[Node], parameters: Tuple[str]):
            pass
        try:
            @transformation_factory(tuple)
            def bad_transformation(context: List[Node], parameters: AbstractSet[str]):
                pass
            assert False, "mismatching types not recognized by transform.transformation_factory()"
        except TypeError:
            pass

    def test_forbidden_generic_types_in_decorator(self):
        try:
            @transformation_factory(AbstractSet[str])
            def forbidden_transformation(context: List[Node], parameters: AbstractSet[str]):
                pass
            assert False, "use of generics not recognized in transform.transformation_factory()"
        except TypeError:
            pass

    def test_forbidden_mutable_sequence_types_in_decorator(self):
        try:
            @transformation_factory(collections.abc.Sequence)
            def parameterized_transformation(context: List[Node], parameters: Sequence[str]):
                pass
            _ = parameterized_transformation('a', 'b', 'c')
            assert False, ("use of mutable sequences not recognized in "
                           "transform.transformation_factory()")
        except TypeError:
            pass

    def test_parameter_set_expansion1(self):
        save = None
        @transformation_factory(collections.abc.Set)
        def parameterized_transformation(context: List[Node], parameters: AbstractSet[str]):
            nonlocal save
            save = parameters
        transformation = parameterized_transformation('a', 'b', 'c')
150
        transformation([PLACEHOLDER])
151
152
153
154
155
156
157
158
159
        assert save == {'a', 'b', 'c'}

    def test_parameter_set_expansion2(self):
        save = None
        @transformation_factory(tuple)
        def parameterized_transformation(context: List[Node], parameters: Tuple[str]):
            nonlocal save
            save = parameters
        transformation = parameterized_transformation('a', 'b', 'c')
160
        transformation([PLACEHOLDER])
161
        assert save == ('a', 'b', 'c'), str(save)
162

163
164
165
166

class TestConditionalTransformations:
    """Tests conditional transformations."""

167
    def test_has_parent(self):
168
169
170
        context = [Node('A', 'alpha'),
                   Node('B', 'beta'),
                   Node('C', 'gamma')]
171
172
173
174
        assert has_parent(context, {'A'})
        assert has_parent(context, {'B'})
        assert not has_parent(context, {'C'})

175
    def test_traverse_locally(self):
176
        cst = parse_sxpr("""
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
            (Lemma
                (LemmaVariante
                    (LAT_WORT
                        (:RegExp
                            "facitercula"
                        )
                        (:Whitespace
                            " "
                        )
                    )
                    (Zusatz
                        (DEU_WORT
                            "sim."
                        )
                    )
                )
                (Hinweis
                    (LAT_WORT
                        (:RegExp
                            "bona"
                        )
                        (:Whitespace
                            " "
                        )
                    )
                    (LAT_WORT
                        (:RegExp
                            "fide"
                        )
                    )
                )
            )""")
        LemmaVariante_transformations = {
            "LAT_WORT": [remove_whitespace, reduce_single_child],
            "Zusatz": [reduce_single_child]
        }
        global_tansformations = {
            "LemmaVariante": [traverse_locally(LemmaVariante_transformations)],
            "Hinweis": [collapse]
        }
        traverse(cst, global_tansformations)
        # whitespace after "facitergula", but not after "bona" should have been removed
        assert str(cst) == "faciterculasim.bona fide"

221
class TestComplexTransformations:
di68kap's avatar
di68kap committed
222
    def setup(self):
223
        self.Text = 'Text'  # TOKEN_PTYPE
di68kap's avatar
di68kap committed
224
225

    def test_collapse_if_plain(self):
226
227
228
229
        xml = "<EINZEILER><DEU_WORT>spectat</DEU_WORT><WS> </WS><DEU_WORT>ad</DEU_WORT>" +\
              "<WS> </WS><DEU_WORT>gravitatem</DEU_WORT><TEIL_SATZZEICHEN>,</TEIL_SATZZEICHEN>" +\
              "<WS> </WS><DEU_WORT>momentum</DEU_WORT></EINZEILER>"
        tree = parse_xml(xml)
di68kap's avatar
di68kap committed
230
231
232
233
234
235
236
237
238
239
240
241
242
243
        assert tree.as_xml(inline_tags={'EINZEILER'}) == xml
        collapse_if([tree], lambda l: True, self.Text)
        assert tree.as_xml(inline_tags={'EINZEILER'}) == \
               "<EINZEILER><Text>spectat ad gravitatem, momentum</Text></EINZEILER>"

    def test_collapse_if_structured(self):
        xml = """<Stelle>
                   <DEU_WORT>p.</DEU_WORT>
                   <SEITENZAHL>26</SEITENZAHL>
                   <HOCHGESTELLT>b</HOCHGESTELLT>
                   <TEIL_SATZZEICHEN>,</TEIL_SATZZEICHEN>
                   <SEITENZAHL>18</SEITENZAHL>
                 </Stelle>"""
        tree = parse_xml(xml)
di68kap's avatar
di68kap committed
244
        # print(flatten_sxpr(tree.as_sxpr()))
di68kap's avatar
di68kap committed
245
246
247
        collapse_if([tree], lambda context: context[-1].tag_name != 'HOCHGESTELLT', self.Text)
        assert tree.as_xml(inline_tags={'Stelle'}) == \
               "<Stelle><Text>p.26</Text><HOCHGESTELLT>b</HOCHGESTELLT><Text>,18</Text></Stelle>"
248
249


250
251
252
253
254
255
256
257
class TestWhitespaceTransformations:
    def test_move_adjacent(self):
        sentence = parse_sxpr('(SENTENCE (WORD (LETTERS "To") (:Whitespace " ")) '
                              '(WORD (LETTERS "be") (:Whitespace " ")) '
                              '(WORD (LETTERS "or") (:Whitespace " ")) '
                              '(WORD (LETTERS "not") (:Whitespace " ")) '
                              '(WORD (LETTERS "to") (:Whitespace " "))'
                              '(WORD (LETTERS "be") (:Whitespace " ")))')
258
        transformations = {'WORD': move_adjacent(is_insignificant_whitespace)}
259
260
261
262
263
264
265
266
267
268
269
        traverse(sentence, transformations)
        assert tree_sanity_check(sentence)
        assert all(i % 2 == 0 or node.tag_name == ':Whitespace' for i, node in enumerate(sentence))

    def test_move_and_merge_adjacent(self):
        sentence = parse_sxpr('(SENTENCE (WORD (LETTERS "To") (:Whitespace " ")) '
                              '(WORD (:Whitespace " ") (LETTERS "be") (:Whitespace " ")) '
                              '(WORD (:Whitespace " ") (LETTERS "or") (:Whitespace " ")) '
                              '(WORD (:Whitespace " ") (LETTERS "not") (:Whitespace " ")) '
                              '(WORD (:Whitespace " ") (LETTERS "to") (:Whitespace " "))'
                              '(WORD (:Whitespace " ") (LETTERS "be") (:Whitespace " ")))')
270
        transformations = {'WORD': move_adjacent(is_insignificant_whitespace)}
271
272
273
274
275
        traverse(sentence, transformations)
        assert tree_sanity_check(sentence)
        assert all(i % 2 == 0 or node.tag_name == ':Whitespace' for i, node in enumerate(sentence))


276
277
278
if __name__ == "__main__":
    from DHParser.testing import runner
    runner("", globals())