tst_LaTeX_docs.py 4.92 KB
Newer Older
1
#!/usr/bin/env python3
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

"""tst_LaTeX_doc.py - tests with full documents in subdir 'testdata'

Author: Eckhart Arnold <arnold@badw.de>

Copyright 2017 Bavarian Academy of Sciences and Humanities

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

22
import fnmatch
23
24
25
import os
import sys

26

27
28
29
30
31
scriptpath = os.path.dirname(__file__) or '.'
for path in (os.path.join('..', '..'), '.'):
    fullpath = os.path.abspath(os.path.join(scriptpath, path))
    if fullpath not in sys.path:
        sys.path.append(fullpath)
32

eckhart's avatar
eckhart committed
33
import DHParser.configuration
34
import DHParser.dsl
35
from DHParser.error import ERROR
36
import DHParser.log
37
38
from DHParser.log import log_parsing_history

39

40
LOGGING = 'LOGS'
eckhart's avatar
eckhart committed
41

42
43
44
grammar_path = os.path.join(fullpath, 'LaTeX.ebnf')
if not DHParser.dsl.recompile_grammar(grammar_path, force=False):
    # recompiles Grammar only if it has changed
45
    print('\nErrors while recompiling "LaTeX.ebnf":\n--------------------------------------\n\n')
46
    with open('LaTeX_ebnf_ERRORS.txt', encoding="utf-8") as f:
47
48
49
50
        print(f.read())
    sys.exit(1)


51
from LaTeXParser import get_preprocessor, get_grammar, get_transformer, get_compiler
52

53
preprocessor = get_preprocessor()
54
55
parser = get_grammar()
transformer = get_transformer()
56
compiler = get_compiler()
57

58

59
60
def fail_on_error(src, result):
    if result.error_flag:
61
        for e in result.errors_sorted:
62
            print(str(e))
63
64
        if result.error_flag >= ERROR:
            sys.exit(1)
65

66

67
68
69
70
71
72
73
def tree_size(tree) -> int:
    """
    Recursively counts the number of nodes in the tree including the root node.
    """
    return sum(tree_size(child) for child in tree.children) + 1


74
def tst_func():
75
    DHParser.log.start_logging(LOGGING)
76
77
    test_dir = os.path.join(fullpath, 'testdata')
    files = os.listdir(test_dir)
78
79
80
    files.sort()
    for file in files:
        if fnmatch.fnmatch(file, '*.tex') and file.lower().find('error') < 0:
81
            filepath = os.path.join(test_dir, file)
eckhart's avatar
eckhart committed
82
            with open(filepath, 'r', encoding='utf-8') as f:
83
84
                doc = f.read()

85
86
87
88
            print(f'\n\nPreprocessing document: "{file}"')
            preprocessed, source_mapper = preprocessor(doc, file)
            print(f'\n\nParsing document: "{file}"')
            result = parser(preprocessed)
89
90
91
92
93
            print("Number of CST-nodes: " + str(tree_size(result)))
            # print("Number of empty nodes: " + str(count_nodes(result,
            #                                                 lambda n: not bool(n.result))))
            if DHParser.log.is_logging():
                print('Saving CST')
eckhart's avatar
eckhart committed
94
95
                logs = DHParser.log.log_dir().rstrip('/') + '/'
                with open(logs + file[:-4] + '.cst', 'w', encoding='utf-8') as f:
Eckhart Arnold's avatar
Eckhart Arnold committed
96
                    f.write(result.as_sxpr(compact=False))
97
98
99
100
101
102
103
104
105
                print('Saving parsing history')
                log_parsing_history(parser, os.path.basename(file), html=True)

            print('\nTransforming document: "%s"' % file)
            fail_on_error(doc, result)
            transformer(result)
            fail_on_error(doc, result)
            print("Number of AST-nodes: " + str(tree_size(result)))
            if DHParser.log.is_logging():
eckhart's avatar
eckhart committed
106
                logs = DHParser.log.log_dir().rstrip('/') + '/'
107
                print('Saving AST')
eckhart's avatar
eckhart committed
108
                with open(logs + file[:-4] + '.ast', 'w', encoding='utf-8') as f:
109
                    f.write(result.as_sxpr(compact=True))
eckhart's avatar
eckhart committed
110
111
                # with open(logs + file[:-4] + '.tex', 'w', encoding='utf-8') as f:
                #     f.write(str(result))
112
113
114

            print('\nCompiling document: "%s"' % file)
            output = compiler(result)
eckhart's avatar
eckhart committed
115
            with open(os.path.splitext(filepath)[0] + '.xml', 'w', encoding='utf-8') as f:
di68kap's avatar
di68kap committed
116
                f.write(output.customized_XML())
eckhart's avatar
eckhart committed
117
            with open(os.path.splitext(filepath)[0] + '.sxpr', 'w', encoding='utf-8') as f:
118
                f.write(output.serialize('S-expression'))
119
120

def cpu_profile(func):
di68kap's avatar
di68kap committed
121
122
    import cProfile as profile
    import pstats
123
124
    pr = profile.Profile()
    pr.enable()
125
    func()
126
127
128
    pr.disable()
    st = pstats.Stats(pr)
    st.strip_dirs()
129
    st.sort_stats('time').print_stats(40)
130
131
132
133
134
135
136
137
138


def mem_profile(func):
    import tracemalloc
    tracemalloc.start()
    func()
    snapshot = tracemalloc.take_snapshot()
    top_stats = snapshot.statistics('lineno')
    print("[ Top 20 ]")
139
    for stat in top_stats[:40]:
140
141
        print(stat)

142

143
if __name__ == "__main__":
Eckhart Arnold's avatar
Eckhart Arnold committed
144
    cpu_profile(tst_func)
di68kap's avatar
di68kap committed
145
146
147
    from DHParser import syntaxtree
#    for k, v in syntaxtree.CALLERS.items():
#        print(k, v)
148