Commit db176029 authored by eckhart's avatar eckhart

- factored out logging to a new module 'log'

parent 10d5bdb4
......@@ -22,6 +22,7 @@ from .dsl import *
from .ebnf import *
# Flat namespace for the DHParser Package. Is this a good idea...?
from .error import *
from .log import *
from .parse import *
from .preprocess import *
from .stringview import *
......
......@@ -29,8 +29,9 @@ from DHParser.error import Error, is_error, has_errors, only_errors
from DHParser.parse import Grammar, Compiler, compile_source
from DHParser.preprocess import nil_preprocessor, PreprocessorFunc
from DHParser.syntaxtree import Node, TransformationFunc
from DHParser.toolkit import logging, load_if_file, is_python_code, compile_python_object, \
from DHParser.toolkit import load_if_file, is_python_code, compile_python_object, \
re
from DHParser.log import logging
__all__ = ('DHPARSER_IMPORTS',
'GrammarError',
......
This diff is collapsed.
This diff is collapsed.
......@@ -18,12 +18,11 @@ permissions and limitations under the License.
import collections.abc
import copy
import os
from functools import partial
from DHParser.error import Error, linebreaks, line_col
from DHParser.stringview import StringView
from DHParser.toolkit import is_logging, log_dir, re, typing
from DHParser.toolkit import re, typing
from typing import Any, Callable, cast, Iterator, List, Union, Tuple, Hashable, Optional
......@@ -500,7 +499,7 @@ class Node(collections.abc.Sized):
given the position of the element in the text will be
reported as line and column.
compact: If True a compact representation is returned where
brackets are ommited and only the indentation indicates the
brackets are omitted and only the indentation indicates the
tree structure.
"""
......@@ -601,18 +600,6 @@ class Node(collections.abc.Sized):
return sum(child.tree_size() for child in self.children) + 1
def log(self, log_file_name):
"""
Writes an S-expression-representation of the tree with root `self` to a file.
"""
if is_logging():
path = os.path.join(log_dir(), log_file_name)
if os.path.exists(path):
print('WARNING: Log-file "%s" already exists and will be overwritten!' % path)
with open(path, "w", encoding="utf-8") as f:
f.write(self.as_sxpr())
def mock_syntax_tree(sxpr):
"""
Generates a tree of nodes from an S-expression. The main purpose of this is
......
......@@ -23,7 +23,8 @@ import inspect
import json
import os
from DHParser.toolkit import is_logging, clear_logs, re
from DHParser.toolkit import re
from DHParser.log import is_logging, clear_logs, log_ST, log_parsing_history
from DHParser.syntaxtree import Node, mock_syntax_tree, flatten_sxpr, ZOMBIE_PARSER
from DHParser.parse import UnknownParserError
from DHParser.error import is_error, adjust_error_locations
......@@ -177,13 +178,13 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
cst = parser(test_code, parser_name)
except UnknownParserError as upe:
cst = Node(ZOMBIE_PARSER, "").add_error(str(upe)).init_pos(0)
cst.log("match_%s_%s.cst" % (parser_name, test_name))
log_ST(cst, "match_%s_%s.cst" % (parser_name, test_name))
tests.setdefault('__cst__', {})[test_name] = cst
if "ast" in tests or report:
ast = copy.deepcopy(cst)
transform(ast)
tests.setdefault('__ast__', {})[test_name] = ast
ast.log("match_%s_%s.ast" % (parser_name, test_name))
log_ST(ast, "match_%s_%s.ast" % (parser_name, test_name))
if is_error(cst.error_flag):
errors = adjust_error_locations(cst.collect_errors(), test_code)
errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.: %s\n\n\t%s\n\n' %
......@@ -192,7 +193,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
tests.setdefault('__err__', {})[test_name] = errata[-1]
# write parsing-history log only in case of failure!
if is_logging():
parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
log_parsing_history(parser, "match_%s_%s.log" % (parser_name, test_name))
elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
(test_name, parser_name, cst.as_sxpr()))
......@@ -225,7 +226,7 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
tests.setdefault('__err__', {})[test_name] = errata[-1]
# write parsing-history log only in case of test-failure
if is_logging():
parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
log_parsing_history(parser, "fail_%s_%s.log" % (parser_name, test_name))
if verbose:
print(infostr + ("OK" if len(errata) == errflag else "FAIL"))
......
......@@ -31,10 +31,8 @@ already exists.
"""
import codecs
import contextlib
import hashlib
import io
import os
import parser
try:
......@@ -51,12 +49,7 @@ except ImportError:
from typing import Any, Iterable, Sequence, Set, Union, cast
__all__ = ('logging',
'is_logging',
'clear_logs',
'log_dir',
'logfile_basename',
'escape_re',
__all__ = ('escape_re',
'escape_control_characters',
'is_filename',
'lstrip_docstring',
......@@ -71,96 +64,11 @@ __all__ = ('logging',
#######################################################################
#
# logging
# miscellaneous (generic)
#
#######################################################################
def log_dir() -> str:
"""Creates a directory for log files (if it does not exist) and
returns its path.
WARNING: Any files in the log dir will eventually be overwritten.
Don't use a directory name that could be the name of a directory
for other purposes than logging.
Returns:
name of the logging directory
"""
# the try-except clauses in the following are precautions for multiprocessing
global LOGGING
try:
dirname = LOGGING # raises a name error if LOGGING is not defined
if not dirname:
raise NameError # raise a name error if LOGGING evaluates to False
except NameError:
raise NameError("No access to log directory before logging has been "
"turned on within the same thread/process.")
if os.path.exists(dirname) and not os.path.isdir(dirname):
raise IOError('"' + dirname + '" cannot be used as log directory, '
'because it is not a directory!')
else:
try:
os.mkdir(dirname)
except FileExistsError:
pass
info_file_name = os.path.join(dirname, 'info.txt')
if not os.path.exists(info_file_name):
with open(info_file_name, 'w') as f:
f.write("This directory has been created by DHParser to store log files from\n"
"parsing. ANY FILE IN THIS DIRECTORY CAN BE OVERWRITTEN! Therefore,\n"
"do not place any files here and do not bother editing files in this\n"
"directory as any changes will get lost.\n")
return dirname
@contextlib.contextmanager
def logging(dirname="LOGS"):
"""Context manager. Log files within this context will be stored in
directory ``dirname``. Logging is turned off if name is empty.
Args:
dirname: the name for the log directory or the empty string to
turn logging of
"""
global LOGGING
if dirname and not isinstance(dirname, str):
dirname = "LOGS" # be fail tolerant here...
try:
save = LOGGING
except NameError:
save = ""
LOGGING = dirname or ""
yield
LOGGING = save
def is_logging() -> bool:
"""-> True, if logging is turned on."""
global LOGGING
try:
return bool(LOGGING)
except NameError:
return False
def clear_logs(logfile_types=frozenset(['.cst', '.ast', '.log'])):
"""Removes all logs from the log-directory and removes the
log-directory if it is empty.
"""
log_dirname = log_dir()
files = os.listdir(log_dirname)
only_log_files = True
for file in files:
path = os.path.join(log_dirname, file)
if os.path.splitext(file)[1] in logfile_types or file == 'info.txt':
os.remove(path)
else:
only_log_files = False
if only_log_files:
os.rmdir(log_dirname)
def escape_re(strg: str) -> str:
"""
Returns the string with all regular expression special characters escaped.
......@@ -200,21 +108,6 @@ def is_filename(strg: str) -> bool:
# and strg.find('*') < 0 and strg.find('?') < 0
def logfile_basename(filename_or_text, function_or_class_or_instance) -> str:
"""Generates a reasonable logfile-name (without extension) based on
the given information.
"""
if is_filename(filename_or_text):
return os.path.basename(os.path.splitext(filename_or_text)[0])
else:
try:
name = function_or_class_or_instance.__qualname.__
except AttributeError:
name = function_or_class_or_instance.__class__.__name__
i = name.find('.')
return name[:i] + '_out' if i >= 0 else name
#######################################################################
#
# loading and compiling
......@@ -395,7 +288,7 @@ def expand_table(compact_table):
#######################################################################
#
# miscellaneous
# miscellaneous (DHParser-specific)
#
#######################################################################
......
......@@ -26,7 +26,7 @@ import sys
from DHParser.dsl import compileDSL, compile_on_disk
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
from DHParser.parse import compile_source
from DHParser.toolkit import logging
from DHParser.log import logging
EBNF_TEMPLATE = r"""-grammar
......
......@@ -17,7 +17,7 @@ except ImportError:
sys.path.extend(['../../', '../', './'])
from DHParser import logging, is_filename, load_if_file, \
from DHParser import is_filename, load_if_file, \
Grammar, Compiler, nil_preprocessor, \
Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym, \
Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \
......@@ -30,6 +30,7 @@ from DHParser import logging, is_filename, load_if_file, \
is_empty, is_expendable, collapse, replace_content, remove_nodes, remove_content, remove_brackets, replace_parser, \
keep_children, is_one_of, has_content, apply_if, remove_first, remove_last, \
WHITESPACE_PTYPE, TOKEN_PTYPE
from DHParser.log import logging
#######################################################################
......
......@@ -21,6 +21,8 @@ limitations under the License.
import sys
import DHParser.log
sys.path.extend(['../../', '../'])
import DHParser.dsl
......@@ -37,7 +39,7 @@ sys.path.append('./')
# must be appended after module creation, because otherwise an ImportError is raised under Windows
from BibTeXCompiler import get_grammar, get_transformer
with toolkit.logging(True):
with DHParser.log.logging(True):
error_report = testing.grammar_suite('grammar_tests', get_grammar,
get_transformer, report=True, verbose=True)
if error_report:
......
......@@ -14,7 +14,7 @@ try:
import regex as re
except ImportError:
import re
from DHParser import logging, is_filename, load_if_file, \
from DHParser import is_filename, load_if_file, \
Grammar, Compiler, nil_preprocessor, \
Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym, \
Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \
......@@ -27,6 +27,7 @@ from DHParser import logging, is_filename, load_if_file, \
is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
remove_nodes, remove_content, remove_brackets, replace_parser, \
keep_children, is_one_of, has_content, apply_if, remove_first, remove_last
from DHParser.log import logging
#######################################################################
......
......@@ -15,7 +15,7 @@ try:
import regex as re
except ImportError:
import re
from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, Alternative, Pop, \
from DHParser import is_filename, Grammar, Compiler, Lookbehind, Alternative, Pop, \
Token, Synonym, \
Option, NegativeLookbehind, OneOrMore, RegExp, Series, RE, Capture, \
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
......@@ -24,6 +24,7 @@ from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, Altern
reduce_single_child, replace_by_single_child, remove_whitespace, \
flatten, is_empty, collapse, replace_content, remove_brackets, is_one_of, remove_first, \
remove_tokens, remove_nodes, TOKEN_PTYPE
from DHParser.log import logging
#######################################################################
......
......@@ -4,3 +4,5 @@ LaTeX
This is going to be a compiler for a subset of LaTeX. TeX/LaTeX in
general is not a context free language ADD REFERENCE HERE.
Thus, only a subset.
Presently, parser and AST-Transformation works.
\ No newline at end of file
......@@ -25,7 +25,7 @@ import sys
sys.path.extend(['../../', '../', './'])
from DHParser.toolkit import logging
from DHParser.log import logging
from DHParser.dsl import recompile_grammar
......
......@@ -25,10 +25,13 @@ import os
import pstats
import sys
import DHParser.log
sys.path.extend(['../../', '../', './'])
import DHParser.dsl
from DHParser import toolkit
from DHParser.log import log_parsing_history
if not DHParser.dsl.recompile_grammar('LaTeX.ebnf', force=False): # recompiles Grammar only if it has changed
print('\nErrors while recompiling "LaTeX.ebnf":\n--------------------------------------\n\n')
......@@ -51,7 +54,7 @@ def fail_on_error(src, result):
def tst_func():
with toolkit.logging(False):
with DHParser.log.logging(False):
if not os.path.exists('REPORT'):
os.mkdir('REPORT')
files = os.listdir('testdata')
......@@ -62,13 +65,13 @@ def tst_func():
doc = f.read()
print('\n\nParsing document: "%s"\n' % file)
result = parser(doc)
if toolkit.is_logging():
if DHParser.log.is_logging():
with open('REPORT/' + file[:-4] + '.cst', 'w', encoding='utf-8') as f:
f.write(result.as_sxpr(compact=False))
transformer(result)
with open('REPORT/' + file[:-4] + '.ast', 'w', encoding='utf-8') as f:
f.write(result.as_sxpr(compact=False))
parser.log_parsing_history__()
log_parsing_history(parser)
fail_on_error(doc, result)
transformer(result)
fail_on_error(doc, result)
......
......@@ -21,6 +21,8 @@ limitations under the License.
import sys
import DHParser.log
sys.path.extend(['../../', '../', './'])
from DHParser import dsl
......@@ -29,7 +31,7 @@ from DHParser import toolkit
# print(dir(dsl))
with toolkit.logging(False):
with DHParser.log.logging(False):
if not dsl.recompile_grammar('LaTeX.ebnf', force=True): # recompiles Grammar only if it has
# changed
print(
......@@ -40,7 +42,7 @@ with toolkit.logging(False):
from LaTeXCompiler import get_grammar, get_transformer
with toolkit.logging(True):
with DHParser.log.logging(True):
error_report = testing.grammar_suite('grammar_tests', get_grammar, get_transformer,
fn_patterns=['*_test_*.ini'],
report=True, verbose=True)
......
......@@ -17,7 +17,7 @@ try:
import regex as re
except ImportError:
import re
from DHParser import logging, is_filename, load_if_file, \
from DHParser import is_filename, load_if_file, \
Grammar, Compiler, nil_preprocessor, \
Lookbehind, Lookahead, Alternative, Pop, Token, Synonym, AllOf, SomeOf, \
Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \
......@@ -31,6 +31,7 @@ from DHParser import logging, is_filename, load_if_file, \
remove_brackets, replace_parser, traverse_locally, remove_nodes, \
keep_children, is_one_of, has_content, apply_if, remove_first, remove_last, \
lstrip, rstrip, strip, keep_nodes, remove_anonymous_empty
from DHParser.log import logging
#######################################################################
......
......@@ -14,7 +14,7 @@ try:
import regex as re
except ImportError:
import re
from DHParser import logging, is_filename, load_if_file, \
from DHParser import is_filename, load_if_file, \
Grammar, Compiler, nil_preprocessor, \
Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym, \
Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \
......@@ -27,6 +27,7 @@ from DHParser import logging, is_filename, load_if_file, \
is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
remove_nodes, remove_content, remove_brackets, replace_parser, \
keep_children, is_one_of, has_content, apply_if, remove_first, remove_last
from DHParser.log import logging
#######################################################################
......
......@@ -22,6 +22,9 @@ limitations under the License.
import os
import sys
import DHParser.log
sys.path.append(os.path.abspath('../../../'))
import DHParser.toolkit as toolkit
from DHParser.ebnf import grammar_changed
......@@ -40,7 +43,7 @@ if (not os.path.exists(MLW_compiler) or
print('\n'.join(errors))
sys.exit(1)
with toolkit.logging():
with DHParser.log.logging():
errors = compile_on_disk("fascitergula.mlw", MLW_compiler, ".sxpr")
if errors:
print('\n'.join(errors))
......
......@@ -21,6 +21,8 @@ limitations under the License.
import sys
import DHParser.log
sys.path.extend(['../../', '../', './'])
from DHParser import dsl
......@@ -34,7 +36,7 @@ if not dsl.recompile_grammar('MLW.ebnf', force=True): # recompiles Grammar only
from MLWCompiler import get_grammar, get_transformer
with toolkit.logging(True):
with DHParser.log.logging(True):
error_report = testing.grammar_suite('grammar_tests', get_grammar, get_transformer,
fn_patterns=['*_test*'],
verbose=True)
......
......@@ -8,7 +8,7 @@ import sys
sys.path.extend(['../../', '../', './'])
from DHParser.toolkit import logging
from DHParser.log import logging
from DHParser.parse import compile_source
from DHParser.dsl import recompile_grammar
......
......@@ -16,7 +16,7 @@ try:
import regex as re
except ImportError:
import re
from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, \
from DHParser import is_filename, Grammar, Compiler, Lookbehind, \
Alternative, Pop, Token, Synonym, \
Option, NegativeLookbehind, OneOrMore, RegExp, Series, RE, Capture, \
ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
......@@ -26,7 +26,7 @@ from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, \
flatten, is_empty, collapse, replace_content, remove_brackets, \
is_one_of, remove_first, remove_last, remove_tokens, remove_nodes, \
is_whitespace, TOKEN_PTYPE
from DHParser.log import logging
#######################################################################
......
......@@ -137,6 +137,7 @@ class TestCompilerGeneration:
sys.path.append(self.tmp)
from TestCompilerGenerationCompiler import compile_src
print(compile_src)
result, errors, ast = compile_src(self.trivial_text)
......
......@@ -24,7 +24,8 @@ from functools import partial
sys.path.extend(['../', './'])
from DHParser.toolkit import is_logging, logging, compile_python_object
from DHParser.toolkit import compile_python_object
from DHParser.log import logging, is_logging, log_ST
from DHParser.stringview import StringView
from DHParser.error import Error
from DHParser.syntaxtree import mock_syntax_tree
......@@ -49,7 +50,7 @@ class TestInfiLoopsAndRecursion:
assert not syntax_tree.error_flag, str(syntax_tree.collect_errors())
assert snippet == str(syntax_tree)
if is_logging():
syntax_tree.log("test_LeftRecursion_direct.cst")
log_ST(syntax_tree, "test_LeftRecursion_direct.cst")
parser.log_parsing_history__("test_LeftRecursion_direct")
def test_direct_left_recursion2(self):
......@@ -86,7 +87,7 @@ class TestInfiLoopsAndRecursion:
assert not syntax_tree.error_flag, syntax_tree.collect_errors()
assert snippet == str(syntax_tree)
if is_logging():
syntax_tree.log("test_LeftRecursion_indirect.cst")
log_ST(syntax_tree, "test_LeftRecursion_indirect.cst")
parser.log_parsing_history__("test_LeftRecursion_indirect")
def test_inifinite_loops(self):
......@@ -204,7 +205,7 @@ class TestRegex:
assert not messages
parser = compile_python_object(DHPARSER_IMPORTS + result, '\w+Grammar$')()
result = parser(testdoc)
# parser.log_parsing_history__("test.log")
# parser.log_parsing_history("test.log")
assert not result.error_flag
......@@ -452,7 +453,7 @@ class TestPopRetrieve:
pop = str(next(syntax_tree.find(self.closing_delimiter)))
assert delim == pop
if is_logging():
syntax_tree.log("test_PopRetrieve_single_line.cst")
log_ST(syntax_tree, "test_PopRetrieve_single_line.cst")
def test_multi_line(self):
teststr = """
......@@ -469,7 +470,7 @@ class TestPopRetrieve:
pop = str(next(syntax_tree.find(self.closing_delimiter)))
assert delim == pop
if is_logging():
syntax_tree.log("test_PopRetrieve_multi_line.cst")
log_ST(syntax_tree, "test_PopRetrieve_multi_line.cst")
def test_single_line_complement(self):
teststr = "Anfang {{{code block }} <- keine Ende-Zeichen ! }}} Ende"
......@@ -479,7 +480,7 @@ class TestPopRetrieve:
pop = str(next(syntax_tree.find(self.closing_delimiter)))
assert len(delim) == len(pop) and delim != pop
if is_logging():
syntax_tree.log("test_PopRetrieve_single_line.cst")
log_ST(syntax_tree, "test_PopRetrieve_single_line.cst")
def test_multi_line_complement(self):
teststr = """
......@@ -496,7 +497,7 @@ class TestPopRetrieve:
pop = str(next(syntax_tree.find(self.closing_delimiter)))
assert len(delim) == len(pop) and delim != pop
if is_logging():
syntax_tree.log("test_PopRetrieve_multi_line.cst")
log_ST(syntax_tree, "test_PopRetrieve_multi_line.cst")
class TestWhitespaceHandling:
......
......@@ -26,8 +26,9 @@ import sys
sys.path.extend(['../', './'])
from DHParser.toolkit import has_fenced_code, load_if_file, logging, log_dir, is_logging, re, \
from DHParser.toolkit import has_fenced_code, load_if_file, re, \
lstrip_docstring
from DHParser.log import log_dir, logging, is_logging
class TestLoggingAndLoading:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment