- testing.py: LOGS now written in testsuite directory; History-Logs only writen for tests that fail

DHParser/parser.py

@@ -722,20 +722,23 @@ class Grammar:
             elif os.path.exists(path):
                 os.remove(path)
 
-        if not log_file_name:
-            name = self.__class__.__name__
-            log_file_name = name[:-7] if name.lower().endswith('grammar') else name
-        full_history, match_history, errors_only = [], [], []
-        for record in self.history__:
-            line = ";  ".join(prepare_line(record))
-            full_history.append(line)
-            if record.node and record.node.parser.ptype != WHITESPACE_PTYPE:
-                match_history.append(line)
-                if record.node.error_flag:
-                    errors_only.append(line)
-        write_log(full_history, log_file_name + '_full')
-        write_log(match_history, log_file_name + '_match')
-        write_log(errors_only, log_file_name + '_errors')
+        if is_logging():
+            assert self.history__, \
+                "Parser did not yet run or logging was turned off when running parser!"
+            if not log_file_name:
+                name = self.__class__.__name__
+                log_file_name = name[:-7] if name.lower().endswith('grammar') else name
+            full_history, match_history, errors_only = [], [], []
+            for record in self.history__:
+                line = ";  ".join(prepare_line(record))
+                full_history.append(line)
+                if record.node and record.node.parser.ptype != WHITESPACE_PTYPE:
+                    match_history.append(line)
+                    if record.node.error_flag:
+                        errors_only.append(line)
+            write_log(full_history, log_file_name + '_full')
+            write_log(match_history, log_file_name + '_match')
+            write_log(errors_only, log_file_name + '_errors')
 
 
 def dsl_error_msg(parser: Parser, error_str: str) -> str:

DHParser/syntaxtree.py

@@ -31,7 +31,7 @@ except ImportError:
     from .typing34 import AbstractSet, Any, ByteString, Callable, cast, Container, Dict, \
         Iterator, List, NamedTuple, Sequence, Union, Text, Tuple
 
-from DHParser.toolkit import log_dir, line_col, identity
+from DHParser.toolkit import is_logging, log_dir, line_col, identity
 
 __all__ = ('WHITESPACE_PTYPE',
            'TOKEN_PTYPE',
@@ -472,9 +472,10 @@ class Node:
     #     return nav(path.split('/'))
 
     def log(self, log_file_name):
-        st_file_name = log_file_name
-        with open(os.path.join(log_dir(), st_file_name), "w", encoding="utf-8") as f:
-            f.write(self.as_sxpr())
+        if is_logging():
+            st_file_name = log_file_name
+            with open(os.path.join(log_dir(), st_file_name), "w", encoding="utf-8") as f:
+                f.write(self.as_sxpr())
 
 
 def mock_syntax_tree(sxpr):

DHParser/testing.py

@@ -146,22 +146,21 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
                 infostr = '    match-test "' + test_name + '" ... '
                 errflag = len(errata)
             cst = parser(test_code, parser_name)
-            if is_logging():
-                cst.log("match_%s_%s.cst" % (parser_name, test_name))
-                parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
+            cst.log("match_%s_%s.cst" % (parser_name, test_name))
             tests.setdefault('__cst__', {})[test_name] = cst
             if "ast" in tests or report:
                 ast = copy.deepcopy(cst)
                 transform(ast)
                 tests.setdefault('__ast__', {})[test_name] = ast
-                if is_logging():
-                    ast.log("match_%s_%s.ast" % (parser_name, test_name))
+                ast.log("match_%s_%s.ast" % (parser_name, test_name))
             if cst.error_flag:
                 errata.append('Match test "%s" for parser "%s" failed:\n\tExpr.:  %s\n\n\t%s' %
                               (test_name, parser_name, '\n\t'.join(test_code.split('\n')),
                                '\n\t'.join(m.replace('\n', '\n\t\t') for m in
                                            error_messages(test_code, cst.collect_errors()))))
                 tests.setdefault('__err__', {})[test_name] = errata[-1]
+                # write parsing-history log only in case of failure!
+                parser.log_parsing_history__("match_%s_%s.log" % (parser_name, test_name))
             elif "cst" in tests and mock_syntax_tree(tests["cst"][test_name]) != cst:
                     errata.append('Concrete syntax tree test "%s" for parser "%s" failed:\n%s' %
                                   (test_name, parser_name, cst.as_sxpr()))
@@ -184,19 +183,20 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
                 infostr = '    fail-test  "' + test_name + '" ... '
                 errflag = len(errata)
             cst = parser(test_code, parser_name)
-            if is_logging():
-                cst.log("fail_%s_%s.cst" % (parser_name, test_name))
-                parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
+            # doesn't make sense to write cst for fail-tests
+            # cst.log("fail_%s_%s.cst" % (parser_name, test_name))
             if not cst.error_flag:
                 errata.append('Fail test "%s" for parser "%s" yields match instead of '
                               'expected failure!' % (test_name, parser_name))
                 tests.setdefault('__err__', {})[test_name] = errata[-1]
+                # write parsing-history log only in case of test-failure
+                parser.log_parsing_history__("fail_%s_%s.log" % (parser_name, test_name))
             if verbose:
                 print(infostr + "OK" if len(errata) == errflag else "FAIL")
 
     # write test-report
     if report:
-        report_dir = os.path.join(unit_dir, "REPORT")
+        report_dir = "REPORT"
         if not os.path.exists(report_dir):
             os.mkdir(report_dir)
         with open(os.path.join(report_dir, unit_name + '.report'), 'w') as f:
@@ -214,18 +214,21 @@ def grammar_suite(directory, parser_factory, transformer_factory, ignore_unknown
     all_errors = collections.OrderedDict()
     if verbose:
         print("\nScanning test-directory: " + directory)
-    for filename in sorted(os.listdir(directory)):
+    save_cwd = os.getcwd()
+    os.chdir(directory)
+    for filename in sorted(os.listdir()):
         if filename.lower().find("test") >= 0:
             try:
                 if verbose:
                     print("\nRunning grammar tests from: " + filename)
-                errata = grammar_unit(os.path.join(directory, filename),
-                                      parser_factory, transformer_factory, report, verbose)
+                errata = grammar_unit(filename, parser_factory,
+                                      transformer_factory, report, verbose)
                 if errata:
                     all_errors[filename] = errata
             except ValueError as e:
                 if not ignore_unknown_filetypes or str(e).find("Unknown") < 0:
                     raise e
+    os.chdir(save_cwd)
     error_report = []
     if all_errors:
         for filename in all_errors:

examples/LaTeX/LaTeX.ebnf

@@ -57,8 +57,8 @@ block_environment   = known_environment | generic_block
 known_environment   = itemize | enumerate | figure | table | quotation
                     | verbatim
 generic_block       = begin_generic_block sequence §end_generic_block
-begin_generic_block = -&LB begin_environment -&LB
-end_generic_block   = -&LB  end_environment  -&LB
+begin_generic_block = -&LB begin_environment (EOF | -&LB)
+end_generic_block   = -&LB  end_environment  (EOF | -&LB)
 
 itemize             = "\begin{itemize}" [PARSEP] { item } §"\end{itemize}"
 enumerate           = "\begin{enumerate}" [PARSEP] {item } §"\end{enumerate}"
@@ -87,7 +87,7 @@ inline_environment  = known_inline_env | generic_inline_env
 known_inline_env    = inline_math
 generic_inline_env  = begin_inline_env { text_elements }+ §end_inline_env
 begin_inline_env    = (-!LB begin_environment) | (begin_environment -!LB)
-end_inline_env      = (-!LB end_environment)   | (end_environment   -!LB)
+end_inline_env      = (-!LB end_environment)   | (end_environment   -!LB)       # ambiguity with genric_block when EOF
 begin_environment   = "\begin{" §NAME §"}"
 end_environment     = "\end{" §::NAME §"}"
 

examples/LaTeX/LaTeXCompiler.py

@@ -108,8 +108,8 @@ class LaTeXGrammar(Grammar):
     known_environment   = itemize | enumerate | figure | table | quotation
                         | verbatim
     generic_block       = begin_generic_block sequence §end_generic_block
-    begin_generic_block = -&LB begin_environment -&LB
-    end_generic_block   = -&LB  end_environment  -&LB
+    begin_generic_block = -&LB begin_environment (EOF | -&LB)
+    end_generic_block   = -&LB  end_environment  (EOF | -&LB)
     
     itemize             = "\begin{itemize}" [PARSEP] { item } §"\end{itemize}"
     enumerate           = "\begin{enumerate}" [PARSEP] {item } §"\end{enumerate}"
@@ -138,7 +138,7 @@ class LaTeXGrammar(Grammar):
     known_inline_env    = inline_math
     generic_inline_env  = begin_inline_env { text_elements }+ §end_inline_env
     begin_inline_env    = (-!LB begin_environment) | (begin_environment -!LB)
-    end_inline_env      = (-!LB end_environment)   | (end_environment   -!LB)
+    end_inline_env      = (-!LB end_environment)   | (end_environment   -!LB)       # ambiguity with genric_block when EOF
     begin_environment   = "\begin{" §NAME §"}"
     end_environment     = "\end{" §::NAME §"}"
     
@@ -205,7 +205,7 @@ class LaTeXGrammar(Grammar):
     block_of_paragraphs = Forward()
     end_generic_block = Forward()
     text_elements = Forward()
-    source_hash__ = "cf722f798fd396d8094fcc28831d9dcd"
+    source_hash__ = "f941997b8aca0a8aa2d2f38cb52818eb"
     parser_initialization__ = "upon instantiation"
     COMMENT__ = r'%.*(?:\n|$)'
     WSP__ = mixin_comment(whitespace=r'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?', comment=r'%.*(?:\n|$)')
@@ -256,8 +256,8 @@ class LaTeXGrammar(Grammar):
     item = Series(Token("\\item"), Optional(PARSEP), sequence)
     enumerate = Series(Token("\\begin{enumerate}"), Optional(PARSEP), ZeroOrMore(item), Required(Token("\\end{enumerate}")))
     itemize = Series(Token("\\begin{itemize}"), Optional(PARSEP), ZeroOrMore(item), Required(Token("\\end{itemize}")))
-    end_generic_block.set(Series(Lookbehind(LB), end_environment, Lookbehind(LB)))
-    begin_generic_block.set(Series(Lookbehind(LB), begin_environment, Lookbehind(LB)))
+    end_generic_block.set(Series(Lookbehind(LB), end_environment, Alternative(EOF, Lookbehind(LB))))
+    begin_generic_block.set(Series(Lookbehind(LB), begin_environment, Alternative(EOF, Lookbehind(LB))))
     generic_block = Series(begin_generic_block, sequence, Required(end_generic_block))
     known_environment = Alternative(itemize, enumerate, figure, table, quotation, verbatim)
     block_environment.set(Alternative(known_environment, generic_block))