bug fixes; started code for systematic testing

.gitignore

@@ -16,6 +16,6 @@ testdata/*.pdf
 DEBUG*
 LOGS/
 external_resources/
-tmp/
+tmp/*
 build/
 dist/

DHParser/dsl.py

@@ -396,4 +396,3 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
 
     return []
 
-

DHParser/ebnf.py

@@ -140,7 +140,7 @@ EBNF_transformation_table = {
     (TOKEN_KEYWORD, WHITESPACE_KEYWORD):
         [remove_expendables, reduce_single_child],
     "list_":
-        [partial(remove_tokens, tokens={','})],
+        [flatten, partial(remove_tokens, tokens={','})],
     "":
         [remove_expendables, replace_by_single_child]
 }
@@ -188,8 +188,8 @@ class EBNFCompiler(CompilerBase):
 
     def _reset(self):
         self.rules = set()
-        self.symbols = set()
         self.variables = set()
+        self.symbol_nodes = []
         self.definition_names = []
         self.recursive = set()
         self.root = ""
@@ -302,10 +302,10 @@ class EBNFCompiler(CompilerBase):
                 declarations += [symbol + '.set(' + statement + ')']
             else:
                 declarations += [symbol + ' = ' + statement]
-        for nd in self.symbols:
+        for nd in self.symbol_nodes:
             if nd.result not in self.rules:
                 nd.add_error("Missing production for symbol '%s'" % nd.result)
-        if self.root and 'root__' not in self.symbols:
+        if self.root and 'root__' not in self.rules:
             declarations.append('root__ = ' + self.root)
         declarations.append('')
         return '\n    '.join(declarations)
@@ -443,7 +443,6 @@ class EBNFCompiler(CompilerBase):
         if prefix in {'::', ':'}:
             assert len(node.result) == 2
             arg = node.result[-1]
-            argstr = str(arg)
             if arg.parser.name != 'symbol':
                 node.add_error(('Retrieve Operator "%s" requires a symbol, '
                                 'and not a %s.') % (prefix, str(arg.parser)))
@@ -487,7 +486,7 @@ class EBNFCompiler(CompilerBase):
         if node.result in self.directives['tokens']:
             return 'ScannerToken("' + node.result + '")'
         else:
-            self.symbols.add(node)
+            self.symbol_nodes.append(node)
             if node.result in self.rules:
                 self.recursive.add(node.result)
             return node.result

DHParser/parsers.py

@@ -55,11 +55,10 @@ try:
     import regex as re
 except ImportError:
     import re
-import sys
 
-from .toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name, smart_list
+from .toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name
 from .syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \
-    traverse
+    mock_syntax_tree
 from DHParser.toolkit import load_if_file, error_messages
 
 __all__ = ['HistoryRecord',
@@ -1070,3 +1069,14 @@ def full_compilation(source, scanner, parser, transform, compiler):
     messages = error_messages(source_text, errors)
     return result, messages, syntax_tree
 
+
+def test_grammar(test_suite, parse_function, transform):
+    for parser_name, tests in test_suite.items():
+        assert set(tests.keys()).issubset({'match', 'fail', 'ast', 'cst'})
+        for test_name, test_code in tests['match'].items():
+            cst = parse_function(test_code, parser_name)
+            if not cst.error_flag:
+                yield "Test %s for parser %s did not match" % (test_name, parser_name)
+            if "cst" in tests:
+                if tests["cst"][test_name] != mock_syntax_tree(cst):
+                    pass # TO BE CONTINUED
\ No newline at end of file

DHParser/syntaxtree.py

@@ -17,6 +17,7 @@ implied.  See the License for the specific language governing
 permissions and limitations under the License.
 """
 
+import copy
 import itertools
 import os
 from functools import partial
@@ -166,9 +167,18 @@ class Node:
     def __eq__(self, other):
         return str(self.parser) == str(other.parser) and self.result == other.result
 
+    def __hash__(self):
+        return hash((str(self.parser), ))
+
+    def __deepcopy__(self, memodict={}):
+        result = copy.deepcopy(self.result)
+        other = Node(self.parser, result)
+        other._pos = self._pos
+        return other
+
     @property
     def tag_name(self):
-        return str(self.parser)
+        return self.parser.name or self.parser.__class__.__name__
         # ONLY FOR DEBUGGING: return self.parser.name + ':' + self.parser.__class__.__name__
 
     @property

DHParser/toolkit.py

@@ -54,7 +54,7 @@ __all__ = ['logging_on',
            'sane_parser_name']
 
 
-LOGGING: str = "LOGS"  # LOGGING = "" turns logging off!
+LOGGING: str = ""  # "LOGS"  # LOGGING = "" turns logging off!
 
 
 def logging_on(log_subdir="LOGS"):

OLDSTUFF/ParserCombinators_obsolete.py

@@ -740,7 +740,7 @@ class Parser(metaclass=ParserMetaClass):
 
     def apply(self, func):
         """Applies function `func(parser)` recursively to this parser and all
-        descendendants of the tree of parsers. The same function can never
+        descendants of the tree of parsers. The same function can never
         be applied twice between calls of the ``reset()``-method!
         """
         if func in self.cycle_detection:

test/test_dsl.py

@@ -31,11 +31,12 @@ class TestCompilerGeneration:
         word = /\w+/
         WSPC = /\s+/
         """
+    tmp = 'tmp/' if os.path.isdir('tmp') else ('test/tmp/')
     trivial_text = """Es war ein König in Thule."""
-    grammar_name = "tmp/TestCompilerGeneration.ebnf"
-    compiler_name = "tmp/TestCompilerGeneration_compiler.py"
-    text_name = "tmp/TestCompilerGeneration_text.txt"
-    result_name = "tmp/TestCompilerGeneration_text.xml"
+    grammar_name = tmp + "TestCompilerGeneration.ebnf"
+    compiler_name = tmp + "TestCompilerGeneration_compiler.py"
+    text_name = tmp + "TestCompilerGeneration_text.txt"
+    result_name = tmp + "TestCompilerGeneration_text.xml"
 
     def setup(self):
         with open(self.grammar_name, "w") as f:
@@ -71,7 +72,7 @@ class TestCompilerGeneration:
         result = run_compiler(self.trivial_text, self.compiler_name)
         assert output == result.as_xml(), str(result)
 
-        sys.path.append('tmp')
+        sys.path.append(self.tmp)
         from TestCompilerGeneration_compiler import compile_TestCompilerGeneration
         result, errors, ast = compile_TestCompilerGeneration(self.trivial_text)
 

test/test_ebnf.py

@@ -81,9 +81,21 @@ class TestDirectives:
 
 
 class TestEBNFParser:
-    test_json = [
-""
-]
+    test_json = {
+        "list_": {
+            "match": {
+                1: "hund",
+                2: "hund, katze,maus",
+                3: "hund , katze"
+            },
+            "fail": {
+                1: "123",
+                2: '"literal"',
+                3: "/regexp/"
+            }
+        }
+    }
+
 
     def setup(self):
         self.EBNF = EBNFGrammar()

test/test_parsers.py

@@ -105,4 +105,4 @@ class TestRegex:
 
 if __name__ == "__main__":
     from run import runner
-    runner("", globals())
+    runner("TestInfiLoopsAndRecursion", globals())

test/test_syntaxtree.py

@@ -19,6 +19,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import copy
 import os
 import sys
 sys.path.append(os.path.abspath('../../'))
@@ -90,6 +91,13 @@ class TestNode:
         assert self.recurr_tree != self.unique_tree
         assert mock_syntax_tree('(a (b c))') != mock_syntax_tree('(a (b d))')
 
+    def test_copy(self):
+        cpy = copy.deepcopy(self.unique_tree)
+        assert cpy == self.unique_tree
+        assert cpy.result[0].result != "epsilon" # just to make sure...
+        cpy.result[0].result = "epsilon"
+        assert cpy != self.unique_tree
+
 
 class TestErrorHandling:
     def test_error_flag_propagation(self):
@@ -106,4 +114,4 @@ class TestErrorHandling:
 
 if __name__ == "__main__":
     from run import runner
-    runner("TestNode", globals())
+    runner("", globals())

test/test_toolkit.py

@@ -26,7 +26,7 @@ import sys
 from DHParser.toolkit import load_if_file
 
 class TestToolkit:
-    filename = "tmp/test.py"
+    filename = "tmp/test.py" if os.path.isdir('tmp') else "test/tmp/test.py"
     code1 = "x = 46"
     code2 = "def f():\n    return 46"