Commit 9ef2e918 authored by Eckhart Arnold's avatar Eckhart Arnold

bug fixes; started code for systematic testing

parent c9fe392a
...@@ -16,6 +16,6 @@ testdata/*.pdf ...@@ -16,6 +16,6 @@ testdata/*.pdf
DEBUG* DEBUG*
LOGS/ LOGS/
external_resources/ external_resources/
tmp/ tmp/*
build/ build/
dist/ dist/
...@@ -396,4 +396,3 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"): ...@@ -396,4 +396,3 @@ def compile_on_disk(source_file, compiler_suite="", extension=".xml"):
return [] return []
...@@ -140,7 +140,7 @@ EBNF_transformation_table = { ...@@ -140,7 +140,7 @@ EBNF_transformation_table = {
(TOKEN_KEYWORD, WHITESPACE_KEYWORD): (TOKEN_KEYWORD, WHITESPACE_KEYWORD):
[remove_expendables, reduce_single_child], [remove_expendables, reduce_single_child],
"list_": "list_":
[partial(remove_tokens, tokens={','})], [flatten, partial(remove_tokens, tokens={','})],
"": "":
[remove_expendables, replace_by_single_child] [remove_expendables, replace_by_single_child]
} }
...@@ -188,8 +188,8 @@ class EBNFCompiler(CompilerBase): ...@@ -188,8 +188,8 @@ class EBNFCompiler(CompilerBase):
def _reset(self): def _reset(self):
self.rules = set() self.rules = set()
self.symbols = set()
self.variables = set() self.variables = set()
self.symbol_nodes = []
self.definition_names = [] self.definition_names = []
self.recursive = set() self.recursive = set()
self.root = "" self.root = ""
...@@ -302,10 +302,10 @@ class EBNFCompiler(CompilerBase): ...@@ -302,10 +302,10 @@ class EBNFCompiler(CompilerBase):
declarations += [symbol + '.set(' + statement + ')'] declarations += [symbol + '.set(' + statement + ')']
else: else:
declarations += [symbol + ' = ' + statement] declarations += [symbol + ' = ' + statement]
for nd in self.symbols: for nd in self.symbol_nodes:
if nd.result not in self.rules: if nd.result not in self.rules:
nd.add_error("Missing production for symbol '%s'" % nd.result) nd.add_error("Missing production for symbol '%s'" % nd.result)
if self.root and 'root__' not in self.symbols: if self.root and 'root__' not in self.rules:
declarations.append('root__ = ' + self.root) declarations.append('root__ = ' + self.root)
declarations.append('') declarations.append('')
return '\n '.join(declarations) return '\n '.join(declarations)
...@@ -443,7 +443,6 @@ class EBNFCompiler(CompilerBase): ...@@ -443,7 +443,6 @@ class EBNFCompiler(CompilerBase):
if prefix in {'::', ':'}: if prefix in {'::', ':'}:
assert len(node.result) == 2 assert len(node.result) == 2
arg = node.result[-1] arg = node.result[-1]
argstr = str(arg)
if arg.parser.name != 'symbol': if arg.parser.name != 'symbol':
node.add_error(('Retrieve Operator "%s" requires a symbol, ' node.add_error(('Retrieve Operator "%s" requires a symbol, '
'and not a %s.') % (prefix, str(arg.parser))) 'and not a %s.') % (prefix, str(arg.parser)))
...@@ -487,7 +486,7 @@ class EBNFCompiler(CompilerBase): ...@@ -487,7 +486,7 @@ class EBNFCompiler(CompilerBase):
if node.result in self.directives['tokens']: if node.result in self.directives['tokens']:
return 'ScannerToken("' + node.result + '")' return 'ScannerToken("' + node.result + '")'
else: else:
self.symbols.add(node) self.symbol_nodes.append(node)
if node.result in self.rules: if node.result in self.rules:
self.recursive.add(node.result) self.recursive.add(node.result)
return node.result return node.result
......
...@@ -55,11 +55,10 @@ try: ...@@ -55,11 +55,10 @@ try:
import regex as re import regex as re
except ImportError: except ImportError:
import re import re
import sys
from .toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name, smart_list from .toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name
from .syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \ from .syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \
traverse mock_syntax_tree
from DHParser.toolkit import load_if_file, error_messages from DHParser.toolkit import load_if_file, error_messages
__all__ = ['HistoryRecord', __all__ = ['HistoryRecord',
...@@ -1070,3 +1069,14 @@ def full_compilation(source, scanner, parser, transform, compiler): ...@@ -1070,3 +1069,14 @@ def full_compilation(source, scanner, parser, transform, compiler):
messages = error_messages(source_text, errors) messages = error_messages(source_text, errors)
return result, messages, syntax_tree return result, messages, syntax_tree
def test_grammar(test_suite, parse_function, transform):
for parser_name, tests in test_suite.items():
assert set(tests.keys()).issubset({'match', 'fail', 'ast', 'cst'})
for test_name, test_code in tests['match'].items():
cst = parse_function(test_code, parser_name)
if not cst.error_flag:
yield "Test %s for parser %s did not match" % (test_name, parser_name)
if "cst" in tests:
if tests["cst"][test_name] != mock_syntax_tree(cst):
pass # TO BE CONTINUED
\ No newline at end of file
...@@ -17,6 +17,7 @@ implied. See the License for the specific language governing ...@@ -17,6 +17,7 @@ implied. See the License for the specific language governing
permissions and limitations under the License. permissions and limitations under the License.
""" """
import copy
import itertools import itertools
import os import os
from functools import partial from functools import partial
...@@ -166,9 +167,18 @@ class Node: ...@@ -166,9 +167,18 @@ class Node:
def __eq__(self, other): def __eq__(self, other):
return str(self.parser) == str(other.parser) and self.result == other.result return str(self.parser) == str(other.parser) and self.result == other.result
def __hash__(self):
return hash((str(self.parser), ))
def __deepcopy__(self, memodict={}):
result = copy.deepcopy(self.result)
other = Node(self.parser, result)
other._pos = self._pos
return other
@property @property
def tag_name(self): def tag_name(self):
return str(self.parser) return self.parser.name or self.parser.__class__.__name__
# ONLY FOR DEBUGGING: return self.parser.name + ':' + self.parser.__class__.__name__ # ONLY FOR DEBUGGING: return self.parser.name + ':' + self.parser.__class__.__name__
@property @property
......
...@@ -54,7 +54,7 @@ __all__ = ['logging_on', ...@@ -54,7 +54,7 @@ __all__ = ['logging_on',
'sane_parser_name'] 'sane_parser_name']
LOGGING: str = "LOGS" # LOGGING = "" turns logging off! LOGGING: str = "" # "LOGS" # LOGGING = "" turns logging off!
def logging_on(log_subdir="LOGS"): def logging_on(log_subdir="LOGS"):
......
...@@ -740,7 +740,7 @@ class Parser(metaclass=ParserMetaClass): ...@@ -740,7 +740,7 @@ class Parser(metaclass=ParserMetaClass):
def apply(self, func): def apply(self, func):
"""Applies function `func(parser)` recursively to this parser and all """Applies function `func(parser)` recursively to this parser and all
descendendants of the tree of parsers. The same function can never descendants of the tree of parsers. The same function can never
be applied twice between calls of the ``reset()``-method! be applied twice between calls of the ``reset()``-method!
""" """
if func in self.cycle_detection: if func in self.cycle_detection:
......
...@@ -31,11 +31,12 @@ class TestCompilerGeneration: ...@@ -31,11 +31,12 @@ class TestCompilerGeneration:
word = /\w+/ word = /\w+/
WSPC = /\s+/ WSPC = /\s+/
""" """
tmp = 'tmp/' if os.path.isdir('tmp') else ('test/tmp/')
trivial_text = """Es war ein König in Thule.""" trivial_text = """Es war ein König in Thule."""
grammar_name = "tmp/TestCompilerGeneration.ebnf" grammar_name = tmp + "TestCompilerGeneration.ebnf"
compiler_name = "tmp/TestCompilerGeneration_compiler.py" compiler_name = tmp + "TestCompilerGeneration_compiler.py"
text_name = "tmp/TestCompilerGeneration_text.txt" text_name = tmp + "TestCompilerGeneration_text.txt"
result_name = "tmp/TestCompilerGeneration_text.xml" result_name = tmp + "TestCompilerGeneration_text.xml"
def setup(self): def setup(self):
with open(self.grammar_name, "w") as f: with open(self.grammar_name, "w") as f:
...@@ -71,7 +72,7 @@ class TestCompilerGeneration: ...@@ -71,7 +72,7 @@ class TestCompilerGeneration:
result = run_compiler(self.trivial_text, self.compiler_name) result = run_compiler(self.trivial_text, self.compiler_name)
assert output == result.as_xml(), str(result) assert output == result.as_xml(), str(result)
sys.path.append('tmp') sys.path.append(self.tmp)
from TestCompilerGeneration_compiler import compile_TestCompilerGeneration from TestCompilerGeneration_compiler import compile_TestCompilerGeneration
result, errors, ast = compile_TestCompilerGeneration(self.trivial_text) result, errors, ast = compile_TestCompilerGeneration(self.trivial_text)
......
...@@ -81,9 +81,21 @@ class TestDirectives: ...@@ -81,9 +81,21 @@ class TestDirectives:
class TestEBNFParser: class TestEBNFParser:
test_json = [ test_json = {
"" "list_": {
] "match": {
1: "hund",
2: "hund, katze,maus",
3: "hund , katze"
},
"fail": {
1: "123",
2: '"literal"',
3: "/regexp/"
}
}
}
def setup(self): def setup(self):
self.EBNF = EBNFGrammar() self.EBNF = EBNFGrammar()
......
...@@ -105,4 +105,4 @@ class TestRegex: ...@@ -105,4 +105,4 @@ class TestRegex:
if __name__ == "__main__": if __name__ == "__main__":
from run import runner from run import runner
runner("", globals()) runner("TestInfiLoopsAndRecursion", globals())
...@@ -19,6 +19,7 @@ See the License for the specific language governing permissions and ...@@ -19,6 +19,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
""" """
import copy
import os import os
import sys import sys
sys.path.append(os.path.abspath('../../')) sys.path.append(os.path.abspath('../../'))
...@@ -90,6 +91,13 @@ class TestNode: ...@@ -90,6 +91,13 @@ class TestNode:
assert self.recurr_tree != self.unique_tree assert self.recurr_tree != self.unique_tree
assert mock_syntax_tree('(a (b c))') != mock_syntax_tree('(a (b d))') assert mock_syntax_tree('(a (b c))') != mock_syntax_tree('(a (b d))')
def test_copy(self):
cpy = copy.deepcopy(self.unique_tree)
assert cpy == self.unique_tree
assert cpy.result[0].result != "epsilon" # just to make sure...
cpy.result[0].result = "epsilon"
assert cpy != self.unique_tree
class TestErrorHandling: class TestErrorHandling:
def test_error_flag_propagation(self): def test_error_flag_propagation(self):
...@@ -106,4 +114,4 @@ class TestErrorHandling: ...@@ -106,4 +114,4 @@ class TestErrorHandling:
if __name__ == "__main__": if __name__ == "__main__":
from run import runner from run import runner
runner("TestNode", globals()) runner("", globals())
...@@ -26,7 +26,7 @@ import sys ...@@ -26,7 +26,7 @@ import sys
from DHParser.toolkit import load_if_file from DHParser.toolkit import load_if_file
class TestToolkit: class TestToolkit:
filename = "tmp/test.py" filename = "tmp/test.py" if os.path.isdir('tmp') else "test/tmp/test.py"
code1 = "x = 46" code1 = "x = 46"
code2 = "def f():\n return 46" code2 = "def f():\n return 46"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment