Commit 318987ab authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

- bug fixes; further unit tests

parent e3a60af3
......@@ -30,7 +30,7 @@ except ImportError:
from .__init__ import __version__
from .EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler
from .toolkit import IS_LOGGING, load_if_file, is_python_code, md5, compile_python_object
from .toolkit import load_if_file, is_python_code, md5, compile_python_object
from .parsercombinators import GrammarBase, CompilerBase, full_compilation, nil_scanner
from .syntaxtree import Node
......@@ -189,7 +189,8 @@ def load_compiler_suite(compiler_suite):
def compileDSL(text_or_file, dsl_grammar, ast_pipeline, compiler,
scanner=nil_scanner):
"""Compiles a text in a domain specific language (DSL) with an
EBNF-specified grammar. Returns the compiled text.
EBNF-specified grammar. Returns the compiled text or raises a
compilation error.
"""
assert isinstance(text_or_file, str)
assert isinstance(compiler, CompilerBase)
......@@ -201,7 +202,7 @@ def compileDSL(text_or_file, dsl_grammar, ast_pipeline, compiler,
return result
def compileEBNF(ebnf_src, ebnf_grammar_obj=None):
def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
"""Compiles an EBNF source file into a Grammar class
Args:
......@@ -211,13 +212,16 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None):
DHParser.EBNFcompiler.EBNFGrammar object. This can speed
up compilation, because no new EBNFGrammar object needs to
be instantiated.
source_only (bool): If True, the source code of the Grammar
class is returned instead of the class itself.
Returns:
A Grammar class that can be instantiated for parsing a text
which conforms to the language defined by ``ebnf_src``
"""
grammar = ebnf_grammar_obj or EBNFGrammar()
grammar_src = compileDSL(ebnf_src, grammar, EBNF_ASTPipeline, EBNFCompiler())
return compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
return grammar_src if source_only else \
compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
def run_compiler(source_file, compiler_suite="", extension=".xml"):
......
......@@ -61,8 +61,8 @@ except ImportError:
from .toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name, sequence
from .syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \
error_messages, traverse
traverse
from DHParser.toolkit import error_messages
__all__ = ['HistoryRecord',
'Parser',
......@@ -286,13 +286,13 @@ class GrammarBase:
if self.wspL__:
self.wsp_left_parser__ = RegExp(self.wspL__, WHITESPACE_KEYWORD)
self.wsp_left_parser__.grammar = self
self.all_parsers.add(self.wsp_left_parser__)
self.all_parsers.add(self.wsp_left_parser__) # don't you forget about me...
else:
self.wsp_left_parser__ = ZOMBIE_PARSER
if self.wspR__:
self.wsp_right_parser__ = RegExp(self.wspR__, WHITESPACE_KEYWORD)
self.wsp_right_parser__.grammar = self
self.all_parsers.add(self.wsp_right_parser__)
self.all_parsers.add(self.wsp_right_parser__) # don't you forget about me...
else:
self.wsp_right_parser__ = ZOMBIE_PARSER
self.root__.apply(self._add_parser)
......@@ -309,7 +309,8 @@ class GrammarBase:
"""Adds the copy of the classes parser object to this
particular instance of GrammarBase.
"""
setattr(self, parser.name, parser)
if parser.name:
setattr(self, parser.name, parser)
self.all_parsers.add(parser)
parser.grammar = self
......@@ -469,7 +470,7 @@ class RegExp(Parser):
regexp = self.regexp.pattern
duplicate = RegExp(regexp, self.name)
duplicate.name = self.name # this ist needed!!!!
duplicate.regexp = self.regexp
# duplicate.regexp = self.regexp
duplicate.grammar = self.grammar
duplicate.visited = copy.deepcopy(self.visited, memo)
duplicate.recursion_counter = copy.deepcopy(self.recursion_counter, memo)
......@@ -818,7 +819,7 @@ class Forward(Parser):
def set(self, parser):
assert isinstance(parser, Parser)
self.name = parser.name # redundant, because of constructor of GrammarBase
self.name = parser.name # redundant, see GrammarBase-constructor
self.parser = parser
def apply(self, func):
......
......@@ -19,7 +19,6 @@ implied. See the License for the specific language governing
permissions and limitations under the License.
"""
import collections
import itertools
import os
from functools import partial
......@@ -37,7 +36,6 @@ __all__ = ['WHITESPACE_KEYWORD',
'ZOMBIE_PARSER',
'Error',
'Node',
'error_messages',
'compact_sexpr',
'traverse',
'no_operation',
......@@ -356,18 +354,6 @@ class Node:
return nav(path.split('/'))
def error_messages(text, errors):
"""
Converts the list of ``errors`` collected from the root node of the
parse tree of `text` into a human readable (and IDE or editor
parsable text) with line an column numbers. Error messages are
separated by an empty line.
"""
return "\n\n".join("line: %i, column: %i, error: %s" %
(*line_col(text, err.pos), err.msg)
for err in sorted(list(errors)))
########################################################################
#
# syntax tree transformation functions
......
......@@ -46,6 +46,7 @@ __all__ = ['logging_on',
'IS_LOGGING',
'LOGS_DIR',
'line_col',
'error_messages',
'escape_re',
'load_if_file',
'is_python_code',
......@@ -118,6 +119,18 @@ def line_col(text, pos):
return line, column
def error_messages(text, errors):
"""
Converts the list of ``errors`` collected from the root node of the
parse tree of `text` into a human readable (and IDE or editor
parsable text) with line an column numbers. Error messages are
separated by an empty line.
"""
return "\n\n".join("line: %i, column: %i, error: %s" %
(*line_col(text, err.pos), err.msg)
for err in sorted(list(errors)))
def compact_sexpr(s):
"""Returns S-expression ``s`` as a one liner without unnecessary
whitespace.
......@@ -225,3 +238,4 @@ def compile_python_object(python_src, catch_obj_regex):
raise AssertionError("Ambigous matches for %s : %s" %
(str(catch_obj_regex), str(matches)))
return namespace[matches[0]] if matches else None
......@@ -44,10 +44,8 @@ class TestDirectives:
parser = MinilangParser()
assert parser
syntax_tree = parser.parse("3 + 4 * 12")
parser.log_parsing_history('WSP1')
assert not syntax_tree.collect_errors()
syntax_tree = parser.parse("3 + 4 \n * 12")
parser.log_parsing_history('WSP2')
assert not syntax_tree.collect_errors()
def test_whitespace_standard(self):
......
......@@ -25,7 +25,10 @@ import os
import sys
sys.path.append(os.path.abspath('../../'))
from DHParser.DSLsupport import compileEBNF
from DHParser.toolkit import compile_python_object
from DHParser.parsercombinators import full_compilation
from DHParser.EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler
from DHParser.DSLsupport import compileEBNF, DHPARSER_IMPORTS
WRITE_LOGS = True
......@@ -33,10 +36,10 @@ WRITE_LOGS = True
class TestLeftRecursion:
mini_language1 = """
@ whitespace = linefeed
formula = expr "."
formula = [ //~ ] expr
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/
factor = /[0-9]+/~
# example: "5 + 3 * 4"
"""
......@@ -47,17 +50,37 @@ class TestLeftRecursion:
assert self.minilang_parser1
def test_direct_left_recursion(self):
syntax_tree = self.minilang_parser1.parse("5 + 3 * 4")
snippet = "5 + 3 * 4"
syntax_tree = self.minilang_parser1.parse(snippet)
assert not syntax_tree.collect_errors()
assert snippet == str(syntax_tree)
if WRITE_LOGS:
syntax_tree.log("test_LeftRecursion_direct", '.cst')
self.minilang_parser1.log_parsing_history("test_LeftRecursion_direct")
def test_indirect_left_recursion(self):
pass
class TestRegex:
def test_multilineRegex(self):
mlregex = r"""
regex = /\w+ # one or more alphabetical characters including the underscore
[+] # followed by a plus sign
\w* # possibly followed by more alpha chracters/
"""
result, messages, syntax_tree = full_compilation(mlregex, EBNFGrammar(), EBNF_ASTPipeline,
EBNFCompiler('MultilineRegexTest'))
assert result is not None, messages
assert not messages
parser = compile_python_object(DHPARSER_IMPORTS + result, '\w+Grammar$')()
node, rest = parser.regex('abc+def')
assert rest == ''
assert node.parser.name == "regex"
assert str(node) == 'abc+def'
if __name__ == "__main__":
from run import run_tests
run_tests("TestLeftRecursion", globals())
\ No newline at end of file
run_tests("TestRegex", globals())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment