The expiration time for new job artifacts in CI/CD pipelines is now 30 days (GitLab default). Previously generated artifacts in already completed jobs will not be affected by the change. The latest artifacts for all jobs in the latest successful pipelines will be kept. More information: https://gitlab.lrz.de/help/user/admin_area/settings/continuous_integration.html#default-artifacts-expiration

Commit 318987ab authored by Eckhart Arnold's avatar Eckhart Arnold
Browse files

- bug fixes; further unit tests

parent e3a60af3
......@@ -30,7 +30,7 @@ except ImportError:
from .__init__ import __version__
from .EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler
from .toolkit import IS_LOGGING, load_if_file, is_python_code, md5, compile_python_object
from .toolkit import load_if_file, is_python_code, md5, compile_python_object
from .parsercombinators import GrammarBase, CompilerBase, full_compilation, nil_scanner
from .syntaxtree import Node
......@@ -189,7 +189,8 @@ def load_compiler_suite(compiler_suite):
def compileDSL(text_or_file, dsl_grammar, ast_pipeline, compiler,
scanner=nil_scanner):
"""Compiles a text in a domain specific language (DSL) with an
EBNF-specified grammar. Returns the compiled text.
EBNF-specified grammar. Returns the compiled text or raises a
compilation error.
"""
assert isinstance(text_or_file, str)
assert isinstance(compiler, CompilerBase)
......@@ -201,7 +202,7 @@ def compileDSL(text_or_file, dsl_grammar, ast_pipeline, compiler,
return result
def compileEBNF(ebnf_src, ebnf_grammar_obj=None):
def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
"""Compiles an EBNF source file into a Grammar class
Args:
......@@ -211,13 +212,16 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None):
DHParser.EBNFcompiler.EBNFGrammar object. This can speed
up compilation, because no new EBNFGrammar object needs to
be instantiated.
source_only (bool): If True, the source code of the Grammar
class is returned instead of the class itself.
Returns:
A Grammar class that can be instantiated for parsing a text
which conforms to the language defined by ``ebnf_src``
"""
grammar = ebnf_grammar_obj or EBNFGrammar()
grammar_src = compileDSL(ebnf_src, grammar, EBNF_ASTPipeline, EBNFCompiler())
return compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
return grammar_src if source_only else \
compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
def run_compiler(source_file, compiler_suite="", extension=".xml"):
......
......@@ -61,8 +61,8 @@ except ImportError:
from .toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name, sequence
from .syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \
error_messages, traverse
traverse
from DHParser.toolkit import error_messages
__all__ = ['HistoryRecord',
'Parser',
......@@ -286,13 +286,13 @@ class GrammarBase:
if self.wspL__:
self.wsp_left_parser__ = RegExp(self.wspL__, WHITESPACE_KEYWORD)
self.wsp_left_parser__.grammar = self
self.all_parsers.add(self.wsp_left_parser__)
self.all_parsers.add(self.wsp_left_parser__) # don't you forget about me...
else:
self.wsp_left_parser__ = ZOMBIE_PARSER
if self.wspR__:
self.wsp_right_parser__ = RegExp(self.wspR__, WHITESPACE_KEYWORD)
self.wsp_right_parser__.grammar = self
self.all_parsers.add(self.wsp_right_parser__)
self.all_parsers.add(self.wsp_right_parser__) # don't you forget about me...
else:
self.wsp_right_parser__ = ZOMBIE_PARSER
self.root__.apply(self._add_parser)
......@@ -309,7 +309,8 @@ class GrammarBase:
"""Adds the copy of the classes parser object to this
particular instance of GrammarBase.
"""
setattr(self, parser.name, parser)
if parser.name:
setattr(self, parser.name, parser)
self.all_parsers.add(parser)
parser.grammar = self
......@@ -469,7 +470,7 @@ class RegExp(Parser):
regexp = self.regexp.pattern
duplicate = RegExp(regexp, self.name)
duplicate.name = self.name # this ist needed!!!!
duplicate.regexp = self.regexp
# duplicate.regexp = self.regexp
duplicate.grammar = self.grammar
duplicate.visited = copy.deepcopy(self.visited, memo)
duplicate.recursion_counter = copy.deepcopy(self.recursion_counter, memo)
......@@ -818,7 +819,7 @@ class Forward(Parser):
def set(self, parser):
assert isinstance(parser, Parser)
self.name = parser.name # redundant, because of constructor of GrammarBase
self.name = parser.name # redundant, see GrammarBase-constructor
self.parser = parser
def apply(self, func):
......
......@@ -19,7 +19,6 @@ implied. See the License for the specific language governing
permissions and limitations under the License.
"""
import collections
import itertools
import os
from functools import partial
......@@ -37,7 +36,6 @@ __all__ = ['WHITESPACE_KEYWORD',
'ZOMBIE_PARSER',
'Error',
'Node',
'error_messages',
'compact_sexpr',
'traverse',
'no_operation',
......@@ -356,18 +354,6 @@ class Node:
return nav(path.split('/'))
def error_messages(text, errors):
"""
Converts the list of ``errors`` collected from the root node of the
parse tree of `text` into a human readable (and IDE or editor
parsable text) with line an column numbers. Error messages are
separated by an empty line.
"""
return "\n\n".join("line: %i, column: %i, error: %s" %
(*line_col(text, err.pos), err.msg)
for err in sorted(list(errors)))
########################################################################
#
# syntax tree transformation functions
......
......@@ -46,6 +46,7 @@ __all__ = ['logging_on',
'IS_LOGGING',
'LOGS_DIR',
'line_col',
'error_messages',
'escape_re',
'load_if_file',
'is_python_code',
......@@ -118,6 +119,18 @@ def line_col(text, pos):
return line, column
def error_messages(text, errors):
"""
Converts the list of ``errors`` collected from the root node of the
parse tree of `text` into a human readable (and IDE or editor
parsable text) with line an column numbers. Error messages are
separated by an empty line.
"""
return "\n\n".join("line: %i, column: %i, error: %s" %
(*line_col(text, err.pos), err.msg)
for err in sorted(list(errors)))
def compact_sexpr(s):
"""Returns S-expression ``s`` as a one liner without unnecessary
whitespace.
......@@ -225,3 +238,4 @@ def compile_python_object(python_src, catch_obj_regex):
raise AssertionError("Ambigous matches for %s : %s" %
(str(catch_obj_regex), str(matches)))
return namespace[matches[0]] if matches else None
......@@ -44,10 +44,8 @@ class TestDirectives:
parser = MinilangParser()
assert parser
syntax_tree = parser.parse("3 + 4 * 12")
parser.log_parsing_history('WSP1')
assert not syntax_tree.collect_errors()
syntax_tree = parser.parse("3 + 4 \n * 12")
parser.log_parsing_history('WSP2')
assert not syntax_tree.collect_errors()
def test_whitespace_standard(self):
......
......@@ -25,7 +25,10 @@ import os
import sys
sys.path.append(os.path.abspath('../../'))
from DHParser.DSLsupport import compileEBNF
from DHParser.toolkit import compile_python_object
from DHParser.parsercombinators import full_compilation
from DHParser.EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler
from DHParser.DSLsupport import compileEBNF, DHPARSER_IMPORTS
WRITE_LOGS = True
......@@ -33,10 +36,10 @@ WRITE_LOGS = True
class TestLeftRecursion:
mini_language1 = """
@ whitespace = linefeed
formula = expr "."
formula = [ //~ ] expr
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/
factor = /[0-9]+/~
# example: "5 + 3 * 4"
"""
......@@ -47,17 +50,37 @@ class TestLeftRecursion:
assert self.minilang_parser1
def test_direct_left_recursion(self):
syntax_tree = self.minilang_parser1.parse("5 + 3 * 4")
snippet = "5 + 3 * 4"
syntax_tree = self.minilang_parser1.parse(snippet)
assert not syntax_tree.collect_errors()
assert snippet == str(syntax_tree)
if WRITE_LOGS:
syntax_tree.log("test_LeftRecursion_direct", '.cst')
self.minilang_parser1.log_parsing_history("test_LeftRecursion_direct")
def test_indirect_left_recursion(self):
pass
class TestRegex:
def test_multilineRegex(self):
mlregex = r"""
regex = /\w+ # one or more alphabetical characters including the underscore
[+] # followed by a plus sign
\w* # possibly followed by more alpha chracters/
"""
result, messages, syntax_tree = full_compilation(mlregex, EBNFGrammar(), EBNF_ASTPipeline,
EBNFCompiler('MultilineRegexTest'))
assert result is not None, messages
assert not messages
parser = compile_python_object(DHPARSER_IMPORTS + result, '\w+Grammar$')()
node, rest = parser.regex('abc+def')
assert rest == ''
assert node.parser.name == "regex"
assert str(node) == 'abc+def'
if __name__ == "__main__":
from run import run_tests
run_tests("TestLeftRecursion", globals())
\ No newline at end of file
run_tests("TestRegex", globals())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment