- bug fixes; further unit tests

DHParser/DSLsupport.py

@@ -30,7 +30,7 @@ except ImportError:
 
 from .__init__ import __version__
 from .EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler
-from .toolkit import IS_LOGGING, load_if_file, is_python_code, md5, compile_python_object
+from .toolkit import load_if_file, is_python_code, md5, compile_python_object
 from .parsercombinators import GrammarBase, CompilerBase, full_compilation, nil_scanner
 from .syntaxtree import Node
 
@@ -189,7 +189,8 @@ def load_compiler_suite(compiler_suite):
 def compileDSL(text_or_file, dsl_grammar, ast_pipeline, compiler,
                scanner=nil_scanner):
     """Compiles a text in a domain specific language (DSL) with an
-    EBNF-specified grammar. Returns the compiled text.
+    EBNF-specified grammar. Returns the compiled text or raises a
+    compilation error.
     """
     assert isinstance(text_or_file, str)
     assert isinstance(compiler, CompilerBase)
@@ -201,7 +202,7 @@ def compileDSL(text_or_file, dsl_grammar, ast_pipeline, compiler,
     return result
 
 
-def compileEBNF(ebnf_src, ebnf_grammar_obj=None):
+def compileEBNF(ebnf_src, ebnf_grammar_obj=None, source_only=False):
     """Compiles an EBNF source file into a Grammar class
 
     Args:
@@ -211,13 +212,16 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None):
             DHParser.EBNFcompiler.EBNFGrammar object. This can speed
             up compilation, because no new EBNFGrammar object needs to
             be instantiated.
+        source_only (bool):  If True, the source code of the Grammar
+            class is returned instead of the class itself.
     Returns:
         A Grammar class that can be instantiated for parsing a text
         which conforms to the language defined by ``ebnf_src``
     """
     grammar = ebnf_grammar_obj or EBNFGrammar()
     grammar_src = compileDSL(ebnf_src, grammar, EBNF_ASTPipeline, EBNFCompiler())
-    return compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
+    return grammar_src if source_only else \
+        compile_python_object(DHPARSER_IMPORTS + grammar_src, '\w*Grammar$')
 
 
 def run_compiler(source_file, compiler_suite="", extension=".xml"):

DHParser/parsercombinators.py

@@ -61,8 +61,8 @@ except ImportError:
 
 from .toolkit import IS_LOGGING, LOGS_DIR, escape_re, sane_parser_name, sequence
 from .syntaxtree import WHITESPACE_KEYWORD, TOKEN_KEYWORD, ZOMBIE_PARSER, Node, \
-    error_messages, traverse
-
+    traverse
+from DHParser.toolkit import error_messages
 
 __all__ = ['HistoryRecord',
            'Parser',
@@ -286,13 +286,13 @@ class GrammarBase:
         if self.wspL__:
             self.wsp_left_parser__ = RegExp(self.wspL__, WHITESPACE_KEYWORD)
             self.wsp_left_parser__.grammar = self
-            self.all_parsers.add(self.wsp_left_parser__)
+            self.all_parsers.add(self.wsp_left_parser__)  # don't you forget about me...
         else:
             self.wsp_left_parser__ = ZOMBIE_PARSER
         if self.wspR__:
             self.wsp_right_parser__ = RegExp(self.wspR__, WHITESPACE_KEYWORD)
             self.wsp_right_parser__.grammar = self
-            self.all_parsers.add(self.wsp_right_parser__)
+            self.all_parsers.add(self.wsp_right_parser__)  # don't you forget about me...
         else:
             self.wsp_right_parser__ = ZOMBIE_PARSER
         self.root__.apply(self._add_parser)
@@ -309,7 +309,8 @@ class GrammarBase:
         """Adds the copy of the classes parser object to this
         particular instance of GrammarBase.
         """
-        setattr(self, parser.name, parser)
+        if parser.name:
+            setattr(self, parser.name, parser)
         self.all_parsers.add(parser)
         parser.grammar = self
 
@@ -469,7 +470,7 @@ class RegExp(Parser):
             regexp = self.regexp.pattern
         duplicate = RegExp(regexp, self.name)
         duplicate.name = self.name  # this ist needed!!!!
-        duplicate.regexp = self.regexp
+        # duplicate.regexp = self.regexp
         duplicate.grammar = self.grammar
         duplicate.visited = copy.deepcopy(self.visited, memo)
         duplicate.recursion_counter = copy.deepcopy(self.recursion_counter, memo)
@@ -818,7 +819,7 @@ class Forward(Parser):
 
     def set(self, parser):
         assert isinstance(parser, Parser)
-        self.name = parser.name  # redundant, because of constructor of GrammarBase
+        self.name = parser.name  # redundant, see GrammarBase-constructor
         self.parser = parser
 
     def apply(self, func):

DHParser/syntaxtree.py

@@ -19,7 +19,6 @@ implied.  See the License for the specific language governing
 permissions and limitations under the License.
 """
 
-import collections
 import itertools
 import os
 from functools import partial
@@ -37,7 +36,6 @@ __all__ = ['WHITESPACE_KEYWORD',
            'ZOMBIE_PARSER',
            'Error',
            'Node',
-           'error_messages',
            'compact_sexpr',
            'traverse',
            'no_operation',
@@ -356,18 +354,6 @@ class Node:
         return nav(path.split('/'))
 
 
-def error_messages(text, errors):
-    """
-    Converts the list of ``errors`` collected from the root node of the
-    parse tree of `text` into a human readable (and IDE or editor
-    parsable text) with line an column numbers. Error messages are
-    separated by an empty line.
-    """
-    return "\n\n".join("line: %i, column: %i, error: %s" %
-                       (*line_col(text, err.pos), err.msg)
-                       for err in sorted(list(errors)))
-
-
 ########################################################################
 #
 # syntax tree transformation functions

DHParser/toolkit.py

@@ -46,6 +46,7 @@ __all__ = ['logging_on',
            'IS_LOGGING',
            'LOGS_DIR',
            'line_col',
+           'error_messages',
            'escape_re',
            'load_if_file',
            'is_python_code',
@@ -118,6 +119,18 @@ def line_col(text, pos):
     return line, column
 
 
+def error_messages(text, errors):
+    """
+    Converts the list of ``errors`` collected from the root node of the
+    parse tree of `text` into a human readable (and IDE or editor
+    parsable text) with line an column numbers. Error messages are
+    separated by an empty line.
+    """
+    return "\n\n".join("line: %i, column: %i, error: %s" %
+                       (*line_col(text, err.pos), err.msg)
+                       for err in sorted(list(errors)))
+
+
 def compact_sexpr(s):
     """Returns S-expression ``s`` as a one liner without unnecessary
     whitespace.
@@ -225,3 +238,4 @@ def compile_python_object(python_src, catch_obj_regex):
         raise AssertionError("Ambigous matches for %s : %s" %
                              (str(catch_obj_regex), str(matches)))
     return namespace[matches[0]] if matches else None
+

tests/test_EBNFcompiler.py

@@ -44,10 +44,8 @@ class TestDirectives:
         parser = MinilangParser()
         assert parser
         syntax_tree = parser.parse("3 + 4 * 12")
-        parser.log_parsing_history('WSP1')
         assert not syntax_tree.collect_errors()
         syntax_tree = parser.parse("3 + 4 \n * 12")
-        parser.log_parsing_history('WSP2')
         assert not syntax_tree.collect_errors()
 
     def test_whitespace_standard(self):

tests/test_parsercombinators.py

@@ -25,7 +25,10 @@ import os
 import sys
 
 sys.path.append(os.path.abspath('../../'))
-from DHParser.DSLsupport import compileEBNF
+from DHParser.toolkit import compile_python_object
+from DHParser.parsercombinators import full_compilation
+from DHParser.EBNFcompiler import EBNFGrammar, EBNF_ASTPipeline, EBNFCompiler
+from DHParser.DSLsupport import compileEBNF, DHPARSER_IMPORTS
 
 WRITE_LOGS = True
 
@@ -33,10 +36,10 @@ WRITE_LOGS = True
 class TestLeftRecursion:
     mini_language1 = """
         @ whitespace = linefeed
-        formula = expr "." 
+        formula = [ //~ ] expr
         expr = expr ("+"|"-") term | term
         term = term ("*"|"/") factor | factor
-        factor = /[0-9]+/
+        factor = /[0-9]+/~
         # example:  "5 + 3 * 4"
         """
 
@@ -47,17 +50,37 @@ class TestLeftRecursion:
         assert self.minilang_parser1
 
     def test_direct_left_recursion(self):
-        syntax_tree = self.minilang_parser1.parse("5 + 3 * 4")
+        snippet = "5 + 3 * 4"
+        syntax_tree = self.minilang_parser1.parse(snippet)
         assert not syntax_tree.collect_errors()
+        assert snippet == str(syntax_tree)
         if WRITE_LOGS:
             syntax_tree.log("test_LeftRecursion_direct", '.cst')
             self.minilang_parser1.log_parsing_history("test_LeftRecursion_direct")
 
-
     def test_indirect_left_recursion(self):
         pass
 
 
+class TestRegex:
+    def test_multilineRegex(self):
+        mlregex = r"""
+        regex =  /\w+    # one or more alphabetical characters including the underscore
+                  [+]    # followed by a plus sign
+                  \w*    # possibly followed by more alpha chracters/
+        """
+        result, messages, syntax_tree = full_compilation(mlregex, EBNFGrammar(), EBNF_ASTPipeline,
+                                                         EBNFCompiler('MultilineRegexTest'))
+        assert result is not None, messages
+        assert not messages
+        parser = compile_python_object(DHPARSER_IMPORTS + result, '\w+Grammar$')()
+        node, rest = parser.regex('abc+def')
+        assert rest == ''
+        assert node.parser.name == "regex"
+        assert str(node) == 'abc+def'
+
+
 if __name__ == "__main__":
     from run import run_tests
-    run_tests("TestLeftRecursion", globals())
\ No newline at end of file
+
+    run_tests("TestRegex", globals())