- some bugfixes

DHParser/ebnf.py

@@ -786,6 +786,8 @@ class EBNFCompiler(Compiler):
                 self.symbols[symbol] = node  # remember first use of symbol
             if symbol in self.rules:
                 self.recursive.add(symbol)
+            if symbol in (EBNFCompiler.WHITESPACE_KEYWORD, EBNFCompiler.COMMENT_KEYWORD):
+                return "RegExp(%s)" % symbol
             return symbol
 
 

DHParser/parser.py

@@ -197,9 +197,6 @@ def add_parser_guard(parser_func):
                 grammar.moving_forward__ = True
                 grammar.left_recursion_encountered__ = False
 
-            if grammar.history_tracking__:
-                grammar.call_stack__.append(parser)
-
             # if location has already been visited by the current parser,
             # return saved result
             if location in parser.visited:
@@ -210,6 +207,9 @@ def add_parser_guard(parser_func):
                 grammar.left_recursion_encountered__ = True
                 return None, text
 
+            if grammar.history_tracking__:
+                grammar.call_stack__.append(parser)
+
             parser.recursion_counter[location] += 1
 
             # run original __call__ method
@@ -1032,7 +1032,7 @@ class UnaryOperator(Parser):
     """
     def __init__(self, parser: Parser, name: str = '') -> None:
         super(UnaryOperator, self).__init__(name)
-        # assert isinstance(parser, Parser)
+        assert isinstance(parser, Parser), str(parser)
         self.parser = parser  # type: Parser
 
     def __deepcopy__(self, memo):
@@ -1057,7 +1057,7 @@ class NaryOperator(Parser):
     """
     def __init__(self, *parsers: Parser, name: str = '') -> None:
         super(NaryOperator, self).__init__(name)
-        # assert all([isinstance(parser, Parser) for parser in parsers]), str(parsers)
+        assert all([isinstance(parser, Parser) for parser in parsers]), str(parsers)
         self.parsers = parsers  # type: Tuple[Parser, ...]
 
     def __deepcopy__(self, memo):

examples/LaTeX/LaTeX.ebnf

@@ -76,10 +76,8 @@ tabular_config      = "{" /[lcr|]+/~ §"}"
 
 block_of_paragraphs = /{/~ sequence §/}/
 sequence            = { (paragraph | block_environment ) [PARSEP] }+
-
 paragraph           = { !blockcmd text_element //~ }+
-text_element       = command | text | block | inline_environment
-
+text_element        = text | block | inline_environment | command
 
 #### inline enivronments ####
 
@@ -146,7 +144,8 @@ TEXTCHUNK  = /[^\\%$&\{\}\[\]\s\n]+/        # some piece of text excluding white
                                             # linefeed and special characters
 LF         = !GAP /[ \t]*\n[ \t]*/          # linefeed but not an empty line
 LFF        = //~ -&LB [ WSPC ]              # at least one linefeed
-WSPC       = { ~/\s+/~ }+                   # arbitrary horizontal or vertical whitespace
+WSPC       = { COMMENT__ | /\s+/ }+
+# WSPC       = { /\s+/~ | ~/\s+/ }+           # arbitrary horizontal or vertical whitespace
 PARSEP     = { GAP }+                       # paragraph separator
 GAP        = /[ \t]*(?:\n[ \t]*)+\n/~       # at least one empty line, i.e.
                                             # [whitespace] linefeed [whitespace] linefeed

examples/LaTeX/LaTeXCompiler.py

@@ -7,26 +7,22 @@
 #######################################################################
 
 
-from functools import partial
 import os
 import sys
+from functools import partial
+
 try:
     import regex as re
 except ImportError:
     import re
-from DHParser import logging, is_filename, load_if_file, \
-    Grammar, Compiler, nil_preprocessor, \
-    Lookbehind, Lookahead, Alternative, Pop, Required, Token, Synonym, \
-    Optional, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, RE, Capture, \
+from DHParser import logging, is_filename, Grammar, Compiler, Lookbehind, Alternative, Pop, \
+    Required, Token, Synonym, \
+    Optional, NegativeLookbehind, OneOrMore, RegExp, Series, RE, Capture, \
     ZeroOrMore, Forward, NegativeLookahead, mixin_comment, compile_source, \
-    last_value, counterpart, accumulate, PreprocessorFunc, \
-    Node, TransformationFunc, MockParser, \
-    traverse, remove_children_if, merge_children, TRUE_CONDITION, is_anonymous, \
+    PreprocessorFunc, \
+    Node, TransformationFunc, traverse, remove_children_if, is_anonymous, \
     reduce_single_child, replace_by_single_child, remove_whitespace, \
-    remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \
-    is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
-    remove_parser, remove_content, remove_brackets, replace_parser, \
-    keep_children, is_one_of, has_content, apply_if, remove_first, remove_last
+    flatten, is_empty, collapse, replace_content, remove_brackets, is_one_of, remove_first
 
 
 #######################################################################
@@ -129,10 +125,8 @@ class LaTeXGrammar(Grammar):
     
     block_of_paragraphs = /{/~ sequence §/}/
     sequence            = { (paragraph | block_environment ) [PARSEP] }+
-    
     paragraph           = { !blockcmd text_element //~ }+
-    text_element       = command | text | block | inline_environment
-    
+    text_element        = text | block | inline_environment | command
     
     #### inline enivronments ####
     
@@ -199,7 +193,8 @@ class LaTeXGrammar(Grammar):
                                                 # linefeed and special characters
     LF         = !GAP /[ \t]*\n[ \t]*/          # linefeed but not an empty line
     LFF        = //~ -&LB [ WSPC ]              # at least one linefeed
-    WSPC       = { ~/\s+/~ }+                   # arbitrary horizontal or vertical whitespace
+    WSPC       = { COMMENT__ | /\s+/ }+
+    # WSPC       = { /\s+/~ | ~/\s+/ }+           # arbitrary horizontal or vertical whitespace
     PARSEP     = { GAP }+                       # paragraph separator
     GAP        = /[ \t]*(?:\n[ \t]*)+\n/~       # at least one empty line, i.e.
                                                 # [whitespace] linefeed [whitespace] linefeed
@@ -215,7 +210,7 @@ class LaTeXGrammar(Grammar):
     end_generic_block = Forward()
     paragraph = Forward()
     text_element = Forward()
-    source_hash__ = "773d8d68e38663befc9488f7e0cb60e4"
+    source_hash__ = "529c853d5829c3016605e4ee7ed69ddb"
     parser_initialization__ = "upon instantiation"
     COMMENT__ = r'%.*(?:\n|$)'
     WSP__ = mixin_comment(whitespace=r'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?', comment=r'%.*(?:\n|$)')
@@ -226,7 +221,7 @@ class LaTeXGrammar(Grammar):
     LB = RegExp('\\s*?\\n|$')
     GAP = RE('[ \\t]*(?:\\n[ \\t]*)+\\n')
     PARSEP = OneOrMore(GAP)
-    WSPC = OneOrMore(RE('\\s+', wL=WSP__))
+    WSPC = OneOrMore(Alternative(RegExp(COMMENT__), RegExp('\\s+')))
     LFF = Series(RE(''), Lookbehind(LB), Optional(WSPC))
     LF = Series(NegativeLookahead(GAP), RegExp('[ \\t]*\\n[ \\t]*'))
     TEXTCHUNK = RegExp('[^\\\\%$&\\{\\}\\[\\]\\s\\n]+')
@@ -256,7 +251,7 @@ class LaTeXGrammar(Grammar):
     generic_inline_env = Series(begin_inline_env, RE(''), paragraph, Required(end_inline_env))
     known_inline_env = Synonym(inline_math)
     inline_environment = Alternative(known_inline_env, generic_inline_env)
-    text_element.set(Alternative(command, text, block, inline_environment))
+    text_element.set(Alternative(text, block, inline_environment, command))
     paragraph.set(OneOrMore(Series(NegativeLookahead(blockcmd), text_element, RE(''))))
     sequence = OneOrMore(Series(Alternative(paragraph, block_environment), Optional(PARSEP)))
     block_of_paragraphs.set(Series(RE('{'), sequence, Required(RegExp('}'))))

test/test_dsl.py

@@ -77,7 +77,17 @@ class TestCompilerGeneration:
         for name in (self.grammar_name, self.compiler_name, self.text_name, self.result_name):
             if os.path.exists(name):
                 os.remove(name)
-        pass
+        if os.path.exists('LOGS'):
+            files = os.listdir('LOGS')
+            flag = False
+            for file in files:
+                if file.startswith('TestCompilerGenerationCompiler') or file == "info.txt":
+                    os.remove(os.path.join('LOGS', file))
+                else:
+                    flag = True
+            if not flag:
+                os.rmdir('LOGS')
+
 
     def test_load_compiler_suite(self):
         src = compileEBNF(self.trivial_lang, "Trivial")

test/test_ebnf.py

@@ -83,6 +83,35 @@ class TestDirectives:
         assert syntax_tree.collect_errors()
 
 
+class TestReservedSymbols:
+    def test_comment_usage(self):
+        lang = r"""
+        @comment = /#.*(?:\n|$)/
+        document = text [ COMMENT__ ]
+        text = /[^#]+/
+        """
+        parser = grammar_provider(lang)()
+
+    def test_whitespace(self):
+        lang = r"""
+        @whitespace = /\s*/
+        document = WSP__ { word WSP__ }
+        word = /\w+/ 
+        """
+        parser = grammar_provider(lang)()
+
+    def test_mixin(self):
+        lang = r"""
+        @comment = /#.*(?:\n|$)/
+        @whitespace = /\s*/
+        document = WSP__ { word WSP__ }
+        word = /\w+/ 
+        """
+        parser = grammar_provider(lang)()
+        result = parser("test # kommentar")
+        assert not result.error_flag, str(result.as_sxpr())
+
+
 class TestEBNFParser:
     cases = {
         "list_": {

test/test_testing.py

@@ -19,6 +19,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import os
+import re
 import sys
 from functools import partial
 
@@ -102,6 +104,18 @@ class TestGrammarTest:
         }
     }
 
+    def teardown(self):
+        if os.path.exists('REPORT'):
+            files = os.listdir('REPORT')
+            flag = False
+            for file in files:
+                if re.match(r'\d+\.md', file):
+                    os.remove(os.path.join('REPORT', file))
+                else:
+                    flag = True
+            if not flag:
+                os.rmdir('REPORT')
+
     def test_testing_grammar(self):
         parser_fac = grammar_provider(ARITHMETIC_EBNF)
         trans_fac = lambda : ARITHMETIC_EBNFTransform