further LaTeX.ebnf tests

DHParser/transform.py

@@ -432,9 +432,9 @@ def remove_tokens(node, tokens: AbstractSet[str] = frozenset()):
 
 
 @transformation_factory
-def remove_parser(node, regexp: str):
-    """Removes children by 'tag name'."""
-    remove_children_if(node, partial(is_one_of, regexp=regexp))
+def remove_parser(node, tag_names: AbstractSet[str]):
+    """Removes children by tag name."""
+    remove_children_if(node, partial(is_one_of, tag_name_set=tag_names))
 
 
 @transformation_factory

examples/LaTeX/LaTeX.ebnf

@@ -144,8 +144,9 @@ ESCAPED    = /\\[%$&_\/]/
 BRACKETS   = /[\[\]]/                       # left or right square bracket: [ ]
 TEXTCHUNK  = /[^\\%$&\{\}\[\]\s\n]+/        # some piece of text excluding whitespace,
                                             # linefeed and special characters
-LF         = !PARSEP /[ \t]*\n[ \t]*/       # linefeed but not an empty line
-PARSEP     = /[ \t]*(?:\n[ \t]*)+\n[ \t]*/  # at least one empty line, i.e.
+LF         = !GAP /[ \t]*\n[ \t]*/          # linefeed but not an empty line
+PARSEP     = { GAP }+                       # paragraph separator
+GAP        = /[ \t]*(?:\n[ \t]*)+\n/~       # at least one empty line, i.e.
                                             # [whitespace] linefeed [whitespace] linefeed
 LB         = /\s*?\n|$/                     # backwards line break for Lookbehind-Operator
                                             # beginning of text marker '$' added for test code

examples/LaTeX/LaTeXCompiler.py

@@ -155,7 +155,7 @@ class LaTeXGrammar(Grammar):
     generic_command     = !no_command CMDNAME [[ //~ config ] //~ block ]
     
     footnote            = "\footnote" block_of_paragraphs
-    includegraphics     = "\includegraphics" config block
+    includegraphics     = "\includegraphics" [ config ] block
     caption             = "\caption" block
     
     
@@ -197,8 +197,9 @@ class LaTeXGrammar(Grammar):
     BRACKETS   = /[\[\]]/                       # left or right square bracket: [ ]
     TEXTCHUNK  = /[^\\%$&\{\}\[\]\s\n]+/        # some piece of text excluding whitespace,
                                                 # linefeed and special characters
-    LF         = !PARSEP /[ \t]*\n[ \t]*/       # linefeed but not an empty line
-    PARSEP     = /[ \t]*(?:\n[ \t]*)+\n[ \t]*/  # at least one empty line, i.e.
+    LF         = !GAP /[ \t]*\n[ \t]*/          # linefeed but not an empty line
+    PARSEP     = { GAP }+                       # paragraph separator
+    GAP        = /[ \t]*(?:\n[ \t]*)+\n/~       # at least one empty line, i.e.
                                                 # [whitespace] linefeed [whitespace] linefeed
     LB         = /\s*?\n|$/                     # backwards line break for Lookbehind-Operator
                                                 # beginning of text marker '$' added for test code
@@ -211,7 +212,7 @@ class LaTeXGrammar(Grammar):
     block_of_paragraphs = Forward()
     end_generic_block = Forward()
     text_element = Forward()
-    source_hash__ = "4e001b31490278efccfe43953bfdcf58"
+    source_hash__ = "9cdeab7d908861b396d3667373fdcb9a"
     parser_initialization__ = "upon instantiation"
     COMMENT__ = r'%.*(?:\n|$)'
     WSP__ = mixin_comment(whitespace=r'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?', comment=r'%.*(?:\n|$)')
@@ -220,8 +221,9 @@ class LaTeXGrammar(Grammar):
     EOF = RegExp('(?!.)')
     BACKSLASH = RegExp('[\\\\]')
     LB = RegExp('\\s*?\\n|$')
-    PARSEP = RegExp('[ \\t]*(?:\\n[ \\t]*)+\\n[ \\t]*')
-    LF = Series(NegativeLookahead(PARSEP), RegExp('[ \\t]*\\n[ \\t]*'))
+    GAP = RE('[ \\t]*(?:\\n[ \\t]*)+\\n')
+    PARSEP = OneOrMore(GAP)
+    LF = Series(NegativeLookahead(GAP), RegExp('[ \\t]*\\n[ \\t]*'))
     TEXTCHUNK = RegExp('[^\\\\%$&\\{\\}\\[\\]\\s\\n]+')
     BRACKETS = RegExp('[\\[\\]]')
     ESCAPED = RegExp('\\\\[%$&_/]')
@@ -236,7 +238,7 @@ class LaTeXGrammar(Grammar):
     block = Series(RegExp('{'), ZeroOrMore(text_element), Required(RegExp('}')))
     config = Series(Token("["), cfgtext, Required(Token("]")))
     caption = Series(Token("\\caption"), block)
-    includegraphics = Series(Token("\\includegraphics"), config, block)
+    includegraphics = Series(Token("\\includegraphics"), Optional(config), block)
     footnote = Series(Token("\\footnote"), block_of_paragraphs)
     generic_command = Series(NegativeLookahead(no_command), CMDNAME, Optional(Series(Optional(Series(RE(''), config)), RE(''), block)))
     known_command = Alternative(footnote, includegraphics, caption)
@@ -324,7 +326,8 @@ def watch(node):
 
 LaTeX_AST_transformation_table = {
     # AST Transformations for the LaTeX-grammar
-    "+": remove_empty,
+    "+": remove_children_if(lambda node: is_empty(node) or is_one_of(node, {'PARSEP'})),
+    # remove_empty,
     "latexdoc": [],
     "preamble": [],
     "document": [],
@@ -383,6 +386,7 @@ LaTeX_AST_transformation_table = {
     "TEXTCHUNK": [],
     "LF": [],
     "PARSEP": replace_content(lambda node: '\n\n'),
+    "GAP": [],
     "LB": [],
     "BACKSLASH": [],
     "EOF": [],

examples/LaTeX/grammar_tests/test_environment.ini

@@ -96,16 +96,41 @@
     \item Only that the bullets are numbers.
     \end{enumerate}
 
-2: \begin{enumerate}
-   \item \begin{itemize}
-         \item Item-lists and
-         \item Enumeration-lists
-            \begin{enumerate}
-               \item can be nested
-               \item arbitrarily
-            \end{enumerate}
-         \item Another item
-         \end{itemize}
-   \item Plain numerated item.
-   \end{enumerate}
+2 : \begin{enumerate}
+    \item \begin{itemize}
+          \item Item-lists and
+          \item Enumeration-lists
+             \begin{enumerate}
+                \item can be nested
+                \item arbitrarily
+             \end{enumerate}
+          \item Another item
+          \end{itemize}
+    \item Plain numerated item.
+    \end{enumerate}
+
+3 : \begin{enumerate} % comment
+
+
+    % more comments and paragraph separators
+    % yet some more
+
+
+    \item %another comment
+          finally, the first item
+
+
+    % comment
+
+
+    \end{enumerate}
+
+4 : \begin{enumerate}
+    \item An item
+
+          \begin{itemize}
+          \item with an enumeration
+          \end{itemize}
+         as a separate paragraph
+    \end{enumerate}
 

examples/LaTeX/grammar_tests/test_paragraph.ini

@@ -49,8 +49,19 @@
 
     The parser should accept this, too.
 
+3 : Paragraphs can be delimited by  % comment
 
-3 : Sequences of paragraphs may
+    % sequences of separators
+
+
+    % and comments
+
+
+    In the end such a sequence counts
+    % merely as one comment
+
+
+4 : Sequences of paragraphs may
     \begin{quotation}
     include block environments
     \end{quotation}