- more hints for AST-transformation from EBNFCompiler

DHParser/dsl.py

@@ -82,7 +82,7 @@ from DHParser import logging, is_filename, load_if_file, \\
     last_value, counterpart, accumulate, PreprocessorFunc, \\
     Node, TransformationFunc, TRUE_CONDITION, \\
     traverse, remove_children_if, merge_children, is_anonymous, \\
-    reduce_single_child, replace_by_single_child, remove_whitespace, \\
+    reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \\
     remove_expendables, remove_empty, remove_tokens, flatten, is_whitespace, \\
     is_empty, is_expendable, collapse, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \\
     remove_parser, remove_content, remove_brackets, replace_parser, \\

DHParser/ebnf.py

@@ -330,6 +330,9 @@ class EBNFCompiler(Compiler):
         recursive - A set of symbols that are used recursively and
                 therefore require a `Forward`-operator.
 
+        definitions - A dictionary of definitions. Other than `rules`
+                this maps the symbols to their compiled definienda.
+
         deferred_taks - A list of callables that is filled during
                 compilatation, but that will be executed only after
                 compilation has finished. Typically, it contains
@@ -367,6 +370,7 @@ class EBNFCompiler(Compiler):
         self.symbols = {}           # type: Dict[str, Node]
         self.variables = set()      # type: Set[str]
         self.recursive = set()      # type: Set[str]
+        self.definitions = {}       # type: Dict[str, str]
         self.deferred_tasks = []    # type: List[Callable]
         self.root = ""              # type: str
         self.directives = {'whitespace': self.WHITESPACE['horizontal'],
@@ -407,9 +411,15 @@ class EBNFCompiler(Compiler):
                       self.grammar_name + '-grammar']
         transtable.append('    "+": remove_empty,')
         for name in self.rules:
-            transtable.append('    "' + name + '": [],')
+            tf = '[]'
+            rule = self.definitions[name]
+            if rule.startswith('Alternative'):
+                tf = '[replace_or_reduce]'
+            elif rule.startswith('Synonym'):
+                tf = '[replace_by_single_child]'
+            transtable.append('    "' + name + '": %s,' % tf)
         transtable.append('    ":Token, :RE": reduce_single_child,')
-        transtable += ['    # "*": replace_by_single_child', '}', '', tf_name +
+        transtable += ['    "*": replace_by_single_child', '}', '', tf_name +
                        ' = partial(traverse, processing_table=%s)' % tt_name, '']
         transtable += [TRANSFORMER_FACTORY.format(NAME=self.grammar_name)]
         return '\n'.join(transtable)
@@ -560,6 +570,7 @@ class EBNFCompiler(Compiler):
                 assert nd.parser.name == "directive", nd.as_sxpr()
                 self.compile(nd)
                 node.error_flag = node.error_flag or nd.error_flag
+        self.definitions.update(definitions)
 
         return self.assemble_parser(definitions, node)
 

DHParser/transform.py

@@ -39,8 +39,10 @@ __all__ = ('transformation_factory',
            'key_parser_name',
            'key_tag_name',
            'traverse',
+           'is_named',
            'replace_by_single_child',
            'reduce_single_child',
+           'replace_or_reduce',
            'replace_parser',
            'collapse',
            'merge_children',
@@ -235,6 +237,21 @@ def TRUE_CONDITION(node):
     return True
 
 
+def replace_child(node):
+    assert len(node.children) == 1
+    if not node.children[0].parser.name:
+        node.children[0].parser.name = node.parser.name
+    node.parser = node.children[0].parser
+    node._errors.extend(node.children[0]._errors)
+    node.result = node.result[0].result
+
+
+def reduce_child(node):
+    assert len(node.children) == 1
+    node._errors.extend(node.children[0]._errors)
+    node.result = node.result[0].result
+
+
 @transformation_factory(Callable)
 def replace_by_single_child(node, condition=TRUE_CONDITION):
     """Remove single branch node, replacing it by its immediate descendant
@@ -242,12 +259,8 @@ def replace_by_single_child(node, condition=TRUE_CONDITION):
     (In case the descendant's name is empty (i.e. anonymous) the
     name of this node's parser is kept.)
     """
-    if node.children and len(node.result) == 1 and condition(node.children[0]):
-        if not node.result[0].parser.name:
-            node.result[0].parser.name = node.parser.name
-        node.parser = node.result[0].parser
-        node._errors.extend(node.result[0]._errors)
-        node.result = node.result[0].result
+    if len(node.children) == 1 and condition(node.children[0]):
+        replace_child(node)
 
 
 @transformation_factory(Callable)
@@ -257,9 +270,23 @@ def reduce_single_child(node, condition=TRUE_CONDITION):
     If the condition evaluates to false on the descendant, it will not
     be reduced.
     """
-    if node.children and len(node.result) == 1 and condition(node.children[0]):
-        node._errors.extend(node.result[0]._errors)
-        node.result = node.result[0].result
+    if len(node.children) == 1 and condition(node.children[0]):
+        reduce_child(node)
+
+
+def is_named(node):
+    return node.parser.name
+
+
+@transformation_factory(Callable)
+def replace_or_reduce(node, condition=is_named):
+    """Replaces node by a single child, if condition is met on child,
+    otherwise (i.e. if the child is anonymous) reduces the child.
+    """
+    if len(node.children) == 1 and condition(node.children[0]):
+        replace_child(node)
+    else:
+        reduce_child(node)
 
 
 @transformation_factory

dhparser.py

@@ -77,6 +77,7 @@ def selftest() -> bool:
     print("\n\nSTAGE 2: Selfhosting-test: Trying to compile EBNF-Grammar with generated parser...\n")
     selfhosted_ebnf_parser = compileDSL(ebnf_src, None, generated_ebnf_parser,
                                         ebnf_transformer, ebnf_compiler)
+    ebnf_compiler.gen_transformer_skeleton()
     print(selfhosted_ebnf_parser)
     print("\n\n Selftest SUCCEEDED :-)\n\n")
     return True

examples/LaTeX/LaTeX.ebnf

@@ -64,12 +64,12 @@ itemize             = "\begin{itemize}" [PARSEP] { item } §"\end{itemize}"
 enumerate           = "\begin{enumerate}" [PARSEP] {item } §"\end{enumerate}"
 item                = "\item" [PARSEP] sequence
 
-figure              = "\begin{figure}" sequence "\end{figure}"
-quotation           = ("\begin{quotation}" sequence "\end{quotation}")
-                    | ("\begin{quote}" sequence "\end{quote}")
-verbatim            = "\begin{verbatim}" sequence "\end{verbatim}"
-table               = "\begin{tabular}" table_config sequence "\end{tabular}"
-table_config        = "{" /[lcr|]+/~ "}"
+figure              = "\begin{figure}" sequence §"\end{figure}"
+quotation           = ("\begin{quotation}" sequence §"\end{quotation}")
+                    | ("\begin{quote}" sequence §"\end{quote}")
+verbatim            = "\begin{verbatim}" sequence §"\end{verbatim}"
+table               = "\begin{tabular}" table_config sequence §"\end{tabular}"
+table_config        = "{" /[lcr|]+/~ §"}"
 
 
 #### paragraphs and sequences of paragraphs ####
@@ -92,7 +92,7 @@ end_inline_env      = end_environment
 begin_environment   = "\begin{" §NAME §"}"
 end_environment     = "\end{" §::NAME §"}"
 
-inline_math         = "$" /[^$]*/ "$"
+inline_math         = "$" /[^$]*/ §"$"
 
 
 #### commands ####