- new example case: Alternative arithmetic, new TODO: infnite loop protection

examples/Arithmetic_alternative/Arithmetic.ebnf

+# Arithmetic-grammar
+
+#######################################################################
+#
+#  EBNF-Directives
+#
+#######################################################################
+
+@ whitespace  = vertical           # implicit whitespace, includes any number of line feeds
+@ literalws   = right              # literals have implicit whitespace on the right hand side
+@ comment     = /#.*/              # comments range from a '#'-character to the end of the line
+@ ignorecase  = False              # literals and regular expressions are case-sensitive
+@ drop        = whitespace, token  # drop anonymous whitespace
+
+#######################################################################
+#
+#: Expressions
+#
+#######################################################################
+
+expression     = addition | subtraction
+addition       = (expression | term) "+" (expression | term)
+subtraction    = (expression | term) "-" (expression | term)
+
+
+#######################################################################
+#
+#: Terms
+#
+#######################################################################
+
+term           = multiplication | division
+multiplication = (term | factor) "*" (term | factor)
+division       = (term | factor) "/" (term | factor)
+
+
+#######################################################################
+#
+#: Factors
+#
+#######################################################################
+
+factor         = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
+group          = "(" expression ")"
+
+
+#######################################################################
+#
+#: Tokens
+#
+#######################################################################
+
+SIGN       = /[+-]/
+NUMBER     = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
+VARIABLE   = /[A-Za-z]/~

examples/Arithmetic_alternative/ArithmeticCompiler.py

+#!/usr/bin/python3
+
+#######################################################################
+#
+# SYMBOLS SECTION - Can be edited. Changes will be preserved.
+#
+#######################################################################
+
+
+import collections
+from functools import partial
+import os
+import sys
+
+sys.path.append(r'/home/eckhart/Entwicklung/DHParser')
+
+try:
+    import regex as re
+except ImportError:
+    import re
+from DHParser import logging, is_filename, load_if_file, \
+    Grammar, Compiler, nil_preprocessor, PreprocessorToken, Whitespace, DropWhitespace, \
+    Lookbehind, Lookahead, Alternative, Pop, Token, DropToken, Synonym, AllOf, SomeOf, \
+    Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
+    ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
+    grammar_changed, last_value, counterpart, accumulate, PreprocessorFunc, \
+    Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
+    remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
+    reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
+    remove_expendables, remove_empty, remove_tokens, flatten, is_insignificant_whitespace, is_empty, \
+    is_expendable, collapse, collapse_if, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
+    remove_nodes, remove_content, remove_brackets, replace_parser, remove_anonymous_tokens, \
+    keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \
+    remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \
+    replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \
+    error_on, recompile_grammar, flatten_anonymous_nodes, GLOBALS
+
+
+#######################################################################
+#
+# PREPROCESSOR SECTION - Can be edited. Changes will be preserved.
+#
+#######################################################################
+
+def ArithmeticPreprocessor(text):
+    return text, lambda i: i
+
+def get_preprocessor() -> PreprocessorFunc:
+    return ArithmeticPreprocessor
+
+
+#######################################################################
+#
+# PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!
+#
+#######################################################################
+
+class ArithmeticGrammar(Grammar):
+    r"""Parser for an Arithmetic source file.
+    """
+    expression = Forward()
+    term = Forward()
+    source_hash__ = "ba85985ea3917dbb90568f216cb1cbb2"
+    static_analysis_pending__ = [True]
+    parser_initialization__ = ["upon instantiation"]
+    resume_rules__ = {}
+    COMMENT__ = r'#.*'
+    WHITESPACE__ = r'\s*'
+    WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
+    dwsp__ = DropWhitespace(WSP_RE__)
+    wsp__ = Whitespace(WSP_RE__)
+    VARIABLE = Series(RegExp('[A-Za-z]'), dwsp__)
+    NUMBER = Series(RegExp('(?:0|(?:[1-9]\\d*))(?:\\.\\d+)?'), dwsp__)
+    SIGN = RegExp('[+-]')
+    group = Series(Series(DropToken("("), dwsp__), expression, Series(DropToken(")"), dwsp__))
+    factor = Series(Option(SIGN), Alternative(NUMBER, VARIABLE, group), ZeroOrMore(Alternative(VARIABLE, group)))
+    division = Series(Alternative(term, factor), Series(DropToken("/"), dwsp__), Alternative(term, factor))
+    multiplication = Series(Alternative(term, factor), Series(DropToken("*"), dwsp__), Alternative(term, factor))
+    term.set(Alternative(multiplication, division))
+    subtraction = Series(Alternative(expression, term), Series(DropToken("-"), dwsp__), Alternative(expression, term))
+    addition = Series(Alternative(expression, term), Series(DropToken("+"), dwsp__), Alternative(expression, term))
+    expression.set(Alternative(addition, subtraction))
+    root__ = expression
+    
+def get_grammar() -> ArithmeticGrammar:
+    global GLOBALS
+    try:
+        grammar = GLOBALS.Arithmetic_00000001_grammar_singleton
+    except AttributeError:
+        GLOBALS.Arithmetic_00000001_grammar_singleton = ArithmeticGrammar()
+        if hasattr(get_grammar, 'python_src__'):
+            GLOBALS.Arithmetic_00000001_grammar_singleton.python_src__ = get_grammar.python_src__
+        grammar = GLOBALS.Arithmetic_00000001_grammar_singleton
+    return grammar
+
+
+#######################################################################
+#
+# AST SECTION - Can be edited. Changes will be preserved.
+#
+#######################################################################
+
+
+Arithmetic_AST_transformation_table = {
+    # AST Transformations for the Arithmetic-grammar
+    # "<": flatten_anonymous_nodes,
+    "expression": [],
+    "term": [],
+    "factor": [],
+    "group": [],
+    "NUMBER": [],
+    "VARIABLE": []
+    # ":Token": reduce_single_child,
+    # "*": replace_by_single_child
+}
+
+
+def ArithmeticTransform() -> TransformationDict:
+    return partial(traverse, processing_table=Arithmetic_AST_transformation_table.copy())
+
+def get_transformer() -> TransformationFunc:
+    try:
+        transformer = GLOBALS.Arithmetic_00000001_transformer_singleton
+    except AttributeError:
+        GLOBALS.Arithmetic_00000001_transformer_singleton = ArithmeticTransform()
+        transformer = GLOBALS.Arithmetic_00000001_transformer_singleton
+    return transformer
+
+
+#######################################################################
+#
+# COMPILER SECTION - Can be edited. Changes will be preserved.
+#
+#######################################################################
+
+class ArithmeticCompiler(Compiler):
+    """Compiler for the abstract-syntax-tree of a Arithmetic source file.
+    """
+
+    def __init__(self):
+        super(ArithmeticCompiler, self).__init__()
+
+    def _reset(self):
+        super()._reset()
+        # initialize your variables here, not in the constructor!
+    def on_expression(self, node):
+        return self.fallback_compiler(node)
+
+    # def on_term(self, node):
+    #     return node
+
+    # def on_factor(self, node):
+    #     return node
+
+    # def on_NUMBER(self, node):
+    #     return node
+
+    # def on_VARIABLE(self, node):
+    #     return node
+
+
+def get_compiler() -> ArithmeticCompiler:
+    try:
+        compiler = GLOBALS.Arithmetic_00000001_compiler_singleton
+    except AttributeError:
+        GLOBALS.Arithmetic_00000001_compiler_singleton = ArithmeticCompiler()
+        compiler = GLOBALS.Arithmetic_00000001_compiler_singleton
+    return compiler
+
+
+#######################################################################
+#
+# END OF DHPARSER-SECTIONS
+#
+#######################################################################
+
+
+def compile_src(source, log_dir=''):
+    """Compiles ``source`` and returns (result, errors, ast).
+    """
+    with logging(log_dir):
+        compiler = get_compiler()
+        cname = compiler.__class__.__name__
+        result_tuple = compile_source(source, get_preprocessor(),
+                                      get_grammar(),
+                                      get_transformer(), compiler)
+    return result_tuple
+
+
+if __name__ == "__main__":
+    # recompile grammar if needed
+    grammar_path = os.path.abspath(__file__).replace('Compiler.py', '.ebnf')
+    if os.path.exists(grammar_path):
+        if not recompile_grammar(grammar_path, force=False,
+                                  notify=lambda:print('recompiling ' + grammar_path)):
+            error_file = os.path.basename(__file__).replace('Compiler.py', '_ebnf_ERRORS.txt')
+            with open(error_file, encoding="utf-8") as f:
+                print(f.read())
+            sys.exit(1)
+    else:
+        print('Could not check whether grammar requires recompiling, '
+              'because grammar was not found at: ' + grammar_path)
+
+    if len(sys.argv) > 1:
+        # compile file
+        file_name, log_dir = sys.argv[1], ''
+        if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
+            file_name, log_dir = sys.argv[2], 'LOGS'
+        result, errors, ast = compile_src(file_name, log_dir)
+        if errors:
+            cwd = os.getcwd()
+            rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
+            for error in errors:
+                print(rel_path + ':' + str(error))
+            sys.exit(1)
+        else:
+            print(result.as_xml() if isinstance(result, Node) else result)
+    else:
+        print("Usage: ArithmeticCompiler.py [FILENAME]")

examples/Arithmetic_alternative/README.md

+# Arithmetic
+
+PLACE A SHORT DESCRIPTION HERE
+
+Author: AUTHOR'S NAME <EMAIL>, AFFILIATION
+
+
+## License
+
+Arithmetic is open source software under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0)
+
+Copyright YEAR AUTHOR'S NAME <EMAIL>, AFFILIATION
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

examples/Arithmetic_alternative/grammar_tests/01_test_Tokens.ini

+
+[match:SIGN]
+M1: '-'
+M2: '+'
+
+[match:NUMBER]
+M1: 1
+M2: 99
+M3: 3.14156
+M5: 2
+M6: 105.23
+
+[fail:NUMBER]
+F1: +22
+F2: 003
+F4: 2.2.6
+F5: -22
+
+
+[match:VARIABLE]
+M1: a
+M2: Z
+
+[fail:VARIABLE]
+F1: ä
+F2: ab

examples/Arithmetic_alternative/grammar_tests/02_test_Factors.ini

+[match:factor]
+M1: "-2"
+M2: "+22"
+M3: "-2.71828"
+M4: "-x"
+M5: "(2 + x)"
+M6: "-(a * b)"
+M7: "4x"
+M8: "-2x"
+
+[fail:factor]
+F1: "x4"
+F2: "- 2"
+
+[match:group]
+M1: "(2 * 4)"
+M2: "(2 + 4)"
+
+[ast:group]
+
+[fail:group]
+

examples/Arithmetic_alternative/grammar_tests/03_test_Terms.ini

+
+[match:term]
+M1: "2 * 4"
+M2: "2 / 4"
+M3: "(2 + 4) * (2 - 4)"
+M4: "(a + b)(a - b)"
+
+[ast:term]
+
+[fail:term]
+
+
+[match:multiplication]
+M1:  "2 * 4"
+
+[ast:multiplication]
+
+[fail:multiplication]
+
+
+[match:division]
+M1: "2 / 4"
+M2: "-x / 7"
+
+[ast:division]
+
+[fail:division]
+

examples/Arithmetic_alternative/grammar_tests/04_test_Expressions.ini

+
+[match:expression]
+M1: "2 + x"
+
+[ast:expression]
+
+[fail:expression]
+
+
+[match:addition]
+M1: "2 + x"
+
+[ast:addition]
+
+[fail:addition]
+
+
+[match:subtraction]
+
+[ast:subtraction]
+
+[fail:subtraction]
+

examples/Arithmetic_alternative/tst_Arithmetic_grammar.py

+#!/usr/bin/python3
+
+"""tst_Arithmetic_grammar.py - runs the unit tests for the Arithmetic-grammar
+"""
+
+import os
+import sys
+
+LOGGING = True
+
+sys.path.append(r'/home/eckhart/Entwicklung/DHParser')
+
+scriptpath = os.path.dirname(__file__)
+
+
+try:
+    from DHParser import dsl
+    import DHParser.log
+    from DHParser import testing, create_test_templates, CONFIG_PRESET
+except ModuleNotFoundError:
+    print('Could not import DHParser. Please adjust sys.path in file '
+          '"%s" manually' % __file__)
+    sys.exit(1)
+
+
+CONFIG_PRESET['ast_serialization'] = "S-expression"
+CONFIG_PRESET['test_parallelization'] = False
+
+
+def recompile_grammar(grammar_src, force):
+    grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
+    if not os.path.exists(grammar_tests_dir) \
+            or not any(os.path.isfile(os.path.join(grammar_tests_dir, entry))
+                       for entry in os.listdir(grammar_tests_dir)):
+        print('No grammar-tests found, generating test templates.')
+        create_test_templates(grammar_src, grammar_tests_dir)
+    with DHParser.log.logging(LOGGING):
+        # recompiles Grammar only if it has changed
+        name = os.path.splitext(os.path.basename(grammar_src))[0]
+        if not dsl.recompile_grammar(grammar_src, force=force):
+            print('\nErrors while recompiling "{}":'.format(grammar_src) +
+                  '\n--------------------------------------\n\n')
+            with open('{}_ebnf_ERRORS.txt'.format(name)) as f:
+                print(f.read())
+            sys.exit(1)
+
+
+def run_grammar_tests(glob_pattern):
+    with DHParser.log.logging(LOGGING):
+        error_report = testing.grammar_suite(
+            os.path.join(scriptpath, 'grammar_tests'),
+            get_grammar, get_transformer,
+            fn_patterns=[glob_pattern], report=True, verbose=True)
+    return error_report
+
+
+if __name__ == '__main__':
+    argv = sys.argv[:]
+    if len(argv) > 1 and sys.argv[1] == "--debug":
+        LOGGING = True
+        del argv[1]
+    if (len(argv) >= 2 and (argv[1].endswith('.ebnf') or
+        os.path.splitext(argv[1])[1].lower() in testing.TEST_READERS.keys())):
+        # if called with a single filename that is either an EBNF file or a known
+        # test file type then use the given argument
+        arg = argv[1]
+    else:
+        # otherwise run all tests in the test directory
+        arg = '*_test_*.ini'
+    if arg.endswith('.ebnf'):
+        recompile_grammar(arg, force=True)
+    else:
+        recompile_grammar(os.path.join(scriptpath, 'Arithmetic.ebnf'),
+                          force=False)
+        sys.path.append('.')
+        from ArithmeticCompiler import get_grammar, get_transformer
+        error_report = run_grammar_tests(glob_pattern=arg)
+        if error_report:
+            print('\n')
+            print(error_report)
+            sys.exit(1)
+        print('ready.\n')