- examples: json added

DHParser/ebnf.py

@@ -180,7 +180,7 @@ class EBNFGrammar(Grammar):
     EOF = NegativeLookahead(RegExp('.'))
     whitespace = Series(RegExp('~'), wsp__)
     regexp = Series(RegExp('/(?:(?<!\\\\)\\\\(?:/)|[^/])*?/'), wsp__)
-    plaintext = Series(RegExp('`(?:(?<!\\\\)\\\\`|[^"])*?`'), wsp__)
+    plaintext = Series(RegExp('`(?:(?<!\\\\)\\\\`|[^`])*?`'), wsp__)
     literal = Alternative(Series(RegExp('"(?:(?<!\\\\)\\\\"|[^"])*?"'), wsp__),
                           Series(RegExp("'(?:(?<!\\\\)\\\\'|[^'])*?'"), wsp__))
     symbol = Series(RegExp('(?!\\d)\\w+'), wsp__)

examples/EBNF/EBNF.ebnf

@@ -45,7 +45,7 @@ option     = "[" §expression "]"
 symbol     = /(?!\d)\w+/~                       # e.g. expression, factor, parameter_list
 literal    = /"(?:(?<!\\)\\"|[^"])*?"/~         # e.g. "(", '+', 'while'
            | /'(?:(?<!\\)\\'|[^'])*?'/~         # whitespace following literals will be ignored tacitly.
-plaintext  = /`(?:(?<!\\)\\`|[^"])*?`/~         # like literal but does not eat whitespace
+plaintext  = /`(?:(?<!\\)\\`|[^`])*?`/~         # like literal but does not eat whitespace
 regexp     = /\/(?:(?<!\\)\\(?:\/)|[^\/])*?\//~     # e.g. /\w+/, ~/#.*(?:\n|$)/~
 whitespace = /~/~                               # insignificant whitespace
 

examples/json/README.md

+# json
+
+PLACE A SHORT DESCRIPTION HERE
+
+Author: AUTHOR'S NAME <EMAIL>, AFFILIATION
+
+
+## License
+
+json is open source software under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0)
+
+Copyright YEAR AUTHOR'S NAME <EMAIL>, AFFILIATION
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

examples/json/example.json

+// json-example
+
+{
+  "object": { "one": 1, "two": 2, "three": ["3"] },
+  "array": ["one", 2, 3],
+  "string": "string example",
+  "true": true,
+  "false": false,
+  "null": null
+}

examples/json/grammar_tests/01_test_JSON_primitives.ini

+
+[match:CHARACTERS]
+
+[ast:CHARACTERS]
+
+[fail:CHARACTERS]
+
+
+[match:ESCAPE]
+
+[ast:ESCAPE]
+
+[fail:ESCAPE]
+
+
+[match:HEX]
+
+[ast:HEX]
+
+[fail:HEX]
+
+
+[match:INT]
+
+[ast:INT]
+
+[fail:INT]
+
+
+[match:FRAC]
+
+[ast:FRAC]
+
+[fail:FRAC]
+
+
+[match:EXP]
+
+[ast:EXP]
+
+[fail:EXP]
+
+
+[match:EOF]
+
+[ast:EOF]
+
+[fail:EOF]
+

examples/json/grammar_tests/02_test_JSON_elements.ini

+
+[match:json]
+
+[ast:json]
+
+[fail:json]
+
+
+[match:element]
+
+[ast:element]
+
+[fail:element]
+
+
+[match:value]
+
+[ast:value]
+
+[fail:value]
+
+
+[match:object]
+
+[ast:object]
+
+[fail:object]
+
+
+[match:member]
+
+[ast:member]
+
+[fail:member]
+
+
+[match:array]
+
+[ast:array]
+
+[fail:array]
+
+
+[match:string]
+
+[ast:string]
+
+[fail:string]
+
+
+[match:number]
+M1: "1"
+M2: 1.1
+M3: 0
+M4: 1.43E+22
+
+
+[ast:number]
+
+[fail:number]
+

examples/json/json.ebnf

+# json-grammar  see: https://json.org/
+
+#######################################################################
+#
+#  EBNF-Directives
+#
+#######################################################################
+
+@ whitespace  = vertical        # implicit whitespace, includes any number of line feeds
+@ literalws   = right           # literals have implicit whitespace on the right hand side
+@ comment     = /\/\/.*/        # comments range from a '#'-character to the end of the line
+@ ignorecase  = False           # literals and regular expressions are case-sensitive
+
+
+#######################################################################
+#
+#:  JSON elements
+#
+#######################################################################
+
+json       = ~ element EOF
+element    = value
+value      = object | array | string | number | "true" | "false" | "null"
+object     = "{" [member { "," member }] "}"
+member     = string ":" element
+array      = "[" [value { "," value }] "]"
+string     = `"` CHARACTERS `"` ~
+number     = INT FRAC EXP ~
+
+#######################################################################
+#
+#:  JSON primitives
+#
+#######################################################################
+
+CHARACTERS = { /[^"\\]+/ | ESCAPE }
+ESCAPE     = /\\[\/bnrt\\]/ | /\\u/ HEX HEX HEX HEX
+HEX        = /[0-9a-fA-F]/
+
+INT        = [`-`] /[0-9]/ | /[1-9][0-9]+/
+FRAC       = [ `.` /[0-9]+/ ]
+EXP        = [ (`E`|`e`) [`+`|`-`] /[0-9]+/ ]
+
+EOF      =  !/./        # no more characters ahead, end of file reached
+

examples/json/jsonCompiler.py

+#!/usr/bin/python3
+
+#######################################################################
+#
+# SYMBOLS SECTION - Can be edited. Changes will be preserved.
+#
+#######################################################################
+
+
+import collections
+from functools import partial
+import os
+import sys
+
+sys.path.append(r'/home/eckhart/Entwicklung/DHParser')
+
+try:
+    import regex as re
+except ImportError:
+    import re
+from DHParser import logging, is_filename, load_if_file, \
+    Grammar, Compiler, nil_preprocessor, PreprocessorToken, Whitespace, DropWhitespace, \
+    Lookbehind, Lookahead, Alternative, Pop, Token, DropToken, Synonym, AllOf, SomeOf, \
+    Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
+    ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
+    grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, \
+    Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
+    remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
+    reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
+    replace_by_children, remove_empty, remove_tokens, flatten, is_insignificant_whitespace, \
+    collapse, collapse_if, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
+    remove_nodes, remove_content, remove_brackets, change_tag_name, remove_anonymous_tokens, \
+    keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \
+    remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \
+    replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \
+    error_on, recompile_grammar, left_associative, lean_left, GLOBALS
+
+
+#######################################################################
+#
+# PREPROCESSOR SECTION - Can be edited. Changes will be preserved.
+#
+#######################################################################
+
+def jsonPreprocessor(text):
+    return text, lambda i: i
+
+def get_preprocessor() -> PreprocessorFunc:
+    return jsonPreprocessor
+
+
+#######################################################################
+#
+# PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!
+#
+#######################################################################
+
+class jsonGrammar(Grammar):
+    r"""Parser for a json source file.
+    """
+    element = Forward()
+    value = Forward()
+    source_hash__ = "06e67712cd6ccd3f0acd62f0cbe70ade"
+    static_analysis_pending__ = [True]
+    parser_initialization__ = ["upon instantiation"]
+    resume_rules__ = {}
+    COMMENT__ = r'\/\/.*'
+    WHITESPACE__ = r'\s*'
+    WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
+    wsp__ = Whitespace(WSP_RE__)
+    EOF = NegativeLookahead(RegExp('.'))
+    EXP = Option(Series(Alternative(Token("E"), Token("e")), Option(Alternative(Token("+"), Token("-"))), RegExp('[0-9]+')))
+    FRAC = Option(Series(Token("."), RegExp('[0-9]+')))
+    INT = Alternative(Series(Option(Token("-")), RegExp('[0-9]')), RegExp('[1-9][0-9]+'))
+    HEX = RegExp('[0-9a-fA-F]')
+    ESCAPE = Alternative(RegExp('\\\\[/bnrt\\\\]'), Series(RegExp('\\\\u'), HEX, HEX, HEX, HEX))
+    CHARACTERS = ZeroOrMore(Alternative(RegExp('[^"\\\\]+'), ESCAPE))
+    number = Series(INT, FRAC, EXP, wsp__)
+    string = Series(Token('"'), CHARACTERS, Token('"'), wsp__)
+    array = Series(Series(Token("["), wsp__), Option(Series(value, ZeroOrMore(Series(Series(Token(","), wsp__), value)))), Series(Token("]"), wsp__))
+    member = Series(string, Series(Token(":"), wsp__), element)
+    object = Series(Series(Token("{"), wsp__), Option(Series(member, ZeroOrMore(Series(Series(Token(","), wsp__), member)))), Series(Token("}"), wsp__))
+    value.set(Alternative(object, array, string, number, Series(Token("true"), wsp__), Series(Token("false"), wsp__), Series(Token("null"), wsp__)))
+    element.set(Synonym(value))
+    json = Series(wsp__, element, EOF)
+    root__ = json
+    
+def get_grammar() -> jsonGrammar:
+    """Returns a thread/process-exclusive jsonGrammar-singleton."""
+    try:
+        grammar = GLOBALS.json_00000001_grammar_singleton
+    except AttributeError:
+        GLOBALS.json_00000001_grammar_singleton = jsonGrammar()
+        if hasattr(get_grammar, 'python_src__'):
+            GLOBALS.json_00000001_grammar_singleton.python_src__ = get_grammar.python_src__
+        grammar = GLOBALS.json_00000001_grammar_singleton
+    return grammar
+
+
+#######################################################################
+#
+# AST SECTION - Can be edited. Changes will be preserved.
+#
+#######################################################################
+
+json_AST_transformation_table = {
+    # AST Transformations for the json-grammar
+    "<": flatten,
+    "json": [],
+    "element": [],
+    "value": [],
+    "object": [],
+    "member": [],
+    "array": [],
+    "string": [],
+    "number": [],
+    "CHARACTERS": [],
+    "ESCAPE": [],
+    "HEX": [],
+    "INT": [],
+    "FRAC": [],
+    "EXP": [],
+    "EOF": [],
+    "*": replace_by_single_child
+}
+
+
+def CreatejsonTransformer() -> TransformationFunc:
+    """Creates a transformation function that does not share state with other
+    threads or processes."""
+    return partial(traverse, processing_table=json_AST_transformation_table.copy())
+
+def get_transformer() -> TransformationFunc:
+    """Returns a thread/process-exclusive transformation function."""
+    try:
+        transformer = GLOBALS.json_00000001_transformer_singleton
+    except AttributeError:
+        GLOBALS.json_00000001_transformer_singleton = CreatejsonTransformer()
+        transformer = GLOBALS.json_00000001_transformer_singleton
+    return transformer
+
+
+#######################################################################
+#
+# COMPILER SECTION - Can be edited. Changes will be preserved.
+#
+#######################################################################
+
+class jsonCompiler(Compiler):
+    """Compiler for the abstract-syntax-tree of a json source file.
+    """
+
+    def __init__(self):
+        super(jsonCompiler, self).__init__()
+
+    def _reset(self):
+        super()._reset()
+        # initialize your variables here, not in the constructor!
+
+    def on_json(self, node):
+        return self.fallback_compiler(node)
+
+    # def on_element(self, node):
+    #     return node
+
+    # def on_value(self, node):
+    #     return node
+
+    # def on_object(self, node):
+    #     return node
+
+    # def on_member(self, node):
+    #     return node
+
+    # def on_array(self, node):
+    #     return node
+
+    # def on_string(self, node):
+    #     return node
+
+    # def on_number(self, node):
+    #     return node
+
+    # def on_CHARACTERS(self, node):
+    #     return node
+
+    # def on_ESCAPE(self, node):
+    #     return node
+
+    # def on_HEX(self, node):
+    #     return node
+
+    # def on_INT(self, node):
+    #     return node
+
+    # def on_FRAC(self, node):
+    #     return node
+
+    # def on_EXP(self, node):
+    #     return node
+
+    # def on_EOF(self, node):
+    #     return node
+
+
+def get_compiler() -> jsonCompiler:
+    """Returns a thread/process-exclusive jsonCompiler-singleton."""
+    try:
+        compiler = GLOBALS.json_00000001_compiler_singleton
+    except AttributeError:
+        GLOBALS.json_00000001_compiler_singleton = jsonCompiler()
+        compiler = GLOBALS.json_00000001_compiler_singleton
+    return compiler
+
+
+#######################################################################
+#
+# END OF DHPARSER-SECTIONS
+#
+#######################################################################
+
+
+def compile_src(source, log_dir=''):
+    """Compiles ``source`` and returns (result, errors, ast).
+    """
+    with logging(log_dir):
+        compiler = get_compiler()
+        result_tuple = compile_source(source, get_preprocessor(),
+                                      get_grammar(),
+                                      get_transformer(), compiler)
+    return result_tuple
+
+
+if __name__ == "__main__":
+    # recompile grammar if needed
+    grammar_path = os.path.abspath(__file__).replace('Compiler.py', '.ebnf')
+    if os.path.exists(grammar_path):
+        if not recompile_grammar(grammar_path, force=False,
+                                  notify=lambda:print('recompiling ' + grammar_path)):
+            error_file = os.path.basename(__file__).replace('Compiler.py', '_ebnf_ERRORS.txt')
+            with open(error_file, encoding="utf-8") as f:
+                print(f.read())
+            sys.exit(1)
+    else:
+        print('Could not check whether grammar requires recompiling, '
+              'because grammar was not found at: ' + grammar_path)
+
+    if len(sys.argv) > 1:
+        # compile file
+        file_name, log_dir = sys.argv[1], ''
+        if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
+            file_name, log_dir = sys.argv[2], 'LOGS'
+        result, errors, _ = compile_src(file_name, log_dir)
+        if errors:
+            cwd = os.getcwd()
+            rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
+            for error in errors:
+                print(rel_path + ':' + str(error))
+            sys.exit(1)
+        else:
+            print(result.as_xml() if isinstance(result, Node) else result)
+    else:
+        print("Usage: jsonCompiler.py [FILENAME]")

examples/json/tst_json_grammar.py

+#!/usr/bin/python3
+
+"""tst_json_grammar.py - runs the unit tests for the json-grammar
+"""
+
+import os
+import sys
+
+LOGGING = False
+
+sys.path.append(r'/home/eckhart/Entwicklung/DHParser')
+
+scriptpath = os.path.dirname(__file__)
+
+
+try:
+    from DHParser import dsl
+    import DHParser.log
+    from DHParser import testing
+except ModuleNotFoundError:
+    print('Could not import DHParser. Please adjust sys.path in file '
+          '"%s" manually' % __file__)
+    sys.exit(1)
+
+
+def recompile_grammar(grammar_src, force):
+    grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
+    testing.create_test_templates(grammar_src, grammar_tests_dir)
+    with DHParser.log.logging(False):
+        # recompiles Grammar only if it has changed
+        if not dsl.recompile_grammar(grammar_src, force=force,
+                notify=lambda: print('recompiling ' + grammar_src)):
+            print('\nErrors while recompiling "%s":' % grammar_src +
+                  '\n--------------------------------------\n\n')
+            with open('json_ebnf_ERRORS.txt') as f:
+                print(f.read())
+            sys.exit(1)
+
+
+def run_grammar_tests(glob_pattern):
+    with DHParser.log.logging(LOGGING):
+        error_report = testing.grammar_suite(
+            os.path.join(scriptpath, 'grammar_tests'),
+            get_grammar, get_transformer,
+            fn_patterns=[glob_pattern], report=True, verbose=True)
+    return error_report
+
+
+if __name__ == '__main__':
+    argv = sys.argv[:]
+    if len(argv) > 1 and sys.argv[1] == "--debug":
+        LOGGING = True
+        del argv[1]
+    if (len(argv) >= 2 and (argv[1].endswith('.ebnf') or
+        os.path.splitext(argv[1])[1].lower() in testing.TEST_READERS.keys())):
+        # if called with a single filename that is either an EBNF file or a known
+        # test file type then use the given argument
+        arg = argv[1]
+    else:
+        # otherwise run all tests in the test directory
+        arg = '*_test_*.ini'
+    if arg.endswith('.ebnf'):
+        recompile_grammar(arg, force=True)
+    else:
+        recompile_grammar(os.path.join(scriptpath, 'json.ebnf'),
+                          force=False)
+        sys.path.append('.')
+        from jsonCompiler import get_grammar, get_transformer
+        error_report = run_grammar_tests(glob_pattern=arg)
+        if error_report:
+            print('\n')
+            print(error_report)
+            sys.exit(1)
+        print('ready.\n')

scripts/dhparser.py

@@ -18,11 +18,17 @@ implied.  See the License for the specific language governing
 permissions and limitations under the License.
 """
 
-#  TODO: This is still a stub...
-
 import os
 import sys
 
+scriptdir = os.path.dirname(os.path.realpath(__file__))
+i = scriptdir.find('DHParser')
+if i >= 0:
+    dhparserdir = scriptdir[:i + 8]
+    sys.path.append(dhparserdir)
+else:
+    dhparserdir = ''
+
 from DHParser.compile import compile_source
 from DHParser.dsl import compileDSL, compile_on_disk  # , recompile_grammar
 from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
@@ -32,8 +38,6 @@ from typing import cast
 
 LOGGING = False
 
-dhparserdir = os.path.dirname(os.path.realpath(__file__))
-
 EBNF_TEMPLATE = r"""-grammar
 
 #######################################################################