2.12.2021, 9:00 - 11:00: Due to updates GitLab may be unavailable for some minutes between 09:00 and 11:00.

Commit 59844e1a authored by eckhart's avatar eckhart
Browse files

- examples: json added

parent 54275a30
......@@ -180,7 +180,7 @@ class EBNFGrammar(Grammar):
EOF = NegativeLookahead(RegExp('.'))
whitespace = Series(RegExp('~'), wsp__)
regexp = Series(RegExp('/(?:(?<!\\\\)\\\\(?:/)|[^/])*?/'), wsp__)
plaintext = Series(RegExp('`(?:(?<!\\\\)\\\\`|[^"])*?`'), wsp__)
plaintext = Series(RegExp('`(?:(?<!\\\\)\\\\`|[^`])*?`'), wsp__)
literal = Alternative(Series(RegExp('"(?:(?<!\\\\)\\\\"|[^"])*?"'), wsp__),
Series(RegExp("'(?:(?<!\\\\)\\\\'|[^'])*?'"), wsp__))
symbol = Series(RegExp('(?!\\d)\\w+'), wsp__)
......
......@@ -45,7 +45,7 @@ option = "[" §expression "]"
symbol = /(?!\d)\w+/~ # e.g. expression, factor, parameter_list
literal = /"(?:(?<!\\)\\"|[^"])*?"/~ # e.g. "(", '+', 'while'
| /'(?:(?<!\\)\\'|[^'])*?'/~ # whitespace following literals will be ignored tacitly.
plaintext = /`(?:(?<!\\)\\`|[^"])*?`/~ # like literal but does not eat whitespace
plaintext = /`(?:(?<!\\)\\`|[^`])*?`/~ # like literal but does not eat whitespace
regexp = /\/(?:(?<!\\)\\(?:\/)|[^\/])*?\//~ # e.g. /\w+/, ~/#.*(?:\n|$)/~
whitespace = /~/~ # insignificant whitespace
......
# json
PLACE A SHORT DESCRIPTION HERE
Author: AUTHOR'S NAME <EMAIL>, AFFILIATION
## License
json is open source software under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0)
Copyright YEAR AUTHOR'S NAME <EMAIL>, AFFILIATION
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
// json-example
{
"object": { "one": 1, "two": 2, "three": ["3"] },
"array": ["one", 2, 3],
"string": "string example",
"true": true,
"false": false,
"null": null
}
[match:CHARACTERS]
[ast:CHARACTERS]
[fail:CHARACTERS]
[match:ESCAPE]
[ast:ESCAPE]
[fail:ESCAPE]
[match:HEX]
[ast:HEX]
[fail:HEX]
[match:INT]
[ast:INT]
[fail:INT]
[match:FRAC]
[ast:FRAC]
[fail:FRAC]
[match:EXP]
[ast:EXP]
[fail:EXP]
[match:EOF]
[ast:EOF]
[fail:EOF]
[match:json]
[ast:json]
[fail:json]
[match:element]
[ast:element]
[fail:element]
[match:value]
[ast:value]
[fail:value]
[match:object]
[ast:object]
[fail:object]
[match:member]
[ast:member]
[fail:member]
[match:array]
[ast:array]
[fail:array]
[match:string]
[ast:string]
[fail:string]
[match:number]
M1: "1"
M2: 1.1
M3: 0
M4: 1.43E+22
[ast:number]
[fail:number]
# json-grammar see: https://json.org/
#######################################################################
#
# EBNF-Directives
#
#######################################################################
@ whitespace = vertical # implicit whitespace, includes any number of line feeds
@ literalws = right # literals have implicit whitespace on the right hand side
@ comment = /\/\/.*/ # comments range from a '#'-character to the end of the line
@ ignorecase = False # literals and regular expressions are case-sensitive
#######################################################################
#
#: JSON elements
#
#######################################################################
json = ~ element EOF
element = value
value = object | array | string | number | "true" | "false" | "null"
object = "{" [member { "," member }] "}"
member = string ":" element
array = "[" [value { "," value }] "]"
string = `"` CHARACTERS `"` ~
number = INT FRAC EXP ~
#######################################################################
#
#: JSON primitives
#
#######################################################################
CHARACTERS = { /[^"\\]+/ | ESCAPE }
ESCAPE = /\\[\/bnrt\\]/ | /\\u/ HEX HEX HEX HEX
HEX = /[0-9a-fA-F]/
INT = [`-`] /[0-9]/ | /[1-9][0-9]+/
FRAC = [ `.` /[0-9]+/ ]
EXP = [ (`E`|`e`) [`+`|`-`] /[0-9]+/ ]
EOF = !/./ # no more characters ahead, end of file reached
#!/usr/bin/python3
#######################################################################
#
# SYMBOLS SECTION - Can be edited. Changes will be preserved.
#
#######################################################################
import collections
from functools import partial
import os
import sys
sys.path.append(r'/home/eckhart/Entwicklung/DHParser')
try:
import regex as re
except ImportError:
import re
from DHParser import logging, is_filename, load_if_file, \
Grammar, Compiler, nil_preprocessor, PreprocessorToken, Whitespace, DropWhitespace, \
Lookbehind, Lookahead, Alternative, Pop, Token, DropToken, Synonym, AllOf, SomeOf, \
Unordered, Option, NegativeLookbehind, OneOrMore, RegExp, Retrieve, Series, Capture, \
ZeroOrMore, Forward, NegativeLookahead, Required, mixin_comment, compile_source, \
grammar_changed, last_value, counterpart, PreprocessorFunc, is_empty, \
Node, TransformationFunc, TransformationDict, transformation_factory, traverse, \
remove_children_if, move_adjacent, normalize_whitespace, is_anonymous, matches_re, \
reduce_single_child, replace_by_single_child, replace_or_reduce, remove_whitespace, \
replace_by_children, remove_empty, remove_tokens, flatten, is_insignificant_whitespace, \
collapse, collapse_if, replace_content, WHITESPACE_PTYPE, TOKEN_PTYPE, \
remove_nodes, remove_content, remove_brackets, change_tag_name, remove_anonymous_tokens, \
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last, \
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip, \
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator, \
error_on, recompile_grammar, left_associative, lean_left, GLOBALS
#######################################################################
#
# PREPROCESSOR SECTION - Can be edited. Changes will be preserved.
#
#######################################################################
def jsonPreprocessor(text):
return text, lambda i: i
def get_preprocessor() -> PreprocessorFunc:
return jsonPreprocessor
#######################################################################
#
# PARSER SECTION - Don't edit! CHANGES WILL BE OVERWRITTEN!
#
#######################################################################
class jsonGrammar(Grammar):
r"""Parser for a json source file.
"""
element = Forward()
value = Forward()
source_hash__ = "06e67712cd6ccd3f0acd62f0cbe70ade"
static_analysis_pending__ = [True]
parser_initialization__ = ["upon instantiation"]
resume_rules__ = {}
COMMENT__ = r'\/\/.*'
WHITESPACE__ = r'\s*'
WSP_RE__ = mixin_comment(whitespace=WHITESPACE__, comment=COMMENT__)
wsp__ = Whitespace(WSP_RE__)
EOF = NegativeLookahead(RegExp('.'))
EXP = Option(Series(Alternative(Token("E"), Token("e")), Option(Alternative(Token("+"), Token("-"))), RegExp('[0-9]+')))
FRAC = Option(Series(Token("."), RegExp('[0-9]+')))
INT = Alternative(Series(Option(Token("-")), RegExp('[0-9]')), RegExp('[1-9][0-9]+'))
HEX = RegExp('[0-9a-fA-F]')
ESCAPE = Alternative(RegExp('\\\\[/bnrt\\\\]'), Series(RegExp('\\\\u'), HEX, HEX, HEX, HEX))
CHARACTERS = ZeroOrMore(Alternative(RegExp('[^"\\\\]+'), ESCAPE))
number = Series(INT, FRAC, EXP, wsp__)
string = Series(Token('"'), CHARACTERS, Token('"'), wsp__)
array = Series(Series(Token("["), wsp__), Option(Series(value, ZeroOrMore(Series(Series(Token(","), wsp__), value)))), Series(Token("]"), wsp__))
member = Series(string, Series(Token(":"), wsp__), element)
object = Series(Series(Token("{"), wsp__), Option(Series(member, ZeroOrMore(Series(Series(Token(","), wsp__), member)))), Series(Token("}"), wsp__))
value.set(Alternative(object, array, string, number, Series(Token("true"), wsp__), Series(Token("false"), wsp__), Series(Token("null"), wsp__)))
element.set(Synonym(value))
json = Series(wsp__, element, EOF)
root__ = json
def get_grammar() -> jsonGrammar:
"""Returns a thread/process-exclusive jsonGrammar-singleton."""
try:
grammar = GLOBALS.json_00000001_grammar_singleton
except AttributeError:
GLOBALS.json_00000001_grammar_singleton = jsonGrammar()
if hasattr(get_grammar, 'python_src__'):
GLOBALS.json_00000001_grammar_singleton.python_src__ = get_grammar.python_src__
grammar = GLOBALS.json_00000001_grammar_singleton
return grammar
#######################################################################
#
# AST SECTION - Can be edited. Changes will be preserved.
#
#######################################################################
json_AST_transformation_table = {
# AST Transformations for the json-grammar
"<": flatten,
"json": [],
"element": [],
"value": [],
"object": [],
"member": [],
"array": [],
"string": [],
"number": [],
"CHARACTERS": [],
"ESCAPE": [],
"HEX": [],
"INT": [],
"FRAC": [],
"EXP": [],
"EOF": [],
"*": replace_by_single_child
}
def CreatejsonTransformer() -> TransformationFunc:
"""Creates a transformation function that does not share state with other
threads or processes."""
return partial(traverse, processing_table=json_AST_transformation_table.copy())
def get_transformer() -> TransformationFunc:
"""Returns a thread/process-exclusive transformation function."""
try:
transformer = GLOBALS.json_00000001_transformer_singleton
except AttributeError:
GLOBALS.json_00000001_transformer_singleton = CreatejsonTransformer()
transformer = GLOBALS.json_00000001_transformer_singleton
return transformer
#######################################################################
#
# COMPILER SECTION - Can be edited. Changes will be preserved.
#
#######################################################################
class jsonCompiler(Compiler):
"""Compiler for the abstract-syntax-tree of a json source file.
"""
def __init__(self):
super(jsonCompiler, self).__init__()
def _reset(self):
super()._reset()
# initialize your variables here, not in the constructor!
def on_json(self, node):
return self.fallback_compiler(node)
# def on_element(self, node):
# return node
# def on_value(self, node):
# return node
# def on_object(self, node):
# return node
# def on_member(self, node):
# return node
# def on_array(self, node):
# return node
# def on_string(self, node):
# return node
# def on_number(self, node):
# return node
# def on_CHARACTERS(self, node):
# return node
# def on_ESCAPE(self, node):
# return node
# def on_HEX(self, node):
# return node
# def on_INT(self, node):
# return node
# def on_FRAC(self, node):
# return node
# def on_EXP(self, node):
# return node
# def on_EOF(self, node):
# return node
def get_compiler() -> jsonCompiler:
"""Returns a thread/process-exclusive jsonCompiler-singleton."""
try:
compiler = GLOBALS.json_00000001_compiler_singleton
except AttributeError:
GLOBALS.json_00000001_compiler_singleton = jsonCompiler()
compiler = GLOBALS.json_00000001_compiler_singleton
return compiler
#######################################################################
#
# END OF DHPARSER-SECTIONS
#
#######################################################################
def compile_src(source, log_dir=''):
"""Compiles ``source`` and returns (result, errors, ast).
"""
with logging(log_dir):
compiler = get_compiler()
result_tuple = compile_source(source, get_preprocessor(),
get_grammar(),
get_transformer(), compiler)
return result_tuple
if __name__ == "__main__":
# recompile grammar if needed
grammar_path = os.path.abspath(__file__).replace('Compiler.py', '.ebnf')
if os.path.exists(grammar_path):
if not recompile_grammar(grammar_path, force=False,
notify=lambda:print('recompiling ' + grammar_path)):
error_file = os.path.basename(__file__).replace('Compiler.py', '_ebnf_ERRORS.txt')
with open(error_file, encoding="utf-8") as f:
print(f.read())
sys.exit(1)
else:
print('Could not check whether grammar requires recompiling, '
'because grammar was not found at: ' + grammar_path)
if len(sys.argv) > 1:
# compile file
file_name, log_dir = sys.argv[1], ''
if file_name in ['-d', '--debug'] and len(sys.argv) > 2:
file_name, log_dir = sys.argv[2], 'LOGS'
result, errors, _ = compile_src(file_name, log_dir)
if errors:
cwd = os.getcwd()
rel_path = file_name[len(cwd):] if file_name.startswith(cwd) else file_name
for error in errors:
print(rel_path + ':' + str(error))
sys.exit(1)
else:
print(result.as_xml() if isinstance(result, Node) else result)
else:
print("Usage: jsonCompiler.py [FILENAME]")
#!/usr/bin/python3
"""tst_json_grammar.py - runs the unit tests for the json-grammar
"""
import os
import sys
LOGGING = False
sys.path.append(r'/home/eckhart/Entwicklung/DHParser')
scriptpath = os.path.dirname(__file__)
try:
from DHParser import dsl
import DHParser.log
from DHParser import testing
except ModuleNotFoundError:
print('Could not import DHParser. Please adjust sys.path in file '
'"%s" manually' % __file__)
sys.exit(1)
def recompile_grammar(grammar_src, force):
grammar_tests_dir = os.path.join(scriptpath, 'grammar_tests')
testing.create_test_templates(grammar_src, grammar_tests_dir)
with DHParser.log.logging(False):
# recompiles Grammar only if it has changed
if not dsl.recompile_grammar(grammar_src, force=force,
notify=lambda: print('recompiling ' + grammar_src)):
print('\nErrors while recompiling "%s":' % grammar_src +
'\n--------------------------------------\n\n')
with open('json_ebnf_ERRORS.txt') as f:
print(f.read())
sys.exit(1)
def run_grammar_tests(glob_pattern):
with DHParser.log.logging(LOGGING):
error_report = testing.grammar_suite(
os.path.join(scriptpath, 'grammar_tests'),
get_grammar, get_transformer,
fn_patterns=[glob_pattern], report=True, verbose=True)
return error_report
if __name__ == '__main__':
argv = sys.argv[:]
if len(argv) > 1 and sys.argv[1] == "--debug":
LOGGING = True
del argv[1]
if (len(argv) >= 2 and (argv[1].endswith('.ebnf') or
os.path.splitext(argv[1])[1].lower() in testing.TEST_READERS.keys())):
# if called with a single filename that is either an EBNF file or a known
# test file type then use the given argument
arg = argv[1]
else:
# otherwise run all tests in the test directory
arg = '*_test_*.ini'
if arg.endswith('.ebnf'):
recompile_grammar(arg, force=True)
else:
recompile_grammar(os.path.join(scriptpath, 'json.ebnf'),
force=False)
sys.path.append('.')
from jsonCompiler import get_grammar, get_transformer
error_report = run_grammar_tests(glob_pattern=arg)
if error_report:
print('\n')
print(error_report)
sys.exit(1)
print('ready.\n')
......@@ -18,11 +18,17 @@ implied. See the License for the specific language governing
permissions and limitations under the License.
"""
# TODO: This is still a stub...
import os
import sys
scriptdir = os.path.dirname(os.path.realpath(__file__))
i = scriptdir.find('DHParser')
if i >= 0:
dhparserdir = scriptdir[:i + 8]
sys.path.append(dhparserdir)
else:
dhparserdir = ''
from DHParser.compile import compile_source
from DHParser.dsl import compileDSL, compile_on_disk # , recompile_grammar
from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compiler
......@@ -32,8 +38,6 @@ from typing import cast
LOGGING = False
dhparserdir = os.path.dirname(os.path.realpath(__file__))
EBNF_TEMPLATE = r"""-grammar
#######################################################################
......
File mode changed from 100644 to 100755
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment