Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
D
DHParser
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Issues
0
Issues
0
List
Boards
Labels
Service Desk
Milestones
Iterations
Merge Requests
0
Merge Requests
0
Requirements
Requirements
List
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Operations
Operations
Incidents
Analytics
Analytics
Code Review
Insights
Issue
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
badw-it
DHParser
Commits
ff94e91c
Commit
ff94e91c
authored
Sep 18, 2017
by
Eckhart Arnold
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
- cleanup of error handling
parent
7fa46da0
Changes
10
Hide whitespace changes
Inline
Side-by-side
Showing
10 changed files
with
73 additions
and
192 deletions
+73
-192
DHParser/dsl.py
DHParser/dsl.py
+10
-9
DHParser/ebnf.py
DHParser/ebnf.py
+17
-22
DHParser/syntaxtree.py
DHParser/syntaxtree.py
+18
-70
DHParser/toolkit.py
DHParser/toolkit.py
+2
-26
examples/LaTeX/LaTeX.ebnf
examples/LaTeX/LaTeX.ebnf
+6
-1
examples/LaTeX/LaTeXCompiler.py
examples/LaTeX/LaTeXCompiler.py
+7
-2
examples/MLW/MLW.ebnf
examples/MLW/MLW.ebnf
+0
-1
examples/MLW/recompile_grammar.py
examples/MLW/recompile_grammar.py
+0
-49
test/test_ebnf.py
test/test_ebnf.py
+12
-11
test/test_parser.py
test/test_parser.py
+1
-1
No files found.
DHParser/dsl.py
View file @
ff94e91c
...
...
@@ -26,16 +26,16 @@ try:
except
ImportError
:
import
re
try
:
from
typing
import
Any
,
cast
,
Tuple
,
Union
,
Iterable
from
typing
import
Any
,
cast
,
Tuple
,
Union
,
Itera
tor
,
Itera
ble
except
ImportError
:
from
.typing34
import
Any
,
cast
,
Tuple
,
Union
,
Iterable
from
.typing34
import
Any
,
cast
,
Tuple
,
Union
,
Itera
tor
,
Itera
ble
from
DHParser.ebnf
import
EBNFCompiler
,
grammar_changed
,
\
get_ebnf_preprocessor
,
get_ebnf_grammar
,
get_ebnf_transformer
,
get_ebnf_compiler
,
\
PreprocessorFactoryFunc
,
ParserFactoryFunc
,
TransformerFactoryFunc
,
CompilerFactoryFunc
from
DHParser.toolkit
import
logging
,
load_if_file
,
is_python_code
,
compile_python_object
from
DHParser.parser
import
Grammar
,
Compiler
,
compile_source
,
nil_preprocessor
,
PreprocessorFunc
from
DHParser.syntaxtree
import
Error
,
is_error
,
has_errors
,
Node
,
TransformationFunc
from
DHParser.syntaxtree
import
Error
,
is_error
,
has_errors
,
only_errors
,
Node
,
TransformationFunc
__all__
=
(
'GrammarError'
,
'CompilationError'
,
...
...
@@ -125,7 +125,8 @@ class DSLException(Exception):
Base class for DSL-exceptions.
"""
def
__init__
(
self
,
errors
):
assert
isinstance
(
errors
,
list
)
or
isinstance
(
errors
,
tuple
)
assert
isinstance
(
errors
,
Iterator
)
or
isinstance
(
errors
,
list
)
\
or
isinstance
(
errors
,
tuple
)
self
.
errors
=
errors
def
__str__
(
self
):
...
...
@@ -180,7 +181,7 @@ def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
parser_py
,
messages
,
AST
=
compile_source
(
grammar_src
,
None
,
get_ebnf_grammar
(),
get_ebnf_transformer
(),
get_ebnf_compiler
())
if
has_errors
(
messages
):
raise
GrammarError
(
messages
,
grammar_src
)
raise
GrammarError
(
only_errors
(
messages
)
,
grammar_src
)
parser_root
=
compile_python_object
(
DHPARSER_IMPORTS
+
parser_py
,
'\w+Grammar$'
)()
else
:
# assume that dsl_grammar is a ParserHQ-object or Grammar class
...
...
@@ -214,7 +215,7 @@ def compileDSL(text_or_file: str,
ast_transformation
,
compiler
)
if
has_errors
(
messages
):
src
=
load_if_file
(
text_or_file
)
raise
CompilationError
(
messages
,
src
,
grammar_src
,
AST
,
result
)
raise
CompilationError
(
only_errors
(
messages
)
,
src
,
grammar_src
,
AST
,
result
)
return
result
...
...
@@ -317,7 +318,7 @@ def load_compiler_suite(compiler_suite: str) -> \
compile_py
,
messages
,
AST
=
compile_source
(
source
,
None
,
get_ebnf_grammar
(),
get_ebnf_transformer
(),
get_ebnf_compiler
())
if
has_errors
(
messages
):
raise
GrammarError
(
messages
,
source
)
raise
GrammarError
(
only_errors
(
messages
)
,
source
)
preprocessor
=
get_ebnf_preprocessor
parser
=
get_ebnf_grammar
ast
=
get_ebnf_transformer
...
...
@@ -533,9 +534,9 @@ def recompile_grammar(ebnf_filename, force=False) -> bool:
messages
=
compile_on_disk
(
ebnf_filename
)
if
messages
:
# print("Errors while compiling: " + ebnf_filename + '!')
with
open
(
error_file_name
,
'w'
)
as
f
:
with
open
(
error_file_name
,
'w'
,
encoding
=
"UTF-8"
)
as
f
:
for
e
in
messages
:
f
.
write
(
e
)
f
.
write
(
str
(
e
)
)
f
.
write
(
'
\n
'
)
if
has_errors
(
messages
):
return
False
...
...
DHParser/ebnf.py
View file @
ff94e91c
...
...
@@ -33,7 +33,7 @@ from DHParser.toolkit import load_if_file, escape_re, md5, sane_parser_name
from
DHParser.parser
import
Grammar
,
mixin_comment
,
nil_preprocessor
,
Forward
,
RE
,
NegativeLookahead
,
\
Alternative
,
Series
,
Option
,
Required
,
OneOrMore
,
ZeroOrMore
,
Token
,
Compiler
,
\
PreprocessorFunc
from
DHParser.syntaxtree
import
WHITESPACE_PTYPE
,
TOKEN_PTYPE
,
Node
,
TransformationFunc
from
DHParser.syntaxtree
import
WHITESPACE_PTYPE
,
TOKEN_PTYPE
,
Error
,
Node
,
TransformationFunc
from
DHParser.transform
import
TransformationDict
,
traverse
,
remove_brackets
,
\
reduce_single_child
,
replace_by_single_child
,
remove_expendables
,
\
remove_tokens
,
flatten
,
forbid
,
assert_content
,
remove_infix_operator
...
...
@@ -397,8 +397,7 @@ class EBNFCompiler(Compiler):
'literalws'
:
[
'right'
],
'tokens'
:
set
(),
# alt. 'preprocessor_tokens'
'filter'
:
dict
(),
# alt. 'filter'
'ignorecase'
:
False
,
'testing'
:
False
}
'ignorecase'
:
False
}
@
property
def
result
(
self
)
->
str
:
...
...
@@ -544,22 +543,18 @@ class EBNFCompiler(Compiler):
# check for unconnected rules
if
not
self
.
directives
[
'testing'
]:
defined_symbols
.
difference_update
(
self
.
RESERVED_SYMBOLS
)
def
remove_connections
(
symbol
):
if
symbol
in
defined_symbols
:
defined_symbols
.
remove
(
symbol
)
for
related
in
self
.
rules
[
symbol
][
1
:]:
remove_connections
(
str
(
related
))
remove_connections
(
self
.
root_symbol
)
for
leftover
in
defined_symbols
:
self
.
rules
[
leftover
][
0
].
add_error
((
'Rule "%s" is not connected to parser '
'root "%s" !'
)
%
(
leftover
,
self
.
root_symbol
)
+
' (Use directive "@testing=True" '
'to supress this error message.)'
)
# root_node.error_flag = True
defined_symbols
.
difference_update
(
self
.
RESERVED_SYMBOLS
)
def
remove_connections
(
symbol
):
if
symbol
in
defined_symbols
:
defined_symbols
.
remove
(
symbol
)
for
related
in
self
.
rules
[
symbol
][
1
:]:
remove_connections
(
str
(
related
))
remove_connections
(
self
.
root_symbol
)
for
leftover
in
defined_symbols
:
self
.
rules
[
leftover
][
0
].
add_error
((
'Rule "%s" is not connected to '
'parser root "%s" !'
)
%
(
leftover
,
self
.
root_symbol
),
Error
.
WARNING
)
# set root_symbol parser and assemble python grammar definition
...
...
@@ -679,9 +674,9 @@ class EBNFCompiler(Compiler):
if
value
:
self
.
re_flags
.
add
(
'i'
)
elif
key
==
'testing'
:
value
=
str
(
node
.
children
[
1
])
self
.
directives
[
'testing'
]
=
value
.
lower
()
not
in
{
"off"
,
"false"
,
"no"
}
#
elif key == 'testing':
#
value = str(node.children[1])
#
self.directives['testing'] = value.lower() not in {"off", "false", "no"}
elif
key
==
'literalws'
:
value
=
{
item
.
lower
()
for
item
in
self
.
compile
(
node
.
children
[
1
])}
...
...
DHParser/syntaxtree.py
View file @
ff94e91c
...
...
@@ -27,10 +27,10 @@ except ImportError:
import
re
try
:
from
typing
import
AbstractSet
,
Any
,
ByteString
,
Callable
,
cast
,
Container
,
Dict
,
\
Iterator
,
Iterable
,
List
,
NamedTuple
,
Sequence
,
Union
,
Text
,
Tuple
Iterator
,
Iterable
,
List
,
NamedTuple
,
Sequence
,
Union
,
Text
,
Tuple
,
Hashable
except
ImportError
:
from
.typing34
import
AbstractSet
,
Any
,
ByteString
,
Callable
,
cast
,
Container
,
Dict
,
\
Iterator
,
Iterable
,
List
,
NamedTuple
,
Sequence
,
Union
,
Text
,
Tuple
Iterator
,
Iterable
,
List
,
NamedTuple
,
Sequence
,
Union
,
Text
,
Tuple
,
Hashable
from
DHParser.toolkit
import
is_logging
,
log_dir
,
StringView
,
linebreaks
,
line_col
,
identity
...
...
@@ -133,7 +133,7 @@ class Error:
ERROR
=
1000
HIGHEST
=
ERROR
def
__init__
(
self
,
message
:
str
,
level
:
int
=
ERROR
,
code
:
str
=
''
):
def
__init__
(
self
,
message
:
str
,
level
:
int
=
ERROR
,
code
:
Hashable
=
0
):
self
.
message
=
message
assert
level
>=
0
self
.
level
=
level
or
Error
.
ERROR
...
...
@@ -143,19 +143,14 @@ class Error:
self
.
column
=
-
1
def
__str__
(
self
):
return
(
"line: %3i, column: %2i"
%
(
self
.
line
,
self
.
column
)
+
", %s: %s"
%
(
self
.
level_str
,
self
.
message
))
@
staticmethod
def
from_template
(
template
:
str
,
level
:
int
=
ERROR
,
content
:
Union
[
tuple
,
dict
]
=
()):
if
isinstance
(
content
,
tuple
):
return
Error
((
template
%
content
)
if
content
else
template
,
level
,
template
)
else
:
return
Error
(
template
.
format
(
**
content
),
level
,
template
)
prefix
=
''
if
self
.
line
>
0
:
prefix
=
"line: %3i, column: %2i, "
%
(
self
.
line
,
self
.
column
)
return
prefix
+
"%s: %s"
%
(
self
.
level_str
,
self
.
message
)
@
property
def
level_str
(
self
):
return
"
warning"
if
is_warning
(
self
.
level
)
else
"e
rror"
return
"
Warning"
if
is_warning
(
self
.
level
)
else
"E
rror"
def
is_warning
(
level
:
int
)
->
bool
:
...
...
@@ -177,6 +172,14 @@ def has_errors(messages: Iterable[Error], level: int=Error.ERROR) -> bool:
return
False
def
only_errors
(
messages
:
Iterable
[
Error
],
level
:
int
=
Error
.
ERROR
)
->
Iterator
[
Error
]:
"""
Returns an Iterator that yields only those messages that have
at least the given error level.
"""
return
(
err
for
err
in
messages
if
err
.
level
>=
level
)
ChildrenType
=
Tuple
[
'Node'
,
...]
StrictResultType
=
Union
[
ChildrenType
,
StringView
,
str
]
...
...
@@ -344,22 +347,8 @@ class Node(collections.abc.Sized):
return
self
.
_errors
.
copy
()
# def add_error(self, error_str: str) -> 'Node':
# assert isinstance(error_str, str)
# self._errors.append(error_str)
# self.error_flag = True
# return self
def
add_error
(
self
:
'Node'
,
template
:
Union
[
str
,
Error
],
level
:
int
=
0
,
content
:
Union
[
tuple
,
dict
]
=
())
->
'Node'
:
if
isinstance
(
template
,
Error
):
assert
not
(
bool
(
level
)
or
bool
(
content
))
self
.
_errors
.
append
(
template
)
else
:
self
.
_errors
.
append
(
Error
.
from_template
(
template
,
level
,
content
))
def
add_error
(
self
,
message
:
str
,
level
:
int
=
Error
.
ERROR
,
code
:
Hashable
=
0
)
->
'Node'
:
self
.
_errors
.
append
(
Error
(
message
,
level
,
code
))
self
.
error_flag
=
max
(
self
.
error_flag
,
self
.
_errors
[
-
1
].
level
)
return
self
...
...
@@ -540,47 +529,6 @@ class Node(collections.abc.Sized):
yield
nd
# def range(self, match_first, match_last):
# """Iterates over the range of nodes, starting from the first
# node for which ``match_first`` becomes True until the first node
# after this one for which ``match_last`` becomes true or until
# the end if it never does.
#
# Args:
# match_first (function): A function that takes as Node
# object as argument and returns True or False
# match_last (function): A function that takes as Node
# object as argument and returns True or False
# Yields:
# Node: all nodes of the tree for which
# ``match_function(node)`` returns True
# """
# def navigate(self, path):
# """Yields the results of all descendant elements matched by
# ``path``, e.g.
# 'd/s' yields 'l' from (d (s l)(e (r x1) (r x2))
# 'e/r' yields 'x1', then 'x2'
# 'e' yields (r x1)(r x2)
#
# Args:
# path (str): The path of the object, e.g. 'a/b/c'. The
# components of ``path`` can be regular expressions
#
# Returns:
# The object at the path, either a string or a Node or
# ``None``, if the path did not match.
# """
# def nav(node, pl):
# if pl:
# return itertools.chain(nav(child, pl[1:]) for child in node.children
# if re.match(pl[0], child.tag_name))
# else:
# return self.result,
# return nav(path.split('/'))
def
tree_size
(
self
)
->
int
:
"""Recursively counts the number of nodes in the tree including the root node."""
return
sum
(
child
.
tree_size
()
for
child
in
self
.
children
)
+
1
...
...
DHParser/toolkit.py
View file @
ff94e91c
...
...
@@ -57,9 +57,6 @@ __all__ = ('logging',
'sv_match'
,
'sv_index'
,
'sv_search'
,
# 'supress_warnings',
# 'warnings',
# 'repr_call',
'linebreaks'
,
'line_col'
,
'error_messages'
,
...
...
@@ -159,7 +156,7 @@ def clear_logs(logfile_types={'.cst', '.ast', '.log'}):
class
StringView
(
collections
.
abc
.
Sized
):
""""A rudimentary StringView class, just enough for the use cases
in pars
w
er.py.
in parser.py.
Slicing Python-strings always yields copies of a segment of the original
string. See: https://mail.python.org/pipermail/python-dev/2008-May/079699.html
...
...
@@ -275,27 +272,6 @@ def sv_search(regex, sv: StringView):
EMPTY_STRING_VIEW
=
StringView
(
''
)
# def repr_call(f, parameter_list) -> str:
# """Turns a list of items into a string resembling the parameter
# list of a function call by omitting default values at the end:
# >>> def f(a, b=1): print(a, b)
# >>> repr_call(f, (5,1))
# 'f(5)'
# >>> repr_call(f, (5,2))
# 'f(5, 2)'
# """
# i = 0
# defaults = f.__defaults__ if f.__defaults__ is not None else []
# for parameter, default in zip(reversed(parameter_list), reversed(defaults)):
# if parameter != default:
# break
# i -= 1
# if i < 0:
# parameter_list = parameter_list[:i]
# name = f.__self__.__class__.__name__ if f.__name__ == '__init__' else f.__name__
# return "%s(%s)" % (name, ", ".merge_children(repr(item) for item in parameter_list))
def
linebreaks
(
text
:
Union
[
StringView
,
str
]):
lb
=
[
-
1
]
i
=
text
.
find
(
'
\n
'
,
0
)
...
...
@@ -344,7 +320,7 @@ def error_messages(source_text, errors) -> List[str]:
string starts with "line: [Line-No], column: [Column-No]
"""
for
err
in
errors
:
if
err
.
pos
>=
0
and
err
.
line
<
0
:
if
err
.
pos
>=
0
and
err
.
line
<
=
0
:
err
.
line
,
err
.
column
=
line_col
(
source_text
,
err
.
pos
)
return
[
str
(
err
)
for
err
in
sorted
(
errors
,
key
=
lambda
err
:
err
.
pos
)]
...
...
examples/LaTeX/LaTeX.ebnf
View file @
ff94e91c
# LaTeX-Grammar for DHParser
@ testing = True
@ whitespace = /[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?/ # optional whitespace, including at most one linefeed
@ comment = /%.*/
########################################################################
#
# outer document structure
#
########################################################################
latexdoc = preamble document
preamble = { [WSPC] command }+
...
...
examples/LaTeX/LaTeXCompiler.py
View file @
ff94e91c
...
...
@@ -49,11 +49,16 @@ class LaTeXGrammar(Grammar):
# LaTeX-Grammar for DHParser
@ testing = True
@ whitespace = /[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?/ # optional whitespace, including at most one linefeed
@ comment = /%.*/
########################################################################
#
# outer document structure
#
########################################################################
latexdoc = preamble document
preamble = { [WSPC] command }+
...
...
@@ -223,7 +228,7 @@ class LaTeXGrammar(Grammar):
paragraph
=
Forward
()
tabular_config
=
Forward
()
text_element
=
Forward
()
source_hash__
=
"
939c094e994677d2ab894169c013cf58
"
source_hash__
=
"
37585004123d6b80ecf8f67217b43479
"
parser_initialization__
=
"upon instantiation"
COMMENT__
=
r
'%.*'
WHITESPACE__
=
r
'[ \t]*(?:\n(?![ \t]*\n)[ \t]*)?'
...
...
examples/MLW/MLW.ebnf
View file @
ff94e91c
# EBNF-Syntax für MLW-Artikel
@ testing = True
@ comment = /#.*/ # Kommentare beginnen mit '#' und reichen bis zum Zeilenende
# ohne das Zeilenende zu beinhalten
...
...
examples/MLW/recompile_grammar.py
deleted
100644 → 0
View file @
7fa46da0
#!/usr/bin/python3
"""recompile_grammar.py - recompiles any .ebnf files in the current
directory if necessary
Author: Eckhart Arnold <arnold@badw.de>
Copyright 2017 Bavarian Academy of Sciences and Humanities
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from
DHParser.dsl
import
recompile_grammar
recompile_grammar
(
'.'
)
# import os
#
# from DHParser.ebnf import grammar_changed
# from DHParser.dsl import compile_on_disk
#
#
# def compile(name):
# base, ext = os.path.splitext(name)
# compiler_name = base + '_compiler.py'
# if (not os.path.exists(compiler_name) or
# grammar_changed(compiler_name, name)):
# print("recompiling parser for: " + name)
# errors = compile_on_disk(name)
# if errors:
# print("Errors while compiling: " + name + '!')
# with open(base + '_errors.txt', 'w') as f:
# for e in errors:
# f.write(e)
# f.write('\n')
#
# for entry in os.listdir():
# if entry.lower().endswith('.ebnf') and os.path.isfile(entry):
# compile(entry)
test/test_ebnf.py
View file @
ff94e91c
...
...
@@ -30,6 +30,7 @@ from multiprocessing import Pool
sys
.
path
.
extend
([
'../'
,
'./'
])
from
DHParser.toolkit
import
compile_python_object
from
DHParser.syntaxtree
import
has_errors
from
DHParser.parser
import
compile_source
,
WHITESPACE_PTYPE
,
nil_preprocessor
from
DHParser.ebnf
import
get_ebnf_grammar
,
get_ebnf_transformer
,
EBNFTransform
,
get_ebnf_compiler
from
DHParser.dsl
import
CompilationError
,
compileDSL
,
DHPARSER_IMPORTS
,
grammar_provider
...
...
@@ -297,13 +298,19 @@ class TestBoundaryCases:
ebnf
=
"""root = /.*/
unconnected = /.*/
"""
try
:
grammar
=
grammar_provider
(
ebnf
)()
assert
False
,
"EBNF compiler should complain about unconnected rules."
except
CompilationError
as
err
:
grammar_src
=
err
.
result
result
,
messages
,
AST
=
compile_source
(
ebnf
,
nil_preprocessor
,
get_ebnf_grammar
(),
get_ebnf_transformer
(),
get_ebnf_compiler
())
if
messages
:
assert
not
has_errors
(
messages
),
"Unconnected rules should result in a warning, "
\
"not an error: "
+
str
(
messages
)
grammar_src
=
result
grammar
=
compile_python_object
(
DHPARSER_IMPORTS
+
grammar_src
,
'get_(?:\w+_)?grammar$'
)()
else
:
assert
False
,
"EBNF compiler should warn about unconnected rules."
assert
grammar
[
'root'
],
"Grammar objects should be subscriptable by parser names!"
try
:
unconnected
=
grammar
[
'unconnected'
]
...
...
@@ -315,12 +322,6 @@ class TestBoundaryCases:
"a non-existant parser name!"
except
KeyError
:
pass
ebnf_testing
=
"@testing = True
\n
"
+
ebnf
try
:
grammar
=
grammar_provider
(
ebnf_testing
)()
except
CompilationError
:
assert
False
,
"EBNF compiler should not complain about unconnected "
\
"rules when directive @testing is set."
class
TestSynonymDetection
:
...
...
test/test_parser.py
View file @
ff94e91c
...
...
@@ -392,7 +392,7 @@ class TestPopRetrieve:
class
TestWhitespaceHandling
:
minilang
=
"""
@testing = True
minilang
=
"""
doc = A B
A = "A"
B = "B"
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment