Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
badw-it
DHParser
Commits
2bdc97c9
Commit
2bdc97c9
authored
Feb 18, 2019
by
eckhart
Browse files
Merge remote-tracking branch 'origin/development' into development
parents
3663e8ae
cf39d143
Changes
10
Hide whitespace changes
Inline
Side-by-side
DHParser/dsl.py
View file @
2bdc97c9
...
...
@@ -470,7 +470,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
f
=
None
if
RX_WHITESPACE
.
fullmatch
(
intro
):
intro
=
'#!/usr/bin/python'
intro
=
'#!/usr/bin/python
3
'
if
RX_WHITESPACE
.
fullmatch
(
outro
):
outro
=
DHPARSER_MAIN
.
format
(
NAME
=
compiler_name
)
if
RX_WHITESPACE
.
fullmatch
(
imports
):
...
...
DHParser/ebnf.py
View file @
2bdc97c9
...
...
@@ -936,7 +936,7 @@ class EBNFCompiler(Compiler):
rule
=
node
.
children
[
0
].
content
if
rule
in
self
.
rules
:
first
=
self
.
rules
[
rule
][
0
]
if
not
first
.
errors
:
if
not
id
(
first
)
in
self
.
tree
.
error
_node
s
:
self
.
tree
.
new_error
(
first
,
'First definition of rule "%s" '
'followed by illegal redefinitions.'
%
rule
)
self
.
tree
.
new_error
(
node
,
'A rule "%s" has already been defined earlier.'
%
rule
)
...
...
DHParser/error.py
View file @
2bdc97c9
...
...
@@ -73,7 +73,7 @@ class Error:
REDECLARED_TOKEN_WARNING
=
ErrorCode
(
120
)
UNUSED_ERROR_HANDLING_WARNING
=
ErrorCode
(
130
)
LEFT_RECURSION_WARING
=
ErrorCode
(
140
)
LEFT_RECURSION_WAR
N
ING
=
ErrorCode
(
140
)
UNDEFINED_SYMBOL_IN_TRANSTABLE_WARNING
=
ErrorCode
(
610
)
...
...
DHParser/log.py
View file @
2bdc97c9
...
...
@@ -52,12 +52,12 @@ import collections
import
contextlib
import
html
import
os
from
typing
import
List
,
Tuple
,
Union
,
Optional
from
DHParser.error
import
Error
from
DHParser.stringview
import
StringView
from
DHParser.syntaxtree
import
Node
from
DHParser.toolkit
import
is_filename
,
escape_control_characters
,
GLOBALS
,
typing
from
typing
import
List
,
Tuple
,
Union
from
DHParser.toolkit
import
is_filename
,
escape_control_characters
,
GLOBALS
__all__
=
(
'log_dir'
,
'logging'
,
...
...
@@ -222,13 +222,13 @@ class HistoryRecord:
HTML_LEAD_OUT
=
'
\n
</body>
\n
</html>
\n
'
def
__init__
(
self
,
call_stack
:
List
[
str
],
node
:
Node
,
node
:
Optional
[
Node
]
,
text
:
StringView
,
line_col
:
Tuple
[
int
,
int
],
errors
:
List
[
Error
]
=
[])
->
None
:
# copy call stack, dropping uninformative Forward-Parsers
self
.
call_stack
=
[
tn
for
tn
in
call_stack
if
tn
!=
":Forward"
]
# type: List[str]
self
.
node
=
node
# type: Node
self
.
node
=
node
# type:
Optional[
Node
]
self
.
text
=
text
# type: StringView
self
.
line_col
=
line_col
# type: Tuple[int, int]
self
.
errors
=
errors
# type: List[Error]
...
...
DHParser/parse.py
View file @
2bdc97c9
...
...
@@ -340,10 +340,11 @@ class Parser:
if
location
in
grammar
.
recursion_locations__
:
if
location
in
self
.
visited
:
node
,
rest
=
self
.
visited
[
location
]
if
location
!=
grammar
.
last_recursion_location__
:
grammar
.
tree__
.
add_error
(
node
,
Error
(
"Left recursion encountered. "
"Refactor grammar to avoid slow parsing."
,
node
.
pos
,
Error
.
LEFT_RECURSION_WARING
))
if
node
and
location
!=
grammar
.
last_recursion_location__
:
grammar
.
tree__
.
add_error
(
node
,
Error
(
"Left recursion encountered. "
"Refactor grammar to avoid slow parsing."
,
node
.
pos
,
Error
.
LEFT_RECURSION_WARNING
))
grammar
.
last_recursion_location__
=
location
# don't overwrite any positive match (i.e. node not None) in the cache
# and don't add empty entries for parsers returning from left recursive calls!
...
...
@@ -515,7 +516,7 @@ class UnknownParserError(KeyError):
is referred to that does not exist."""
GrammarErrorType
=
List
[
Tuple
[
str
,
Parser
,
Error
]
]
# TODO: replace with a named tuple?
GrammarErrorType
=
Tuple
[
str
,
Parser
,
Error
]
# TODO: replace with a named tuple?
class
GrammarError
(
Exception
):
...
...
@@ -915,9 +916,9 @@ class Grammar:
"""
last_record
=
self
.
history__
[
-
2
]
if
len
(
self
.
history__
)
>
1
else
None
# type: Optional[HistoryRecord]
return
last_record
and
parser
!=
self
.
root_parser__
\
and
any
(
self
.
history__
[
i
].
status
==
HistoryRecord
.
MATCH
\
and
self
.
history__
[
i
].
node
.
pos
\
+
len
(
self
.
history__
[
i
].
node
)
>=
len
(
self
.
document__
)
\
and
any
(
self
.
history__
[
i
].
status
==
HistoryRecord
.
MATCH
and
((
self
.
history__
[
i
].
node
.
pos
+
len
(
self
.
history__
[
i
].
node
))
if
self
.
history__
[
i
].
node
else
0
)
>=
len
(
self
.
document__
)
and
any
(
tn
in
self
and
isinstance
(
self
[
tn
],
Lookahead
)
or
tn
[
0
]
==
':'
and
issubclass
(
eval
(
tn
[
1
:]),
Lookahead
)
for
tn
in
self
.
history__
[
i
].
call_stack
)
...
...
@@ -1376,7 +1377,7 @@ class MetaParser(Parser):
return
Node
(
self
.
tag_name
,
node
)
return
node
elif
self
.
pname
:
return
Node
(
self
.
tag_name
,
())
# type: Node
return
Node
(
self
.
tag_name
,
())
return
EMPTY_NODE
# avoid creation of a node object for anonymous empty nodes
return
Node
(
self
.
tag_name
,
node
or
())
# unoptimized code
...
...
@@ -1392,7 +1393,7 @@ class MetaParser(Parser):
N
=
len
(
results
)
if
N
>
1
:
if
self
.
grammar
.
flatten_tree__
:
nr
=
[]
nr
=
[]
# type: List[Node]
for
child
in
results
:
if
child
.
children
and
child
.
tag_name
[
0
]
==
':'
:
# faster than c.is_anonymous():
nr
.
extend
(
child
.
children
)
...
...
DHParser/stringview.py
View file @
2bdc97c9
...
...
@@ -42,7 +42,7 @@ try:
cython_optimized
=
cython
.
compiled
# type: bool
except
ImportError
:
# import DHParser.Shadow as cython
cython_optimized
=
False
# type: bool
cython_optimized
=
False
import
DHParser.shadow_cython
as
cython
...
...
DHParser/syntaxtree.py
View file @
2bdc97c9
...
...
@@ -25,11 +25,11 @@ parser classes are defined in the ``parse`` module.
from
collections
import
OrderedDict
import
copy
from
typing
import
Callable
,
cast
,
Iterator
,
List
,
AbstractSet
,
Set
,
Union
,
Tuple
,
Optional
,
Dict
from
DHParser.error
import
Error
,
ErrorCode
,
linebreaks
,
line_col
from
DHParser.stringview
import
StringView
from
DHParser.toolkit
import
get_config_value
,
re
from
typing
import
Callable
,
cast
,
Iterator
,
List
,
AbstractSet
,
Set
,
Union
,
Tuple
,
Optional
__all__
=
(
'WHITESPACE_PTYPE'
,
...
...
@@ -544,9 +544,9 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
left_bracket
,
right_bracket
,
density
=
(
''
,
''
,
1
)
if
compact
else
(
'('
,
'
\n
)'
,
0
)
lbreaks
=
linebreaks
(
src
)
if
src
else
[]
# type: List[int]
root
=
cast
(
RootNode
,
self
)
if
isinstance
(
self
,
RootNode
)
else
None
# type: Optional[Node]
root
=
cast
(
RootNode
,
self
)
if
isinstance
(
self
,
RootNode
)
else
None
# type: Optional[
Root
Node]
def
opening
(
node
)
->
str
:
def
opening
(
node
:
Node
)
->
str
:
"""Returns the opening string for the representation of `node`."""
txt
=
[
left_bracket
,
node
.
tag_name
]
# s += " '(pos %i)" % node.add_pos
...
...
@@ -560,11 +560,11 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
txt
.
append
(
" `(err `%s)"
%
' '
.
join
(
str
(
err
)
for
err
in
root
.
get_errors
(
node
)))
return
""
.
join
(
txt
)
+
'
\n
'
def
closing
(
node
)
->
str
:
def
closing
(
node
:
Node
)
->
str
:
"""Returns the closing string for the representation of `node`."""
return
right_bracket
def
pretty
(
strg
)
:
def
pretty
(
strg
:
str
)
->
str
:
"""Encloses `strg` with the right kind of quotation marks."""
return
'"%s"'
%
strg
if
strg
.
find
(
'"'
)
<
0
\
else
"'%s'"
%
strg
if
strg
.
find
(
"'"
)
<
0
\
...
...
@@ -596,9 +596,9 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
empty_tags: A set of tags which shall be rendered as empty elements, e.g.
"<empty/>" instead of "<empty><empty>".
"""
root
=
cast
(
RootNode
,
self
)
if
isinstance
(
self
,
RootNode
)
else
None
# type: Optional[Node]
root
=
cast
(
RootNode
,
self
)
if
isinstance
(
self
,
RootNode
)
else
None
# type: Optional[
Root
Node]
def
opening
(
node
)
->
str
:
def
opening
(
node
:
Node
)
->
str
:
"""Returns the opening string for the representation of `node`."""
if
node
.
tag_name
in
omit_tags
:
return
''
...
...
@@ -620,7 +620,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
ending
=
">
\n
"
return
""
.
join
(
txt
+
[
ending
])
def
closing
(
node
):
def
closing
(
node
:
Node
):
"""Returns the closing string for the representation of `node`."""
if
node
.
tag_name
in
omit_tags
or
node
.
tag_name
in
empty_tags
:
return
''
...
...
@@ -632,8 +632,7 @@ class Node: # (collections.abc.Sized): Base class omitted for cython-compatibil
content
=
content
.
replace
(
'<'
,
'<'
).
replace
(
'>'
,
'>'
)
return
content
def
inlining
(
node
):
def
inlining
(
node
:
Node
):
"""Returns True, if `node`'s tag name is contained in `inline_tags`,
thereby signalling that the children of this node shall not be
printed on several lines to avoid unwanted gaps in the output.
...
...
@@ -808,7 +807,7 @@ def tree_sanity_check(tree: Node) -> bool:
:param tree: the root of the tree to be checked
:return: True, if the tree is `sane`, False otherwise.
"""
node_set
=
set
()
node_set
=
set
()
# type: Set[Node]
for
node
in
tree
.
select
(
lambda
nd
:
True
,
include_root
=
True
):
if
node
in
node_set
or
isinstance
(
Node
,
FrozenNode
):
return
False
...
...
@@ -832,6 +831,12 @@ class RootNode(Node):
processing (i.e. parsing, AST-transformation, compiling)
of this tree.
error_nodes (dict): A mapping of node-ids to a list of errors that
occurred on the node with the respective id.
error_positions (dict): A mapping of locations to a set of ids of
nodes that contain an error at that particular location
error_flag (int): the highest warning or error level of all errors
that occurred.
...
...
@@ -844,7 +849,7 @@ class RootNode(Node):
def
__init__
(
self
,
node
:
Optional
[
Node
]
=
None
):
super
().
__init__
(
'__not_yet_ready__'
,
''
)
self
.
errors
=
[]
# type: List[Error]
self
.
errors
=
[]
# type: List[Error]
self
.
error_nodes
=
dict
()
# type: Dict[int, List[Error]] # id(node) -> error list
self
.
error_positions
=
dict
()
# type: Dict[int, Set[int]] # pos -> set of id(node)
self
.
error_flag
=
0
...
...
@@ -852,8 +857,8 @@ class RootNode(Node):
self
.
swallow
(
node
)
# customization for XML-Representation
self
.
inline_tags
=
set
()
# type: Set[str]
self
.
omit_tags
=
set
()
# type: Set[str]
self
.
empty_tags
=
set
()
# type: Set[str]
self
.
omit_tags
=
set
()
# type: Set[str]
self
.
empty_tags
=
set
()
# type: Set[str]
def
__deepcopy__
(
self
,
memodict
=
{}):
duplicate
=
self
.
__class__
(
None
)
...
...
DHParser/toolkit.py
View file @
2bdc97c9
...
...
@@ -42,7 +42,7 @@ try:
if
cython_optimized
:
import
DHParser.shadow_cython
as
cython
except
ImportError
:
cython_optimized
=
False
# type: bool
cython_optimized
=
False
import
DHParser.shadow_cython
as
cython
from
DHParser.configuration
import
CONFIG_PRESET
...
...
DHParser/transform.py
View file @
2bdc97c9
...
...
@@ -591,7 +591,7 @@ def flatten_anonymous_nodes(context: List[Node]):
"""
node
=
context
[
-
1
]
if
node
.
children
:
new_result
=
[]
new_result
=
[]
# type: List[Node]
for
child
in
node
.
children
:
if
child
.
is_anonymous
():
if
child
.
children
:
...
...
@@ -819,22 +819,35 @@ def merge_whitespace(context):
@
transformation_factory
(
collections
.
abc
.
Callable
)
def
move_adjacent
(
context
,
condition
:
Callable
=
is_insignificant_whitespace
):
def
move_adjacent
(
context
:
List
[
Node
]
,
condition
:
Callable
=
is_insignificant_whitespace
):
"""
Moves adjacent nodes that fulfill the given condition to the parent node.
"""
def
join_results
(
a
:
Node
,
b
:
Node
,
c
:
Node
)
->
bool
:
"""Joins the results of node `a` and `b` and write them to the result
of `c` type-safely, if possible. Return True, if join was possible
and done, False otherwise."""
if
a
.
children
and
b
.
children
:
c
.
result
=
cast
(
Tuple
[
Node
,
...],
a
.
result
)
+
cast
(
Tuple
[
Node
,
...],
b
.
result
)
return
True
elif
not
a
.
children
and
not
b
.
children
:
c
.
result
=
cast
(
str
,
a
.
result
)
+
cast
(
str
,
b
.
result
)
return
True
return
False
node
=
context
[
-
1
]
if
len
(
context
)
<=
1
or
not
node
.
children
:
return
parent
=
context
[
-
2
]
children
=
node
.
children
if
condition
([
children
[
0
]]):
before
=
(
children
[
0
],)
before
=
(
children
[
0
],)
# type: Tuple[Node, ...]
children
=
children
[
1
:]
else
:
before
=
()
if
children
and
condition
([
children
[
-
1
]]):
after
=
(
children
[
-
1
],)
after
=
(
children
[
-
1
],)
# type: Tuple[Node, ...]
children
=
children
[:
-
1
]
else
:
after
=
tuple
()
...
...
@@ -849,11 +862,15 @@ def move_adjacent(context, condition: Callable = is_insignificant_whitespace):
prevN
=
parent
.
children
[
i
-
1
]
if
i
>
0
else
None
nextN
=
parent
.
children
[
i
+
1
]
if
i
<
len
(
parent
.
children
)
-
1
else
None
if
before
and
prevN
and
condition
([
prevN
]):
prevN
.
result
=
prevN
.
result
+
before
[
0
].
result
before
=
()
# prevN.result = prevN.result + before[0].result
# before = ()
if
join_results
(
prevN
,
before
[
0
],
prevN
):
before
=
()
if
after
and
nextN
and
condition
([
nextN
]):
nextN
.
result
=
after
[
0
].
result
+
nextN
.
result
after
=
()
# nextN.result = after[0].result + nextN.result
# after = ()
if
join_results
(
after
[
0
],
nextN
,
nextN
):
after
=
()
parent
.
result
=
parent
.
children
[:
i
]
+
before
+
(
node
,)
+
after
+
parent
.
children
[
i
+
1
:]
...
...
setup.py
View file @
2bdc97c9
...
...
@@ -52,7 +52,7 @@ setup(
],
entry_points
=
{
'console_scripts'
:
[
'dhparser = dhparser'
'dhparser = dhparser
.py
'
]
}
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment