Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
badw-it
DHParser
Commits
d9fbc53c
Commit
d9fbc53c
authored
Feb 25, 2019
by
Eckhart Arnold
Browse files
- BUG: named nodes might be dropped by Series (fixed), ZeroOrMore and potentially other parsers
parent
6ef3b9fe
Changes
8
Show whitespace changes
Inline
Side-by-side
DHParser/ebnf.py
View file @
d9fbc53c
...
@@ -99,7 +99,7 @@ from DHParser import logging, is_filename, load_if_file, \\
...
@@ -99,7 +99,7 @@ from DHParser import logging, is_filename, load_if_file, \\
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last,
\\
keep_children, is_one_of, not_one_of, has_content, apply_if, remove_first, remove_last,
\\
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip,
\\
remove_anonymous_empty, keep_nodes, traverse_locally, strip, lstrip, rstrip,
\\
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator,
\\
replace_content, replace_content_by, forbid, assert_content, remove_infix_operator,
\\
error_on, recompile_grammar, GLOBALS
error_on, recompile_grammar,
left_associative, swing_left,
GLOBALS
'''
.
format
(
dhparserdir
=
dhparserdir
)
'''
.
format
(
dhparserdir
=
dhparserdir
)
...
...
DHParser/parse.py
View file @
d9fbc53c
...
@@ -1748,7 +1748,7 @@ class Series(NaryParser):
...
@@ -1748,7 +1748,7 @@ class Series(NaryParser):
else
:
else
:
results
+=
(
node
,)
results
+=
(
node
,)
break
break
if
node
.
_result
or
parser
.
pname
:
# optimization
if
node
.
_result
or
parser
.
pname
or
node
.
tag_name
[
0
:
1
]
!=
':'
:
# optimization
results
+=
(
node
,)
results
+=
(
node
,)
# assert len(results) <= len(self.parsers) \
# assert len(results) <= len(self.parsers) \
# or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG])
# or len(self.parsers) >= len([p for p in results if p.tag_name != ZOMBIE_TAG])
...
...
DHParser/transform.py
View file @
d9fbc53c
...
@@ -58,6 +58,8 @@ __all__ = ('TransformationDict',
...
@@ -58,6 +58,8 @@ __all__ = ('TransformationDict',
'replace_content_by'
,
'replace_content_by'
,
'normalize_whitespace'
,
'normalize_whitespace'
,
'move_adjacent'
,
'move_adjacent'
,
'left_associative'
,
'swing_left'
,
'apply_if'
,
'apply_if'
,
'apply_unless'
,
'apply_unless'
,
'traverse_locally'
,
'traverse_locally'
,
...
@@ -878,12 +880,12 @@ def left_associative(context: List[Node]):
...
@@ -878,12 +880,12 @@ def left_associative(context: List[Node]):
Rearranges a flat node with infix operators into a left associative tree.
Rearranges a flat node with infix operators into a left associative tree.
"""
"""
node
=
context
[
-
1
]
node
=
context
[
-
1
]
assert
len
(
node
.
children
)
>=
3
if
len
(
node
.
children
)
>=
3
:
assert
(
len
(
node
.
children
)
+
1
)
%
2
==
0
assert
(
len
(
node
.
children
)
+
1
)
%
2
==
0
rest
=
list
(
node
.
result
)
rest
=
list
(
node
.
result
)
left
,
rest
=
rest
[
0
],
rest
[
1
:]
left
,
rest
=
rest
[
0
],
rest
[
1
:]
while
rest
:
while
rest
:
infix
,
right
,
rest
=
rest
[
:
2
],
rest
[
2
:]
infix
,
right
,
rest
=
rest
[
0
],
rest
[
1
],
rest
[
2
:]
assert
not
infix
.
children
assert
not
infix
.
children
assert
infix
.
tag_name
[
0
:
1
]
!=
":"
assert
infix
.
tag_name
[
0
:
1
]
!=
":"
left
=
Node
(
infix
.
tag_name
,
(
left
,
right
))
left
=
Node
(
infix
.
tag_name
,
(
left
,
right
))
...
@@ -891,7 +893,7 @@ def left_associative(context: List[Node]):
...
@@ -891,7 +893,7 @@ def left_associative(context: List[Node]):
@
transformation_factory
(
collections
.
abc
.
Set
)
@
transformation_factory
(
collections
.
abc
.
Set
)
def
left_
swing
(
context
:
List
[
Node
],
operators
:
AbstractSet
[
str
]):
def
swing
_left
(
context
:
List
[
Node
],
operators
:
AbstractSet
[
str
]):
"""
"""
Rearranges a node that contains a sub-node on the right
Rearranges a node that contains a sub-node on the right
with a left-associative operator so that the tree structure
with a left-associative operator so that the tree structure
...
...
examples/Arithmetic/Arithmetic.ebnf
View file @
d9fbc53c
...
@@ -19,7 +19,7 @@
...
@@ -19,7 +19,7 @@
#######################################################################
#######################################################################
expression = term { (PLUS|MINUS) term}
expression = term { (PLUS|MINUS) term}
term = factor
{ (DIV|
[
MUL
]
) factor}
term = factor { (DIV|MUL) factor}
factor = [sign] ( NUMBER | VARIABLE | group )
factor = [sign] ( NUMBER | VARIABLE | group )
sign = POSITIVE | NEGATIVE
sign = POSITIVE | NEGATIVE
group = "(" expression ")"
group = "(" expression ")"
...
@@ -32,7 +32,7 @@ group = "(" expression ")"
...
@@ -32,7 +32,7 @@ group = "(" expression ")"
PLUS = "+"
PLUS = "+"
MINUS = "-"
MINUS = "-"
MUL = "*"
MUL = "*"
| &factor # TODO: higher precedence of &factor
DIV = "/"
DIV = "/"
POSITIVE = /[+]/ # no implicit whitespace after signs
POSITIVE = /[+]/ # no implicit whitespace after signs
...
...
examples/Arithmetic/ArithmeticCompiler.py
View file @
d9fbc53c
...
@@ -33,7 +33,7 @@ from DHParser import logging, is_filename, load_if_file, \
...
@@ -33,7 +33,7 @@ from DHParser import logging, is_filename, load_if_file, \
keep_children
,
is_one_of
,
not_one_of
,
has_content
,
apply_if
,
remove_first
,
remove_last
,
\
keep_children
,
is_one_of
,
not_one_of
,
has_content
,
apply_if
,
remove_first
,
remove_last
,
\
remove_anonymous_empty
,
keep_nodes
,
traverse_locally
,
strip
,
lstrip
,
rstrip
,
\
remove_anonymous_empty
,
keep_nodes
,
traverse_locally
,
strip
,
lstrip
,
rstrip
,
\
replace_content
,
replace_content_by
,
forbid
,
assert_content
,
remove_infix_operator
,
\
replace_content
,
replace_content_by
,
forbid
,
assert_content
,
remove_infix_operator
,
\
error_on
,
recompile_grammar
,
GLOBALS
error_on
,
recompile_grammar
,
left_associative
,
GLOBALS
#######################################################################
#######################################################################
...
@@ -59,7 +59,7 @@ class ArithmeticGrammar(Grammar):
...
@@ -59,7 +59,7 @@ class ArithmeticGrammar(Grammar):
r
"""Parser for an Arithmetic source file.
r
"""Parser for an Arithmetic source file.
"""
"""
expression
=
Forward
()
expression
=
Forward
()
source_hash__
=
"
9f06b2623e1d797c32efc3b864fec5bd
"
source_hash__
=
"
a8142ddc723ae56bbdf6c898efc7af45
"
static_analysis_pending__
=
[
True
]
static_analysis_pending__
=
[
True
]
parser_initialization__
=
[
"upon instantiation"
]
parser_initialization__
=
[
"upon instantiation"
]
resume_rules__
=
{}
resume_rules__
=
{}
...
@@ -79,7 +79,7 @@ class ArithmeticGrammar(Grammar):
...
@@ -79,7 +79,7 @@ class ArithmeticGrammar(Grammar):
group
=
Series
(
Series
(
DropToken
(
"("
),
dwsp__
),
expression
,
Series
(
DropToken
(
")"
),
dwsp__
))
group
=
Series
(
Series
(
DropToken
(
"("
),
dwsp__
),
expression
,
Series
(
DropToken
(
")"
),
dwsp__
))
sign
=
Alternative
(
POSITIVE
,
NEGATIVE
)
sign
=
Alternative
(
POSITIVE
,
NEGATIVE
)
factor
=
Series
(
Option
(
sign
),
Alternative
(
NUMBER
,
VARIABLE
,
group
))
factor
=
Series
(
Option
(
sign
),
Alternative
(
NUMBER
,
VARIABLE
,
group
))
term
=
Series
(
factor
,
ZeroOrMore
(
Series
(
Alternative
(
DIV
,
Option
(
MUL
)
)
,
factor
)))
term
=
Series
(
factor
,
ZeroOrMore
(
Series
(
Alternative
(
DIV
,
MUL
),
factor
)))
expression
.
set
(
Series
(
term
,
ZeroOrMore
(
Series
(
Alternative
(
PLUS
,
MINUS
),
term
))))
expression
.
set
(
Series
(
term
,
ZeroOrMore
(
Series
(
Alternative
(
PLUS
,
MINUS
),
term
))))
root__
=
expression
root__
=
expression
...
@@ -101,17 +101,17 @@ def get_grammar() -> ArithmeticGrammar:
...
@@ -101,17 +101,17 @@ def get_grammar() -> ArithmeticGrammar:
#
#
#######################################################################
#######################################################################
def
group_no_asterix_mul
(
context
:
List
[
Node
]):
pass
# TODO: Find an algorithm, here
Arithmetic_AST_transformation_table
=
{
Arithmetic_AST_transformation_table
=
{
# AST Transformations for the Arithmetic-grammar
# AST Transformations for the Arithmetic-grammar
"<"
:
flatten
,
"expression"
:
[
left_associative
,
replace_by_single_child
],
"expression"
:
[],
"term"
:
[
left_associative
,
replace_by_single_child
],
"term"
:
[
reduce_single_child
],
"factor"
:
[
replace_by_single_child
],
"factor"
:
[
reduce_single_child
],
"group"
:
[
remove_tokens
(
'('
,
')'
),
replace_by_single_child
],
"group"
:
[
remove_tokens
(
'('
,
')'
),
replace_by_single_child
],
"NUMBER"
:
[],
"sign"
:
[
replace_by_single_child
]
"VARIABLE"
:
[],
":Token"
:
reduce_single_child
,
"*"
:
replace_by_single_child
}
}
...
...
examples/Arithmetic/grammar_tests/02_test_components.ini
View file @
d9fbc53c
...
@@ -23,11 +23,13 @@ F2: "- 2"
...
@@ -23,11 +23,13 @@ F2: "- 2"
M1:
"2
*
4"
M1:
"2
*
4"
M2:
"3x"
M2:
"3x"
M3:
"5
/
2"
M3:
"5
/
2"
M4:
"5
/
2x"
M4
*
:
"5
/
2x"
M5:
"5
/
-2x"
M5:
"5
/
-2x"
M6:
"-3*2y"
M6:
"-3*2y"
M7:
"4x"
M7:
"4x"
M8:
"-2x"
M8:
"-2x"
M9:
"20
/
2
*
2"
M10:
"20
/
(2
*
2)"
[fail:term]
[fail:term]
F1:
"2
+
4"
F1:
"2
+
4"
...
@@ -39,6 +41,8 @@ M2: "-5 - -4x"
...
@@ -39,6 +41,8 @@ M2: "-5 - -4x"
M3:
"(a
+
b)(a
-
b)"
M3:
"(a
+
b)(a
-
b)"
M4:
"a
-
5
+
b"
M4:
"a
-
5
+
b"
M5:
"-5
-
+4x"
M5:
"-5
-
+4x"
M6:
"5
-
4
+
3"
M7:
"5
-
(4
+
3)"
[fail:expression]
[fail:expression]
F1:
"-5
-
-
4x"
F1:
"-5
-
-
4x"
...
...
examples/Arithmetic/tst_Arithmetic_grammar.py
View file @
d9fbc53c
...
@@ -24,7 +24,7 @@ except ModuleNotFoundError:
...
@@ -24,7 +24,7 @@ except ModuleNotFoundError:
CONFIG_PRESET
[
'ast_serialization'
]
=
"S-expression"
CONFIG_PRESET
[
'ast_serialization'
]
=
"S-expression"
CONFIG_PRESET
[
'test_parallelization'
]
=
False
def
recompile_grammar
(
grammar_src
,
force
):
def
recompile_grammar
(
grammar_src
,
force
):
grammar_tests_dir
=
os
.
path
.
join
(
scriptpath
,
'grammar_tests'
)
grammar_tests_dir
=
os
.
path
.
join
(
scriptpath
,
'grammar_tests'
)
...
...
test/test_parse.py
View file @
d9fbc53c
...
@@ -119,26 +119,26 @@ class TestInfiLoopsAndRecursion:
...
@@ -119,26 +119,26 @@ class TestInfiLoopsAndRecursion:
log_ST
(
syntax_tree
,
"test_LeftRecursion_indirect.cst"
)
log_ST
(
syntax_tree
,
"test_LeftRecursion_indirect.cst"
)
log_parsing_history
(
parser
,
"test_LeftRecursion_indirect"
)
log_parsing_history
(
parser
,
"test_LeftRecursion_indirect"
)
# BEWARE: EXPERIMENTAL TEST can be long running
#
#
BEWARE: EXPERIMENTAL TEST can be long running
def
test_indirect_left_recursion2
(
self
):
#
def test_indirect_left_recursion2(self):
arithmetic_syntax
=
"""
#
arithmetic_syntax = """
expression = addition | subtraction
#
expression = addition | subtraction
addition = (expression | term) "+" (expression | term)
#
addition = (expression | term) "+" (expression | term)
subtraction = (expression | term) "-" (expression | term)
#
subtraction = (expression | term) "-" (expression | term)
term = multiplication | division
#
term = multiplication | division
multiplication = (term | factor) "*" (term | factor)
#
multiplication = (term | factor) "*" (term | factor)
division = (term | factor) "/" (term | factor)
#
division = (term | factor) "/" (term | factor)
factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
#
factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group }
group = "("
§
expression ")"
#
group = "(" expression ")"
SIGN = /[+-]/
#
SIGN = /[+-]/
NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
#
NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
VARIABLE = /[A-Za-z]/~
#
VARIABLE = /[A-Za-z]/~
"""
#
"""
arithmetic
=
grammar_provider
(
arithmetic_syntax
)()
#
arithmetic = grammar_provider(arithmetic_syntax)()
arithmetic
.
left_recursion_depth__
=
2
#
arithmetic.left_recursion_depth__ = 2
assert
arithmetic
#
assert arithmetic
syntax_tree
=
arithmetic
(
"(a + b) * (a - b)"
)
#
syntax_tree = arithmetic("(a + b) * (a - b)")
assert
syntax_tree
.
errors
#
assert syntax_tree.errors
def
test_break_inifnite_loop_ZeroOrMore
(
self
):
def
test_break_inifnite_loop_ZeroOrMore
(
self
):
forever
=
ZeroOrMore
(
RegExp
(
''
))
forever
=
ZeroOrMore
(
RegExp
(
''
))
...
@@ -886,6 +886,18 @@ class TestMetaParser:
...
@@ -886,6 +886,18 @@ class TestMetaParser:
rv
=
self
.
mp
.
_return_values
((
Node
(
'tag'
,
'content'
),
EMPTY_NODE
))
rv
=
self
.
mp
.
_return_values
((
Node
(
'tag'
,
'content'
),
EMPTY_NODE
))
assert
rv
[
-
1
].
tag_name
!=
EMPTY_NODE
.
tag_name
,
rv
[
-
1
].
tag_name
assert
rv
[
-
1
].
tag_name
!=
EMPTY_NODE
.
tag_name
,
rv
[
-
1
].
tag_name
def
test_in_context
(
self
):
minilang
=
"""
term = factor { (DIV|MUL) factor}
factor = NUMBER | VARIABLE
MUL = "*" | &factor
DIV = "/"
NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~
VARIABLE = /[A-Za-z]/~
"""
gr
=
grammar_provider
(
minilang
)()
cst
=
gr
(
"2x"
)
assert
bool
(
cst
.
pick
(
'MUL'
))
if
__name__
==
"__main__"
:
if
__name__
==
"__main__"
:
from
DHParser.testing
import
runner
from
DHParser.testing
import
runner
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment