Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
9.2.2023: Due to updates GitLab will be unavailable for some minutes between 9:00 and 11:00.
Open sidebar
badw-it
DHParser
Commits
318987ab
Commit
318987ab
authored
Apr 13, 2017
by
Eckhart Arnold
Browse files
- bug fixes; further unit tests
parent
e3a60af3
Changes
6
Hide whitespace changes
Inline
Side-by-side
DHParser/DSLsupport.py
View file @
318987ab
...
...
@@ -30,7 +30,7 @@ except ImportError:
from
.__init__
import
__version__
from
.EBNFcompiler
import
EBNFGrammar
,
EBNF_ASTPipeline
,
EBNFCompiler
from
.toolkit
import
IS_LOGGING
,
load_if_file
,
is_python_code
,
md5
,
compile_python_object
from
.toolkit
import
load_if_file
,
is_python_code
,
md5
,
compile_python_object
from
.parsercombinators
import
GrammarBase
,
CompilerBase
,
full_compilation
,
nil_scanner
from
.syntaxtree
import
Node
...
...
@@ -189,7 +189,8 @@ def load_compiler_suite(compiler_suite):
def
compileDSL
(
text_or_file
,
dsl_grammar
,
ast_pipeline
,
compiler
,
scanner
=
nil_scanner
):
"""Compiles a text in a domain specific language (DSL) with an
EBNF-specified grammar. Returns the compiled text.
EBNF-specified grammar. Returns the compiled text or raises a
compilation error.
"""
assert
isinstance
(
text_or_file
,
str
)
assert
isinstance
(
compiler
,
CompilerBase
)
...
...
@@ -201,7 +202,7 @@ def compileDSL(text_or_file, dsl_grammar, ast_pipeline, compiler,
return
result
def
compileEBNF
(
ebnf_src
,
ebnf_grammar_obj
=
None
):
def
compileEBNF
(
ebnf_src
,
ebnf_grammar_obj
=
None
,
source_only
=
False
):
"""Compiles an EBNF source file into a Grammar class
Args:
...
...
@@ -211,13 +212,16 @@ def compileEBNF(ebnf_src, ebnf_grammar_obj=None):
DHParser.EBNFcompiler.EBNFGrammar object. This can speed
up compilation, because no new EBNFGrammar object needs to
be instantiated.
source_only (bool): If True, the source code of the Grammar
class is returned instead of the class itself.
Returns:
A Grammar class that can be instantiated for parsing a text
which conforms to the language defined by ``ebnf_src``
"""
grammar
=
ebnf_grammar_obj
or
EBNFGrammar
()
grammar_src
=
compileDSL
(
ebnf_src
,
grammar
,
EBNF_ASTPipeline
,
EBNFCompiler
())
return
compile_python_object
(
DHPARSER_IMPORTS
+
grammar_src
,
'\w*Grammar$'
)
return
grammar_src
if
source_only
else
\
compile_python_object
(
DHPARSER_IMPORTS
+
grammar_src
,
'\w*Grammar$'
)
def
run_compiler
(
source_file
,
compiler_suite
=
""
,
extension
=
".xml"
):
...
...
DHParser/parsercombinators.py
View file @
318987ab
...
...
@@ -61,8 +61,8 @@ except ImportError:
from
.toolkit
import
IS_LOGGING
,
LOGS_DIR
,
escape_re
,
sane_parser_name
,
sequence
from
.syntaxtree
import
WHITESPACE_KEYWORD
,
TOKEN_KEYWORD
,
ZOMBIE_PARSER
,
Node
,
\
error_messages
,
traverse
traverse
from
DHParser.toolkit
import
error_messages
__all__
=
[
'HistoryRecord'
,
'Parser'
,
...
...
@@ -286,13 +286,13 @@ class GrammarBase:
if
self
.
wspL__
:
self
.
wsp_left_parser__
=
RegExp
(
self
.
wspL__
,
WHITESPACE_KEYWORD
)
self
.
wsp_left_parser__
.
grammar
=
self
self
.
all_parsers
.
add
(
self
.
wsp_left_parser__
)
self
.
all_parsers
.
add
(
self
.
wsp_left_parser__
)
# don't you forget about me...
else
:
self
.
wsp_left_parser__
=
ZOMBIE_PARSER
if
self
.
wspR__
:
self
.
wsp_right_parser__
=
RegExp
(
self
.
wspR__
,
WHITESPACE_KEYWORD
)
self
.
wsp_right_parser__
.
grammar
=
self
self
.
all_parsers
.
add
(
self
.
wsp_right_parser__
)
self
.
all_parsers
.
add
(
self
.
wsp_right_parser__
)
# don't you forget about me...
else
:
self
.
wsp_right_parser__
=
ZOMBIE_PARSER
self
.
root__
.
apply
(
self
.
_add_parser
)
...
...
@@ -309,7 +309,8 @@ class GrammarBase:
"""Adds the copy of the classes parser object to this
particular instance of GrammarBase.
"""
setattr
(
self
,
parser
.
name
,
parser
)
if
parser
.
name
:
setattr
(
self
,
parser
.
name
,
parser
)
self
.
all_parsers
.
add
(
parser
)
parser
.
grammar
=
self
...
...
@@ -469,7 +470,7 @@ class RegExp(Parser):
regexp
=
self
.
regexp
.
pattern
duplicate
=
RegExp
(
regexp
,
self
.
name
)
duplicate
.
name
=
self
.
name
# this ist needed!!!!
duplicate
.
regexp
=
self
.
regexp
#
duplicate.regexp = self.regexp
duplicate
.
grammar
=
self
.
grammar
duplicate
.
visited
=
copy
.
deepcopy
(
self
.
visited
,
memo
)
duplicate
.
recursion_counter
=
copy
.
deepcopy
(
self
.
recursion_counter
,
memo
)
...
...
@@ -818,7 +819,7 @@ class Forward(Parser):
def
set
(
self
,
parser
):
assert
isinstance
(
parser
,
Parser
)
self
.
name
=
parser
.
name
# redundant,
because of constructor of GrammarBase
self
.
name
=
parser
.
name
# redundant,
see GrammarBase-constructor
self
.
parser
=
parser
def
apply
(
self
,
func
):
...
...
DHParser/syntaxtree.py
View file @
318987ab
...
...
@@ -19,7 +19,6 @@ implied. See the License for the specific language governing
permissions and limitations under the License.
"""
import
collections
import
itertools
import
os
from
functools
import
partial
...
...
@@ -37,7 +36,6 @@ __all__ = ['WHITESPACE_KEYWORD',
'ZOMBIE_PARSER'
,
'Error'
,
'Node'
,
'error_messages'
,
'compact_sexpr'
,
'traverse'
,
'no_operation'
,
...
...
@@ -356,18 +354,6 @@ class Node:
return
nav
(
path
.
split
(
'/'
))
def
error_messages
(
text
,
errors
):
"""
Converts the list of ``errors`` collected from the root node of the
parse tree of `text` into a human readable (and IDE or editor
parsable text) with line an column numbers. Error messages are
separated by an empty line.
"""
return
"
\n\n
"
.
join
(
"line: %i, column: %i, error: %s"
%
(
*
line_col
(
text
,
err
.
pos
),
err
.
msg
)
for
err
in
sorted
(
list
(
errors
)))
########################################################################
#
# syntax tree transformation functions
...
...
DHParser/toolkit.py
View file @
318987ab
...
...
@@ -46,6 +46,7 @@ __all__ = ['logging_on',
'IS_LOGGING'
,
'LOGS_DIR'
,
'line_col'
,
'error_messages'
,
'escape_re'
,
'load_if_file'
,
'is_python_code'
,
...
...
@@ -118,6 +119,18 @@ def line_col(text, pos):
return
line
,
column
def
error_messages
(
text
,
errors
):
"""
Converts the list of ``errors`` collected from the root node of the
parse tree of `text` into a human readable (and IDE or editor
parsable text) with line an column numbers. Error messages are
separated by an empty line.
"""
return
"
\n\n
"
.
join
(
"line: %i, column: %i, error: %s"
%
(
*
line_col
(
text
,
err
.
pos
),
err
.
msg
)
for
err
in
sorted
(
list
(
errors
)))
def
compact_sexpr
(
s
):
"""Returns S-expression ``s`` as a one liner without unnecessary
whitespace.
...
...
@@ -225,3 +238,4 @@ def compile_python_object(python_src, catch_obj_regex):
raise
AssertionError
(
"Ambigous matches for %s : %s"
%
(
str
(
catch_obj_regex
),
str
(
matches
)))
return
namespace
[
matches
[
0
]]
if
matches
else
None
tests/test_EBNFcompiler.py
View file @
318987ab
...
...
@@ -44,10 +44,8 @@ class TestDirectives:
parser
=
MinilangParser
()
assert
parser
syntax_tree
=
parser
.
parse
(
"3 + 4 * 12"
)
parser
.
log_parsing_history
(
'WSP1'
)
assert
not
syntax_tree
.
collect_errors
()
syntax_tree
=
parser
.
parse
(
"3 + 4
\n
* 12"
)
parser
.
log_parsing_history
(
'WSP2'
)
assert
not
syntax_tree
.
collect_errors
()
def
test_whitespace_standard
(
self
):
...
...
tests/test_parsercombinators.py
View file @
318987ab
...
...
@@ -25,7 +25,10 @@ import os
import
sys
sys
.
path
.
append
(
os
.
path
.
abspath
(
'../../'
))
from
DHParser.DSLsupport
import
compileEBNF
from
DHParser.toolkit
import
compile_python_object
from
DHParser.parsercombinators
import
full_compilation
from
DHParser.EBNFcompiler
import
EBNFGrammar
,
EBNF_ASTPipeline
,
EBNFCompiler
from
DHParser.DSLsupport
import
compileEBNF
,
DHPARSER_IMPORTS
WRITE_LOGS
=
True
...
...
@@ -33,10 +36,10 @@ WRITE_LOGS = True
class
TestLeftRecursion
:
mini_language1
=
"""
@ whitespace = linefeed
formula =
expr "."
formula =
[ //~ ] expr
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/
factor = /[0-9]+/
~
# example: "5 + 3 * 4"
"""
...
...
@@ -47,17 +50,37 @@ class TestLeftRecursion:
assert
self
.
minilang_parser1
def
test_direct_left_recursion
(
self
):
syntax_tree
=
self
.
minilang_parser1
.
parse
(
"5 + 3 * 4"
)
snippet
=
"5 + 3 * 4"
syntax_tree
=
self
.
minilang_parser1
.
parse
(
snippet
)
assert
not
syntax_tree
.
collect_errors
()
assert
snippet
==
str
(
syntax_tree
)
if
WRITE_LOGS
:
syntax_tree
.
log
(
"test_LeftRecursion_direct"
,
'.cst'
)
self
.
minilang_parser1
.
log_parsing_history
(
"test_LeftRecursion_direct"
)
def
test_indirect_left_recursion
(
self
):
pass
class
TestRegex
:
def
test_multilineRegex
(
self
):
mlregex
=
r
"""
regex = /\w+ # one or more alphabetical characters including the underscore
[+] # followed by a plus sign
\w* # possibly followed by more alpha chracters/
"""
result
,
messages
,
syntax_tree
=
full_compilation
(
mlregex
,
EBNFGrammar
(),
EBNF_ASTPipeline
,
EBNFCompiler
(
'MultilineRegexTest'
))
assert
result
is
not
None
,
messages
assert
not
messages
parser
=
compile_python_object
(
DHPARSER_IMPORTS
+
result
,
'\w+Grammar$'
)()
node
,
rest
=
parser
.
regex
(
'abc+def'
)
assert
rest
==
''
assert
node
.
parser
.
name
==
"regex"
assert
str
(
node
)
==
'abc+def'
if
__name__
==
"__main__"
:
from
run
import
run_tests
run_tests
(
"TestLeftRecursion"
,
globals
())
\ No newline at end of file
run_tests
(
"TestRegex"
,
globals
())
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment