Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
badw-it
DHParser
Commits
def999dd
Commit
def999dd
authored
Nov 15, 2017
by
eckhart
Browse files
cleanups finished
parent
83a0b5dd
Changes
3
Hide whitespace changes
Inline
Side-by-side
DHParser/dsl.py
View file @
def999dd
...
...
@@ -169,14 +169,15 @@ def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
# read grammar
grammar_src
=
load_if_file
(
grammar_representation
)
if
is_python_code
(
grammar_src
):
parser_py
,
messages
,
AST
=
grammar_src
,
[]
,
None
# type: str, List[Error]
, Node
parser_py
,
messages
=
grammar_src
,
[]
# type: str, List[Error]
else
:
with
logging
(
False
):
parser_py
,
messages
,
AST
=
compile_source
(
grammar_src
,
None
,
parser_py
,
messages
,
_
=
compile_source
(
grammar_src
,
None
,
get_ebnf_grammar
(),
get_ebnf_transformer
(),
get_ebnf_compiler
())
if
has_errors
(
messages
):
raise
GrammarError
(
only_errors
(
messages
),
grammar_src
)
parser_root
=
compile_python_object
(
DHPARSER_IMPORTS
+
parser_py
,
'\w+Grammar$'
)()
parser_root
=
compile_python_object
(
DHPARSER_IMPORTS
+
parser_py
,
r
'\w+Grammar$'
)()
else
:
# assume that dsl_grammar is a ParserHQ-object or Grammar class
grammar_src
=
''
...
...
@@ -197,7 +198,7 @@ def compileDSL(text_or_file: str,
Compiles a text in a domain specific language (DSL) with an
EBNF-specified grammar. Returns the compiled text or raises a
compilation error.
Raises:
CompilationError if any errors occurred during compilation
"""
...
...
@@ -218,16 +219,16 @@ def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
Compiles an EBNF grammar file and returns the compiler object
that was used and which can now be queried for the result as well
as skeleton code for preprocessor, transformer and compiler objects.
Args:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str): Branding name for the compiler suite source
code.
code.
Returns:
An instance of class ``ebnf.EBNFCompiler``
Raises:
CompilationError if any errors occurred during compilation
CompilationError if any errors occurred during compilation
"""
grammar
=
get_ebnf_grammar
()
compiler
=
get_ebnf_compiler
(
branding
,
ebnf_src
)
...
...
@@ -246,11 +247,11 @@ def compileEBNF(ebnf_src: str, branding="DSL") -> str:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str): Branding name for the compiler suite source
code.
code.
Returns:
The complete compiler suite skeleton as Python source code.
Raises:
CompilationError if any errors occurred during compilation
CompilationError if any errors occurred during compilation
"""
compiler
=
raw_compileEBNF
(
ebnf_src
,
branding
)
src
=
[
"#/usr/bin/python
\n
"
,
...
...
@@ -272,23 +273,23 @@ def grammar_provider(ebnf_src: str, branding="DSL") -> Grammar:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str or bool): Branding name for the compiler
suite source code.
suite source code.
Returns:
A provider function for a grammar object for texts in the
language defined by ``ebnf_src``.
"""
grammar_src
=
compileDSL
(
ebnf_src
,
nil_preprocessor
,
get_ebnf_grammar
(),
get_ebnf_transformer
(),
get_ebnf_compiler
(
branding
))
return
compile_python_object
(
DHPARSER_IMPORTS
+
grammar_src
,
'get_(?:\w+_)?grammar$'
)
return
compile_python_object
(
DHPARSER_IMPORTS
+
grammar_src
,
r
'get_(?:\w+_)?grammar$'
)
def
load_compiler_suite
(
compiler_suite
:
str
)
->
\
Tuple
[
PreprocessorFactoryFunc
,
ParserFactoryFunc
,
TransformerFactoryFunc
,
CompilerFactoryFunc
]:
Tuple
[
PreprocessorFactoryFunc
,
ParserFactoryFunc
,
TransformerFactoryFunc
,
CompilerFactoryFunc
]:
"""
Extracts a compiler suite from file or string ``compiler suite``
and returns it as a tuple (preprocessor, parser, ast, compiler).
Returns:
4-tuple (preprocessor function, parser class, ast transformer function, compiler class)
"""
...
...
@@ -298,26 +299,27 @@ def load_compiler_suite(compiler_suite: str) -> \
imports
=
DHPARSER_IMPORTS
if
is_python_code
(
compiler_suite
):
try
:
intro
,
imports
,
preprocessor_py
,
parser_py
,
ast_py
,
compiler_py
,
outro
=
\
_
,
imports
,
preprocessor_py
,
parser_py
,
ast_py
,
compiler_py
,
_
=
\
RX_SECTION_MARKER
.
split
(
source
)
except
ValueError
as
error
:
except
ValueError
:
raise
AssertionError
(
'File "'
+
compiler_suite
+
'" seems to be corrupted. '
'Please delete or repair file manually.'
)
# TODO: Compile in one step and pick parts from namespace later ?
preprocessor
=
compile_python_object
(
imports
+
preprocessor_py
,
'get_(?:\w+_)?preprocessor$'
)
parser
=
compile_python_object
(
imports
+
parser_py
,
'get_(?:\w+_)?grammar$'
)
ast
=
compile_python_object
(
imports
+
ast_py
,
'get_(?:\w+_)?transformer$'
)
preprocessor
=
compile_python_object
(
imports
+
preprocessor_py
,
r
'get_(?:\w+_)?preprocessor$'
)
parser
=
compile_python_object
(
imports
+
parser_py
,
r
'get_(?:\w+_)?grammar$'
)
ast
=
compile_python_object
(
imports
+
ast_py
,
r
'get_(?:\w+_)?transformer$'
)
else
:
# assume source is an ebnf grammar. Is there really any reasonable application case for this?
with
logging
(
False
):
compile_py
,
messages
,
AST
=
compile_source
(
source
,
None
,
get_ebnf_grammar
(),
get_ebnf_transformer
(),
get_ebnf_compiler
())
compile
r
_py
,
messages
,
_
=
compile_source
(
source
,
None
,
get_ebnf_grammar
(),
get_ebnf_transformer
(),
get_ebnf_compiler
())
if
has_errors
(
messages
):
raise
GrammarError
(
only_errors
(
messages
),
source
)
preprocessor
=
get_ebnf_preprocessor
parser
=
get_ebnf_grammar
ast
=
get_ebnf_transformer
compiler
=
compile_python_object
(
imports
+
compiler_py
,
'get_(?:\w+_)?compiler$'
)
compiler
=
compile_python_object
(
imports
+
compiler_py
,
r
'get_(?:\w+_)?compiler$'
)
return
preprocessor
,
parser
,
ast
,
compiler
...
...
@@ -325,12 +327,12 @@ def load_compiler_suite(compiler_suite: str) -> \
def
is_outdated
(
compiler_suite
:
str
,
grammar_source
:
str
)
->
bool
:
"""
Returns ``True`` if the ``compile_suite`` needs to be updated.
An update is needed, if either the grammar in the compieler suite
does not reflect the latest changes of ``grammar_source`` or if
An update is needed, if either the grammar in the compieler suite
does not reflect the latest changes of ``grammar_source`` or if
sections from the compiler suite have diligently been overwritten
with whitespace order to trigger their recreation. Note: Do not
delete or overwrite the section marker itself.
delete or overwrite the section marker itself.
Args:
compiler_suite: the parser class representing the grammar
...
...
@@ -342,7 +344,7 @@ def is_outdated(compiler_suite: str, grammar_source: str) -> bool:
True, if ``compiler_suite`` seems to be out of date.
"""
try
:
preprocessor
,
grammar
,
ast
,
compiler
=
load_compiler_suite
(
compiler_suite
)
_
,
grammar
,
_
,
_
=
load_compiler_suite
(
compiler_suite
)
return
grammar_changed
(
grammar
(),
grammar_source
)
except
ValueError
:
return
True
...
...
@@ -353,17 +355,17 @@ def run_compiler(text_or_file: str, compiler_suite: str) -> Any:
Args:
text_or_file (str): Either the file name of the source code or
the source code directly. (Which is determined by
the source code directly. (Which is determined by
heuristics. If ``text_or_file`` contains at least on
linefeed then it is always assumed to be a source text and
not a file name.)
compiler_suite(str): File name of the compiler suite to be
used.
Returns:
The result of the compilation, the form and type of which
The result of the compilation, the form and type of which
depends entirely on the compiler.
Raises:
CompilerError
"""
...
...
@@ -398,7 +400,7 @@ def compile_on_disk(source_file: str, compiler_suite="", extension=".xml") -> It
is written to a file with the same name but a different
extension than the source file. This parameter sets the
extension.
Returns:
A (potentially empty) list of error or warning messages.
"""
...
...
DHParser/ebnf.py
View file @
def999dd
...
...
@@ -430,13 +430,13 @@ class EBNFCompiler(Compiler):
' # AST Transformations for the '
+
self
.
grammar_name
+
'-grammar'
]
transtable
.
append
(
' "+": remove_empty,'
)
for
name
in
self
.
rules
:
t
f
=
'[]'
t
ransformations
=
'[]'
rule
=
self
.
definitions
[
name
]
if
rule
.
startswith
(
'Alternative'
):
t
f
=
'[replace_or_reduce]'
t
ransformations
=
'[replace_or_reduce]'
elif
rule
.
startswith
(
'Synonym'
):
t
f
=
'[replace_by_single_child]'
transtable
.
append
(
' "'
+
name
+
'": %s,'
%
t
f
)
t
ransformations
=
'[replace_by_single_child]'
transtable
.
append
(
' "'
+
name
+
'": %s,'
%
t
ransformations
)
transtable
.
append
(
' ":Token, :RE": reduce_single_child,'
)
transtable
+=
[
' "*": replace_by_single_child'
,
'}'
,
''
]
transtable
+=
[
TRANSFORMER_FACTORY
.
format
(
NAME
=
self
.
grammar_name
)]
...
...
@@ -459,7 +459,7 @@ class EBNFCompiler(Compiler):
self
.
grammar_name
+
'", grammar_source=""):'
,
' super('
+
self
.
grammar_name
+
'Compiler, self).__init__(grammar_name, grammar_source)'
,
" assert re.match('\w+\Z', grammar_name)"
,
''
]
r
" assert re.match('\w+\Z', grammar_name)"
,
''
]
for
name
in
self
.
rules
:
method_name
=
Compiler
.
method_name
(
name
)
if
name
==
self
.
root_symbol
:
...
...
@@ -548,6 +548,8 @@ class EBNFCompiler(Compiler):
defined_symbols
.
difference_update
(
self
.
RESERVED_SYMBOLS
)
def
remove_connections
(
symbol
):
"""Recursiviely removes all symbols which appear in the
definiens of a particular symbol."""
if
symbol
in
defined_symbols
:
defined_symbols
.
remove
(
symbol
)
for
related
in
self
.
rules
[
symbol
][
1
:]:
...
...
@@ -555,8 +557,9 @@ class EBNFCompiler(Compiler):
remove_connections
(
self
.
root_symbol
)
for
leftover
in
defined_symbols
:
self
.
rules
[
leftover
][
0
].
add_error
((
'Rule "%s" is not connected to '
'parser root "%s" !'
)
%
(
leftover
,
self
.
root_symbol
),
Error
.
WARNING
)
self
.
rules
[
leftover
][
0
].
add_error
(
(
'Rule "%s" is not connected to parser root "%s" !'
)
%
(
leftover
,
self
.
root_symbol
),
Error
.
WARNING
)
# set root_symbol parser and assemble python grammar definition
...
...
@@ -573,6 +576,12 @@ class EBNFCompiler(Compiler):
def
on_syntax
(
self
,
node
:
Node
)
->
str
:
definitions
=
[]
# type: List[Tuple[str, str]]
# transfer directives to re_flags where needed
if
self
.
directives
[
'ignorecase'
]:
self
.
re_flags
.
add
(
'i'
)
elif
'i'
in
self
.
re_flags
:
self
.
re_flags
.
remove
(
'i'
)
# drop the wrapping sequence node
if
len
(
node
.
children
)
==
1
and
not
node
.
children
[
0
].
parser
.
name
:
node
=
node
.
children
[
0
]
...
...
@@ -678,6 +687,8 @@ class EBNFCompiler(Compiler):
self
.
directives
[
'ignorecase'
]
==
value
if
value
:
self
.
re_flags
.
add
(
'i'
)
elif
'i'
in
self
.
re_flags
:
self
.
re_flags
.
remove
(
'i'
)
# elif key == 'testing':
# value = str(node.children[1])
...
...
@@ -685,14 +696,14 @@ class EBNFCompiler(Compiler):
elif
key
==
'literalws'
:
value
=
{
item
.
lower
()
for
item
in
self
.
compile
(
node
.
children
[
1
])}
if
(
len
(
value
-
{
'left'
,
'right'
,
'both'
,
'none'
})
>
0
if
((
value
-
{
'left'
,
'right'
,
'both'
,
'none'
})
or
(
'none'
in
value
and
len
(
value
)
>
1
)):
node
.
add_error
(
'Directive "literalws" allows the values '
'`left`, `right`, `both` or `none`, '
'but not `%s`'
%
", "
.
join
(
value
))
ws
=
{
'left'
,
'right'
}
if
'both'
in
value
\
ws
p
=
{
'left'
,
'right'
}
if
'both'
in
value
\
else
{}
if
'none'
in
value
else
value
self
.
directives
[
key
]
=
list
(
ws
)
self
.
directives
[
key
]
=
list
(
ws
p
)
elif
key
in
{
'tokens'
,
'preprocessor_tokens'
}:
self
.
directives
[
'tokens'
]
|=
self
.
compile
(
node
.
children
[
1
])
...
...
@@ -747,7 +758,7 @@ class EBNFCompiler(Compiler):
i
+=
1
saved_result
=
node
.
result
node
.
result
=
tuple
(
filtered_children
)
custom_args
=
[
'mandatory=%i'
%
mandatory_marker
[
0
]]
if
mandatory_marker
else
[]
custom_args
=
[
'mandatory=%i'
%
mandatory_marker
[
0
]]
if
mandatory_marker
else
[]
compiled
=
self
.
non_terminal
(
node
,
'Series'
,
custom_args
)
node
.
result
=
saved_result
return
compiled
...
...
@@ -832,10 +843,10 @@ class EBNFCompiler(Compiler):
# return self.non_terminal(node, 'Unordered')
assert
len
(
node
.
children
)
==
1
nd
=
node
.
children
[
0
]
for
r
in
nd
.
children
:
if
r
.
parser
.
ptype
==
TOKEN_PTYPE
and
str
(
nd
)
==
"§"
:
for
child
in
nd
.
children
:
if
child
.
parser
.
ptype
==
TOKEN_PTYPE
and
str
(
nd
)
==
"§"
:
node
.
add_error
(
"Unordered parser lists cannot contain mandatory (§) items."
)
args
=
', '
.
join
(
self
.
compile
(
r
)
for
r
in
nd
.
children
)
args
=
', '
.
join
(
self
.
compile
(
child
)
for
child
in
nd
.
children
)
if
nd
.
parser
.
name
==
"term"
:
return
"AllOf("
+
args
+
")"
elif
nd
.
parser
.
name
==
"expression"
:
...
...
DHParser/testing.py
View file @
def999dd
...
...
@@ -189,8 +189,8 @@ def grammar_unit(test_unit, parser_factory, transformer_factory, report=True, ve
# write parsing-history log only in case of failure!
parser
.
log_parsing_history__
(
"match_%s_%s.log"
%
(
parser_name
,
test_name
))
elif
"cst"
in
tests
and
mock_syntax_tree
(
tests
[
"cst"
][
test_name
])
!=
cst
:
errata
.
append
(
'Concrete syntax tree test "%s" for parser "%s" failed:
\n
%s'
%
(
test_name
,
parser_name
,
cst
.
as_sxpr
()))
errata
.
append
(
'Concrete syntax tree test "%s" for parser "%s" failed:
\n
%s'
%
(
test_name
,
parser_name
,
cst
.
as_sxpr
()))
elif
"ast"
in
tests
:
compare
=
mock_syntax_tree
(
tests
[
"ast"
][
test_name
])
if
compare
!=
ast
:
...
...
@@ -245,7 +245,8 @@ def grammar_suite(directory, parser_factory, transformer_factory,
print
(
"
\n
Scanning test-directory: "
+
directory
)
save_cwd
=
os
.
getcwd
()
os
.
chdir
(
directory
)
if
is_logging
():
clear_logs
()
if
is_logging
():
clear_logs
()
for
filename
in
sorted
(
os
.
listdir
()):
if
any
(
fnmatch
.
fnmatch
(
filename
,
pattern
)
for
pattern
in
fn_patterns
):
try
:
...
...
@@ -259,7 +260,8 @@ def grammar_suite(directory, parser_factory, transformer_factory,
if
not
ignore_unknown_filetypes
or
str
(
e
).
find
(
"Unknown"
)
<
0
:
raise
e
os
.
chdir
(
save_cwd
)
error_report
=
[];
err_N
=
0
error_report
=
[]
err_N
=
0
if
all_errors
:
for
filename
in
all_errors
:
error_report
.
append
(
'Errors found by unit test "%s":'
%
filename
)
...
...
@@ -267,16 +269,18 @@ def grammar_suite(directory, parser_factory, transformer_factory,
for
error
in
all_errors
[
filename
]:
error_report
.
append
(
'
\t
'
+
'
\n\t
'
.
join
(
error
.
split
(
'
\n
'
)))
if
error_report
:
if
verbose
:
print
(
"
\n
FAILURE! %i error%s found!
\n
"
%
(
err_N
,
's'
if
err_N
>
1
else
''
))
if
verbose
:
print
(
"
\n
FAILURE! %i error%s found!
\n
"
%
(
err_N
,
's'
if
err_N
>
1
else
''
))
return
(
'Test suite "%s" revealed some errors:
\n\n
'
%
directory
)
+
'
\n
'
.
join
(
error_report
)
if
verbose
:
print
(
"
\n
SUCCESS! All tests passed :-)
\n
"
)
if
verbose
:
print
(
"
\n
SUCCESS! All tests passed :-)
\n
"
)
return
''
def
runner
(
tests
,
namespace
):
"""
Runs all or some selected Python unit tests found in the
namespace. To run all tests in a module, call
namespace. To run all tests in a module, call
``runner("", globals())`` from within that module.
Args:
...
...
@@ -284,9 +288,9 @@ def runner(tests, namespace):
names of test or test classes. Each test and, in the case
of a test class, all tests within the test class will be
run.
namespace: The namespace for running the test, usually
namespace: The namespace for running the test, usually
``globals()`` should be used.
Example:
class TestSomething()
def setup(self):
...
...
@@ -295,10 +299,10 @@ def runner(tests, namespace):
pass
def test_something(self):
pass
if __name__ == "__main__":
from DHParser.testing import runner
runner("", globals())
runner("", globals())
"""
def
instantiate
(
cls_name
):
exec
(
"obj = "
+
cls_name
+
"()"
,
namespace
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment