Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
9.2.2023: Due to updates GitLab will be unavailable for some minutes between 9:00 and 11:00.
Open sidebar
badw-it
DHParser
Commits
2e058188
Commit
2e058188
authored
May 15, 2017
by
Eckhart Arnold
Browse files
- can read tests from config file now
parent
a3a5b54a
Changes
11
Hide whitespace changes
Inline
Side-by-side
DHParser/testing.py
View file @
2e058188
...
...
@@ -15,9 +15,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License.
"""
import
configparser
import
copy
import
inspect
import
regex
as
re
try
:
import
regex
as
re
except
ImportError
:
import
re
from
DHParser
import
Node
,
error_messages
from
DHParser.syntaxtree
import
MockParser
...
...
@@ -78,14 +82,38 @@ def mock_syntax_tree(sexpr):
return
Node
(
MockParser
(
name
,
':'
+
class_name
),
result
)
def
test_grammar
(
test_suite
,
parser_factory
,
transformer_factory
):
SUITE_STAGES
=
{
'match'
,
'fail'
,
'ast'
,
'cst'
,
'__ast__'
,
'__cst__'
}
def
suite_from_configfile
(
config_filename
):
"""Reads a grammar test suite from a config file.
"""
cfg
=
configparser
.
ConfigParser
cfg
.
read
(
config_filename
)
suite
=
{}
for
section
in
cfg
.
sections
():
symbol
,
stage
=
section
.
split
(
':'
)
if
stage
not
in
SUITE_STAGES
:
if
symbol
in
SUITE_STAGES
:
symbol
,
stage
=
stage
,
symbol
else
:
raise
ValueError
(
'Stage %s not in: '
%
(
stage
,
str
(
SUITE_STAGES
)))
for
testkey
,
testcode
in
cfg
[
section
].
items
():
suite
.
setdefault
(
symbol
,
{}).
setdefault
(
stage
,
{})[
testkey
]
=
testcode
return
suite
def
unit_grammar
(
test_suite
,
parser_factory
,
transformer_factory
):
"""Unit tests for a grammar-parser and ast transformations.
"""
if
isinstance
(
test_suite
,
str
):
test_suite
=
suite_from_configfile
(
test_suite
)
errata
=
[]
parser
=
parser_factory
()
transform
=
transformer_factory
()
for
parser_name
,
tests
in
test_suite
.
items
():
assert
set
(
tests
.
keys
()).
issubset
(
{
'match'
,
'fail'
,
'ast'
,
'cst'
,
'__ast__'
,
'__cst__'
}
)
assert
set
(
tests
.
keys
()).
issubset
(
SUITE_STAGES
)
for
test_name
,
test_code
in
tests
.
get
(
'match'
,
dict
()).
items
():
cst
=
parser
(
test_code
,
parser_name
)
...
...
examples/LaTeX/OLDSTUFF/LaTeX.ebnf
0 → 100644
View file @
2e058188
# latex Grammar
@ whitespace = /[ \t]*\n?(?!\s*\n)[ \t]*/ # whitespace, including at most one linefeed
@ comment = /%.*(?:\n|$)/
latexdoc = preamble document
preamble = { command }+
genericenv = beginenv sequence §endenv
beginenv = "\begin" §( "{" name "}" )
endenv = "\end" §( "{" ::name "}" )
name = /\w+/~
comand = cmdname [ config ] block
cmdname = /\\\w+/
config = "[" cfgtext §"]"
sequence = { partext | parblock }
parblock = "{" { partext | parblock } §"}"
block = "{" { text | block } §"}"
partext = text | PARSEP
text = cfgtext | brackets
cfgtext = chunk | escaped | WSPC
ESCAPED = /\\[%$&]/
BRACKET = /[\[\]]/ # left or right square bracket: [ ]
TEXTCHUNK = /[^\\%$&\{\}\[\]\s\n]+/ # some piece of text excluding whitespace,
# linefeed and special characters
WSPC = /[ \t]*\n?(?!\s*\n)[ \t]*/ # whitespace, including at most one linefeed
LF = /[ \t]*\n(?!\s*\n)/ # a linefeed, but not an empty line (i.e. par)
PARSEP = /\s*\n\s*\n/ # at least one empty line, i.e.
# [whitespace] linefeed [whitespace] linefeed
examples/MLW/test_MLW_grammar.py
View file @
2e058188
...
...
@@ -66,7 +66,7 @@ MLW_TEST_CASES_LEMMA_POSITION = {
class
TestMLWGrammar
:
def
test_lemma_position
(
self
):
errata
=
testing
.
tes
t_grammar
(
MLW_TEST_CASES_LEMMA_POSITION
,
errata
=
testing
.
uni
t_grammar
(
MLW_TEST_CASES_LEMMA_POSITION
,
get_MLW_grammar
,
get_MLW_transformer
)
assert
not
errata
,
str
(
errata
)
...
...
test/run.py
View file @
2e058188
import
inspect
class
SelfTest
:
def
setup
(
self
):
print
(
"setup"
)
def
teardown
(
self
):
print
(
"teardown"
)
def
test1
(
self
):
print
(
"test1"
)
def
test2
(
self
):
print
(
"test2"
)
def
runner
(
tests
,
namespace
):
""" Runs only some selected tests from a test suite. To run all
tests in a module, call ``runner("", globals())`` from within
that module.
Args:
tests: Either a string or a list of strings that contains the
names of test or test classes. Each test and, in the case
of a test class, all tests within the test class will be
run.
namespace: The namespace for running the test, usually
``globals()`` should be used.
"""
def
instantiate
(
cls_name
):
exec
(
"obj = "
+
cls_name
+
"()"
,
namespace
)
obj
=
namespace
[
"obj"
]
if
"setup"
in
dir
(
obj
):
obj
.
setup
()
return
obj
if
tests
:
if
isinstance
(
tests
,
str
):
tests
=
tests
.
split
(
" "
)
else
:
# collect all test classes, in case no methods or classes have been passed explicitly
tests
=
[]
for
name
in
namespace
.
keys
():
if
name
.
lower
().
startswith
(
'test'
)
and
inspect
.
isclass
(
namespace
[
name
]):
tests
.
append
(
name
)
obj
=
None
for
test
in
tests
:
try
:
if
test
.
find
(
'.'
)
>=
0
:
cls_name
,
method_name
=
test
.
split
(
'.'
)
obj
=
instantiate
(
cls_name
)
print
(
"Running "
+
cls_name
+
"."
+
method_name
)
exec
(
'obj.'
+
method_name
+
'()'
)
else
:
obj
=
instantiate
(
test
)
for
name
in
dir
(
obj
):
if
name
.
lower
().
startswith
(
"test"
):
print
(
"Running "
+
test
+
"."
+
name
)
exec
(
'obj.'
+
name
+
'()'
)
finally
:
if
"teardown"
in
dir
(
obj
):
obj
.
teardown
()
if
__name__
==
"__main__"
:
# runner("", globals())
# runner("TestSelf.test1 TestSelf", globals())
import
os
os
.
chdir
(
'..'
)
...
...
test/test_DHParser.py
View file @
2e058188
...
...
@@ -31,6 +31,6 @@ from DHParser.dsl import *
if
__name__
==
"__main__"
:
from
run
import
runner
from
DHParser.testing
import
runner
runner
(
""
,
globals
())
\ No newline at end of file
test/test_dsl.py
View file @
2e058188
...
...
@@ -123,5 +123,5 @@ class TestCompilerGeneration:
if
__name__
==
"__main__"
:
from
run
import
runner
from
DHParser.testing
import
runner
runner
(
""
,
globals
())
\ No newline at end of file
test/test_ebnf.py
View file @
2e058188
...
...
@@ -347,5 +347,5 @@ class TestBoundaryCases:
if
__name__
==
"__main__"
:
from
run
import
runner
runner
(
"
TestCompilerErrors
"
,
globals
())
from
DHParser.testing
import
runner
runner
(
""
,
globals
())
test/test_parsers.py
View file @
2e058188
...
...
@@ -22,8 +22,6 @@ limitations under the License.
from
functools
import
partial
import
sys
import
DHParser.testing
sys
.
path
.
extend
([
'../'
,
'./'
])
from
DHParser
import
parsers
...
...
@@ -58,65 +56,6 @@ ARITHMETIC_EBNF_transformation_table = {
ARITHMETIC_EBNFTransform
=
partial
(
traverse
,
processing_table
=
ARITHMETIC_EBNF_transformation_table
)
class
TestGrammarTest
:
cases
=
{
"factor"
:
{
"match"
:
{
1
:
"0"
,
2
:
"314"
,
},
"fail"
:
{
3
:
"21F"
,
4
:
"G123"
}
},
"term"
:
{
"match"
:
{
1
:
"4 * 5"
,
2
:
"20 / 4"
,
3
:
"20 / 4 * 3"
},
"ast"
:
{
1
:
"(term (factor 4) (:Token *) (factor 5))"
,
2
:
"(term (factor 20) (:Token /) (factor 4))"
,
3
:
"(term (term (factor 20) (:Token /) (factor 4)) (:Token *) (factor 3))"
},
"fail"
:
{
4
:
"4 + 5"
,
5
:
"20 / 4 - 3"
}
}
}
failure_cases
=
{
"term"
:
{
"match"
:
{
1
:
"4 + 5"
,
# error: this should fail
2
:
"20 / 4"
,
3
:
"20 / 4 * 3"
},
"ast"
:
{
1
:
"(term (factor 4) (:Token *) (factor 5))"
,
2
:
"(term (factor 20) (:Token /) (factor 4))"
,
3
:
"(term (term (factor 19) (:Token /) (factor 4)) (:Token *) (factor 3))"
# error 19 != 20
},
"fail"
:
{
4
:
"4 * 5"
,
# error: this should match
5
:
"20 / 4 - 3"
}
}
}
def
test_testing_grammar
(
self
):
parser_fac
=
parser_factory
(
ARITHMETIC_EBNF
)
trans_fac
=
lambda
:
ARITHMETIC_EBNFTransform
errata
=
DHParser
.
testing
.
test_grammar
(
self
.
cases
,
parser_fac
,
trans_fac
)
assert
not
errata
,
str
(
errata
)
errata
=
DHParser
.
testing
.
test_grammar
(
self
.
failure_cases
,
parser_fac
,
trans_fac
)
# for e in errata:
# print(e)
assert
len
(
errata
)
==
3
class
TestInfiLoopsAndRecursion
:
def
test_direct_left_recursion
(
self
):
minilang
=
ARITHMETIC_EBNF
...
...
@@ -182,5 +121,5 @@ class TestRegex:
if
__name__
==
"__main__"
:
from
run
import
runner
from
DHParser.testing
import
runner
runner
(
""
,
globals
())
test/test_syntaxtree.py
View file @
2e058188
...
...
@@ -23,7 +23,6 @@ import copy
import
sys
sys
.
path
.
extend
([
'../'
,
'./'
])
from
DHParser.toolkit
import
compact_sexpr
,
logging
from
DHParser.syntaxtree
import
traverse
,
reduce_single_child
,
\
replace_by_single_child
,
flatten
,
remove_expendables
,
TOKEN_PTYPE
from
DHParser.testing
import
mock_syntax_tree
...
...
@@ -31,50 +30,6 @@ from DHParser.ebnf import get_ebnf_grammar, get_ebnf_transformer, get_ebnf_compi
from
DHParser.dsl
import
parser_factory
class
MockParser
:
def
__init__
(
self
,
name
=
''
):
self
.
name
=
name
def
__str__
(
self
):
return
self
.
name
or
self
.
__class__
.
__name__
def
__call__
(
self
,
text
):
return
None
,
text
class
TestSExpr
:
"""
Tests for S-expression handling.
"""
def
test_compact_sexpr
(
self
):
assert
compact_sexpr
(
"(a
\n
(b
\n
c
\n
)
\n
)
\n
"
)
==
"(a (b c))"
def
test_mock_syntax_tree
(
self
):
sexpr
=
'(a (b c) (d e) (f (g h)))'
tree
=
mock_syntax_tree
(
sexpr
)
assert
compact_sexpr
(
tree
.
as_sexpr
().
replace
(
'"'
,
''
))
==
sexpr
# test different quotation marks
sexpr
=
'''(a (b """c""" 'k' "l") (d e) (f (g h)))'''
sexpr_stripped
=
'(a (b c k l) (d e) (f (g h)))'
tree
=
mock_syntax_tree
(
sexpr
)
assert
compact_sexpr
(
tree
.
as_sexpr
().
replace
(
'"'
,
''
))
==
sexpr_stripped
sexpr_clean
=
'(a (b "c" "k" "l") (d "e") (f (g "h")))'
tree
=
mock_syntax_tree
(
sexpr_clean
)
assert
compact_sexpr
(
tree
.
as_sexpr
())
==
sexpr_clean
tree
=
mock_syntax_tree
(
sexpr_stripped
)
assert
compact_sexpr
(
tree
.
as_sexpr
())
==
'(a (b "c k l") (d "e") (f (g "h")))'
def
test_mock_syntax_tree_with_classes
(
self
):
sexpr
=
'(a:class1 (b:class2 x) (:class3 y) (c z))'
tree
=
mock_syntax_tree
(
sexpr
)
assert
tree
.
tag_name
==
'a'
assert
tree
.
result
[
0
].
tag_name
==
'b'
assert
tree
.
result
[
1
].
tag_name
==
':class3'
assert
tree
.
result
[
2
].
tag_name
==
'c'
class
TestNode
:
"""
Tests for class Node
...
...
@@ -159,5 +114,5 @@ class TestErrorHandling:
if
__name__
==
"__main__"
:
from
run
import
runner
from
DHParser.testing
import
runner
runner
(
""
,
globals
())
test/test_testing.py
0 → 100644
View file @
2e058188
#!/usr/bin/python3
"""test_parsers.py - tests of the parsers-module of DHParser
Author: Eckhart Arnold <arnold@badw.de>
Copyright 2017 Bavarian Academy of Sciences and Humanities
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from
functools
import
partial
import
sys
sys
.
path
.
extend
([
'../'
,
'./'
])
from
DHParser
import
parsers
from
DHParser.toolkit
import
is_logging
,
compile_python_object
,
compact_sexpr
from
DHParser.syntaxtree
import
no_operation
,
traverse
,
remove_expendables
,
\
replace_by_single_child
,
reduce_single_child
,
flatten
,
TOKEN_PTYPE
from
DHParser.parsers
import
compile_source
from
DHParser.ebnf
import
get_ebnf_grammar
,
get_ebnf_transformer
,
get_ebnf_compiler
from
DHParser.dsl
import
parser_factory
,
DHPARSER_IMPORTS
from
DHParser.testing
import
unit_grammar
,
mock_syntax_tree
ARITHMETIC_EBNF
=
"""
@ whitespace = linefeed
formula = [ //~ ] expr
expr = expr ("+"|"-") term | term
term = term ("*"|"/") factor | factor
factor = /[0-9]+/~
# example: "5 + 3 * 4"
"""
ARITHMETIC_EBNF_transformation_table
=
{
# AST Transformations for the DSL-grammar
"formula"
:
[
remove_expendables
],
"term, expr"
:
[
replace_by_single_child
,
flatten
],
"factor"
:
[
remove_expendables
,
reduce_single_child
],
(
TOKEN_PTYPE
):
[
remove_expendables
,
reduce_single_child
],
"*"
:
[
remove_expendables
,
replace_by_single_child
]
}
ARITHMETIC_EBNFTransform
=
partial
(
traverse
,
processing_table
=
ARITHMETIC_EBNF_transformation_table
)
class
TestGrammarTest
:
cases
=
{
"factor"
:
{
"match"
:
{
1
:
"0"
,
2
:
"314"
,
},
"fail"
:
{
3
:
"21F"
,
4
:
"G123"
}
},
"term"
:
{
"match"
:
{
1
:
"4 * 5"
,
2
:
"20 / 4"
,
3
:
"20 / 4 * 3"
},
"ast"
:
{
1
:
"(term (factor 4) (:Token *) (factor 5))"
,
2
:
"(term (factor 20) (:Token /) (factor 4))"
,
3
:
"(term (term (factor 20) (:Token /) (factor 4)) (:Token *) (factor 3))"
},
"fail"
:
{
4
:
"4 + 5"
,
5
:
"20 / 4 - 3"
}
}
}
failure_cases
=
{
"term"
:
{
"match"
:
{
1
:
"4 + 5"
,
# error: this should fail
2
:
"20 / 4"
,
3
:
"20 / 4 * 3"
},
"ast"
:
{
1
:
"(term (factor 4) (:Token *) (factor 5))"
,
2
:
"(term (factor 20) (:Token /) (factor 4))"
,
3
:
"(term (term (factor 19) (:Token /) (factor 4)) (:Token *) (factor 3))"
# error 19 != 20
},
"fail"
:
{
4
:
"4 * 5"
,
# error: this should match
5
:
"20 / 4 - 3"
}
}
}
def
test_testing_grammar
(
self
):
parser_fac
=
parser_factory
(
ARITHMETIC_EBNF
)
trans_fac
=
lambda
:
ARITHMETIC_EBNFTransform
errata
=
unit_grammar
(
self
.
cases
,
parser_fac
,
trans_fac
)
assert
not
errata
,
str
(
errata
)
errata
=
unit_grammar
(
self
.
failure_cases
,
parser_fac
,
trans_fac
)
# for e in errata:
# print(e)
assert
len
(
errata
)
==
3
class
TestSExpr
:
"""
Tests for S-expression handling.
"""
def
test_compact_sexpr
(
self
):
assert
compact_sexpr
(
"(a
\n
(b
\n
c
\n
)
\n
)
\n
"
)
==
"(a (b c))"
def
test_mock_syntax_tree
(
self
):
sexpr
=
'(a (b c) (d e) (f (g h)))'
tree
=
mock_syntax_tree
(
sexpr
)
assert
compact_sexpr
(
tree
.
as_sexpr
().
replace
(
'"'
,
''
))
==
sexpr
# test different quotation marks
sexpr
=
'''(a (b """c""" 'k' "l") (d e) (f (g h)))'''
sexpr_stripped
=
'(a (b c k l) (d e) (f (g h)))'
tree
=
mock_syntax_tree
(
sexpr
)
assert
compact_sexpr
(
tree
.
as_sexpr
().
replace
(
'"'
,
''
))
==
sexpr_stripped
sexpr_clean
=
'(a (b "c" "k" "l") (d "e") (f (g "h")))'
tree
=
mock_syntax_tree
(
sexpr_clean
)
assert
compact_sexpr
(
tree
.
as_sexpr
())
==
sexpr_clean
tree
=
mock_syntax_tree
(
sexpr_stripped
)
assert
compact_sexpr
(
tree
.
as_sexpr
())
==
'(a (b "c k l") (d "e") (f (g "h")))'
def
test_mock_syntax_tree_with_classes
(
self
):
sexpr
=
'(a:class1 (b:class2 x) (:class3 y) (c z))'
tree
=
mock_syntax_tree
(
sexpr
)
assert
tree
.
tag_name
==
'a'
assert
tree
.
result
[
0
].
tag_name
==
'b'
assert
tree
.
result
[
1
].
tag_name
==
':class3'
assert
tree
.
result
[
2
].
tag_name
==
'c'
if
__name__
==
"__main__"
:
from
DHParser.testing
import
runner
runner
(
""
,
globals
())
test/test_toolkit.py
View file @
2e058188
...
...
@@ -113,5 +113,5 @@ class TestToolkit:
if
__name__
==
"__main__"
:
from
run
import
runner
from
DHParser.testing
import
runner
runner
(
""
,
globals
())
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment