Skip to content

GitLab

Commit e71359de authored by Hossain Mahmud's avatar Hossain Mahmud
Browse files

removed run#, copy artifact to failed/, show number of pixel diff

parent ed2d2410
Hide whitespace changes
Inline Side-by-side
test/main.cpp
View file @ e71359de
......@@ -144,7 +144,7 @@ GTEST_API_ int main(int argc, char **argv) {
app = new QApplication(argc, argv);
//testing::InitGoogleTest(&argc, argv);
int _argc = 2;
char *options[] = {static_cast<char *>("THIS DOESN'T HAVE ANY EFFECT"), static_cast<char *>("--gtest_output=xml:visregtests/result.xml")};
char *options[] = {"THIS DOESN'T HAVE ANY EFFECT", "--gtest_output=xml:visregtests/result.xml"};
testing::InitGoogleTest(&_argc, options);
int ret;
......
test/modules/pipelineresultimagetest.cpp
View file @ e71359de
......@@ -60,16 +60,18 @@ protected:
tgt::FileSystem::createDirectory(_basePath);
}
std::vector<std::string> filelist = tgt::FileSystem::listSubDirectories(_basePath, true);
std::string testRunNo = "";
std::string caseNo = "1/";
if (_prevNoCases++ == 0) {
testRunNo = filelist.size() > 0 ? StringUtils::toString(StringUtils::fromString<int>(filelist[filelist.size()-1])+1) : "1";
tgt::FileSystem::createDirectoryRecursive(_basePath+testRunNo+"/"+ caseNo);
}
else
testRunNo = filelist[filelist.size()-1];
_basePath += testRunNo+"/"+caseNo;
//std::string testRunNo = "";
//if (_prevNoCases++ == 0) {
// testRunNo = filelist.size() > 0 ? StringUtils::toString(StringUtils::fromString<int>(filelist[filelist.size()-1])+1) : "1";
// tgt::FileSystem::createDirectoryRecursive(_basePath+testRunNo+"/"+ caseNo);
//}
//else
// testRunNo = filelist[filelist.size()-1];
//
//_basePath += testRunNo+"/"+caseNo;
tgt::FileSystem::createDirectoryRecursive(_basePath+ caseNo);
_basePath += caseNo;
}
~PipelineWriteResultImageTest() {
......
test/visregtests/result.xml deleted 100644 → 0
View file @ ed2d2410
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="46" failures="0" disabled="0" errors="0" timestamp="2014-08-24T09:57:56" time="2.545" name="AllTests">
<testsuite name="DataContainerTest" tests="3" failures="0" disabled="0" errors="0" time="0.002">
<testcase name="miscellaneousTest" status="run" time="0.001" classname="DataContainerTest" />
<testcase name="getDataHandlesCopyTest" status="run" time="0" classname="DataContainerTest" />
<testcase name="concurrentAccessTest" status="run" time="0" classname="DataContainerTest" />
</testsuite>
<testsuite name="DataHandleTest" tests="2" failures="0" disabled="0" errors="0" time="0">
<testcase name="getDataTest" status="run" time="0" classname="DataHandleTest" />
<testcase name="getTimestampTest" status="run" time="0" classname="DataHandleTest" />
</testsuite>
<testsuite name="ImageDataTest" tests="2" failures="0" disabled="0" errors="0" time="0.002">
<testcase name="miscellaneousTest" status="run" time="0" classname="ImageDataTest" />
<testcase name="cloneTest" status="run" time="0" classname="ImageDataTest" />
</testsuite>
<testsuite name="ImageRepresentationTest" tests="3" failures="0" disabled="0" errors="0" time="0.029">
<testcase name="conversion_disk_local_gl_test" status="run" time="0.003" classname="ImageRepresentationTest" />
<testcase name="conversion_disk_gl_local_test" status="run" time="0.001" classname="ImageRepresentationTest" />
<testcase name="basetype_conversion_test" status="run" time="0.022" classname="ImageRepresentationTest" />
</testsuite>
<testsuite name="ImageSeriesTest" tests="2" failures="0" disabled="0" errors="0" time="0.002">
<testcase name="miscellaneousTest" status="run" time="0" classname="ImageSeriesTest" />
<testcase name="cloneTest" status="run" time="0" classname="ImageSeriesTest" />
</testsuite>
<testsuite name="NumericPropertyTest" tests="4" failures="0" disabled="0" errors="0" time="0.003">
<testcase name="maxTest" status="run" time="0" classname="NumericPropertyTest" />
<testcase name="minTest" status="run" time="0" classname="NumericPropertyTest" />
<testcase name="increaseTest" status="run" time="0" classname="NumericPropertyTest" />
<testcase name="decreaseTest" status="run" time="0" classname="NumericPropertyTest" />
</testsuite>
<testsuite name="StringPropertyTest" tests="3" failures="0" disabled="0" errors="0" time="0.003">
<testcase name="getSetValueTest" status="run" time="0" classname="StringPropertyTest" />
<testcase name="sharedPropertyTest" status="run" time="0" classname="StringPropertyTest" />
<testcase name="unlockTest" status="run" time="0" classname="StringPropertyTest" />
</testsuite>
<testsuite name="ConcurrentHistogram1DTest" tests="1" failures="0" disabled="0" errors="0" time="0.002">
<testcase name="concurrentAddSampleTest" status="run" time="0.001" classname="ConcurrentHistogram1DTest" />
</testsuite>
<testsuite name="ConcurrentHistogram2DTest" tests="1" failures="0" disabled="0" errors="0" time="0.085">
<testcase name="concurrentAddSampleTest" status="run" time="0.085" classname="ConcurrentHistogram2DTest" />
</testsuite>
<testsuite name="ConcurrentHistogram1DTestSpecific" tests="1" failures="0" disabled="0" errors="0" time="0.109">
<testcase name="concurrentAddSampleTest" status="run" time="0.109" classname="ConcurrentHistogram1DTestSpecific" />
</testsuite>
<testsuite name="ConcurrentHistogram2DTestSpecific" tests="1" failures="0" disabled="0" errors="0" time="0.129">
<testcase name="concurrentAddSampleTest" status="run" time="0.129" classname="ConcurrentHistogram2DTestSpecific" />
</testsuite>
<testsuite name="StringUtilTest" tests="6" failures="0" disabled="0" errors="0" time="0.006">
<testcase name="caseOperationsTest" status="run" time="0" classname="StringUtilTest" />
<testcase name="parseFloatsTest" status="run" time="0" classname="StringUtilTest" />
<testcase name="splitTest" status="run" time="0.001" classname="StringUtilTest" />
<testcase name="replaceTest" status="run" time="0" classname="StringUtilTest" />
<testcase name="trimTest" status="run" time="0" classname="StringUtilTest" />
<testcase name="toStringTest" status="run" time="0" classname="StringUtilTest" />
</testsuite>
<testsuite name="AbstractProcessorTest" tests="2" failures="0" disabled="0" errors="0" time="0.003">
<testcase name="invalidationTest" status="run" time="0" classname="AbstractProcessorTest" />
<testcase name="lockingTest" status="run" time="0.001" classname="AbstractProcessorTest" />
</testsuite>
<testsuite name="PipelineWriteResultImageTest" tests="3" failures="0" disabled="0" errors="0" time="2.034">
<testcase name="VolumeExplorerDemo" status="run" time="1.447" classname="PipelineWriteResultImageTest" />
<testcase name="GeometryRendererDemo" status="run" time="0.268" classname="PipelineWriteResultImageTest" />
<testcase name="SliceVis" status="run" time="0.315" classname="PipelineWriteResultImageTest" />
</testsuite>
<testsuite name="2DTest/GlReductionTest" tests="6" failures="0" disabled="0" errors="0" time="0.062">
<testcase name="isScopedLockWorking/0" value_param="2" status="run" time="0" classname="2DTest/GlReductionTest" />
<testcase name="localRepCreationTest/0" value_param="2" status="run" time="0" classname="2DTest/GlReductionTest" />
<testcase name="minTest/0" value_param="2" status="run" time="0.016" classname="2DTest/GlReductionTest" />
<testcase name="maxTest/0" value_param="2" status="run" time="0.012" classname="2DTest/GlReductionTest" />
<testcase name="sumTest/0" value_param="2" status="run" time="0.012" classname="2DTest/GlReductionTest" />
<testcase name="multTest/0" value_param="2" status="run" time="0.014" classname="2DTest/GlReductionTest" />
</testsuite>
<testsuite name="3DTest/GlReductionTest" tests="6" failures="0" disabled="0" errors="0" time="0.056">
<testcase name="isScopedLockWorking/0" value_param="3" status="run" time="0" classname="3DTest/GlReductionTest" />
<testcase name="localRepCreationTest/0" value_param="3" status="run" time="0" classname="3DTest/GlReductionTest" />
<testcase name="minTest/0" value_param="3" status="run" time="0.012" classname="3DTest/GlReductionTest" />
<testcase name="maxTest/0" value_param="3" status="run" time="0.012" classname="3DTest/GlReductionTest" />
<testcase name="sumTest/0" value_param="3" status="run" time="0.011" classname="3DTest/GlReductionTest" />
<testcase name="multTest/0" value_param="3" status="run" time="0.011" classname="3DTest/GlReductionTest" />
</testsuite>
</testsuites>
test/visregtests/script.py
View file @ e71359de
......@@ -36,6 +36,7 @@ from skimage import io#, img_as_float, data
#from skimage.measure import structural_similarity as ssim
import xml.etree.ElementTree as et
import numpy as np
import shutil as fio
from xml.etree import ElementTree
from xml.dom import minidom
......@@ -49,25 +50,26 @@ def prettify(elem):
refDir = 'reference/';
testDir = 'testruns/';
resultDir = 'results/';
failedDir = 'failed/';
if (not os.path.exists(resultDir)) :
os.mkdir(resultDir)
casesDir = os.listdir(refDir);
prevRunDirs = os.listdir(testDir);
prevResDirs = os.listdir(resultDir);
#prevRunDirs = os.listdir(testDir);
#prevResDirs = os.listdir(resultDir);
# Create a new directory to store result
curRun = 0;
for i in range(len(prevResDirs)) :
prevRunDirs[i] = int(prevRunDirs[i][0:])
prevResDirs[i] = int(prevResDirs[i][0:])
if (prevRunDirs[i] == prevResDirs[i]) :
curRun += 1;
continue;
break;
#curRun = 0;
#for i in range(len(prevResDirs)) :
# prevRunDirs[i] = int(prevRunDirs[i][0:])
# prevResDirs[i] = int(prevResDirs[i][0:])
# if (prevRunDirs[i] == prevResDirs[i]) :
# curRun += 1;
# continue;
# break;
# List the test runs that are not computed yet
newTestDirs = prevRunDirs[curRun:];
#newTestDirs = prevRunDirs[curRun:];
# Find or create results.xml file created by test-campvis
xmlFile = "result.xml";
......@@ -79,73 +81,90 @@ else :
"errors":"0", "timestamp":"2014-08-24T01:35:42", "time":"0", "name":"AllTests"});
tree = et.ElementTree(root);
'''
for test in newTestDirs :
curTestDir = testDir + str(prevRunDirs[curRun]) + "/";
resultSaveDir = resultDir + str(prevRunDirs[curRun]) + "/";
curRun += 1;
os.mkdir(resultSaveDir)
os.mkdir(resultSaveDir)'''
for case in casesDir :
refCaseDir = refDir + case + "/";
testCaseDir = curTestDir + case + "/";
resCaseDir = resultSaveDir + case + "/";
# if no corresponding test directory - continue
if (not os.path.exists(testCaseDir)) :
continue;
curTestDir = testDir;
resultSaveDir = resultDir;
for case in casesDir :
refCaseDir = refDir + case + "/";
testCaseDir = curTestDir + case + "/";
resCaseDir = resultSaveDir + case + "/";
# if no corresponding test directory - continue
if (not os.path.exists(testCaseDir)) :
continue;
if (not os.path.exists(resCaseDir)) :
os.mkdir(resCaseDir)
files = os.listdir(refCaseDir)
if (len(files) != 0) :
suite = et.SubElement(root, "testsuite", {"name":refCaseDir, "tests":"0",
"failures":"0", "disabled":"0", "errors":"0", "time":"0"});
i = 0;
for file in files :
refFilePath = refCaseDir + file;
testFilePath = testCaseDir + file;
resFilePath = resCaseDir + file;
# Check existence of test file
if (not os.path.isfile(testFilePath)) :
continue;
if (refFilePath[-4:] != ".jpg" and refFilePath[-4:] != ".png"
and refFilePath[-4:] != ".tif") :
continue;
ref = io.imread(refFilePath);
test = io.imread(testFilePath);
# Check dimension of the file before finding difference
if (ref.shape == test.shape) :
test = ref-test;
else :
test = ref;
io.imsave(resFilePath, test);
#(x, y, z) = test.shape
#plt.figure(figsize=(4, 4))
#plt.imshow(test)#, cmap='gray', interpolation='nearest')
#plt.axis('off')
#plt.tight_layout()
#plt.show()
files = os.listdir(refCaseDir)
if (len(files) != 0) :
suite = et.SubElement(root, "testsuite", {"name":refCaseDir, "tests":"0",
"failures":"0", "disabled":"0", "errors":"0", "time":"0"});
i = 0;
for file in files :
refFilePath = refCaseDir + file;
testFilePath = testCaseDir + file;
resFilePath = resCaseDir + file;
# Check existence of test file
if (not os.path.isfile(testFilePath)) :
continue;
if (refFilePath[-4:] != ".jpg" and refFilePath[-4:] != ".png"
and refFilePath[-4:] != ".tif") :
continue;
ref = io.imread(refFilePath);
test = io.imread(testFilePath);
# Check dimension of the file before finding difference
if (ref.shape == test.shape) :
test = ref-test;
else :
test = ref;
io.imsave(resFilePath, test);
#(x, y, z) = test.shape
#plt.figure(figsize=(4, 4))
#plt.imshow(test)#, cmap='gray', interpolation='nearest')
#plt.axis('off')
#plt.tight_layout()
#plt.show()
case = et.SubElement(suite, "testcase", {"name":file, "status":"run",
"time":"0", "classname":refCaseDir});
suite.set("tests", str(int(suite.get("tests"))+1));
root.set("tests", str(int(root.get("tests"))+1));
if (np.sum(test) != 0) :
failure = et.SubElement(case, "failure", {"message":"", "type":""});
failure.set("message", "Image difference is not 0");
'''failure.text = "<![CDATA[" + \
"MESSAGE" + \
"]]>";'''
failure.text = "Reference and test images differ in " + str(sum(1 for x in test if x.any() > 0)) + " pixel/s";
suite.set("failures", str(int(suite.get("failures")) + 1));
root.set("failures", str(int(root.get("failures")) + 1));
case = et.SubElement(suite, "testcase", {"name":file, "status":"run",
"time":"0", "classname":refCaseDir});
suite.set("tests", str(int(suite.get("tests"))+1));
root.set("tests", str(int(root.get("tests"))+1));
print "failed";
if (not os.path.exists(failedDir + refCaseDir)) :
os.makedirs(failedDir + refCaseDir);
fio.copy(refFilePath, failedDir + refFilePath);
if (not os.path.exists(failedDir + testCaseDir)) :
os.makedirs(failedDir + testCaseDir);
fio.copy(testFilePath, failedDir + testFilePath);
if (np.sum(test) != 0) :#
failure = et.SubElement(case, "failure", {"message":"", "type":""});
failure.set("message", "Image difference is not 0");
failure.text = "<![CDATA[" + \
"MESSAGE" + \
"]]>";
suite.set("failures", str(int(suite.get("failures")) + 1));
root.set("failures", str(int(root.get("failures")) + 1));
if (not os.path.exists(failedDir + resCaseDir)) :
os.makedirs(failedDir + resCaseDir);
fio.copy(resFilePath, failedDir + resFilePath);
tree.write(xmlFile);
print ""
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment