Commit a66402ab authored by Christian Schulte zu Berge's avatar Christian Schulte zu Berge

Merge branch 'visregtest' into development

* Introducing visual regression testing framework
* Added PipelineResultImageTest case to run pipelines and write their rendering results to PNGs.
* Updated DevilImageWriter, added property to make writing depth images optional.
parents 558e2873 cac55a2e
......@@ -46,10 +46,12 @@ namespace campvis {
DevilImageWriter::DevilImageWriter()
: AbstractProcessor()
, p_inputImage("InputImage", "Input Image ID", "DevilImageWriter.input", DataNameProperty::READ)
, p_url("url", "Image URL", "", StringProperty::SAVE_FILENAME)
, p_url("Url", "Image URL", "", StringProperty::SAVE_FILENAME)
, p_writeDepthImage("WriteDepthImage", "Write Depth Image", false)
{
addProperty(p_inputImage);
addProperty(p_url);
addProperty(p_writeDepthImage);
}
DevilImageWriter::~DevilImageWriter() {
......@@ -64,7 +66,7 @@ namespace campvis {
std::string filebase = tgt::FileSystem::fullBaseName(filename);
if (extension.empty()) {
LINFO("Filename has no extension, defaulting to .PNG.");
extension = ".png";
extension = "png";
}
for (size_t i = 0; i < image->getNumColorTextures(); ++i) {
......@@ -75,16 +77,16 @@ namespace campvis {
}
WeaklyTypedPointer wtp = id->getWeaklyTypedPointer();
writeIlImage(wtp, id->getSize().xy(), filebase + ".color" + ((image->getNumColorTextures() > 1) ? StringUtils::toString(i) : "") + extension);
writeIlImage(wtp, id->getSize().xy(), filebase + ((image->getNumColorTextures() > 1) ? StringUtils::toString(i) : "") + "." + extension);
}
if (image->hasDepthTexture()) {
if (p_writeDepthImage.getValue() && image->hasDepthTexture()) {
const ImageRepresentationLocal* id = image->getDepthTexture()->getRepresentation<ImageRepresentationLocal>(true);
if (id == 0) {
LERROR("Could not download depth texture from RenderData, skipping.");
}
else {
WeaklyTypedPointer wtp = id->getWeaklyTypedPointer();
writeIlImage(wtp, id->getSize().xy(), filebase + ".depth" + extension);
writeIlImage(wtp, id->getSize().xy(), filebase + ".depth." + extension);
}
}
}
......@@ -106,12 +108,13 @@ namespace campvis {
ILboolean success = ilSaveImage(filename.c_str());
ilDeleteImages(1, &img);
if (success != IL_NO_ERROR) {
if (! success) {
ILenum errorcode;
while ((errorcode = ilGetError()) != IL_NO_ERROR) {
LERROR("Error while writing '" << filename << "': "<< iluErrorString(errorcode));
}
}
}
}
\ No newline at end of file
......@@ -70,6 +70,7 @@ namespace campvis {
DataNameProperty p_inputImage; ///< image ID for image to write
StringProperty p_url; ///< URL for file to write
BoolProperty p_writeDepthImage; ///< Flag whether to save also depth image
protected:
/// \see AbstractProcessor::updateResult
......
......@@ -10,7 +10,8 @@ FILE(GLOB TestCampvisSources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
core/datastructures/*.cpp
core/properties/*.cpp
core/tools/*.cpp
core/pipeline/*.cpp
core/pipeline/*.cpp
modules/*.cpp
)
# Summary of tuple support for Microsoft Visual Studio:
......@@ -37,3 +38,4 @@ if (CAMPVIS_GROUP_SOURCE_FILES)
DEFINE_SOURCE_GROUPS_FROM_SUBDIR(TestCampvisSources ${CampvisHome}/test "")
DEFINE_SOURCE_GROUPS_FROM_SUBDIR(TestCampvisHeaders ${CampvisHome}/test "")
ENDIF()
......@@ -80,7 +80,8 @@ void init() {
tgt::GLContextScopedLock lock(_localContext);
tgt::GlContextManager::getRef().registerContextAndInitGlew(_localContext);
campvis::OpenGLJobProcessor::getRef().registerContext(_localContext);
GLJobProc.iKnowWhatImDoingSetThisThreadOpenGlThread();
GLJobProc.registerContext(_localContext);
tgt::initGL(featureset);
ShdrMgr.setDefaultGlslVersion("330");
......@@ -142,7 +143,10 @@ GTEST_API_ int main(int argc, char **argv) {
app = new QApplication(argc, argv);
testing::InitGoogleTest(&argc, argv);
//testing::InitGoogleTest(&argc, argv);
int _argc = 2;
char *options[] = {"THIS DOESN'T HAVE ANY EFFECT", "--gtest_output=xml:visregtests/result.xml"};
testing::InitGoogleTest(&_argc, options);
int ret;
......
// ================================================================================================
//
// This file is part of the CAMPVis Software Framework.
//
// If not explicitly stated otherwise: Copyright (C) 2012-2014, all rights reserved,
// Christian Schulte zu Berge <christian.szb@in.tum.de>
// Chair for Computer Aided Medical Procedures
// Technische Universitaet Muenchen
// Boltzmannstr. 3, 85748 Garching b. Muenchen, Germany
//
// For a full list of authors and contributors, please refer to the file "AUTHORS.txt".
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// ================================================================================================
#include "gtest/gtest.h"
#ifdef CAMPVIS_HAS_MODULE_DEVIL
#include "tgt/filesystem.h"
#include "core/datastructures/datacontainer.h"
#include "core/datastructures/renderdata.h"
#include "core/pipeline/abstractpipeline.h"
#include "core/tools/opengljobprocessor.h"
#include "core/tools/stringutils.h"
#include "modules/pipelinefactory.h"
#include "modules/devil/processors/devilimagewriter.h"
using namespace campvis;
/**
* Test class for creating test images for visual regression test
*/
class PipelineWriteResultImageTest : public ::testing::Test {
protected:
PipelineWriteResultImageTest()
: _dataContainer("Test Container")
, _pipeline(nullptr)
, _wroteFile(false)
{
_basePath = "visregtests/";
if ( ! tgt::FileSystem::dirExists(_basePath)) {
tgt::FileSystem::createDirectory(_basePath);
}
_basePath = "visregtests/testruns/";
if ( ! tgt::FileSystem::dirExists(_basePath)) {
tgt::FileSystem::createDirectory(_basePath);
}
std::vector<std::string> filelist = tgt::FileSystem::listSubDirectories(_basePath, true);
std::string caseNo = "1/";
tgt::FileSystem::createDirectoryRecursive(_basePath+ caseNo);
_basePath += caseNo;
}
~PipelineWriteResultImageTest() {
delete _pipeline;
}
virtual void SetUp() {
}
virtual void TearDown() {
if (_pipeline != nullptr) {
_pipeline->setEnabled(false);
_pipeline->deinit();
}
}
void init() {
// create pipeline
_pipeline = PipelineFactory::getRef().createPipeline(_pipelineName, &_dataContainer);
if (_pipeline != nullptr) {
// setup pipeline
_pipeline->setCanvas(GLJobProc.iKnowWhatImDoingGetArbitraryContext());
_pipeline->init();
_pipeline->setEnabled(true);
_pipeline->setRenderTargetSize(tgt::ivec2(1024, 1024));
}
}
void execute() {
if (_pipeline != nullptr) {
// invalidate each processor
std::vector<AbstractProcessor*> processors = _pipeline->getProcessors();
for (size_t i = 0; i < processors.size(); ++i) {
processors[i]->invalidate(AbstractProcessor::INVALID_RESULT);
}
// execute each processor (we do this n*n times, as we might have a complex dependency graph)
for (size_t i = 0; i < processors.size(); ++i) {
for (size_t i = 0; i < processors.size(); ++i) {
processors[i]->process(_dataContainer);
}
}
// write result image
_imageWriter.p_inputImage.setValue(_pipeline->getRenderTargetID());
_imageWriter.p_url.setValue(_fileName);
_imageWriter.process(_dataContainer);
_wroteFile = tgt::FileSystem::fileExists(_fileName);
}
}
protected:
std::string _pipelineName;
std::string _fileName;
std::string _basePath;
static int _prevNoCases;
DataContainer _dataContainer;
AbstractPipeline* _pipeline;
DevilImageWriter _imageWriter;
bool _wroteFile;
};
int PipelineWriteResultImageTest::_prevNoCases = 0;
TEST_F(PipelineWriteResultImageTest, VolumeExplorerDemo) {
_pipelineName = "VolumeExplorerDemo";
_fileName = _basePath +"volumeexplorerdemo.png";
init();
execute();
EXPECT_TRUE(_wroteFile);
}
TEST_F(PipelineWriteResultImageTest, GeometryRendererDemo) {
_pipelineName = "GeometryRendererDemo";
_fileName = _basePath +"geometryrendererdemo.png";
init();
execute();
EXPECT_TRUE(_wroteFile);
}
TEST_F(PipelineWriteResultImageTest, SliceVis) {
_pipelineName = "SliceVis";
_fileName = _basePath +"slicevis.png";
init();
execute();
EXPECT_TRUE(_wroteFile);
}
#endif
# ================================================================================================
#
# This file is part of the CAMPVis Software Framework.
#
# If not explicitly stated otherwise: Copyright (C) 2012-2014, all rights reserved,
# Christian Schulte zu Berge <christian.szb@in.tum.de>
# Chair for Computer Aided Medical Procedures
# Technische Universitaet Muenchen
# Boltzmannstr. 3, 85748 Garching b. Muenchen, Germany
#
# For a full list of authors and contributors, please refer to the file "AUTHORS.txt".
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the
# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific language governing permissions
# and limitations under the License.
#
# ================================================================================================
import os
import os.path
import xml.etree.ElementTree as et
import numpy as np
import shutil as fio
from skimage import io, color
from xml.etree import ElementTree
from xml.dom import minidom
from skimage.measure import structural_similarity as ssim
# Return a pretty-printed XML string for the Element
def prettify(elem):
rough_string = ElementTree.tostring(elem, 'utf-8')
reparsed = minidom.parseString(rough_string)
return reparsed.toprettyxml(indent=" ")
refDir = 'reference/';
testDir = 'testruns/';
resultDir = 'results/';
failedDir = 'failed/';
if (not os.path.exists(resultDir)) :
os.mkdir(resultDir)
casesDir = os.listdir(refDir);
# Find or create results.xml file created by test-campvis
xmlFile = "result.xml";
if (os.path.isfile(xmlFile)) :
tree = et.parse(xmlFile);
root = tree.getroot();
else :
# XML
root = et.Element("testsuites", {"tests":"0", "failures":"0", "disabled":"0",
"errors":"0", "timestamp":"2014-08-24T01:35:42", "time":"0", "name":"AllTests"});
tree = et.ElementTree(root);
curTestDir = testDir;
resultSaveDir = resultDir;
for case in casesDir :
refCaseDir = refDir + case + "/";
testCaseDir = curTestDir + case + "/";
resCaseDir = resultSaveDir + case + "/";
# if no corresponding test directory - continue
if (not os.path.exists(testCaseDir)) :
continue;
if (not os.path.exists(resCaseDir)) :
os.mkdir(resCaseDir)
files = os.listdir(refCaseDir)
if (len(files) != 0) :
# XML
suite = et.SubElement(root, "testsuite", {"name":refCaseDir, "tests":"0",
"failures":"0", "disabled":"0", "errors":"0", "time":"0"});
i = 0;
for file in files :
refFilePath = refCaseDir + file;
testFilePath = testCaseDir + file;
resFilePath = resCaseDir + file;
alphaFilePath = resFilePath[:-4]+"_aplha"+resFilePath[-4:];
# Check existence of test file
if (not os.path.isfile(testFilePath)) :
continue;
if (refFilePath[-4:] != ".jpg" and refFilePath[-4:] != ".png"
and refFilePath[-4:] != ".tif") :
continue;
ref = io.imread(refFilePath);
testim = io.imread(testFilePath);
# Check dimension of the file before finding difference
if (ref.shape == testim.shape) :
test = np.abs(ref.astype(int)-testim.astype(int));
else :
test = ref;
# Store fully opaque image
rgb = test[:, :, : 3];
alpha = test[:, :, 3:];
opaque = [[[255]*alpha.shape[2]] * alpha.shape[1]] * alpha.shape[0]
#io.imsave(resFilePath, rgb);
io.imsave(resFilePath, np.concatenate((rgb, opaque), axis=2));
io.imsave(alphaFilePath, alpha);
# Calculate MSE and SSIM
mse = np.linalg.norm(test);
reff = color.rgb2gray(ref);
testf = color.rgb2gray(test.astype(float));
ssimval = ssim(reff, testf);
# XML
case = et.SubElement(suite, "testcase", {"name":file, "status":"run",
"time":"0", "classname":refCaseDir});
suite.set("tests", str(int(suite.get("tests"))+1));
root.set("tests", str(int(root.get("tests"))+1));
if (np.sum(test) != 0) :
# Prepare and write messages to show in stacktrace
failure = et.SubElement(case, "failure", {"message":"", "type":""});
failure.set("message", "Image difference is not 0");
alphamsg = "differ in both RGB and aplha channels";
if (sum(1 for x in rgb if x.any() > 0) == 0):
alphamsg = "differ in transparency level only";
elif (sum(1 for x in alpha if x.any() > 0) == 0) :
alphamsg = "differ in RGB channels only";
failure.text = "Reference and test images differ in " \
+ str(sum(1 for x in test if x.any() > 0)) + " pixel/s\n" \
+ "and images " + alphamsg \
+ "\nMSE: " + str(mse) + " SSIM: " + str(ssimval);
suite.set("failures", str(int(suite.get("failures")) + 1));
root.set("failures", str(int(root.get("failures")) + 1));
# Copy artifacts to failed directory
if (not os.path.exists(failedDir + refCaseDir)) :
os.makedirs(failedDir + refCaseDir);
fio.copy(refFilePath, failedDir + refFilePath);
if (not os.path.exists(failedDir + testCaseDir)) :
os.makedirs(failedDir + testCaseDir);
fio.copy(testFilePath, failedDir + testFilePath);
if (not os.path.exists(failedDir + resCaseDir)) :
os.makedirs(failedDir + resCaseDir);
fio.copy(resFilePath, failedDir + resFilePath);
fio.copy(alphaFilePath, failedDir + alphaFilePath);
#break;
tree.write(xmlFile);
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment