Merge branch 'visregtest' into development

* Introducing visual regression testing framework
* Added PipelineResultImageTest case to run pipelines and write their rendering results to PNGs.
* Updated DevilImageWriter, added property to make writing depth images optional.

modules/devil/processors/devilimagewriter.cpp

@@ -46,10 +46,12 @@ namespace campvis {
     DevilImageWriter::DevilImageWriter()
         : AbstractProcessor()
         , p_inputImage("InputImage", "Input Image ID", "DevilImageWriter.input", DataNameProperty::READ)
-        , p_url("url", "Image URL", "", StringProperty::SAVE_FILENAME)
+        , p_url("Url", "Image URL", "", StringProperty::SAVE_FILENAME)
+        , p_writeDepthImage("WriteDepthImage", "Write Depth Image", false)
     {
         addProperty(p_inputImage);
         addProperty(p_url);
+        addProperty(p_writeDepthImage);
     }
 
     DevilImageWriter::~DevilImageWriter() {
@@ -64,7 +66,7 @@ namespace campvis {
             std::string filebase = tgt::FileSystem::fullBaseName(filename);
             if (extension.empty()) {
                 LINFO("Filename has no extension, defaulting to .PNG.");
-                extension = ".png";
+                extension = "png";
             }
 
             for (size_t i = 0; i < image->getNumColorTextures(); ++i) {
@@ -75,16 +77,16 @@ namespace campvis {
                 }
 
                 WeaklyTypedPointer wtp = id->getWeaklyTypedPointer();
-                writeIlImage(wtp, id->getSize().xy(), filebase + ".color" + ((image->getNumColorTextures() > 1) ? StringUtils::toString(i) : "") + extension);
+                writeIlImage(wtp, id->getSize().xy(), filebase + ((image->getNumColorTextures() > 1) ? StringUtils::toString(i) : "") + "." + extension);
             }
-            if (image->hasDepthTexture()) {
+            if (p_writeDepthImage.getValue() && image->hasDepthTexture()) {
                 const ImageRepresentationLocal* id = image->getDepthTexture()->getRepresentation<ImageRepresentationLocal>(true);
                 if (id == 0) {
                     LERROR("Could not download depth texture from RenderData, skipping.");
                 }
                 else {
                     WeaklyTypedPointer wtp = id->getWeaklyTypedPointer();
-                    writeIlImage(wtp, id->getSize().xy(), filebase + ".depth" + extension);
+                    writeIlImage(wtp, id->getSize().xy(), filebase + ".depth." + extension);
                 }
             }
         }
@@ -106,12 +108,13 @@ namespace campvis {
         ILboolean success = ilSaveImage(filename.c_str());
         ilDeleteImages(1, &img);
 
-        if (success != IL_NO_ERROR) {
+        if (! success) {
             ILenum errorcode;
             while ((errorcode = ilGetError()) != IL_NO_ERROR) {
                 LERROR("Error while writing '" << filename << "': "<< iluErrorString(errorcode));
             } 
         }
+
     }
 
 }
\ No newline at end of file

modules/devil/processors/devilimagewriter.h

@@ -70,6 +70,7 @@ namespace campvis {
 
         DataNameProperty p_inputImage;      ///< image ID for image to write
         StringProperty p_url;               ///< URL for file to write
+        BoolProperty p_writeDepthImage;     ///< Flag whether to save also depth image
 
     protected:
         /// \see AbstractProcessor::updateResult

test/CMakeLists.txt

@@ -10,7 +10,8 @@ FILE(GLOB TestCampvisSources RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
     core/datastructures/*.cpp
     core/properties/*.cpp
     core/tools/*.cpp
-	core/pipeline/*.cpp
+    core/pipeline/*.cpp
+    modules/*.cpp
 )
 
 # Summary of tuple support for Microsoft Visual Studio:
@@ -37,3 +38,4 @@ if (CAMPVIS_GROUP_SOURCE_FILES)
     DEFINE_SOURCE_GROUPS_FROM_SUBDIR(TestCampvisSources ${CampvisHome}/test "")
     DEFINE_SOURCE_GROUPS_FROM_SUBDIR(TestCampvisHeaders ${CampvisHome}/test "")
 ENDIF()
+

test/main.cpp

@@ -80,7 +80,8 @@ void init() {
     
     tgt::GLContextScopedLock lock(_localContext);
     tgt::GlContextManager::getRef().registerContextAndInitGlew(_localContext);
-    campvis::OpenGLJobProcessor::getRef().registerContext(_localContext);
+    GLJobProc.iKnowWhatImDoingSetThisThreadOpenGlThread();
+    GLJobProc.registerContext(_localContext);
 
     tgt::initGL(featureset);
     ShdrMgr.setDefaultGlslVersion("330");
@@ -142,7 +143,10 @@ GTEST_API_ int main(int argc, char **argv) {
 
 
     app = new QApplication(argc, argv);
-    testing::InitGoogleTest(&argc, argv);
+    //testing::InitGoogleTest(&argc, argv);
+    int _argc = 2;
+    char *options[] = {"THIS DOESN'T HAVE ANY EFFECT", "--gtest_output=xml:visregtests/result.xml"};
+    testing::InitGoogleTest(&_argc, options);
 
     int ret;
 

test/modules/pipelineresultimagetest.cpp

+// ================================================================================================
+// 
+// This file is part of the CAMPVis Software Framework.
+// 
+// If not explicitly stated otherwise: Copyright (C) 2012-2014, all rights reserved,
+//      Christian Schulte zu Berge <christian.szb@in.tum.de>
+//      Chair for Computer Aided Medical Procedures
+//      Technische Universitaet Muenchen
+//      Boltzmannstr. 3, 85748 Garching b. Muenchen, Germany
+// 
+// For a full list of authors and contributors, please refer to the file "AUTHORS.txt".
+// 
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 
+// except in compliance with the License. You may obtain a copy of the License at
+// 
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software distributed under the 
+// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 
+// either express or implied. See the License for the specific language governing permissions 
+// and limitations under the License.
+// 
+// ================================================================================================
+
+#include "gtest/gtest.h"
+
+#ifdef CAMPVIS_HAS_MODULE_DEVIL
+
+#include "tgt/filesystem.h"
+
+#include "core/datastructures/datacontainer.h"
+#include "core/datastructures/renderdata.h"
+#include "core/pipeline/abstractpipeline.h"
+#include "core/tools/opengljobprocessor.h"
+#include "core/tools/stringutils.h"
+
+#include "modules/pipelinefactory.h"
+#include "modules/devil/processors/devilimagewriter.h"
+
+using namespace campvis;
+
+
+/**
+ * Test class for creating test images for visual regression test
+ */
+
+class PipelineWriteResultImageTest : public ::testing::Test {
+protected:
+    PipelineWriteResultImageTest() 
+        : _dataContainer("Test Container")
+        , _pipeline(nullptr)
+        , _wroteFile(false)
+    {
+        _basePath = "visregtests/";
+        if ( ! tgt::FileSystem::dirExists(_basePath)) {
+            tgt::FileSystem::createDirectory(_basePath);
+        }
+        _basePath = "visregtests/testruns/";
+        if ( ! tgt::FileSystem::dirExists(_basePath)) {
+            tgt::FileSystem::createDirectory(_basePath);
+        }
+        std::vector<std::string> filelist = tgt::FileSystem::listSubDirectories(_basePath, true);
+        std::string caseNo = "1/";
+        tgt::FileSystem::createDirectoryRecursive(_basePath+ caseNo);
+        _basePath += caseNo;
+    }
+
+    ~PipelineWriteResultImageTest() {
+        delete _pipeline;
+    }
+
+    virtual void SetUp() {
+    }
+
+    virtual void TearDown() {
+        if (_pipeline != nullptr) {
+            _pipeline->setEnabled(false);
+            _pipeline->deinit();
+        }
+    }
+
+    void init() {
+        // create pipeline
+        _pipeline = PipelineFactory::getRef().createPipeline(_pipelineName, &_dataContainer);
+
+        if (_pipeline != nullptr) {
+            // setup pipeline
+            _pipeline->setCanvas(GLJobProc.iKnowWhatImDoingGetArbitraryContext());
+            _pipeline->init();
+            _pipeline->setEnabled(true);
+            _pipeline->setRenderTargetSize(tgt::ivec2(1024, 1024));
+        }
+    }
+
+    void execute() {
+        if (_pipeline != nullptr) {
+            // invalidate each processor
+            std::vector<AbstractProcessor*> processors = _pipeline->getProcessors();
+            for (size_t i = 0; i < processors.size(); ++i) {
+                processors[i]->invalidate(AbstractProcessor::INVALID_RESULT);
+            }
+
+            // execute each processor (we do this n*n times, as we might have a complex dependency graph)
+            for (size_t i = 0; i < processors.size(); ++i) {
+                for (size_t i = 0; i < processors.size(); ++i) {
+                    processors[i]->process(_dataContainer);
+                }
+            }
+
+            // write result image
+            _imageWriter.p_inputImage.setValue(_pipeline->getRenderTargetID());
+            _imageWriter.p_url.setValue(_fileName);
+            _imageWriter.process(_dataContainer);
+
+            _wroteFile = tgt::FileSystem::fileExists(_fileName);
+        }
+    }
+
+protected:
+    std::string _pipelineName;
+    std::string _fileName;
+    std::string _basePath;
+    static int _prevNoCases;
+
+    DataContainer _dataContainer;
+    AbstractPipeline* _pipeline;
+    DevilImageWriter _imageWriter;
+
+    bool _wroteFile;
+};
+int PipelineWriteResultImageTest::_prevNoCases = 0;
+
+TEST_F(PipelineWriteResultImageTest, VolumeExplorerDemo) {
+    _pipelineName = "VolumeExplorerDemo";
+    _fileName = _basePath +"volumeexplorerdemo.png";
+    init();
+    execute();
+    EXPECT_TRUE(_wroteFile);
+}
+
+TEST_F(PipelineWriteResultImageTest, GeometryRendererDemo) {
+    _pipelineName = "GeometryRendererDemo";
+    _fileName = _basePath +"geometryrendererdemo.png";
+    init();
+    execute();
+    EXPECT_TRUE(_wroteFile);
+}
+
+TEST_F(PipelineWriteResultImageTest, SliceVis) {
+    _pipelineName = "SliceVis";
+    _fileName = _basePath +"slicevis.png";
+    init();
+    execute();
+    EXPECT_TRUE(_wroteFile);
+}
+
+#endif

test/visregtests/script.py

+# ================================================================================================
+# 
+# This file is part of the CAMPVis Software Framework.
+# 
+# If not explicitly stated otherwise: Copyright (C) 2012-2014, all rights reserved,
+#      Christian Schulte zu Berge <christian.szb@in.tum.de>
+#      Chair for Computer Aided Medical Procedures
+#      Technische Universitaet Muenchen
+#      Boltzmannstr. 3, 85748 Garching b. Muenchen, Germany
+# 
+# For a full list of authors and contributors, please refer to the file "AUTHORS.txt".
+# 
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 
+# except in compliance with the License. You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software distributed under the 
+# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 
+# either express or implied. See the License for the specific language governing permissions 
+# and limitations under the License.
+# 
+# ================================================================================================
+
+import os
+import os.path
+import xml.etree.ElementTree as et
+import numpy as np
+import shutil as fio
+
+from skimage import io, color
+from xml.etree import ElementTree
+from xml.dom import minidom
+from skimage.measure import structural_similarity as ssim
+
+# Return a pretty-printed XML string for the Element
+def prettify(elem):
+    rough_string = ElementTree.tostring(elem, 'utf-8')
+    reparsed = minidom.parseString(rough_string)
+    return reparsed.toprettyxml(indent="  ")
+
+refDir = 'reference/';
+testDir = 'testruns/';
+resultDir = 'results/';
+failedDir = 'failed/';
+if (not os.path.exists(resultDir)) :
+        os.mkdir(resultDir)
+
+casesDir = os.listdir(refDir);
+
+# Find or create results.xml file created by test-campvis
+xmlFile = "result.xml";
+if (os.path.isfile(xmlFile)) :
+    tree = et.parse(xmlFile);
+    root = tree.getroot();
+else :
+    # XML
+    root = et.Element("testsuites", {"tests":"0", "failures":"0", "disabled":"0", 
+    "errors":"0", "timestamp":"2014-08-24T01:35:42", "time":"0", "name":"AllTests"}); 
+    tree = et.ElementTree(root);
+    
+curTestDir = testDir;
+resultSaveDir = resultDir;
+for case in casesDir :
+    refCaseDir = refDir + case + "/";
+    testCaseDir = curTestDir + case + "/";
+    resCaseDir = resultSaveDir + case + "/";
+    # if no corresponding test directory - continue
+    if (not os.path.exists(testCaseDir)) :
+        continue;
+    if (not os.path.exists(resCaseDir)) :
+        os.mkdir(resCaseDir)
+
+    files = os.listdir(refCaseDir)
+    if (len(files) != 0) :
+        # XML
+        suite = et.SubElement(root, "testsuite", {"name":refCaseDir, "tests":"0", 
+        "failures":"0", "disabled":"0", "errors":"0", "time":"0"});
+
+    i = 0;
+    for file in files :
+        refFilePath = refCaseDir + file;
+        testFilePath = testCaseDir + file;
+        resFilePath = resCaseDir + file;
+        alphaFilePath = resFilePath[:-4]+"_aplha"+resFilePath[-4:];
+        # Check existence of test file       
+        if (not os.path.isfile(testFilePath)) :
+            continue;
+
+        if (refFilePath[-4:] != ".jpg" and refFilePath[-4:] != ".png" 
+        and refFilePath[-4:] != ".tif") :
+            continue;
+
+        ref = io.imread(refFilePath);
+        testim = io.imread(testFilePath);
+        # Check dimension of the file before finding difference
+        if (ref.shape == testim.shape) :
+            test = np.abs(ref.astype(int)-testim.astype(int));
+        else :
+            test = ref;
+        # Store fully opaque image    
+        rgb = test[:, :, : 3];
+        alpha = test[:, :, 3:];
+        opaque = [[[255]*alpha.shape[2]] * alpha.shape[1]] * alpha.shape[0]
+        #io.imsave(resFilePath, rgb);
+        io.imsave(resFilePath, np.concatenate((rgb, opaque), axis=2));
+        io.imsave(alphaFilePath, alpha);
+        # Calculate MSE and SSIM
+        mse = np.linalg.norm(test);
+        reff = color.rgb2gray(ref);
+        testf = color.rgb2gray(test.astype(float));
+        ssimval = ssim(reff, testf);
+
+        # XML
+        case = et.SubElement(suite, "testcase", {"name":file, "status":"run", 
+        "time":"0", "classname":refCaseDir});
+        suite.set("tests", str(int(suite.get("tests"))+1));
+        root.set("tests", str(int(root.get("tests"))+1));
+
+        if (np.sum(test) != 0) :
+            # Prepare and write messages to show in stacktrace
+            failure = et.SubElement(case, "failure", {"message":"", "type":""});
+            failure.set("message", "Image difference is not 0");
+            alphamsg = "differ in both RGB and aplha channels";
+            if (sum(1 for x in rgb if x.any() > 0) == 0):
+                alphamsg = "differ in transparency level only";
+            elif (sum(1 for x in alpha if x.any() > 0) == 0) :
+                alphamsg = "differ in RGB channels only";
+
+            failure.text = "Reference and test images differ in " \
+            + str(sum(1 for x in test if x.any() > 0)) + " pixel/s\n" \
+            + "and images " + alphamsg \
+            + "\nMSE: " + str(mse) + " SSIM: " + str(ssimval);
+
+            suite.set("failures", str(int(suite.get("failures")) + 1));
+            root.set("failures", str(int(root.get("failures")) + 1));
+            
+            # Copy artifacts to failed directory
+            if (not os.path.exists(failedDir + refCaseDir)) :
+                os.makedirs(failedDir + refCaseDir);
+            fio.copy(refFilePath, failedDir + refFilePath);
+            
+            if (not os.path.exists(failedDir + testCaseDir)) :
+                os.makedirs(failedDir + testCaseDir);
+            fio.copy(testFilePath, failedDir + testFilePath);
+            
+            if (not os.path.exists(failedDir + resCaseDir)) :
+                os.makedirs(failedDir + resCaseDir);
+            fio.copy(resFilePath, failedDir + resFilePath);
+            fio.copy(alphaFilePath, failedDir + alphaFilePath);
+        #break;
+tree.write(xmlFile);