DataHandlerGPU: Build and CI changes (#21)

.gitlab-ci.yml

@@ -14,7 +14,8 @@ variables:
   CLANG_IMAGE: "elsa/clang:9"
   CUDA_IMAGE: "elsa/cuda:10.2"
   UBUNTU_IMAGE: "elsa/ubuntu:18.04"
-  
+  GPU_ONLY_IMAGE: "elsa/cuda:10.0"
+
 
 ### job templates ###
 
@@ -24,11 +25,11 @@ variables:
 
 .job_template: &build_job
   stage: compile
-  cache: 
+  cache:
     key: "$CI_JOB_NAME"
     paths:
       - build/
-      
+
 .job_template: &build_job_artifact
   stage: compile
   cache:
@@ -46,7 +47,7 @@ variables:
     - mkdir -p build
     - cd build
     - cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
-    - ninja 
+    - ninja
 
 .buildjob_script_normal:
   script: &buildjob_normal
@@ -106,7 +107,7 @@ build-gcc:
     - linux
     - elsa
     - gcc
-    
+
 build-clang:
   <<: *build_job_artifact
   image: $CLANG_IMAGE
@@ -126,6 +127,20 @@ build-cuda:
     - gcc
     - cuda
 
+build-gpu-only:
+  <<: *build_job_artifact
+  image: $GPU_ONLY_IMAGE
+  script:
+    - mkdir -p build
+    - cd build
+    - cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi" -DELSA_CUDA_VECTOR=ON
+    - ninja
+  tags:
+    - linux
+    - elsa
+    - gcc
+    - cuda
+
 
 ### test jobs ###
 
@@ -133,7 +148,7 @@ test-ubuntu:
   <<: *test_job_artifact
   stage: test
   image: $UBUNTU_IMAGE
-  dependencies: 
+  dependencies:
     - build-ubuntu
   tags:
     - linux
@@ -149,7 +164,7 @@ test-gcc:
     - linux
     - elsa
     - gcc
-    
+
 test-clang:
   <<: *test_job_artifact
   stage: test
@@ -173,6 +188,28 @@ test-cuda:
     - gcc
     - cuda
 
+test-gpu-only:
+  stage: test
+  image: $GPU_ONLY_IMAGE
+  dependencies:
+    - build-gpu-only
+  script:
+    - cd build
+    - cmake -GNinja -DELSA_CREATE_JUNIT_REPORTS=ON ..
+    - ninja tests
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    paths:
+      - build/
+    expire_in: 30 mins
+    reports:
+      junit: "build/test_reports/test_*.xml"
+  tags:
+    - linux
+    - elsa
+    - gcc
+    - cuda
+
 
 ### test coverage ###
 
@@ -198,8 +235,22 @@ test-coverage:
     - gcc
     - cuda
 
+
 ### Sanitizers ###
 
+cuda-memcheck:
+  stage: sanitizer
+  image: $GPU_ONLY_IMAGE
+  script:
+    ./tools/ci_scripts/cuda-memcheck.sh
+  dependencies:
+    - test-gpu-only
+  tags:
+    - linux
+    - elsa
+    - gcc
+    - cuda
+
 #thread-sanitizer:
 #  <<: *nightly_job
 #  stage: sanitizer

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "thirdparty/spdlog"]
 	path = thirdparty/spdlog
 	url = https://github.com/gabime/spdlog.git
+[submodule "thirdparty/quickvec"]
+	path = thirdparty/quickvec
+	url = https://gitlab.lrz.de/IP/quickvec.git

CMakeLists.txt

@@ -31,6 +31,7 @@ option(ELSA_BUILD_WITH_MORE_WARNINGS "Enable all and extra warnings when buildin
 option(ELSA_SANITIZE_THREAD "Build elsa with thread sanitizers (TSAN)" OFF)
 option(ELSA_SANITIZE_ADDRESS "Build elsa with address and undefined-behavior sanitizers (ASAN and UBSAN)" OFF)
 
+option(ELSA_CUDA_VECTOR "Build elsa with GPU DataContainer support and default" OFF)
 
 # ------------ general setup -----------
 # ------------
@@ -63,6 +64,12 @@ if(ELSA_MASTER_PROJECT)
 
     # setup Dnnl library
     find_package(Dnnl)
+
+    if(ELSA_CUDA_VECTOR)
+        find_package(CUDA REQUIRED)
+        find_package(Quickvec REQUIRED)
+        message(STATUS "Using Quickvec submodule at ${Quickvec_INCLUDE_DIR}")
+    endif()
 else()
     message(STATUS "    No dependencies added for elsa, as it is not stand-alone")
 endif()
@@ -78,6 +85,32 @@ set(INSTALL_CONFIG_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/elsa)
 include(InstallElsaModule)
 
 
+# ------------ setup VectorCUDA --------
+# ------------
+if(ELSA_CUDA_VECTOR)
+    set(CMAKE_CXX_EXTENSIONS OFF)
+
+    include(CheckLanguage)
+    check_language(CUDA)
+
+    if (CMAKE_CUDA_COMPILER AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+        enable_language(CUDA)
+
+        message(STATUS "CUDA support enabled")
+        message(STATUS "Enabling DataHandlerGPU")
+        message(STATUS "The default container type will be on the GPU")
+
+        add_definitions(-DELSA_CUDA_VECTOR)
+        add_subdirectory(thirdparty/quickvec)
+    else()
+        message(STATUS "CUDA compiler not found or not compiling with clang!")
+        message(STATUS "Compiling without a DataHandlerGPU")
+        message(STATUS "The default container type will be CPU")
+        set(ELSA_CUDA_VECTOR OFF)
+    endif()
+endif()
+
+
 # ------------ setup testing -----------
 # ------------
 
@@ -174,4 +207,4 @@ add_custom_target(uninstall
     COMMAND ${CMAKE_COMMAND} -P "${PROJECT_SOURCE_DIR}/cmake/Uninstall.cmake"
     COMMENT "Uninstalling elsa (Removing all installed files)"
     DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt"
-    )
\ No newline at end of file
+    )

cmake/FindQuickvec.cmake

+# Find Quickvec library which implements GPU based vector arithmetic
+#
+# The following variables are set if Quickvec is found. Quickvec_FOUND - True when the include directory is found.
+# Quickvec_INCLUDE_DIR - The path to the headers
+#
+# This module reads hints about search locations from the following environment variables:
+#
+# Quickvec_ROOT Quickvec_ROOT_DIR
+#
+# as well as in the following subdirectories
+#
+# include thirdparty ThirdParty external
+#
+# If
+#
+# Quickvec_INCLUDE_DIR
+#
+# is explicitly set, the search falls back to the values in these variables
+
+include(FindPackageHandleStandardArgs)
+
+macro(_quickvec_find_include)
+  find_path(Quickvec_INCLUDE_DIR
+            NAMES quickvec/src/Quickvec.h
+            HINTS ENV
+                  Quickvec_ROOT
+                  ENV
+                  Quickvec_ROOT_DIR
+                  include
+                  thirdparty
+                  ThirdParty
+                  external
+            DOC "Quickvec header files"
+            PATH_SUFFIXES quickvec)
+endmacro(_quickvec_find_include)
+
+if(NOT EXISTS "${Quickvec_INCLUDE_DIR}")
+  _quickvec_find_include()
+endif()
+
+find_package_handle_standard_args(Quickvec
+                                  DEFAULT_MSG
+                                  Quickvec_INCLUDE_DIR)
+
+mark_as_advanced(Quickvec_INCLUDE_DIR)
+

cmake/UpdateGitSubmodules.cmake

@@ -6,7 +6,7 @@ find_package(Git QUIET)
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
     if(GIT_SUBMODULE) # do this only if the GIT_SUBMODULE option is enabled
         message(STATUS "Git submodule update")
-        execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive
+        execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init
                         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
                         RESULT_VARIABLE GIT_SUBMODULE_RESULT)
         if(NOT GIT_SUBMODULE_RESULT EQUAL "0")
@@ -20,4 +20,4 @@ if (NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/Catch2/CMakeLists.txt" OR
     NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/eigen3/CMakeLists.txt" OR
     NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/spdlog/CMakeLists.txt")
     message(FATAL_ERROR "The git submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
\ No newline at end of file
+endif()

elsa/core/CMakeLists.txt

@@ -35,6 +35,17 @@ set(MODULE_SOURCES
         DataHandlerMapCPU.cpp
         LinearOperator.cpp)
 
+if(${ELSA_CUDA_VECTOR})
+    set(MODULE_HEADERS
+        ${MODULE_HEADERS}
+        DataHandlerGPU.h
+        DataHandlerMapGPU.h)
+
+    set(MODULE_SOURCES
+        ${MODULE_SOURCES}
+        DataHandlerGPU.cpp
+        DataHandlerMapGPU.cpp)
+endif()
 
 # build the module library
 add_library(${ELSA_MODULE_TARGET_NAME} ${MODULE_HEADERS} ${MODULE_SOURCES})
@@ -48,6 +59,10 @@ if(OpenMP_CXX_FOUND)
     target_link_libraries(${ELSA_MODULE_TARGET_NAME} PRIVATE OpenMP::OpenMP_CXX)
 endif()
 
+# link Quickvec for DataHandlerGPU
+if(ELSA_CUDA_VECTOR AND Quickvec_FOUND)
+    target_link_libraries(${ELSA_MODULE_TARGET_NAME} PUBLIC Quickvec::quickvec)
+endif()
 
 target_include_directories(${ELSA_MODULE_TARGET_NAME}
         PUBLIC

tools/ci_scripts/clang-tidy.sh

@@ -5,7 +5,7 @@ exit_flag=false
 # for compilation database
 mkdir -p build
 cd build
-cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DELSA_CUDA_VECTOR=ON
 cd ..
 
 target_branch="master"
@@ -28,11 +28,11 @@ filelist=`git diff origin/${target_branch} --name-only`
 
 # check list of files
 for f in $filelist; do
-    if checkCPP $f; then
+    # check if .cpp file and in compilation DB
+    if checkCPP $f && [[ -n $(grep $f build/compile_commands.json) ]]; then
         echo "Checking matching file ${f}"
-        # apply the clang-format script
         touch output.txt
-        clang-tidy-8 -p=build ${f} > output.txt
+        clang-tidy-8 -p=build ${f} --extra-arg=--cuda-host-only > output.txt
         # decide if error or warning fail
         if [[ -n $(grep "warning: " output.txt) ]] || [[ -n $(grep "error: " output.txt) ]]; then
             echo ""

tools/ci_scripts/cuda-memcheck.sh

+#!/bin/bash
+
+FILES="build/elsa/core/tests/test_ExpressionTemplates"
+
+if type cuda-memcheck &> /dev/null; then
+    echo "cuda-memcheck available in version:"
+    cuda-memcheck --version
+else
+    echo "cuda-memcheck not correctly installed"
+    exit 1
+fi
+
+for f in $FILES
+do
+    echo
+    echo "Processing $f"
+    if [ ! -x $f ]; then
+        echo "File $f not available, error!"
+        exit 1
+    else
+        cuda-memcheck --leak-check full --error-exitcode 1 $f
+        if [ $? -ne 0 ]; then
+            echo
+            echo "cuda-memcheck errors reported"
+            exit 1
+        fi
+    fi
+done
+
+echo
+echo "All cuda-memcheck tests pass"
+exit 0