DataHandlerGPU: Build and CI changes (#21)

825e8c5b · Jens Petit · 9223a5a6 · 825e8c5b · 825e8c5b · 825e8c5b
Commit 825e8c5b authored Mar 16, 2020 by Jens Petit
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,8 @@ variables:
  CLANG_IMAGE: "elsa/clang:9"
  CUDA_IMAGE: "elsa/cuda:10.2"
  UBUNTU_IMAGE: "elsa/ubuntu:18.04"
-  
+  GPU_ONLY_IMAGE: "elsa/cuda:10.0"
+

 ### job templates ###

@@ -24,11 +25,11 @@ variables:

 .job_template: &build_job
  stage: compile
-  cache: 
+  cache:
    key: "$CI_JOB_NAME"
    paths:
      - build/
-      
+
 .job_template: &build_job_artifact
  stage: compile
  cache:
@@ -46,7 +47,7 @@ variables:
    - mkdir -p build
    - cd build
    - cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
-    - ninja 
+    - ninja

 .buildjob_script_normal:
  script: &buildjob_normal
@@ -106,7 +107,7 @@ build-gcc:
    - linux
    - elsa
    - gcc
-    
+
 build-clang:
  <<: *build_job_artifact
  image: $CLANG_IMAGE
@@ -126,6 +127,20 @@ build-cuda:
    - gcc
    - cuda

+build-gpu-only:
+  <<: *build_job_artifact
+  image: $GPU_ONLY_IMAGE
+  script:
+    - mkdir -p build
+    - cd build
+    - cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi" -DELSA_CUDA_VECTOR=ON
+    - ninja
+  tags:
+    - linux
+    - elsa
+    - gcc
+    - cuda
+

 ### test jobs ###

@@ -133,7 +148,7 @@ test-ubuntu:
  <<: *test_job_artifact
  stage: test
  image: $UBUNTU_IMAGE
-  dependencies: 
+  dependencies:
    - build-ubuntu
  tags:
    - linux
@@ -149,7 +164,7 @@ test-gcc:
    - linux
    - elsa
    - gcc
-    
+
 test-clang:
  <<: *test_job_artifact
  stage: test
@@ -173,6 +188,28 @@ test-cuda:
    - gcc
    - cuda

+test-gpu-only:
+  stage: test
+  image: $GPU_ONLY_IMAGE
+  dependencies:
+    - build-gpu-only
+  script:
+    - cd build
+    - cmake -GNinja -DELSA_CREATE_JUNIT_REPORTS=ON ..
+    - ninja tests
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    paths:
+      - build/
+    expire_in: 30 mins
+    reports:
+      junit: "build/test_reports/test_*.xml"
+  tags:
+    - linux
+    - elsa
+    - gcc
+    - cuda
+

 ### test coverage ###

@@ -198,8 +235,22 @@ test-coverage:
    - gcc
    - cuda

+
 ### Sanitizers ###

+cuda-memcheck:
+  stage: sanitizer
+  image: $GPU_ONLY_IMAGE
+  script:
+    ./tools/ci_scripts/cuda-memcheck.sh
+  dependencies:
+    - test-gpu-only
+  tags:
+    - linux
+    - elsa
+    - gcc
+    - cuda
+
 #thread-sanitizer:
 #  <<: *nightly_job
 #  stage: sanitizer

--- a/.gitmodules
+++ b/.gitmodules
@@ -7,3 +7,6 @@
 [submodule "thirdparty/spdlog"]
 	path = thirdparty/spdlog
 	url = https://github.com/gabime/spdlog.git
+[submodule "thirdparty/quickvec"]
+	path = thirdparty/quickvec
+	url = https://gitlab.lrz.de/IP/quickvec.git
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -31,6 +31,7 @@ option(ELSA_BUILD_WITH_MORE_WARNINGS "Enable all and extra warnings when buildin
 option(ELSA_SANITIZE_THREAD "Build elsa with thread sanitizers (TSAN)" OFF)
 option(ELSA_SANITIZE_ADDRESS "Build elsa with address and undefined-behavior sanitizers (ASAN and UBSAN)" OFF)

+option(ELSA_CUDA_VECTOR "Build elsa with GPU DataContainer support and default" OFF)

 # ------------ general setup -----------
 # ------------
@@ -63,6 +64,12 @@ if(ELSA_MASTER_PROJECT)

    # setup Dnnl library
    find_package(Dnnl)
+
+    if(ELSA_CUDA_VECTOR)
+        find_package(CUDA REQUIRED)
+        find_package(Quickvec REQUIRED)
+        message(STATUS "Using Quickvec submodule at ${Quickvec_INCLUDE_DIR}")
+    endif()
 else()
    message(STATUS "    No dependencies added for elsa, as it is not stand-alone")
 endif()
@@ -78,6 +85,32 @@ set(INSTALL_CONFIG_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/elsa)
 include(InstallElsaModule)


+# ------------ setup VectorCUDA --------
+# ------------
+if(ELSA_CUDA_VECTOR)
+    set(CMAKE_CXX_EXTENSIONS OFF)
+
+    include(CheckLanguage)
+    check_language(CUDA)
+
+    if (CMAKE_CUDA_COMPILER AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+        enable_language(CUDA)
+
+        message(STATUS "CUDA support enabled")
+        message(STATUS "Enabling DataHandlerGPU")
+        message(STATUS "The default container type will be on the GPU")
+
+        add_definitions(-DELSA_CUDA_VECTOR)
+        add_subdirectory(thirdparty/quickvec)
+    else()
+        message(STATUS "CUDA compiler not found or not compiling with clang!")
+        message(STATUS "Compiling without a DataHandlerGPU")
+        message(STATUS "The default container type will be CPU")
+        set(ELSA_CUDA_VECTOR OFF)
+    endif()
+endif()
+
+
 # ------------ setup testing -----------
 # ------------

@@ -174,4 +207,4 @@ add_custom_target(uninstall
    COMMAND ${CMAKE_COMMAND} -P "${PROJECT_SOURCE_DIR}/cmake/Uninstall.cmake"
    COMMENT "Uninstalling elsa (Removing all installed files)"
    DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt"
-    )
\ No newline at end of file
+    )
--- a/cmake/FindQuickvec.cmake
+++ b/cmake/FindQuickvec.cmake
+# Find Quickvec library which implements GPU based vector arithmetic
+#
+# The following variables are set if Quickvec is found. Quickvec_FOUND - True when the include directory is found.
+# Quickvec_INCLUDE_DIR - The path to the headers
+#
+# This module reads hints about search locations from the following environment variables:
+#
+# Quickvec_ROOT Quickvec_ROOT_DIR
+#
+# as well as in the following subdirectories
+#
+# include thirdparty ThirdParty external
+#
+# If
+#
+# Quickvec_INCLUDE_DIR
+#
+# is explicitly set, the search falls back to the values in these variables
+
+include(FindPackageHandleStandardArgs)
+
+macro(_quickvec_find_include)
+  find_path(Quickvec_INCLUDE_DIR
+            NAMES quickvec/src/Quickvec.h
+            HINTS ENV
+                  Quickvec_ROOT
+                  ENV
+                  Quickvec_ROOT_DIR
+                  include
+                  thirdparty
+                  ThirdParty
+                  external
+            DOC "Quickvec header files"
+            PATH_SUFFIXES quickvec)
+endmacro(_quickvec_find_include)
+
+if(NOT EXISTS "${Quickvec_INCLUDE_DIR}")
+  _quickvec_find_include()
+endif()
+
+find_package_handle_standard_args(Quickvec
+                                  DEFAULT_MSG
+                                  Quickvec_INCLUDE_DIR)
+
+mark_as_advanced(Quickvec_INCLUDE_DIR)
+
--- a/cmake/UpdateGitSubmodules.cmake
+++ b/cmake/UpdateGitSubmodules.cmake
@@ -6,7 +6,7 @@ find_package(Git QUIET)
 if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
    if(GIT_SUBMODULE) # do this only if the GIT_SUBMODULE option is enabled
        message(STATUS "Git submodule update")
-        execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive
+        execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init
                        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
                        RESULT_VARIABLE GIT_SUBMODULE_RESULT)
        if(NOT GIT_SUBMODULE_RESULT EQUAL "0")
@@ -20,4 +20,4 @@ if (NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/Catch2/CMakeLists.txt" OR
    NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/eigen3/CMakeLists.txt" OR
    NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/spdlog/CMakeLists.txt")
    message(FATAL_ERROR "The git submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
-endif()
\ No newline at end of file
+endif()
--- a/elsa/core/CMakeLists.txt
+++ b/elsa/core/CMakeLists.txt
@@ -35,6 +35,17 @@ set(MODULE_SOURCES
        DataHandlerMapCPU.cpp
        LinearOperator.cpp)

+if(${ELSA_CUDA_VECTOR})
+    set(MODULE_HEADERS
+        ${MODULE_HEADERS}
+        DataHandlerGPU.h
+        DataHandlerMapGPU.h)
+
+    set(MODULE_SOURCES
+        ${MODULE_SOURCES}
+        DataHandlerGPU.cpp
+        DataHandlerMapGPU.cpp)
+endif()

 # build the module library
 add_library(${ELSA_MODULE_TARGET_NAME} ${MODULE_HEADERS} ${MODULE_SOURCES})
@@ -48,6 +59,10 @@ if(OpenMP_CXX_FOUND)
    target_link_libraries(${ELSA_MODULE_TARGET_NAME} PRIVATE OpenMP::OpenMP_CXX)
 endif()

+# link Quickvec for DataHandlerGPU
+if(ELSA_CUDA_VECTOR AND Quickvec_FOUND)
+    target_link_libraries(${ELSA_MODULE_TARGET_NAME} PUBLIC Quickvec::quickvec)
+endif()

 target_include_directories(${ELSA_MODULE_TARGET_NAME}
        PUBLIC

--- a/tools/ci_scripts/clang-tidy.sh
+++ b/tools/ci_scripts/clang-tidy.sh
@@ -5,7 +5,7 @@ exit_flag=false
 # for compilation database
 mkdir -p build
 cd build
-cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DELSA_CUDA_VECTOR=ON
 cd ..

 target_branch="master"
@@ -28,11 +28,11 @@ filelist=`git diff origin/${target_branch} --name-only`

 # check list of files
 for f in $filelist; do
-    if checkCPP $f; then
+    # check if .cpp file and in compilation DB
+    if checkCPP $f && [[ -n $(grep $f build/compile_commands.json) ]]; then
        echo "Checking matching file ${f}"
-        # apply the clang-format script
        touch output.txt
-        clang-tidy-8 -p=build ${f} > output.txt
+        clang-tidy-8 -p=build ${f} --extra-arg=--cuda-host-only > output.txt
        # decide if error or warning fail
        if [[ -n $(grep "warning: " output.txt) ]] || [[ -n $(grep "error: " output.txt) ]]; then
            echo ""

--- a/tools/ci_scripts/cuda-memcheck.sh
+++ b/tools/ci_scripts/cuda-memcheck.sh
+#!/bin/bash
+
+FILES="build/elsa/core/tests/test_ExpressionTemplates"
+
+if type cuda-memcheck &> /dev/null; then
+    echo "cuda-memcheck available in version:"
+    cuda-memcheck --version
+else
+    echo "cuda-memcheck not correctly installed"
+    exit 1
+fi
+
+for f in $FILES
+do
+    echo
+    echo "Processing $f"
+    if [ ! -x $f ]; then
+        echo "File $f not available, error!"
+        exit 1
+    else
+        cuda-memcheck --leak-check full --error-exitcode 1 $f
+        if [ $? -ne 0 ]; then
+            echo
+            echo "cuda-memcheck errors reported"
+            exit 1
+        fi
+    fi
+done
+
+echo
+echo "All cuda-memcheck tests pass"
+exit 0