Commit 6950275c authored by David Frank's avatar David Frank Committed by Tobias Lasser
Browse files

Reworked CI caching; Reworked CMake submodule update routine

- Chaching is introduced for for all jobs, it caches for a branch.
only submodules are chached, as build folders should be cached using
artifacts, which is also done. We also cache the .git/modules folder
as anyway we have to reclone everything everytime.
- CMake UpdateGitSubmodules now goes over all submodules and check
individually if they have to be initialized, or just updated.
Also work around a specific error in the CI pipeline, if
.git/modules/thirdparty/<submodule> is not present (which by default
dosent get cached), we have to deinit it and then reinit it.
parent f445e5a9
Pipeline #316559 passed with stages
in 15 minutes and 25 seconds
......@@ -16,6 +16,18 @@ variables:
UBUNTU_IMAGE: "elsa/ubuntu:18.04"
GPU_ONLY_IMAGE: "elsa/cuda:10.0"
### Cache setup ###
# Caches should only be used for caching
# between pipelines not jobs, artifacts should be prefered for that
# Therefore, we cache the submodules, so that they don't have to be
# downloaded so frequently
cache:
key: "$CI_COMMIT_REF_SLUG"
paths:
- thirdparty/
- .git/modules
### job templates ###
......@@ -58,13 +70,13 @@ variables:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
paths:
- build/
expire_in: 30 mins
expire_in: 60 mins
.buildjob_script_clang:
script: &buildjob_clang
- mkdir -p build
- cd build
- cmake .. -GNinja -DELSA_BENCHMARKS=ON -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
- cmake .. -GNinja -DELSA_BENCHMARKS=ON -DELSA_CREATE_JUNIT_REPORTS=ON -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
- ninja
- ninja build-benchmarks
- ninja build-examples
......@@ -73,7 +85,7 @@ variables:
script: &buildjob_normal
- mkdir -p build
- cd build
- cmake .. -GNinja -DELSA_BENCHMARKS=ON
- cmake .. -GNinja -DELSA_BENCHMARKS=ON -DELSA_CREATE_JUNIT_REPORTS=ON
- ninja
- ninja build-benchmarks
- ninja build-examples
......@@ -82,9 +94,12 @@ variables:
<<: *run_always
script:
- cd build
- cmake -GNinja -DELSA_CREATE_JUNIT_REPORTS=ON ..
- ninja tests
artifacts:
name: "$CI_COMMIT_REF_SLUG-$CI_JOB_NAME"
paths:
- build/
expire_in: 60 mins
reports:
junit: "build/test_reports/test_*.xml"
......@@ -216,22 +231,11 @@ test-cuda:
- cuda
test-gpu-only:
<<: *run_always
<<: *test_job_artifact
stage: test
image: $GPU_ONLY_IMAGE
dependencies:
- build-gpu-only
script:
- cd build
- cmake -GNinja -DELSA_CREATE_JUNIT_REPORTS=ON ..
- ninja tests
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
paths:
- build/
expire_in: 30 mins
reports:
junit: "build/test_reports/test_*.xml"
tags:
- linux
- elsa
......@@ -259,6 +263,7 @@ asan-ubsan:
<<: *run_on_merge_requests
stage: sanitizer
image: $CUDA_IMAGE
dependencies: []
script:
- mkdir -p build
- cd build
......@@ -278,18 +283,24 @@ test-coverage:
<<: *run_on_merge_requests
stage: coverage
image: $CUDA_IMAGE
dependencies: []
dependencies: []
script:
- mkdir -p build
- cd build
- cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DELSA_COVERAGE=ON ../
- ninja tests
- ninja test_coverage
cache:
key: "$CI_COMMIT_REF_SLUG-$CI_JOB_NAME"
paths:
- thirdparty/
- .git/modules
- build/
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-coverage"
paths:
- build/test_coverage/
expire_in: 30 mins
expire_in: 60 mins
tags:
- linux
- elsa
......
......@@ -51,6 +51,13 @@ include(SetDefaultBuildType)
# ------------ dependencies ------------
# ------------
# Setup list of submodules for elsa
list(APPEND ELSA_SUBMODULES Catch2 eigen3 pybind11 spdlog)
if(ELSA_CUDA_VECTOR)
list(APPEND ELSA_SUBMODULES quickvec)
endif(ELSA_CUDA_VECTOR)
# only add the dependencies if elsa is stand-alone
if(ELSA_MASTER_PROJECT)
......
# update/init git submodules if necessary
# idea from https://cliutils.gitlab.io/modern-cmake/chapters/projects/submodule.html
# try to perform the git submodule update
find_package(Git QUIET)
if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git")
if(GIT_SUBMODULE) # do this only if the GIT_SUBMODULE option is enabled
message(STATUS "Git submodule update")
execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE GIT_SUBMODULE_RESULT)
# Return early if git is not found, or submodule flag is not set, or if this is not a git project
if(NOT GIT_FOUND OR NOT GIT_SUBMODULE OR NOT EXISTS "${PROJECT_SOURCE_DIR}/.git")
return()
endif()
# Loop over all submodules
foreach(module ${ELSA_SUBMODULES})
set(SUBMODULE_GIT_PATH "${PROJECT_SOURCE_DIR}/.git/modules/thirdparty/${module}")
set(SUBMODULE_PATH "${PROJECT_SOURCE_DIR}/thirdparty/${module}")
set(GIT_SUBMODULE_RESULT "0")
if(NOT EXISTS ${SUBMODULE_GIT_PATH})
message(STATUS "Init submodule ${module}")
execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --force thirdparty/${module}
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
RESULT_VARIABLE GIT_SUBMODULE_RESULT
ERROR_QUIET)
# If init for some reason didn't work, we deinit and init again
# This seems to resolve some problems
# This usually happens in the pipeline, if we copy over the thirdparty stuff, but not the .git/modules
# folder
if(NOT GIT_SUBMODULE_RESULT EQUAL "0")
message(WARNING "git submodule update --init failed with ${GIT_SUBMODULE_RESULT}")
message(STATUS "Init failed, trying to deinit it and init again")
# reset error
set(GIT_SUBMODULE_RESULT "0")
# Deinit it
execute_process(COMMAND ${GIT_EXECUTABLE} submodule deinit --force thirdparty/${module}
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
OUTPUT_QUIET ERROR_QUIET)
# And reinit and update it
execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init thirdparty/${module}
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
RESULT_VARIABLE GIT_SUBMODULE_RESULT)
if(NOT GIT_SUBMODULE_RESULT EQUAL "0")
message(WARNING "Init submodule ${module} failed with ${GIT_SUBMODULE_RESULT}")
else()
message(STATUS "Reinit worked")
endif()
endif()
endif()
endif()
if(NOT EXISTS "${SUBMODULE_PATH}/CMakeLists.txt")
message(STATUS "Update submodule ${module}")
execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --force thirdparty/${module}
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
RESULT_VARIABLE GIT_SUBMODULE_RESULT)
endif()
# Check return value for error
if(NOT GIT_SUBMODULE_RESULT EQUAL "0")
message(WARNING "Updating submodule ${module} failed with ${GIT_SUBMODULE_RESULT}")
endif()
# If we still have no CMakeLists file something is wrong
if(NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/${module}/CMakeLists.txt")
message(FATAL_ERROR "The git submodule ${module} is not present! Either GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
endif()
endforeach()
# check the results
if (NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/Catch2/CMakeLists.txt" OR
NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/eigen3/CMakeLists.txt" OR
NOT EXISTS "${PROJECT_SOURCE_DIR}/thirdparty/spdlog/CMakeLists.txt")
message(FATAL_ERROR "The git submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.")
endif()
#!/bin/bash
exit_flag=false
target_branch="master"
# Retrieve list of cpp-files that were changed in source branch with respect to master (target branch)
filelist=($(git diff origin/${target_branch} --name-only | grep ".cpp"))
if [[ "${#filelist[@]}" -eq "0" ]]; then
echo "==> No cpp files found"
echo "==> clang-tidy has nothing to do, stop early"
exit 0
else
echo "==> Found ${#filelist[@]} cpp files"
echo "==> Let's start our clang-tidy check"
fi
# for compilation database
mkdir -p build
......@@ -8,7 +23,6 @@ cd build
cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DELSA_CUDA_VECTOR=ON -DELSA_BENCHMARKS=ON
cd ..
target_branch="master"
echo
echo "clang-tidy checking changed files compared to target branch ${target_branch}"
......@@ -23,8 +37,6 @@ function checkCPP(){
clang-tidy-8 --version
echo
# Retrieve list of files that were changed in source branch with respect to master (target branch)
filelist=`git diff origin/${target_branch} --name-only`
filesWithErrors=()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment