Commit fbb361be authored by David Frank's avatar David Frank Committed by Tobias Lasser
Browse files

Fully modularized Docker images, updated CI pipeline accordingly

Build image hierarchy. Using install scripts in the scripts/ folder, buildDockerContainers.sh also runs tests on the containers.
Added an image for static analysis and documentation builds. Added hidden builds in .gitlab-ci.yml for local testing, which build and run tests.
parent 7f5e4a50
Pipeline #221164 passed with stages
in 29 minutes and 34 seconds
...@@ -6,13 +6,22 @@ stages: ...@@ -6,13 +6,22 @@ stages:
- sanitizer - sanitizer
- docs - docs
### variables ###
variables:
ANALYSIS_IMAGE: "elsa/static-analysis:8"
GCC_IMAGE: "elsa/gcc:9"
CLANG_IMAGE: "elsa/clang:9"
CUDA_IMAGE: "elsa/cuda:10.2"
UBUNTU_IMAGE: "elsa/ubuntu:18.04"
### job templates ### ### job templates ###
.job_template: &nightly_job .job_template: &nightly_job
only: only:
- schedules - schedules
.job_template: &build_job .job_template: &build_job
stage: compile stage: compile
cache: cache:
...@@ -37,20 +46,20 @@ stages: ...@@ -37,20 +46,20 @@ stages:
- mkdir -p build - mkdir -p build
- cd build - cd build
- cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi" - cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
- ninja -j 8 - ninja
.buildjob_script_normal: .buildjob_script_normal:
script: &buildjob_normal script: &buildjob_normal
- mkdir -p build - mkdir -p build
- cd build - cd build
- cmake .. -GNinja - cmake .. -GNinja
- ninja -j 8 - ninja
.job_template: &test_job_artifact .job_template: &test_job_artifact
script: script:
- cd build - cd build
- cmake -GNinja -DELSA_CREATE_JUNIT_REPORTS=ON .. - cmake -GNinja -DELSA_CREATE_JUNIT_REPORTS=ON ..
- ninja tests -j 8 - ninja tests
artifacts: artifacts:
reports: reports:
junit: "build/test_reports/test_*.xml" junit: "build/test_reports/test_*.xml"
...@@ -60,7 +69,7 @@ stages: ...@@ -60,7 +69,7 @@ stages:
clang-format: clang-format:
stage: static-test stage: static-test
image: elsa/clang:8 image: $ANALYSIS_IMAGE
script: script:
./tools/ci_scripts/clang-format-test.sh ./tools/ci_scripts/clang-format-test.sh
tags: tags:
...@@ -70,7 +79,7 @@ clang-format: ...@@ -70,7 +79,7 @@ clang-format:
clang-tidy: clang-tidy:
stage: static-test stage: static-test
image: elsa/clang-tidy:8 image: $ANALYSIS_IMAGE
script: script:
./tools/ci_scripts/clang-tidy.sh ./tools/ci_scripts/clang-tidy.sh
tags: tags:
...@@ -83,33 +92,33 @@ clang-tidy: ...@@ -83,33 +92,33 @@ clang-tidy:
build-ubuntu: build-ubuntu:
<<: *build_job_artifact <<: *build_job_artifact
image: elsa/ubuntu:18.04 image: $UBUNTU_IMAGE
script: *buildjob_normal script: *buildjob_normal
tags: tags:
- linux - linux
- elsa - elsa
build-gcc9: build-gcc:
<<: *build_job_artifact <<: *build_job_artifact
image: elsa/gcc:9 image: $GCC_IMAGE
script: *buildjob_normal script: *buildjob_normal
tags: tags:
- linux - linux
- elsa - elsa
- gcc - gcc
build-clang8: build-clang:
<<: *build_job_artifact <<: *build_job_artifact
image: elsa/clang:8 image: $CLANG_IMAGE
script: *buildjob_clang script: *buildjob_clang
tags: tags:
- linux - linux
- elsa - elsa
- clang - clang
build-cuda92: build-cuda:
<<: *build_job_artifact <<: *build_job_artifact
image: elsa/cuda:9.2 image: $CUDA_IMAGE
script: *buildjob_normal script: *buildjob_normal
tags: tags:
- linux - linux
...@@ -123,41 +132,41 @@ build-cuda92: ...@@ -123,41 +132,41 @@ build-cuda92:
test-ubuntu: test-ubuntu:
<<: *test_job_artifact <<: *test_job_artifact
stage: test stage: test
image: elsa/ubuntu:18.04 image: $UBUNTU_IMAGE
dependencies: dependencies:
- build-ubuntu - build-ubuntu
tags: tags:
- linux - linux
- elsa - elsa
test-gcc9: test-gcc:
<<: *test_job_artifact <<: *test_job_artifact
stage: test stage: test
image: elsa/gcc:9 image: $GCC_IMAGE
dependencies: dependencies:
- build-gcc9 - build-gcc
tags: tags:
- linux - linux
- elsa - elsa
- gcc - gcc
test-clang8: test-clang:
<<: *test_job_artifact <<: *test_job_artifact
stage: test stage: test
image: elsa/clang:8 image: $CLANG_IMAGE
dependencies: dependencies:
- build-clang8 - build-clang
tags: tags:
- linux - linux
- elsa - elsa
- clang - clang
test-cuda92: test-cuda:
<<: *test_job_artifact <<: *test_job_artifact
stage: test stage: test
image: elsa/cuda:9.2 image: $CUDA_IMAGE
dependencies: dependencies:
- build-cuda92 - build-cuda
tags: tags:
- linux - linux
- elsa - elsa
...@@ -170,14 +179,14 @@ test-cuda92: ...@@ -170,14 +179,14 @@ test-cuda92:
test-coverage: test-coverage:
<<: *nightly_job <<: *nightly_job
stage: coverage stage: coverage
image: elsa/cuda:9.2 image: $CUDA_IMAGE
dependencies: [] dependencies: []
script: script:
- mkdir -p build - mkdir -p build
- cd build - cd build
- cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DELSA_COVERAGE=ON ../ - cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DELSA_COVERAGE=ON ../
- ninja tests -j 8 - ninja tests
- ninja test_coverage -j 8 - ninja test_coverage
artifacts: artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-coverage" name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-coverage"
paths: paths:
...@@ -189,13 +198,12 @@ test-coverage: ...@@ -189,13 +198,12 @@ test-coverage:
- gcc - gcc
- cuda - cuda
### Sanitizers ### ### Sanitizers ###
#thread-sanitizer: #thread-sanitizer:
# <<: *nightly_job # <<: *nightly_job
# stage: sanitizer # stage: sanitizer
# image: elsa/cuda:9.2 # image: $CUDA_IMAGE
# script: # script:
# - mkdir -p build # - mkdir -p build
# - cd build # - cd build
...@@ -213,7 +221,7 @@ test-coverage: ...@@ -213,7 +221,7 @@ test-coverage:
#address-sanitizer: #address-sanitizer:
# <<: *nightly_job # <<: *nightly_job
# stage: sanitizer # stage: sanitizer
# image: elsa/cuda:9.2 # image: $CUDA_IMAGE
# script: # script:
# - mkdir -p build # - mkdir -p build
# - cd build # - cd build
...@@ -247,3 +255,35 @@ deploy-docs: ...@@ -247,3 +255,35 @@ deploy-docs:
tags: tags:
- elsa-docs-deploy - elsa-docs-deploy
### build, test and deploy docker images ###
.deploy-docker:
image: elsa/base:latest
<<: *nightly_job
stage: docker
script:
- cd tools/docker
- ./buildDockerContainers.sh
### Test jobs ###
# These jobs, are for local build and test all in one job, should not be run with pipeline
.all-clang:
image: $UBUNTU_IMAGE
script:
- mkdir -p build
- cd build
- cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
- ninja
- ninja tests
.all-gcc:
image: $GCC_IMAGE
script:
- mkdir -p build
- cd build
- cmake .. -GNinja
- ninja
- ninja tests
...@@ -164,3 +164,14 @@ if(ELSA_INSTALL) ...@@ -164,3 +164,14 @@ if(ELSA_INSTALL)
# ...but we rely on the paths in the installed version. sigh. commented out for now. # ...but we rely on the paths in the installed version. sigh. commented out for now.
#export(PACKAGE elsa) #export(PACKAGE elsa)
endif(ELSA_INSTALL) endif(ELSA_INSTALL)
##
# Add uninstall target
# Remove all headers, library and CMake files, which where copied to the install location
add_custom_target(uninstall
COMMAND ${CMAKE_COMMAND} -P "${PROJECT_SOURCE_DIR}/cmake/Uninstall.cmake"
COMMENT "Uninstalling elsa (Removing all installed files)"
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt"
)
\ No newline at end of file
##
# Uninstall routine
# Removes all files in the <build_folder>/install_manifest.txt
# Uses cmake -E remove <file> to remove file in a cross platform way (hopefully)
set(MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt")
if(NOT EXISTS ${MANIFEST})
message(FATAL_ERROR "Cannot find install manifest: '${MANIFEST}'")
endif()
file(STRINGS ${MANIFEST} files)
foreach(file ${files})
if(EXISTS ${file})
message(STATUS "Removing file: '${file}'")
exec_program(
${CMAKE_COMMAND} ARGS "-E remove ${file}"
OUTPUT_VARIABLE stdout
RETURN_VALUE result
)
if(NOT "${result}" STREQUAL 0)
message(FATAL_ERROR "Failed to remove file: '${file}'.")
endif()
else()
MESSAGE(STATUS "File '${file}' does not exist.")
endif()
endforeach(file)
FROM ubuntu:18.04 as base
MAINTAINER Tobias Lasser <lasser@in.tum.de>
MAINTAINER David Frank <dfrank@in.tum.de>
LABEL Description="Base image for elsa build system (git, wget, ninja)"
ADD scripts/ /tmp/scripts/
ADD tests/ /tmp/tests/
ENV DEBIAN_FRONTEND=noninteractive
RUN /tmp/scripts/install_base.sh
CMD bash
FROM elsa/base:latest as builder
MAINTAINER Tobias Lasser <lasser@in.tum.de>
MAINTAINER David Frank <dfrank@in.tum.de>
LABEL Description="CMake-Base image for elsa build system"
# install CMAKE
ARG CMAKE_VERSION=3.16
ARG CMAKE_VERSION_PATCH=4
ARG CMAKE_DIR=cmake-${CMAKE_VERSION}.${CMAKE_VERSION_PATCH}
RUN bash /tmp/scripts/install_cmake.sh "${CMAKE_VERSION}" "${CMAKE_DIR}"
FROM elsa/base:latest
COPY --from=builder /tmp/cmake-install /usr/local
CMD bash
FROM clang:6
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang6 on Debian with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget ca-certificates pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& export CC=clang \
&& export CXX=clang++ \
&& export CXXFLAGS="-std=c++17 -stdlib=libc++" \
&& export LDFLAGS="-lc++ -lc++abi" \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# fix libc++ paths
RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
&& ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
FROM clang:7
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang7 on Debian with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget ca-certificates pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& export CC=clang \
&& export CXX=clang++ \
&& export CXXFLAGS="-std=c++17 -stdlib=libc++" \
&& export LDFLAGS="-lc++ -lc++abi" \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# fix libc++ paths
RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
&& ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
FROM clang:8
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang8 on Debian with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget ca-certificates pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& export CC=clang \
&& export CXX=clang++ \
&& export CXXFLAGS="-std=c++17 -stdlib=libc++" \
&& export LDFLAGS="-lc++ -lc++abi" \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# fix libc++ paths
RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
&& ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
FROM clang:9
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang9 on Debian with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget ca-certificates pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& export CC=clang \
&& export CXX=clang++ \
&& export CXXFLAGS="-std=c++17 -stdlib=libc++" \
&& export LDFLAGS="-lc++ -lc++abi" \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# fix libc++ paths
RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
&& ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
### Clone step of LLVM/clang ###
FROM elsa/base-cmake:latest as clone
MAINTAINER Tobias Lasser <lasser@in.tum.de>
MAINTAINER David Frank <dfrank@in.tum.de>
LABEL Description="Clang image based on Ubuntu"
# Provide an argument to choose the compiler version from the command line
ARG COMPILER_VERSION
ARG CLANG_TAG=llvmorg-${COMPILER_VERSION}
# Clone llvm-project with the specific tag
RUN git clone --depth 1 --branch ${CLANG_TAG} https://github.com/llvm/llvm-project.git /tmp/llvm_project/
### Configuration step of LLVM/clang ###
FROM elsa/base-cmake:latest as configure
COPY --from=clone /tmp/llvm_project/ /tmp/llvm_project/
RUN mkdir -p /tmp/llvm_build/ && mkdir -p /tmp/llvm_install/ \
&& cmake -S /tmp/llvm_project/llvm/ -B /tmp/llvm_build/ -GNinja -DLLVM_ENABLE_PROJECTS="clang;libcxx;libcxxabi;openmp" \
-DCMAKE_INSTALL_PREFIX=/tmp/llvm_install/ -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON
### Build LLVM/clang
FROM elsa/base-cmake:latest as build
COPY --from=configure /tmp/llvm_project/ /tmp/llvm_project/
COPY --from=configure /tmp/llvm_build/ /tmp/llvm_build/
# install a reduced version of clang (due to LLVM_INSTALL_TOOLCHAIN_ONLY=ON)