Commit fbb361be authored by David Frank's avatar David Frank Committed by Tobias Lasser
Browse files

Fully modularized Docker images, updated CI pipeline accordingly

Build image hierarchy. Using install scripts in the scripts/ folder, buildDockerContainers.sh also runs tests on the containers.
Added an image for static analysis and documentation builds. Added hidden builds in .gitlab-ci.yml for local testing, which build and run tests.
parent 7f5e4a50
Pipeline #221164 passed with stages
in 29 minutes and 34 seconds
......@@ -6,13 +6,22 @@ stages:
- sanitizer
- docs
### variables ###
variables:
ANALYSIS_IMAGE: "elsa/static-analysis:8"
GCC_IMAGE: "elsa/gcc:9"
CLANG_IMAGE: "elsa/clang:9"
CUDA_IMAGE: "elsa/cuda:10.2"
UBUNTU_IMAGE: "elsa/ubuntu:18.04"
### job templates ###
.job_template: &nightly_job
only:
- schedules
.job_template: &build_job
stage: compile
cache:
......@@ -37,20 +46,20 @@ stages:
- mkdir -p build
- cd build
- cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
- ninja -j 8
- ninja
.buildjob_script_normal:
script: &buildjob_normal
- mkdir -p build
- cd build
- cmake .. -GNinja
- ninja -j 8
- ninja
.job_template: &test_job_artifact
script:
- cd build
- cmake -GNinja -DELSA_CREATE_JUNIT_REPORTS=ON ..
- ninja tests -j 8
- ninja tests
artifacts:
reports:
junit: "build/test_reports/test_*.xml"
......@@ -60,7 +69,7 @@ stages:
clang-format:
stage: static-test
image: elsa/clang:8
image: $ANALYSIS_IMAGE
script:
./tools/ci_scripts/clang-format-test.sh
tags:
......@@ -70,7 +79,7 @@ clang-format:
clang-tidy:
stage: static-test
image: elsa/clang-tidy:8
image: $ANALYSIS_IMAGE
script:
./tools/ci_scripts/clang-tidy.sh
tags:
......@@ -83,33 +92,33 @@ clang-tidy:
build-ubuntu:
<<: *build_job_artifact
image: elsa/ubuntu:18.04
image: $UBUNTU_IMAGE
script: *buildjob_normal
tags:
- linux
- elsa
build-gcc9:
build-gcc:
<<: *build_job_artifact
image: elsa/gcc:9
image: $GCC_IMAGE
script: *buildjob_normal
tags:
- linux
- elsa
- gcc
build-clang8:
build-clang:
<<: *build_job_artifact
image: elsa/clang:8
image: $CLANG_IMAGE
script: *buildjob_clang
tags:
- linux
- elsa
- clang
build-cuda92:
build-cuda:
<<: *build_job_artifact
image: elsa/cuda:9.2
image: $CUDA_IMAGE
script: *buildjob_normal
tags:
- linux
......@@ -123,41 +132,41 @@ build-cuda92:
test-ubuntu:
<<: *test_job_artifact
stage: test
image: elsa/ubuntu:18.04
image: $UBUNTU_IMAGE
dependencies:
- build-ubuntu
tags:
- linux
- elsa
test-gcc9:
test-gcc:
<<: *test_job_artifact
stage: test
image: elsa/gcc:9
image: $GCC_IMAGE
dependencies:
- build-gcc9
- build-gcc
tags:
- linux
- elsa
- gcc
test-clang8:
test-clang:
<<: *test_job_artifact
stage: test
image: elsa/clang:8
image: $CLANG_IMAGE
dependencies:
- build-clang8
- build-clang
tags:
- linux
- elsa
- clang
test-cuda92:
test-cuda:
<<: *test_job_artifact
stage: test
image: elsa/cuda:9.2
image: $CUDA_IMAGE
dependencies:
- build-cuda92
- build-cuda
tags:
- linux
- elsa
......@@ -170,14 +179,14 @@ test-cuda92:
test-coverage:
<<: *nightly_job
stage: coverage
image: elsa/cuda:9.2
image: $CUDA_IMAGE
dependencies: []
script:
- mkdir -p build
- cd build
- cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DELSA_COVERAGE=ON ../
- ninja tests -j 8
- ninja test_coverage -j 8
- ninja tests
- ninja test_coverage
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-coverage"
paths:
......@@ -189,13 +198,12 @@ test-coverage:
- gcc
- cuda
### Sanitizers ###
#thread-sanitizer:
# <<: *nightly_job
# stage: sanitizer
# image: elsa/cuda:9.2
# image: $CUDA_IMAGE
# script:
# - mkdir -p build
# - cd build
......@@ -213,7 +221,7 @@ test-coverage:
#address-sanitizer:
# <<: *nightly_job
# stage: sanitizer
# image: elsa/cuda:9.2
# image: $CUDA_IMAGE
# script:
# - mkdir -p build
# - cd build
......@@ -247,3 +255,35 @@ deploy-docs:
tags:
- elsa-docs-deploy
### build, test and deploy docker images ###
.deploy-docker:
image: elsa/base:latest
<<: *nightly_job
stage: docker
script:
- cd tools/docker
- ./buildDockerContainers.sh
### Test jobs ###
# These jobs, are for local build and test all in one job, should not be run with pipeline
.all-clang:
image: $UBUNTU_IMAGE
script:
- mkdir -p build
- cd build
- cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
- ninja
- ninja tests
.all-gcc:
image: $GCC_IMAGE
script:
- mkdir -p build
- cd build
- cmake .. -GNinja
- ninja
- ninja tests
......@@ -164,3 +164,14 @@ if(ELSA_INSTALL)
# ...but we rely on the paths in the installed version. sigh. commented out for now.
#export(PACKAGE elsa)
endif(ELSA_INSTALL)
##
# Add uninstall target
# Remove all headers, library and CMake files, which where copied to the install location
add_custom_target(uninstall
COMMAND ${CMAKE_COMMAND} -P "${PROJECT_SOURCE_DIR}/cmake/Uninstall.cmake"
COMMENT "Uninstalling elsa (Removing all installed files)"
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt"
)
\ No newline at end of file
##
# Uninstall routine
# Removes all files in the <build_folder>/install_manifest.txt
# Uses cmake -E remove <file> to remove file in a cross platform way (hopefully)
set(MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt")
if(NOT EXISTS ${MANIFEST})
message(FATAL_ERROR "Cannot find install manifest: '${MANIFEST}'")
endif()
file(STRINGS ${MANIFEST} files)
foreach(file ${files})
if(EXISTS ${file})
message(STATUS "Removing file: '${file}'")
exec_program(
${CMAKE_COMMAND} ARGS "-E remove ${file}"
OUTPUT_VARIABLE stdout
RETURN_VALUE result
)
if(NOT "${result}" STREQUAL 0)
message(FATAL_ERROR "Failed to remove file: '${file}'.")
endif()
else()
MESSAGE(STATUS "File '${file}' does not exist.")
endif()
endforeach(file)
FROM ubuntu:18.04 as base
MAINTAINER Tobias Lasser <lasser@in.tum.de>
MAINTAINER David Frank <dfrank@in.tum.de>
LABEL Description="Base image for elsa build system (git, wget, ninja)"
ADD scripts/ /tmp/scripts/
ADD tests/ /tmp/tests/
ENV DEBIAN_FRONTEND=noninteractive
RUN /tmp/scripts/install_base.sh
CMD bash
FROM elsa/base:latest as builder
MAINTAINER Tobias Lasser <lasser@in.tum.de>
MAINTAINER David Frank <dfrank@in.tum.de>
LABEL Description="CMake-Base image for elsa build system"
# install CMAKE
ARG CMAKE_VERSION=3.16
ARG CMAKE_VERSION_PATCH=4
ARG CMAKE_DIR=cmake-${CMAKE_VERSION}.${CMAKE_VERSION_PATCH}
RUN bash /tmp/scripts/install_cmake.sh "${CMAKE_VERSION}" "${CMAKE_DIR}"
FROM elsa/base:latest
COPY --from=builder /tmp/cmake-install /usr/local
CMD bash
FROM clang:6
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang6 on Debian with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget ca-certificates pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& export CC=clang \
&& export CXX=clang++ \
&& export CXXFLAGS="-std=c++17 -stdlib=libc++" \
&& export LDFLAGS="-lc++ -lc++abi" \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# fix libc++ paths
RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
&& ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
FROM clang:7
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang7 on Debian with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget ca-certificates pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& export CC=clang \
&& export CXX=clang++ \
&& export CXXFLAGS="-std=c++17 -stdlib=libc++" \
&& export LDFLAGS="-lc++ -lc++abi" \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# fix libc++ paths
RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
&& ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
FROM clang:8
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang8 on Debian with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget ca-certificates pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& export CC=clang \
&& export CXX=clang++ \
&& export CXXFLAGS="-std=c++17 -stdlib=libc++" \
&& export LDFLAGS="-lc++ -lc++abi" \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# fix libc++ paths
RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
&& ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
FROM clang:9
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang9 on Debian with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget ca-certificates pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& export CC=clang \
&& export CXX=clang++ \
&& export CXXFLAGS="-std=c++17 -stdlib=libc++" \
&& export LDFLAGS="-lc++ -lc++abi" \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# fix libc++ paths
RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
&& ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
### Clone step of LLVM/clang ###
FROM elsa/base-cmake:latest as clone
MAINTAINER Tobias Lasser <lasser@in.tum.de>
MAINTAINER David Frank <dfrank@in.tum.de>
LABEL Description="Clang image based on Ubuntu"
# Provide an argument to choose the compiler version from the command line
ARG COMPILER_VERSION
ARG CLANG_TAG=llvmorg-${COMPILER_VERSION}
# Clone llvm-project with the specific tag
RUN git clone --depth 1 --branch ${CLANG_TAG} https://github.com/llvm/llvm-project.git /tmp/llvm_project/
### Configuration step of LLVM/clang ###
FROM elsa/base-cmake:latest as configure
COPY --from=clone /tmp/llvm_project/ /tmp/llvm_project/
RUN mkdir -p /tmp/llvm_build/ && mkdir -p /tmp/llvm_install/ \
&& cmake -S /tmp/llvm_project/llvm/ -B /tmp/llvm_build/ -GNinja -DLLVM_ENABLE_PROJECTS="clang;libcxx;libcxxabi;openmp" \
-DCMAKE_INSTALL_PREFIX=/tmp/llvm_install/ -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON
### Build LLVM/clang
FROM elsa/base-cmake:latest as build
COPY --from=configure /tmp/llvm_project/ /tmp/llvm_project/
COPY --from=configure /tmp/llvm_build/ /tmp/llvm_build/
# install a reduced version of clang (due to LLVM_INSTALL_TOOLCHAIN_ONLY=ON)
WORKDIR /tmp/llvm_build/
RUN ninja install
WORKDIR /
### Set base for clang ###
FROM elsa/base-cmake:latest as base
COPY --from=build /tmp/llvm_install/ /usr/local/
# Set environment variables
ENV CC="/usr/local/bin/clang"
ENV CXX="/usr/local/bin/clang++"
ENV PATH="$PATH:/usr/local/bin/"
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/:/usr/local/lib/"
### Download, configure, build and install Intel DNNL
FROM base as build-dnnl
# Intel DNNL Versions
ARG DNNL_VERSION=1.1.1
# Add test scripts
ADD scripts/ /tmp/scripts/
# Run install script for DNNL
RUN ./tmp/scripts/install_inteldnnl.sh ${DNNL_VERSION}
### Final stage ###
FROM base
# Copy Intel DNNL over
COPY --from=build-dnnl /tmp/dnnl_install /usr/local/
# setup conan home directory, to be in the cache of our CI pipeline
ENV CONAN_USER_HOME=/cache
# setup conan for clang
RUN conan profile new default --detect \
&& conan profile update settings.compiler.libcxx=libc++ default \
&& conan profile update settings.compiler.cppstd=17 default \
&& conan profile update env.CC=clang default \
&& conan profile update env.CXX=clang++ default \
&& conan profile update env.CONAN_CMAKE_GENERATOR=Ninja default \
&& conan profile show default
# Add test scripts
ADD tests/ /tmp/tests/
CMD bash
FROM nvidia/cuda:9.2-devel-ubuntu18.04
MAINTAINER Tobias Lasser <lasser@in.tum.de>
LABEL Description="clang-tidy 8 (with cuda9.2) develop with cmake/git"
# install wget
RUN apt-get -qq update \
&& apt-get install -qqy wget git pkg-config
# install ninja
RUN apt-get install -qqy ninja-build
# install clang-tidy-8
RUN apt-get install -qqy clang-tidy-8
# install OpenSSL for CMake
RUN apt-get install -qqy libssl-dev
# install CMAKE
ARG cmake_version=3.16.1
ARG cmake_version_short=3.16
ARG cmake_dir=cmake-${cmake_version}
RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
&& tar -xvzf ${cmake_dir}.tar.gz \
&& rm ${cmake_dir}.tar.gz \
&& cd ${cmake_dir} \
&& ./configure --parallel=28 \
&& make -j28 && make install \
&& cd .. && rm -rf ${cmake_dir} && ldconfig
# install Intel MKL-DNN
ARG mkldnn_version=1.1.1
ARG mkldnn_folder=v${mkldnn_version}
RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
&& tar -xvzf ${mkldnn_folder}.tar.gz \
&& rm ${mkldnn_folder}.tar.gz \
&& cd mkl-dnn-${mkldnn_version} \
&& mkdir build && cd build \
&& cmake .. \
&& make -j28 && make install \
&& cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
CMD bash
# Provide an argument to choose the compiler version from the command line
ARG CUDA_VERSION=10.2