Fully modularized Docker images, updated CI pipeline accordingly

Build image hierarchy. Using install scripts in the scripts/ folder, buildDockerContainers.sh also runs tests on the containers.
Added an image for static analysis and documentation builds. Added hidden builds in .gitlab-ci.yml for local testing, which build and run tests.

.gitlab-ci.yml

@@ -6,13 +6,22 @@ stages:
 - sanitizer
 - docs
 
+### variables ###
+
+variables:
+  ANALYSIS_IMAGE: "elsa/static-analysis:8"
+  GCC_IMAGE: "elsa/gcc:9"
+  CLANG_IMAGE: "elsa/clang:9"
+  CUDA_IMAGE: "elsa/cuda:10.2"
+  UBUNTU_IMAGE: "elsa/ubuntu:18.04"
+  
+
 ### job templates ###
 
 .job_template: &nightly_job
   only:
     - schedules
 
-
 .job_template: &build_job
   stage: compile
   cache: 
@@ -37,20 +46,20 @@ stages:
     - mkdir -p build
     - cd build
     - cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
-    - ninja -j 8 
+    - ninja 
 
 .buildjob_script_normal:
   script: &buildjob_normal
     - mkdir -p build
     - cd build
     - cmake .. -GNinja
-    - ninja -j 8
+    - ninja
 
 .job_template: &test_job_artifact
   script:
     - cd build
     - cmake -GNinja -DELSA_CREATE_JUNIT_REPORTS=ON ..
-    - ninja tests -j 8
+    - ninja tests
   artifacts:
     reports:
       junit: "build/test_reports/test_*.xml"
@@ -60,7 +69,7 @@ stages:
 
 clang-format:
   stage: static-test
-  image: elsa/clang:8
+  image: $ANALYSIS_IMAGE
   script:
     ./tools/ci_scripts/clang-format-test.sh
   tags:
@@ -70,7 +79,7 @@ clang-format:
 
 clang-tidy:
   stage: static-test
-  image: elsa/clang-tidy:8
+  image: $ANALYSIS_IMAGE
   script:
     ./tools/ci_scripts/clang-tidy.sh
   tags:
@@ -83,33 +92,33 @@ clang-tidy:
 
 build-ubuntu:
   <<: *build_job_artifact
-  image: elsa/ubuntu:18.04
+  image: $UBUNTU_IMAGE
   script: *buildjob_normal
   tags:
     - linux
     - elsa
 
-build-gcc9:
+build-gcc:
   <<: *build_job_artifact
-  image: elsa/gcc:9
+  image: $GCC_IMAGE
   script: *buildjob_normal
   tags:
     - linux
     - elsa
     - gcc
     
-build-clang8:
+build-clang:
   <<: *build_job_artifact
-  image: elsa/clang:8
+  image: $CLANG_IMAGE
   script: *buildjob_clang
   tags:
     - linux
     - elsa
     - clang
 
-build-cuda92:
+build-cuda:
   <<: *build_job_artifact
-  image: elsa/cuda:9.2
+  image: $CUDA_IMAGE
   script: *buildjob_normal
   tags:
     - linux
@@ -123,41 +132,41 @@ build-cuda92:
 test-ubuntu:
   <<: *test_job_artifact
   stage: test
-  image: elsa/ubuntu:18.04
+  image: $UBUNTU_IMAGE
   dependencies: 
     - build-ubuntu
   tags:
     - linux
     - elsa
 
-test-gcc9:
+test-gcc:
   <<: *test_job_artifact
   stage: test
-  image: elsa/gcc:9
+  image: $GCC_IMAGE
   dependencies:
-    - build-gcc9
+    - build-gcc
   tags:
     - linux
     - elsa
     - gcc
     
-test-clang8:
+test-clang:
   <<: *test_job_artifact
   stage: test
-  image: elsa/clang:8
+  image: $CLANG_IMAGE
   dependencies:
-    - build-clang8
+    - build-clang
   tags:
     - linux
     - elsa
     - clang
 
-test-cuda92:
+test-cuda:
   <<: *test_job_artifact
   stage: test
-  image: elsa/cuda:9.2
+  image: $CUDA_IMAGE
   dependencies:
-    - build-cuda92
+    - build-cuda
   tags:
     - linux
     - elsa
@@ -170,14 +179,14 @@ test-cuda92:
 test-coverage:
   <<: *nightly_job
   stage: coverage
-  image: elsa/cuda:9.2
+  image: $CUDA_IMAGE
   dependencies: []
   script:
     - mkdir -p build
     - cd build
     - cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DELSA_COVERAGE=ON ../
-    - ninja tests -j 8
-    - ninja test_coverage -j 8
+    - ninja tests
+    - ninja test_coverage
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-coverage"
     paths:
@@ -189,13 +198,12 @@ test-coverage:
     - gcc
     - cuda
 
-
 ### Sanitizers ###
 
 #thread-sanitizer:
 #  <<: *nightly_job
 #  stage: sanitizer
-#  image: elsa/cuda:9.2
+#  image: $CUDA_IMAGE
 #  script:
 #    - mkdir -p build
 #    - cd build
@@ -213,7 +221,7 @@ test-coverage:
 #address-sanitizer:
 #  <<: *nightly_job
 #  stage: sanitizer
-#  image: elsa/cuda:9.2
+#  image: $CUDA_IMAGE
 #  script:
 #    - mkdir -p build
 #    - cd build
@@ -247,3 +255,35 @@ deploy-docs:
   tags:
     - elsa-docs-deploy
 
+
+### build, test and deploy docker images ###
+.deploy-docker:
+  image: elsa/base:latest
+  <<: *nightly_job
+  stage: docker
+  script:
+    - cd tools/docker
+    - ./buildDockerContainers.sh
+
+### Test jobs ###
+
+# These jobs, are for local build and test all in one job, should not be run with pipeline
+
+.all-clang:
+  image: $UBUNTU_IMAGE
+  script:
+    - mkdir -p build
+    - cd build
+    - cmake .. -GNinja -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
+    - ninja
+    - ninja tests
+
+.all-gcc:
+  image: $GCC_IMAGE
+  script:
+    - mkdir -p build
+    - cd build
+    - cmake .. -GNinja
+    - ninja
+    - ninja tests
+

CMakeLists.txt

@@ -164,3 +164,14 @@ if(ELSA_INSTALL)
     # ...but we rely on the paths in the installed version. sigh. commented out for now.
     #export(PACKAGE elsa)
 endif(ELSA_INSTALL)
+
+
+##
+# Add uninstall target
+# Remove all headers, library and CMake files, which where copied to the install location
+
+add_custom_target(uninstall
+    COMMAND ${CMAKE_COMMAND} -P "${PROJECT_SOURCE_DIR}/cmake/Uninstall.cmake"
+    COMMENT "Uninstalling elsa (Removing all installed files)"
+    DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt"
+    )
\ No newline at end of file

cmake/Uninstall.cmake

+##
+# Uninstall routine
+# Removes all files in the <build_folder>/install_manifest.txt
+# Uses cmake -E remove <file> to remove file in a cross platform way (hopefully)
+
+set(MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt")
+
+if(NOT EXISTS ${MANIFEST})
+    message(FATAL_ERROR "Cannot find install manifest: '${MANIFEST}'")
+endif()
+
+file(STRINGS ${MANIFEST} files)
+foreach(file ${files})
+    if(EXISTS ${file})
+        message(STATUS "Removing file: '${file}'")
+
+        exec_program(
+            ${CMAKE_COMMAND} ARGS "-E remove ${file}"
+            OUTPUT_VARIABLE stdout
+            RETURN_VALUE result
+        )
+        
+        if(NOT "${result}" STREQUAL 0)
+            message(FATAL_ERROR "Failed to remove file: '${file}'.")
+        endif()
+    else()
+        MESSAGE(STATUS "File '${file}' does not exist.")
+    endif()
+endforeach(file)

tools/docker/DocFileBase

+FROM ubuntu:18.04 as base
+MAINTAINER Tobias Lasser <lasser@in.tum.de>
+MAINTAINER David Frank <dfrank@in.tum.de>
+
+LABEL Description="Base image for elsa build system (git, wget, ninja)"
+
+ADD scripts/ /tmp/scripts/
+ADD tests/ /tmp/tests/
+
+ENV DEBIAN_FRONTEND=noninteractive 
+
+RUN /tmp/scripts/install_base.sh
+
+CMD bash

tools/docker/DocFileBaseCMake

+FROM elsa/base:latest as builder
+MAINTAINER Tobias Lasser <lasser@in.tum.de>
+MAINTAINER David Frank <dfrank@in.tum.de>
+
+LABEL Description="CMake-Base image for elsa build system"
+
+# install CMAKE
+ARG CMAKE_VERSION=3.16
+ARG CMAKE_VERSION_PATCH=4
+ARG CMAKE_DIR=cmake-${CMAKE_VERSION}.${CMAKE_VERSION_PATCH}
+
+RUN bash /tmp/scripts/install_cmake.sh "${CMAKE_VERSION}" "${CMAKE_DIR}"
+
+FROM elsa/base:latest
+COPY --from=builder /tmp/cmake-install /usr/local
+
+CMD bash

tools/docker/DocFileClang6

-FROM clang:6
-MAINTAINER Tobias Lasser <lasser@in.tum.de>
-LABEL Description="clang6 on Debian with cmake/git"
-
-# install wget
-RUN apt-get -qq update \
- && apt-get install -qqy wget ca-certificates pkg-config
-
-# install ninja
-RUN apt-get install -qqy ninja-build
-
-# install OpenSSL for CMake
-RUN apt-get install -qqy libssl-dev
-
-# install CMAKE
-ARG cmake_version=3.16.1
-ARG cmake_version_short=3.16
-ARG cmake_dir=cmake-${cmake_version}
-
-RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
- && tar -xvzf ${cmake_dir}.tar.gz \
- && rm ${cmake_dir}.tar.gz \
- && cd ${cmake_dir} \
- && export CC=clang \
- && export CXX=clang++ \
- && export CXXFLAGS="-std=c++17 -stdlib=libc++" \
- && export LDFLAGS="-lc++ -lc++abi" \
- && ./configure --parallel=28 \
- && make -j28 && make install \
- && cd .. && rm -rf ${cmake_dir} && ldconfig
-
-# fix libc++ paths
-RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
- && ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
-
-# install Intel MKL-DNN
-ARG mkldnn_version=1.1.1
-ARG mkldnn_folder=v${mkldnn_version}
-
-RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
- && tar -xvzf ${mkldnn_folder}.tar.gz \
- && rm ${mkldnn_folder}.tar.gz \
- && cd mkl-dnn-${mkldnn_version} \
- && mkdir build && cd build \
- && cmake .. \
- && make -j28 && make install \
- && cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
-
-CMD bash

tools/docker/DocFileClang7

-FROM clang:7
-MAINTAINER Tobias Lasser <lasser@in.tum.de>
-LABEL Description="clang7 on Debian with cmake/git"
-
-# install wget
-RUN apt-get -qq update \
- && apt-get install -qqy wget ca-certificates pkg-config
-
-# install ninja
-RUN apt-get install -qqy ninja-build
-
-# install OpenSSL for CMake
-RUN apt-get install -qqy libssl-dev
-
-# install CMAKE
-ARG cmake_version=3.16.1
-ARG cmake_version_short=3.16
-ARG cmake_dir=cmake-${cmake_version}
-
-RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
- && tar -xvzf ${cmake_dir}.tar.gz \
- && rm ${cmake_dir}.tar.gz \
- && cd ${cmake_dir} \
- && export CC=clang \
- && export CXX=clang++ \
- && export CXXFLAGS="-std=c++17 -stdlib=libc++" \
- && export LDFLAGS="-lc++ -lc++abi" \
- && ./configure --parallel=28 \
- && make -j28 && make install \
- && cd .. && rm -rf ${cmake_dir} && ldconfig
-
-# fix libc++ paths
-RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
- && ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
-
-# install Intel MKL-DNN
-ARG mkldnn_version=1.1.1
-ARG mkldnn_folder=v${mkldnn_version}
-
-RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
- && tar -xvzf ${mkldnn_folder}.tar.gz \
- && rm ${mkldnn_folder}.tar.gz \
- && cd mkl-dnn-${mkldnn_version} \
- && mkdir build && cd build \
- && cmake .. \
- && make -j28 && make install \
- && cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
-
-CMD bash

tools/docker/DocFileClang8

-FROM clang:8
-MAINTAINER Tobias Lasser <lasser@in.tum.de>
-LABEL Description="clang8 on Debian with cmake/git"
-
-# install wget
-RUN apt-get -qq update \
- && apt-get install -qqy wget ca-certificates pkg-config
-
-# install ninja
-RUN apt-get install -qqy ninja-build
-
-# install OpenSSL for CMake
-RUN apt-get install -qqy libssl-dev
-
-# install CMAKE
-ARG cmake_version=3.16.1
-ARG cmake_version_short=3.16
-ARG cmake_dir=cmake-${cmake_version}
-
-RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
- && tar -xvzf ${cmake_dir}.tar.gz \
- && rm ${cmake_dir}.tar.gz \
- && cd ${cmake_dir} \
- && export CC=clang \
- && export CXX=clang++ \
- && export CXXFLAGS="-std=c++17 -stdlib=libc++" \
- && export LDFLAGS="-lc++ -lc++abi" \
- && ./configure --parallel=28 \
- && make -j28 && make install \
- && cd .. && rm -rf ${cmake_dir} && ldconfig
-
-# fix libc++ paths
-RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
- && ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
-
-# install Intel MKL-DNN
-ARG mkldnn_version=1.1.1
-ARG mkldnn_folder=v${mkldnn_version}
-
-RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
- && tar -xvzf ${mkldnn_folder}.tar.gz \
- && rm ${mkldnn_folder}.tar.gz \
- && cd mkl-dnn-${mkldnn_version} \
- && mkdir build && cd build \
- && cmake .. \
- && make -j28 && make install \
- && cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
-
-CMD bash

tools/docker/DocFileClang9

-FROM clang:9
-MAINTAINER Tobias Lasser <lasser@in.tum.de>
-LABEL Description="clang9 on Debian with cmake/git"
-
-# install wget
-RUN apt-get -qq update \
- && apt-get install -qqy wget ca-certificates pkg-config
-
-# install ninja
-RUN apt-get install -qqy ninja-build
-
-# install OpenSSL for CMake
-RUN apt-get install -qqy libssl-dev
-
-# install CMAKE
-ARG cmake_version=3.16.1
-ARG cmake_version_short=3.16
-ARG cmake_dir=cmake-${cmake_version}
-
-RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
- && tar -xvzf ${cmake_dir}.tar.gz \
- && rm ${cmake_dir}.tar.gz \
- && cd ${cmake_dir} \
- && export CC=clang \
- && export CXX=clang++ \
- && export CXXFLAGS="-std=c++17 -stdlib=libc++" \
- && export LDFLAGS="-lc++ -lc++abi" \
- && ./configure --parallel=28 \
- && make -j28 && make install \
- && cd .. && rm -rf ${cmake_dir} && ldconfig
-
-# fix libc++ paths
-RUN ln -sf /usr/local/lib/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a \
- && ln -sf /usr/local/lib/libc++.so.1.0 /usr/lib/x86_64-linux-gnu/libc++.so.1.0
-
-# install Intel MKL-DNN
-ARG mkldnn_version=1.1.1
-ARG mkldnn_folder=v${mkldnn_version}
-
-RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
- && tar -xvzf ${mkldnn_folder}.tar.gz \
- && rm ${mkldnn_folder}.tar.gz \
- && cd mkl-dnn-${mkldnn_version} \
- && mkdir build && cd build \
- && cmake .. \
- && make -j28 && make install \
- && cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
-
-CMD bash

tools/docker/DocFileClangBase

+### Clone step of LLVM/clang ###
+FROM elsa/base-cmake:latest as clone
+MAINTAINER Tobias Lasser <lasser@in.tum.de>
+MAINTAINER David Frank <dfrank@in.tum.de>
+
+LABEL Description="Clang image based on Ubuntu"
+
+# Provide an argument to choose the compiler version from the command line
+ARG COMPILER_VERSION
+ARG CLANG_TAG=llvmorg-${COMPILER_VERSION}
+
+# Clone llvm-project with the specific tag
+RUN git clone --depth 1 --branch ${CLANG_TAG} https://github.com/llvm/llvm-project.git /tmp/llvm_project/
+
+### Configuration step of LLVM/clang ###
+FROM elsa/base-cmake:latest as configure
+COPY --from=clone /tmp/llvm_project/ /tmp/llvm_project/
+
+RUN mkdir -p /tmp/llvm_build/ && mkdir -p /tmp/llvm_install/ \
+  && cmake -S /tmp/llvm_project/llvm/ -B /tmp/llvm_build/ -GNinja -DLLVM_ENABLE_PROJECTS="clang;libcxx;libcxxabi;openmp" \
+           -DCMAKE_INSTALL_PREFIX=/tmp/llvm_install/ -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD=host -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON
+
+### Build LLVM/clang
+FROM elsa/base-cmake:latest as build
+COPY --from=configure /tmp/llvm_project/ /tmp/llvm_project/
+COPY --from=configure /tmp/llvm_build/ /tmp/llvm_build/
+
+
+# install a reduced version of clang (due to LLVM_INSTALL_TOOLCHAIN_ONLY=ON)
+WORKDIR /tmp/llvm_build/
+RUN ninja install
+WORKDIR /
+
+### Set base for clang ###
+FROM elsa/base-cmake:latest as base
+COPY --from=build /tmp/llvm_install/ /usr/local/
+
+# Set environment variables
+ENV CC="/usr/local/bin/clang"
+ENV CXX="/usr/local/bin/clang++"
+ENV PATH="$PATH:/usr/local/bin/"
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/:/usr/local/lib/"
+
+### Download, configure, build and install Intel DNNL
+FROM base as build-dnnl
+
+# Intel DNNL Versions
+ARG DNNL_VERSION=1.1.1
+
+# Add test scripts
+ADD scripts/ /tmp/scripts/
+
+# Run install script for DNNL
+RUN ./tmp/scripts/install_inteldnnl.sh ${DNNL_VERSION}
+
+### Final stage ###
+FROM base
+
+# Copy Intel DNNL over
+COPY --from=build-dnnl /tmp/dnnl_install /usr/local/
+
+# setup conan home directory, to be in the cache of our CI pipeline
+ENV CONAN_USER_HOME=/cache
+
+# setup conan for clang
+RUN conan profile new default --detect \
+ && conan profile update settings.compiler.libcxx=libc++ default \
+ && conan profile update settings.compiler.cppstd=17 default \
+ && conan profile update env.CC=clang default \
+ && conan profile update env.CXX=clang++ default \
+ && conan profile update env.CONAN_CMAKE_GENERATOR=Ninja default \
+ && conan profile show default
+
+# Add test scripts
+ADD tests/ /tmp/tests/
+
+CMD bash

tools/docker/DocFileClangTidy8

-FROM nvidia/cuda:9.2-devel-ubuntu18.04 
-MAINTAINER Tobias Lasser <lasser@in.tum.de>
-LABEL Description="clang-tidy 8 (with cuda9.2) develop with cmake/git"
-
-# install wget
-RUN apt-get -qq update \
- && apt-get install -qqy wget git pkg-config
-
-# install ninja
-RUN apt-get install -qqy ninja-build
-
-# install clang-tidy-8
-RUN apt-get install -qqy clang-tidy-8
-
-# install OpenSSL for CMake
-RUN apt-get install -qqy libssl-dev
-
-# install CMAKE
-ARG cmake_version=3.16.1
-ARG cmake_version_short=3.16
-ARG cmake_dir=cmake-${cmake_version}
-
-RUN wget https://cmake.org/files/v${cmake_version_short}/${cmake_dir}.tar.gz \
- && tar -xvzf ${cmake_dir}.tar.gz \
- && rm ${cmake_dir}.tar.gz \
- && cd ${cmake_dir} \
- && ./configure --parallel=28 \
- && make -j28 && make install \
- && cd .. && rm -rf ${cmake_dir} && ldconfig
-
-# install Intel MKL-DNN
-ARG mkldnn_version=1.1.1
-ARG mkldnn_folder=v${mkldnn_version}
-
-RUN wget https://github.com/intel/mkl-dnn/archive/${mkldnn_folder}.tar.gz \
- && tar -xvzf ${mkldnn_folder}.tar.gz \
- && rm ${mkldnn_folder}.tar.gz \
- && cd mkl-dnn-${mkldnn_version} \
- && mkdir build && cd build \
- && cmake .. \
- && make -j28 && make install \
- && cd ../.. && rm -rf mkl-dnn-${mkldnn_version} && ldconfig
-
-CMD bash
-

tools/docker/DocFileCuda

+# Provide an argument to choose the compiler version from the command line
+ARG CUDA_VERSION=10.2
+
+FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 as base
+MAINTAINER Tobias Lasser <lasser@in.tum.de>
+LABEL Description="Cuda develop with cmake/git and clang"
+
+ADD scripts/ /tmp/scripts/
+RUN /tmp/scripts/install_base.sh
+RUN apt-get update -qq && apt-get --no-install-recommends install -qq swig libncurses5-dev libedit-dev libpthread-stubs0-dev lcov \
+  && rm -rf /var/lib/apt/lists/*
+
+FROM base as build-cmake
+
+# CMake version
+ARG CMAKE_VERSION=3.16
+ARG CMAKE_VERSION_PATCH=4
+ARG CMAKE_DIR=cmake-${CMAKE_VERSION}.${CMAKE_VERSION_PATCH}
+
+RUN bash /tmp/scripts/install_cmake.sh "${CMAKE_VERSION}" "${CMAKE_DIR}"
+
+FROM base as cmake
+COPY --from=build-cmake /tmp/cmake-install /usr/local
+
+FROM cmake as clone-clang
+
+ARG CLANG_VERSION=9.0.1
+ARG CLANG_TAG=llvmorg-${CLANG_VERSION}