.gitlab-ci.yml 12.2 KB
Newer Older
Tobias Lasser's avatar
Tobias Lasser committed
1
stages:
Jens Petit's avatar
Jens Petit committed
2
- static-test
Tobias Lasser's avatar
Tobias Lasser committed
3
4
- compile
- test
5
- sanitizer
6
- coverage
Tobias Lasser's avatar
Tobias Lasser committed
7
- docs
David Frank's avatar
David Frank committed
8
- docker
Tobias Lasser's avatar
Tobias Lasser committed
9

10
11
12
### variables ###

variables:
13
14
15
16
17
18
  DOCKER_REGISTRY_URL: "docker.ciip.in.tum.de:5000" 
  CLANG_FORMAT_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/clang-format:10"
  CLANG_TIDY_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/clang-tidy:8"
  GCC_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/gcc-pybinds:10"
  CLANG9_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/clang-pybinds:9"
  CLANG10_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/clang-pybinds:10"
Nikola Dinev's avatar
Nikola Dinev committed
19
  CUDA_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/cuda:11.2"
20
21
22
23
  UBUNTU_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/ubuntu:20.04"
  GPU_ONLY_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/cuda-clang:10.0"
  SANITIZER_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/gcc:10"
  COVERAGE_IMAGE: "${DOCKER_REGISTRY_URL}/elsa/coverage:10.2"
24

25
26
### Cache setup ###

27
28
29
# Caches should only be used for caching between pipelines not jobs
# By default, have a unique and separate cache for each branch and job in the CI, otherwhise it can happen that 
# different build configurations are spilled into different jobs and sporadic build failure occures.
30
cache:
31
  key: "$CI_COMMIT_REF_SLUG-$CI_JOB_STAGE-$CI_JOB_NAME" 
32
  paths:
33
    - build/_deps/*
34

Tobias Lasser's avatar
Tobias Lasser committed
35
36
### job templates ###

David Frank's avatar
David Frank committed
37
38
39
#### Schedule jobs ####

# run job only only on schedules
David Frank's avatar
David Frank committed
40
.job_template: &nightly_job
41
42
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "schedule"'
David Frank's avatar
David Frank committed
43

44
# run job if connected to a schedule, merge request or some other things
45
.job_template: &run_on_merge_requests
46
47
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
48
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "schedule"'
49
    - when: never
50
51
52

# always run this job
.job_template: &run_always
53
54
  rules:
    - when: always
55

David Frank's avatar
David Frank committed
56
#### Build job templates ####
David Frank's avatar
David Frank committed
57

Tobias Lasser's avatar
Tobias Lasser committed
58
.job_template: &build_job_artifact
David Frank's avatar
David Frank committed
59
  <<: *run_always
Tobias Lasser's avatar
Tobias Lasser committed
60
61
62
63
  stage: compile
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
    paths:
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
      # The actual build artifacts
      - build/bin
      - build/lib
      - build/pyelsa/
      - build/elsa/elsaConfig.cmake
      - build/elsa/elsaConfigVersion.cmake

      # If we ever generate headers, store them as well
      - build/elsa/**/*.h

      # CTest files
      - build/**/CTestTestfile.cmake
      - build/**/tests/test_*include-*.cmake
      - build/**/tests/test_*tests-*.cmake
      - build/Testing
79
      - build/junit_report.xml
80

81
      # Add elsa install to artifact
82
83
84
85
86
      - install-elsa/include/elsa/**/*.h
      - install-elsa/include/elsa/**/*.hpp
      - install-elsa/include/elsa/**/*.cuh
      - install-elsa/lib/cmake/elsa/**/*.cmake
      - install-elsa/lib/libelsa*.a
87
      # Add quickvec install to artifact
88
89
90
      - install-elsa/include/quickvec/**/*.h
      - install-elsa/include/quickvec/**/*.hpp
      - install-elsa/lib/cmake/quickvec/*.cmake
91
      - install-elsa/lib/libquickvec.a
92
      # Add doctest install to artifact
93
94
      - install-elsa/include/doctest/**/*.h
      - install-elsa/lib/cmake/doctest/**/*.cmake
95
      # Add spdlog install to artifact
96
97
98
99
100
101
      - install-elsa/include/spdlog/**/*.h
      - install-elsa/lib/cmake/spdlog/**/*.cmake
      - install-elsa/lib/libspdlog.a
      - install-elsa/lib/pkgconfig/spdlog.pc
    exclude:
      - build/_deps/*
102
    expire_in: 60 mins
103

Tobias Lasser's avatar
Tobias Lasser committed
104
105
106
107
.buildjob_script_clang:
  script: &buildjob_clang
    - mkdir -p build
    - cd build
108
    - cmake .. -GNinja -DELSA_BENCHMARKS=ON  -DCMAKE_INSTALL_PREFIX="../install-elsa" -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
109
    - ninja
110
    - ninja build-tests
David Frank's avatar
David Frank committed
111
112
    - ninja build-benchmarks
    - ninja build-examples
113
    - ninja install
Tobias Lasser's avatar
Tobias Lasser committed
114
115
116
117
118

.buildjob_script_normal:
  script: &buildjob_normal
    - mkdir -p build
    - cd build
119
    - cmake .. -GNinja -DELSA_BENCHMARKS=ON -DCMAKE_INSTALL_PREFIX="../install-elsa"
120
    - ninja
121
    - ninja build-tests
David Frank's avatar
David Frank committed
122
123
    - ninja build-benchmarks
    - ninja build-examples
124
    - ninja install
Tobias Lasser's avatar
Tobias Lasser committed
125

David Frank's avatar
David Frank committed
126
127
128
#### Test job templates ####

.testjob_template: &test_job_artifact
David Frank's avatar
David Frank committed
129
130
  <<: *run_always
  script:
131
    - cd build
132
    - ctest --schedule-random --output-junit junit_report.xml || ctest --rerun-failed --output-on-failure
133
  artifacts:
134
135
136
137
    name: "$CI_COMMIT_REF_SLUG-$CI_JOB_NAME"
    paths:
      - build/
    expire_in: 60 mins
138
    reports:
139
      junit: "build/junit_report.xml"
140

David Frank's avatar
David Frank committed
141
#### Install job templates ####
David Frank's avatar
David Frank committed
142

David Frank's avatar
David Frank committed
143
144
145
146
147
.installjob_clang_anchors:
  script: &installjob_script_clang
    - git clone https://gitlab.lrz.de/IP/elsa_testing.git /tmp/elsa_testing
    - cd /tmp/elsa_testing
    - mkdir -p build/ && cd build/
148
    - cmake .. -GNinja -DELSA_INSTALL_DIR="$CI_PROJECT_DIR/install-elsa" -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
David Frank's avatar
David Frank committed
149
    - ninja
David Frank's avatar
David Frank committed
150
151
152
153
154
155

.installjob_default_anchors:
  script: &installjob_script
    - git clone https://gitlab.lrz.de/IP/elsa_testing.git /tmp/elsa_testing
    - cd /tmp/elsa_testing
    - mkdir -p build/ && cd build/
156
    - cmake .. -GNinja -DELSA_INSTALL_DIR="$CI_PROJECT_DIR/install-elsa"
David Frank's avatar
David Frank committed
157
    - ninja
158

David Frank's avatar
David Frank committed
159
.installjob_anchors: &install_job
David Frank's avatar
David Frank committed
160
161
  <<: *run_on_merge_requests

Jens Petit's avatar
Jens Petit committed
162
163
164
### static test ###

clang-format:
David Frank's avatar
David Frank committed
165
  <<: *run_always
Jens Petit's avatar
Jens Petit committed
166
  stage: static-test
167
  image: $CLANG_FORMAT_IMAGE
Jens Petit's avatar
Jens Petit committed
168
169
170
171
172
173
  script:
    ./tools/ci_scripts/clang-format-test.sh
  tags:
    - linux
    - elsa
    - clang
Tobias Lasser's avatar
Tobias Lasser committed
174

175
clang-tidy:
David Frank's avatar
David Frank committed
176
  <<: *run_always
177
  stage: static-test
178
  image: $CLANG_TIDY_IMAGE
179
180
181
182
183
184
185
  script:
    ./tools/ci_scripts/clang-tidy.sh
  tags:
    - linux
    - elsa
    - clang

186
187
188
189
190
191
192
193
194
195
196
comment-formating:
  <<: *run_always
  stage: static-test
  image: $CLANG_FORMAT_IMAGE
  script:
    ./tools/ci_scripts/check-comment-style.sh
  allow_failure: true
  tags:
    - linux
    - elsa
    - clang
197

198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
.cmake-format:
  <<: *run_always
  stage: static-test
  image: $CLANG_FORMAT_IMAGE
  script:
    ./tools/ci_scripts/cmake-format-test.sh
  tags:
    - linux
    - elsa
    - clang

cmake-lint:
  <<: *run_always
  stage: static-test
  image: $CLANG_FORMAT_IMAGE
  script:
    ./tools/ci_scripts/cmake-lint-test.sh
  tags:
    - linux
    - elsa
    - clang

Tobias Lasser's avatar
Tobias Lasser committed
220
221
222
223
### compile jobs ###

build-ubuntu:
  <<: *build_job_artifact
224
  image: $UBUNTU_IMAGE
Tobias Lasser's avatar
Tobias Lasser committed
225
  script: *buildjob_normal
Tobias Lasser's avatar
Tobias Lasser committed
226
227
228
  tags:
    - linux
    - elsa
Jens Petit's avatar
Jens Petit committed
229

230
build-gcc:
Tobias Lasser's avatar
Tobias Lasser committed
231
  <<: *build_job_artifact
232
  image: $GCC_IMAGE
233
234
  script:
  - *buildjob_normal
Tobias Lasser's avatar
Tobias Lasser committed
235
236
237
238
  tags:
    - linux
    - elsa
    - gcc
239

240
241
242
build-clang9:
  <<: *build_job_artifact
  image: $CLANG9_IMAGE
243
244
245
246
  script:
  - *buildjob_clang
  - pip3 install matplotlib
  - PYTHONPATH="$PYTHONPATH:$(pwd)" python3 ../examples/solver_experiments.py --headless --max-iterations 1 --problem-size 16
247
248
249
250
251
252
  tags:
    - linux
    - elsa
    - clang

build-clang10:
Tobias Lasser's avatar
Tobias Lasser committed
253
  <<: *build_job_artifact
254
  image: $CLANG10_IMAGE
255
256
257
258
  script:
  - *buildjob_clang
  - pip3 install matplotlib
  - PYTHONPATH="$PYTHONPATH:$(pwd)" python3 ../examples/solver_experiments.py --headless --max-iterations 1 --problem-size 16
Tobias Lasser's avatar
Tobias Lasser committed
259
260
261
262
263
  tags:
    - linux
    - elsa
    - clang

264
build-cuda:
Tobias Lasser's avatar
Tobias Lasser committed
265
  <<: *build_job_artifact
266
  image: $CUDA_IMAGE
Nikola Dinev's avatar
Nikola Dinev committed
267
268
269
270
271
272
273
274
275
  script:
    - mkdir -p build
    - cd build
    - cmake .. -GNinja -DELSA_CUDA_VECTOR=ON -DELSA_BENCHMARKS=ON -DCMAKE_INSTALL_PREFIX="../install-elsa"
    - ninja
    - ninja build-tests
    - ninja build-benchmarks
    - ninja build-examples
    - ninja install
Tobias Lasser's avatar
Tobias Lasser committed
276
277
278
279
280
281
  tags:
    - linux
    - elsa
    - gcc
    - cuda

282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
# TODO: CUDA compilation with clang no longer supported, change to nvcc build with clang as host compiler later
# build-gpu-only:
#   <<: *build_job_artifact
#   image: $GPU_ONLY_IMAGE
#   script:
#     - mkdir -p build
#     - cd build
#     - cmake .. -GNinja -DELSA_BENCHMARKS=ON -DELSA_CUDA_VECTOR=ON -DCMAKE_INSTALL_PREFIX="../install-elsa" -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
#     - ninja
#     - ninja build-tests
#     - ninja build-benchmarks
#     - ninja build-examples
#     - ninja install
#   tags:
#     - linux
#     - elsa
#     - gcc
#     - cuda
300

Tobias Lasser's avatar
Tobias Lasser committed
301
302
303
304

### test jobs ###

test-ubuntu:
305
  <<: *test_job_artifact
Tobias Lasser's avatar
Tobias Lasser committed
306
  stage: test
307
  image: $UBUNTU_IMAGE
308
  dependencies:
Tobias Lasser's avatar
Tobias Lasser committed
309
310
311
312
    - build-ubuntu
  tags:
    - linux
    - elsa
Jens Petit's avatar
Jens Petit committed
313

314
test-gcc:
315
  <<: *test_job_artifact
Tobias Lasser's avatar
Tobias Lasser committed
316
  stage: test
317
  image: $GCC_IMAGE
Tobias Lasser's avatar
Tobias Lasser committed
318
  dependencies:
319
    - build-gcc
Tobias Lasser's avatar
Tobias Lasser committed
320
321
322
323
  tags:
    - linux
    - elsa
    - gcc
324

325
test-clang9:
326
  <<: *test_job_artifact
Tobias Lasser's avatar
Tobias Lasser committed
327
  stage: test
328
  image: $CLANG9_IMAGE
Tobias Lasser's avatar
Tobias Lasser committed
329
  dependencies:
330
331
332
333
334
335
336
337
338
339
340
341
    - build-clang9
  tags:
    - linux
    - elsa
    - clang

test-clang10:
  <<: *test_job_artifact
  stage: test
  image: $CLANG10_IMAGE
  dependencies:
    - build-clang10
Tobias Lasser's avatar
Tobias Lasser committed
342
343
344
345
  tags:
    - linux
    - elsa
    - clang
Tobias Lasser's avatar
Tobias Lasser committed
346

347
test-cuda:
348
  <<: *test_job_artifact
Tobias Lasser's avatar
Tobias Lasser committed
349
  stage: test
350
  image: $CUDA_IMAGE
Tobias Lasser's avatar
Tobias Lasser committed
351
  dependencies:
352
    - build-cuda
Tobias Lasser's avatar
Tobias Lasser committed
353
354
355
356
357
358
  tags:
    - linux
    - elsa
    - gcc
    - cuda

359
360
361
362
363
364
365
366
367
368
369
# test-gpu-only:
#   <<: *test_job_artifact
#   stage: test
#   image: $GPU_ONLY_IMAGE
#   dependencies:
#     - build-gpu-only
#   tags:
#     - linux
#     - elsa
#     - gcc
#     - cuda
370

David Frank's avatar
David Frank committed
371
372
373
374
375
376
377
378
379
380
381
382
install-gcc:
  <<: *install_job
  stage: test
  image: $GCC_IMAGE
  dependencies:
    - build-gcc
  script: *installjob_script
  tags:
    - linux
    - elsa
    - gcc

383
install-clang9:
David Frank's avatar
David Frank committed
384
385
  <<: *install_job
  stage: test
386
  image: $CLANG9_IMAGE
David Frank's avatar
David Frank committed
387
  dependencies:
388
389
390
391
392
393
394
395
396
397
398
399
400
    - build-clang9
  script: *installjob_script_clang
  tags:
    - linux
    - elsa
    - clang

install-clang10:
  <<: *install_job
  stage: test
  image: $CLANG10_IMAGE
  dependencies:
    - build-clang10
David Frank's avatar
David Frank committed
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
  script: *installjob_script_clang
  tags:
    - linux
    - elsa
    - clang

install-cuda:
  <<: *install_job
  stage: test
  image: $CUDA_IMAGE
  dependencies:
    - build-cuda
  script: *installjob_script
  tags:
    - linux
    - elsa
    - gcc
    - cuda

420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
# install-gpu-only:
#   <<: *install_job
#   stage: test
#   image: $GPU_ONLY_IMAGE
#   dependencies:
#     - build-gpu-only
#   script:
#     - git clone https://gitlab.lrz.de/IP/elsa_testing.git /tmp/elsa_testing
#     - cd /tmp/elsa_testing
#     - mkdir -p build/ && cd build/
#     - cmake .. -GNinja -DELSA_INSTALL_DIR="$CI_PROJECT_DIR/install-elsa" -Dquickvec_DIR="$CI_PROJECT_DIR/install-elsa/lib/cmake/quickvec" -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi" -DELSA_CUDA_VECTOR=ON
#     - ninja
#   tags:
#     - linux
#     - elsa
#     - gcc
#     - cuda
Tobias Lasser's avatar
Tobias Lasser committed
437

David Frank's avatar
David Frank committed
438
### sanitizers ###
439

440
cuda-memcheck:
441
  <<: *run_on_merge_requests
442
  stage: sanitizer
443
  image: $CUDA_IMAGE
444
445
446
  script:
    ./tools/ci_scripts/cuda-memcheck.sh
  dependencies:
447
    - test-cuda
448
449
450
451
452
453
  tags:
    - linux
    - elsa
    - gcc
    - cuda

454
# Be sure to run this job with container which has privaledge mode set
455
asan-ubsan:
456
  <<: *run_on_merge_requests
David Frank's avatar
David Frank committed
457
  stage: sanitizer
David Frank's avatar
David Frank committed
458
  image: $SANITIZER_IMAGE
David Frank's avatar
David Frank committed
459
  dependencies: []
David Frank's avatar
David Frank committed
460
461
462
  script:
    - mkdir -p build
    - cd build
David Frank's avatar
David Frank committed
463
    - cmake -GNinja -DELSA_BUILD_CUDA_PROJECTORS=OFF -DELSA_BUILD_PYTHON_BINDINGS=OFF -DCMAKE_BUILD_TYPE=Debug -DELSA_SANITIZER="Address;Undefined" ..
David Frank's avatar
David Frank committed
464
465
466
467
468
469
470
    - ninja tests
  tags:
    - linux
    - elsa
    - gcc
    - cuda

Jens Petit's avatar
Jens Petit committed
471

472
473
### test coverage ###

David Frank's avatar
David Frank committed
474
# Enforce GCC for test coverage, as our coverage only works with gcov and acts weird with clang
475
test-coverage:
476
  <<: *run_on_merge_requests
477
  stage: coverage
478
  image: $COVERAGE_IMAGE
David Frank's avatar
David Frank committed
479
  dependencies: []
480
481
482
  script:
    - mkdir -p build
    - cd build
David Frank's avatar
David Frank committed
483
    - CXX=g++ cmake .. -GNinja -DELSA_BUILD_PYTHON_BINDINGS=OFF -DCMAKE_BUILD_TYPE=Debug -DELSA_COVERAGE=ON
484
485
486
487
488
489
    - ninja tests
    - ninja test_coverage
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME-coverage"
    paths:
      - build/test_coverage/
490
    expire_in: 60 mins
491
492
493
494
495
496
497
  tags:
    - linux
    - elsa
    - gcc
    - cuda


Jens Petit's avatar
Jens Petit committed
498
### deploy docs and coverage report ###
Tobias Lasser's avatar
Tobias Lasser committed
499
500

deploy-docs:
David Frank's avatar
David Frank committed
501
  <<: *nightly_job
Tobias Lasser's avatar
Tobias Lasser committed
502
  stage: docs
Jens Petit's avatar
Jens Petit committed
503
504
  dependencies:
    - test-coverage
Tobias Lasser's avatar
Tobias Lasser committed
505
506
507
  script:
    - mkdir -p build
    - cd build
Tobias Lasser's avatar
Tobias Lasser committed
508
509
    - cmake .. -GNinja
    - ninja docs
Tobias Lasser's avatar
Tobias Lasser committed
510
    - cp -r docs/sphinx/* /var/www/ciip/elsadocs/
Jens Petit's avatar
Jens Petit committed
511
    - cd ..
Tobias Lasser's avatar
Tobias Lasser committed
512
    - cp -r build/test_coverage/* /var/www/ciip/elsacoverage/
Tobias Lasser's avatar
Tobias Lasser committed
513
514
  tags:
    - elsa-docs-deploy
Tobias Lasser's avatar
Tobias Lasser committed
515

516
517
518
519
520

### build, test and deploy docker images ###
.deploy-docker:
  <<: *nightly_job
  stage: docker
David Frank's avatar
David Frank committed
521
  dependencies: []
522
523
524
525
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PIPELINE_SOURCE == "schedule"'
    - changes:
      - tools/docker/*
526
527
528
  script:
    - cd tools/docker
    - ./buildDockerContainers.sh
529
530
531
532
  tags:
    - elsa
    - shell

533
534
535
536
537
538

### Test jobs ###

# These jobs, are for local build and test all in one job, should not be run with pipeline

.all-clang:
539
540
541
  image: $CLANG9_IMAGE
  rules:
    - when: never
David Frank's avatar
David Frank committed
542
  script:
543
544
    - mkdir -p build
    - cd build
545
    - cmake .. -GNinja -DELSA_BENCHMARKS=ON -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_EXE_LINKER_FLAGS="-lc++abi"
546
547
    - ninja
    - ninja tests
548
549
550

.all-gcc:
  image: $GCC_IMAGE
551
552
  rules:
    - when: never
David Frank's avatar
David Frank committed
553
  script:
554
555
556
    - *buildjob_normal
    - ninja tests
    - *installjob_script