Commit 73a99e62 authored by Benedikt Kleinmeier's avatar Benedikt Kleinmeier

Added 1-click-deployment in ".gitlab-ci.yml".

- Deployment can be triggered via play button in web interface: https://gitlab.lrz.de/vadere/vadere/pipelines
- Link to deployed Vadere package can be found in web interface: https://gitlab.lrz.de/vadere/vadere/environments

Configure ci setup #150

See merge request !43
parents ccba5d9d cd36f5a8
Pipeline #89400 passed with stages
in 3 minutes and 33 seconds
# # GitLab CI Jobs
# # GitLab CI Job Configuration
#
# Each of the jobs defined here is triggered by GitLab's CI pipeline.
# Goal: Run all jobs defined here on all available runners (GNU/Linux and
# Windows).
#
# Procedure: Use GitLab job templates prefixed with a "." and use them in the
# actual job description with the "extend" keyword.
#
# ## GitLab YAML Job Description Syntax
# ## Job Description Syntax
#
# <job_name>: # Job name which is displayed in web interface.
# stage: <stage_name> # <build|test|deploy>: these stages are executed
# # one after another. A stage can contain multiple
# # jobs.
# stage: <stage_name> # Run "script" commands in this stage. The
# # stages are executed sequentially.
# script:
# - <arbitrary_commands> # Here you can execute arbitratry terminal
# # commands. If any of the commands return a
# # non-zero exit code, the job fails.
#
# ## Maven Syntax for Skipping Unit Tests
# ## References
#
# General syntax to execute unit tests:
#
# ```
# mvn test
# ```
#
# The comma-separated list of tests after "-Dtest=<list>" is skipped. All other
# tests are executed normally:
#
# ```
# mvn -Dtest=!TestClassA,!TestClassB,!TestClassC
# ```
#
# **Note:** on some CI instances the exclamation mark must be escaped by a
# backslash. Otherwise, it is interpreted by the executing shell as special
# command.
#
# ## Unit Test Limitations
# - https://docs.gitlab.com/ee/ci/yaml/README.html
# Variable Definitions
variables:
VADERE_DEPLOYMENT_BASE_URL: "http://www.vadere.org/builds"
VADERE_PACKAGE_NAME_BRANCHES: "vadere.${CI_COMMIT_REF_NAME}.${CI_RUNNER_TAGS}.zip"
VADERE_PACKAGE_NAME_RELEASES: "vadere.${CI_COMMIT_TAG}.${CI_RUNNER_TAGS}.zip"
# Stage Definitions
stages:
- unit_test
- integration_test
- deploy
# PLEASE, OMIT FOLLOWING GITLAB PITFALLS:
#
# **Watch out:** OpenCL code cannot be run on virtual machines where tests are
# executed. Therefore, skip this tests.
# 1. "include:" statement does not work properly and seems really buggy.
# Therefore, define all job templates and jobs directly here.
# "include: .gitlab/ci/templates/test_stage_templates.yml" does not work!
# 2. Variables from "variables" section are not expanded correctly in "url"
# section of an "environment".
unit_tests_with_coverage:
stage: test
# Job Template Definitions
.template_unit_tests:
stage: unit_test
# Skip OpenCL tests by using "-Dtest=!Test1,!Test2,..." because GitLab
# runners do not support OpenCL.
script:
- mvn clean
- mvn -Dtest=!TestConvolution,!TestBitonicSort,!TestCLLinkedList,!TestCLOptimalStepsModel test
......@@ -48,11 +53,13 @@ unit_tests_with_coverage:
when: on_success
paths:
- "*/target/site/coverage-reports"
run_scenario_files:
stage: deploy
expire_in: 1 week
when: on_success
.template_scenario_files:
stage: integration_test
script:
- mvn clean
- mvn -Dmaven.test.skip=true package
- python3 Tools/ContinuousIntegration/run_vadere_console_with_all_scenario_files.py
artifacts:
......@@ -60,28 +67,105 @@ run_scenario_files:
paths:
- "log_dir"
expire_in: 1 week
when: manual
# exclude. Finde solution an then add this test again.
# run_origin_test:
# stage: deploy
# script:
# - Documentation/version-control/git-hook-vadere-software
# - mvn clean
# - mvn -Dmaven.test.skip=true package
# - python3 Tools/VadereAnalysisTools/VadereAnalysisTool/setup.py install --user
# - python3 Tools/ContinuousIntegration/run_orign_translation_test.py
run_seed_test:
stage: deploy
.template_seed_test:
stage: integration_test
script:
- mvn clean
- mvn -Dmaven.test.skip=true package
- python3 Tools/VadereAnalysisTools/VadereAnalysisTool/setup.py install --user
- python3 Tools/ContinuousIntegration/run_seed_comparison_test.py
- python3 Tools/ContinuousIntegration/run_seed_comparison_test.py
artifacts:
when: on_failure
paths:
- "Tools/ContinuousIntegration/run_seed_comparison_test.d/output"
expire_in: 1 week
when: manual
.template_deploy_master_automatically:
stage: deploy
script:
- mvn clean
- mvn -Dmaven.test.skip=true package
- python3 -m zipfile -c ${VADERE_PACKAGE_NAME_BRANCHES} VadereModelTests/ VadereGui/target/vadere.jar VadereSimulator/target/vadere-console.jar
- scp ${VADERE_PACKAGE_NAME_BRANCHES} di49mur@webdev-mwn.lrz.de:~/webserver/htdocs/builds/master/${VADERE_PACKAGE_NAME_BRANCHES}
only:
refs:
- master
when: on_success
.template_deploy_branch_manually:
stage: deploy
script:
- mvn clean
- mvn -Dmaven.test.skip=true package
- python3 -m zipfile -c ${VADERE_PACKAGE_NAME_BRANCHES} VadereModelTests/ VadereGui/target/vadere.jar VadereSimulator/target/vadere-console.jar
- scp ${VADERE_PACKAGE_NAME_BRANCHES} di49mur@webdev-mwn.lrz.de:~/webserver/htdocs/builds/branches/${VADERE_PACKAGE_NAME_BRANCHES}
when: manual
# Job Definitions
## Jobs for GNU/Linux runner (which is labelled with tag "linux")
run_unit_tests_on_linux:
extends: .template_unit_tests
tags:
- linux
run_scenario_files_on_linux:
extends: .template_scenario_files
tags:
- linux
run_seed_test_on_linux:
extends: .template_seed_test
tags:
- linux
deploy_master_on_linux:
extends: .template_deploy_master_automatically
tags:
- linux
environment:
name: master/linux
url: ${VADERE_DEPLOYMENT_BASE_URL}/master/vadere.${CI_COMMIT_REF_NAME}.${CI_RUNNER_TAGS}.zip
deploy_branch_on_linux:
extends: .template_deploy_branch_manually
tags:
- linux
environment:
name: branches/${CI_COMMIT_REF_NAME}/linux
url: ${VADERE_DEPLOYMENT_BASE_URL}/branches/vadere.${CI_COMMIT_REF_NAME}.${CI_RUNNER_TAGS}.zip
## Jobs for Windows runner (which is labelled with tag "windows")
run_unit_tests_on_windows:
extends: .template_unit_tests
tags:
- windows
run_scenario_files_on_windows:
extends: .template_scenario_files
tags:
- windows
run_seed_test_on_windows:
extends: .template_seed_test
tags:
- windows
deploy_master_on_windows:
extends: .template_deploy_master_automatically
tags:
- windows
environment:
name: master/windows
url: ${VADERE_DEPLOYMENT_BASE_URL}/master/vadere.${CI_COMMIT_REF_NAME}.${CI_RUNNER_TAGS}.zip
deploy_branch_on_windows:
extends: .template_deploy_branch_manually
tags:
- windows
environment:
name: branches/${CI_COMMIT_REF_NAME}/windows
url: ${VADERE_DEPLOYMENT_BASE_URL}/branches/vadere.${CI_COMMIT_REF_NAME}.${CI_RUNNER_TAGS}.zip
......@@ -6,6 +6,18 @@
### Added
- Configured 1-click-deployment in ".gitlab-ci.yml".
* The build pipeline consists of three stages: unit_test, integration_test and deploy.
* These stages run sequentially. The build pipeline is stopped if one stage fails.
* The jobs in "integration_test" and "deploy" are marked as "manual". I.e., these jobs
must be triggered via the play button in the GitLab web interface:
https://gitlab.lrz.de/vadere/vadere/pipelines
* The jobs in the "integration_test" stage run manual because of a bug in GitLab which
does not allow to run manual jobs (e.g., from deploy stage) when the pipeline fails.
In Vadere, this situation arises because of our seed test. Link to issue on GitLab website:
https://gitlab.com/gitlab-org/gitlab-ce/issues/51645
* The link to the deployed items can be found in the GitLab web interface:
https://gitlab.lrz.de/vadere/vadere/environments
- Create a Facade for the logging in vadere. This will remove 3rd party
dependencies in source most files. Only a small set of facade classes
will have the dependency.
......
......@@ -16,4 +16,4 @@ from suqc.parameter.postchanges import ScenarioChanges
from suqc.qoi import *
from suqc.request import *
__version__ = "1.0"
__version__ = "1.01"
......@@ -73,7 +73,7 @@ class ParameterVariation(metaclass=abc.ABCMeta):
ret = dict(row)
delete_keys = list()
for k, v in ret.items():
if np.isnan(v):
if isinstance(v, np.float) and np.isnan(v):
delete_keys.append(k)
for dk in delete_keys:
......
......@@ -26,7 +26,7 @@ class FileDataInfo(object):
# changes in Vadere. See also vadere issue #199 and #201.
map_outputtype2index = {"IdOutputFile": 1,
"LogEventOutputFile": 1,
"NotDataKeyOutputFile": 0,
"NoDataKeyOutputFile": 0,
"PedestrianIdOutputFile": 1,
"TimestepOutputFile": 1,
"TimestepPedestrianIdOutputFile": 2,
......@@ -125,9 +125,12 @@ class QuantityOfInterest(object):
nr_row_indices = req_qoi.nr_row_indices
df = pd.read_csv(filepath, delimiter=" ", header=[0], comment="#")
idx_keys = df.columns[:nr_row_indices]
return df.set_index(idx_keys.tolist())
if req_qoi.output_key.__eq__("NoDataKeyOutputFile"):
return df
else:
idx_keys = df.columns[:nr_row_indices]
return df.set_index(idx_keys.tolist())
def _add_parid2idx(self, df, par_id):
# from https://stackoverflow.com/questions/14744068/prepend-a-level-to-a-pandas-multiindex
......
......@@ -116,42 +116,6 @@
</executions>
</plugin>
<plugin>
<groupId>com.akathist.maven.plugins.launch4j</groupId>
<artifactId>launch4j-maven-plugin</artifactId>
<executions>
<execution>
<id>vadere-gui</id>
<phase>package</phase>
<goals><goal>launch4j</goal></goals>
<configuration>
<headerType>gui</headerType>
<outfile>target/vadere.exe</outfile>
<jar>target/vadere.jar</jar>
<errTitle>vadere</errTitle>
<icon>../vadere-gui.ico</icon>
<classPath>
<mainClass>org.vadere.gui.projectview.VadereApplication</mainClass>
</classPath>
<jre>
<minVersion>1.8.0</minVersion>
</jre>
<versionInfo>
<fileVersion>0.0.0.6</fileVersion>
<txtFileVersion>0.0.0.6</txtFileVersion>
<fileDescription>Vadere is an open source framework for the simulation of microscopic pedestrian dynamics.</fileDescription>
<copyright>GNU Lesser General Public License (LGPL)</copyright>
<productVersion>0.0.0.6</productVersion>
<txtProductVersion>0.0.0.6</txtProductVersion>
<productName>Vadere</productName>
<internalName>vadere</internalName>
<originalFilename>vadere.exe</originalFilename>
</versionInfo>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
......@@ -296,4 +260,4 @@
</dependency>
</dependencies>
</project>
\ No newline at end of file
</project>
......@@ -162,41 +162,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>com.akathist.maven.plugins.launch4j</groupId>
<artifactId>launch4j-maven-plugin</artifactId>
<executions>
<execution>
<id>vadere-console</id>
<phase>package</phase>
<goals><goal>launch4j</goal></goals>
<configuration>
<headerType>console</headerType>
<outfile>target/vadere-console.exe</outfile>
<jar>target/vadere-console.jar</jar>
<errTitle>vadere</errTitle>
<icon>../vadere-cmd.ico</icon>
<classPath>
<mainClass>org.vadere.gui.projectview.VadereApplication</mainClass>
</classPath>
<jre>
<minVersion>1.8.0</minVersion>
</jre>
<versionInfo>
<fileVersion>0.0.0.6</fileVersion>
<txtFileVersion>0.0.0.6</txtFileVersion>
<fileDescription>Vadere is an open source framework for the simulation of microscopic pedestrian dynamics.</fileDescription>
<copyright>GNU Lesser General Public License (LGPL)</copyright>
<productVersion>0.0.0.6</productVersion>
<txtProductVersion>0.0.0.6</txtProductVersion>
<productName>Vadere-console</productName>
<internalName>vadere-console</internalName>
<originalFilename>vadere-console.exe</originalFilename>
</versionInfo>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<pluginManagement>
<plugins>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment