diff options
author | Philippe Lafoucrière <plafoucriere@gitlab.com> | 2019-06-25 11:57:33 +0300 |
---|---|---|
committer | Rémy Coutable <remy@rymai.me> | 2019-06-25 11:57:33 +0300 |
commit | ab696cdf28064ec088de0292290461b57fd900f5 (patch) | |
tree | 88f870554d441e3b992ebd47b8f3451aab627275 | |
parent | 3115c9fc121743dea29aa92a603a0c782eb3d75b (diff) |
Fix reports jobs timing out because of cache
Note: This commit relies on
https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26801
which brings support to multiple extends.
While `cache: {}` can disable a cache being set globally, it doesn't
work with extends, which will perform a reverse deep merge based on the
keys.
The cache defined in the base `.default-cache` job
won't be disabled in the report jobs.
As a side effect, the `code_quality`, `sast`, and `dependency_scanning`
jobs are running on a larger code base than expected, leading to
timeouts.
fixes https://gitlab.com/gitlab-org/gitlab-ee/issues/11303
fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/60879
fixes https://gitlab.com/gitlab-org/gitlab-ee/issues/12021
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 14 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 88 | ||||
-rw-r--r-- | changelogs/unreleased/60879-fix-reports-timing-out.yml | 5 |
3 files changed, 25 insertions, 82 deletions
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index b7ef4b31743..eb50f08c1a7 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -28,11 +28,15 @@ policy: pull stage: test -.dedicated-no-docs-pull-cache-job: - extends: .dedicated-pull-cache-job +.no-docs: except: - /(^docs[\/-].*|.*-docs$)/ +.dedicated-no-docs-pull-cache-job: + extends: + - .dedicated-pull-cache-job + - .no-docs + .dedicated-no-docs-and-no-qa-pull-cache-job: extends: .dedicated-pull-cache-job except: @@ -45,6 +49,12 @@ variables: SETUP_DB: "false" +# Jobs that need a dedicated runner, with no cache +.dedicated-no-docs: + extends: + - .dedicated-runner + - .no-docs + .single-script-job-dedicated-runner: extends: .dedicated-runner image: ruby:2.6-alpine diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index d0e09dbf2f8..89b5ae38072 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -1,98 +1,26 @@ include: - template: Code-Quality.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml code_quality: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: .dedicated-no-docs # gitlab-org runners set `privileged: false` but we need to have it set to true # since we're using Docker in Docker tags: [] before_script: [] cache: {} dependencies: [] - variables: - SETUP_DB: "false" sast: - extends: .dedicated-no-docs-no-db-pull-cache-job - image: docker:stable + extends: .dedicated-no-docs + before_script: [] + tags: [] variables: SAST_CONFIDENCE_LEVEL: 2 DOCKER_DRIVER: overlay2 - allow_failure: true - tags: [] - before_script: [] - cache: {} - dependencies: [] - services: - - docker:stable-dind - script: - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } - - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - | - docker run \ - $(propagate_env_vars \ - SAST_ANALYZER_IMAGES \ - SAST_ANALYZER_IMAGE_PREFIX \ - SAST_ANALYZER_IMAGE_TAG \ - SAST_DEFAULT_ANALYZERS \ - SAST_BRAKEMAN_LEVEL \ - SAST_GOSEC_LEVEL \ - SAST_FLAWFINDER_LEVEL \ - SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ - SAST_RUN_ANALYZER_TIMEOUT \ - ) \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code - artifacts: - reports: - sast: gl-sast-report.json dependency_scanning: - extends: .dedicated-no-docs-no-db-pull-cache-job - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - allow_failure: true - tags: [] before_script: [] - cache: {} - dependencies: [] - services: - - docker:stable-dind - script: - - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } - - | - docker run \ - $(propagate_env_vars \ - DS_ANALYZER_IMAGES \ - DS_ANALYZER_IMAGE_PREFIX \ - DS_ANALYZER_IMAGE_TAG \ - DS_DEFAULT_ANALYZERS \ - DEP_SCAN_DISABLE_REMOTE_CHECKS \ - DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - DS_PULL_ANALYZER_IMAGE_TIMEOUT \ - DS_RUN_ANALYZER_TIMEOUT \ - ) \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code - artifacts: - reports: - dependency_scanning: gl-dependency-scanning-report.json + tags: [] + extends: .dedicated-no-docs diff --git a/changelogs/unreleased/60879-fix-reports-timing-out.yml b/changelogs/unreleased/60879-fix-reports-timing-out.yml new file mode 100644 index 00000000000..845162fe10f --- /dev/null +++ b/changelogs/unreleased/60879-fix-reports-timing-out.yml @@ -0,0 +1,5 @@ +--- +title: Fix reports jobs timing out because of cache +merge_request: 29780 +author: +type: fixed |