From 63dad2c39c23a0714bd35cd84497a4ee36fd30fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 14 May 2019 23:32:58 +0200 Subject: Optimize the use of cache and dependencies in CI pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The idea is to only pull & push from/to cache from the `setup-test-env` and `compile-assets` jobs, and only pull the required artifacts from all the other jobs. Signed-off-by: Rémy Coutable --- .gitlab-ci.yml | 1 - .gitlab/ci/frontend.gitlab-ci.yml | 126 +++++++++++++++----------------------- .gitlab/ci/global.gitlab-ci.yml | 41 ++++++++----- .gitlab/ci/pages.gitlab-ci.yml | 2 +- .gitlab/ci/rails.gitlab-ci.yml | 60 ++++++++---------- .gitlab/ci/reports.gitlab-ci.yml | 91 +++++++++++++++++++++++++++ .gitlab/ci/review.gitlab-ci.yml | 16 ++--- .gitlab/ci/setup.gitlab-ci.yml | 2 +- scripts/gitaly_test.rb | 2 +- 9 files changed, 204 insertions(+), 137 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f926cbc2939..de29963b1ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,6 @@ variables: BUILD_ASSETS_IMAGE: "false" before_script: - - date - source scripts/utils.sh - source scripts/prepare_build.sh - date diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 5c3278fcf53..49eb62f4a8f 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -1,20 +1,11 @@ -.assets-compile-cache: &assets-compile-cache - cache: - key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v6" - paths: - - vendor/ruby/ - - .yarn-cache/ - - tmp/cache/assets/sprockets - .use-pg: &use-pg services: - name: postgres:9.6.14 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine -.gitlab:assets:compile-metadata: - <<: *assets-compile-cache - extends: .dedicated-no-docs-pull-cache-job +gitlab:assets:compile: + extends: .dedicated-no-docs-no-cache image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1 dependencies: - setup-test-env @@ -42,6 +33,10 @@ - install_api_client_dependencies_with_apt - play_job "review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played - play_job "schedule:review-build-cng" || true # this job might not exist so ignore the failure if it cannot be played + cache: + key: "assets-compile:tmp_cache_assets_sprockets" + paths: + - tmp/cache/assets/sprockets artifacts: name: webpack-report expire_in: 31d @@ -57,68 +52,49 @@ - docker - gitlab-org -gitlab:assets:compile: - extends: .gitlab:assets:compile-metadata - cache: - policy: pull-push - only: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - -gitlab:assets:compile pull-cache: - extends: .gitlab:assets:compile-metadata - cache: - policy: pull - except: - refs: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - /(^docs[\/-].+|.+-docs$)/ - -.compile-assets-metadata: - extends: .dedicated-runner +compile-assets: + extends: + - .dedicated-runner-default-cache + - .no-docs <<: *use-pg - <<: *assets-compile-cache stage: prepare script: - node --version - retry yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline - free -m - retry bundle exec rake gitlab:assets:compile - - scripts/clean-old-cached-assets variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 artifacts: expire_in: 7d paths: - - node_modules - - public/assets + - node_modules/ + - public/assets/ -compile-assets: - extends: .compile-assets-metadata - cache: - policy: pull-push +gitlab:ui:visual: + extends: .dedicated-no-docs-no-cache + before_script: [] + allow_failure: true + dependencies: + - compile-assets + script: + # Remove node modules from GitLab that may conflict with gitlab-ui + - rm -r node_modules + - git clone https://gitlab.com/gitlab-org/gitlab-ui.git + - cp public/assets/application-*.css gitlab-ui/styles/application.css + - cd gitlab-ui + - yarn install + - CSS_URL=./application.css yarn test only: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee -compile-assets pull-cache: - extends: .compile-assets-metadata - cache: - policy: pull - except: - refs: - - master@gitlab-org/gitlab-ce - - master@gitlab-org/gitlab-ee - - /(^docs[\/-].+|.+-docs$)/ - karma: - extends: .dedicated-no-docs-pull-cache-job + extends: .dedicated-no-docs-no-cache <<: *use-pg dependencies: - compile-assets - - compile-assets pull-cache - setup-test-env variables: # we override the max_old_space_size to prevent OOM errors @@ -142,11 +118,10 @@ karma: junit: junit_karma.xml jest: - extends: .dedicated-no-docs-and-no-qa-pull-cache-job + extends: .dedicated-no-docs-no-qa-no-cache <<: *use-pg dependencies: - compile-assets - - compile-assets pull-cache - setup-test-env script: - scripts/gitaly-test-spawn @@ -168,42 +143,39 @@ jest: key: jest paths: - tmp/jest/jest/ - policy: pull-push qa:internal: - extends: .dedicated-no-docs-no-db-pull-cache-job - services: [] + extends: .dedicated-no-docs-no-db-no-cache + before_script: [] + dependencies: + - setup-test-env script: + - mkdir -p qa/vendor && mv vendor/ruby qa/vendor/ruby - cd qa/ - - bundle install + - time bundle install --without=production --jobs=$(nproc) --path=vendor --retry=3 --quiet - bundle exec rspec - dependencies: - - setup-test-env qa:selectors: - extends: .dedicated-no-docs-no-db-pull-cache-job - services: [] + extends: .dedicated-no-docs-no-db-no-cache + before_script: [] + dependencies: + - setup-test-env script: + - mkdir -p qa/vendor && mv vendor/ruby qa/vendor/ruby - cd qa/ - - bundle install + - time bundle install --without=production --jobs=$(nproc) --path=vendor --retry=3 --quiet - bundle exec bin/qa Test::Sanity::Selectors - dependencies: - - setup-test-env .qa-frontend-node: &qa-frontend-node - extends: .dedicated-no-docs-no-db-pull-cache-job - stage: test + extends: .dedicated-no-docs-no-db-no-cache + before_script: [] + dependencies: [] cache: key: "$CI_JOB_NAME" paths: - .yarn-cache/ - policy: pull-push - dependencies: [] - before_script: [] script: - - date - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline - - date - yarn run webpack-prod qa-frontend-node:8: @@ -220,12 +192,12 @@ qa-frontend-node:latest: allow_failure: true lint:javascript:report: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: .dedicated-no-docs-no-db-no-cache stage: post-test - dependencies: [] before_script: [] + dependencies: + - compile-assets script: - - date - yarn run eslint-report || true # ignore exit code artifacts: name: eslint-report @@ -234,14 +206,12 @@ lint:javascript:report: - eslint-report.html jsdoc: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: .dedicated-no-docs-no-db-no-cache stage: post-test + before_script: [] dependencies: - compile-assets - - compile-assets pull-cache - before_script: [] script: - - date - yarn run jsdoc || true # ignore exit code artifacts: name: jsdoc diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 78ef346d417..6e1be400f54 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -9,23 +9,22 @@ - gitlab-org .default-cache: &default-cache - key: "debian-stretch-ruby-2.6.3-node-12.x" - paths: - - vendor/ruby - - .yarn-cache/ - - vendor/gitaly-ruby + cache: + key: "debian-stretch-ruby-2.6.3-node-12.x" + paths: + - vendor/ruby/ + - .yarn-cache/ + - vendor/gitaly-ruby/ + - tmp/cache/assets/sprockets .dedicated-runner-default-cache: extends: .dedicated-runner - cache: - <<: *default-cache + <<: *default-cache -# Jobs that only need to pull cache -.dedicated-pull-cache-job: +# Jobs that do not need cache +.dedicated-no-cache: extends: .dedicated-runner - cache: - <<: *default-cache - policy: pull + cache: {} stage: test .no-docs: @@ -33,12 +32,24 @@ refs: - /(^docs[\/-].+|.+-docs$)/ +# Jobs that shouldn't run for docs branches and do not need cache +.dedicated-no-docs-no-cache: + extends: + - .dedicated-no-cache + - .no-docs + .no-docs-and-no-qa: except: refs: - /(^docs[\/-].+|.+-docs$)/ - /(^qa[\/-].*|.*-qa$)/ +# Jobs that shouldn't run for docs nor qa branches and do not need cache +.dedicated-no-docs-no-qa-no-cache: + extends: + - .dedicated-no-cache + - .no-docs-and-no-qa + .dedicated-no-docs-pull-cache-job: extends: - .dedicated-pull-cache-job @@ -49,9 +60,9 @@ - .dedicated-pull-cache-job - .no-docs-and-no-qa -# Jobs that do not need a DB -.dedicated-no-docs-no-db-pull-cache-job: - extends: .dedicated-no-docs-pull-cache-job +# Jobs that shouldn't run for docs and do not need DB nor cache +.dedicated-no-docs-no-db-no-cache: + extends: .dedicated-no-docs-no-cache variables: SETUP_DB: "false" diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index f7b18b809b4..cab84642828 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -1,5 +1,5 @@ pages: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: .dedicated-no-docs-no-db-no-cache before_script: [] stage: pages dependencies: diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 50476b43dd6..74bfe37b4bc 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -21,8 +21,10 @@ .gitlab-setup: &gitlab-setup extends: - - .dedicated-no-docs-and-no-qa-pull-cache-job + - .dedicated-no-docs-no-qa-no-cache - .use-pg + dependencies: + - setup-test-env variables: SETUP_DB: "false" script: @@ -39,13 +41,13 @@ - log/development.log .rake-exec: &rake-exec - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: .dedicated-no-docs-no-db-no-cache script: - bundle exec rake $CI_JOB_NAME .rspec-metadata: &rspec-metadata extends: - - .dedicated-pull-cache-job + - .dedicated-no-cache - .no-docs-and-no-qa stage: test script: @@ -94,14 +96,16 @@ # DB migration, rollback, and seed jobs .db-migrate-reset: &db-migrate-reset - extends: .dedicated-no-docs-and-no-qa-pull-cache-job + extends: .dedicated-no-docs-no-qa-no-cache script: - bundle exec rake db:migrate:reset dependencies: - setup-test-env .migration-paths: &migration-paths - extends: .dedicated-no-docs-and-no-qa-pull-cache-job + extends: .dedicated-no-docs-no-qa-no-cache + dependencies: + - setup-test-env variables: SETUP_DB: "false" script: @@ -121,8 +125,6 @@ - . scripts/prepare_build.sh - date - bundle exec rake db:migrate - dependencies: - - setup-test-env setup-test-env: extends: @@ -131,14 +133,18 @@ setup-test-env: - .use-pg stage: prepare script: + - date - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' + - date - scripts/gitaly-test-build # Do not use 'bundle exec' here + - date artifacts: expire_in: 7d paths: - tmp/tests - config/secrets.yml - - vendor/gitaly-ruby + - vendor/ruby/ + - vendor/gitaly-ruby/ rspec unit pg: <<: *rspec-metadata-pg @@ -185,32 +191,28 @@ rspec quarantine pg: allow_failure: true static-analysis: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: .dedicated-no-docs-no-db-no-cache dependencies: - compile-assets - compile-assets pull-cache - setup-test-env - script: - - scripts/static-analysis cache: - key: "debian-stretch-ruby-2.6.3-node-12.x-and-rubocop" + key: "rubocop_cache" paths: - - vendor/ruby - - .yarn-cache/ - tmp/rubocop_cache - policy: pull-push + script: + - scripts/static-analysis downtime_check: <<: *rake-exec - except: - refs: - - master - - tags - - /^[\d-]+-stable(-ee)?$/ - - /(^docs[\/-].+|.+-docs$)/ - - /(^qa[\/-].*|.*-qa$)/ dependencies: - setup-test-env + except: + - master + - tags + - /^[\d-]+-stable(-ee)?$/ + - /(^docs[\/-].+|.+-docs$)/ + - /(^qa[\/-].*|.*-qa$)/ ee_compat_check: <<: *rake-exec @@ -247,7 +249,7 @@ migration:path-pg: <<: *use-pg .db-rollback: &db-rollback - extends: .dedicated-no-docs-and-no-qa-pull-cache-job + extends: .dedicated-no-docs-no-qa-no-cache script: - bundle exec rake db:migrate VERSION=20180101160629 - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true @@ -261,20 +263,12 @@ db:rollback-pg: gitlab:setup-pg: <<: *gitlab-setup <<: *use-pg - dependencies: - - setup-test-env coverage: - # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to - # download artifacts from all the rspec jobs instead of from setup-test-env only - extends: - - .dedicated-runner-default-cache - - .no-docs-and-no-qa - cache: - policy: pull + extends: .dedicated-no-docs-no-qa-no-cache + stage: post-test variables: SETUP_DB: "false" - stage: post-test script: - bundle exec scripts/merge-simplecov - bundle exec scripts/gather-test-memory-data diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index ca55bbd32a7..35956c302f7 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -4,11 +4,16 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml code_quality: +<<<<<<< HEAD extends: .dedicated-no-docs +======= + extends: .dedicated-no-docs-no-db-no-cache +>>>>>>> Optimize the use of cache and dependencies in CI pipeline # gitlab-org runners set `privileged: false` but we need to have it set to true # since we're using Docker in Docker tags: [] before_script: [] +<<<<<<< HEAD cache: {} sast: @@ -24,3 +29,89 @@ dependency_scanning: tags: [] before_script: [] cache: {} +======= + dependencies: [] + +sast: + extends: .dedicated-no-docs-no-db-no-cache + image: docker:stable + variables: + SAST_CONFIDENCE_LEVEL: 2 + DOCKER_DRIVER: overlay2 + allow_failure: true + tags: [] + before_script: [] + dependencies: [] + services: + - docker:stable-dind + script: + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + - | + docker run \ + $(propagate_env_vars \ + SAST_ANALYZER_IMAGES \ + SAST_ANALYZER_IMAGE_PREFIX \ + SAST_ANALYZER_IMAGE_TAG \ + SAST_DEFAULT_ANALYZERS \ + SAST_BRAKEMAN_LEVEL \ + SAST_GOSEC_LEVEL \ + SAST_FLAWFINDER_LEVEL \ + SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ + SAST_RUN_ANALYZER_TIMEOUT \ + ) \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code + artifacts: + reports: + sast: gl-sast-report.json + +dependency_scanning: + extends: .dedicated-no-docs-no-db-no-cache + image: docker:stable + variables: + DOCKER_DRIVER: overlay2 + allow_failure: true + tags: [] + before_script: [] + dependencies: [] + services: + - docker:stable-dind + script: + - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + DS_ANALYZER_IMAGES \ + DS_ANALYZER_IMAGE_PREFIX \ + DS_ANALYZER_IMAGE_TAG \ + DS_DEFAULT_ANALYZERS \ + DEP_SCAN_DISABLE_REMOTE_CHECKS \ + DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ + DS_PULL_ANALYZER_IMAGE_TIMEOUT \ + DS_RUN_ANALYZER_TIMEOUT \ + ) \ + --volume "$PWD:/code" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json +>>>>>>> Optimize the use of cache and dependencies in CI pipeline diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 4a9269ffd82..33ec6a9a98b 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -27,8 +27,8 @@ extends: .dedicated-runner <<: *review-only image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - cache: {} dependencies: [] + cache: {} before_script: - source scripts/utils.sh @@ -254,11 +254,16 @@ schedule:review-cleanup: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb danger-review: - extends: .dedicated-pull-cache-job + extends: .dedicated-no-cache image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger stage: test - dependencies: [] - before_script: [] + dependencies: + - setup-test-env + - compile-assets + before_script: + - git version + - node --version + - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline only: variables: - $DANGER_GITLAB_API_TOKEN @@ -271,7 +276,4 @@ danger-review: - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/ - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/ script: - - git version - - node --version - - yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline - danger --fail-on-errors=true diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index c1fc3a893ca..635507c8ebf 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -1,7 +1,7 @@ # Insurance in case a gem needed by one of our releases gets yanked from # rubygems.org in the future. cache gems: - extends: .dedicated-no-docs-no-db-pull-cache-job + extends: .dedicated-no-docs-no-db-no-cache script: - bundle package --all --all-platforms artifacts: diff --git a/scripts/gitaly_test.rb b/scripts/gitaly_test.rb index b5cc5118530..b528e7cbbb8 100644 --- a/scripts/gitaly_test.rb +++ b/scripts/gitaly_test.rb @@ -31,7 +31,7 @@ module GitalyTest if ENV['CI'] bundle_path = File.expand_path('../vendor/gitaly-ruby', __dir__) - env_hash['BUNDLE_FLAGS'] << " --path=#{bundle_path}" + env_hash['BUNDLE_FLAGS'] << " --path=#{bundle_path} --quiet" end env_hash -- cgit v1.2.3