Add latest changes from gitlab-org/gitlab@master

5 files changed, 125 insertions, 90 deletions

diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index 5e9aea52d96..33aab8554e7 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -58,31 +58,27 @@ compile-test-assets as-if-foss:
 update-assets-compile-production-cache:
   extends:
     - compile-production-assets
+    - .assets-compile-cache-push
     - .shared:rules:update-cache
   stage: prepare
   artifacts: {}  # This job's purpose is only to update the cache.
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 update-assets-compile-test-cache:
   extends:
     - compile-test-assets
+    - .assets-compile-cache-push
     - .shared:rules:update-cache
   stage: prepare
   artifacts: {}  # This job's purpose is only to update the cache.
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 update-yarn-cache:
   extends:
     - .default-retry
-    - .yarn-cache
+    - .yarn-cache-push
     - .shared:rules:update-cache
   stage: prepare
   script:
     - *yarn-install
-  cache:
-    policy: push
 
 .frontend-fixtures-base:
   extends:
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml
index 0cadce5cc20..489b02fe2f6 100644
--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -16,75 +16,147 @@
     - source scripts/utils.sh
     - source scripts/prepare_build.sh
 
+.ruby-gems-cache: &ruby-gems-cache
+  key: "ruby-gems-v1"
+  paths:
+    - vendor/ruby/
+  policy: pull
+
+.ruby-gems-cache-push: &ruby-gems-cache-push
+  <<: *ruby-gems-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.gitaly-ruby-gems-cache: &gitaly-ruby-gems-cache
+  key: "gitaly-ruby-gems-v1"
+  paths:
+    - vendor/gitaly-ruby/
+  policy: pull
+
+.gitaly-ruby-gems-cache-push: &gitaly-ruby-gems-cache-push
+  <<: *gitaly-ruby-gems-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.go-pkg-cache: &go-pkg-cache
+  key: "go-pkg-v1"
+  paths:
+    - .go/pkg/mod/
+  policy: pull
+
+.go-pkg-cache-push: &go-pkg-cache-push
+  <<: *go-pkg-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.node-modules-cache: &node-modules-cache
+  key: "node-modules-${NODE_ENV}-v1"
+  paths:
+    - node_modules/
+    - tmp/cache/webpack-dlls/
+  policy: pull
+
+.node-modules-cache-push: &node-modules-cache-push
+  <<: *node-modules-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.assets-cache: &assets-cache
+  key: "assets-${NODE_ENV}-v1"
+  paths:
+    - assets-hash.txt
+    - public/assets/webpack/
+    - tmp/cache/assets/sprockets/
+    - tmp/cache/babel-loader/
+    - tmp/cache/vue-loader/
+  policy: pull
+
+.assets-cache-push: &assets-cache-push
+  <<: *assets-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.rubocop-cache: &rubocop-cache
+  key: "rubocop-v1"
+  paths:
+    - tmp/rubocop_cache/
+  policy: pull
+
+.rubocop-cache-push: &rubocop-cache-push
+  <<: *rubocop-cache
+  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
+  # for keeping only the N latest cache files, so we take advantage of it with `pull-push`.
+  policy: pull-push
+
+.qa-ruby-gems-cache: &qa-ruby-gems-cache
+  key: "qa-ruby-gems-v1"
+  paths:
+    - qa/vendor/ruby/
+  policy: pull
+
+.qa-ruby-gems-cache-push: &qa-ruby-gems-cache-push
+  <<: *qa-ruby-gems-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
 .setup-test-env-cache:
   cache:
-    key: "setup-test-env-v1"
-    paths:
-      - vendor/ruby/
-      - vendor/gitaly-ruby/
-      - .go/pkg/mod/
-    policy: pull
+    - *ruby-gems-cache
+    - *gitaly-ruby-gems-cache
+    - *go-pkg-cache
+
+.setup-test-env-cache-push:
+  cache:
+    - *ruby-gems-cache-push
+    - *gitaly-ruby-gems-cache-push
+    - *go-pkg-cache-push
 
 .rails-cache:
   cache:
-    key: "rails-v5"
-    paths:
-      - vendor/ruby/
-      - vendor/gitaly-ruby/
-    policy: pull
+    - *ruby-gems-cache
+    - *gitaly-ruby-gems-cache
 
 .static-analysis-cache:
   cache:
-    key: "static-analysis-v2"
-    paths:
-      - vendor/ruby/
-      - node_modules/
-      - tmp/rubocop_cache/
-    policy: pull
+    - *ruby-gems-cache
+    - *node-modules-cache
+    - *rubocop-cache
+
+.static-analysis-cache-push:
+  cache:
+    - *ruby-gems-cache  # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
+    - *rubocop-cache-push
 
 .coverage-cache:
   cache:
-    key: "coverage-cache-v1"
-    paths:
-      - vendor/ruby/
-    policy: pull
+    - *ruby-gems-cache
 
 .danger-review-cache:
   cache:
-    key: "danger-review-v1"
-    paths:
-      - vendor/ruby/
-      - node_modules/
-    policy: pull
+    - *ruby-gems-cache
+    - *node-modules-cache
 
 .qa-cache:
   cache:
-    key: "qa-v2"
-    paths:
-      - qa/vendor/ruby/
-    policy: pull
+    - *qa-ruby-gems-cache
+
+.qa-cache-push:
+  cache:
+    - *qa-ruby-gems-cache-push
 
 .yarn-cache:
   cache:
-    key: "yarn-v1"
-    paths:
-      - node_modules/
-      - tmp/cache/webpack-dlls/
-    policy: pull
+    - *node-modules-cache
+
+.yarn-cache-push:
+  cache:
+    - *node-modules-cache-push
 
 .assets-compile-cache:
   cache:
-    key: "assets-compile-${NODE_ENV}-v1"
-    paths:
-      - vendor/ruby/
-      - node_modules/
-      - assets-hash.txt
-      - public/assets/webpack/
-      - tmp/cache/assets/sprockets/
-      - tmp/cache/babel-loader/
-      - tmp/cache/vue-loader/
-      - tmp/cache/webpack-dlls/
-    policy: pull
+    - *ruby-gems-cache
+    - *node-modules-cache
+    - *assets-cache
+
+.assets-compile-cache-push:
+  cache:
+    - *ruby-gems-cache  # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
+    - *node-modules-cache-push
+    - *assets-cache-push
 
 .use-pg11:
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36"
diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml
index 5097fd28eeb..8bbd7dbf075 100644
--- a/.gitlab/ci/qa.gitlab-ci.yml
+++ b/.gitlab/ci/qa.gitlab-ci.yml
@@ -41,12 +41,11 @@ qa:selectors-as-if-foss:
 update-qa-cache:
   extends:
     - .qa-job-base
+    - .qa-cache-push
     - .shared:rules:update-cache
   stage: prepare
   script:
     - echo "Cache has been updated and ready to be uploaded."
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 .package-and-qa-base:
   image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 4a3cbf89edb..5cd64baf4d3 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -187,18 +187,10 @@ setup-test-env:
 update-setup-test-env-cache:
   extends:
     - setup-test-env
+    - .setup-test-env-cache-push
     - .shared:rules:update-cache
   artifacts:
     paths: []  # This job's purpose is only to update the cache.
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-
-update-rails-cache:
-  extends:
-    - update-setup-test-env-cache
-    - .rails-cache
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 .coverage-base:
   extends:
@@ -209,16 +201,6 @@ update-rails-cache:
     SETUP_DB: "false"
     USE_BUNDLE_INSTALL: "false"
 
-update-coverage-cache:
-  extends:
-    - .coverage-base
-    - .shared:rules:update-cache
-  stage: prepare
-  script:
-    - !reference [.minimal-bundle-install, script]
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-
 .static-analysis-base:
   extends:
     - .default-retry
@@ -232,16 +214,11 @@ update-coverage-cache:
 update-static-analysis-cache:
   extends:
     - .static-analysis-base
+    - .static-analysis-cache-push
     - .shared:rules:update-cache
   stage: prepare
   script:
-    - rm -rf ./node_modules  # We remove node_modules because there's no mechanism to remove stall entries.
-    - run_timed_command "retry yarn install --frozen-lockfile"
-    - run_timed_command "bundle exec rubocop --parallel"  # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks,
-  cache:
-    # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
-    # for keeping only the N latest cache files, so we take advantage of it with `pull-push` and removing `node_modules` at the start of the job.
-    policy: pull-push
+    - run_timed_command "bundle exec rubocop --parallel"  # For the moment we only cache `tmp/rubocop_cache` so we don't need to run all the tasks.
 
 static-analysis:
   extends:
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 1eccc9cabfa..5decc83da2b 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -230,12 +230,3 @@ danger-review:
       else
         run_timed_command "bundle exec danger --fail-on-errors=true --verbose"
       fi
-
-update-danger-review-cache:
-  extends:
-    - danger-review
-    - .shared:rules:update-cache
-  stage: prepare
-  script: echo 'Cache is fresh!'
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.