Add latest changes from gitlab-org/gitlab@master

5 files changed, 131 insertions, 47 deletions

diff --git a/.gitlab/ci/caching.gitlab-ci.yml b/.gitlab/ci/caching.gitlab-ci.yml
new file mode 100644
index 00000000000..c7b8d49e9d1
--- /dev/null
+++ b/.gitlab/ci/caching.gitlab-ci.yml
@@ -0,0 +1,55 @@
+cache-workhorse:
+  extends:
+    - .default-retry
+    - .default-before_script
+    - .rails-cache
+    - .setup-test-env-cache
+    - .caching:rules:cache-workhorse
+  stage: prepare
+  variables:
+    SETUP_DB: "false"
+  script:
+    - source scripts/gitlab_component_helpers.sh
+    - 'gitlab_workhorse_archive_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }'
+    - run_timed_command "scripts/setup-test-env"
+    - run_timed_command "select_gitlab_workhorse_essentials"
+    - run_timed_command "create_gitlab_workhorse_package"
+    - run_timed_command "upload_gitlab_workhorse_package"
+  artifacts:
+    expire_in: 7d
+    paths:
+      - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/
+
+.cache-assets-base:
+  extends:
+    - .compile-assets-base
+    - .caching:rules:cache-assets
+  stage: prepare
+  variables:
+    WEBPACK_REPORT: "false"
+  script:
+    - !reference [.yarn-install, script]
+    - export GITLAB_ASSETS_HASH=$(bundle exec rake gitlab:assets:hash_sum | tee assets-hash.txt)
+    - source scripts/gitlab_component_helpers.sh
+    - 'gitlab_assets_archive_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }'
+    # If we still use the legacy cache, we don't want to create a package from it as we don't check the sha sum in gitlab:assets:compile_with_new_strategy.
+    # This line can be removed once we stop downloading the legacy cache.
+    - rm -rf public/assets
+    - run_timed_command "bin/rake gitlab:assets:compile_with_new_strategy"
+    - run_timed_command "scripts/clean-old-cached-assets"
+    - run_timed_command "create_gitlab_assets_package"
+    - run_timed_command "upload_gitlab_assets_package"
+
+cache-assets:test:
+  extends: .cache-assets-base
+
+cache-assets:test as-if-foss:
+  extends:
+    - .cache-assets-base
+    - .as-if-foss
+
+cache-assets:production:
+  extends: .cache-assets-base
+  variables:
+    NODE_ENV: "production"
+    RAILS_ENV: "production"
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index 3bd65b565e4..fc2bf5540e4 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -1,10 +1,12 @@
-.yarn-install: &yarn-install
-  - source scripts/utils.sh
-  - run_timed_command "retry yarn install --frozen-lockfile"
+.yarn-install:
+  script:
+    - source scripts/utils.sh
+    - run_timed_command "retry yarn install --frozen-lockfile"
 
-.storybook-yarn-install: &storybook-yarn-install
-  - source scripts/utils.sh
-  - run_timed_command "retry yarn run storybook:install --frozen-lockfile"
+.storybook-yarn-install:
+  script:
+    - source scripts/utils.sh
+    - run_timed_command "retry yarn run storybook:install --frozen-lockfile"
 
 .compile-assets-base:
   extends:
@@ -21,8 +23,17 @@
     WEBPACK_COMPILE_LOG_PATH: "tmp/webpack-output.log"
   stage: prepare
   script:
-    - *yarn-install
-    - run_timed_command "bin/rake gitlab:assets:compile"
+    - !reference [.yarn-install, script]
+    - |
+      # The new strategy to cache assets as generic packages is experimental and can be disabled by removing the `CACHE_ASSETS_AS_PACKAGE` variable
+      if [[ "${CACHE_ASSETS_AS_PACKAGE}" == "true" ]]; then
+        export GITLAB_ASSETS_HASH=$(bundle exec rake gitlab:assets:hash_sum | tee assets-hash.txt)
+        source scripts/gitlab_component_helpers.sh
+        run_timed_command "download_and_extract_gitlab_assets" || true
+        run_timed_command "bin/rake gitlab:assets:compile_with_new_strategy"
+      else
+        run_timed_command "bin/rake gitlab:assets:compile"
+      fi
     - run_timed_command "scripts/clean-old-cached-assets"
 
 compile-production-assets:
@@ -42,8 +53,6 @@ compile-production-assets:
       - public/assets/
       - "${WEBPACK_COMPILE_LOG_PATH}"
     when: always
-  before_script:
-    - !reference [.default-before_script, before_script]
   after_script:
     - rm -f /etc/apt/sources.list.d/google*.list  # We don't need to update Chrome here
 
@@ -88,7 +97,7 @@ update-yarn-cache:
     - .shared:rules:update-cache
   stage: prepare
   script:
-    - *yarn-install
+    - !reference [.yarn-install, script]
 
 update-storybook-yarn-cache:
   extends:
@@ -97,7 +106,7 @@ update-storybook-yarn-cache:
     - .shared:rules:update-cache
   stage: prepare
   script:
-    - *storybook-yarn-install
+    - !reference [.storybook-yarn-install, script]
 
 .frontend-fixtures-base:
   extends:
@@ -180,7 +189,7 @@ graphql-schema-dump as-if-foss:
     SETUP_DB: "false"
   before_script:
     - !reference [.default-before_script, before_script]
-    - *yarn-install
+    - !reference [.yarn-install, script]
   stage: test
 
 .jest-base:
@@ -256,7 +265,7 @@ coverage-frontend:
       optional: true
   stage: post-test
   before_script:
-    - *yarn-install
+    - !reference [.yarn-install, script]
   script:
     - run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js"
     # Removing the individual coverage results, as we just merged them.
@@ -282,7 +291,7 @@ coverage-frontend:
   stage: test
   dependencies: []
   script:
-    - *yarn-install
+    - !reference [.yarn-install, script]
     - run_timed_command "retry yarn run webpack-prod"
 
 qa-frontend-node:14:
@@ -310,7 +319,7 @@ webpack-dev-server:
     WEBPACK_MEMORY_TEST: "true"
     WEBPACK_VENDOR_DLL: "true"
   script:
-    - *yarn-install
+    - !reference [.yarn-install, script]
     - run_timed_command "retry yarn webpack-vendor"
     - run_timed_command "node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js"
   artifacts:
@@ -328,7 +337,7 @@ bundle-size-review:
   stage: test
   needs: []
   script:
-    - *yarn-install
+    - !reference [.yarn-install, script]
     - scripts/bundle_size_review
   artifacts:
     when: always
@@ -366,7 +375,7 @@ startup-css-check as-if-foss:
     - .frontend-test-base
     - .storybook-yarn-cache
   script:
-    - *storybook-yarn-install
+    - !reference [.storybook-yarn-install, script]
     - yarn run storybook:build
   needs: ["graphql-schema-dump"]
 
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml
index 7bdca62b68e..9418d2462d3 100644
--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -79,10 +79,12 @@
   policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 .assets-cache: &assets-cache
-  key: "assets-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}-v2"
+  key: "assets-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}-v3"
+  # This list should match GITLAB_ASSETS_PATHS_LIST from scripts/gitlab_component_helpers.sh
   paths:
     - assets-hash.txt
-    - public/assets/webpack/
+    - app/assets/javascripts/locale/**/app.js
+    - public/assets/
     - tmp/cache/assets/sprockets/
     - tmp/cache/babel-loader/
     - tmp/cache/vue-loader/
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index b2ce4152918..be84cf904d7 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -168,7 +168,7 @@ setup-test-env:
     SETUP_DB: "false"
   script:
     - echo $CI_MERGE_REQUEST_APPROVED
-    - source scripts/gitlab_workhorse_component_helpers.sh
+    - source scripts/gitlab_component_helpers.sh
     - run_timed_command "download_and_extract_gitlab_workhorse_package" || true
     - run_timed_command "scripts/setup-test-env"
     - run_timed_command "select_gitlab_workhorse_essentials"
@@ -197,22 +197,6 @@ setup-test-env:
       - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/
     when: always
 
-build-components:
-  extends:
-    - setup-test-env
-    - .rails:rules:build-components
-  script:
-    - source scripts/gitlab_workhorse_component_helpers.sh
-    - 'gitlab_workhorse_archive_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }'
-    - run_timed_command "scripts/setup-test-env"
-    - run_timed_command "select_gitlab_workhorse_essentials"
-    - run_timed_command "create_gitlab_workhorse_package"
-    - run_timed_command "upload_gitlab_workhorse_package"
-  artifacts:
-    expire_in: 7d
-    paths:
-      - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/
-
 update-setup-test-env-cache:
   extends:
     - setup-test-env
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index 203aa779e5b..e2bf3e02eea 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -249,8 +249,21 @@
   - "babel.config.js"
   - "config/webpack.config.js"
   - "config/**/*.js"
+  - "{,ee/,jh/}app/assets/**/*"
   - "vendor/assets/**/*"
+
+# This list should match the list in Tasks::Gitlab::Assets.assets_impacting_compilation
+.assets-compilation-patterns: &assets-compilation-patterns
+  - "{package.json,yarn.lock}"
+  - ".browserslistrc"
+  - "babel.config.js"
+  - "config/webpack.config.js"
+  - "*.js"
+  - "config/**/*.js"
+  - "locale/**/gitlab.po"
   - "{,ee/,jh/}app/assets/**/*"
+  - "fixtures/emojis/**/*"
+  - "vendor/assets/**/*"
 
 .frontend-patterns-for-as-if-foss: &frontend-patterns-for-as-if-foss
   - "{package.json,yarn.lock}"
@@ -652,6 +665,37 @@
     - changes: *ci-build-images-patterns
     - changes: *code-qa-patterns
 
+#################
+# Caching rules #
+#################
+.caching:rules:cache-workhorse:
+  rules:
+    - <<: *if-dot-com-ee-schedule
+    - <<: *if-dot-com-gitlab-org-default-branch
+      changes: ["workhorse/**/*"]
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes:
+        - ".gitlab/ci/caching.gitlab-ci.yml"
+        - "scripts/gitlab_component_helpers.sh"
+      when: manual
+      allow_failure: true
+
+.caching:rules:cache-assets:
+  rules:
+    # The new strategy to cache assets as generic packages is experimental and can be disabled by removing the `CACHE_ASSETS_AS_PACKAGE` variable
+    - if: '$CACHE_ASSETS_AS_PACKAGE != "true"'
+      when: never
+    - <<: *if-dot-com-ee-schedule
+    - <<: *if-dot-com-gitlab-org-default-branch
+      changes: *assets-compilation-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes:
+        - ".gitlab/ci/caching.gitlab-ci.yml"
+        - "scripts/gitlab_component_helpers.sh"
+        - "lib/tasks/gitlab/assets.rake"
+      when: manual
+      allow_failure: true
+
 ######################
 # CI Templates Rules #
 ######################
@@ -969,16 +1013,6 @@
 ###############
 # Rails rules #
 ###############
-.rails:rules:build-components:
-  rules:
-    - <<: *if-dot-com-ee-schedule
-    - <<: *if-dot-com-gitlab-org-default-branch
-      changes:
-        - "workhorse/**/*"
-    - <<: *if-dot-com-gitlab-org-merge-request
-      when: manual
-      allow_failure: true
-
 .rails:rules:setup-test-env:
   rules:
     - changes: *setup-test-env-patterns