Add latest changes from gitlab-org/gitlab@master

3 files changed, 80 insertions, 18 deletions

diff --git a/scripts/build_assets_image b/scripts/build_assets_image
index 8aa6526061a..7482a170fe7 100755
--- a/scripts/build_assets_image
+++ b/scripts/build_assets_image
@@ -1,36 +1,82 @@
+#!/bin/sh
+
+. scripts/utils.sh
+
 # Exit early if we don't want to build the image
-if [[ "${BUILD_ASSETS_IMAGE}" != "true" ]]
+if [ "${BUILD_ASSETS_IMAGE}" != "true" ]
 then
   exit 0
 fi
 
+get_repository_id() {
+  repository_name="${1}"
+  repositories_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories"
+
+  curl --header "PRIVATE-TOKEN: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" "${repositories_url}" | jq "map(select(.name == \"${repository_name}\")) | .[0].id"
+}
+
 # Generate the image name based on the project this is being run in
 ASSETS_IMAGE_NAME="gitlab-assets-ce"
+
 # `dev.gitlab-org` still has gitlab-ee.
-if [[ "${CI_PROJECT_NAME}" == "gitlab" ]] || [[ "${CI_PROJECT_NAME}" == "gitlab-ee" ]]
+if [ "${CI_PROJECT_NAME}" = "gitlab" ] || [ "${CI_PROJECT_NAME}" = "gitlab-ee" ]
 then
   ASSETS_IMAGE_NAME="gitlab-assets-ee"
 fi
 
-ASSETS_IMAGE_PATH=${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME}
+ASSETS_IMAGE_PATH="${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME}"
+COMMIT_ASSETS_HASH_TAG="$(assets_image_tag)"
+COMMIT_ASSETS_HASH_DESTINATION="${ASSETS_IMAGE_PATH}:${COMMIT_ASSETS_HASH_TAG}"
 
-mkdir -p assets_container.build/public
-cp -r public/assets assets_container.build/public/
-cp Dockerfile.assets assets_container.build/
+DESTINATIONS="--destination=${COMMIT_ASSETS_HASH_DESTINATION}"
+
+SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="true"
 
-COMMIT_REF_SLUG_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG}
+# Also tag the image with GitLab version, if running on a tag pipeline
+# (and thus skip the performance optimization in that case), for back-compatibility.
+if [ -n "${CI_COMMIT_TAG}" ]; then
+  COMMIT_REF_NAME_DESTINATION="${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}"
+  DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION"
+  SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="false"
+fi
 
-COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA}
-COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}
+# The auto-deploy branch process still fetch assets image tagged with $CI_COMMIT_SHA,
+# so we need to push the image with it (and thus skip the performance optimization in that case),
+# for back-compatibility.
+if echo "${CI_COMMIT_BRANCH}" | grep -Eq "^[0-9]+-[0-9]+-auto-deploy-[0-9]+$"; then
+  COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA}
+  DESTINATIONS="$DESTINATIONS --destination=$COMMIT_SHA_DESTINATION"
+  SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="false"
+fi
 
-DESTINATIONS="--destination=$COMMIT_REF_SLUG_DESTINATION --destination=$COMMIT_SHA_DESTINATION"
+if [ "${SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST}" = "true" ] && [ -n "${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" ]; then
+  echoinfo "Checking if the ${COMMIT_ASSETS_HASH_DESTINATION} image exists..."
+  repository_id=$(get_repository_id "${ASSETS_IMAGE_NAME}")
 
-# Also tag the image with GitLab version, if running on a tag pipeline, so
-# other projects can simply use that instead of computing the slug.
-if [ -n "$CI_COMMIT_TAG" ]; then
-  DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION"
+  if [ -n "${repository_id}" ]; then
+    api_image_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${repository_id}/tags/${COMMIT_ASSETS_HASH_TAG}"
+    echoinfo "api_image_url: ${api_image_url}"
+
+    if test_url "${api_image_url}" "--header \"PRIVATE-TOKEN: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}\""; then
+      echosuccess "Image ${COMMIT_ASSETS_HASH_DESTINATION} already exists, no need to rebuild it."
+      exit 0
+    else
+      echoinfo "Image ${COMMIT_ASSETS_HASH_DESTINATION} doesn't exist, we'll need to build it."
+    fi
+  else
+    echoerr "Repository ID couldn't be found for the '${ASSETS_IMAGE_NAME}' image!"
+  fi
+else
+  echoinfo "The 'PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE' variable is not present, so we cannot check if the image already exists."
 fi
 
-echo "building assets image for destinations: $DESTINATIONS"
+mkdir -p assets_container.build/public
+cp -r public/assets assets_container.build/public/
+cp Dockerfile.assets assets_container.build/
+
+echo "Building assets image for destinations: ${DESTINATIONS}"
 
-/kaniko/executor --context=assets_container.build --dockerfile=assets_container.build/Dockerfile.assets $DESTINATIONS
+/kaniko/executor \
+  --context="assets_container.build" \
+  --dockerfile="assets_container.build/Dockerfile.assets" \
+  ${DESTINATIONS}
diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb
index 897ca9f473e..8dfab8dd2eb 100755
--- a/scripts/trigger-build.rb
+++ b/scripts/trigger-build.rb
@@ -160,6 +160,8 @@ module Trigger
   end
 
   class CNG < Base
+    ASSETS_HASH = "cached-assets-hash.txt"
+
     def variables
       # Delete variables that aren't useful when using native triggers.
       super.tap do |hash|
@@ -187,7 +189,6 @@ module Trigger
         "TRIGGER_BRANCH" => ref,
         "GITLAB_VERSION" => ENV['CI_COMMIT_SHA'],
         "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
-        "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_SHA'],
         "FORCE_RAILS_IMAGE_BUILDS" => 'true',
         "CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
         "EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
diff --git a/scripts/utils.sh b/scripts/utils.sh
index ea2b390f249..dae65ac8156 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -15,9 +15,11 @@ function retry() {
 
 function test_url() {
   local url="${1}"
+  local curl_args="${2}"
   local status
+  local cmd="curl ${curl_args} --output /dev/null -L -s -w ''%{http_code}'' \"${url}\""
 
-  status=$(curl --output /dev/null -L -s -w ''%{http_code}'' "${url}")
+  status=$(eval "${cmd}")
 
   if [[ $status == "200" ]]; then
     return 0
@@ -203,3 +205,16 @@ function danger_as_local() {
   # We need to base SHA to help danger determine the base commit for this shallow clone.
   bundle exec danger dry_run --fail-on-errors=true --verbose --base="${CI_MERGE_REQUEST_DIFF_BASE_SHA}" --head="${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --dangerfile="${DANGER_DANGERFILE:-Dangerfile}"
 }
+
+# We're inlining this function in `.gitlab/ci/package-and-test/main.gitlab-ci.yml` since this file can be included in other projects.
+function assets_image_tag() {
+  local cache_assets_hash_file="cached-assets-hash.txt"
+
+  if [[ -n "${CI_COMMIT_TAG}" ]]; then
+    echo -n "${CI_COMMIT_REF_NAME}"
+  elif [[ -f "${cache_assets_hash_file}" ]]; then
+    echo -n "assets-hash-$(cat ${cache_assets_hash_file} | cut -c1-10)"
+  else
+    echo -n "${CI_COMMIT_SHA}"
+  fi
+}