diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-10 18:08:00 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-10 18:08:00 +0300 |
commit | 609943de5e2ee6c3bf0f09d3fb1d5fc38ed5a4ed (patch) | |
tree | d6c366584972d84f9171881fa5977e59b123bff6 /scripts | |
parent | be2696666feee6e1045e0991309b71ce7f38a413 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/build_assets_image | 78 | ||||
-rwxr-xr-x | scripts/trigger-build.rb | 3 | ||||
-rw-r--r-- | scripts/utils.sh | 17 |
3 files changed, 80 insertions, 18 deletions
diff --git a/scripts/build_assets_image b/scripts/build_assets_image index 8aa6526061a..7482a170fe7 100755 --- a/scripts/build_assets_image +++ b/scripts/build_assets_image @@ -1,36 +1,82 @@ +#!/bin/sh + +. scripts/utils.sh + # Exit early if we don't want to build the image -if [[ "${BUILD_ASSETS_IMAGE}" != "true" ]] +if [ "${BUILD_ASSETS_IMAGE}" != "true" ] then exit 0 fi +get_repository_id() { + repository_name="${1}" + repositories_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories" + + curl --header "PRIVATE-TOKEN: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" "${repositories_url}" | jq "map(select(.name == \"${repository_name}\")) | .[0].id" +} + # Generate the image name based on the project this is being run in ASSETS_IMAGE_NAME="gitlab-assets-ce" + # `dev.gitlab-org` still has gitlab-ee. -if [[ "${CI_PROJECT_NAME}" == "gitlab" ]] || [[ "${CI_PROJECT_NAME}" == "gitlab-ee" ]] +if [ "${CI_PROJECT_NAME}" = "gitlab" ] || [ "${CI_PROJECT_NAME}" = "gitlab-ee" ] then ASSETS_IMAGE_NAME="gitlab-assets-ee" fi -ASSETS_IMAGE_PATH=${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME} +ASSETS_IMAGE_PATH="${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME}" +COMMIT_ASSETS_HASH_TAG="$(assets_image_tag)" +COMMIT_ASSETS_HASH_DESTINATION="${ASSETS_IMAGE_PATH}:${COMMIT_ASSETS_HASH_TAG}" -mkdir -p assets_container.build/public -cp -r public/assets assets_container.build/public/ -cp Dockerfile.assets assets_container.build/ +DESTINATIONS="--destination=${COMMIT_ASSETS_HASH_DESTINATION}" + +SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="true" -COMMIT_REF_SLUG_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG} +# Also tag the image with GitLab version, if running on a tag pipeline +# (and thus skip the performance optimization in that case), for back-compatibility. +if [ -n "${CI_COMMIT_TAG}" ]; then + COMMIT_REF_NAME_DESTINATION="${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}" + DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION" + SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="false" +fi -COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA} -COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME} +# The auto-deploy branch process still fetch assets image tagged with $CI_COMMIT_SHA, +# so we need to push the image with it (and thus skip the performance optimization in that case), +# for back-compatibility. +if echo "${CI_COMMIT_BRANCH}" | grep -Eq "^[0-9]+-[0-9]+-auto-deploy-[0-9]+$"; then + COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA} + DESTINATIONS="$DESTINATIONS --destination=$COMMIT_SHA_DESTINATION" + SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST="false" +fi -DESTINATIONS="--destination=$COMMIT_REF_SLUG_DESTINATION --destination=$COMMIT_SHA_DESTINATION" +if [ "${SKIP_ASSETS_IMAGE_BUILDING_IF_ALREADY_EXIST}" = "true" ] && [ -n "${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" ]; then + echoinfo "Checking if the ${COMMIT_ASSETS_HASH_DESTINATION} image exists..." + repository_id=$(get_repository_id "${ASSETS_IMAGE_NAME}") -# Also tag the image with GitLab version, if running on a tag pipeline, so -# other projects can simply use that instead of computing the slug. -if [ -n "$CI_COMMIT_TAG" ]; then - DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION" + if [ -n "${repository_id}" ]; then + api_image_url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${repository_id}/tags/${COMMIT_ASSETS_HASH_TAG}" + echoinfo "api_image_url: ${api_image_url}" + + if test_url "${api_image_url}" "--header \"PRIVATE-TOKEN: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}\""; then + echosuccess "Image ${COMMIT_ASSETS_HASH_DESTINATION} already exists, no need to rebuild it." + exit 0 + else + echoinfo "Image ${COMMIT_ASSETS_HASH_DESTINATION} doesn't exist, we'll need to build it." + fi + else + echoerr "Repository ID couldn't be found for the '${ASSETS_IMAGE_NAME}' image!" + fi +else + echoinfo "The 'PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE' variable is not present, so we cannot check if the image already exists." fi -echo "building assets image for destinations: $DESTINATIONS" +mkdir -p assets_container.build/public +cp -r public/assets assets_container.build/public/ +cp Dockerfile.assets assets_container.build/ + +echo "Building assets image for destinations: ${DESTINATIONS}" -/kaniko/executor --context=assets_container.build --dockerfile=assets_container.build/Dockerfile.assets $DESTINATIONS +/kaniko/executor \ + --context="assets_container.build" \ + --dockerfile="assets_container.build/Dockerfile.assets" \ + ${DESTINATIONS} diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index 897ca9f473e..8dfab8dd2eb 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -160,6 +160,8 @@ module Trigger end class CNG < Base + ASSETS_HASH = "cached-assets-hash.txt" + def variables # Delete variables that aren't useful when using native triggers. super.tap do |hash| @@ -187,7 +189,6 @@ module Trigger "TRIGGER_BRANCH" => ref, "GITLAB_VERSION" => ENV['CI_COMMIT_SHA'], "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. - "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_SHA'], "FORCE_RAILS_IMAGE_BUILDS" => 'true', "CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. "EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. diff --git a/scripts/utils.sh b/scripts/utils.sh index ea2b390f249..dae65ac8156 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -15,9 +15,11 @@ function retry() { function test_url() { local url="${1}" + local curl_args="${2}" local status + local cmd="curl ${curl_args} --output /dev/null -L -s -w ''%{http_code}'' \"${url}\"" - status=$(curl --output /dev/null -L -s -w ''%{http_code}'' "${url}") + status=$(eval "${cmd}") if [[ $status == "200" ]]; then return 0 @@ -203,3 +205,16 @@ function danger_as_local() { # We need to base SHA to help danger determine the base commit for this shallow clone. bundle exec danger dry_run --fail-on-errors=true --verbose --base="${CI_MERGE_REQUEST_DIFF_BASE_SHA}" --head="${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --dangerfile="${DANGER_DANGERFILE:-Dangerfile}" } + +# We're inlining this function in `.gitlab/ci/package-and-test/main.gitlab-ci.yml` since this file can be included in other projects. +function assets_image_tag() { + local cache_assets_hash_file="cached-assets-hash.txt" + + if [[ -n "${CI_COMMIT_TAG}" ]]; then + echo -n "${CI_COMMIT_REF_NAME}" + elif [[ -f "${cache_assets_hash_file}" ]]; then + echo -n "assets-hash-$(cat ${cache_assets_hash_file} | cut -c1-10)" + else + echo -n "${CI_COMMIT_SHA}" + fi +} |