diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-24 12:11:46 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-24 12:11:46 +0300 |
commit | 8883c54feaf1ca2d5a859ecd634043b36dfb8832 (patch) | |
tree | 22cc323511d7fde2e3f3c50ba13b50817c5a7bb1 /scripts | |
parent | 555532c942a339983ab09730be3f1b72eaec38d1 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/build_assets_image | 75 | ||||
-rwxr-xr-x | scripts/trigger-build.rb | 3 | ||||
-rw-r--r-- | scripts/utils.sh | 17 |
3 files changed, 75 insertions, 20 deletions
diff --git a/scripts/build_assets_image b/scripts/build_assets_image index 8aa6526061a..ee8623c826e 100755 --- a/scripts/build_assets_image +++ b/scripts/build_assets_image @@ -1,36 +1,75 @@ +#!/bin/sh + +set -e + +# This script builds an image that contains assets, that's then used by: +# - The `CNG` downstream pipelines (triggered from `gitlab-org/gitlab` via the `review-build-cng` job): +# https://gitlab.com/gitlab-org/gitlab/-/blob/c34e0834b01cd45c1f69a01b5e38dd6bc505f903/.gitlab/ci/review-apps/main.gitlab-ci.yml#L69. +# - The `omnibus-gitlab` downstream pipelines (triggered from `gitlab-org/gitlab` via the `e2e:package-and-test` job): +# https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/dfd1ad475868fc84e91ab7b5706aa03e46dc3a86/.gitlab-ci.yml#L130. +# - The `gitlab-org/charts/gitlab` `master` pipelines via `gitlab-org/build/CNG`, +# which pull `registry.gitlab.com/gitlab-org/gitlab/gitlab-assets-ee:master` +# - The `omnibus-gitlab` and CNG `master`/stable-branch pipelines, for both gitlab.com and dev.gitlab.org, +# which pull `registry.gitlab.com/gitlab-org/gitlab/gitlab-assets-ee:${CI_COMMIT_REF_SLUG}`. +# - The `omnibus-gitlab` tag pipelines, for both gitlab.com and dev.gitlab.org, +# which pull `registry.gitlab.com/gitlab-org/gitlab/gitlab-assets-ee:${CI_COMMIT_REF_SLUG}`. +# - The CNG tag pipelines, for both gitlab.com and dev.gitlab.org, +# which pull `registry.gitlab.com/gitlab-org/gitlab/gitlab-assets-ee:${CI_COMMIT_REF_NAME}`. +# - The auto-deploy pipelines, which pull `registry.gitlab.com/gitlab-org/gitlab/gitlab-assets-ee:${CI_COMMIT_SHA}`. + +. scripts/utils.sh + # Exit early if we don't want to build the image -if [[ "${BUILD_ASSETS_IMAGE}" != "true" ]] +if [ "${BUILD_ASSETS_IMAGE}" != "true" ] then exit 0 fi # Generate the image name based on the project this is being run in ASSETS_IMAGE_NAME="gitlab-assets-ce" + # `dev.gitlab-org` still has gitlab-ee. -if [[ "${CI_PROJECT_NAME}" == "gitlab" ]] || [[ "${CI_PROJECT_NAME}" == "gitlab-ee" ]] -then +if [ "${CI_PROJECT_NAME}" = "gitlab" ] || [ "${CI_PROJECT_NAME}" = "gitlab-ee" ]; then ASSETS_IMAGE_NAME="gitlab-assets-ee" fi -ASSETS_IMAGE_PATH=${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME} +ASSETS_IMAGE_PATH="${CI_REGISTRY}/${CI_PROJECT_PATH}/${ASSETS_IMAGE_NAME}" -mkdir -p assets_container.build/public -cp -r public/assets assets_container.build/public/ -cp Dockerfile.assets assets_container.build/ +# Used in MR pipelines +COMMIT_ASSETS_HASH_DESTINATION="${ASSETS_IMAGE_PATH}:$(assets_image_tag)" +# Used by other projects's master pipelines +COMMIT_REF_SLUG_DESTINATION="${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG}" +# Used by auto-deploy pipelines: https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md +COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA} +# Used for CNG tag pipelines +COMMIT_REF_NAME_DESTINATION="${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME}" -COMMIT_REF_SLUG_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_SLUG} +if skopeo inspect "docker://${COMMIT_ASSETS_HASH_DESTINATION}" > /dev/null; then + echosuccess "Image ${COMMIT_ASSETS_HASH_DESTINATION} already exists, no need to rebuild it." -COMMIT_SHA_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_SHA} -COMMIT_REF_NAME_DESTINATION=${ASSETS_IMAGE_PATH}:${CI_COMMIT_REF_NAME} + skopeo copy "docker://${COMMIT_ASSETS_HASH_DESTINATION}" "docker://${COMMIT_REF_SLUG_DESTINATION}" + skopeo copy "docker://${COMMIT_ASSETS_HASH_DESTINATION}" "docker://${COMMIT_SHA_DESTINATION}" -DESTINATIONS="--destination=$COMMIT_REF_SLUG_DESTINATION --destination=$COMMIT_SHA_DESTINATION" + if [ -n "${CI_COMMIT_TAG}" ]; then + skopeo copy "docker://${COMMIT_ASSETS_HASH_DESTINATION}" "docker://${COMMIT_REF_NAME_DESTINATION}" + fi +else + echoinfo "Image ${COMMIT_ASSETS_HASH_DESTINATION} doesn't exist, we'll need to build it." -# Also tag the image with GitLab version, if running on a tag pipeline, so -# other projects can simply use that instead of computing the slug. -if [ -n "$CI_COMMIT_TAG" ]; then - DESTINATIONS="$DESTINATIONS --destination=$COMMIT_REF_NAME_DESTINATION" -fi + DESTINATIONS="--destination=${COMMIT_ASSETS_HASH_DESTINATION} --destination=${COMMIT_REF_SLUG_DESTINATION} --destination=${COMMIT_SHA_DESTINATION}" -echo "building assets image for destinations: $DESTINATIONS" + if [ -n "${CI_COMMIT_TAG}" ]; then + DESTINATIONS="$DESTINATIONS --destination=${COMMIT_REF_NAME_DESTINATION}" + fi -/kaniko/executor --context=assets_container.build --dockerfile=assets_container.build/Dockerfile.assets $DESTINATIONS + mkdir -p assets_container.build/public + cp -r public/assets assets_container.build/public/ + cp Dockerfile.assets assets_container.build/ + + echo "Building assets image for destinations: ${DESTINATIONS}" + + /kaniko/executor \ + --context="assets_container.build" \ + --dockerfile="assets_container.build/Dockerfile.assets" \ + ${DESTINATIONS} +fi diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index 897ca9f473e..8dfab8dd2eb 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -160,6 +160,8 @@ module Trigger end class CNG < Base + ASSETS_HASH = "cached-assets-hash.txt" + def variables # Delete variables that aren't useful when using native triggers. super.tap do |hash| @@ -187,7 +189,6 @@ module Trigger "TRIGGER_BRANCH" => ref, "GITLAB_VERSION" => ENV['CI_COMMIT_SHA'], "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. - "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_SHA'], "FORCE_RAILS_IMAGE_BUILDS" => 'true', "CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. "EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. diff --git a/scripts/utils.sh b/scripts/utils.sh index 50ca7f558f6..b9b40d3dbd5 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -15,9 +15,11 @@ function retry() { function test_url() { local url="${1}" + local curl_args="${2}" local status + local cmd="curl ${curl_args} --output /dev/null -L -s -w ''%{http_code}'' \"${url}\"" - status=$(curl --output /dev/null -L -s -w ''%{http_code}'' "${url}") + status=$(eval "${cmd}") if [[ $status == "200" ]]; then return 0 @@ -203,3 +205,16 @@ function danger_as_local() { # We need to base SHA to help danger determine the base commit for this shallow clone. bundle exec danger dry_run --fail-on-errors=true --verbose --base="${CI_MERGE_REQUEST_DIFF_BASE_SHA}" --head="${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --dangerfile="${DANGER_DANGERFILE:-Dangerfile}" } + +# We're inlining this function in `.gitlab/ci/package-and-test/main.gitlab-ci.yml` so make sure to reflect any changes there +function assets_image_tag() { + local cache_assets_hash_file="cached-assets-hash.txt" + + if [[ -n "${CI_COMMIT_TAG}" ]]; then + echo -n "${CI_COMMIT_REF_NAME}" + elif [[ -f "${cache_assets_hash_file}" ]]; then + echo -n "assets-hash-$(cat ${cache_assets_hash_file} | cut -c1-10)" + else + echo -n "${CI_COMMIT_SHA}" + fi +} |