diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/api/download_job_artifact.rb | 94 | ||||
| -rwxr-xr-x | scripts/build_qa_image | 27 | ||||
| -rw-r--r-- | scripts/rspec_helpers.sh | 96 |
3 files changed, 27 insertions, 190 deletions
diff --git a/scripts/api/download_job_artifact.rb b/scripts/api/download_job_artifact.rb deleted file mode 100755 index 394ad8f3a3d..00000000000 --- a/scripts/api/download_job_artifact.rb +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'optparse' -require 'fileutils' -require 'uri' -require 'cgi' -require 'net/http' -require_relative 'default_options' - -class ArtifactFinder - def initialize(options) - @project = options.delete(:project) - @job_id = options.delete(:job_id) - @api_token = options.delete(:api_token) - @endpoint = options.delete(:endpoint) || API::DEFAULT_OPTIONS[:endpoint] - @artifact_path = options.delete(:artifact_path) - - warn "No API token given." unless api_token - end - - def execute - url = "#{endpoint}/projects/#{CGI.escape(project)}/jobs/#{job_id}/artifacts" - - if artifact_path - FileUtils.mkdir_p(File.dirname(artifact_path)) - url += "/#{artifact_path}" - end - - fetch(url) - end - - private - - attr_reader :project, :job_id, :api_token, :endpoint, :artifact_path - - def fetch(uri_str, limit = 10) - raise 'Too many HTTP redirects' if limit == 0 - - uri = URI(uri_str) - request = Net::HTTP::Get.new(uri) - request['Private-Token'] = api_token if api_token - - Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http| - http.request(request) do |response| - case response - when Net::HTTPSuccess then - File.open(artifact_path || 'artifacts.zip', 'w') do |file| - response.read_body(&file.method(:write)) - end - when Net::HTTPRedirection then - location = response['location'] - warn "Redirected (#{limit - 1} redirections remaining)." - fetch(location, limit - 1) - else - raise "Unexpected response: #{response.value}" - end - end - end - end -end - -if $PROGRAM_NAME == __FILE__ - options = API::DEFAULT_OPTIONS.dup - - OptionParser.new do |opts| - opts.on("-p", "--project PROJECT", String, "Project where to find the job (defaults to $CI_PROJECT_ID)") do |value| - options[:project] = value - end - - opts.on("-j", "--job-id JOB_ID", String, "A job ID") do |value| - options[:job_id] = value - end - - opts.on("-a", "--artifact-path ARTIFACT_PATH", String, "A valid artifact path") do |value| - options[:artifact_path] = value - end - - opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope") do |value| - options[:api_token] = value - end - - opts.on("-E", "--endpoint ENDPOINT", String, "The API endpoint for the API token. (defaults to $CI_API_V4_URL and fallback to https://gitlab.com/api/v4)") do |value| - options[:endpoint] = value - end - - opts.on("-h", "--help", "Prints this help") do - puts opts - exit - end - end.parse! - - ArtifactFinder.new(options).execute -end diff --git a/scripts/build_qa_image b/scripts/build_qa_image index 4b7eb73e784..3728608e32c 100755 --- a/scripts/build_qa_image +++ b/scripts/build_qa_image @@ -1,9 +1,11 @@ -#!/bin/sh +#!/bin/bash QA_IMAGE_NAME="gitlab-ee-qa" +QA_BUILD_TARGET="ee" -if [ "${CI_PROJECT_NAME}" == "gitlabhq" ] || [ "${CI_PROJECT_NAME}" == "gitlab-foss" ]; then +if [[ "${CI_PROJECT_NAME}" == "gitlabhq" || "${CI_PROJECT_NAME}" == "gitlab-foss" ]]; then QA_IMAGE_NAME="gitlab-ce-qa" + QA_BUILD_TARGET="foss" fi # Tag with commit SHA by default @@ -15,24 +17,29 @@ IMAGE_TAG=${CI_COMMIT_TAG#v} IMAGE_TAG=${IMAGE_TAG:-$CI_COMMIT_REF_SLUG} QA_IMAGE_BRANCH="${CI_REGISTRY}/${CI_PROJECT_PATH}/${QA_IMAGE_NAME}:${IMAGE_TAG}" +QA_IMAGE_MASTER="${CI_REGISTRY}/${CI_PROJECT_PATH}/${QA_IMAGE_NAME}:master" -DESTINATIONS="--destination=${QA_IMAGE} --destination=${QA_IMAGE_BRANCH}" +DESTINATIONS="--tag ${QA_IMAGE} --tag ${QA_IMAGE_BRANCH}" # Auto-deploy tag format uses first 12 letters of commit SHA. Tag with that # reference also for EE images. if [ "${QA_IMAGE_NAME}" == "gitlab-ee-qa" ]; then QA_IMAGE_FOR_AUTO_DEPLOY="${CI_REGISTRY}/${CI_PROJECT_PATH}/${QA_IMAGE_NAME}:${CI_COMMIT_SHA:0:11}" - DESTINATIONS="${DESTINATIONS} --destination=$QA_IMAGE_FOR_AUTO_DEPLOY" + DESTINATIONS="${DESTINATIONS} --tag $QA_IMAGE_FOR_AUTO_DEPLOY" fi echo "Building QA image for destinations: ${DESTINATIONS}" -/kaniko/executor \ - --context="${CI_PROJECT_DIR}" \ - --dockerfile="${CI_PROJECT_DIR}/qa/Dockerfile" \ +docker buildx build \ + --cache-to=type=inline \ + --cache-from="$QA_IMAGE_BRANCH" \ + --cache-from="$QA_IMAGE_MASTER" \ + --platform=${ARCH:-amd64} \ --build-arg=CHROME_VERSION="${CHROME_VERSION}" \ --build-arg=DOCKER_VERSION="${DOCKER_VERSION}" \ --build-arg=RUBY_VERSION="${RUBY_VERSION}" \ - --build-arg=QA_BUILD_TARGET="${QA_BUILD_TARGET:-qa}" \ - --cache=true \ - ${DESTINATIONS} + --build-arg=QA_BUILD_TARGET="${QA_BUILD_TARGET}" \ + --file="${CI_PROJECT_DIR}/qa/Dockerfile" \ + --push \ + ${DESTINATIONS} \ + ${CI_PROJECT_DIR} diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index 14c5b94e921..cdf397ed72d 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -3,44 +3,14 @@ function retrieve_tests_metadata() { mkdir -p $(dirname "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}") $(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}") "${RSPEC_PROFILING_FOLDER_PATH}" - if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then - if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then - curl --location -o "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || - echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" - fi - - if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then - curl --location -o "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || - echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" - fi - else - # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to - # always target the canonical project here, so the branch must be hardcoded - local project_path="gitlab-org/gitlab" - local artifact_branch="master" - local username="gitlab-bot" - local job_name="update-tests-metadata" - local test_metadata_job_id - - # Ruby - test_metadata_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=${username}" -Q "scope=success" --job-name "${job_name}") - - if [[ -n "${test_metadata_job_id}" ]]; then - echo "test_metadata_job_id: ${test_metadata_job_id}" - - if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then - scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" - fi - - if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then - scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || - echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" - fi - else - echo "test_metadata_job_id couldn't be found!" + if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then + curl --location -o "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" + fi + + if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then + curl --location -o "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" - fi fi } @@ -74,31 +44,8 @@ function update_tests_metadata() { function retrieve_tests_mapping() { mkdir -p $(dirname "$RSPEC_PACKED_TESTS_MAPPING_PATH") - if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then - if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then - (curl --location -o "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "https://gitlab-org.gitlab.io/gitlab/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}" - fi - else - # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to - # always target the canonical project here, so the branch must be hardcoded - local project_path="gitlab-org/gitlab" - local artifact_branch="master" - local username="gitlab-bot" - local job_name="update-tests-metadata" - local test_metadata_with_mapping_job_id - - test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=${username}" -Q "scope=success" --job-name "${job_name}") - - if [[ -n "${test_metadata_with_mapping_job_id}" ]]; then - echo "test_metadata_with_mapping_job_id: ${test_metadata_with_mapping_job_id}" - - if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then - (scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}" - fi - else - echo "test_metadata_with_mapping_job_id couldn't be found!" - echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}" - fi + if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then + (curl --location -o "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "https://gitlab-org.gitlab.io/gitlab/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}" fi scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}" @@ -107,31 +54,8 @@ function retrieve_tests_mapping() { function retrieve_frontend_fixtures_mapping() { mkdir -p $(dirname "$FRONTEND_FIXTURES_MAPPING_PATH") - if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then - if [[ ! -f "${FRONTEND_FIXTURES_MAPPING_PATH}" ]]; then - (curl --location -o "${FRONTEND_FIXTURES_MAPPING_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FRONTEND_FIXTURES_MAPPING_PATH}") || echo "{}" > "${FRONTEND_FIXTURES_MAPPING_PATH}" - fi - else - # ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to - # always target the canonical project here, so the branch must be hardcoded - local project_path="gitlab-org/gitlab" - local artifact_branch="master" - local username="gitlab-bot" - local job_name="generate-frontend-fixtures-mapping" - local test_metadata_with_mapping_job_id - - test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "ref=${artifact_branch}" -q "username=${username}" -Q "scope=success" --job-name "${job_name}") - - if [[ $? -eq 0 ]] && [[ -n "${test_metadata_with_mapping_job_id}" ]]; then - echo "test_metadata_with_mapping_job_id: ${test_metadata_with_mapping_job_id}" - - if [[ ! -f "${FRONTEND_FIXTURES_MAPPING_PATH}" ]]; then - (scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${FRONTEND_FIXTURES_MAPPING_PATH}") || echo "{}" > "${FRONTEND_FIXTURES_MAPPING_PATH}" - fi - else - echo "test_metadata_with_mapping_job_id couldn't be found!" - echo "{}" > "${FRONTEND_FIXTURES_MAPPING_PATH}" - fi + if [[ ! -f "${FRONTEND_FIXTURES_MAPPING_PATH}" ]]; then + (curl --location -o "${FRONTEND_FIXTURES_MAPPING_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FRONTEND_FIXTURES_MAPPING_PATH}") || echo "{}" > "${FRONTEND_FIXTURES_MAPPING_PATH}" fi } |