diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-09-20 14:18:08 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-09-20 14:18:08 +0300 |
commit | 5afcbe03ead9ada87621888a31a62652b10a7e4f (patch) | |
tree | 9918b67a0d0f0bafa6542e839a8be37adf73102d /scripts | |
parent | c97c0201564848c1f53226fe19d71fdcc472f7d0 (diff) |
Add latest changes from gitlab-org/gitlab@16-4-stable-eev16.4.0-rc42
Diffstat (limited to 'scripts')
-rwxr-xr-x | scripts/build_gdk_image | 13 | ||||
-rwxr-xr-x | scripts/decomposition/generate-loose-foreign-key | 2 | ||||
-rwxr-xr-x | scripts/frontend/create_jsconfig.js | 29 | ||||
-rw-r--r-- | scripts/frontend/preinstall.mjs | 59 | ||||
-rwxr-xr-x | scripts/frontend/webpack_dev_server.js | 14 | ||||
-rwxr-xr-x | scripts/generate-message-to-run-e2e-pipeline.rb | 2 | ||||
-rwxr-xr-x | scripts/gitaly-test-spawn | 2 | ||||
-rwxr-xr-x | scripts/lint-doc.sh | 2 | ||||
-rwxr-xr-x | scripts/lint-docs-redirects.rb | 203 | ||||
-rwxr-xr-x | scripts/process_custom_semgrep_results.sh | 2 | ||||
-rwxr-xr-x | scripts/remote_development/run-e2e-tests.sh | 7 | ||||
-rwxr-xr-x | scripts/remote_development/run-smoke-test-suite.sh | 2 | ||||
-rw-r--r-- | scripts/rspec_helpers.sh | 1 | ||||
-rwxr-xr-x | scripts/setup-test-env | 1 | ||||
-rwxr-xr-x | scripts/setup/as-if-jh.sh | 53 | ||||
-rwxr-xr-x | scripts/trigger-build.rb | 2 |
16 files changed, 359 insertions, 35 deletions
diff --git a/scripts/build_gdk_image b/scripts/build_gdk_image index 8b1fdec2d7d..cb1dbd03adb 100755 --- a/scripts/build_gdk_image +++ b/scripts/build_gdk_image @@ -8,10 +8,6 @@ REGISTRY="${CI_REGISTRY}/${CI_PROJECT_PATH}" SHA_TAG="${CI_COMMIT_SHA}" BRANCH_TAG="${CI_COMMIT_REF_SLUG}" -# Fetch ruby version based on contents of .ruby-version -RUBY_VERSION=$(cat .ruby-version) -RUBY_VERSION_MINOR=$(echo "$RUBY_VERSION" | awk -F. '{$3=0; OFS="."; print $1,$2,$3}') - IMAGE="${REGISTRY}/gitlab-qa-gdk" if [[ -n "${CI}" ]]; then @@ -20,17 +16,16 @@ else OUTPUT_OPTION="--load" fi -echoinfo "Building GDK image with GDK sha: '${GDK_SHA}'" "yes" +echoinfo "Building GDK image" "yes" docker buildx build \ - --cache-to="type=inline" \ - --cache-from="${IMAGE}:${BRANCH_TAG}" \ - --cache-from="${IMAGE}:master" \ + --cache-to="type=registry,mode=max,image-manifest=true,ref=${IMAGE}/cache:${BRANCH_TAG}" \ + --cache-from="${IMAGE}/cache:${BRANCH_TAG}" \ + --cache-from="${IMAGE}/cache:master" \ --file="qa/gdk/Dockerfile.gdk" \ --platform=${ARCH:-amd64} \ --tag="${IMAGE}:${SHA_TAG}" \ --tag="${IMAGE}:${BRANCH_TAG}" \ - --build-arg="GEM_CACHE=/home/gdk/.asdf/installs/ruby/${RUBY_VERSION}/lib/ruby/gems/${RUBY_VERSION_MINOR}/cache" \ ${OUTPUT_OPTION} \ . diff --git a/scripts/decomposition/generate-loose-foreign-key b/scripts/decomposition/generate-loose-foreign-key index 66781343411..d52fa2b4f3f 100755 --- a/scripts/decomposition/generate-loose-foreign-key +++ b/scripts/decomposition/generate-loose-foreign-key @@ -167,8 +167,6 @@ def generate_migration(definition) FOREIGN_KEY_NAME = "#{definition.name}" def up - return unless foreign_key_exists?(:#{definition.from_table}, :#{definition.to_table}, name: FOREIGN_KEY_NAME) - with_lock_retries do remove_foreign_key_if_exists(:#{definition.from_table}, :#{definition.to_table}, name: FOREIGN_KEY_NAME, reverse_lock_order: true) diff --git a/scripts/frontend/create_jsconfig.js b/scripts/frontend/create_jsconfig.js index 7e2d0eb02c4..be95c5cb2d3 100755 --- a/scripts/frontend/create_jsconfig.js +++ b/scripts/frontend/create_jsconfig.js @@ -30,8 +30,19 @@ async function createJsConfig() { // eslint-disable-next-line global-require const webpackConfig = require('../../config/webpack.config'); - const paths = {}; // aliases - const WEBPACK_ALIAS_EXCEPTIONS = ['jquery$', '@gitlab/svgs/dist/icons.svg']; + // Aliases + const paths = { + // NOTE: Sentry is exposed via a wrapper, which has a limited API. + '@sentry/browser': [ + path.relative(PATH_PROJECT_ROOT, 'app/assets/javascripts/sentry/sentry_browser_wrapper.js'), + ], + }; + const WEBPACK_ALIAS_EXCEPTIONS = [ + 'jquery$', + '@gitlab/svgs/dist/icons.svg', + '@apollo/client$', + '@sentry/browser$', + ]; Object.entries(webpackConfig.resolve.alias) .filter(([key]) => !WEBPACK_ALIAS_EXCEPTIONS.includes(key)) .forEach(([key, value]) => { @@ -40,6 +51,7 @@ async function createJsConfig() { paths[alias] = target; }); + // JS/TS config. See more: https://www.typescriptlang.org/tsconfig const jsConfig = { // As we're introducing jsconfig to the project, as a precaution we add both: // 'include' and 'exclude' options. This might be simplified in the future. @@ -52,13 +64,22 @@ async function createJsConfig() { 'ee/app/assets/javascripts', 'spec/frontend', 'ee/spec/frontend', - 'tmp/tests/frontend/fixtures/', - 'tmp/tests/frontend/fixtures-ee/', + 'tmp/tests/frontend/fixtures', + 'tmp/tests/frontend/fixtures-ee', ], + + // Explicitly enable automatic type acquisition + // See more: https://www.typescriptlang.org/tsconfig#type-acquisition + typeAcquisition: { + enable: true, + }, + compilerOptions: { baseUrl: '.', // Define the project root checkJs: false, // Disable type checking on JavaScript files disableSizeLimit: true, // Disable memory size limit for the language server + skipLibCheck: true, // Skip type checking all .d.ts files + resolveJsonModule: true, // Enable importing .json files paths, // Aliases }, }; diff --git a/scripts/frontend/preinstall.mjs b/scripts/frontend/preinstall.mjs new file mode 100644 index 00000000000..09d980344ea --- /dev/null +++ b/scripts/frontend/preinstall.mjs @@ -0,0 +1,59 @@ +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { readFile, rm } from 'node:fs/promises'; + +const ROOT_PATH = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const NODE_MODULES = join(ROOT_PATH, 'node_modules'); +const INTEGRITY_FILE = join(NODE_MODULES, '.yarn-integrity'); +const PACKAGE_JSON = join(ROOT_PATH, 'package.json'); + +function isAliasedDependency(x) { + return x.includes('@npm:'); +} +function serializeAliasedDependencyPatterns(obj) { + return Object.entries(obj).map(([key, version]) => `${key}@${version}`); +} + +async function readJSON(file) { + return JSON.parse(await readFile(file, { encoding: 'utf-8' })); +} + +async function getPrevTopLevelPatterns() { + try { + return (await readJSON(INTEGRITY_FILE))?.topLevelPatterns?.filter(isAliasedDependency); + } catch { + return []; + } +} +async function getCurrentTopLevelPatterns() { + try { + const { dependencies, devDependencies } = await readJSON(PACKAGE_JSON); + + return serializeAliasedDependencyPatterns(dependencies) + .concat(serializeAliasedDependencyPatterns(devDependencies)) + .filter(isAliasedDependency); + } catch { + return []; + } +} + +function arraysHaveSameItems(a1, a2) { + return JSON.stringify(a1.sort()) === JSON.stringify(a2.sort()); +} + +const [prevTopLevelPatterns, currentTopLevelPatterns] = await Promise.all([ + getPrevTopLevelPatterns(), + getCurrentTopLevelPatterns(), +]); + +/** + * Yarn seems to have problems at times, if one uses an <alias>@npm:<name> + * and those packages are being updated. In case one switches branches the + * node_modules folder seems to end up being a corrupted somehow + */ +if (!arraysHaveSameItems(prevTopLevelPatterns, currentTopLevelPatterns)) { + console.error( + '[WARNING] package.json changed significantly. Removing node_modules to be sure there are no problems.', + ); + await rm(NODE_MODULES, { recursive: true, force: true }); +} diff --git a/scripts/frontend/webpack_dev_server.js b/scripts/frontend/webpack_dev_server.js index a76e6dc024a..ae73c14b501 100755 --- a/scripts/frontend/webpack_dev_server.js +++ b/scripts/frontend/webpack_dev_server.js @@ -30,17 +30,15 @@ if (STATIC_MODE) { // run webpack through webpack-dev-server, optionally compiling a DLL to reduce memory else { - const watch = ['config/webpack.config.js']; + const watch = [ + 'config/webpack.config.js', + // ensure we refresh when running yarn install + 'node_modules/.yarn-integrity', + ]; // if utilizing the vendor DLL, we need to restart the process when dependency changes occur if (DLL_MODE) { - watch.push( - 'config/webpack.vendor.config.js', - // ensure we refresh when running yarn install - 'node_modules/.yarn-integrity', - 'package.json', - 'yarn.lock', - ); + watch.push('config/webpack.vendor.config.js', 'package.json', 'yarn.lock'); } nodemon({ exec: 'webpack-dev-server --config config/webpack.config.js', diff --git a/scripts/generate-message-to-run-e2e-pipeline.rb b/scripts/generate-message-to-run-e2e-pipeline.rb index cfe480ac5c7..ccbaba8a3eb 100755 --- a/scripts/generate-message-to-run-e2e-pipeline.rb +++ b/scripts/generate-message-to-run-e2e-pipeline.rb @@ -83,7 +83,7 @@ class GenerateMessageToRunE2ePipeline Once done, apply the ✅ emoji on this comment. - For any questions or help, reach out on the internal #quality Slack channel. + **Team members only:** for any questions or help, reach out on the internal `#quality` Slack channel. <!-- Run e2e warning end --> MARKDOWN end diff --git a/scripts/gitaly-test-spawn b/scripts/gitaly-test-spawn index 9285b561ae0..f3688a6e88e 100755 --- a/scripts/gitaly-test-spawn +++ b/scripts/gitaly-test-spawn @@ -10,6 +10,8 @@ class GitalyTestSpawn include GitalySetup def run + ensure_gitlab_shell_secret! + # Run Praefect migrations setup_praefect diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh index e6b63925fc6..6683802c2fe 100755 --- a/scripts/lint-doc.sh +++ b/scripts/lint-doc.sh @@ -149,7 +149,7 @@ else if [ -n "${MD_DOC_PATH}" ] then # shellcheck disable=2059 - printf "${COLOR_GREEN}INFO: Merged results pipeline detected. Testing only the following files:${COLOR_RESET}\n${MD_DOC_PATH}" + printf "${COLOR_GREEN}INFO: Merged results pipeline detected. Testing only the following files:${COLOR_RESET}\n${MD_DOC_PATH}\n" fi fi fi diff --git a/scripts/lint-docs-redirects.rb b/scripts/lint-docs-redirects.rb new file mode 100755 index 00000000000..fb4ac19981d --- /dev/null +++ b/scripts/lint-docs-redirects.rb @@ -0,0 +1,203 @@ +#!/usr/bin/env ruby + +# frozen_string_literal: true + +# +# https://docs.gitlab.com/ee/development/documentation/redirects.html +# +require 'net/http' +require 'uri' +require 'json' +require 'cgi' + +class LintDocsRedirect + COLOR_CODE_RED = "\e[31m" + COLOR_CODE_RESET = "\e[0m" + # All the projects we want this script to run + PROJECT_PATHS = ['gitlab-org/gitlab', + 'gitlab-org/gitlab-runner', + 'gitlab-org/omnibus-gitlab', + 'gitlab-org/charts/gitlab', + 'gitlab-org/cloud-native/gitlab-operator'].freeze + + def execute + return unless project_supported? + + abort_unless_merge_request_iid_exists + + check_renamed_deleted_files + end + + private + + # Project slug based on project path + # Taken from https://gitlab.com/gitlab-org/gitlab/-/blob/daaa5b6f79049e5bb28cdafaa11d3a0a84d64ab3/scripts/trigger-build.rb#L298-313 + def project_slug + case ENV['CI_PROJECT_PATH'] + when 'gitlab-org/gitlab' + 'ee' + when 'gitlab-org/gitlab-runner' + 'runner' + when 'gitlab-org/omnibus-gitlab' + 'omnibus' + when 'gitlab-org/charts/gitlab' + 'charts' + when 'gitlab-org/cloud-native/gitlab-operator' + 'operator' + end + end + + def navigation_file + @navigation_file ||= begin + url = URI('https://gitlab.com/gitlab-org/gitlab-docs/-/raw/main/content/_data/navigation.yaml') + response = Net::HTTP.get_response(url) + + raise "Could not download navigation.yaml. Response code: #{response.code}" if response.code != '200' + + # response.body should be memoized in a method, so that it doesn't + # need to be downloaded multiple times in one CI job. + response.body + end + end + + ## + ## Check if the deleted/renamed file exists in + ## https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/content/_data/navigation.yaml. + ## + ## We need to first convert the Markdown file to HTML. There are two cases: + ## + ## - A source doc entry with index.md looks like: doc/administration/index.md + ## The navigation.yaml equivalent is: ee/administration/ + ## - A source doc entry without index.md looks like: doc/administration/appearance.md + ## The navigation.yaml equivalent is: ee/administration/appearance.html + ## + def check_for_missing_nav_entry(file) + file_sub = file["old_path"].gsub('doc', project_slug).gsub('index.md', '').gsub('.md', '.html') + + result = navigation_file.include?(file_sub) + return unless result + + warning(file) + + abort + end + + def warning(file) + warn <<~WARNING + #{COLOR_CODE_RED}✖ ERROR: Missing redirect for a deleted or moved page#{COLOR_CODE_RESET} + + The following file is linked in the global navigation for docs.gitlab.com: + + => #{file['old_path']} + + Unless you add a redirect or remove the page from the global navigation, + this change will break pipelines in the 'gitlab/gitlab-docs' project. + + #{rake_command(file)} + + For more information, see: + - Create a redirect : https://docs.gitlab.com/ee/development/documentation/redirects.html + - Edit the global nav : https://docs.gitlab.com/ee/development/documentation/site_architecture/global_nav.html#add-a-navigation-entry + WARNING + end + + # Rake task to use depending on the file being deleted or renamed + def rake_command(file) + # The Rake task is only available for gitlab-org/gitlab + return unless project_slug == 'ee' + + if renamed_doc_file?(file) + rake = "bundle exec rake \"gitlab:docs:redirect[#{file['old_path']}, #{file['new_path']}]\"" + msg = "It seems you renamed a page, run the following Rake task locally and commit the changes.\n" + elsif deleted_doc_file?(file) + rake = "bundle exec rake \"gitlab:docs:redirect[#{file['old_path']}, doc/new/path.md]\"" + msg = "It seems you deleted a page. Run the following Rake task by replacing\n" \ + "'doc/new/path.md' with the page to redirect to, and commit the changes.\n" + end + + <<~MSG + #{msg} + #{rake} + MSG + end + + # GitLab API URL + def gitlab_api_url + ENV.fetch('CI_API_V4_URL', 'https://gitlab.com/api/v4') + end + + # Take the project path from the CI_PROJECT_PATH predefined variable. + def url_encoded_project_path + project_path = ENV.fetch('CI_PROJECT_PATH', nil) + return unless project_path + + CGI.escape(project_path) + end + + # Take the merge request ID from the CI_MERGE_REQUEST_IID predefined + # variable. + def merge_request_iid + ENV.fetch('CI_MERGE_REQUEST_IID', nil) + end + + def abort_unless_merge_request_iid_exists + abort("Error: CI_MERGE_REQUEST_IID environment variable is missing") if merge_request_iid.nil? + end + + # Skip if CI_PROJECT_PATH is not in the designated project paths + def project_supported? + PROJECT_PATHS.include? ENV['CI_PROJECT_PATH'] + end + + # Fetch the merge request diff JSON object + def merge_request_diff + @merge_request_diff ||= begin + uri = URI.parse( + "#{gitlab_api_url}/projects/#{url_encoded_project_path}/merge_requests/#{merge_request_iid}/diffs?per_page=30" + ) + response = Net::HTTP.get_response(uri) + + unless response.code == '200' + raise "API call to get MR diffs failed. Response code: #{response.code}. Response message: #{response.message}" + end + + JSON.parse(response.body) + end + end + + def renamed_doc_file?(file) + file['renamed_file'] == true && file['old_path'].start_with?('doc') + end + + def deleted_doc_file?(file) + file['deleted_file'] == true && file['old_path'].start_with?('doc') + end + + # Create a list of hashes of the renamed documentation files + def check_renamed_deleted_files + renamed_files = merge_request_diff.select do |file| + renamed_doc_file?(file) + end + + deleted_files = merge_request_diff.select do |file| + deleted_doc_file?(file) + end + + # Merge the two arrays + all_files = renamed_files + deleted_files + + return if all_files.empty? + + all_files.each do |file| + status = deleted_doc_file?(file) ? 'deleted' : 'renamed' + puts "Checking #{status} file..." + puts "=> Old_path: #{file['old_path']}" + puts "=> New_path: #{file['new_path']}" + puts + + check_for_missing_nav_entry(file) + end + end +end + +LintDocsRedirect.new.execute if $PROGRAM_NAME == __FILE__ diff --git a/scripts/process_custom_semgrep_results.sh b/scripts/process_custom_semgrep_results.sh index 1fdd8e486f3..0eccef00973 100755 --- a/scripts/process_custom_semgrep_results.sh +++ b/scripts/process_custom_semgrep_results.sh @@ -22,7 +22,7 @@ jq -crM '.vulnerabilities | echo "Resulting file:" cat findings.txt -EXISTING_COMMENT_ID=$(curl "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/merge_requests/$CI_MERGE_REQUEST_IID/notes" \ +EXISTING_COMMENT_ID=$(curl "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/merge_requests/$CI_MERGE_REQUEST_IID/notes?per_page=100" \ --header "Private-Token: $CUSTOM_SAST_RULES_BOT_PAT" | jq -crM 'map( select( .author.id == (env.BOT_USER_ID | tonumber) ) | .id ) | first') diff --git a/scripts/remote_development/run-e2e-tests.sh b/scripts/remote_development/run-e2e-tests.sh index 63d828d323c..1c1fb07ea53 100755 --- a/scripts/remote_development/run-e2e-tests.sh +++ b/scripts/remote_development/run-e2e-tests.sh @@ -7,7 +7,7 @@ # them to be overridden. # # For details on how to run this, see the documentation comments at the top of -# qa/qa/specs/features/ee/browser_ui/3_create/remote_development/create_new_workspace_and_terminate_spec.rb +# qa/qa/specs/features/ee/browser_ui/3_create/remote_development/with_prerequisite_done/workspace_actions_with_prerequisite_done_spec.rb DEFAULT_PASSWORD='5iveL!fe' @@ -28,9 +28,8 @@ echo "TEST_INSTANCE_URL: ${TEST_INSTANCE_URL}" working_directory="$(git rev-parse --show-toplevel)/qa" -# TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/397005 -# Remove the '--tag quarantine' from below once this test is removed from quarantine +# This test is currently quarantined as its only used for local testing, so we have to use the '--tag quarantine' (cd "$working_directory" && \ bundle && \ bundle exec bin/qa Test::Instance::All "$TEST_INSTANCE_URL" -- \ - --tag quarantine qa/specs/features/ee/browser_ui/3_create/remote_development/without_setup) + --tag quarantine qa/specs/features/ee/browser_ui/3_create/remote_development/with_prerequisite_done/workspace_actions_with_prerequisite_done_spec.rb) diff --git a/scripts/remote_development/run-smoke-test-suite.sh b/scripts/remote_development/run-smoke-test-suite.sh index 9ab692e6a15..14c50678fba 100755 --- a/scripts/remote_development/run-smoke-test-suite.sh +++ b/scripts/remote_development/run-smoke-test-suite.sh @@ -32,7 +32,7 @@ printf "${Color_Off}" printf "${BBlue}Running Remote Development backend specs${Color_Off}\n\n" bin/rspec -r spec_helper \ -$(find . -path '**/remote_development/**/*_spec.rb') \ +$(find . -path './**/remote_development/*_spec.rb' | grep -v 'qa/qa') \ ee/spec/graphql/types/query_type_spec.rb \ ee/spec/graphql/types/subscription_type_spec.rb \ ee/spec/requests/api/internal/kubernetes_spec.rb \ diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index cab1ac10f35..eefd9ed4993 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -166,7 +166,6 @@ function debug_shell_options() { } function debug_rspec_variables() { - echoinfo "SKIP_FLAKY_TESTS_AUTOMATICALLY: ${SKIP_FLAKY_TESTS_AUTOMATICALLY:-}" echoinfo "RETRY_FAILED_TESTS_IN_NEW_PROCESS: ${RETRY_FAILED_TESTS_IN_NEW_PROCESS:-}" echoinfo "KNAPSACK_GENERATE_REPORT: ${KNAPSACK_GENERATE_REPORT:-}" diff --git a/scripts/setup-test-env b/scripts/setup-test-env index 50bec46b71a..a9d1be7a0ce 100755 --- a/scripts/setup-test-env +++ b/scripts/setup-test-env @@ -24,6 +24,7 @@ require_relative '../lib/system_check/helpers' # Required for config/initializers/1_settings.rb require 'omniauth' require 'omniauth-github' +require 'omniauth-saml' require 'etc' require 'gitlab/utils/all' require 'gitlab/safe_request_store' diff --git a/scripts/setup/as-if-jh.sh b/scripts/setup/as-if-jh.sh index ffc3c6582db..6701f12e64c 100755 --- a/scripts/setup/as-if-jh.sh +++ b/scripts/setup/as-if-jh.sh @@ -1,6 +1,6 @@ #!/bin/sh -prepare_jh_branch() { +set_jh_branch_env_variable() { set -eu # https://explainshell.com/explain?cmd=set+-eu JH_BRANCH="$(./scripts/setup/find-jh-branch.rb)" @@ -9,7 +9,15 @@ prepare_jh_branch() { echoinfo "JH_BRANCH: ${JH_BRANCH}" } -download_jh_path() { +download_jh_files() { + if [ "${JH_BRANCH}" = "main-jh" ]; then + download_jh_files_from_api "$@" + else + download_jh_files_from_git_clone "$@" + fi +} + +download_jh_files_from_api() { set -eu # https://explainshell.com/explain?cmd=set+-eu for path in "$@"; do @@ -17,11 +25,50 @@ download_jh_path() { # shellcheck disable=SC3043 local output="${path}.tar.gz" - echoinfo "Downloading ${path}" + echoinfo "Downloading ${path} via API" + # Note: We are limited to 5 downloads/minute on this endpoint. + # (see https://docs.gitlab.com/ee/api/repositories.html#get-file-archive) + # + # If we run this command more than 5 times/minute, we will receive HTTP 429 errors. + # + # If this problem happens too often, we might want to either download files from + # another endpoint, and only download the folders with this endpoint. We could also + # do a git clone in all cases. curl -f --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive" tar -zxf "${output}" --strip-component 1 rm "${output}" done } + +# The JiHu mirror project is private, so we would need to be authenticated to download files from the API. +# +# When being authenticated and downloading files via the API, we are limited to 5 requests per minute +# (see https://docs.gitlab.com/ee/api/repositories.html#get-file-archive), and we would need to download 6 files +# (3 archives for two branches). This job can also be run in parallel between many pipelines. +download_jh_files_from_git_clone() { + return_code=0 + git_merge_status_code=0 + + echoinfo "Cloning JH mirror repo to download JH files" + + git config --global user.email "${GITLAB_USER_EMAIL}"; + git config --global user.name "${GITLAB_USER_NAME}"; + + git clone --filter=tree:0 "${JH_MIRROR_REPOSITORY}" gitlab-jh + cd gitlab-jh + git checkout "${JH_BRANCH}" + + git merge main-jh || git_merge_status_code=$? + if [ "${git_merge_status_code}" -ne 0 ]; then + git merge --abort || true + return_code=3 + fi + + mv ${JH_FILES_TO_COMMIT} ./.. + cd .. + + # We explicitly use exit instead of return, otherwise the job would exit with a 1 error code + exit "${return_code}" +} diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index 8f509399fd4..98ca8112d62 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -307,6 +307,8 @@ module Trigger 'omnibus' when 'gitlab-org/charts/gitlab' 'charts' + when 'gitlab-org/cloud-native/gitlab-operator' + 'operator' end end |