Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-09-20 14:18:08 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-09-20 14:18:08 +0300
commit5afcbe03ead9ada87621888a31a62652b10a7e4f (patch)
tree9918b67a0d0f0bafa6542e839a8be37adf73102d /scripts
parentc97c0201564848c1f53226fe19d71fdcc472f7d0 (diff)
Add latest changes from gitlab-org/gitlab@16-4-stable-eev16.4.0-rc42
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/build_gdk_image13
-rwxr-xr-xscripts/decomposition/generate-loose-foreign-key2
-rwxr-xr-xscripts/frontend/create_jsconfig.js29
-rw-r--r--scripts/frontend/preinstall.mjs59
-rwxr-xr-xscripts/frontend/webpack_dev_server.js14
-rwxr-xr-xscripts/generate-message-to-run-e2e-pipeline.rb2
-rwxr-xr-xscripts/gitaly-test-spawn2
-rwxr-xr-xscripts/lint-doc.sh2
-rwxr-xr-xscripts/lint-docs-redirects.rb203
-rwxr-xr-xscripts/process_custom_semgrep_results.sh2
-rwxr-xr-xscripts/remote_development/run-e2e-tests.sh7
-rwxr-xr-xscripts/remote_development/run-smoke-test-suite.sh2
-rw-r--r--scripts/rspec_helpers.sh1
-rwxr-xr-xscripts/setup-test-env1
-rwxr-xr-xscripts/setup/as-if-jh.sh53
-rwxr-xr-xscripts/trigger-build.rb2
16 files changed, 359 insertions, 35 deletions
diff --git a/scripts/build_gdk_image b/scripts/build_gdk_image
index 8b1fdec2d7d..cb1dbd03adb 100755
--- a/scripts/build_gdk_image
+++ b/scripts/build_gdk_image
@@ -8,10 +8,6 @@ REGISTRY="${CI_REGISTRY}/${CI_PROJECT_PATH}"
SHA_TAG="${CI_COMMIT_SHA}"
BRANCH_TAG="${CI_COMMIT_REF_SLUG}"
-# Fetch ruby version based on contents of .ruby-version
-RUBY_VERSION=$(cat .ruby-version)
-RUBY_VERSION_MINOR=$(echo "$RUBY_VERSION" | awk -F. '{$3=0; OFS="."; print $1,$2,$3}')
-
IMAGE="${REGISTRY}/gitlab-qa-gdk"
if [[ -n "${CI}" ]]; then
@@ -20,17 +16,16 @@ else
OUTPUT_OPTION="--load"
fi
-echoinfo "Building GDK image with GDK sha: '${GDK_SHA}'" "yes"
+echoinfo "Building GDK image" "yes"
docker buildx build \
- --cache-to="type=inline" \
- --cache-from="${IMAGE}:${BRANCH_TAG}" \
- --cache-from="${IMAGE}:master" \
+ --cache-to="type=registry,mode=max,image-manifest=true,ref=${IMAGE}/cache:${BRANCH_TAG}" \
+ --cache-from="${IMAGE}/cache:${BRANCH_TAG}" \
+ --cache-from="${IMAGE}/cache:master" \
--file="qa/gdk/Dockerfile.gdk" \
--platform=${ARCH:-amd64} \
--tag="${IMAGE}:${SHA_TAG}" \
--tag="${IMAGE}:${BRANCH_TAG}" \
- --build-arg="GEM_CACHE=/home/gdk/.asdf/installs/ruby/${RUBY_VERSION}/lib/ruby/gems/${RUBY_VERSION_MINOR}/cache" \
${OUTPUT_OPTION} \
.
diff --git a/scripts/decomposition/generate-loose-foreign-key b/scripts/decomposition/generate-loose-foreign-key
index 66781343411..d52fa2b4f3f 100755
--- a/scripts/decomposition/generate-loose-foreign-key
+++ b/scripts/decomposition/generate-loose-foreign-key
@@ -167,8 +167,6 @@ def generate_migration(definition)
FOREIGN_KEY_NAME = "#{definition.name}"
def up
- return unless foreign_key_exists?(:#{definition.from_table}, :#{definition.to_table}, name: FOREIGN_KEY_NAME)
-
with_lock_retries do
remove_foreign_key_if_exists(:#{definition.from_table}, :#{definition.to_table},
name: FOREIGN_KEY_NAME, reverse_lock_order: true)
diff --git a/scripts/frontend/create_jsconfig.js b/scripts/frontend/create_jsconfig.js
index 7e2d0eb02c4..be95c5cb2d3 100755
--- a/scripts/frontend/create_jsconfig.js
+++ b/scripts/frontend/create_jsconfig.js
@@ -30,8 +30,19 @@ async function createJsConfig() {
// eslint-disable-next-line global-require
const webpackConfig = require('../../config/webpack.config');
- const paths = {}; // aliases
- const WEBPACK_ALIAS_EXCEPTIONS = ['jquery$', '@gitlab/svgs/dist/icons.svg'];
+ // Aliases
+ const paths = {
+ // NOTE: Sentry is exposed via a wrapper, which has a limited API.
+ '@sentry/browser': [
+ path.relative(PATH_PROJECT_ROOT, 'app/assets/javascripts/sentry/sentry_browser_wrapper.js'),
+ ],
+ };
+ const WEBPACK_ALIAS_EXCEPTIONS = [
+ 'jquery$',
+ '@gitlab/svgs/dist/icons.svg',
+ '@apollo/client$',
+ '@sentry/browser$',
+ ];
Object.entries(webpackConfig.resolve.alias)
.filter(([key]) => !WEBPACK_ALIAS_EXCEPTIONS.includes(key))
.forEach(([key, value]) => {
@@ -40,6 +51,7 @@ async function createJsConfig() {
paths[alias] = target;
});
+ // JS/TS config. See more: https://www.typescriptlang.org/tsconfig
const jsConfig = {
// As we're introducing jsconfig to the project, as a precaution we add both:
// 'include' and 'exclude' options. This might be simplified in the future.
@@ -52,13 +64,22 @@ async function createJsConfig() {
'ee/app/assets/javascripts',
'spec/frontend',
'ee/spec/frontend',
- 'tmp/tests/frontend/fixtures/',
- 'tmp/tests/frontend/fixtures-ee/',
+ 'tmp/tests/frontend/fixtures',
+ 'tmp/tests/frontend/fixtures-ee',
],
+
+ // Explicitly enable automatic type acquisition
+ // See more: https://www.typescriptlang.org/tsconfig#type-acquisition
+ typeAcquisition: {
+ enable: true,
+ },
+
compilerOptions: {
baseUrl: '.', // Define the project root
checkJs: false, // Disable type checking on JavaScript files
disableSizeLimit: true, // Disable memory size limit for the language server
+ skipLibCheck: true, // Skip type checking all .d.ts files
+ resolveJsonModule: true, // Enable importing .json files
paths, // Aliases
},
};
diff --git a/scripts/frontend/preinstall.mjs b/scripts/frontend/preinstall.mjs
new file mode 100644
index 00000000000..09d980344ea
--- /dev/null
+++ b/scripts/frontend/preinstall.mjs
@@ -0,0 +1,59 @@
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { readFile, rm } from 'node:fs/promises';
+
+const ROOT_PATH = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const NODE_MODULES = join(ROOT_PATH, 'node_modules');
+const INTEGRITY_FILE = join(NODE_MODULES, '.yarn-integrity');
+const PACKAGE_JSON = join(ROOT_PATH, 'package.json');
+
+function isAliasedDependency(x) {
+ return x.includes('@npm:');
+}
+function serializeAliasedDependencyPatterns(obj) {
+ return Object.entries(obj).map(([key, version]) => `${key}@${version}`);
+}
+
+async function readJSON(file) {
+ return JSON.parse(await readFile(file, { encoding: 'utf-8' }));
+}
+
+async function getPrevTopLevelPatterns() {
+ try {
+ return (await readJSON(INTEGRITY_FILE))?.topLevelPatterns?.filter(isAliasedDependency);
+ } catch {
+ return [];
+ }
+}
+async function getCurrentTopLevelPatterns() {
+ try {
+ const { dependencies, devDependencies } = await readJSON(PACKAGE_JSON);
+
+ return serializeAliasedDependencyPatterns(dependencies)
+ .concat(serializeAliasedDependencyPatterns(devDependencies))
+ .filter(isAliasedDependency);
+ } catch {
+ return [];
+ }
+}
+
+function arraysHaveSameItems(a1, a2) {
+ return JSON.stringify(a1.sort()) === JSON.stringify(a2.sort());
+}
+
+const [prevTopLevelPatterns, currentTopLevelPatterns] = await Promise.all([
+ getPrevTopLevelPatterns(),
+ getCurrentTopLevelPatterns(),
+]);
+
+/**
+ * Yarn seems to have problems at times, if one uses an <alias>@npm:<name>
+ * and those packages are being updated. In case one switches branches the
+ * node_modules folder seems to end up being a corrupted somehow
+ */
+if (!arraysHaveSameItems(prevTopLevelPatterns, currentTopLevelPatterns)) {
+ console.error(
+ '[WARNING] package.json changed significantly. Removing node_modules to be sure there are no problems.',
+ );
+ await rm(NODE_MODULES, { recursive: true, force: true });
+}
diff --git a/scripts/frontend/webpack_dev_server.js b/scripts/frontend/webpack_dev_server.js
index a76e6dc024a..ae73c14b501 100755
--- a/scripts/frontend/webpack_dev_server.js
+++ b/scripts/frontend/webpack_dev_server.js
@@ -30,17 +30,15 @@ if (STATIC_MODE) {
// run webpack through webpack-dev-server, optionally compiling a DLL to reduce memory
else {
- const watch = ['config/webpack.config.js'];
+ const watch = [
+ 'config/webpack.config.js',
+ // ensure we refresh when running yarn install
+ 'node_modules/.yarn-integrity',
+ ];
// if utilizing the vendor DLL, we need to restart the process when dependency changes occur
if (DLL_MODE) {
- watch.push(
- 'config/webpack.vendor.config.js',
- // ensure we refresh when running yarn install
- 'node_modules/.yarn-integrity',
- 'package.json',
- 'yarn.lock',
- );
+ watch.push('config/webpack.vendor.config.js', 'package.json', 'yarn.lock');
}
nodemon({
exec: 'webpack-dev-server --config config/webpack.config.js',
diff --git a/scripts/generate-message-to-run-e2e-pipeline.rb b/scripts/generate-message-to-run-e2e-pipeline.rb
index cfe480ac5c7..ccbaba8a3eb 100755
--- a/scripts/generate-message-to-run-e2e-pipeline.rb
+++ b/scripts/generate-message-to-run-e2e-pipeline.rb
@@ -83,7 +83,7 @@ class GenerateMessageToRunE2ePipeline
Once done, apply the ✅ emoji on this comment.
- For any questions or help, reach out on the internal #quality Slack channel.
+ **Team members only:** for any questions or help, reach out on the internal `#quality` Slack channel.
<!-- Run e2e warning end -->
MARKDOWN
end
diff --git a/scripts/gitaly-test-spawn b/scripts/gitaly-test-spawn
index 9285b561ae0..f3688a6e88e 100755
--- a/scripts/gitaly-test-spawn
+++ b/scripts/gitaly-test-spawn
@@ -10,6 +10,8 @@ class GitalyTestSpawn
include GitalySetup
def run
+ ensure_gitlab_shell_secret!
+
# Run Praefect migrations
setup_praefect
diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh
index e6b63925fc6..6683802c2fe 100755
--- a/scripts/lint-doc.sh
+++ b/scripts/lint-doc.sh
@@ -149,7 +149,7 @@ else
if [ -n "${MD_DOC_PATH}" ]
then
# shellcheck disable=2059
- printf "${COLOR_GREEN}INFO: Merged results pipeline detected. Testing only the following files:${COLOR_RESET}\n${MD_DOC_PATH}"
+ printf "${COLOR_GREEN}INFO: Merged results pipeline detected. Testing only the following files:${COLOR_RESET}\n${MD_DOC_PATH}\n"
fi
fi
fi
diff --git a/scripts/lint-docs-redirects.rb b/scripts/lint-docs-redirects.rb
new file mode 100755
index 00000000000..fb4ac19981d
--- /dev/null
+++ b/scripts/lint-docs-redirects.rb
@@ -0,0 +1,203 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+#
+# https://docs.gitlab.com/ee/development/documentation/redirects.html
+#
+require 'net/http'
+require 'uri'
+require 'json'
+require 'cgi'
+
+class LintDocsRedirect
+ COLOR_CODE_RED = "\e[31m"
+ COLOR_CODE_RESET = "\e[0m"
+ # All the projects we want this script to run
+ PROJECT_PATHS = ['gitlab-org/gitlab',
+ 'gitlab-org/gitlab-runner',
+ 'gitlab-org/omnibus-gitlab',
+ 'gitlab-org/charts/gitlab',
+ 'gitlab-org/cloud-native/gitlab-operator'].freeze
+
+ def execute
+ return unless project_supported?
+
+ abort_unless_merge_request_iid_exists
+
+ check_renamed_deleted_files
+ end
+
+ private
+
+ # Project slug based on project path
+ # Taken from https://gitlab.com/gitlab-org/gitlab/-/blob/daaa5b6f79049e5bb28cdafaa11d3a0a84d64ab3/scripts/trigger-build.rb#L298-313
+ def project_slug
+ case ENV['CI_PROJECT_PATH']
+ when 'gitlab-org/gitlab'
+ 'ee'
+ when 'gitlab-org/gitlab-runner'
+ 'runner'
+ when 'gitlab-org/omnibus-gitlab'
+ 'omnibus'
+ when 'gitlab-org/charts/gitlab'
+ 'charts'
+ when 'gitlab-org/cloud-native/gitlab-operator'
+ 'operator'
+ end
+ end
+
+ def navigation_file
+ @navigation_file ||= begin
+ url = URI('https://gitlab.com/gitlab-org/gitlab-docs/-/raw/main/content/_data/navigation.yaml')
+ response = Net::HTTP.get_response(url)
+
+ raise "Could not download navigation.yaml. Response code: #{response.code}" if response.code != '200'
+
+ # response.body should be memoized in a method, so that it doesn't
+ # need to be downloaded multiple times in one CI job.
+ response.body
+ end
+ end
+
+ ##
+ ## Check if the deleted/renamed file exists in
+ ## https://gitlab.com/gitlab-org/gitlab-docs/-/blob/main/content/_data/navigation.yaml.
+ ##
+ ## We need to first convert the Markdown file to HTML. There are two cases:
+ ##
+ ## - A source doc entry with index.md looks like: doc/administration/index.md
+ ## The navigation.yaml equivalent is: ee/administration/
+ ## - A source doc entry without index.md looks like: doc/administration/appearance.md
+ ## The navigation.yaml equivalent is: ee/administration/appearance.html
+ ##
+ def check_for_missing_nav_entry(file)
+ file_sub = file["old_path"].gsub('doc', project_slug).gsub('index.md', '').gsub('.md', '.html')
+
+ result = navigation_file.include?(file_sub)
+ return unless result
+
+ warning(file)
+
+ abort
+ end
+
+ def warning(file)
+ warn <<~WARNING
+ #{COLOR_CODE_RED}✖ ERROR: Missing redirect for a deleted or moved page#{COLOR_CODE_RESET}
+
+ The following file is linked in the global navigation for docs.gitlab.com:
+
+ => #{file['old_path']}
+
+ Unless you add a redirect or remove the page from the global navigation,
+ this change will break pipelines in the 'gitlab/gitlab-docs' project.
+
+ #{rake_command(file)}
+
+ For more information, see:
+ - Create a redirect : https://docs.gitlab.com/ee/development/documentation/redirects.html
+ - Edit the global nav : https://docs.gitlab.com/ee/development/documentation/site_architecture/global_nav.html#add-a-navigation-entry
+ WARNING
+ end
+
+ # Rake task to use depending on the file being deleted or renamed
+ def rake_command(file)
+ # The Rake task is only available for gitlab-org/gitlab
+ return unless project_slug == 'ee'
+
+ if renamed_doc_file?(file)
+ rake = "bundle exec rake \"gitlab:docs:redirect[#{file['old_path']}, #{file['new_path']}]\""
+ msg = "It seems you renamed a page, run the following Rake task locally and commit the changes.\n"
+ elsif deleted_doc_file?(file)
+ rake = "bundle exec rake \"gitlab:docs:redirect[#{file['old_path']}, doc/new/path.md]\""
+ msg = "It seems you deleted a page. Run the following Rake task by replacing\n" \
+ "'doc/new/path.md' with the page to redirect to, and commit the changes.\n"
+ end
+
+ <<~MSG
+ #{msg}
+ #{rake}
+ MSG
+ end
+
+ # GitLab API URL
+ def gitlab_api_url
+ ENV.fetch('CI_API_V4_URL', 'https://gitlab.com/api/v4')
+ end
+
+ # Take the project path from the CI_PROJECT_PATH predefined variable.
+ def url_encoded_project_path
+ project_path = ENV.fetch('CI_PROJECT_PATH', nil)
+ return unless project_path
+
+ CGI.escape(project_path)
+ end
+
+ # Take the merge request ID from the CI_MERGE_REQUEST_IID predefined
+ # variable.
+ def merge_request_iid
+ ENV.fetch('CI_MERGE_REQUEST_IID', nil)
+ end
+
+ def abort_unless_merge_request_iid_exists
+ abort("Error: CI_MERGE_REQUEST_IID environment variable is missing") if merge_request_iid.nil?
+ end
+
+ # Skip if CI_PROJECT_PATH is not in the designated project paths
+ def project_supported?
+ PROJECT_PATHS.include? ENV['CI_PROJECT_PATH']
+ end
+
+ # Fetch the merge request diff JSON object
+ def merge_request_diff
+ @merge_request_diff ||= begin
+ uri = URI.parse(
+ "#{gitlab_api_url}/projects/#{url_encoded_project_path}/merge_requests/#{merge_request_iid}/diffs?per_page=30"
+ )
+ response = Net::HTTP.get_response(uri)
+
+ unless response.code == '200'
+ raise "API call to get MR diffs failed. Response code: #{response.code}. Response message: #{response.message}"
+ end
+
+ JSON.parse(response.body)
+ end
+ end
+
+ def renamed_doc_file?(file)
+ file['renamed_file'] == true && file['old_path'].start_with?('doc')
+ end
+
+ def deleted_doc_file?(file)
+ file['deleted_file'] == true && file['old_path'].start_with?('doc')
+ end
+
+ # Create a list of hashes of the renamed documentation files
+ def check_renamed_deleted_files
+ renamed_files = merge_request_diff.select do |file|
+ renamed_doc_file?(file)
+ end
+
+ deleted_files = merge_request_diff.select do |file|
+ deleted_doc_file?(file)
+ end
+
+ # Merge the two arrays
+ all_files = renamed_files + deleted_files
+
+ return if all_files.empty?
+
+ all_files.each do |file|
+ status = deleted_doc_file?(file) ? 'deleted' : 'renamed'
+ puts "Checking #{status} file..."
+ puts "=> Old_path: #{file['old_path']}"
+ puts "=> New_path: #{file['new_path']}"
+ puts
+
+ check_for_missing_nav_entry(file)
+ end
+ end
+end
+
+LintDocsRedirect.new.execute if $PROGRAM_NAME == __FILE__
diff --git a/scripts/process_custom_semgrep_results.sh b/scripts/process_custom_semgrep_results.sh
index 1fdd8e486f3..0eccef00973 100755
--- a/scripts/process_custom_semgrep_results.sh
+++ b/scripts/process_custom_semgrep_results.sh
@@ -22,7 +22,7 @@ jq -crM '.vulnerabilities |
echo "Resulting file:"
cat findings.txt
-EXISTING_COMMENT_ID=$(curl "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/merge_requests/$CI_MERGE_REQUEST_IID/notes" \
+EXISTING_COMMENT_ID=$(curl "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/merge_requests/$CI_MERGE_REQUEST_IID/notes?per_page=100" \
--header "Private-Token: $CUSTOM_SAST_RULES_BOT_PAT" |
jq -crM 'map( select( .author.id == (env.BOT_USER_ID | tonumber) ) | .id ) | first')
diff --git a/scripts/remote_development/run-e2e-tests.sh b/scripts/remote_development/run-e2e-tests.sh
index 63d828d323c..1c1fb07ea53 100755
--- a/scripts/remote_development/run-e2e-tests.sh
+++ b/scripts/remote_development/run-e2e-tests.sh
@@ -7,7 +7,7 @@
# them to be overridden.
#
# For details on how to run this, see the documentation comments at the top of
-# qa/qa/specs/features/ee/browser_ui/3_create/remote_development/create_new_workspace_and_terminate_spec.rb
+# qa/qa/specs/features/ee/browser_ui/3_create/remote_development/with_prerequisite_done/workspace_actions_with_prerequisite_done_spec.rb
DEFAULT_PASSWORD='5iveL!fe'
@@ -28,9 +28,8 @@ echo "TEST_INSTANCE_URL: ${TEST_INSTANCE_URL}"
working_directory="$(git rev-parse --show-toplevel)/qa"
-# TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/397005
-# Remove the '--tag quarantine' from below once this test is removed from quarantine
+# This test is currently quarantined as its only used for local testing, so we have to use the '--tag quarantine'
(cd "$working_directory" && \
bundle && \
bundle exec bin/qa Test::Instance::All "$TEST_INSTANCE_URL" -- \
- --tag quarantine qa/specs/features/ee/browser_ui/3_create/remote_development/without_setup)
+ --tag quarantine qa/specs/features/ee/browser_ui/3_create/remote_development/with_prerequisite_done/workspace_actions_with_prerequisite_done_spec.rb)
diff --git a/scripts/remote_development/run-smoke-test-suite.sh b/scripts/remote_development/run-smoke-test-suite.sh
index 9ab692e6a15..14c50678fba 100755
--- a/scripts/remote_development/run-smoke-test-suite.sh
+++ b/scripts/remote_development/run-smoke-test-suite.sh
@@ -32,7 +32,7 @@ printf "${Color_Off}"
printf "${BBlue}Running Remote Development backend specs${Color_Off}\n\n"
bin/rspec -r spec_helper \
-$(find . -path '**/remote_development/**/*_spec.rb') \
+$(find . -path './**/remote_development/*_spec.rb' | grep -v 'qa/qa') \
ee/spec/graphql/types/query_type_spec.rb \
ee/spec/graphql/types/subscription_type_spec.rb \
ee/spec/requests/api/internal/kubernetes_spec.rb \
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index cab1ac10f35..eefd9ed4993 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -166,7 +166,6 @@ function debug_shell_options() {
}
function debug_rspec_variables() {
- echoinfo "SKIP_FLAKY_TESTS_AUTOMATICALLY: ${SKIP_FLAKY_TESTS_AUTOMATICALLY:-}"
echoinfo "RETRY_FAILED_TESTS_IN_NEW_PROCESS: ${RETRY_FAILED_TESTS_IN_NEW_PROCESS:-}"
echoinfo "KNAPSACK_GENERATE_REPORT: ${KNAPSACK_GENERATE_REPORT:-}"
diff --git a/scripts/setup-test-env b/scripts/setup-test-env
index 50bec46b71a..a9d1be7a0ce 100755
--- a/scripts/setup-test-env
+++ b/scripts/setup-test-env
@@ -24,6 +24,7 @@ require_relative '../lib/system_check/helpers'
# Required for config/initializers/1_settings.rb
require 'omniauth'
require 'omniauth-github'
+require 'omniauth-saml'
require 'etc'
require 'gitlab/utils/all'
require 'gitlab/safe_request_store'
diff --git a/scripts/setup/as-if-jh.sh b/scripts/setup/as-if-jh.sh
index ffc3c6582db..6701f12e64c 100755
--- a/scripts/setup/as-if-jh.sh
+++ b/scripts/setup/as-if-jh.sh
@@ -1,6 +1,6 @@
#!/bin/sh
-prepare_jh_branch() {
+set_jh_branch_env_variable() {
set -eu # https://explainshell.com/explain?cmd=set+-eu
JH_BRANCH="$(./scripts/setup/find-jh-branch.rb)"
@@ -9,7 +9,15 @@ prepare_jh_branch() {
echoinfo "JH_BRANCH: ${JH_BRANCH}"
}
-download_jh_path() {
+download_jh_files() {
+ if [ "${JH_BRANCH}" = "main-jh" ]; then
+ download_jh_files_from_api "$@"
+ else
+ download_jh_files_from_git_clone "$@"
+ fi
+}
+
+download_jh_files_from_api() {
set -eu # https://explainshell.com/explain?cmd=set+-eu
for path in "$@"; do
@@ -17,11 +25,50 @@ download_jh_path() {
# shellcheck disable=SC3043
local output="${path}.tar.gz"
- echoinfo "Downloading ${path}"
+ echoinfo "Downloading ${path} via API"
+ # Note: We are limited to 5 downloads/minute on this endpoint.
+ # (see https://docs.gitlab.com/ee/api/repositories.html#get-file-archive)
+ #
+ # If we run this command more than 5 times/minute, we will receive HTTP 429 errors.
+ #
+ # If this problem happens too often, we might want to either download files from
+ # another endpoint, and only download the folders with this endpoint. We could also
+ # do a git clone in all cases.
curl -f --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive"
tar -zxf "${output}" --strip-component 1
rm "${output}"
done
}
+
+# The JiHu mirror project is private, so we would need to be authenticated to download files from the API.
+#
+# When being authenticated and downloading files via the API, we are limited to 5 requests per minute
+# (see https://docs.gitlab.com/ee/api/repositories.html#get-file-archive), and we would need to download 6 files
+# (3 archives for two branches). This job can also be run in parallel between many pipelines.
+download_jh_files_from_git_clone() {
+ return_code=0
+ git_merge_status_code=0
+
+ echoinfo "Cloning JH mirror repo to download JH files"
+
+ git config --global user.email "${GITLAB_USER_EMAIL}";
+ git config --global user.name "${GITLAB_USER_NAME}";
+
+ git clone --filter=tree:0 "${JH_MIRROR_REPOSITORY}" gitlab-jh
+ cd gitlab-jh
+ git checkout "${JH_BRANCH}"
+
+ git merge main-jh || git_merge_status_code=$?
+ if [ "${git_merge_status_code}" -ne 0 ]; then
+ git merge --abort || true
+ return_code=3
+ fi
+
+ mv ${JH_FILES_TO_COMMIT} ./..
+ cd ..
+
+ # We explicitly use exit instead of return, otherwise the job would exit with a 1 error code
+ exit "${return_code}"
+}
diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb
index 8f509399fd4..98ca8112d62 100755
--- a/scripts/trigger-build.rb
+++ b/scripts/trigger-build.rb
@@ -307,6 +307,8 @@ module Trigger
'omnibus'
when 'gitlab-org/charts/gitlab'
'charts'
+ when 'gitlab-org/cloud-native/gitlab-operator'
+ 'operator'
end
end