1 files changed, 50 insertions, 3 deletions

diff --git a/scripts/setup/as-if-jh.sh b/scripts/setup/as-if-jh.sh
index ffc3c6582db..6701f12e64c 100755
--- a/scripts/setup/as-if-jh.sh
+++ b/scripts/setup/as-if-jh.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-prepare_jh_branch() {
+set_jh_branch_env_variable() {
   set -eu # https://explainshell.com/explain?cmd=set+-eu
 
   JH_BRANCH="$(./scripts/setup/find-jh-branch.rb)"
@@ -9,7 +9,15 @@ prepare_jh_branch() {
   echoinfo "JH_BRANCH: ${JH_BRANCH}"
 }
 
-download_jh_path() {
+download_jh_files() {
+  if [ "${JH_BRANCH}" = "main-jh" ]; then
+    download_jh_files_from_api "$@"
+  else
+    download_jh_files_from_git_clone "$@"
+  fi
+}
+
+download_jh_files_from_api() {
   set -eu # https://explainshell.com/explain?cmd=set+-eu
 
   for path in "$@"; do
@@ -17,11 +25,50 @@ download_jh_path() {
     # shellcheck disable=SC3043
     local output="${path}.tar.gz"
 
-    echoinfo "Downloading ${path}"
+    echoinfo "Downloading ${path} via API"
 
+    # Note: We are limited to 5 downloads/minute on this endpoint.
+    # (see https://docs.gitlab.com/ee/api/repositories.html#get-file-archive)
+    #
+    # If we run this command more than 5 times/minute, we will receive HTTP 429 errors.
+    #
+    # If this problem happens too often, we might want to either download files from
+    # another endpoint, and only download the folders with this endpoint. We could also
+    # do a git clone in all cases.
     curl -f --location --output "${output}" --header "Private-Token: ${ADD_JH_FILES_TOKEN}" --get --data-urlencode "sha=${JH_BRANCH}" --data-urlencode "path=${path}" "https://gitlab.com/api/v4/projects/${GITLAB_JH_MIRROR_PROJECT}/repository/archive"
 
     tar -zxf "${output}" --strip-component 1
     rm "${output}"
   done
 }
+
+# The JiHu mirror project is private, so we would need to be authenticated to download files from the API.
+#
+# When being authenticated and downloading files via the API, we are limited to 5 requests per minute
+# (see https://docs.gitlab.com/ee/api/repositories.html#get-file-archive), and we would need to download 6 files
+# (3 archives for two branches). This job can also be run in parallel between many pipelines.
+download_jh_files_from_git_clone() {
+  return_code=0
+  git_merge_status_code=0
+
+  echoinfo "Cloning JH mirror repo to download JH files"
+
+  git config --global user.email "${GITLAB_USER_EMAIL}";
+  git config --global user.name "${GITLAB_USER_NAME}";
+
+  git clone --filter=tree:0 "${JH_MIRROR_REPOSITORY}" gitlab-jh
+  cd gitlab-jh
+  git checkout "${JH_BRANCH}"
+
+  git merge main-jh || git_merge_status_code=$?
+  if [ "${git_merge_status_code}" -ne 0 ]; then
+    git merge --abort || true
+    return_code=3
+  fi
+
+  mv ${JH_FILES_TO_COMMIT} ./..
+  cd ..
+
+  # We explicitly use exit instead of return, otherwise the job would exit with a 1 error code
+  exit "${return_code}"
+}