Add latest changes from gitlab-org/gitlab@16-3-stable-ee

5 files changed, 86 insertions, 133 deletions

diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml
index 1361f1fe412..d4a9582ded3 100644
--- a/.gitlab/ci/build-images.gitlab-ci.yml
+++ b/.gitlab/ci/build-images.gitlab-ci.yml
@@ -34,7 +34,7 @@ build-gdk-image:
     - .base-image-build-buildx
     - .build-images:rules:build-gdk-image
   tags:
-    - high-cpu
+    - saas-linux-xlarge-amd64
   stage: build-images
   needs: []
   script:
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index 264950ae1cc..304544468ac 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -906,17 +906,13 @@
 .build-images:rules:build-gdk-image:
   rules:
     - if: '$QA_RUN_TESTS_ON_GDK !~ /true|yes|1/i'
-      when: never
+      when: manual
+      allow_failure: true
     - !reference [".qa:rules:package-and-test-never-run", rules]
     - <<: *if-default-branch-schedule-nightly # already executed in the 2-hourly schedule
       when: never
-    - <<: *if-default-branch-refs # Includes scheduled pipelines
-      variables:
-        BUILD_GDK_BASE: "true"
-    # We want to also rebuild the base image if MRs change certain components.
+    - <<: *if-default-branch-refs
     - <<: *if-merge-request
-      variables:
-        BUILD_GDK_BASE: "true"
       changes: *gdk-component-patterns
     # The rest are included to be consistent with .qa:rules:e2e:test-on-gdk
     - <<: *if-merge-request-targeting-stable-branch
diff --git a/qa/gdk/Dockerfile.gdk b/qa/gdk/Dockerfile.gdk
index 51e058a7088..0926883d00f 100644
--- a/qa/gdk/Dockerfile.gdk
+++ b/qa/gdk/Dockerfile.gdk
@@ -1,99 +1,66 @@
-# Multi-stage Dockerfile for packaging gdk as executable docker image
-# Each stage can be executed in parallel and cached separately based on changes for respective component
-# Caches are cleaned for each stage to reduce the footprint of exported cache layers
-
-ARG BASE_TAG=master
-
-FROM registry.gitlab.com/gitlab-org/gitlab-build-images/debian-bullseye-slim-ruby-3.1-golang-1.19-rust-1.65-node-16.14-postgresql-13:rubygems-3.4-git-2.36-lfs-2.9-yarn-1.22 as gdk-base
-
-RUN set -eux; \
-    groupadd gdk -g 1000; \
-    useradd gdk -m -s /bin/bash -u 1000 -g 1000
-
-ENV GEM_HOME=/home/gdk/.gem \
-    GEM_PATH=/home/gdk/.gem \
-    PATH=$PATH:/home/gdk/.gem/bin
-
-WORKDIR /home/gdk
-
-# Reinstall libpcre2 and install postgresql
-# See: https://gitlab.com/gitlab-org/gitaly/-/issues/4085
-RUN set -eux; \
-    rm -f /usr/lib/libpcre2*; \
-    apt-get update && apt-get install -y --reinstall --no-install-recommends \
-    libpcre2-16-0 \
-    libpcre2-32-0 \
-    libpcre2-8-0 \
-    libpcre2-dev \
-    libpcre2-posix2 \
-    && apt-get install -y --no-install-recommends postgresql-13; \
-    apt-get autoclean -y
-
-# Clone GDK and install system dependencies, purge system git
-ARG GDK_SHA
-ENV GDK_SHA=${GDK_SHA:-main}
-RUN set -eux; \
-    git -c advice.detachedHead=false clone --depth 1 https://gitlab.com/gitlab-org/gitlab-development-kit.git; \
-    git -C gitlab-development-kit fetch --depth 1 origin ${GDK_SHA}; \
-    git -C gitlab-development-kit -c advice.detachedHead=false checkout ${GDK_SHA}; \
-    mkdir -p gitlab-development-kit/gitlab && chown -R gdk:gdk gitlab-development-kit; \
-    apt-get update && apt-get install -y --no-install-recommends $(grep -o '^[^#]*' gitlab-development-kit/packages_debian.txt); \
-    apt-get remove -y git git-lfs; \
-    apt-get autoclean -y && apt-get autoremove -y
+FROM registry.gitlab.com/gitlab-org/gitlab-development-kit/asdf-bootstrapped-verify:main@sha256:14fa752a80df21f840fc48f4be8561bee21b78886ac718652582fdd788d34c32
+
+ENV GITLAB_LICENSE_MODE=test \
+    GDK_KILL_CONFIRM=true
 
 # Allow passwordless /etc/hosts update by gdk user
+USER root
 RUN echo "gdk ALL=(ALL) NOPASSWD: /usr/bin/tee -a /etc/hosts" >> /etc/sudoers
 
 USER gdk
-WORKDIR /home/gdk/gitlab-development-kit
 
-# Install GDK and gem dependencies
-ARG GDK_VERSION
-ENV GDK_VERSION=${GDK_VERSION:-0.2.16}
-RUN set -eux; \
-    gem install gitlab-development-kit -v ${GDK_VERSION} && touch .gitlab-gdk-gem; \
-    bundle install; \
-    rm -rf ${GEM_HOME}/cache
+# Clone GDK at specific sha and bootstrap packages
+#
+ARG GDK_SHA=747ab64be815f5c239d5d63209527a42bd838e83
+ARG GEM_CACHE=/home/gdk/.asdf/installs/ruby/3.1.4/lib/ruby/gems/3.1.0/cache
+RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \
+    set -eux; \
+    git clone --depth 1 https://gitlab.com/gitlab-org/gitlab-development-kit.git && cd gitlab-development-kit; \
+    git fetch --depth 1 origin ${GDK_SHA} && git -c advice.detachedHead=false checkout ${GDK_SHA}; \
+    mkdir gitlab && make bootstrap
 
-COPY --chown=gdk:gdk qa/gdk/gdk.yml ./
+WORKDIR /home/gdk/gitlab-development-kit
 
-# Build gitaly
-#
-COPY --chown=gdk:gdk GITALY_SERVER_VERSION ./gitlab/
-RUN set -eux; \
-    make gitaly-setup; \
-    rm -rf gitaly/_build/cache \
-           gitaly/_build/deps/git/source \
-           gitaly/_build/deps/libgit2/source \
-           gitaly/_build/deps \
-           gitaly/_build/intermediate \
-           ${GEM_HOME}/cache \
-    && go clean -cache
+COPY --chown=gdk:gdk qa/gdk/gdk.yml ./
 
 # Build gitlab-shell
 #
 COPY --chown=gdk:gdk GITLAB_SHELL_VERSION ./gitlab/
-RUN set -eux; \
-    make gitlab-shell-setup; \
-    rm -rf ${GEM_HOME}/cache \
-    && go clean -cache
+RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \
+    set -eux; \
+    make gitlab-shell-setup \
+    && cd gitlab-shell && go clean -cache -modcache -r
 
 # Build gitlab-workhorse
 #
 COPY --chown=gdk:gdk VERSION GITLAB_WORKHORSE_VERSION ./gitlab/
 COPY --chown=gdk:gdk workhorse ./gitlab/workhorse
-RUN set -eux; \
-    make gitlab-workhorse-setup \
-    && mv gitlab/workhorse ./; \
-    rm -rf ${GEM_HOME}/cache \
-    && go clean -cache
+RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \
+    set -eux; \
+    make gitlab-workhorse-setup && mv gitlab/workhorse ./ \
+    && cd workhorse && go clean -cache -modcache -r
+
+# Build gitaly
+#
+COPY --chown=gdk:gdk GITALY_SERVER_VERSION ./gitlab/
+RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \
+    set -eux; \
+    make gitaly-setup; \
+    cd gitaly \
+    && go clean -cache -modcache -r \
+    && rm -rf _build/cache \
+              _build/deps/git/source \
+              _build/deps/libgit2/source \
+              _build/deps \
+              _build/intermediate
 
 # Install gitlab gem dependencies
 #
 COPY --chown=gdk:gdk Gemfile Gemfile.lock ./gitlab/
 COPY --chown=gdk:gdk vendor/gems/ ./gitlab/vendor/gems/
 COPY --chown=gdk:gdk gems/ ./gitlab/gems/
-RUN make .gitlab-bundle && rm -rf ${GEM_HOME}/cache
+RUN --mount=type=cache,target=${GEM_CACHE},uid=1000,gid=1000 \
+    make .gitlab-bundle
 
 # Install gitlab npm dependencies
 #
@@ -101,13 +68,6 @@ COPY --chown=gdk:gdk package.json yarn.lock ./gitlab/
 COPY --chown=gdk:gdk scripts/frontend/postinstall.js ./gitlab/scripts/frontend/postinstall.js
 RUN make .gitlab-yarn && yarn cache clean
 
-# Executable gdk image
-#
-FROM registry.gitlab.com/gitlab-org/gitlab/gitlab-qa-gdk-base:${BASE_TAG} as gdk
-
-ENV GITLAB_LICENSE_MODE=test \
-    GDK_KILL_CONFIRM=true
-
 # Copy code
 COPY --chown=gdk:gdk ./ ./gitlab/
 COPY --chown=gdk:gdk qa/gdk/entrypoint ../
@@ -117,18 +77,9 @@ RUN mkdir -p gitlab/tmp/pids \
     && rsync -a --remove-source-files workhorse/ gitlab/workhorse/
 
 # Set up GDK
-RUN make \
-    redis/redis.conf \
-    all \
-    && gdk kill \
-    && rm -rf ${GEM_HOME}/cache \
-           gitaly/_build/cache \
-           gitaly/_build/deps/git/source \
-           gitaly/_build/deps/libgit2/source \
-           gitaly/_build/deps \
-           gitaly/_build/intermediate \
-    && go clean -modcache \
-    && go clean -cache
+RUN SKIP_WORKHORSE_SETUP=true SKIP_GITLAB_SHELL_SETUP=true SKIP_GITALY_SETUP=true \
+    make redis/redis.conf all \
+    && gdk kill
 
 ENTRYPOINT [ "/home/gdk/entrypoint" ]
 CMD [ "gdk", "tail" ]
diff --git a/qa/gdk/Dockerfile.gdk.dockerignore b/qa/gdk/Dockerfile.gdk.dockerignore
index ef1074ed833..26062339c21 100644
--- a/qa/gdk/Dockerfile.gdk.dockerignore
+++ b/qa/gdk/Dockerfile.gdk.dockerignore
@@ -1,4 +1,10 @@
 .bundle/
+.gitlab/
+.lefthook/
+.rubocop_todo/
+.vscode/
+builds/
+changelogs/
 danger/
 doc/
 log/*.log
@@ -6,6 +12,21 @@ node_modules/
 rubocop/
 tmp/*
 
+.eslint*
+.gitlab-ci.yml
+.haml-lint*
+.prettier*
+.rubocop*
+.stylelintrc
+.yamllint
+docker-compose.yml
+Dockerfile.assets
+jest*
+lefthook.yml
+tests.yml
+
+*.md
+
 db/fixtures/
 !db/fixtures/development/01_admin.rb
 !db/fixtures/development/02_application_settings.rb
diff --git a/scripts/build_gdk_image b/scripts/build_gdk_image
index 292d315b5ec..8b1fdec2d7d 100755
--- a/scripts/build_gdk_image
+++ b/scripts/build_gdk_image
@@ -7,11 +7,12 @@ source "$(dirname "$0")/utils.sh"
 REGISTRY="${CI_REGISTRY}/${CI_PROJECT_PATH}"
 SHA_TAG="${CI_COMMIT_SHA}"
 BRANCH_TAG="${CI_COMMIT_REF_SLUG}"
-BASE_TAG=$([ "${BUILD_GDK_BASE}" == "true" ] && echo "${SHA_TAG}" || echo "master")
 
-if [[ -z "${GDK_SHA}" ]]; then
-  GDK_SHA=$(git ls-remote https://gitlab.com/gitlab-org/gitlab-development-kit.git main | cut -f 1)
-fi
+# Fetch ruby version based on contents of .ruby-version
+RUBY_VERSION=$(cat .ruby-version)
+RUBY_VERSION_MINOR=$(echo "$RUBY_VERSION" | awk -F. '{$3=0; OFS="."; print $1,$2,$3}')
+
+IMAGE="${REGISTRY}/gitlab-qa-gdk"
 
 if [[ -n "${CI}" ]]; then
   OUTPUT_OPTION="--push"
@@ -19,34 +20,18 @@ else
   OUTPUT_OPTION="--load"
 fi
 
-function build_image() {
-  local image=$1
-  local target=$2
-
-  echoinfo "Using GDK at SHA ${GDK_SHA}"
-
-  docker buildx build \
-    --cache-to="type=inline" \
-    --cache-from="${image}:${BRANCH_TAG}" \
-    --cache-from="${image}:master" \
-    --file="qa/gdk/Dockerfile.gdk" \
-    --target="${target}" \
-    --platform=${ARCH:-amd64} \
-    --tag="${image}:${SHA_TAG}" \
-    --tag="${image}:${BRANCH_TAG}" \
-    --build-arg="BASE_TAG=${BASE_TAG}" \
-    --build-arg="GDK_SHA=${GDK_SHA:-main}" \
-    ${OUTPUT_OPTION} \
-    .
-}
-
-# Rebuild base image when BUILD_GDK_BASE is set to true
-if [[ "${BUILD_GDK_BASE}" == "true" ]]; then
-  echoinfo "Building GDK base image", "yes"
-  build_image "${REGISTRY}/gitlab-qa-gdk-base" "gdk-base"
-fi
-
-echoinfo "Building GDK image", "yes"
-build_image "${REGISTRY}/gitlab-qa-gdk" "gdk"
+echoinfo "Building GDK image with GDK sha: '${GDK_SHA}'" "yes"
+
+docker buildx build \
+  --cache-to="type=inline" \
+  --cache-from="${IMAGE}:${BRANCH_TAG}" \
+  --cache-from="${IMAGE}:master" \
+  --file="qa/gdk/Dockerfile.gdk" \
+  --platform=${ARCH:-amd64} \
+  --tag="${IMAGE}:${SHA_TAG}" \
+  --tag="${IMAGE}:${BRANCH_TAG}" \
+  --build-arg="GEM_CACHE=/home/gdk/.asdf/installs/ruby/${RUBY_VERSION}/lib/ruby/gems/${RUBY_VERSION_MINOR}/cache" \
+  ${OUTPUT_OPTION} \
+  .
 
 echosuccess "Built image '${REGISTRY}/gitlab-qa-gdk:${SHA_TAG}'"