Add latest changes from gitlab-org/gitlab@master

6 files changed, 186 insertions, 89 deletions

diff --git a/qa/gdk/Dockerfile b/qa/gdk/Dockerfile
deleted file mode 100644
index ed8f3f317eb..00000000000
--- a/qa/gdk/Dockerfile
+++ /dev/null
@@ -1,50 +0,0 @@
-FROM registry.gitlab.com/gitlab-org/gitlab-development-kit/asdf-bootstrapped-gdk-installed-gitlab-e2e:ml-create-image-for-gitlab-qa-tests
-
-ENV CHROME_DRIVER_VERSION="107.0.5304.62"
-ENV CHROME_VERSION="107.0.5304.87-1"
-ENV CHROME_DEB="google-chrome-stable_${CHROME_VERSION}_amd64.deb"
-ENV CHROME_URL="https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab-build-images/packages/generic/google-chrome-stable/${CHROME_VERSION}/${CHROME_DEB}"
-
-WORKDIR /home/gdk/gdk
-
-COPY --chown=gdk qa/gdk/gdk.yml .
-
-RUN cat gdk.yml && \
-    gdk update && \
-    gdk restart && \
-    ./support/test_url http://gdk.test:3000 && \
-    gdk stop && sleep 5 && \
-    GDK_KILL_CONFIRM=true gdk kill && \
-    ps -ef && \
-    cd gitlab && git reset --hard && \
-    sudo rm -rf "$HOME/gdk/gitaly/_build/deps/git/source" \
-        "$HOME/gdk/gitaly/_build/deps/libgit2/source" \
-        "$HOME/gdk/gitaly/_build/cache" \
-        "$HOME/gdk/gitaly/_build/deps" \
-        "$HOME/gdk/gitaly/_build/intermediate" \
-        "$HOME/.cache/" \
-        "$HOME/gdk/gdk/gitlab" \
-        /tmp/*
-
-# Install Google Chrome version with headless support
-# Download from our local S3 bucket, populated by https://gitlab.com/gitlab-org/gitlab-build-images/-/blob/master/scripts/cache-google-chrome
-#
-RUN echo "${CHROME_URL}" && \
-    curl --silent --show-error --fail -O "${CHROME_URL}" && \
-    sudo apt update && \
-    sudo dpkg -i "./${CHROME_DEB}" || true && \
-    sudo apt install -f -y && \
-    rm -f "./${CHROME_DEB}"
-
-WORKDIR /home/gdk/gdk/gitlab
-
-RUN bundle install --jobs=$(nproc) --retry=3 --quiet
-RUN cd qa && \
-    bundle install --jobs=$(nproc) --retry=3 --quiet && \
-    bundle exec rake -f tasks/webdrivers.rake webdrivers:chromedriver:update[${CHROME_DRIVER_VERSION}]
-
-RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
-
-COPY --chown=gdk qa/gdk/launch .
-
-ENTRYPOINT ["./launch"]
diff --git a/qa/gdk/Dockerfile.gdk b/qa/gdk/Dockerfile.gdk
new file mode 100644
index 00000000000..a9863446e8e
--- /dev/null
+++ b/qa/gdk/Dockerfile.gdk
@@ -0,0 +1,141 @@
+# Multi-stage Dockerfile for packaging gdk as executable docker image
+# Each stage can be executed in parallel and cached separately based on changes for respective component
+# Caches are cleaned for each stage to reduce the footprint of exported cache layers
+
+ARG BASE_TAG=master
+
+FROM registry.gitlab.com/gitlab-org/gitlab-build-images/debian-bullseye-slim-ruby-3.0-golang-1.19-rust-1.65-node-16.14-postgresql-12:rubygems-3.4-git-2.36-lfs-2.9-yarn-1.22 as gdk-base
+
+RUN set -eux; \
+    groupadd gdk -g 1000; \
+    useradd gdk -m -s /bin/bash -u 1000 -g 1000
+
+ENV GEM_HOME=/home/gdk/.gem \
+    GEM_PATH=/home/gdk/.gem \
+    PATH=$PATH:/home/gdk/.gem/bin
+
+WORKDIR /home/gdk
+
+# Reinstall libpcre2 and install postgresql
+# See: https://gitlab.com/gitlab-org/gitaly/-/issues/4085
+RUN set -eux; \
+    rm -f /usr/lib/libpcre2*; \
+    apt-get update && apt-get install -y --reinstall --no-install-recommends \
+    libpcre2-16-0 \
+    libpcre2-32-0 \
+    libpcre2-8-0 \
+    libpcre2-dev \
+    libpcre2-posix2 \
+    && apt-get install -y --no-install-recommends postgresql-12; \
+    apt-get autoclean -y
+
+# Clone GDK and install system dependencies, purge system git
+RUN set -eux; \
+    git -c advice.detachedHead=false clone --depth 1 --branch ${GDK_BRANCH_OR_TAG:-main} https://gitlab.com/gitlab-org/gitlab-development-kit.git; \
+    mkdir -p gitlab-development-kit/gitlab && chown -R gdk:gdk gitlab-development-kit; \
+    apt-get update && apt-get install -y --no-install-recommends $(grep -o '^[^#]*' gitlab-development-kit/packages_debian.txt); \
+    apt-get remove -y git git-lfs; \
+    apt-get autoclean -y && apt-get autoremove -y
+
+# Allow passwordless /etc/hosts update by gdk user
+RUN echo "gdk ALL=(ALL) NOPASSWD: /usr/bin/tee -a /etc/hosts" >> /etc/sudoers
+
+USER gdk
+WORKDIR /home/gdk/gitlab-development-kit
+
+# Install GDK and gem dependencies
+ARG GDK_VERSION
+ENV GDK_VERSION=${GDK_VERSION:-0.2.16}
+RUN set -eux; \
+    gem install gitlab-development-kit -v ${GDK_VERSION} && touch .gitlab-gdk-gem; \
+    bundle install; \
+    rm -rf ${GEM_HOME}/cache
+
+COPY --chown=gdk:gdk qa/gdk/gdk.yml ./
+
+# Build gitaly
+#
+COPY --chown=gdk:gdk GITALY_SERVER_VERSION ./gitlab/
+RUN set -eux; \
+    make gitaly-setup; \
+    rm -rf gitaly/_build/cache \
+           gitaly/_build/deps/git/source \
+           gitaly/_build/deps/libgit2/source \
+           gitaly/_build/deps \
+           gitaly/_build/intermediate \
+           ${GEM_HOME}/cache \
+    && go clean -cache
+
+# Build gitlab-shell
+#
+COPY --chown=gdk:gdk GITLAB_SHELL_VERSION ./gitlab/
+RUN set -eux; \
+    make gitlab-shell-setup; \
+    rm -rf ${GEM_HOME}/cache \
+    && go clean -cache
+
+# Build gitlab-workhorse
+#
+COPY --chown=gdk:gdk VERSION GITLAB_WORKHORSE_VERSION ./gitlab/
+COPY --chown=gdk:gdk workhorse ./gitlab/workhorse
+RUN set -eux; \
+    make gitlab-workhorse-setup \
+    && mv gitlab/workhorse ./; \
+    rm -rf ${GEM_HOME}/cache \
+    && go clean -cache
+
+# Build metrics-exporter
+#
+COPY --chown=gdk:gdk GITLAB_METRICS_EXPORTER_VERSION ./gitlab/
+RUN set -eux; \
+    make gitlab-metrics-exporter-setup; \
+    go clean -cache
+
+# Install gitlab gem dependencies
+#
+COPY --chown=gdk:gdk Gemfile Gemfile.lock ./gitlab/
+COPY --chown=gdk:gdk vendor/gems ./gitlab/vendor/gems
+RUN make .gitlab-bundle && rm -rf ${GEM_HOME}/cache
+
+# Install gitlab npm dependencies
+#
+COPY --chown=gdk:gdk package.json yarn.lock ./gitlab/
+COPY --chown=gdk:gdk scripts/frontend/postinstall.js ./gitlab/scripts/frontend/postinstall.js
+RUN make .gitlab-yarn && yarn cache clean
+
+# Executable gdk image
+#
+FROM registry.gitlab.com/gitlab-org/gitlab/gitlab-qa-gdk-base:${BASE_TAG} as gdk
+
+ENV GITLAB_LICENSE_MODE=test \
+    GDK_KILL_CONFIRM=true
+
+# Copy code
+COPY --chown=gdk:gdk ./ ./gitlab/
+COPY --chown=gdk:gdk qa/gdk/entrypoint ../
+
+# Create missing pids folder and sync compiled workhorse
+RUN mkdir -p gitlab/tmp/pids \
+    && rsync -a --remove-source-files workhorse/ gitlab/workhorse/
+
+# Set up GDK
+RUN make \
+    redis/redis.conf \
+    all \
+    && gdk kill \
+    && rm -rf ${GEM_HOME}/cache \
+           gitaly/_build/cache \
+           gitaly/_build/deps/git/source \
+           gitaly/_build/deps/libgit2/source \
+           gitaly/_build/deps \
+           gitaly/_build/intermediate \
+    && go clean -modcache \
+    && go clean -cache
+
+ENTRYPOINT [ "/home/gdk/entrypoint" ]
+CMD [ "gdk", "tail" ]
+
+HEALTHCHECK --interval=10s --timeout=1s --start-period=5s --retries=17 \
+    CMD curl --fail http://0.0.0.0:3000/users/sign_in || exit 1
+
+EXPOSE 3000
diff --git a/qa/gdk/Dockerfile.gdk.dockerignore b/qa/gdk/Dockerfile.gdk.dockerignore
new file mode 100644
index 00000000000..ef1074ed833
--- /dev/null
+++ b/qa/gdk/Dockerfile.gdk.dockerignore
@@ -0,0 +1,21 @@
+.bundle/
+danger/
+doc/
+log/*.log
+node_modules/
+rubocop/
+tmp/*
+
+db/fixtures/
+!db/fixtures/development/01_admin.rb
+!db/fixtures/development/02_application_settings.rb
+!db/fixtures/development/25_api_personal_access_token.rb
+
+ee/db/fixtures/
+!ee/db/fixtures/development/10_license_file.rb
+
+scripts/
+!scripts/frontend
+
+qa/
+!qa/gdk
diff --git a/qa/gdk/entrypoint b/qa/gdk/entrypoint
new file mode 100755
index 00000000000..3b86f19b9b2
--- /dev/null
+++ b/qa/gdk/entrypoint
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+# host file can't be modified during docker build because it is created on container start
+# this makes for a more portable solution as other alternative is to use `--add-host` option for 'docker run' command
+# We want a non-loopback ip otherwise some services (maybe just workhorse?) bind to localhost and can't be accessed
+# from outside the container
+echo "$(hostname -i) gdk.test" | sudo tee -a /etc/hosts > /dev/null
+
+# make sure we don't override existing BUNDLE_PATH which happens on CI where all job variables are also passed in to service
+unset BUNDLE_PATH
+
+gdk start
+
+# /builds folder is present when running as a service on CI
+if [ -d /builds ]; then
+  exec "$@" | tee -a /builds/gdk.log
+else
+  exec "$@" | tee -a ${HOME}/gitlab-development-kit/gitlab/log/gdk.log
+fi
diff --git a/qa/gdk/gdk.yml b/qa/gdk/gdk.yml
index 0494cd0d3c1..649ac9a60c6 100644
--- a/qa/gdk/gdk.yml
+++ b/qa/gdk/gdk.yml
@@ -6,12 +6,15 @@ webpack:
   live_reload: false
   sourcemaps: false
   incremental: false
+  static: true
 gdk:
   ask_to_restart_after_update: false
   auto_reconfigure: false
   overwrite_changes: true
   quiet: false
 gitlab:
+  cache_classes: true
+  lefthook_enabled: false
   rails:
     bootsnap: false
     hostname: gdk.test
diff --git a/qa/gdk/launch b/qa/gdk/launch
deleted file mode 100755
index 249484b417c..00000000000
--- a/qa/gdk/launch
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-
-COMMIT_REF=${1:-$CI_COMMIT_SHA}
-RSPEC_ARGS=$2
-
-if [ -z "${COMMIT_REF}" ]; then
-    echo "Please provide a commit ref with the code to be tested as the first argument"
-    exit 1
-fi
-
-# Set the GitLab license mode to "test" so that GitLab uses the appropriate encryption key
-export GITLAB_LICENSE_MODE="test"
-
-# Create the temporary directory that screenshots are saved to
-sudo install -m 777 -d /home/gdk/gdk/gitlab/qa/tmp
-
-# Update GDK
-(cd .. ; gdk update ; cat gdk.yml)
-
-# Reset, fetch, and checkout the GitLab repository with the code from the ref to be tested
-git reset --hard
-git fetch origin $COMMIT_REF
-git checkout $COMMIT_REF
-
-# Install the required gems
-bundle install --jobs=$(nproc) --retry=3 --quiet
-
-# Run the database migrations
-bundle exec rake db:migrate
-
-# Restart GDK to be sure any changes are accounted for in running services, start any stopped services, and wait until the GDK is reachable
-(cd .. ; gdk restart ; ./support/test_url http://gdk.test:3000)
-
-# Install the required gems in the QA directory
-cd qa
-bundle install --jobs=$(nproc) --retry=3 --quiet
-
-# Run the tests
-bundle exec bin/qa Test::Instance::All http://gdk.test:3000 -- $RSPEC_ARGS