path: root/.gitlab/ci/global.gitlab-ci.yml

.default-retry:
  retry:
    max: 2  # This is confusing but this means "3 runs at max".
    when:
      - api_failure
      - data_integrity_failure
      - runner_system_failure
      - scheduler_failure
      - stuck_or_timeout_failure
      - unknown_failure

.default-utils-before_script:
  before_script:
    - echo $FOSS_ONLY
    - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
    - export GOPATH=$CI_PROJECT_DIR/.go
    - mkdir -p $GOPATH
    - source scripts/utils.sh

.default-before_script:
  before_script:
    - !reference [.default-utils-before_script, before_script]
    - source scripts/prepare_build.sh

.production:
  variables:
    RAILS_ENV: "production"
    NODE_ENV: "production"
    GITLAB_ALLOW_SEPARATE_CI_DATABASE: "true"

.ruby-gems-cache: &ruby-gems-cache
  key: "ruby-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}"
  paths:
    - vendor/ruby/
  policy: pull

.ruby-gems-cache-push: &ruby-gems-cache-push
  <<: *ruby-gems-cache
  policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

.ruby-coverage-gems-cache: &ruby-coverage-gems-cache
  key: "ruby-coverage-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}"
  paths:
    - vendor/ruby/
  policy: pull

.ruby-coverage-gems-cache-push: &ruby-coverage-gems-cache-push
  <<: *ruby-coverage-gems-cache
  policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

.gitaly-binaries-cache: &gitaly-binaries-cache
  key:
    files:
      - GITALY_SERVER_VERSION
      - lib/gitlab/setup_helper.rb
    prefix: "gitaly-binaries-debian-${DEBIAN_VERSION}"
  paths:
    - ${TMP_TEST_FOLDER}/gitaly/_build/bin/
    - ${TMP_TEST_FOLDER}/gitaly/_build/deps/git/install/
    - ${TMP_TEST_FOLDER}/gitaly/config.toml
    - ${TMP_TEST_FOLDER}/gitaly/gitaly2.config.toml
    - ${TMP_TEST_FOLDER}/gitaly/internal/
    - ${TMP_TEST_FOLDER}/gitaly/run/
    - ${TMP_TEST_FOLDER}/gitaly/run2/
    - ${TMP_TEST_FOLDER}/gitaly/Makefile
    - ${TMP_TEST_FOLDER}/gitaly/praefect.config.toml
    - ${TMP_TEST_FOLDER}/gitaly/praefect-db.config.toml
  policy: pull

.go-pkg-cache: &go-pkg-cache
  key: "go-pkg-${DEBIAN_VERSION}"
  paths:
    - .go/pkg/mod/
  policy: pull

.go-pkg-cache-push: &go-pkg-cache-push
  <<: *go-pkg-cache
  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

.node-modules-cache: &node-modules-cache
  key: "node-modules-${DEBIAN_VERSION}-${NODE_ENV}"
  paths:
    - node_modules/
    - tmp/cache/webpack-dlls/
  policy: pull

.node-modules-cache-push: &node-modules-cache-push
  <<: *node-modules-cache
  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

.assets-tmp-cache: &assets-tmp-cache
  key: "assets-tmp-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}-v1"
  paths:
    - tmp/cache/assets/sprockets/
    - tmp/cache/babel-loader/
    - tmp/cache/vue-loader/
  policy: pull

.assets-tmp-cache-push: &assets-tmp-cache-push
  <<: *assets-tmp-cache
  policy: push  # We want to rebuild the cache from scratch to ensure we don't pile up outdated cache files.

.storybook-node-modules-cache: &storybook-node-modules-cache
  key: "storybook-node-modules-${DEBIAN_VERSION}-${NODE_ENV}"
  paths:
    - storybook/node_modules/
  policy: pull

.storybook-node-modules-cache-push: &storybook-node-modules-cache-push
  <<: *storybook-node-modules-cache
  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

.rubocop-cache: &rubocop-cache
  key: "rubocop-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}"
  paths:
    - tmp/rubocop_cache/
  policy: pull

.rubocop-cache-push: &rubocop-cache-push
  <<: *rubocop-cache
  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
  # for keeping only the N latest cache files, so we take advantage of it with `pull-push`.
  policy: push

.qa-ruby-gems-cache: &qa-ruby-gems-cache
  key:
    prefix: "qa-ruby-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}"
    files:
      - qa/Gemfile.lock
  paths:
    - qa/vendor/ruby
  policy: pull

.qa-ruby-gems-cache-push: &qa-ruby-gems-cache-push
  <<: *qa-ruby-gems-cache
  policy: pull-push

.setup-test-env-cache:
  cache:
    - *ruby-gems-cache
    - *gitaly-binaries-cache
    - *go-pkg-cache

.setup-test-env-cache-push:
  cache:
    - *ruby-gems-cache-push
    - *go-pkg-cache-push

.gitaly-binaries-cache-push:
  cache:
    - <<: *gitaly-binaries-cache
      policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

.ruby-cache:
  cache:
    - *ruby-gems-cache

.static-analysis-cache:
  cache:
    - *ruby-gems-cache
    - *node-modules-cache
    - *rubocop-cache

.rubocop-job-cache:
  cache:
    - *ruby-gems-cache
    - *rubocop-cache

.rubocop-job-cache-push:
  cache:
    - *ruby-gems-cache  # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
    - *rubocop-cache-push

.ruby-gems-coverage-cache:
  cache:
    - *ruby-coverage-gems-cache

.ruby-gems-coverage-cache-push:
  cache:
    - *ruby-coverage-gems-cache-push

.ruby-node-cache:
  cache:
    - *ruby-gems-cache
    - *node-modules-cache

.qa-bundler-variables: &qa-bundler-variables
  variables:
    BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES: "true"
    BUNDLE_SILENCE_ROOT_WARNING: "true"
    BUNDLE_PATH: vendor

.qa-cache:
  <<: *qa-bundler-variables
  cache:
    - *qa-ruby-gems-cache

.qa-cache-push:
  <<: *qa-bundler-variables
  cache:
    - *qa-ruby-gems-cache-push

.yarn-cache:
  cache:
    - *node-modules-cache

.assets-compile-cache:
  cache:
    - *ruby-gems-cache
    - *node-modules-cache
    - *assets-tmp-cache

.assets-compile-cache-push:
  cache:
    - *ruby-gems-cache  # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
    - *node-modules-cache-push
    - *assets-tmp-cache-push

.storybook-yarn-cache:
  cache:
    - *node-modules-cache
    - *storybook-node-modules-cache

.storybook-yarn-cache-push:
  cache:
    - *node-modules-cache  # We don't push this cache as it's already rebuilt by `update-assets-compile-*-cache`
    - *storybook-node-modules-cache-push

.redis-services:
  services:
    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:redis-cluster-6.2.12
      alias: rediscluster # configure connections in config/redis.yml
    - name: redis:${REDIS_VERSION}-alpine

.pg-base-variables:
  variables:
    POSTGRES_HOST_AUTH_METHOD: trust

.db-services:
  services:
    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-${PG_VERSION}-pgvector-0.4.1
      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off", "-c", "max_locks_per_transaction=256"]
      alias: postgres
    - !reference [.redis-services, services]

.db-services-with-auto-explain:
  services:
    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:postgres-${PG_VERSION}-pgvector-0.4.1
      command:
        - postgres
        - -c
        - fsync=off
        - -c
        - synchronous_commit=off
        - -c
        - full_page_writes=off
        - -c
        - max_locks_per_transaction=256
        - -c
        - log_filename=pglog
        - -c
        - log_destination=csvlog
        - -c
        - logging_collector=true
        - -c
        - auto_explain.log_min_duration=0
        - -c
        - auto_explain.log_format=json
        - -c
        - auto_explain.log_timing=off
      alias: postgres
    - !reference [.redis-services, services]

.zoekt-variables:
  variables:
    ZOEKT_INDEX_BASE_URL: http://zoekt-ci-image:6060
    ZOEKT_SEARCH_BASE_URL: http://zoekt-ci-image:6070

.zoekt-services:
  services:
    - name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:zoekt-ci-image-1.2
      alias: zoekt-ci-image

.use-pg12:
  extends:
    - .pg-base-variables
  services:
    - !reference [.db-services, services]
  variables:
    PG_VERSION: "12"
    REDIS_VERSION: "6.2"

.use-pg13:
  extends:
    - .pg-base-variables
  services:
    - !reference [.db-services, services]
  variables:
    PG_VERSION: "13"
    REDIS_VERSION: "6.2"

.use-pg14:
  extends:
    - .pg-base-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
  variables:
    PG_VERSION: "14"
    REDIS_VERSION: "6.2"

.use-pg15:
  extends:
    - .pg-base-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
  variables:
    PG_VERSION: "15"
    REDIS_VERSION: "7.0"

.es7-services:
  services:
    - !reference [.zoekt-services, services]
    - name: elasticsearch:7.17.6
      command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false", "-E", "cluster.routing.allocation.disk.threshold_enabled=false"]

.use-pg12-es7-ee:
  extends:
    - .use-pg12
    - .zoekt-variables
  services:
    - !reference [.db-services, services]
    - !reference [.es7-services, services]

.use-pg13-es7-ee:
  extends:
    - .use-pg13
    - .zoekt-variables
  services:
    - !reference [.db-services, services]
    - !reference [.es7-services, services]

.use-pg14-es7-ee:
  extends:
    - .use-pg14
    - .zoekt-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
    - !reference [.es7-services, services]

.use-pg15-es7-ee:
  extends:
    - .use-pg15
    - .zoekt-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
    - !reference [.es7-services, services]

.es8-services:
  services:
    - !reference [.zoekt-services, services]
    - name: elasticsearch:8.6.2

.es8-variables:
  variables:
    ES_SETTING_DISCOVERY_TYPE: "single-node"
    ES_SETTING_XPACK_SECURITY_ENABLED: "false"
    ES_SETTING_CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD__ENABLED: "false"

.use-pg13-es8-ee:
  extends:
    - .use-pg13
    - .zoekt-variables
    - .es8-variables
  services:
    - !reference [.db-services, services]
    - !reference [.es8-services, services]

.use-pg14-es8-ee:
  extends:
    - .use-pg14
    - .zoekt-variables
    - .es8-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
    - !reference [.es8-services, services]

.use-pg15-es8-ee:
  extends:
    - .use-pg15
    - .zoekt-variables
    - .es8-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
    - !reference [.es8-services, services]

.os1-services:
  services:
    - !reference [.zoekt-services, services]
    - name: opensearchproject/opensearch:1.3.5
      alias: elasticsearch
      command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true", "-E", "cluster.routing.allocation.disk.threshold_enabled=false"]

.use-pg13-opensearch1-ee:
  extends:
    - .use-pg13
    - .zoekt-variables
  services:
    - !reference [.db-services, services]
    - !reference [.os1-services, services]

.use-pg14-opensearch1-ee:
  extends:
    - .use-pg14
    - .zoekt-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
    - !reference [.os1-services, services]

.use-pg15-opensearch1-ee:
  extends:
    - .use-pg15
    - .zoekt-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
    - !reference [.os1-services, services]

.os2-services:
  services:
    - !reference [.zoekt-services, services]
    - name: opensearchproject/opensearch:2.2.1
      alias: elasticsearch
      command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true", "-E", "cluster.routing.allocation.disk.threshold_enabled=false"]

.use-pg13-opensearch2-ee:
  extends:
    - .use-pg13
    - .zoekt-variables
  services:
    - !reference [.db-services, services]
    - !reference [.os2-services, services]

.use-pg14-opensearch2-ee:
  extends:
    - .use-pg14
    - .zoekt-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
    - !reference [.os2-services, services]

.use-pg15-opensearch2-ee:
  extends:
    - .use-pg15
    - .zoekt-variables
  services:
    - !reference [.db-services-with-auto-explain, services]
    - !reference [.os2-services, services]

.use-pg14-clickhouse23:
  extends: .use-pg14
  services:
    - !reference [.db-services-with-auto-explain, services]
    - name: clickhouse/clickhouse-server:23.11.3.23-alpine
      alias: clickhouse
  variables:
    CLICKHOUSE_USER: clickhouse
    CLICKHOUSE_PASSWORD: clickhouse
    CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    CLICKHOUSE_DB: gitlab_clickhouse_test

.use-kaniko:
  image:
    name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:kaniko
    entrypoint: [""]
  before_script:
    - source scripts/utils.sh
    - mkdir -p /kaniko/.docker
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json

.as-if-foss:
  variables:
    FOSS_ONLY: '1'

.use-docker-in-docker:
  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}docker:${DOCKER_VERSION}
  services:
    - docker:${DOCKER_VERSION}-dind
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
  tags:
    # See https://gitlab.com/gitlab-com/www-gitlab-com/-/issues/7019 for tag descriptions
    - gitlab-org-docker

.use-buildx:
  extends: .use-docker-in-docker
  image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-slim:docker-${DOCKER_VERSION}
  variables:
    QEMU_IMAGE: tonistiigi/binfmt:qemu-v7.0.0
  before_script:
    - !reference [.default-utils-before_script, before_script]
    - echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" --password-stdin
    - |
      if [[ "${ARCH}" =~ arm64 ]]; then
        echo -e "\033[1;33mInstalling latest qemu emulators\033[0m"
        docker pull -q ${QEMU_IMAGE};
        docker run --rm --privileged ${QEMU_IMAGE} --uninstall qemu-*;
        docker run --rm --privileged ${QEMU_IMAGE} --install all;
      fi
    - docker buildx create --use  # creates and set's to active buildkit builder

.use-kube-context:
  before_script:
    - export KUBE_CONTEXT="${CI_PROJECT_NAMESPACE}/gitlab:review-apps"
    - kubectl config use-context ${KUBE_CONTEXT}

.fast-no-clone-job:
  variables:
    GIT_STRATEGY: none # We will download the required files for the job from the API
  before_script:
    # Logic taken from scripts/utils.sh in download_files function
    - |
      url="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/repository/files/scripts%2Futils.sh/raw?ref=${CI_COMMIT_SHA}"
      curl -f --header "Private-Token: ${PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE}" "${url}" --create-dirs --output scripts/utils.sh
    - source scripts/utils.sh
    - run_timed_command "download_files ${FILES_TO_DOWNLOAD}"