path: root/.gitlab-ci.yml

stages:
  - build
  - test
  - qa

default:
  image: registry.gitlab.com/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}:git-2.36
  tags:
    - gitlab-org

variables:
  FF_USE_FASTZIP: "true"
  FF_NETWORK_PER_BUILD: "true"
  SAST_DISABLE_DIND: "true"
  SAST_DEFAULT_ANALYZERS: "gosec"
  DEBIAN_VERSION: "bullseye"
  UBI_VERSION: "8.6"
  # We use Gitaly's Git version by default.
  GIT_VERSION: "default"
  GO_VERSION: "1.18"
  RUBY_VERSION: "2.7"
  POSTGRES_VERSION: "12.6-alpine"
  PGBOUNCER_VERSION: "1.16.1"
  BUNDLE_PATH: "${CI_PROJECT_DIR}/.ruby"
  GOPATH: "${CI_PROJECT_DIR}/.go"
  # Run tests with an intercepted home directory so that we detect cases where
  # Gitaly picks up the gitconfig even though it ought not to.
  GITALY_TESTING_INTERCEPT_HOME: "YesPlease"
  # TEST_UID is the user ID we use to run tests in an unprivileged way. 9999 is
  # chosen as a semi-random value so as to not interfer with any preexisting
  # users.
  TEST_UID: 9999

include:
  - template: Workflows/MergeRequest-Pipelines.gitlab-ci.yml
  - template: Security/License-Scanning.gitlab-ci.yml
  - template: Security/SAST.gitlab-ci.yml
  - template: Security/Dependency-Scanning.gitlab-ci.yml
  - template: Security/Secret-Detection.gitlab-ci.yml
  - project: 'gitlab-org/quality/pipeline-common'
    file:
      - '/ci/danger-review.yml'

.cache_deps:
  cache: &cache_deps_configuration
    key:
      files:
        - .gitlab-ci.yml
        - Makefile
      prefix: git-${GIT_VERSION}
    paths:
      - _build/deps
      - _build/tools
    policy: pull

.cache_gems:
  cache: &cache_gems_configuration
    key:
      files:
        - .gitlab-ci.yml
        - ruby/Gemfile.lock
      prefix: debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}
    paths:
      - .ruby
    policy: pull

.cache_go:
  cache: &cache_go_configuration
    key:
      files:
        - .gitlab-ci.yml
        - go.sum
      prefix: go-${GO_VERSION}
    paths:
      - .go/pkg/mod
      - _build/cache
    policy: pull

.test_template: &test_definition
  needs: []
  stage: test
  cache:
    - *cache_deps_configuration
    - *cache_gems_configuration
    - *cache_go_configuration
  services:
    - name: postgres:${POSTGRES_VERSION}
      alias: postgres
      command: ["postgres", "-c", "max_connections=500"]
  variables: &test_variables
    PGHOST: postgres
    PGPORT: 5432
    PGUSER: postgres
    POSTGRES_DB: praefect_test
    POSTGRES_HOST_AUTH_METHOD: trust
    TEST_REPORT: _unprivileged/go-tests-report.xml
    TEST_COVERAGE_DIR: _unprivileged
    TEST_FULL_OUTPUT: /tmp/test-output.log
  before_script: &test_before_script
    - go version
    # Create a directory for the unprivileged user that we're running tests as.
    # This is required so that we can still store test reports successfully.
    - install --directory --owner=${TEST_UID} --group=${TEST_UID} _unprivileged
    # We need to explicitly build all prerequisites so that we can run tests unprivileged.
    - make -j$(nproc) build prepare-tests $(pwd)/_build/tools/gocover-cobertura $(test "${GIT_VERSION}" = default && echo WITH_BUNDLED_GIT=YesPlease)
  script:
    # But the actual tests should run unprivileged. This assures that we pay
    # proper attention to permission bits and that we don't modify the source
    # directory.
    - setpriv --reuid=${TEST_UID} --regid=${TEST_UID} --clear-groups --no-new-privs make ${TEST_TARGET} UNPRIVILEGED_CI_SKIP=YesPlease $(test "${GIT_VERSION}" = default && echo WITH_BUNDLED_GIT=YesPlease)

  after_script:
    - |
      # Checking for panics in ${TEST_FULL_OUTPUT}
      if [ "${CI_JOB_STATUS}" = "failed" ] && grep 'Output":"panic' "${TEST_FULL_OUTPUT}" > /dev/null; then
        echo -e "\e[0Ksection_start:`date +%s`:panic_stack_traces[collapsed=true]\r\e[0K\e[0;31mPanic stack traces\e[0m"
        ruby -e "require 'json'; f = File.read(ENV['TEST_FULL_OUTPUT']); f.lines.each do |l| out = JSON.parse(l); puts out['Output']; end" | awk '/^panic/ || /goroutine/,/^\s*$/'
        echo -e "\e[0Ksection_end:`date +%s`:panic_stack_traces\r\e[0K"
      fi
  artifacts:
    paths:
      - ruby/tmp/gitaly-rspec-test.log
    reports:
      junit: ${TEST_REPORT}
    when: on_failure
    expire_in: 1 week

danger-review:
  needs: []
  stage: build
  allow_failure: true
  variables:
    GITLAB_DANGERFILES_VERSION: "3.1.0"

build:
  needs: []
  stage: build
  cache:
    - <<: *cache_deps_configuration
      policy: pull-push
    - <<: *cache_gems_configuration
      policy: pull-push
    - <<: *cache_go_configuration
      policy: pull-push
  script:
    - go version
    - make -j$(nproc) build $(pwd)/_build/tools/protoc $(test "${GIT_VERSION}" = "default" && echo "build-bundled-git" || echo "git")
    - _support/test-boot . ${TEST_BOOT_ARGS}
  parallel:
    matrix:
      - GO_VERSION: [ "1.17", "1.18" ]
        TEST_BOOT_ARGS: "--bundled-git"
      - GIT_VERSION: "v2.33.0"

build:binaries:
  needs: []
  stage: build
  cache:
    - *cache_deps_configuration
    - *cache_go_configuration
  only:
    - tags
  script:
    # Just in case we start running CI builds on other architectures in future
    - go version
    - make -j$(nproc) build
    - cd _build && sha256sum bin/* | tee checksums.sha256.txt
  artifacts:
    paths:
    - _build/checksums.sha256.txt
    - _build/bin/
    name: "${CI_JOB_NAME}:go-${GO_VERSION}-git-${GIT_VERSION}"
    expire_in: 6 months
  parallel:
    matrix:
      - GO_VERSION: [ "1.17", "1.18" ]

test:
  <<: *test_definition
  parallel:
    matrix:
      # The following jobs all test with our default Git version, which is
      # using bundled Git binaries.
      - GO_VERSION: [ "1.17", "1.18" ]
        TEST_TARGET: test
      - TEST_TARGET: [ test-with-proxies, test-with-praefect, race-go ]
      # We also verify that things work as expected with a non-bundled Git
      # version matching our minimum required Git version.
      - TEST_TARGET: test
        GIT_VERSION: "v2.33.0"
      # Execute tests with our minimum required Postgres version, as well. If
      # the minimum version changes, please change this to the new minimum
      # version. Furthermore, please make sure to update the minimum required
      # version in `datastore.CheckPostgresVersion()`.
      - POSTGRES_VERSION: "11.14-alpine"
        TEST_TARGET: [ test, test-with-praefect ]

test:coverage:
  <<: *test_definition
  coverage: /^total:\t+\(statements\)\t+\d+\.\d+%$/
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: ${TEST_COVERAGE_DIR}/cobertura.xml
  variables:
    <<: *test_variables
    TEST_TARGET: "cover"

test:pgbouncer:
  <<: *test_definition
  services:
    - name: postgres:${POSTGRES_VERSION}
      alias: postgres
      command: ["postgres", "-c", "max_connections=500"]
    - name: bitnami/pgbouncer:${PGBOUNCER_VERSION}
      alias: pgbouncer
  variables:
    <<: *test_variables
    # The following variables are used by PgBouncer to connect to Postgres.
    POSTGRESQL_HOST: "${PGHOST}"
    # The image doesn't support setting `auth_user`, so we're cheating and use
    # "command line injection" here. In any case, `auth_user` is required so
    # that we can connect as a different user, but authenticate as the actual
    # PGUSER. We can fix this when
    # https://github.com/bitnami/bitnami-docker-pgbouncer/pull/22 lands.
    POSTGRESQL_PORT: "${PGPORT} auth_user=${PGUSER}"
    POSTGRESQL_USERNAME: "${PGUSER}"
    # These variables define how PgBouncer itself is configured
    PGBOUNCER_AUTH_TYPE: trust
    PGBOUNCER_DATABASE: "*"
    PGBOUNCER_IGNORE_STARTUP_PARAMETERS: extra_float_digits
    PGBOUNCER_POOL_MODE: transaction
    PGBOUNCER_MAX_DB_CONNECTIONS: 100
    # And these are finally used by Gitaly's tests.
    PGHOST_PGBOUNCER: pgbouncer
    PGPORT_PGBOUNCER: 6432
    # We need to enable per-build networking such that the PgBouncer service
    # can reach Postgres.
    TEST_TARGET: "test-with-praefect"

test:nightly:
  <<: *test_definition
  parallel:
    matrix:
      - GIT_VERSION: [ "master", "next" ]
        TEST_TARGET: [ test, test-with-proxies, test-with-praefect ]
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - when: manual
      allow_failure: true

test:praefect_smoke:
  <<: *test_definition
  script:
    - make -j$(nproc)
    - ruby -rerb -e 'ERB.new(ARGF.read).run' _support/config.praefect.toml.ci-sql-test.erb > config.praefect.toml
    - ./_build/bin/praefect -config config.praefect.toml sql-ping
    - ./_build/bin/praefect -config config.praefect.toml sql-migrate

test:sha256:
  <<: *test_definition
  parallel:
    matrix:
      - TEST_TARGET: [ test, test-with-praefect ]
        GITALY_TESTING_ENABLE_SHA256: "YesPlease"

test:fips:
  <<: *test_definition
  tags:
    - fips
  image: registry.gitlab.com/gitlab-org/gitlab-build-images/ubi-${UBI_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}:git-2.36
  before_script:
    - test "$(cat /proc/sys/crypto/fips_enabled)" = "1" || (echo "System is not running in FIPS mode" && exit 1)
    - *test_before_script
  parallel:
    matrix:
      - TEST_TARGET: [ test, test-with-praefect ]
        FIPS_MODE: "YesPlease"
  rules:
    # When running in a fork of Gitaly we don't have the runners available, and
    # consequentially this job would fail anyway. So we configure the job to be
    # a manual one in that case.
    - if: $CI_PROJECT_PATH != "gitlab-org/gitaly"
      when: manual
      allow_failure: true
    # Otherwise, we automatically run the job when either merging to the
    # default branch or when the merge request has a FIPS label.
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_LABELS =~ /FIPS/
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      when: manual
      allow_failure: true

verify:
  needs: []
  stage: test
  cache:
    - *cache_deps_configuration
    - *cache_gems_configuration
    - *cache_go_configuration
  script:
    # Download the dependencies in case there was no cache hit, otherwise
    # golang-ci lint will timeout downloading them.
    - go mod download
    - make -j$(nproc) verify
  artifacts:
    paths:
      - _build/proto.diff
      - ruby/proto/gitaly/*
      - proto/go/gitalypb/*
    when: on_failure

dbschema:
  needs: []
  stage: test
  cache:
    - *cache_deps_configuration
    - *cache_gems_configuration
    - *cache_go_configuration
  services:
    # The database version we use must match the version of `pg_dump` we have
    # available in the build image.
    - postgres:11.13-alpine
  variables:
    <<: *test_variables
  script:
    - make dump-database-schema no-changes
  artifacts:
    paths:
      - _support/praefect-schema.sql
    when: on_failure

gosec-sast:
  needs: []
  cache:
    - *cache_go_configuration
  variables:
    GOPATH: "/go"
  before_script:
    # Our pipeline places GOPATH to $CI_PROJECT_DIR/.go so it can be cached.
    # This causes gosec-sast to find the module cache and scan all the sources of
    # the dependencies as well. This makes the scan time grow massively. This is
    # avoided by this job moving the GOPATH outside of the project directory along
    # with the cached modules if they were successfully extracted.
    #
    # SAST_EXCLUDED_PATHS is not sufficient as it only filters out the results but
    # still performs the expensive scan.
    - if [ -d .go ]; then mv .go $GOPATH; fi
  rules:
    - if: $SAST_DISABLED
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG

license_scanning:
  needs: []
  rules:
    - if: $LICENSE_SCANNING_DISABLED
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG
  variables:
    LICENSE_FINDER_CLI_OPTS: '--aggregate-paths=. ruby'

gemnasium-dependency_scanning:
  needs: []
  rules:
    - if: $DEPENDENCY_SCANNING_DISABLED
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG

secret_detection:
  needs: []
  rules:
    - if: $SECRET_DETECTION_DISABLED
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG

trigger-qa:
  needs: []
  stage: qa
  trigger:
    project: gitlab-org/build/omnibus-gitlab-mirror
  variables:
    ALTERNATIVE_SOURCES: "true"
    GITALY_SERVER_VERSION: $CI_COMMIT_SHA
    GITALY_SERVER_ALTERNATIVE_REPO: $CI_PROJECT_URL
    ee: "true"
  rules:
    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA != ""
      when: manual
      allow_failure: true
      variables:
        # Downstream pipeline does not fetch the merged result SHA.
        # Fix: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/6482
        GITALY_SERVER_VERSION: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
    - when: manual
      allow_failure: true

qa:nightly-praefect-migration-test:
  needs: []
  stage: qa
  trigger:
    project: gitlab-org/quality/praefect-migration-testing
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'