path: root/.gitlab-ci.yml

stages:
  - build
  - test
  - publish
  - qa

default:
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-golang-1.15-git-2.31
  tags:
    - gitlab-org

variables:
  DOCKER_DRIVER: overlay2
  SAST_DISABLE_DIND: "true"
  SAST_DEFAULT_ANALYZERS: "gosec"
  GIT_VERSION: "v2.33.0"
  GO_VERSION: "1.17"
  RUBY_VERSION: "2.7"
  POSTGRES_VERSION: "12.6-alpine"

include:
  - template: Workflows/MergeRequest-Pipelines.gitlab-ci.yml
  - template: Security/License-Scanning.gitlab-ci.yml
  - template: Security/SAST.gitlab-ci.yml
  - template: Security/Dependency-Scanning.gitlab-ci.yml
  - template: Security/Coverage-Fuzzing.gitlab-ci.yml
  - template: Security/Secret-Detection.gitlab-ci.yml

danger-review:
  image: ruby:2.7
  allow_failure: true
  stage: build
  only:
    - merge_requests
  except:
    - tags
    - master
  variables:
    BUNDLE_GEMFILE: danger/Gemfile
  before_script:
    - bundle install
  script:
    - >
      if [ -z "$DANGER_GITLAB_API_TOKEN" ]; then
        # Force danger to skip CI source GitLab and fallback to "local only git repo".
        unset GITLAB_CI
        # We need access to the base SHA to help danger determine the base commit for this shallow clone.
        bundle exec danger dry_run --fail-on-errors=true --verbose --base='$CI_MERGE_REQUEST_DIFF_BASE_SHA'
      else
        bundle exec danger --fail-on-errors=true --verbose
      fi

.cache: &cache_definition
  cache:
    key:
      files:
        - Makefile
        - ruby/Gemfile.lock
      prefix: git-${GIT_VERSION}-ruby-${RUBY_VERSION}
    paths:
      - _build/deps
      - _build/Makefile.sha256
      - ruby/vendor/bundle

.test_template: &test_definition
  <<: *cache_definition
  stage: test
  # Override the cache definition for pull
  cache:
    key:
      files:
        - Makefile
        - ruby/Gemfile.lock
      prefix: git-${GIT_VERSION}-ruby-${RUBY_VERSION}
    paths:
      - _build/deps
      - _build/Makefile.sha256
      - ruby/vendor/bundle
    policy: pull
  artifacts:
    paths:
    - ruby/tmp/gitaly-rspec-test.log
    when: on_failure
    expire_in: 1 week

.postgres_template: &postgres_definition
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
  services:
    - postgres:${POSTGRES_VERSION}
  variables: &postgres_variables
    PGHOST: postgres
    PGPORT: "5432"
    PGUSER: postgres
    POSTGRES_HOST_AUTH_METHOD: trust
  before_script:
    - go version
    - git version
    - while ! psql -h $PGHOST -U $PGUSER -c 'SELECT 1' > /dev/null; do echo "awaiting Postgres service to be ready..." && sleep 1 ; done && echo "Postgres service is ready!"

.pgbouncer_template: &pgbouncer_definition
  <<: *postgres_definition
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31-pgbouncer-1.14
  variables:
    <<: *postgres_variables
    PGHOST_PGBOUNCER: 0.0.0.0
    PGPORT_PGBOUNCER: "6432"
  before_script:
    - go version
    - git version
    - pgbouncer --version
    - adduser --no-create-home --disabled-password --disabled-login --quiet --force-badname --gecos '' pgbouncer-runner
    - su pgbouncer-runner -c 'pgbouncer internal/praefect/datastore/glsql/testdata/pgbouncer.ini' &
    - for i in {1..10}; do psql -U $PGUSER -c 'select now()' && break; done || { echo 'pgbouncer awaiting failed' ; exit 1; }

verify:
  <<: *cache_definition
  stage: test
  script:
    - make verify

proto:
  <<: *cache_definition
  stage: test
  script:
    - make proto no-changes
  artifacts:
    paths:
    - _build/proto.diff
    - ruby/proto/gitaly/*
    - proto/go/gitalypb/*
    when: on_failure

build:
  <<: *cache_definition
  stage: build
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
  script:
    - go version
    - make all git
    - _support/test-boot .
  parallel:
    matrix:
      # These definitions are for the non-default Git versions. We do not want
      # to apply any additional Git patches on top of those given that they may
      # not cleanly apply and thus set GIT_PATCHES="".
      - GO_VERSION: [ "1.15", "1.16", "1.17" ]
        GIT_VERSION: [ "v2.31.1", "v2.32.0" ]
        GIT_PATCHES: [ "" ]
        RUBY_VERSION: [ "2.7" ]
      # These definitions are for the default Git version where we want to
      # apply our default set of Git patches.
      - GO_VERSION: [ "1.15", "1.16", "1.17" ]
        GIT_VERSION: [ "v2.33.0" ]
        RUBY_VERSION: [ "2.7" ]

binaries:
  <<: *cache_definition
  stage: build
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
  only:
    - tags
  script:
    # Just in case we start running CI builds on other architectures in future
    - go version
    - make build
    - cd _build && sha256sum bin/* | tee checksums.sha256.txt
  artifacts:
    paths:
    - _build/checksums.sha256.txt
    - _build/bin/
    name: "${CI_JOB_NAME}:go-${GO_VERSION}-git-${GIT_VERSION}"
    expire_in: 6 months
  parallel:
    matrix:
      - GO_VERSION: [ "1.15", "1.16", "1.17" ]

test:
  <<: *test_definition
  <<: *postgres_definition
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
  script:
    - _build/deps/git/install/bin/git version
    # This command will make all directories except of our build directory and Ruby code unwritable.
    # The purpose is to verify that there is no test which writes into those directories anymore, as
    # they should all instead use a temporary directory for runtime data.
    - find . -type d \( -path ./_build -o -path ./ruby \) -prune -o -type d -exec chmod a-w {} \;
    - make ${TARGET}
  artifacts:
    paths:
      - _build/reports/go-tests-report-go-${GO_VERSION}-git-${GIT_VERSION}.xml
    reports:
      junit: _build/reports/go-tests-report-go-${GO_VERSION}-git-${GIT_VERSION}.xml
  parallel:
    matrix:
      # These definitions are for the non-default Git versions. We do not want
      # to apply any additional Git patches on top of those given that they may
      # not cleanly apply and thus set GIT_PATCHES="".
      - GO_VERSION: [ "1.15", "1.16", "1.17" ]
        GIT_VERSION: [ "v2.31.1", "v2.32.0" ]
        GIT_PATCHES: [ "" ]
        TARGET: test
      # These definitions are for the default Git version where we want to
      # apply our default set of Git patches.
      - GO_VERSION: [ "1.15", "1.16", "1.17" ]
        GIT_VERSION: [ "v2.33.0" ]
        TARGET: test
      - GO_VERSION: "1.17"
        TARGET: [ test-with-proxies, test-with-praefect, race-go ]

nightly:git:
  <<: *test_definition
  <<: *postgres_definition
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-${RUBY_VERSION}-golang-${GO_VERSION}-git-2.31
  script:
    - go version
    - make all ${TARGET}
  parallel:
    matrix:
      - GO_VERSION: "1.17"
        GIT_VERSION: ["master", "next"]
        GIT_PATCHES: [ "" ]
        TARGET: [ test, test-with-proxies, test-with-praefect ]
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'

cover:
  <<: *cache_definition
  <<: *postgres_definition
  stage: test
  script:
    - make cover
  artifacts:
    reports:
      cobertura: _build/cover/cobertura.xml

gosec-sast:
  before_script:
    - apk add pkgconfig libgit2-dev gcc libc-dev
  rules:
    - if: $SAST_DISABLED
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG

license_scanning:
  before_script:
    - sudo apt-get update
    - sudo apt-get install -y libicu-dev libgit2-dev cmake
  rules:
    - if: $LICENSE_SCANNING_DISABLED
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG
  variables:
    LICENSE_FINDER_CLI_OPTS: '--aggregate-paths=. ruby'

gemnasium-dependency_scanning:
  rules:
    - if: $DEPENDENCY_SCANNING_DISABLED
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG

secret_detection:
  inherit:
    default: false
  rules:
    - if: $SECRET_DETECTION_DISABLED
      when: never
    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG

praefect_sql_connect:
  <<: *test_definition
  services:
    - postgres:${POSTGRES_VERSION}
  variables:
    POSTGRES_DB: praefect_test
    POSTGRES_USER: praefect
    POSTGRES_PASSWORD: sql-password
  script:
    - make
    # Sanity check: direct ping with psql
    - PGPASSWORD=$POSTGRES_PASSWORD psql -h postgres -U $POSTGRES_USER -d $POSTGRES_DB -c 'select now()'
    - ruby -rerb -e 'ERB.new(ARGF.read).run' _support/config.praefect.toml.ci-sql-test.erb > config.praefect.toml
    - ./_build/bin/praefect -config config.praefect.toml sql-ping
    - ./_build/bin/praefect -config config.praefect.toml sql-migrate

pgbouncer_test:
  # This job verifies all operations run without issues if PgBouncer is set in between service and PostgreSQL database.
  <<: *test_definition
  <<: *pgbouncer_definition
  script:
    - make test-postgres

backwards_compatibility_test:
  # The job verifies the old version of the service could run its database migration on the database where the
  # migration of the newest service version was already applied.
  <<: *test_definition
  <<: *postgres_definition
  rules:
    - changes:
      - "internal/praefect/datastore/migrations/*"
  script:
    - git fetch origin 'refs/tags/*:refs/tags/*'
    - git checkout $(_support/get-previous-minor-release)
    - git checkout --no-overlay $CI_COMMIT_SHA -- internal/praefect/datastore/migrations
    - make test-postgres

lint:
  stage: test
  retry: 2
  script:
    - go version
    - make lint

objectinfo_fuzz_test:
  extends: .fuzz_base
  stage: test
  script:
    - apt update && apt install -y clang-7
    - go get github.com/dvyukov/go-fuzz/go-fuzz && go get github.com/dvyukov/go-fuzz/go-fuzz-build
    - /root/go/bin/go-fuzz-build -libfuzzer -o objectinfo_fuzzer.a ./internal/git/catfile
    - clang-7 -fsanitize=fuzzer objectinfo_fuzzer.a -o objectinfo_fuzzer
    - ./gitlab-cov-fuzz run -- ./objectinfo_fuzzer -max_total_time=300

code_navigation:
  allow_failure: true
  script:
    - go get github.com/sourcegraph/lsif-go/cmd/lsif-go@v1.3.1
    - ~/go/bin/lsif-go
  artifacts:
    reports:
      lsif: dump.lsif

trigger-qa:
  stage: qa
  when: manual
  trigger:
    project: gitlab-org/build/omnibus-gitlab-mirror
  variables:
    ALTERNATIVE_SOURCES: "true"
    GITALY_SERVER_VERSION: $CI_COMMIT_SHA
    ee: "true"