path: root/.gitlab/ci/rails.gitlab-ci.yml

#######################
# rspec job base specs
.rails-job-base:
  extends:
    - .default-retry
    - .default-before_script
    - .rails-cache

.base-script: &base-script
  # Only install knapsack after bundle install! Otherwise oddly some native
  # gems could not be found under some circumstance. No idea why, hours wasted.
  - run_timed_command "gem install knapsack --no-document"
  - run_timed_command "scripts/gitaly-test-build"
  - run_timed_command "scripts/gitaly-test-spawn"
  - source ./scripts/rspec_helpers.sh

.rspec-base:
  extends: .rails-job-base
  stage: test
  variables:
    RUBY_GC_MALLOC_LIMIT: 67108864
    RUBY_GC_MALLOC_LIMIT_MAX: 134217728
    CRYSTALBALL: "true"
  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
  script:
    - *base-script
    - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration"
  artifacts:
    expire_in: 31d
    when: always
    paths:
      - coverage/
      - crystalball/
      - knapsack/
      - rspec_flaky/
      - rspec_profiling/
      - tmp/capybara/
      - tmp/memory_test/
      - tmp/feature_flags/
      - log/*.log
    reports:
      junit: junit_rspec.xml

.rspec-base-migration:
  extends: .rails:rules:ee-and-foss-migration
  script:
    - *base-script
    - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration"

.rspec-base-pg11:
  extends:
    - .rspec-base
    - .use-pg11

.rspec-base-pg12:
  extends:
    - .rspec-base
    - .use-pg12

.rspec-base-pg11-as-if-foss:
  extends:
    - .rspec-base
    - .as-if-foss
    - .use-pg11
  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"]

.rspec-ee-base-pg11:
  extends:
    - .rspec-base
    - .use-pg11-ee

.rspec-ee-base-pg12:
  extends:
    - .rspec-base
    - .use-pg12-ee

.rspec-ee-base-geo:
  extends: .rspec-base
  script:
    - *base-script
    - rspec_paralellized_job "--tag ~quarantine --tag geo"

.rspec-ee-base-geo-pg11:
  extends:
    - .rspec-ee-base-geo
    - .use-pg11-ee

.rspec-ee-base-geo-pg12:
  extends:
    - .rspec-ee-base-geo
    - .use-pg12-ee

.db-job-base:
  extends:
    - .rails-job-base
    - .rails:rules:ee-and-foss-migration
    - .use-pg11
  stage: test
  needs: ["setup-test-env"]
# rspec job base specs
######################

############################
# rspec job parallel configs
.rspec-migration-parallel:
  parallel: 5

.rspec-ee-migration-parallel:
  parallel: 2

.rspec-unit-parallel:
  parallel: 20

.rspec-ee-unit-parallel:
  parallel: 10

.rspec-ee-unit-geo-parallel:
  parallel: 2

.rspec-integration-parallel:
  parallel: 8

.rspec-ee-integration-parallel:
  parallel: 4

.rspec-system-parallel:
  parallel: 24

.rspec-ee-system-parallel:
  parallel: 6
# rspec job parallel configs
############################

#######################################################
# EE/FOSS: default refs (MRs, master, schedules) jobs #
setup-test-env:
  extends:
    - .rails-job-base
    - .rails:rules:default-refs-code-backstage-qa
    - .use-pg11
  stage: prepare
  variables:
    GITLAB_TEST_EAGER_LOAD: "0"
  script:
    - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'"
    - run_timed_command "scripts/gitaly-test-build"  # Do not use 'bundle exec' here
    - rm tmp/tests/gitaly/.ruby-bundle  # This file prevents gems from being installed even if vendor/gitaly-ruby is missing
  artifacts:
    expire_in: 7d
    paths:
      - config/secrets.yml
      - tmp/tests/gitaly
      - tmp/tests/gitlab-elasticsearch-indexer
      - tmp/tests/gitlab-shell
      - tmp/tests/gitlab-test-fork
      - tmp/tests/gitlab-test-fork_bare
      - tmp/tests/gitlab-test
      - tmp/tests/gitlab-workhorse
      - tmp/tests/repositories
      - tmp/tests/second_storage
    when: always

update-rails-cache:
  extends:
    - setup-test-env
    - .shared:rules:update-cache
  artifacts: {}  # This job's purpose is only to update the cache.
  cache:
    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

.coverage-base:
  extends:
    - .default-retry
    - .default-before_script
    - .coverage-cache
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"

update-coverage-cache:
  extends:
    - .coverage-base
    - .shared:rules:update-cache
  stage: prepare
  script:
    - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --without default development test production puma unicorn kerberos metrics omnibus ed25519"
  cache:
    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

.static-analysis-base:
  extends:
    - .default-retry
    - .default-before_script
    - .static-analysis-cache
  needs: []
  variables:
    SETUP_DB: "false"
    ENABLE_SPRING: "1"

update-static-analysis-cache:
  extends:
    - .static-analysis-base
    - .shared:rules:update-cache
  stage: prepare
  script:
    - rm -rf ./node_modules  # We remove node_modules because there's no mechanism to remove stall entries.
    - run_timed_command "retry yarn install --frozen-lockfile"
    - run_timed_command "bundle exec rubocop --parallel"  # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks,
  cache:
    # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
    # for keeping only the N latest cache files, so we take advantage of it with `pull-push` and removing `node_modules` at the start of the job.
    policy: pull-push

static-analysis:
  extends:
    - .static-analysis-base
    - .rails:rules:default-refs-code-backstage-qa
  stage: test
  parallel: 4
  script:
    - run_timed_command "retry yarn install --frozen-lockfile"
    - scripts/static-analysis

downtime_check:
  extends:
    - .rails-job-base
    - .rails:rules:downtime_check
  needs: []
  stage: test
  variables:
    SETUP_DB: "false"
  script:
    - bundle exec rake downtime_check

rspec migration pg11:
  extends:
    - .rspec-base-pg11
    - .rspec-base-migration
    - .rspec-migration-parallel

rspec unit pg11:
  extends:
    - .rspec-base-pg11
    - .rails:rules:ee-and-foss-unit
    - .rspec-unit-parallel

rspec integration pg11:
  extends:
    - .rspec-base-pg11
    - .rails:rules:ee-and-foss-integration
    - .rspec-integration-parallel

rspec system pg11:
  extends:
    - .rspec-base-pg11
    - .rails:rules:ee-and-foss-system
    - .rspec-system-parallel

rspec fast_spec_helper:
  extends:
    - .rspec-base-pg11
    - .rails:rules:ee-and-foss-fast_spec_helper
  script:
    - bin/rspec spec/fast_spec_helper.rb

db:migrate:reset:
  extends: .db-job-base
  script:
    - bundle exec rake db:migrate:reset

db:check-schema:
  extends:
    - .db-job-base
    - .rails:rules:ee-mr-and-master-only
  script:
    - source scripts/schema_changed.sh

db:migrate-from-v12.10.0:
  extends: .db-job-base
  variables:
    SETUP_DB: "false"
  script:
    - export PROJECT_TO_CHECKOUT="gitlab"
    - export TAG_TO_CHECKOUT="v12.10.0-ee"
    - '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab-foss"'
    - '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="v12.10.0"'
    - git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT
    - git checkout -f FETCH_HEAD
    - sed -i -e "s/gem 'grpc', '~> 1.24.0'/gem 'grpc', '~> 1.30.2'/" Gemfile  # Update gRPC for Ruby 2.7
    - sed -i -e "s/gem 'google-protobuf', '~> 3.8.0'/gem 'google-protobuf', '~> 3.12.0'/" Gemfile
    - gem install bundler:1.17.3
    - bundle update google-protobuf grpc bootsnap
    - bundle install $BUNDLE_INSTALL_FLAGS
    - date
    - cp config/gitlab.yml.example config/gitlab.yml
    - bundle exec rake db:drop db:create db:structure:load db:seed_fu
    - date
    - git checkout -f $CI_COMMIT_SHA
    - bundle install $BUNDLE_INSTALL_FLAGS
    - date
    - . scripts/prepare_build.sh
    - date
    - bundle exec rake db:migrate

db:rollback:
  extends: .db-job-base
  script:
    - bundle exec rake db:migrate VERSION=20181228175414
    - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true

gitlab:setup:
  extends: .db-job-base
  variables:
    SETUP_DB: "false"
  script:
    # Manually clone gitlab-test and only seed this project in
    # db/fixtures/development/04_project.rb thanks to SIZE=1 below
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - *base-script
    - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/*.log

db:backup_and_restore:
  extends: .db-job-base
  variables:
    SETUP_DB: "false"
    GITLAB_ASSUME_YES: "1"
  script:
    - . scripts/prepare_build.sh
    - bundle exec rake db:drop db:create db:structure:load db:seed_fu
    - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry}
    - bundle exec rake gitlab:backup:create
    - date
    - bundle exec rake gitlab:backup:restore
  rules:
    - changes: ["lib/backup/**/*"]

rspec:coverage:
  extends:
    - .coverage-base
    - .rails:rules:rspec-coverage
  stage: post-test
  # We cannot use needs since it would mean needing 84 jobs (since most are parallelized)
  # so we use `dependencies` here.
  dependencies:
    - setup-test-env
    - rspec migration pg11
    - rspec unit pg11
    - rspec integration pg11
    - rspec system pg11
    - rspec-ee migration pg11
    - rspec-ee unit pg11
    - rspec-ee integration pg11
    - rspec-ee system pg11
    - rspec-ee unit pg11 geo
    - rspec-ee integration pg11 geo
    - rspec-ee system pg11 geo
    - memory-static
    - memory-on-boot
  script:
    - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --without default development test production puma unicorn kerberos metrics omnibus ed25519"
    - run_timed_command "bundle exec scripts/merge-simplecov"
    - run_timed_command "bundle exec scripts/gather-test-memory-data"
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
      - coverage/index.html
      - coverage/assets/
      - tmp/memory_test/
    reports:
      cobertura: coverage/coverage.xml

rspec:feature-flags:
  extends:
    - .coverage-base
    - .rails:rules:rspec-feature-flags
  stage: post-test
  # We cannot use needs since it would mean needing 84 jobs (since most are parallelized)
  # so we use `dependencies` here.
  dependencies:
    - setup-test-env
    - rspec migration pg11
    - rspec unit pg11
    - rspec integration pg11
    - rspec system pg11
    - rspec-ee migration pg11
    - rspec-ee unit pg11
    - rspec-ee integration pg11
    - rspec-ee system pg11
    - rspec-ee unit pg11 geo
    - rspec-ee integration pg11 geo
    - rspec-ee system pg11 geo
    - memory-static
    - memory-on-boot
  script:
    - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --without default development test production puma unicorn kerberos metrics omnibus ed25519"
    - run_timed_command "bundle exec scripts/used-feature-flags"
# EE/FOSS: default refs (MRs, master, schedules) jobs #
#######################################################

##################################################
# EE: default refs (MRs, master, schedules) jobs #
rspec migration pg11-as-if-foss:
  extends:
    - .rspec-base-pg11-as-if-foss
    - .rspec-base-migration
    - .rails:rules:as-if-foss-migration
    - .rspec-migration-parallel

rspec unit pg11-as-if-foss:
  extends:
    - .rspec-base-pg11-as-if-foss
    - .rails:rules:as-if-foss-unit
    - .rspec-unit-parallel

rspec integration pg11-as-if-foss:
  extends:
    - .rspec-base-pg11-as-if-foss
    - .rails:rules:as-if-foss-integration
    - .rspec-integration-parallel

rspec system pg11-as-if-foss:
  extends:
    - .rspec-base-pg11-as-if-foss
    - .rails:rules:as-if-foss-system
    - .rspec-system-parallel

rspec-ee migration pg11:
  extends:
    - .rspec-ee-base-pg11
    - .rspec-base-migration
    - .rails:rules:ee-only-migration
    - .rspec-ee-migration-parallel

rspec-ee unit pg11:
  extends:
    - .rspec-ee-base-pg11
    - .rails:rules:ee-only-unit
    - .rspec-ee-unit-parallel

rspec-ee integration pg11:
  extends:
    - .rspec-ee-base-pg11
    - .rails:rules:ee-only-integration
    - .rspec-ee-integration-parallel

rspec-ee system pg11:
  extends:
    - .rspec-ee-base-pg11
    - .rails:rules:ee-only-system
    - .rspec-ee-system-parallel

rspec-ee unit pg11 geo:
  extends:
    - .rspec-ee-base-geo-pg11
    - .rails:rules:ee-only-unit
    - .rspec-ee-unit-geo-parallel

rspec-ee integration pg11 geo:
  extends:
    - .rspec-ee-base-geo-pg11
    - .rails:rules:ee-only-integration

rspec-ee system pg11 geo:
  extends:
    - .rspec-ee-base-geo-pg11
    - .rails:rules:ee-only-system

db:rollback geo:
  extends:
    - db:rollback
    - .rails:rules:ee-only-migration
  script:
    - bundle exec rake geo:db:migrate VERSION=20170627195211
    - bundle exec rake geo:db:migrate
# EE: default refs (MRs, master, schedules) jobs #
##################################################

##########################################
# EE/FOSS: master nightly scheduled jobs #
rspec migration pg12:
  extends:
    - .rspec-base-pg12
    - .rspec-base-migration
    - .rails:rules:master-schedule-nightly--code-backstage
    - .rspec-migration-parallel

rspec unit pg12:
  extends:
    - .rspec-base-pg12
    - .rails:rules:master-schedule-nightly--code-backstage
    - .rspec-unit-parallel

rspec integration pg12:
  extends:
    - .rspec-base-pg12
    - .rails:rules:master-schedule-nightly--code-backstage
    - .rspec-integration-parallel

rspec system pg12:
  extends:
    - .rspec-base-pg12
    - .rails:rules:master-schedule-nightly--code-backstage
    - .rspec-system-parallel
# EE/FOSS: master nightly scheduled jobs #
##########################################

#####################################
# EE: master nightly scheduled jobs #
rspec-ee migration pg12:
  extends:
    - .rspec-ee-base-pg12
    - .rspec-base-migration
    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
    - .rspec-ee-migration-parallel

rspec-ee unit pg12:
  extends:
    - .rspec-ee-base-pg12
    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
    - .rspec-ee-unit-parallel

rspec-ee integration pg12:
  extends:
    - .rspec-ee-base-pg12
    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
    - .rspec-ee-integration-parallel

rspec-ee system pg12:
  extends:
    - .rspec-ee-base-pg12
    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
    - .rspec-ee-system-parallel

rspec-ee unit pg12 geo:
  extends:
    - .rspec-ee-base-geo-pg12
    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
    - .rspec-ee-unit-geo-parallel

rspec-ee integration pg12 geo:
  extends:
    - .rspec-ee-base-geo-pg12
    - .rails:rules:master-schedule-nightly--code-backstage-ee-only

rspec-ee system pg12 geo:
  extends:
    - .rspec-ee-base-geo-pg12
    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
# EE: master nightly scheduled jobs #
#####################################

##################################################
# EE: Canonical MR pipelines
rspec fail-fast:
  extends:
    - .rspec-ee-base-pg11  # This job also runs EE spec which needs elasticsearch
    - .rails:rules:rspec fail-fast
  stage: test
  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"]
  script:
    - *base-script
    - rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine"
  artifacts:
    expire_in: 7d
    paths:
      - tmp/capybara/

rspec foss-impact:
  extends:
    - .rspec-base-pg11-as-if-foss
    - .rails:rules:rspec-foss-impact
  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"]
  script:
    - *base-script
    - rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine"
  artifacts:
    expire_in: 7d
    paths:
      - tmp/capybara/

fail-pipeline-early:
  extends:
    - .rails:rules:fail-pipeline-early
  stage: post-test
  needs:
    - job: rspec fail-fast
      artifacts: false
  variables:
    GIT_DEPTH: 1
  before_script:
    - source scripts/utils.sh
    - install_api_client_dependencies_with_apt
  script:
    - fail_pipeline_early
# EE: Canonical MR pipelines
##################################################