image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.6-golang-1.9-git-2.16-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6" .dedicated-runner: &dedicated-runner retry: 1 tags: - gitlab-org .default-cache: &default-cache key: "ruby-2.3.6-with-yarn" paths: - vendor/ruby - .yarn-cache/ .push-cache: &push-cache cache: <<: *default-cache policy: push .pull-cache: &pull-cache cache: <<: *default-cache policy: pull variables: MYSQL_ALLOW_EMPTY_PASSWORD: "1" RAILS_ENV: "test" NODE_ENV: "test" SIMPLECOV: "true" GIT_DEPTH: "20" GIT_SUBMODULE_STRATEGY: "none" GET_SOURCES_ATTEMPTS: "3" KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json before_script: - bundle --version - date - source scripts/utils.sh - date - source scripts/prepare_build.sh - date after_script: - date stages: - build - prepare - test - post-test - pages - post-cleanup # Predefined scopes .tests-metadata-state: &tests-metadata-state <<: *dedicated-runner variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: - source scripts/utils.sh artifacts: expire_in: 31d paths: - knapsack/ - rspec_flaky/ .use-pg: &use-pg services: # As of Jan 2018, we don't have a strong reason to upgrade to 9.6 for CI yet, # so using the least common denominator ensures backwards compatibility # (as many users are still using 9.2). - postgres:9.2 - redis:alpine .use-mysql: &use-mysql services: - mysql:5.7 - redis:alpine # Skip all jobs except the ones that begin with 'docs/'. # Used for commits including ONLY documentation changes. # https://docs.gitlab.com/ce/development/writing_documentation.html#testing .except-docs: &except-docs except: - /(^docs[\/-].*|.*-docs$)/ .except-qa: &except-qa except: - /(^qa[\/-].*|.*-qa$)/ .except-docs-and-qa: &except-docs-and-qa except: - /(^docs[\/-].*|.*-docs$)/ - /(^qa[\/-].*|.*-qa$)/ # Jobs that only need to pull cache .dedicated-no-docs-pull-cache-job: &dedicated-no-docs-pull-cache-job <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache dependencies: - setup-test-env stage: test # Jobs that do not need a DB .dedicated-no-docs-no-db-pull-cache-job: &dedicated-no-docs-no-db-pull-cache-job <<: *dedicated-no-docs-pull-cache-job variables: SETUP_DB: "false" .rake-exec: &rake-exec <<: *dedicated-no-docs-no-db-pull-cache-job script: - bundle exec rake $CI_JOB_NAME .rspec-metadata: &rspec-metadata <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test script: - JOB_NAME=( $CI_JOB_NAME ) - export CI_NODE_INDEX=${JOB_NAME[-2]} - export CI_NODE_TOTAL=${JOB_NAME[-1]} - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH} - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${JOB_NAME[0]}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${JOB_NAME[0]}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export FLAKY_RSPEC_GENERATE_REPORT=true - export CACHE_CLASSES=true - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}' - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}' - scripts/gitaly-test-spawn - knapsack rspec "--color --format documentation" artifacts: expire_in: 31d when: always paths: - coverage/ - knapsack/ - rspec_flaky/ - tmp/capybara/ .rspec-metadata-pg: &rspec-metadata-pg <<: *rspec-metadata <<: *use-pg .rspec-metadata-mysql: &rspec-metadata-mysql <<: *rspec-metadata <<: *use-mysql .spinach-metadata: &spinach-metadata <<: *dedicated-runner <<: *except-docs-and-qa <<: *pull-cache stage: test script: - JOB_NAME=( $CI_JOB_NAME ) - export CI_NODE_INDEX=${JOB_NAME[-2]} - export CI_NODE_TOTAL=${JOB_NAME[-1]} - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - export CACHE_CLASSES=true - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - scripts/gitaly-test-spawn - knapsack spinach "-r rerun" -b || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -b -r rerun $(cat tmp/spinach-rerun.txt)' artifacts: expire_in: 31d when: always paths: - coverage/ - knapsack/ - tmp/capybara/ .spinach-metadata-pg: &spinach-metadata-pg <<: *spinach-metadata <<: *use-pg .spinach-metadata-mysql: &spinach-metadata-mysql <<: *spinach-metadata <<: *use-mysql .only-canonical-masters: &only-canonical-masters only: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - master@gitlab/gitlabhq - master@gitlab/gitlab-ee .gitlab-setup: &gitlab-setup <<: *dedicated-no-docs-pull-cache-job <<: *use-pg variables: CREATE_DB_USER: "true" script: # Manually clone gitlab-test and only seed this project in # db/fixtures/development/04_project.rb thanks to SIZE=1 below - git clone https://gitlab.com/gitlab-org/gitlab-test.git /home/git/repositories/gitlab-org/gitlab-test.git - scripts/gitaly-test-spawn - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup artifacts: when: on_failure expire_in: 1d paths: - log/development.log # Review docs base .review-docs: &review-docs <<: *dedicated-runner <<: *except-qa image: ruby:2.4-alpine before_script: - gem install gitlab --no-doc # We need to download the script rather than clone the repo since the # review-docs-cleanup job will not be able to run when the branch gets # deleted (when merging the MR). - apk add --update openssl - wget https://gitlab.com/gitlab-org/gitlab-ce/raw/master/scripts/trigger-build-docs - chmod 755 trigger-build-docs cache: {} dependencies: [] variables: GIT_STRATEGY: none when: manual only: - branches # DB migration, rollback, and seed jobs .db-migrate-reset: &db-migrate-reset <<: *dedicated-no-docs-pull-cache-job script: - bundle exec rake db:migrate:reset .migration-paths: &migration-paths <<: *dedicated-no-docs-pull-cache-job variables: CREATE_DB_USER: "true" script: - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0 - git checkout -f FETCH_HEAD - bundle install $BUNDLE_INSTALL_FLAGS - date - cp config/gitlab.yml.example config/gitlab.yml - bundle exec rake db:drop db:create db:schema:load db:seed_fu - date - git checkout $CI_COMMIT_SHA - bundle install $BUNDLE_INSTALL_FLAGS - date - . scripts/prepare_build.sh - date - bundle exec rake db:migrate ## # Trigger a package build in omnibus-gitlab repository # package-qa: <<: *dedicated-runner image: ruby:2.4-alpine before_script: [] stage: build cache: {} when: manual script: - scripts/trigger-build-omnibus only: - //@gitlab-org/gitlab-ce - //@gitlab-org/gitlab-ee # Trigger a docs build in gitlab-docs # Useful to preview the docs changes live review-docs-deploy: <<: *review-docs stage: build environment: name: review-docs/$CI_COMMIT_REF_NAME # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are secret variables # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693 url: http://$DOCS_GITLAB_REPO_SUFFIX-$CI_COMMIT_REF_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX on_stop: review-docs-cleanup script: - ./trigger-build-docs deploy # Cleanup remote environment of gitlab-docs review-docs-cleanup: <<: *review-docs stage: post-cleanup environment: name: review-docs/$CI_COMMIT_REF_NAME action: stop script: - ./trigger-build-docs cleanup # Retrieve knapsack and rspec_flaky reports retrieve-tests-metadata: <<: *tests-metadata-state <<: *except-docs-and-qa stage: prepare cache: key: tests_metadata policy: pull script: - mkdir -p knapsack/${CI_PROJECT_NAME}/ - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}' - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}' - mkdir -p rspec_flaky/ - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' update-tests-metadata: <<: *tests-metadata-state <<: *only-canonical-masters stage: post-test cache: key: tests_metadata paths: - knapsack/ - rspec_flaky/ policy: push script: - retry gem install fog-aws mime-types - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach-pg_node_*.json - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH' - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH' - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json flaky-examples-check: <<: *dedicated-runner image: ruby:2.4-alpine services: [] before_script: [] variables: SETUP_DB: "false" USE_BUNDLE_INSTALL: "false" NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json stage: post-test allow_failure: true retry: 0 only: - branches except: - master - /(^docs[\/-].*|.*-docs$)/ - /(^qa[\/-].*|.*-qa$)/ artifacts: expire_in: 30d paths: - rspec_flaky/ script: - '[[ -f $NEW_FLAKY_SPECS_REPORT ]] || echo "{}" > ${NEW_FLAKY_SPECS_REPORT}' - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT compile-assets: <<: *dedicated-runner <<: *except-docs <<: *use-pg stage: prepare cache: <<: *default-cache script: - node --version - date - yarn install --frozen-lockfile --cache-folder .yarn-cache - date - bundle exec rake gitlab:assets:compile artifacts: expire_in: 7d paths: - node_modules - public/assets setup-test-env: <<: *dedicated-runner <<: *except-docs <<: *use-pg stage: prepare cache: <<: *default-cache script: - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init' - scripts/gitaly-test-build # Do not use 'bundle exec' here artifacts: expire_in: 7d paths: - tmp/tests - config/secrets.yml rspec-pg 0 28: *rspec-metadata-pg rspec-pg 1 28: *rspec-metadata-pg rspec-pg 2 28: *rspec-metadata-pg rspec-pg 3 28: *rspec-metadata-pg rspec-pg 4 28: *rspec-metadata-pg rspec-pg 5 28: *rspec-metadata-pg rspec-pg 6 28: *rspec-metadata-pg rspec-pg 7 28: *rspec-metadata-pg rspec-pg 8 28: *rspec-metadata-pg rspec-pg 9 28: *rspec-metadata-pg rspec-pg 10 28: *rspec-metadata-pg rspec-pg 11 28: *rspec-metadata-pg rspec-pg 12 28: *rspec-metadata-pg rspec-pg 13 28: *rspec-metadata-pg rspec-pg 14 28: *rspec-metadata-pg rspec-pg 15 28: *rspec-metadata-pg rspec-pg 16 28: *rspec-metadata-pg rspec-pg 17 28: *rspec-metadata-pg rspec-pg 18 28: *rspec-metadata-pg rspec-pg 19 28: *rspec-metadata-pg rspec-pg 20 28: *rspec-metadata-pg rspec-pg 21 28: *rspec-metadata-pg rspec-pg 22 28: *rspec-metadata-pg rspec-pg 23 28: *rspec-metadata-pg rspec-pg 24 28: *rspec-metadata-pg rspec-pg 25 28: *rspec-metadata-pg rspec-pg 26 28: *rspec-metadata-pg rspec-pg 27 28: *rspec-metadata-pg rspec-mysql 0 28: *rspec-metadata-mysql rspec-mysql 1 28: *rspec-metadata-mysql rspec-mysql 2 28: *rspec-metadata-mysql rspec-mysql 3 28: *rspec-metadata-mysql rspec-mysql 4 28: *rspec-metadata-mysql rspec-mysql 5 28: *rspec-metadata-mysql rspec-mysql 6 28: *rspec-metadata-mysql rspec-mysql 7 28: *rspec-metadata-mysql rspec-mysql 8 28: *rspec-metadata-mysql rspec-mysql 9 28: *rspec-metadata-mysql rspec-mysql 10 28: *rspec-metadata-mysql rspec-mysql 11 28: *rspec-metadata-mysql rspec-mysql 12 28: *rspec-metadata-mysql rspec-mysql 13 28: *rspec-metadata-mysql rspec-mysql 14 28: *rspec-metadata-mysql rspec-mysql 15 28: *rspec-metadata-mysql rspec-mysql 16 28: *rspec-metadata-mysql rspec-mysql 17 28: *rspec-metadata-mysql rspec-mysql 18 28: *rspec-metadata-mysql rspec-mysql 19 28: *rspec-metadata-mysql rspec-mysql 20 28: *rspec-metadata-mysql rspec-mysql 21 28: *rspec-metadata-mysql rspec-mysql 22 28: *rspec-metadata-mysql rspec-mysql 23 28: *rspec-metadata-mysql rspec-mysql 24 28: *rspec-metadata-mysql rspec-mysql 25 28: *rspec-metadata-mysql rspec-mysql 26 28: *rspec-metadata-mysql rspec-mysql 27 28: *rspec-metadata-mysql spinach-pg 0 2: *spinach-metadata-pg spinach-pg 1 2: *spinach-metadata-pg spinach-mysql 0 2: *spinach-metadata-mysql spinach-mysql 1 2: *spinach-metadata-mysql static-analysis: <<: *dedicated-no-docs-no-db-pull-cache-job dependencies: - compile-assets - setup-test-env script: - scripts/static-analysis cache: key: "ruby-2.3.6-with-yarn-and-rubocop" paths: - vendor/ruby - .yarn-cache/ - tmp/rubocop_cache # Documentation checks: # - Check validity of relative links # - Make sure cURL examples in API docs use the full switches docs lint: <<: *dedicated-runner <<: *except-qa image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine" stage: test cache: {} dependencies: [] before_script: [] script: - scripts/lint-doc.sh - scripts/lint-changelog-yaml - mv doc/ /nanoc/content/ - cd /nanoc # Build HTML from Markdown - bundle exec nanoc # Check the internal links - bundle exec nanoc check internal_links downtime_check: <<: *rake-exec except: - master - tags - /^[\d-]+-stable(-ee)?$/ - /(^docs[\/-].*|.*-docs$)/ - /(^qa[\/-].*|.*-qa$)/ ee_compat_check: <<: *rake-exec except: - master - tags - /^[\d-]+-stable(-ee)?/ - /^security-/ - branches@gitlab-org/gitlab-ee - branches@gitlab/gitlab-ee retry: 0 artifacts: name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}" when: always expire_in: 10d paths: - ee_compat_check/patches/*.patch db:migrate:reset-pg: <<: *db-migrate-reset <<: *use-pg db:migrate:reset-mysql: <<: *db-migrate-reset <<: *use-mysql db:check-schema-pg: <<: *db-migrate-reset <<: *use-pg script: - source scripts/schema_changed.sh migration:path-pg: <<: *migration-paths <<: *use-pg migration:path-mysql: <<: *migration-paths <<: *use-mysql .db-rollback: &db-rollback <<: *dedicated-no-docs-pull-cache-job script: - bundle exec rake db:rollback STEP=119 - bundle exec rake db:migrate db:rollback-pg: <<: *db-rollback <<: *use-pg db:rollback-mysql: <<: *db-rollback <<: *use-mysql gitlab:setup-pg: <<: *gitlab-setup <<: *use-pg gitlab:setup-mysql: <<: *gitlab-setup <<: *use-mysql # Frontend-related jobs gitlab:assets:compile: <<: *dedicated-no-docs-no-db-pull-cache-job dependencies: [] variables: NODE_ENV: "production" RAILS_ENV: "production" SETUP_DB: "false" SKIP_STORAGE_VALIDATION: "true" WEBPACK_REPORT: "true" NO_COMPRESSION: "true" script: - date - yarn install --frozen-lockfile --production --cache-folder .yarn-cache - date - bundle exec rake gitlab:assets:compile artifacts: name: webpack-report expire_in: 31d paths: - webpack-report/ karma: <<: *dedicated-no-docs-pull-cache-job <<: *use-pg dependencies: - compile-assets - setup-test-env script: - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log - date - scripts/gitaly-test-spawn - date - bundle exec rake karma coverage: '/^Statements *: (\d+\.\d+%)/' artifacts: name: coverage-javascript expire_in: 31d when: always paths: - chrome_debug.log - coverage-javascript/ codequality: <<: *dedicated-no-docs-no-db-pull-cache-job image: docker:latest before_script: [] services: - docker:dind variables: SETUP_DB: "false" DOCKER_DRIVER: overlay2 CODECLIMATE_FORMAT: json cache: {} dependencies: [] script: - apk update && apk add jq - ./scripts/codequality analyze -f json > raw_codeclimate.json || true # The following line keeps only the fields used in the MR widget, reducing the JSON artifact size - jq -c 'map({check_name,description,fingerprint,location})' raw_codeclimate.json > codeclimate.json artifacts: paths: [codeclimate.json] expire_in: 1 week sast: <<: *except-docs image: registry.gitlab.com/gitlab-org/gl-sast:latest variables: CONFIDENCE_LEVEL: 2 before_script: [] script: - /app/bin/run . artifacts: paths: [gl-sast-report.json] qa:internal: <<: *dedicated-no-docs-no-db-pull-cache-job services: [] script: - cd qa/ - bundle install - bundle exec rspec qa:selectors: <<: *dedicated-no-docs-no-db-pull-cache-job services: [] script: - cd qa/ - bundle install - bundle exec bin/qa Test::Sanity::Selectors coverage: <<: *dedicated-no-docs-no-db-pull-cache-job stage: post-test script: - bundle exec scripts/merge-simplecov coverage: '/LOC \((\d+\.\d+%)\) covered.$/' artifacts: name: coverage expire_in: 31d paths: - coverage/index.html - coverage/assets/ lint:javascript:report: <<: *dedicated-no-docs-no-db-pull-cache-job stage: post-test dependencies: - compile-assets - setup-test-env before_script: [] script: - date - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files - date - yarn run eslint-report || true # ignore exit code artifacts: name: eslint-report expire_in: 31d paths: - eslint-report.html pages: <<: *dedicated-no-docs-no-db-pull-cache-job before_script: [] stage: pages dependencies: - coverage - karma - gitlab:assets:compile - lint:javascript:report script: - mv public/ .public/ - mkdir public/ - mv coverage/ public/coverage-ruby/ || true - mv coverage-javascript/ public/coverage-javascript/ || true - mv eslint-report.html public/ || true - mv webpack-report/ public/webpack-report/ || true artifacts: paths: - public only: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee # Insurance in case a gem needed by one of our releases gets yanked from # rubygems.org in the future. cache gems: <<: *dedicated-no-docs-no-db-pull-cache-job script: - bundle package --all --all-platforms artifacts: paths: - vendor/cache only: - master@gitlab-org/gitlab-ce - master@gitlab-org/gitlab-ee - tags gitlab_git_test: <<: *dedicated-runner <<: *except-docs-and-qa variables: SETUP_DB: "false" before_script: [] cache: {} script: - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes