diff options
Diffstat (limited to '.gitlab/ci')
-rw-r--r-- | .gitlab/ci/dast.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/dev-fixtures.gitlab-ci.yml | 13 | ||||
-rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 4 | ||||
-rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 38 | ||||
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 21 | ||||
-rw-r--r-- | .gitlab/ci/notify.gitlab-ci.yml | 17 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 91 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 17 | ||||
-rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 29 | ||||
-rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 35 | ||||
-rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 4 |
11 files changed, 194 insertions, 77 deletions
diff --git a/.gitlab/ci/dast.gitlab-ci.yml b/.gitlab/ci/dast.gitlab-ci.yml index 93f64930822..33778b9cbd0 100644 --- a/.gitlab/ci/dast.gitlab-ci.yml +++ b/.gitlab/ci/dast.gitlab-ci.yml @@ -28,6 +28,8 @@ # Help pages are excluded from scan as they are static pages. # profile/two_factor_auth is excluded from scan to prevent 2FA from being turned on from user profile, which will reduce coverage. - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/help/.*,${DAST_WEBSITE}/profile/two_factor_auth,${DAST_WEBSITE}/users/sign_out"' + # Exclude the automatically generated monitoring project from being tested due to https://gitlab.com/gitlab-org/gitlab/-/issues/260362 + - 'DAST_AUTH_EXCLUDE_URLS="${DAST_AUTH_EXCLUDE_URLS},https://.*\.gitlab-review\.app/gitlab-instance-(administrators-)?[a-zA-Z0-9]{8}/.*"' - enable_rule () { read all_rules; rule=$1; echo $all_rules | sed -r "s/(,)?$rule(,)?/\1-1\2/" ; } # Sort ids in DAST_RULES ascendingly, which is required when using DAST_RULES as argument to enable_rule - 'DAST_RULES=$(echo $DAST_RULES | tr "," "\n" | sort -n | paste -sd ",")' diff --git a/.gitlab/ci/dev-fixtures.gitlab-ci.yml b/.gitlab/ci/dev-fixtures.gitlab-ci.yml index fc3678a7d17..4141cc7f071 100644 --- a/.gitlab/ci/dev-fixtures.gitlab-ci.yml +++ b/.gitlab/ci/dev-fixtures.gitlab-ci.yml @@ -14,14 +14,17 @@ SIZE: 0 # number of external projects to fork, requires network connection # SEED_NESTED_GROUPS: "false" # requires network connection +.run-dev-fixtures-script: &run-dev-fixtures-script + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" + run-dev-fixtures: extends: - .run-dev-fixtures - .dev-fixtures:rules:ee-and-foss script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" + - *run-dev-fixtures-script run-dev-fixtures-ee: extends: @@ -29,7 +32,5 @@ run-dev-fixtures-ee: - .dev-fixtures:rules:ee-only - .use-pg11-ee script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - cp ee/db/fixtures/development/* $FIXTURE_PATH - - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" + - *run-dev-fixtures-script diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 0e0e156a64f..4b25908aa6a 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -4,7 +4,7 @@ - .docs:rules:review-docs image: ruby:2.6-alpine stage: review - dependencies: [] + needs: [] variables: # We're cloning the repo instead of downloading the script for now # because some repos are private and CI_JOB_TOKEN cannot access files. @@ -42,7 +42,7 @@ docs lint: extends: - .default-retry - .docs:rules:docs-lint - image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint:vale-2.3.4-markdownlint-0.23.2" + image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint:ruby-2.7.2-alpine-3.12-vale-2.4.3-markdownlint-0.24.0" stage: test needs: [] script: diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 27f56cd8667..e4c9f85cf62 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -7,19 +7,21 @@ # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 +.yarn-install: &yarn-install + - source scripts/utils.sh + - run_timed_command "retry yarn install --frozen-lockfile" + .compile-assets-base: extends: - .frontend-base - .assets-compile-cache - image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.28-lfs-2.9-node-12.x-yarn-1.21-graphicsmagick-1.3.34 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.28-lfs-2.9-node-12.18-yarn-1.22-graphicsmagick-1.3.34 variables: WEBPACK_VENDOR_DLL: "true" stage: prepare script: - - node --version - - run_timed_command "retry yarn install --frozen-lockfile" - - free -m - - run_timed_command "bin/rake gitlab:assets:compile > assets-compile.log 2>&1" + - *yarn-install + - run_timed_command "bin/rake gitlab:assets:compile" - run_timed_command "scripts/clean-old-cached-assets" compile-production-assets: @@ -34,7 +36,6 @@ compile-production-assets: name: webpack-report expire_in: 31d paths: - - assets-compile.log # These assets are used in multiple locations: # - in `build-assets-image` job to create assets image for packaging systems # - GitLab UI for integration tests: https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1 @@ -51,7 +52,6 @@ compile-test-assets: artifacts: expire_in: 7d paths: - - assets-compile.log - public/assets/ - node_modules/@gitlab/svgs/dist/icons.json # app/helpers/icons_helper.rb uses this file when: always @@ -87,8 +87,7 @@ update-yarn-cache: - .shared:rules:update-cache stage: prepare script: - - source scripts/utils.sh - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install cache: policy: push @@ -139,14 +138,14 @@ eslint-as-if-foss: - .as-if-foss needs: [] script: - - run_timed_command "retry yarn install --frozen-lockfile" - - yarn run eslint + - *yarn-install + - run_timed_command "yarn run eslint" .karma-base: extends: .frontend-test-base script: - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install - run_timed_command "yarn karma" karma: @@ -177,7 +176,7 @@ karma-as-if-foss: .jest-base: extends: .frontend-test-base script: - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install - run_timed_command "yarn jest --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js" jest: @@ -202,7 +201,7 @@ jest-integration: - .frontend-test-base - .frontend:rules:default-frontend-jobs script: - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install - run_timed_command "yarn jest:integration --ci" needs: ["frontend-fixtures"] @@ -222,8 +221,7 @@ coverage-frontend: needs: ["jest"] stage: post-test before_script: - - source scripts/utils.sh - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install script: - run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js" coverage: '/^Statements\s*:\s*?(\d+(?:\.\d+)?)%/' @@ -243,9 +241,8 @@ coverage-frontend: stage: test dependencies: [] script: - - source scripts/utils.sh - - run_timed_command "yarn install --frozen-lockfile" - - run_timed_command "yarn run webpack-prod" + - *yarn-install + - run_timed_command "retry yarn run webpack-prod" qa-frontend-node:10: extends: .qa-frontend-node @@ -268,8 +265,7 @@ webpack-dev-server: WEBPACK_MEMORY_TEST: "true" WEBPACK_VENDOR_DLL: "true" script: - - source scripts/utils.sh - - run_timed_command "retry yarn install --frozen-lockfile" + - *yarn-install - run_timed_command "retry yarn webpack-vendor" - run_timed_command "node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js" artifacts: diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 238059bf972..fea3956bfe8 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -34,6 +34,13 @@ - tmp/rubocop_cache/ policy: pull +.coverage-cache: + cache: + key: "coverage-cache-v1" + paths: + - vendor/ruby/ + policy: pull + .qa-cache: cache: key: "qa-v1" @@ -64,7 +71,7 @@ policy: pull .use-pg11: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-84-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-11-graphicsmagick-1.3.34" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -73,7 +80,7 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg12: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-84-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-12-graphicsmagick-1.3.34" services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] @@ -82,22 +89,24 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg11-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-84-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-11-graphicsmagick-1.3.34" services: - name: postgres:11.6 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:4.0-alpine - - name: elasticsearch:6.4.2 + - name: elasticsearch:7.9.2 + command: ["elasticsearch", "-E", "discovery.type=single-node"] variables: POSTGRES_HOST_AUTH_METHOD: trust .use-pg12-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-84-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.28-lfs-2.9-chrome-85-node-12.18-yarn-1.22-postgresql-12-graphicsmagick-1.3.34" services: - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:4.0-alpine - - name: elasticsearch:6.4.2 + - name: elasticsearch:7.9.2 + command: ["elasticsearch", "-E", "discovery.type=single-node"] variables: POSTGRES_HOST_AUTH_METHOD: trust diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml index 6dcf19da942..e18a092bb8f 100644 --- a/.gitlab/ci/notify.gitlab-ci.yml +++ b/.gitlab/ci/notify.gitlab-ci.yml @@ -3,6 +3,8 @@ stage: notify dependencies: [] cache: {} + variables: + MERGE_REQUEST_URL: ${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID} before_script: - apk update && apk add git curl bash @@ -16,8 +18,19 @@ notify-update-gitaly: variables: NOTIFY_CHANNEL: g_create_gitaly GITALY_UPDATE_BRANCH: release-tools/update-gitaly - MERGE_REQUEST_URL: ${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID} script: - echo "NOTIFY_CHANNEL is ${NOTIFY_CHANNEL}" - echo "CI_PIPELINE_URL is ${CI_PIPELINE_URL}" - - scripts/slack ${NOTIFY_CHANNEL} "☠️ \`${GITALY_UPDATE_BRANCH}\` failed! ☠️ See ${CI_PIPELINE_URL} (triggered from ${MERGE_REQUEST_URL})" ci_failing + - scripts/slack ${NOTIFY_CHANNEL} "☠️ \`${GITALY_UPDATE_BRANCH}\` failed! ☠️ See ${CI_PIPELINE_URL} (triggered from ${MERGE_REQUEST_URL})" ci_failing "GitLab QA Bot" + +notify-security-pipeline: + extends: + - .notify-slack + - .delivery:rules:security-pipeline-merge-result-failure + variables: + NOTIFY_CHANNEL: f_upcoming_release + script: + - echo "NOTIFY_CHANNEL is ${NOTIFY_CHANNEL}" + - echo "CI_PIPELINE_URL is ${CI_PIPELINE_URL}" + # <!subteam^S0127FU8PDE> mentions the `@release-managers` group + - scripts/slack ${NOTIFY_CHANNEL} "<!subteam^S0127FU8PDE> ☠️ Pipeline for merged result failed! ☠️ See ${CI_PIPELINE_URL} (triggered from ${MERGE_REQUEST_URL})" ci_failing "GitLab Release Tools Bot" diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 165476678bb..c4167ce7bcb 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -6,14 +6,23 @@ - .default-before_script - .rails-cache +.base-script: &base-script + # Only install knapsack after bundle install! Otherwise oddly some native + # gems could not be found under some circumstance. No idea why, hours wasted. + - run_timed_command "gem install knapsack --no-document" + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - source ./scripts/rspec_helpers.sh + .rspec-base: extends: .rails-job-base stage: test + variables: + RUBY_GC_MALLOC_LIMIT: 67108864 + RUBY_GC_MALLOC_LIMIT_MAX: 134217728 needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"] script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration" artifacts: expire_in: 31d @@ -25,6 +34,7 @@ - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ + - tmp/feature_flags/ - log/*.log reports: junit: junit_rspec.xml @@ -32,9 +42,7 @@ .rspec-base-migration: extends: .rails:rules:ee-and-foss-migration script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration" .rspec-base-pg11: @@ -67,9 +75,7 @@ .rspec-ee-base-geo: extends: .rspec-base script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_paralellized_job "--tag ~quarantine --tag geo" .rspec-ee-base-geo-pg11: @@ -160,6 +166,25 @@ update-rails-cache: cache: policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. +.coverage-base: + extends: + - .default-retry + - .default-before_script + - .coverage-cache + variables: + SETUP_DB: "false" + USE_BUNDLE_INSTALL: "false" + +update-coverage-cache: + extends: + - .coverage-base + - .shared:rules:update-cache + stage: prepare + script: + - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --without default development test production puma unicorn kerberos metrics omnibus ed25519" + cache: + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + .static-analysis-base: extends: - .default-retry @@ -178,7 +203,7 @@ update-static-analysis-cache: script: - rm -rf ./node_modules # We remove node_modules because there's no mechanism to remove stall entries. - run_timed_command "retry yarn install --frozen-lockfile" - - bundle exec rubocop --parallel # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks, + - run_timed_command "bundle exec rubocop --parallel" # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks, cache: # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism # for keeping only the N latest cache files, so we take advantage of it with `pull-push` and removing `node_modules` at the start of the job. @@ -287,8 +312,7 @@ gitlab:setup: # db/fixtures/development/04_project.rb thanks to SIZE=1 below - git clone https://gitlab.com/gitlab-org/gitlab-test.git /home/git/repositories/gitlab-org/gitlab-test.git - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" + - *base-script - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup artifacts: when: on_failure @@ -313,7 +337,7 @@ db:backup_and_restore: rspec:coverage: extends: - - .rails-job-base + - .coverage-base - .rails:rules:rspec-coverage stage: post-test # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) @@ -333,11 +357,10 @@ rspec:coverage: - rspec-ee system pg11 geo - memory-static - memory-on-boot - variables: - SETUP_DB: "false" script: - - bundle exec scripts/merge-simplecov - - bundle exec scripts/gather-test-memory-data + - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --without default development test production puma unicorn kerberos metrics omnibus ed25519" + - run_timed_command "bundle exec scripts/merge-simplecov" + - run_timed_command "bundle exec scripts/gather-test-memory-data" coverage: '/LOC \((\d+\.\d+%)\) covered.$/' artifacts: name: coverage @@ -348,6 +371,32 @@ rspec:coverage: - tmp/memory_test/ reports: cobertura: coverage/coverage.xml + +rspec:feature-flags: + extends: + - .coverage-base + - .rails:rules:rspec-feature-flags + stage: post-test + # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) + # so we use `dependencies` here. + dependencies: + - setup-test-env + - rspec migration pg11 + - rspec unit pg11 + - rspec integration pg11 + - rspec system pg11 + - rspec-ee migration pg11 + - rspec-ee unit pg11 + - rspec-ee integration pg11 + - rspec-ee system pg11 + - rspec-ee unit pg11 geo + - rspec-ee integration pg11 geo + - rspec-ee system pg11 geo + - memory-static + - memory-on-boot + script: + - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --without default development test production puma unicorn kerberos metrics omnibus ed25519" + - run_timed_command "bundle exec scripts/used-feature-flags" # EE/FOSS: default refs (MRs, master, schedules) jobs # ####################################################### @@ -512,9 +561,7 @@ rspec fail-fast: stage: test needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine" artifacts: expire_in: 7d @@ -527,9 +574,7 @@ rspec foss-impact: - .rails:rules:rspec-foss-impact needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"] script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh + - *base-script - rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine" artifacts: expire_in: 7d diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 0e2f12789db..168f60f0f65 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -151,3 +151,20 @@ dependency_scanning: reports: dependency_scanning: gl-dependency-scanning-report.json expire_in: 1 week # GitLab-specific + +license_scanning: + extends: + - .default-retry + - .reports:rules:license_scanning + stage: test + image: + name: "registry.gitlab.com/gitlab-org/security-products/analyzers/license-finder:3" + entrypoint: [""] + needs: [] + script: + - /run.sh analyze . + artifacts: + reports: + license_scanning: gl-license-scanning-report.json + expire_in: 1 week # GitLab-specific + dependencies: [] diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index d34687cfdad..46a1a957692 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -16,6 +16,11 @@ review-cleanup: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb - gcp_cleanup +.base-before_script: &base-before_script + - source ./scripts/utils.sh + - source ./scripts/review_apps/review-apps.sh + - install_api_client_dependencies_with_apk + review-build-cng: extends: - .default-retry @@ -23,7 +28,7 @@ review-build-cng: image: ruby:2.6-alpine stage: review-prepare before_script: - - source scripts/utils.sh + - source ./scripts/utils.sh - install_api_client_dependencies_with_apk - install_gitlab_gem needs: @@ -62,9 +67,7 @@ review-deploy: - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt - - source ./scripts/utils.sh - - install_api_client_dependencies_with_apk - - source scripts/review_apps/review-apps.sh + - *base-before_script script: - check_kube_domain - ensure_namespace @@ -72,7 +75,7 @@ review-deploy: - download_chart - date - deploy || (display_deployment_debug && exit 1) - - disable_sign_ups + - disable_sign_ups || (delete_release && exit 1) # When the job is manual, review-qa-smoke is also manual and we don't want people # to have to manually start the jobs in sequence, so we do it for them. - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"' @@ -81,10 +84,9 @@ review-deploy: # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. # Set DAST_RUN to true when jobs are manually scheduled. - if [ "$DAST_RUN" == "true" ]; then source scripts/review_apps/seed-dast-test-data.sh; TRACE=1 trigger_proj_user_creation; fi - artifacts: paths: [environment_url.txt] - expire_in: 2 days + expire_in: 7 days when: always .review-stop-base: @@ -98,9 +100,7 @@ review-deploy: # See https://gitlab.com/gitlab-org/gitlab/issues/191273 GIT_DEPTH: 1 before_script: - - apk add --update openssl - - source ./scripts/utils.sh - - source ./scripts/review_apps/review-apps.sh + - *base-before_script review-stop-failed-deployment: extends: @@ -143,8 +143,7 @@ review-stop: - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - echo "${QA_IMAGE}" - - source scripts/utils.sh - - install_api_client_dependencies_with_apk + - *base-before_script - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} artifacts: paths: @@ -174,7 +173,7 @@ review-performance: - .default-retry - .review:rules:review-performance image: - name: sitespeedio/sitespeed.io:6.3.1 + name: sitespeedio/sitespeed.io entrypoint: [""] stage: qa # This is needed so that manual jobs with needs don't block the pipeline. @@ -232,6 +231,6 @@ danger-review: stage: test needs: [] script: - - source scripts/utils.sh - - retry yarn install --frozen-lockfile + - source ./scripts/utils.sh + - run_timed_command "retry yarn install --frozen-lockfile" - danger --fail-on-errors=true --verbose diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index a8e0e1ccaaa..57d4a2a4cb7 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -73,6 +73,12 @@ .if-rspec-fail-fast-skipped: &if-rspec-fail-fast-skipped if: '$CI_MERGE_REQUEST_TITLE =~ /SKIP RSPEC FAIL-FAST/' +# For Security merge requests, the gitlab-release-tools-bot triggers a new +# pipeline for the "Pipelines for merged results" feature. If the pipeline +# fails, we notify release managers. +.if-security-pipeline-merge-result: &if-security-pipeline-merge-result + if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == $CI_DEFAULT_BRANCH && $CI_PROJECT_NAMESPACE == "gitlab-org/security" && $GITLAB_USER_LOGIN == "gitlab-release-tools-bot"' + #################### # Changes patterns # #################### @@ -132,7 +138,10 @@ .db-patterns: &db-patterns - "{,ee/}{,spec/}{db,migrations}/**/*" + - "{,ee/}{,spec/}lib/{,ee/}gitlab/database/**/*" + - "{,ee/}{,spec/}lib/{,ee/}gitlab/database{,_spec}.rb" - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration/**/*" + - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration{,_spec}.rb" - "config/prometheus/common_metrics.yml" # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer - "{,ee/}app/models/project_statistics.rb" # Used to calculate sizes in migration specs @@ -282,6 +291,14 @@ when: manual allow_failure: true +################## +# Delivery rules # +################## +.delivery:rules:security-pipeline-merge-result-failure: + rules: + - <<: *if-security-pipeline-merge-result + when: on_failure + ###################### # Dev fixtures rules # ###################### @@ -336,6 +353,7 @@ .frontend:rules:compile-test-assets: rules: - changes: *code-backstage-qa-patterns + - <<: *if-merge-request-title-run-all-rspec .frontend:rules:compile-test-assets-as-if-foss: rules: @@ -483,6 +501,7 @@ rules: - <<: *if-default-refs changes: *code-backstage-qa-patterns + - <<: *if-merge-request-title-run-all-rspec .rails:rules:ee-only-migration: rules: @@ -628,6 +647,13 @@ - <<: *if-master-schedule-2-hourly - <<: *if-merge-request-title-run-all-rspec +.rails:rules:rspec-feature-flags: + rules: + - <<: *if-not-ee + when: never + - <<: *if-master-schedule-2-hourly + - <<: *if-merge-request-title-run-all-rspec + .rails:rules:master-schedule-nightly--code-backstage: rules: - <<: *if-master-schedule-nightly @@ -702,6 +728,14 @@ - <<: *if-master-schedule-nightly allow_failure: true +.reports:rules:license_scanning: + rules: + - if: '$LICENSE_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\blicense_scanning\b/' + when: never + - <<: *if-default-refs + changes: *code-backstage-qa-patterns + allow_failure: true + ################ # Review rules # ################ @@ -859,6 +893,7 @@ - <<: *if-default-refs changes: *code-backstage-patterns when: on_success + - <<: *if-merge-request-title-run-all-rspec .test-metadata:rules:update-tests-metadata: rules: diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 1764e9136a1..2d83531e1db 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -38,6 +38,6 @@ update-tests-metadata: - rspec-ee integration pg11 geo - rspec-ee system pg11 geo script: - - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document - - source scripts/rspec_helpers.sh + - run_timed_command "retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document" + - source ./scripts/rspec_helpers.sh - update_tests_metadata |