diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
commit | 85dc423f7090da0a52c73eb66faf22ddb20efff9 (patch) | |
tree | 9160f299afd8c80c038f08e1545be119f5e3f1e1 /.gitlab/ci | |
parent | 15c2c8c66dbe422588e5411eee7e68f1fa440bb8 (diff) |
Add latest changes from gitlab-org/gitlab@13-4-stable-ee
Diffstat (limited to '.gitlab/ci')
-rw-r--r-- | .gitlab/ci/dast.gitlab-ci.yml | 203 | ||||
-rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/notify.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 54 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 50 | ||||
-rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 17 | ||||
-rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 100 | ||||
-rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 43 |
8 files changed, 412 insertions, 59 deletions
diff --git a/.gitlab/ci/dast.gitlab-ci.yml b/.gitlab/ci/dast.gitlab-ci.yml new file mode 100644 index 00000000000..93f64930822 --- /dev/null +++ b/.gitlab/ci/dast.gitlab-ci.yml @@ -0,0 +1,203 @@ +.dast_conf: + tags: + - prm + # For scheduling dast job + extends: + - .reports:schedule-dast + image: + name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" + resource_group: dast_scan + variables: + DAST_USERNAME_FIELD: "user[login]" + DAST_PASSWORD_FIELD: "user[password]" + DAST_FULL_SCAN_ENABLED: "true" + DAST_SPIDER_MINS: 0 + # TBD pin to a version + DAST_VERSION: 1.22.1 + # -Xmx is used to set the JVM memory to 6GB to prevent DAST OutOfMemoryError. + DAST_ZAP_CLI_OPTIONS: "-Xmx6144m" + DAST_RULES: "41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,90019,10109,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028,10003,50003,0,2,3,6,7,10010,10011,10015,10017,10019,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,40016,40017,40018,50000,50001,90011,90020,90022,90033" + before_script: + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' + # Below three lines can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/230687 is fixed + - mkdir -p /zap/xml + - 'sed -i "84 s/true/false/" /zap/xml/config.xml' + - cat /zap/xml/config.xml + # Help pages are excluded from scan as they are static pages. + # profile/two_factor_auth is excluded from scan to prevent 2FA from being turned on from user profile, which will reduce coverage. + - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/help/.*,${DAST_WEBSITE}/profile/two_factor_auth,${DAST_WEBSITE}/users/sign_out"' + - enable_rule () { read all_rules; rule=$1; echo $all_rules | sed -r "s/(,)?$rule(,)?/\1-1\2/" ; } + # Sort ids in DAST_RULES ascendingly, which is required when using DAST_RULES as argument to enable_rule + - 'DAST_RULES=$(echo $DAST_RULES | tr "," "\n" | sort -n | paste -sd ",")' + needs: ["review-deploy"] + stage: dast + # Default job timeout set to 90m and dast rules needs 2h to so that it won't timeout. + timeout: 2h + # Add retry because of intermittent connection problems. See https://gitlab.com/gitlab-org/gitlab/-/issues/244313 + retry: 1 + artifacts: + paths: + - gl-dast-report.json # GitLab-specific + reports: + dast: gl-dast-report.json + expire_in: 1 week # GitLab-specific + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset1: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user1" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10019 | enable_rule 10020 | enable_rule 10021 | enable_rule 10023 | enable_rule 10024 | enable_rule 10025 | enable_rule 10037 | enable_rule 10040 | enable_rule 10054 | enable_rule 10055 | enable_rule 10056) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset2: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user2" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90011 | enable_rule 90020 | enable_rule 90022 | enable_rule 90033) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset3: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user3" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40016 | enable_rule 40017 | enable_rule 50000 | enable_rule 50001) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset4: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user4" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 0 | enable_rule 2 | enable_rule 3 | enable_rule 7 ) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset5: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user5" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10010 | enable_rule 10011 | enable_rule 10015 | enable_rule 10017 | enable_rule 10019) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with a subset of Release scan rules. +DAST-fullscan-ruleset6: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user6" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 30001 | enable_rule 40009) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed +# DAST scan with a subset of Beta scan rules. +# DAST-fullscan-ruleset7: +# extends: +# - .dast_conf +# variables: +# DAST_USERNAME: "user7" +# script: +# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10098 | enable_rule 10105 | enable_rule 10202 | enable_rule 30002 | enable_rule 40003 | enable_rule 40008 | enable_rule 40009) +# - echo $DAST_EXCLUDE_RULES +# - /analyze -t $DAST_WEBSITE -d + +# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed +# Below jobs runs DAST scans with one time consuming scan rule. These scan rules are disabled in above jobs so that those jobs won't timeout. +# DAST scan with rule - 20019 External Redirect +# DAST-fullscan-rule-20019: +# extends: +# - .dast_conf +# variables: +# DAST_USERNAME: "user8" +# script: +# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 20019) +# - echo $DAST_EXCLUDE_RULES +# - /analyze -t $DAST_WEBSITE -d + +# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed +# DAST scan with rule - 10107 Httpoxy - Proxy Header Misuse - Active/beta +# DAST-fullscan-rule-10107: +# extends: +# - .dast_conf +# variables: +# DAST_USERNAME: "user9" +# script: +# - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10107) +# - echo $DAST_EXCLUDE_RULES +# - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 90020 Remote OS Command Injection +DAST-fullscan-rule-90020: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user10" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90020) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 40018 SQL Injection - Active/release +DAST-fullscan-rule-40018: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user11" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40018) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 40014 Cross Site Scripting (Persistent) - Active/release +DAST-fullscan-rule-40014: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user12" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40014) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 6 Path travesal +DAST-fullscan-rule-6: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user13" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 6) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d + +# DAST scan with rule - 40012 Cross Site Scripting (Reflected) +DAST-fullscan-rule-40012: + extends: + - .dast_conf + variables: + DAST_USERNAME: "user14" + script: + - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40012) + - echo $DAST_EXCLUDE_RULES + - /analyze -t $DAST_WEBSITE -d diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 62546e59368..0e0e156a64f 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -42,7 +42,7 @@ docs lint: extends: - .default-retry - .docs:rules:docs-lint - image: "registry.gitlab.com/gitlab-org/gitlab-docs:lint" + image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint:vale-2.3.4-markdownlint-0.23.2" stage: test needs: [] script: diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml index fcdd5ee97d2..6dcf19da942 100644 --- a/.gitlab/ci/notify.gitlab-ci.yml +++ b/.gitlab/ci/notify.gitlab-ci.yml @@ -10,7 +10,7 @@ notify-update-gitaly: extends: - .notify-slack rules: - - if: '$CI_MERGE_REQUEST_IID && $CI_COMMIT_BRANCH == $GITALY_UPDATE_BRANCH' + - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $GITALY_UPDATE_BRANCH' when: on_failure allow_failure: true variables: diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 0b54626f690..165476678bb 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -296,6 +296,21 @@ gitlab:setup: paths: - log/*.log +db:backup_and_restore: + extends: .db-job-base + variables: + SETUP_DB: "false" + GITLAB_ASSUME_YES: "1" + script: + - . scripts/prepare_build.sh + - bundle exec rake db:drop db:create db:structure:load db:seed_fu + - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry} + - bundle exec rake gitlab:backup:create + - date + - bundle exec rake gitlab:backup:restore + rules: + - changes: ["lib/backup/**/*"] + rspec:coverage: extends: - .rails-job-base @@ -490,21 +505,50 @@ rspec-ee system pg12 geo: ################################################## # EE: Canonical MR pipelines +rspec fail-fast: + extends: + - .rspec-ee-base-pg11 # This job also runs EE spec which needs elasticsearch + - .rails:rules:rspec fail-fast + stage: test + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] + script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - source scripts/rspec_helpers.sh + - rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine" + artifacts: + expire_in: 7d + paths: + - tmp/capybara/ + rspec foss-impact: extends: - .rspec-base-pg11-as-if-foss - - .rails:rules:ee-mr-only + - .rails:rules:rspec-foss-impact + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"] script: - - install_gitlab_gem - run_timed_command "scripts/gitaly-test-build" - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - - tooling/bin/find_foss_tests tmp/matching_foss_tests.txt - - rspec_matched_tests tmp/matching_foss_tests.txt "--tag ~quarantine" + - rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine" artifacts: expire_in: 7d paths: - - tmp/matching_foss_tests.txt - tmp/capybara/ + +fail-pipeline-early: + extends: + - .rails:rules:fail-pipeline-early + stage: post-test + needs: + - job: rspec fail-fast + artifacts: false + variables: + GIT_DEPTH: 1 + before_script: + - source scripts/utils.sh + - install_api_client_dependencies_with_apt + script: + - fail_pipeline_early # EE: Canonical MR pipelines ################################################## diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index b9f81f2eb0f..0e2f12789db 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -81,7 +81,13 @@ nodejs-scan-sast: secrets-sast: extends: .sast image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG" + name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:3" + artifacts: + paths: + - gl-secret-detection-report.json # GitLab-specific + reports: + sast: gl-secret-detection-report.json + expire_in: 1 week # GitLab-specific # We need to duplicate this job's definition because it seems it's impossible to # override an included `only.refs`. @@ -145,45 +151,3 @@ dependency_scanning: reports: dependency_scanning: gl-dependency-scanning-report.json expire_in: 1 week # GitLab-specific - -# Temporarily disabling review apps -## We need to duplicate this job's definition because it seems it's impossible to -## override an included `only.refs`. -## See https://gitlab.com/gitlab-org/gitlab/issues/31371. -# dast: -# extends: -# - .default-retry -# - .reports:rules:dast -# # This is needed so that manual jobs with needs don't block the pipeline. -# # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. -# dependencies: ["review-deploy"] -# stage: qa # GitLab-specific -# image: -# name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" -# variables: -# # To be done in a later iteration -# # DAST_USERNAME: "root" -# # DAST_USERNAME_FIELD: "user[login]" -# # DAST_PASSWORD_FIELD: "user[passowrd]" -# DAST_VERSION: 1 -# script: -# - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' -# # To be done in a later iteration -# # - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' -# # - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' -# - /analyze -t $DAST_WEBSITE -# timeout: 4h -# artifacts: -# paths: -# - gl-dast-report.json # GitLab-specific -# reports: -# dast: gl-dast-report.json -# expire_in: 1 week # GitLab-specific - -# To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 -# schedule:dast: -# extends: -# - dast -# - .reports:schedule-dast -# variables: -# DAST_FULL_SCAN_ENABLED: "true" diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 4e3a80372a6..d34687cfdad 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -43,9 +43,9 @@ review-build-cng: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" REVIEW_APPS_DOMAIN: "temp.gitlab-review.app" # FIXME: using temporary domain DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "v4.1.3" + GITLAB_HELM_CHART_REF: "v4.3.0" environment: - name: review/${CI_COMMIT_REF_NAME} + name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop auto_stop_in: 48 hours @@ -53,7 +53,7 @@ review-build-cng: review-deploy: extends: - .review-workflow-base - - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise + - .review:rules:review-deploy stage: review dependencies: [] resource_group: "review/${CI_COMMIT_REF_NAME}" @@ -77,6 +77,11 @@ review-deploy: # to have to manually start the jobs in sequence, so we do it for them. - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"' - '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"' + after_script: + # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. + # Set DAST_RUN to true when jobs are manually scheduled. + - if [ "$DAST_RUN" == "true" ]; then source scripts/review_apps/seed-dast-test-data.sh; TRACE=1 trigger_proj_user_creation; fi + artifacts: paths: [environment_url.txt] expire_in: 2 days @@ -108,8 +113,8 @@ review-stop-failed-deployment: review-stop: extends: - .review-stop-base - - .review:rules:mr-only-manual - stage: review + - .review:rules:review-stop + stage: post-qa script: - delete_release @@ -167,7 +172,7 @@ review-qa-all: review-performance: extends: - .default-retry - - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise + - .review:rules:review-performance image: name: sitespeedio/sitespeed.io:6.3.1 entrypoint: [""] diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 839a06862b2..a8e0e1ccaaa 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -67,6 +67,12 @@ .if-cache-credentials-schedule: &if-cache-credentials-schedule if: '$CI_REPO_CACHE_CREDENTIALS && $CI_PIPELINE_SOURCE == "schedule"' +.if-rspec-fail-fast-disabled: &if-rspec-fail-fast-disabled + if: '$RSPEC_FAIL_FAST_ENABLED != "true"' + +.if-rspec-fail-fast-skipped: &if-rspec-fail-fast-skipped + if: '$CI_MERGE_REQUEST_TITLE =~ /SKIP RSPEC FAIL-FAST/' + #################### # Changes patterns # #################### @@ -83,6 +89,7 @@ - ".gitlab/ci/frontend.gitlab-ci.yml" - ".gitlab/ci/build-images.gitlab-ci.yml" - ".gitlab/ci/review.gitlab-ci.yml" + - "scripts/trigger-build" .ci-qa-patterns: &ci-qa-patterns - ".gitlab-ci.yml" @@ -121,11 +128,13 @@ - "{,ee/}spec/**/*.rb" - ".gitlab-ci.yml" - ".gitlab/ci/**/*" + - "*_VERSION" .db-patterns: &db-patterns - "{,ee/}{,spec/}{db,migrations}/**/*" - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration/**/*" - "config/prometheus/common_metrics.yml" # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer + - "{,ee/}app/models/project_statistics.rb" # Used to calculate sizes in migration specs .backstage-patterns: &backstage-patterns - "Dangerfile" @@ -147,6 +156,7 @@ - "*_VERSION" - "Gemfile{,.lock}" - "Rakefile" + - "tests.yml" - "config.ru" - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated @@ -168,6 +178,7 @@ - "*_VERSION" - "Gemfile{,.lock}" - "Rakefile" + - "tests.yml" - "config.ru" - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated @@ -191,6 +202,7 @@ - "*_VERSION" - "Gemfile{,.lock}" - "Rakefile" + - "tests.yml" - "config.ru" - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated @@ -211,6 +223,7 @@ - "*_VERSION" - "Gemfile{,.lock}" - "Rakefile" + - "tests.yml" - "config.ru" - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated @@ -506,6 +519,7 @@ - <<: *if-security-merge-request changes: *db-patterns - <<: *if-merge-request-title-as-if-foss + changes: *db-patterns - <<: *if-merge-request changes: *ci-patterns @@ -516,6 +530,7 @@ - <<: *if-security-merge-request changes: *backend-patterns - <<: *if-merge-request-title-as-if-foss + changes: *backend-patterns - <<: *if-merge-request changes: *ci-patterns @@ -526,6 +541,7 @@ - <<: *if-security-merge-request changes: *backend-patterns - <<: *if-merge-request-title-as-if-foss + changes: *backend-patterns - <<: *if-merge-request changes: *ci-patterns @@ -536,6 +552,7 @@ - <<: *if-security-merge-request changes: *code-backstage-patterns - <<: *if-merge-request-title-as-if-foss + changes: *code-backstage-patterns - <<: *if-merge-request changes: *ci-patterns @@ -549,7 +566,16 @@ - <<: *if-master-refs changes: *code-backstage-patterns -.rails:rules:ee-mr-only: +.rails:rules:detect-tests: + rules: + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *code-backstage-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-backstage-patterns + +.rails:rules:rspec-foss-impact: rules: - <<: *if-not-ee when: never @@ -560,6 +586,34 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *code-backstage-patterns +.rails:rules:rspec fail-fast: + rules: + - <<: *if-rspec-fail-fast-disabled + when: never + - <<: *if-rspec-fail-fast-skipped + when: never + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *code-backstage-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-backstage-patterns + +.rails:rules:fail-pipeline-early: + rules: + - <<: *if-rspec-fail-fast-disabled + when: never + - <<: *if-rspec-fail-fast-skipped + when: never + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *code-backstage-patterns + when: on_failure + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-backstage-patterns + when: on_failure + .rails:rules:downtime_check: rules: - <<: *if-merge-request @@ -569,6 +623,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request + changes: *code-backstage-patterns - <<: *if-master-schedule-2-hourly - <<: *if-merge-request-title-run-all-rspec @@ -643,7 +699,8 @@ rules: - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' when: never - - <<: *if-dot-com-gitlab-org-schedule + - <<: *if-master-schedule-nightly + allow_failure: true ################ # Review rules # @@ -662,8 +719,26 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-schedule -.review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise: +.review:rules:review-deploy: + rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-review-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *frontend-patterns + allow_failure: true + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: manual + allow_failure: true + - <<: *if-dot-com-gitlab-org-schedule + allow_failure: true + +.review:rules:review-performance: rules: + - if: '$DAST_RUN == "true"' # Skip this job when DAST is run + when: never - <<: *if-not-ee when: never - <<: *if-dot-com-gitlab-org-merge-request @@ -719,6 +794,17 @@ - <<: *if-dot-com-gitlab-org-schedule allow_failure: true +.review:rules:review-stop: + rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: manual + allow_failure: true + - <<: *if-dot-com-gitlab-org-schedule + allow_failure: true + .review:rules:danger: rules: - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID' @@ -757,6 +843,14 @@ changes: *code-backstage-patterns when: on_success +.setup:rules:verify-tests-yml: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + ####################### # Test metadata rules # ####################### diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 26c7a2194cc..cf42d2a8a5e 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -48,3 +48,46 @@ no_ee_check: stage: test script: - scripts/no-ee-check + +verify-tests-yml: + extends: + - .setup:rules:verify-tests-yml + image: ruby:2.6-alpine + stage: test + needs: [] + script: + - source scripts/utils.sh + - install_tff_gem + - scripts/verify-tff-mapping + +.detect-test-base: + image: ruby:2.6-alpine + needs: [] + stage: prepare + script: + - source scripts/utils.sh + - install_gitlab_gem + - install_tff_gem + - tooling/bin/find_foss_tests ${MATCHED_TESTS_FILE} + - 'echo "test files affected: $(cat $MATCHED_TESTS_FILE)"' + artifacts: + expire_in: 7d + paths: + - ${MATCHED_TESTS_FILE} + +detect-tests: + extends: + - .detect-test-base + - .rails:rules:detect-tests + variables: + MATCHED_TESTS_FILE: tmp/matching_tests.txt + +detect-tests as-if-foss: + extends: + - .detect-test-base + - .rails:rules:detect-tests + - .as-if-foss + variables: + MATCHED_TESTS_FILE: tmp/matching_foss_tests.txt + before_script: + - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' |