Add latest changes from gitlab-org/gitlab@13-4-stable-ee

8 files changed, 412 insertions, 59 deletions

diff --git a/.gitlab/ci/dast.gitlab-ci.yml b/.gitlab/ci/dast.gitlab-ci.yml
new file mode 100644
index 00000000000..93f64930822
--- /dev/null
+++ b/.gitlab/ci/dast.gitlab-ci.yml
@@ -0,0 +1,203 @@
+.dast_conf:
+  tags:
+    - prm
+  # For scheduling dast job
+  extends:
+    - .reports:schedule-dast
+  image:
+    name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
+  resource_group: dast_scan
+  variables:
+    DAST_USERNAME_FIELD: "user[login]"
+    DAST_PASSWORD_FIELD: "user[password]"
+    DAST_FULL_SCAN_ENABLED: "true"
+    DAST_SPIDER_MINS: 0
+    # TBD pin to a version
+    DAST_VERSION: 1.22.1
+    # -Xmx is used to set the JVM memory to 6GB to prevent DAST OutOfMemoryError.
+    DAST_ZAP_CLI_OPTIONS: "-Xmx6144m"
+    DAST_RULES: "41,42,43,10027,10032,10041,10042,10045,10047,10052,10053,10057,10061,10096,10097,10104,10106,20012,20014,20015,20016,20017,20018,40019,40020,40021,40024,40025,40027,40029,40032,90001,90019,10109,10026,10028,10029,10030,10031,10033,10034,10035,10036,10038,10039,10043,10044,10048,10050,10051,10058,10062,10095,10107,10108,30003,40013,40022,40023,40028,90021,90023,90024,90025,90027,90028,10003,50003,0,2,3,6,7,10010,10011,10015,10017,10019,10020,10021,10023,10024,10025,10037,10040,10054,10055,10056,10098,10105,10202,20019,30001,30002,40003,40008,40009,40012,40014,40016,40017,40018,50000,50001,90011,90020,90022,90033"
+  before_script:
+    - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"'
+    - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"'
+    - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"'
+    # Below three lines can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/230687 is fixed
+    - mkdir -p /zap/xml
+    - 'sed -i "84 s/true/false/" /zap/xml/config.xml'
+    - cat /zap/xml/config.xml
+    # Help pages are excluded from scan as they are static pages.
+    # profile/two_factor_auth is excluded from scan to prevent 2FA from being turned on from user profile, which will reduce coverage.
+    - 'export DAST_AUTH_EXCLUDE_URLS="${DAST_WEBSITE}/help/.*,${DAST_WEBSITE}/profile/two_factor_auth,${DAST_WEBSITE}/users/sign_out"'
+    - enable_rule () { read all_rules; rule=$1; echo $all_rules | sed -r "s/(,)?$rule(,)?/\1-1\2/" ; }
+    # Sort ids in DAST_RULES ascendingly, which is required when using DAST_RULES as argument to enable_rule
+    - 'DAST_RULES=$(echo $DAST_RULES | tr "," "\n" | sort -n | paste -sd ",")'
+  needs: ["review-deploy"]
+  stage: dast
+  # Default job timeout set to 90m and dast rules needs 2h to so that it won't timeout.
+  timeout: 2h
+  # Add retry because of intermittent connection problems. See https://gitlab.com/gitlab-org/gitlab/-/issues/244313
+  retry: 1
+  artifacts:
+    paths:
+      - gl-dast-report.json  # GitLab-specific
+    reports:
+      dast: gl-dast-report.json
+    expire_in: 1 week  # GitLab-specific
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset1:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user1"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10019 | enable_rule 10020 | enable_rule 10021 | enable_rule 10023 | enable_rule 10024 | enable_rule 10025 | enable_rule 10037 | enable_rule 10040 | enable_rule 10054 | enable_rule 10055 | enable_rule 10056)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset2:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user2"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90011 | enable_rule 90020 | enable_rule 90022 | enable_rule 90033)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset3:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user3"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40016 | enable_rule 40017 | enable_rule 50000 | enable_rule 50001)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset4:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user4"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 0 | enable_rule 2 | enable_rule 3 | enable_rule 7 )
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset5:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user5"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10010 | enable_rule 10011 | enable_rule 10015 | enable_rule 10017 | enable_rule 10019)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with a subset of Release scan rules.
+DAST-fullscan-ruleset6:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user6"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 30001 | enable_rule 40009)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed
+# DAST scan with a subset of Beta scan rules.
+# DAST-fullscan-ruleset7:
+#   extends:
+#     - .dast_conf
+#   variables:
+#     DAST_USERNAME: "user7"
+#   script:
+#     - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10098 | enable_rule 10105 | enable_rule 10202 | enable_rule 30002 | enable_rule 40003 | enable_rule 40008 | enable_rule 40009)
+#     - echo $DAST_EXCLUDE_RULES
+#     - /analyze -t $DAST_WEBSITE -d
+
+# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed
+# Below jobs runs DAST scans with one time consuming scan rule. These scan rules are disabled in above jobs so that those jobs won't timeout.
+# DAST scan with rule - 20019 External Redirect
+# DAST-fullscan-rule-20019:
+#   extends:
+#     - .dast_conf
+#   variables:
+#     DAST_USERNAME: "user8"
+#   script:
+#     - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 20019)
+#     - echo $DAST_EXCLUDE_RULES
+#     - /analyze -t $DAST_WEBSITE -d
+
+# Enable when https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39749 is fixed
+# DAST scan with rule -  10107 Httpoxy - Proxy Header Misuse - Active/beta
+# DAST-fullscan-rule-10107:
+#   extends:
+#     - .dast_conf
+#   variables:
+#     DAST_USERNAME: "user9"
+#   script:
+#     - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 10107)
+#     - echo $DAST_EXCLUDE_RULES
+#     - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 90020 Remote OS Command Injection
+DAST-fullscan-rule-90020:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user10"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 90020)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 40018 SQL Injection - Active/release
+DAST-fullscan-rule-40018:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user11"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40018)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 40014 Cross Site Scripting (Persistent) - Active/release
+DAST-fullscan-rule-40014:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user12"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40014)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 6 Path travesal
+DAST-fullscan-rule-6:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user13"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 6)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
+
+# DAST scan with rule - 40012 Cross Site Scripting (Reflected)
+DAST-fullscan-rule-40012:
+  extends:
+    - .dast_conf
+  variables:
+    DAST_USERNAME: "user14"
+  script:
+    - export DAST_EXCLUDE_RULES=$(echo $DAST_RULES | enable_rule 40012)
+    - echo $DAST_EXCLUDE_RULES
+    - /analyze -t $DAST_WEBSITE -d
diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml
index 62546e59368..0e0e156a64f 100644
--- a/.gitlab/ci/docs.gitlab-ci.yml
+++ b/.gitlab/ci/docs.gitlab-ci.yml
@@ -42,7 +42,7 @@ docs lint:
   extends:
     - .default-retry
     - .docs:rules:docs-lint
-  image: "registry.gitlab.com/gitlab-org/gitlab-docs:lint"
+  image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint:vale-2.3.4-markdownlint-0.23.2"
   stage: test
   needs: []
   script:
diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml
index fcdd5ee97d2..6dcf19da942 100644
--- a/.gitlab/ci/notify.gitlab-ci.yml
+++ b/.gitlab/ci/notify.gitlab-ci.yml
@@ -10,7 +10,7 @@ notify-update-gitaly:
   extends:
     - .notify-slack
   rules:
-    - if: '$CI_MERGE_REQUEST_IID && $CI_COMMIT_BRANCH == $GITALY_UPDATE_BRANCH'
+    - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == $GITALY_UPDATE_BRANCH'
       when: on_failure
       allow_failure: true
   variables:
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 0b54626f690..165476678bb 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -296,6 +296,21 @@ gitlab:setup:
     paths:
       - log/*.log
 
+db:backup_and_restore:
+  extends: .db-job-base
+  variables:
+    SETUP_DB: "false"
+    GITLAB_ASSUME_YES: "1"
+  script:
+    - . scripts/prepare_build.sh
+    - bundle exec rake db:drop db:create db:structure:load db:seed_fu
+    - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry}
+    - bundle exec rake gitlab:backup:create
+    - date
+    - bundle exec rake gitlab:backup:restore
+  rules:
+    - changes: ["lib/backup/**/*"]
+
 rspec:coverage:
   extends:
     - .rails-job-base
@@ -490,21 +505,50 @@ rspec-ee system pg12 geo:
 
 ##################################################
 # EE: Canonical MR pipelines
+rspec fail-fast:
+  extends:
+    - .rspec-ee-base-pg11 # This job also runs EE spec which needs elasticsearch
+    - .rails:rules:rspec fail-fast
+  stage: test
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"]
+  script:
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
+    - source scripts/rspec_helpers.sh
+    - rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine"
+  artifacts:
+    expire_in: 7d
+    paths:
+      - tmp/capybara/
+
 rspec foss-impact:
   extends:
     - .rspec-base-pg11-as-if-foss
-    - .rails:rules:ee-mr-only
+    - .rails:rules:rspec-foss-impact
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"]
   script:
-    - install_gitlab_gem
     - run_timed_command "scripts/gitaly-test-build"
     - run_timed_command "scripts/gitaly-test-spawn"
     - source scripts/rspec_helpers.sh
-    - tooling/bin/find_foss_tests tmp/matching_foss_tests.txt
-    - rspec_matched_tests tmp/matching_foss_tests.txt "--tag ~quarantine"
+    - rspec_matched_foss_tests tmp/matching_foss_tests.txt "--tag ~quarantine"
   artifacts:
     expire_in: 7d
     paths:
-      - tmp/matching_foss_tests.txt
       - tmp/capybara/
+
+fail-pipeline-early:
+  extends:
+    - .rails:rules:fail-pipeline-early
+  stage: post-test
+  needs:
+    - job: rspec fail-fast
+      artifacts: false
+  variables:
+    GIT_DEPTH: 1
+  before_script:
+    - source scripts/utils.sh
+    - install_api_client_dependencies_with_apt
+  script:
+    - fail_pipeline_early
 # EE: Canonical MR pipelines
 ##################################################
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index b9f81f2eb0f..0e2f12789db 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -81,7 +81,13 @@ nodejs-scan-sast:
 secrets-sast:
   extends: .sast
   image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG"
+    name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:3"
+  artifacts:
+    paths:
+      - gl-secret-detection-report.json  # GitLab-specific
+    reports:
+      sast: gl-secret-detection-report.json
+    expire_in: 1 week  # GitLab-specific
 
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.
@@ -145,45 +151,3 @@ dependency_scanning:
     reports:
       dependency_scanning: gl-dependency-scanning-report.json
     expire_in: 1 week  # GitLab-specific
-
-# Temporarily disabling review apps
-## We need to duplicate this job's definition because it seems it's impossible to
-## override an included `only.refs`.
-## See https://gitlab.com/gitlab-org/gitlab/issues/31371.
-# dast:
-#   extends:
-#     - .default-retry
-#     - .reports:rules:dast
-#   # This is needed so that manual jobs with needs don't block the pipeline.
-#   # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979.
-#   dependencies: ["review-deploy"]
-#   stage: qa  # GitLab-specific
-#   image:
-#     name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
-#   variables:
-#     # To be done in a later iteration
-#     # DAST_USERNAME: "root"
-#     # DAST_USERNAME_FIELD: "user[login]"
-#     # DAST_PASSWORD_FIELD: "user[passowrd]"
-#     DAST_VERSION: 1
-#   script:
-#     - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"'
-#     # To be done in a later iteration
-#     # - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"'
-#     # - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"'
-#     - /analyze -t $DAST_WEBSITE
-#   timeout: 4h
-#   artifacts:
-#     paths:
-#       - gl-dast-report.json  # GitLab-specific
-#     reports:
-#       dast: gl-dast-report.json
-#     expire_in: 1 week  # GitLab-specific
-
-# To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
-# schedule:dast:
-#   extends:
-#     - dast
-#     - .reports:schedule-dast
-#   variables:
-#     DAST_FULL_SCAN_ENABLED: "true"
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 4e3a80372a6..d34687cfdad 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -43,9 +43,9 @@ review-build-cng:
     HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
     REVIEW_APPS_DOMAIN: "temp.gitlab-review.app"  # FIXME: using temporary domain
     DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
-    GITLAB_HELM_CHART_REF: "v4.1.3"
+    GITLAB_HELM_CHART_REF: "v4.3.0"
   environment:
-    name: review/${CI_COMMIT_REF_NAME}
+    name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY}
     url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
     on_stop: review-stop
     auto_stop_in: 48 hours
@@ -53,7 +53,7 @@ review-build-cng:
 review-deploy:
   extends:
     - .review-workflow-base
-    - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise
+    - .review:rules:review-deploy
   stage: review
   dependencies: []
   resource_group: "review/${CI_COMMIT_REF_NAME}"
@@ -77,6 +77,11 @@ review-deploy:
     # to have to manually start the jobs in sequence, so we do it for them.
     - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"'
     - '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"'
+  after_script:
+    # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan.
+    # Set DAST_RUN to true when jobs are manually scheduled.
+    - if [ "$DAST_RUN" == "true" ]; then source scripts/review_apps/seed-dast-test-data.sh; TRACE=1 trigger_proj_user_creation; fi
+
   artifacts:
     paths: [environment_url.txt]
     expire_in: 2 days
@@ -108,8 +113,8 @@ review-stop-failed-deployment:
 review-stop:
   extends:
     - .review-stop-base
-    - .review:rules:mr-only-manual
-  stage: review
+    - .review:rules:review-stop
+  stage: post-qa
   script:
     - delete_release
 
@@ -167,7 +172,7 @@ review-qa-all:
 review-performance:
   extends:
     - .default-retry
-    - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise
+    - .review:rules:review-performance
   image:
     name: sitespeedio/sitespeed.io:6.3.1
     entrypoint: [""]
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index 839a06862b2..a8e0e1ccaaa 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -67,6 +67,12 @@
 .if-cache-credentials-schedule: &if-cache-credentials-schedule
   if: '$CI_REPO_CACHE_CREDENTIALS && $CI_PIPELINE_SOURCE == "schedule"'
 
+.if-rspec-fail-fast-disabled: &if-rspec-fail-fast-disabled
+  if: '$RSPEC_FAIL_FAST_ENABLED != "true"'
+
+.if-rspec-fail-fast-skipped: &if-rspec-fail-fast-skipped
+  if: '$CI_MERGE_REQUEST_TITLE =~ /SKIP RSPEC FAIL-FAST/'
+
 ####################
 # Changes patterns #
 ####################
@@ -83,6 +89,7 @@
   - ".gitlab/ci/frontend.gitlab-ci.yml"
   - ".gitlab/ci/build-images.gitlab-ci.yml"
   - ".gitlab/ci/review.gitlab-ci.yml"
+  - "scripts/trigger-build"
 
 .ci-qa-patterns: &ci-qa-patterns
   - ".gitlab-ci.yml"
@@ -121,11 +128,13 @@
   - "{,ee/}spec/**/*.rb"
   - ".gitlab-ci.yml"
   - ".gitlab/ci/**/*"
+  - "*_VERSION"
 
 .db-patterns: &db-patterns
   - "{,ee/}{,spec/}{db,migrations}/**/*"
   - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration/**/*"
   - "config/prometheus/common_metrics.yml" # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer
+  - "{,ee/}app/models/project_statistics.rb" # Used to calculate sizes in migration specs
 
 .backstage-patterns: &backstage-patterns
   - "Dangerfile"
@@ -147,6 +156,7 @@
   - "*_VERSION"
   - "Gemfile{,.lock}"
   - "Rakefile"
+  - "tests.yml"
   - "config.ru"
   - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
   - "doc/api/graphql/reference/*"  # Files in this folder are auto-generated
@@ -168,6 +178,7 @@
   - "*_VERSION"
   - "Gemfile{,.lock}"
   - "Rakefile"
+  - "tests.yml"
   - "config.ru"
   - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
   - "doc/api/graphql/reference/*"  # Files in this folder are auto-generated
@@ -191,6 +202,7 @@
   - "*_VERSION"
   - "Gemfile{,.lock}"
   - "Rakefile"
+  - "tests.yml"
   - "config.ru"
   - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
   - "doc/api/graphql/reference/*"  # Files in this folder are auto-generated
@@ -211,6 +223,7 @@
   - "*_VERSION"
   - "Gemfile{,.lock}"
   - "Rakefile"
+  - "tests.yml"
   - "config.ru"
   - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
   - "doc/api/graphql/reference/*"  # Files in this folder are auto-generated
@@ -506,6 +519,7 @@
     - <<: *if-security-merge-request
       changes: *db-patterns
     - <<: *if-merge-request-title-as-if-foss
+      changes: *db-patterns
     - <<: *if-merge-request
       changes: *ci-patterns
 
@@ -516,6 +530,7 @@
     - <<: *if-security-merge-request
       changes: *backend-patterns
     - <<: *if-merge-request-title-as-if-foss
+      changes: *backend-patterns
     - <<: *if-merge-request
       changes: *ci-patterns
 
@@ -526,6 +541,7 @@
     - <<: *if-security-merge-request
       changes: *backend-patterns
     - <<: *if-merge-request-title-as-if-foss
+      changes: *backend-patterns
     - <<: *if-merge-request
       changes: *ci-patterns
 
@@ -536,6 +552,7 @@
     - <<: *if-security-merge-request
       changes: *code-backstage-patterns
     - <<: *if-merge-request-title-as-if-foss
+      changes: *code-backstage-patterns
     - <<: *if-merge-request
       changes: *ci-patterns
 
@@ -549,7 +566,16 @@
     - <<: *if-master-refs
       changes: *code-backstage-patterns
 
-.rails:rules:ee-mr-only:
+.rails:rules:detect-tests:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-security-merge-request
+      changes: *code-backstage-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *code-backstage-patterns
+
+.rails:rules:rspec-foss-impact:
   rules:
     - <<: *if-not-ee
       when: never
@@ -560,6 +586,34 @@
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-backstage-patterns
 
+.rails:rules:rspec fail-fast:
+  rules:
+    - <<: *if-rspec-fail-fast-disabled
+      when: never
+    - <<: *if-rspec-fail-fast-skipped
+      when: never
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-security-merge-request
+      changes: *code-backstage-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *code-backstage-patterns
+
+.rails:rules:fail-pipeline-early:
+  rules:
+    - <<: *if-rspec-fail-fast-disabled
+      when: never
+    - <<: *if-rspec-fail-fast-skipped
+      when: never
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-security-merge-request
+      changes: *code-backstage-patterns
+      when: on_failure
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *code-backstage-patterns
+      when: on_failure
+
 .rails:rules:downtime_check:
   rules:
     - <<: *if-merge-request
@@ -569,6 +623,8 @@
   rules:
     - <<: *if-not-ee
       when: never
+    - <<: *if-merge-request
+      changes: *code-backstage-patterns
     - <<: *if-master-schedule-2-hourly
     - <<: *if-merge-request-title-run-all-rspec
 
@@ -643,7 +699,8 @@
   rules:
     - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/'
       when: never
-    - <<: *if-dot-com-gitlab-org-schedule
+    - <<: *if-master-schedule-nightly
+      allow_failure: true
 
 ################
 # Review rules #
@@ -662,8 +719,26 @@
       allow_failure: true
     - <<: *if-dot-com-gitlab-org-schedule
 
-.review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise:
+.review:rules:review-deploy:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *ci-review-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *frontend-patterns
+      allow_failure: true
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *code-qa-patterns
+      when: manual
+      allow_failure: true
+    - <<: *if-dot-com-gitlab-org-schedule
+      allow_failure: true
+
+.review:rules:review-performance:
   rules:
+    - if: '$DAST_RUN == "true"' # Skip this job when DAST is run
+      when: never
     - <<: *if-not-ee
       when: never
     - <<: *if-dot-com-gitlab-org-merge-request
@@ -719,6 +794,17 @@
     - <<: *if-dot-com-gitlab-org-schedule
       allow_failure: true
 
+.review:rules:review-stop:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *code-qa-patterns
+      when: manual
+      allow_failure: true
+    - <<: *if-dot-com-gitlab-org-schedule
+      allow_failure: true
+
 .review:rules:danger:
   rules:
     - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID'
@@ -757,6 +843,14 @@
       changes: *code-backstage-patterns
       when: on_success
 
+.setup:rules:verify-tests-yml:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-default-refs
+      changes: *code-backstage-patterns
+      when: on_success
+
 #######################
 # Test metadata rules #
 #######################
diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml
index 26c7a2194cc..cf42d2a8a5e 100644
--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -48,3 +48,46 @@ no_ee_check:
   stage: test
   script:
     - scripts/no-ee-check
+
+verify-tests-yml:
+  extends:
+    - .setup:rules:verify-tests-yml
+  image: ruby:2.6-alpine
+  stage: test
+  needs: []
+  script:
+    - source scripts/utils.sh
+    - install_tff_gem
+    - scripts/verify-tff-mapping
+
+.detect-test-base:
+  image: ruby:2.6-alpine
+  needs: []
+  stage: prepare
+  script:
+    - source scripts/utils.sh
+    - install_gitlab_gem
+    - install_tff_gem
+    - tooling/bin/find_foss_tests ${MATCHED_TESTS_FILE}
+    - 'echo "test files affected: $(cat $MATCHED_TESTS_FILE)"'
+  artifacts:
+    expire_in: 7d
+    paths:
+      - ${MATCHED_TESTS_FILE}
+
+detect-tests:
+  extends:
+    - .detect-test-base
+    - .rails:rules:detect-tests
+  variables:
+    MATCHED_TESTS_FILE: tmp/matching_tests.txt
+
+detect-tests as-if-foss:
+  extends:
+    - .detect-test-base
+    - .rails:rules:detect-tests
+    - .as-if-foss
+  variables:
+    MATCHED_TESTS_FILE: tmp/matching_foss_tests.txt
+  before_script:
+    - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'