Add latest changes from gitlab-org/gitlab@master

3 files changed, 79 insertions, 47 deletions

diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index 4cc03fdb1a4..7920d835a29 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -74,16 +74,16 @@ gemnasium-dependency_scanning:
     - apk add jq
     # Lower execa severity based on https://gitlab.com/gitlab-org/gitlab/-/issues/223859#note_452922390
     - jq '(.vulnerabilities[] | select (.cve == "yarn.lock:execa:gemnasium:05cfa2e8-2d0c-42c1-8894-638e2f12ff3d")).severity = "Medium"' gl-dependency-scanning-report.json > temp.json && mv temp.json gl-dependency-scanning-report.json
-  rules: !reference [".reports:rules:dependency_scanning", rules]
+  rules: !reference [".reports:rules:gemnasium-dependency_scanning", rules]
 
 bundler-audit-dependency_scanning:
-  rules: !reference [".reports:rules:dependency_scanning", rules]
+  rules: !reference [".reports:rules:bundler-audit-dependency_scanning", rules]
 
 retire-js-dependency_scanning:
-  rules: !reference [".reports:rules:dependency_scanning", rules]
+  rules: !reference [".reports:rules:retire-js-dependency_scanning", rules]
 
 gemnasium-python-dependency_scanning:
-  rules: !reference [".reports:rules:dependency_scanning", rules]
+  rules: !reference [".reports:rules:gemnasium-python-dependency_scanning", rules]
 
 # Analyze dependencies for malicious behavior
 # See https://gitlab.com/gitlab-com/gl-security/security-research/package-hunter
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 27cec1d599b..5ef248c27e8 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -66,8 +66,6 @@ review-deploy:
     - *base-before_script
   script:
     - check_kube_domain
-    - "ensure_namespace ${KUBE_NAMESPACE}"
-    - install_external_dns
     - download_chart
     - date
     - deploy || (display_deployment_debug && exit 1)
@@ -116,13 +114,14 @@ review-stop:
 .review-qa-base:
   extends:
     - .use-docker-in-docker
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7
+  image:
+    name: ${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}
+    entrypoint: [""]
   stage: qa
-  needs: ["review-deploy"]
+  needs: ["build-qa-image", "review-deploy"]
   variables:
-    QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
-    QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
     QA_DEBUG: "true"
+    QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
     QA_GENERATE_ALLURE_REPORT: "true"
     GITLAB_USERNAME: "root"
     GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
@@ -132,15 +131,16 @@ review-stop:
     EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
     SIGNUP_DISABLED: "true"
   before_script:
-    - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}"
+    # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync.
+    - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then
+        git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA};
+      fi
     - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
-    - echo "${QA_IMAGE}"
-    - *base-before_script
-    - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}
+    - cd qa
   artifacts:
     paths:
-      - ./qa/gitlab-qa-run-*
+      - qa/tmp
     expire_in: 7 days
     when: always
 
@@ -157,7 +157,7 @@ review-stop:
   script:
     - |
       allure-report-publisher upload gcs \
-        --results-glob="qa/gitlab-qa-run-*/**/allure-results/*" \
+        --results-glob="qa/tmp/allure-results/*" \
         --bucket="gitlab-qa-allure-reports" \
         --prefix="$ALLURE_REPORT_PATH_PREFIX/$CI_COMMIT_REF_SLUG" \
         --update-pr="comment" \
@@ -171,7 +171,7 @@ review-qa-smoke:
     - .review:rules:review-qa-smoke
   retry: 1  # This is confusing but this means "2 runs at max".
   script:
-    - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
+    - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}"
 
 review-qa-all:
   extends:
@@ -181,7 +181,14 @@ review-qa-all:
   script:
     - export KNAPSACK_REPORT_PATH=knapsack/master_report.json
     - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
-    - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation
+    - |
+      bin/test Test::Instance::All "${CI_ENVIRONMENT_URL}" \
+        -- \
+        --color --format documentation \
+        --format RspecJunitFormatter --out tmp/rspec.xml
+  artifacts:
+    reports:
+      junit: qa/tmp/rspec.xml
 
 review-performance:
   extends:
@@ -209,32 +216,6 @@ review-performance:
       performance: performance.json
     expire_in: 31d
 
-parallel-spec-reports:
-  extends:
-    - .review:rules:review-qa-all
-  image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine
-  stage: post-qa
-  needs: ["review-qa-all"]
-  variables:
-    NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
-    BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"
-  script:
-    - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/*
-    - gem install nokogiri --no-document
-    - cd qa/gitlab-qa-run-*/gitlab-*
-    - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_)
-    - cd -
-    - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}'
-    - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm
-  artifacts:
-    when: always
-    paths:
-      - qa/report-new.html
-      - qa/gitlab-qa-run-*
-    reports:
-      junit: qa/gitlab-qa-run-*/**/rspec-*.xml
-    expire_in: 31d
-
 allure-report-qa-smoke:
   extends:
     - .allure-report-base
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index fa4a5d3e3d8..a1c4a26df47 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -131,6 +131,30 @@
   - ".markdownlint.yml"
   - "scripts/lint-doc.sh"
 
+.bundler-patterns: &bundler-patterns
+  - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
+
+.nodejs-patterns: &nodejs-patterns
+  - '{package.json,*/package.json,*/*/package.json}'
+
+.python-patterns: &python-patterns
+  - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}'
+  - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}'
+  - '{Pipfile,*/Pipfile,*/*/Pipfile}'
+  - '{requires.txt,*/requires.txt,*/*/requires.txt}'
+  - '{setup.py,*/setup.py,*/*/setup.py}'
+
+.dependency-patterns: &dependency-patterns
+  - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
+  - '{composer.lock,*/composer.lock,*/*/composer.lock}'
+  - '{gems.locked,*/gems.locked,*/*/gems.locked}'
+  - '{go.sum,*/go.sum,*/*/go.sum}'
+  - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}'
+  - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}'
+  - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}'
+  - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}'
+  - '{conan.lock,*/conan.lock,*/*/conan.lock}'
+
 .frontend-dependency-patterns: &frontend-dependency-patterns
   - "{package.json,yarn.lock}"
   - "config/webpack.config.js"
@@ -1027,13 +1051,40 @@
     - changes: *code-backstage-qa-patterns
       allow_failure: true
 
-.reports:rules:dependency_scanning:
+.reports:rules:gemnasium-dependency_scanning:
   rules:
-    - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/'
+    - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/'
       when: never
     # - <<: *if-default-branch-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
     - <<: *if-default-refs
-      changes: *code-backstage-qa-patterns
+      changes: *dependency-patterns
+      allow_failure: true
+
+.reports:rules:bundler-audit-dependency_scanning:
+  rules:
+    - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/'
+      when: never
+    # - <<: *if-default-branch-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
+    - <<: *if-default-refs
+      changes: *bundler-patterns
+      allow_failure: true
+
+.reports:rules:retire-js-dependency_scanning:
+  rules:
+    - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /retire.js/'
+      when: never
+    # - <<: *if-default-branch-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
+    - <<: *if-default-refs
+      changes: *nodejs-patterns
+      allow_failure: true
+
+.reports:rules:gemnasium-python-dependency_scanning:
+  rules:
+    - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/'
+      when: never
+    # - <<: *if-default-branch-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
+    - <<: *if-default-refs
+      changes: *python-patterns
       allow_failure: true
 
 .reports:rules:dast: