diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-19 10:33:21 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-19 10:33:21 +0300 |
commit | 36a59d088eca61b834191dacea009677a96c052f (patch) | |
tree | e4f33972dab5d8ef79e3944a9f403035fceea43f /.gitlab/ci | |
parent | a1761f15ec2cae7c7f7bbda39a75494add0dfd6f (diff) |
Add latest changes from gitlab-org/gitlab@15-0-stable-eev15.0.0-rc42
Diffstat (limited to '.gitlab/ci')
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 39 | ||||
-rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 17 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 213 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 11 | ||||
-rw-r--r-- | .gitlab/ci/review-apps/main.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | .gitlab/ci/review-apps/qa.gitlab-ci.yml | 94 | ||||
-rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 47 | ||||
-rw-r--r-- | .gitlab/ci/static-analysis.gitlab-ci.yml | 6 | ||||
-rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 3 |
9 files changed, 234 insertions, 197 deletions
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 4069dfe9a2b..7e06a4a71bd 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -53,6 +53,7 @@ - ${TMP_TEST_FOLDER}/gitaly/run2/ - ${TMP_TEST_FOLDER}/gitaly/Makefile - ${TMP_TEST_FOLDER}/gitaly/praefect.config.toml + - ${TMP_TEST_FOLDER}/gitaly/praefect-db.config.toml - ${TMP_TEST_FOLDER}/gitaly/ruby/ policy: pull @@ -114,14 +115,16 @@ policy: push .qa-ruby-gems-cache: &qa-ruby-gems-cache - key: "qa-ruby-gems-${DEBIAN_VERSION}" + key: + files: + - qa/Gemfile.lock paths: - - qa/vendor/ruby/ + - qa/vendor/ruby policy: pull .qa-ruby-gems-cache-push: &qa-ruby-gems-cache-push <<: *qa-ruby-gems-cache - policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + policy: pull-push .setup-test-env-cache: cache: @@ -246,7 +249,7 @@ command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:5.0-alpine - name: elasticsearch:7.17.0 - command: ["elasticsearch", "-E", "discovery.type=single-node"] + command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"] variables: POSTGRES_HOST_AUTH_METHOD: trust PG_VERSION: "11" @@ -257,7 +260,7 @@ command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:6.0-alpine - name: elasticsearch:7.17.0 - command: ["elasticsearch", "-E", "discovery.type=single-node"] + command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"] variables: POSTGRES_HOST_AUTH_METHOD: trust PG_VERSION: "12" @@ -268,11 +271,35 @@ command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:5.0-alpine - name: elasticsearch:7.17.0 - command: ["elasticsearch", "-E", "discovery.type=single-node"] + command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"] variables: POSTGRES_HOST_AUTH_METHOD: trust PG_VERSION: "13" +.use-pg12-es8-ee: + services: + - name: postgres:12 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:6.0-alpine + - name: elasticsearch:8.1.1 + variables: + POSTGRES_HOST_AUTH_METHOD: trust + PG_VERSION: "12" + ES_SETTING_DISCOVERY_TYPE: "single-node" + ES_SETTING_XPACK_SECURITY_ENABLED: "false" + +.use-pg12-opensearch1-ee: + services: + - name: postgres:12 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:6.0-alpine + - name: opensearchproject/opensearch:1.2.4 + alias: elasticsearch + command: ["bin/opensearch", "-E", "discovery.type=single-node", "-E", "plugins.security.disabled=true"] + variables: + POSTGRES_HOST_AUTH_METHOD: trust + PG_VERSION: "12" + .use-kaniko: image: name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:kaniko diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 8881a4c486d..1ebc408e0d4 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -1,4 +1,5 @@ .qa-job-base: + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-chrome-99 extends: - .default-retry - .qa-cache @@ -114,13 +115,13 @@ update-qa-cache: .package-and-qa-ff-base: script: - | - feature_flags=$(scripts/changed-feature-flags --files $(cat $CHANGES_FILE | tr ' ' ',') --state $QA_FF_STATE) + feature_flags=$(scripts/changed-feature-flags --files $CHANGES_DIFFS_DIR --state $QA_FF_STATE) if [[ $feature_flags ]]; then export GITLAB_QA_OPTIONS="--set-feature-flags $feature_flags" echo $GITLAB_QA_OPTIONS ./scripts/trigger-build.rb omnibus else - echo "No changed feature flag found to test. The tests are skipped if the flag was removed." + echo "No changed feature flag found to test as $QA_FF_STATE." fi package-and-qa: @@ -134,7 +135,7 @@ package-and-qa-ff-enabled: - .package-and-qa-ff-base - .qa:rules:package-and-qa:feature-flags variables: - QA_FF_STATE: "enable" + QA_FF_STATE: "enabled" package-and-qa-ff-disabled: extends: @@ -142,4 +143,12 @@ package-and-qa-ff-disabled: - .package-and-qa-ff-base - .qa:rules:package-and-qa:feature-flags variables: - QA_FF_STATE: "disable" + QA_FF_STATE: "disabled" + +package-and-qa-ff-deleted: + extends: + - .package-and-qa-base + - .package-and-qa-ff-base + - .qa:rules:package-and-qa:feature-flags + variables: + QA_FF_STATE: "deleted" diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 24b6c6d2773..77bdfda3eac 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -25,6 +25,10 @@ .single-db-rspec: extends: .single-db +.praefect-with-db: + variables: + GITALY_PRAEFECT_WITH_DB: '1' + .rspec-base: extends: - .rails-job-base @@ -38,7 +42,7 @@ needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] script: - !reference [.base-script, script] - - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration" + - rspec_paralellized_job "--tag ~quarantine --tag ~level:migration" .base-artifacts: artifacts: @@ -61,7 +65,7 @@ - .rails:rules:ee-and-foss-migration script: - !reference [.base-script, script] - - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration" + - rspec_paralellized_job "--tag ~quarantine --tag level:migration" .rspec-base-pg11: extends: @@ -101,41 +105,26 @@ - .rspec-base - .use-pg12-ee -.rspec-jh-base-pg12: - extends: - - .rspec-base-pg12-as-if-jh - - .use-pg12-ee - -.rspec-ee-base-pg13: +.rspec-ee-base-pg12-es8: extends: - .rspec-base - - .use-pg13-ee + - .use-pg12-es8-ee + - .rails:rules:run-search-tests -.rspec-ee-base-geo: - extends: .rspec-base - script: - - !reference [.base-script, script] - - rspec_paralellized_job "--tag ~quarantine --tag geo" - -.rspec-ee-base-geo-pg11: +.rspec-ee-base-pg12-opensearch1: extends: - - .rspec-ee-base-geo - - .use-pg11-ee + - .rspec-base + - .use-pg12-opensearch1-ee + - .rails:rules:run-search-tests -.rspec-ee-base-geo-pg12: +.rspec-jh-base-pg12: extends: - - .rspec-ee-base-geo + - .rspec-base-pg12-as-if-jh - .use-pg12-ee -.rspec-jh-base-geo-pg12: - extends: - - .rspec-jh-base-pg12 - script: - - !reference [.rspec-ee-base-geo, script] - -.rspec-ee-base-geo-pg13: +.rspec-ee-base-pg13: extends: - - .rspec-ee-base-geo + - .rspec-base - .use-pg13-ee .db-job-base: @@ -160,10 +149,7 @@ parallel: 22 .rspec-ee-unit-parallel: - parallel: 14 - -.rspec-ee-unit-geo-parallel: - parallel: 2 + parallel: 16 .rspec-integration-parallel: parallel: 10 @@ -210,6 +196,7 @@ setup-test-env: - ${TMP_TEST_FOLDER}/gitaly/run2/ - ${TMP_TEST_FOLDER}/gitaly/Makefile - ${TMP_TEST_FOLDER}/gitaly/praefect.config.toml + - ${TMP_TEST_FOLDER}/gitaly/praefect-db.config.toml - ${TMP_TEST_FOLDER}/gitaly/ruby/ - ${TMP_TEST_FOLDER}/gitlab-elasticsearch-indexer/bin/gitlab-elasticsearch-indexer - ${TMP_TEST_FOLDER}/gitlab-shell/ @@ -280,6 +267,12 @@ rspec migration pg12 single-db: - .single-db-rspec - .rails:rules:single-db +rspec migration pg12 praefect: + extends: + - rspec migration pg12 + - .praefect-with-db + - .rails:rules:praefect-with-db + rspec unit pg12: extends: - .rspec-base-pg12 @@ -298,6 +291,12 @@ rspec unit pg12 single-db: - .single-db-rspec - .rails:rules:single-db +rspec unit pg12 praefect: + extends: + - rspec unit pg12 + - .praefect-with-db + - .rails:rules:praefect-with-db + rspec integration pg12: extends: - .rspec-base-pg12 @@ -316,6 +315,12 @@ rspec integration pg12 single-db: - .single-db-rspec - .rails:rules:single-db +rspec integration pg12 praefect: + extends: + - rspec integration pg12 + - .praefect-with-db + - .rails:rules:praefect-with-db + rspec system pg12: extends: - .rspec-base-pg12 @@ -336,6 +341,12 @@ rspec system pg12 single-db: - .single-db-rspec - .rails:rules:single-db +rspec system pg12 praefect: + extends: + - rspec system pg12 + - .praefect-with-db + - .rails:rules:praefect-with-db + # Dedicated job to test DB library code against PG11. # Note that these are already tested against PG12 in the `rspec unit pg12` / `rspec-ee unit pg12` jobs. rspec db-library-code pg11: @@ -510,9 +521,6 @@ rspec:deprecations: - rspec-ee unit pg12 - rspec-ee integration pg12 - rspec-ee system pg12 - - rspec-ee unit pg12 geo - - rspec-ee integration pg12 geo - - rspec-ee system pg12 geo variables: SETUP_DB: "false" script: @@ -564,14 +572,6 @@ rspec:coverage: - rspec-ee unit pg12 single-db - rspec-ee integration pg12 single-db - rspec-ee system pg12 single-db - # Geo jobs - - rspec-ee unit pg12 geo - - rspec-ee integration pg12 geo - - rspec-ee system pg12 geo - # Geo minimal jobs - - rspec-ee unit pg12 geo minimal - - rspec-ee integration pg12 geo minimal - - rspec-ee system pg12 geo minimal # Memory jobs - memory-on-boot # As-if-FOSS jobs @@ -788,6 +788,16 @@ rspec-ee unit pg12: - .rails:rules:ee-only-unit - .rspec-ee-unit-parallel +rspec-ee unit pg12 es8: + extends: + - .rspec-ee-base-pg12-es8 + - .rspec-ee-unit-parallel + +rspec-ee unit pg12 opensearch1: + extends: + - .rspec-ee-base-pg12-opensearch1 + - .rspec-ee-unit-parallel + rspec-ee unit pg12 minimal: extends: - rspec-ee unit pg12 @@ -806,6 +816,16 @@ rspec-ee integration pg12: - .rails:rules:ee-only-integration - .rspec-ee-integration-parallel +rspec-ee integration pg12 es8: + extends: + - .rspec-ee-base-pg12-es8 + - .rspec-ee-integration-parallel + +rspec-ee integration pg12 opensearch1: + extends: + - .rspec-ee-base-pg12-opensearch1 + - .rspec-ee-integration-parallel + rspec-ee integration pg12 minimal: extends: - rspec-ee integration pg12 @@ -824,6 +844,16 @@ rspec-ee system pg12: - .rails:rules:ee-only-system - .rspec-ee-system-parallel +rspec-ee system pg12 es8: + extends: + - .rspec-ee-base-pg12-es8 + - .rspec-ee-system-parallel + +rspec-ee system pg12 opensearch1: + extends: + - .rspec-ee-base-pg12-opensearch1 + - .rspec-ee-system-parallel + rspec-ee system pg12 minimal: extends: - rspec-ee system pg12 @@ -836,40 +866,6 @@ rspec-ee system pg12 single-db: - .single-db-rspec - .rails:rules:single-db -rspec-ee unit pg12 geo: - extends: - - .rspec-ee-base-geo-pg12 - - .rails:rules:ee-only-unit - - .rspec-ee-unit-geo-parallel - -rspec-ee unit pg12 geo minimal: - extends: - - rspec-ee unit pg12 geo - - .minimal-rspec-tests - - .rails:rules:ee-only-unit:minimal - -rspec-ee integration pg12 geo: - extends: - - .rspec-ee-base-geo-pg12 - - .rails:rules:ee-only-integration - -rspec-ee integration pg12 geo minimal: - extends: - - rspec-ee integration pg12 geo - - .minimal-rspec-tests - - .rails:rules:ee-only-integration:minimal - -rspec-ee system pg12 geo: - extends: - - .rspec-ee-base-geo-pg12 - - .rails:rules:ee-only-system - -rspec-ee system pg12 geo minimal: - extends: - - rspec-ee system pg12 geo - - .minimal-rspec-tests - - .rails:rules:ee-only-system:minimal - rspec-ee migration pg12-as-if-jh: extends: - .rspec-jh-base-pg12 @@ -895,22 +891,6 @@ rspec-ee system pg12-as-if-jh: - .rails:rules:as-if-jh-rspec - .rspec-ee-system-parallel -rspec-ee unit pg12-as-if-jh geo: - extends: - - .rspec-jh-base-geo-pg12 - - .rails:rules:as-if-jh-rspec - - .rspec-ee-unit-geo-parallel - -rspec-ee integration pg12-as-if-jh geo: - extends: - - .rspec-jh-base-geo-pg12 - - .rails:rules:as-if-jh-rspec - -rspec-ee system pg12-as-if-jh geo: - extends: - - .rspec-jh-base-geo-pg12 - - .rails:rules:as-if-jh-rspec - rspec-jh migration pg12-as-if-jh: extends: - .rspec-jh-base-pg12 @@ -932,21 +912,6 @@ rspec-jh system pg12-as-if-jh: - .rspec-jh-base-pg12 - .rails:rules:as-if-jh-rspec -rspec-jh unit pg12-as-if-jh geo: - extends: - - .rspec-jh-base-geo-pg12 - - .rails:rules:as-if-jh-rspec - -rspec-jh integration pg12-as-if-jh geo: - extends: - - .rspec-jh-base-geo-pg12 - - .rails:rules:as-if-jh-rspec - -rspec-jh system pg12-as-if-jh geo: - extends: - - .rspec-jh-base-geo-pg12 - - .rails:rules:as-if-jh-rspec - db:rollback geo: extends: - db:rollback @@ -1044,22 +1009,6 @@ rspec-ee system pg11: - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-system-parallel -rspec-ee unit pg11 geo: - extends: - - .rspec-ee-base-geo-pg11 - - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - - .rspec-ee-unit-geo-parallel - -rspec-ee integration pg11 geo: - extends: - - .rspec-ee-base-geo-pg11 - - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - -rspec-ee system pg11 geo: - extends: - - .rspec-ee-base-geo-pg11 - - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - # PG13 rspec-ee migration pg13: extends: @@ -1085,22 +1034,6 @@ rspec-ee system pg13: - .rspec-ee-base-pg13 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-system-parallel - -rspec-ee unit pg13 geo: - extends: - - .rspec-ee-base-geo-pg13 - - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - - .rspec-ee-unit-geo-parallel - -rspec-ee integration pg13 geo: - extends: - - .rspec-ee-base-geo-pg13 - - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - -rspec-ee system pg13 geo: - extends: - - .rspec-ee-base-geo-pg13 - - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only # EE: default branch nightly scheduled jobs # ##################################### diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 3628013fc9b..107f37ed47d 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -2,8 +2,8 @@ include: - template: Jobs/Code-Quality.gitlab-ci.yml - template: Jobs/SAST.gitlab-ci.yml - template: Jobs/Secret-Detection.gitlab-ci.yml - - template: Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/License-Scanning.gitlab-ci.yml + - template: Jobs/Dependency-Scanning.gitlab-ci.yml + - template: Jobs/License-Scanning.gitlab-ci.yml code_quality: extends: @@ -82,9 +82,8 @@ secret_detection: expire_in: 1 week # GitLab-specific gemnasium-dependency_scanning: - before_script: - # git-lfs is needed for auto-remediation - - apk add git-lfs + variables: + DS_REMEDIATE: "false" rules: !reference [".reports:rules:gemnasium-dependency_scanning", rules] gemnasium-python-dependency_scanning: @@ -103,7 +102,7 @@ yarn-audit-dependency_scanning: extends: .default-retry stage: test image: - name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:1.1.0 + name: registry.gitlab.com/gitlab-org/security-products/package-hunter-cli:v1.3.2@sha256:7529deaef9ea21aab56bfb74ae1abbc121311affdb6ece49ce7b1c360f997ca2 entrypoint: [""] variables: HTR_user: '$PACKAGE_HUNTER_USER' diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index b528a2c7427..dde08b15bc3 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -111,7 +111,6 @@ review-deploy: artifacts: paths: - environment_url.txt - - curl_output.txt expire_in: 7 days when: always diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml index d2192a7511a..47e756eb230 100644 --- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml @@ -1,21 +1,19 @@ include: - project: gitlab-org/quality/pipeline-common - ref: 0.3.6 + ref: 0.6.0 file: - /ci/allure-report.yml - /ci/knapsack-report.yml -.review-qa-base: - extends: - - .use-docker-in-docker - image: - name: ${QA_IMAGE} - entrypoint: [""] - stage: qa - needs: ["review-deploy"] +.bundler_variables: + variables: + BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES: "true" + BUNDLE_SILENCE_ROOT_WARNING: "true" + BUNDLE_PATH: vendor + +.test_variables: variables: QA_DEBUG: "true" - QA_CAN_TEST_GIT_PROTOCOL_V2: "false" QA_GENERATE_ALLURE_REPORT: "true" GITLAB_USERNAME: "root" GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" @@ -23,19 +21,40 @@ include: GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITLAB_QA_ADMIN_ACCESS_TOKEN: "${REVIEW_APPS_ROOT_TOKEN}" GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" - SIGNUP_DISABLED: "true" + +.review-qa-base: + extends: + - .use-docker-in-docker + - .qa-cache + - .test_variables + - .bundler_variables + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-lfs-2.9-chrome-99-docker-20.10.14-gcloud-383-kubectl-1.23 + stage: qa + needs: + - review-deploy + - download-knapsack-report + variables: + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: /certs + DOCKER_CERT_PATH: /certs/client + DOCKER_TLS_VERIFY: 1 before_script: - # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync. - export EE_LICENSE="$(cat $REVIEW_APPS_EE_LICENSE_FILE)" - - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then - git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; - fi - - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - - echo "${CI_ENVIRONMENT_URL}" - - cd qa + - export QA_GITLAB_URL="$(cat environment_url.txt)" + - cd qa && bundle install script: - qa_run_status=0 - - bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" -- --color --format documentation --format RspecJunitFormatter --out tmp/rspec.xml || qa_run_status=$? + - | + bundle exec rake "knapsack:rspec[\ + ${RSPEC_TAGS} \ + --tag ~orchestrated \ + --tag ~transient \ + --tag ~skip_signup_disabled \ + --force-color \ + --order random \ + --format documentation \ + --format RspecJunitFormatter --out tmp/rspec.xml \ + ]" || qa_run_status=$? - if [ ${qa_run_status} -ne 0 ]; then release_sha=$(echo "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-${CI_COMMIT_SHA}}" | cut -c1-11); echo "Errors can be found at https://sentry.gitlab.net/gitlab/gitlab-review-apps/releases/${release_sha}/all-events/."; @@ -58,25 +77,41 @@ include: ALLURE_MERGE_REQUEST_IID: $CI_MERGE_REQUEST_IID ALLURE_RESULTS_GLOB: qa/tmp/allure-results/* +# Store knapsack report as artifact so the same report is reused across all jobs +download-knapsack-report: + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-chrome-99 + extends: + - .qa-cache + - .bundler_variables + - .review:rules:review-qa-reliable + stage: prepare + before_script: + - cd qa && bundle install + script: + - QA_KNAPSACK_REPORT_NAME=review-qa-reliable bundle exec rake "knapsack:download" + - QA_KNAPSACK_REPORT_NAME=review-qa-all bundle exec rake "knapsack:download" + allow_failure: true + artifacts: + paths: + - qa/knapsack/review-qa-*.json + expire_in: 1 day + review-qa-smoke: extends: - .review-qa-base - .review:rules:review-qa-smoke - retry: 1 # This is confusing but this means "2 runs at max". variables: QA_RUN_TYPE: review-qa-smoke - QA_SCENARIO: Test::Instance::Smoke - + RSPEC_TAGS: --tag smoke review-qa-reliable: extends: - .review-qa-base - .review:rules:review-qa-reliable - parallel: 8 - retry: 1 + parallel: 10 variables: QA_RUN_TYPE: review-qa-reliable - QA_SCENARIO: Test::Instance::Reliable + RSPEC_TAGS: --tag reliable review-qa-all: extends: @@ -85,8 +120,7 @@ review-qa-all: parallel: 5 variables: QA_RUN_TYPE: review-qa-all - QA_SCENARIO: Test::Instance::All - QA_SKIP_SMOKE_RELIABLE: "true" + RSPEC_TAGS: --tag ~reliable --tag ~smoke review-performance: extends: @@ -136,9 +170,11 @@ allure-report-qa-all: variables: ALLURE_JOB_NAME: review-qa-all -knapsack-report: +upload-knapsack-report: extends: - .generate-knapsack-report-base stage: post-qa variables: - QA_KNAPSACK_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/tmp/knapsack/*/*.json + # knapsack report upload uses gitlab-qa image with code already there + GIT_STRATEGY: none + QA_KNAPSACK_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/knapsack/*/*.json diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 142341e5741..37593ffd2fc 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -76,6 +76,9 @@ .if-merge-request-labels-jh-contribution: &if-merge-request-labels-jh-contribution if: '$CI_MERGE_REQUEST_LABELS =~ /JiHu contribution/' +.if-merge-request-labels-group-global-search: &if-merge-request-labels-group-global-search + if: '$CI_MERGE_REQUEST_LABELS =~ /group::global search/' + .if-security-merge-request: &if-security-merge-request if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID' @@ -247,6 +250,9 @@ .models-patterns: &models-patterns - "{,ee/,jh/}{app/models}/**/*" +.lib-gitlab-patterns: &lib-gitlab-patterns + - "{,ee/,jh/}lib/{,ee/,jh/}gitlab/**/*" + .startup-css-patterns: &startup-css-patterns - "{,ee/,jh/}app/assets/stylesheets/startup/**/*" @@ -257,7 +263,7 @@ - "config.ru" # List explicitly all the app/ dirs that are backend (i.e. all except app/assets). - "{,ee/,jh/}{app/channels,app/controllers,app/finders,app/graphql,app/helpers,app/mailers,app/models,app/policies,app/presenters,app/serializers,app/services,app/uploaders,app/validators,app/views,app/workers}/**/*" - - "{,ee/,jh/}{bin,cable,config,db,generator_templates,lib}/**/*" + - "{,ee/,jh/}{bin,config,db,generator_templates,lib}/**/*" - "{,ee/,jh/}spec/**/*" # CI changes - ".gitlab-ci.yml" @@ -267,6 +273,14 @@ # Mapped patterns (see tests.yml) - "data/whats_new/*.yml" +.search-backend-patterns: &search-backend-patterns + - "{,jh/}Gemfile.lock" + - "GITLAB_ELASTICSEARCH_INDEXER_VERSION" + # List explicitly all the app/ dirs that are backend (i.e. all except app/assets). + - "{,ee/,jh/}{app/channels,app/controllers,app/finders,app/graphql,app/helpers,app/mailers,app/models,app/policies,app/presenters,app/serializers,app/services,app/uploaders,app/validators,app/views,app/workers}/**/*" + - "{,ee/,jh/}{bin,config,db,generator_templates,lib}/**/*" + - "{,ee/,jh/}spec/**/*" + # DB patterns + .ci-patterns .db-patterns: &db-patterns - "{,ee/,jh/}{,spec/}{db,migrations}/**/*" @@ -513,6 +527,10 @@ - <<: *if-security-merge-request when: never +.rails:rules:run-search-tests: + rules: + - <<: *if-merge-request-labels-group-global-search + changes: *search-backend-patterns .rails:rules:ee-and-foss-default-rules: rules: @@ -604,6 +622,7 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-targeting-stable-branch - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-and-security-merge-request changes: *ci-build-images-patterns @@ -618,6 +637,7 @@ rules: - <<: *if-not-canonical-namespace when: never + - <<: *if-merge-request-targeting-stable-branch - <<: *if-merge-request-labels-run-review-app - <<: *if-auto-deploy-branches - changes: *ci-build-images-patterns @@ -692,6 +712,7 @@ rules: - <<: *if-not-canonical-namespace when: never + - <<: *if-merge-request-targeting-stable-branch - <<: *if-merge-request-labels-run-review-app - <<: *if-auto-deploy-branches - changes: *code-qa-patterns @@ -879,9 +900,8 @@ rules: - <<: *if-not-ee when: never - - <<: *if-dot-com-gitlab-org-and-security-merge-request - changes: *feature-flag-development-config-patterns - when: never + - <<: *if-merge-request-targeting-stable-branch + allow_failure: true - <<: *if-dot-com-gitlab-org-and-security-merge-request changes: *nodejs-patterns allow_failure: true @@ -938,6 +958,11 @@ changes: *db-patterns - <<: *if-default-branch-schedule-nightly +.rails:rules:praefect-with-db: + rules: + - if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:run-praefect-with-db/' + allow_failure: true + .rails:rules:ee-and-foss-migration: rules: - <<: *if-fork-merge-request @@ -1470,24 +1495,32 @@ rules: - if: '$SECRET_DETECTION_DISABLED' when: never + # Scan each commit on master to feed the Vulnerability Reports with detected secrets + - <<: *if-default-branch-refs - changes: *code-backstage-qa-patterns .reports:rules:gemnasium-dependency_scanning: rules: - - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ || $DS_DEFAULT_ANALYZERS !~ /gemnasium([^-]|$)/' + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/' when: never + # Run Dependency Scanning on master until https://gitlab.com/gitlab-org/gitlab/-/issues/361657 is resolved + - <<: *if-default-branch-refs - changes: *dependency-patterns .reports:rules:gemnasium-python-dependency_scanning: rules: - - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ || $DS_DEFAULT_ANALYZERS !~ /gemnasium-python/' + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/' when: never + # Run Dependency Scanning on master until https://gitlab.com/gitlab-org/gitlab/-/issues/361657 is resolved + - <<: *if-default-branch-refs - changes: *python-patterns .reports:rules:yarn-audit-dependency_scanning: rules: - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/' when: never + # Run Dependency Scanning on master until https://gitlab.com/gitlab-org/gitlab/-/issues/361657 is resolved + - <<: *if-default-branch-refs - changes: *nodejs-patterns .reports:rules:schedule-dast: @@ -1535,6 +1568,8 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *models-patterns - <<: *if-dot-com-gitlab-org-merge-request + changes: *lib-gitlab-patterns + - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns diff --git a/.gitlab/ci/static-analysis.gitlab-ci.yml b/.gitlab/ci/static-analysis.gitlab-ci.yml index f5f0dcfe7f8..e1257e778bd 100644 --- a/.gitlab/ci/static-analysis.gitlab-ci.yml +++ b/.gitlab/ci/static-analysis.gitlab-ci.yml @@ -7,6 +7,7 @@ variables: SETUP_DB: "false" ENABLE_SPRING: "1" + SKIP_LOG_INITIALIZER_CONNECTIONS: "1" # Disable warnings in browserslist which can break on backports # https://github.com/browserslist/browserslist/blob/a287ec6/node.js#L367-L384 BROWSERSLIST_IGNORE_OLD_DATA: "true" @@ -75,7 +76,7 @@ eslint: USE_BUNDLE_INSTALL: "false" script: - run_timed_command "retry yarn install --frozen-lockfile" - - run_timed_command "yarn run lint:eslint:all --parser-options=schema:${GRAPHQL_SCHEMA_APOLLO_FILE}" + - run_timed_command "yarn run lint:eslint:all" eslint as-if-foss: extends: @@ -111,7 +112,7 @@ rubocop: script: - run_timed_command "bundle exec rubocop --parallel" -qa:testcases: +qa:metadata-lint: extends: - .static-analysis-base - .static-analysis:rules:ee-and-foss-qa @@ -123,6 +124,7 @@ qa:testcases: - run_timed_command "bundle exec bin/qa Test::Instance::All http://localhost:3000 --test-metadata-only" - cd .. - run_timed_command "./scripts/qa/testcases-check qa/tmp/test-metadata.json" + - run_timed_command "./scripts/qa/quarantine-types-check qa/tmp/test-metadata.json" variables: USE_BUNDLE_INSTALL: "false" SETUP_DB: "false" diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 20cbd759ac6..79fea15690c 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -38,9 +38,6 @@ update-tests-metadata: - rspec-ee unit pg12 - rspec-ee integration pg12 - rspec-ee system pg12 - - rspec-ee unit pg12 geo - - rspec-ee integration pg12 geo - - rspec-ee system pg12 geo script: - run_timed_command "retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document" - source ./scripts/rspec_helpers.sh |