diff options
Diffstat (limited to '.gitlab/ci')
-rw-r--r-- | .gitlab/ci/build-images.gitlab-ci.yml | 10 | ||||
-rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 6 | ||||
-rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 17 | ||||
-rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 38 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 11 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 4 | ||||
-rw-r--r-- | .gitlab/ci/review-apps/dast.gitlab-ci.yml | 118 | ||||
-rw-r--r-- | .gitlab/ci/review-apps/main.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/review-apps/qa.gitlab-ci.yml | 41 | ||||
-rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 5 | ||||
-rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 69 | ||||
-rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 7 | ||||
-rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/workhorse.gitlab-ci.yml | 8 | ||||
-rw-r--r-- | .gitlab/ci/yaml.gitlab-ci.yml | 21 |
16 files changed, 199 insertions, 162 deletions
diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index 6a222d8937f..46d0bb2fb8f 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -29,7 +29,15 @@ build-qa-image: - !reference [.base-image-build, script] - echo $QA_IMAGE - echo $QA_IMAGE_BRANCH - - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --destination=${QA_IMAGE_BRANCH} --cache=true + - | + /kaniko/executor \ + --context=${CI_PROJECT_DIR} \ + --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile \ + --destination=${QA_IMAGE} \ + --destination=${QA_IMAGE_BRANCH} \ + --build-arg=CHROME_VERSION=${CHROME_VERSION} \ + --build-arg=DOCKER_VERSION=${DOCKER_VERSION} \ + --cache=true # This image is used by: # - The `CNG` pipelines (via the `review-build-cng` job): https://gitlab.com/gitlab-org/build/CNG/-/blob/cfc67136d711e1c8c409bf8e57427a644393da2f/.gitlab-ci.yml#L335 diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 217da6506bf..3af156e9bd0 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -2,7 +2,7 @@ extends: - .default-retry - .docs:rules:review-docs - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine + image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine stage: review needs: [] variables: @@ -44,7 +44,7 @@ docs-lint markdown: - .default-retry - .docs:rules:docs-lint # When updating the image version here, update it in /scripts/lint-doc.sh too. - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.15-vale-2.15.5-markdownlint-0.31.1 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-docs/lint-markdown:alpine-3.16-vale-2.17.0-markdownlint-0.31.1 stage: lint needs: [] script: @@ -53,7 +53,7 @@ docs-lint markdown: docs-lint links: extends: - .docs:rules:docs-lint - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.15-ruby-2.7.5-cee62c13 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-docs/lint-html:alpine-3.16-ruby-2.7.6-0bc327a4 stage: lint needs: [] script: diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 4b1194d0fbd..8bfda0e6684 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -11,7 +11,7 @@ - .default-retry - .default-before_script - .assets-compile-cache - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7-git-2.33-lfs-2.9-node-16.14-yarn-1.22-graphicsmagick-1.3.36 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-git-2.33-lfs-2.9-node-16.14-yarn-1.22-graphicsmagick-1.3.36 variables: SETUP_DB: "false" WEBPACK_VENDOR_DLL: "true" diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 7e06a4a71bd..344a31b28d8 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -18,7 +18,7 @@ - source scripts/prepare_build.sh .ruby-gems-cache: &ruby-gems-cache - key: "ruby-gems-${DEBIAN_VERSION}" + key: "ruby-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}" paths: - vendor/ruby/ policy: pull @@ -28,7 +28,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .gitaly-ruby-gems-cache: &gitaly-ruby-gems-cache - key: "gitaly-ruby-gems-${DEBIAN_VERSION}" + key: "gitaly-ruby-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}" paths: - vendor/gitaly-ruby/ policy: pull @@ -42,7 +42,7 @@ files: - GITALY_SERVER_VERSION - lib/gitlab/setup_helper.rb - prefix: "gitaly-binaries-${DEBIAN-VERSION}" + prefix: "gitaly-binaries-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}" paths: - ${TMP_TEST_FOLDER}/gitaly/_build/bin/ - ${TMP_TEST_FOLDER}/gitaly/_build/deps/git/install/ @@ -79,7 +79,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .assets-cache: &assets-cache - key: "assets-${DEBIAN_VERSION}-${NODE_ENV}" + key: "assets-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}" paths: - assets-hash.txt - public/assets/webpack/ @@ -103,7 +103,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .rubocop-cache: &rubocop-cache - key: "rubocop-${DEBIAN_VERSION}" + key: "rubocop-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}" paths: - tmp/rubocop_cache/ policy: pull @@ -116,6 +116,7 @@ .qa-ruby-gems-cache: &qa-ruby-gems-cache key: + prefix: "qa-ruby-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}" files: - qa/Gemfile.lock paths: @@ -238,7 +239,7 @@ services: - name: postgres:13 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:5.0-alpine + - name: redis:6.2-alpine variables: POSTGRES_HOST_AUTH_METHOD: trust PG_VERSION: "13" @@ -269,7 +270,7 @@ services: - name: postgres:13 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - - name: redis:5.0-alpine + - name: redis:6.2-alpine - name: elasticsearch:7.17.0 command: ["elasticsearch", "-E", "discovery.type=single-node", "-E", "xpack.security.enabled=false"] variables: @@ -281,7 +282,7 @@ - name: postgres:12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:6.0-alpine - - name: elasticsearch:8.1.1 + - name: elasticsearch:8.2.0 variables: POSTGRES_HOST_AUTH_METHOD: trust PG_VERSION: "12" diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 1ebc408e0d4..5ca70da352a 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -1,5 +1,5 @@ .qa-job-base: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-chrome-99 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-chrome-${CHROME_VERSION} extends: - .default-retry - .qa-cache @@ -12,7 +12,7 @@ before_script: - !reference [.default-before_script, before_script] - cd qa/ - - bundle_install_script + - bundle install qa:internal: extends: @@ -52,7 +52,6 @@ qa:nightly-auto-quarantine-dequarantine: - bundle exec confiner -r .confiner/nightly.yml allow_failure: true - qa:selectors-as-if-foss: extends: - qa:selectors @@ -68,8 +67,32 @@ update-qa-cache: script: - echo "Cache has been updated and ready to be uploaded." -.package-and-qa-base: +populate-qa-tests-var: + extends: + - .qa:rules:determine-qa-tests image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine + stage: prepare + script: + - tooling/bin/qa/check_if_qa_only_spec_changes ${CHANGES_FILE} ${ONLY_QA_CHANGES_FILE} + - '[ -f $ONLY_QA_CHANGES_FILE ] && export QA_TESTS="`cat $ONLY_QA_CHANGES_FILE`"' + - 'echo "QA_TESTS=$QA_TESTS" >> qa_tests_var.env' + - 'echo "QA_TESTS: $QA_TESTS"' + artifacts: + expire_in: 2d + reports: + dotenv: qa_tests_var.env + paths: + - ${CHANGES_FILE} + - ${ONLY_QA_CHANGES_FILE} + - qa_tests_var.env + variables: + CHANGES_FILE: tmp/changed_files.txt + ONLY_QA_CHANGES_FILE: tmp/qa_only_changed_files.txt + needs: + - detect-tests + +.package-and-qa-base: + image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine stage: qa retry: 0 before_script: @@ -77,8 +100,6 @@ update-qa-cache: - install_gitlab_gem - tooling/bin/find_change_diffs ${CHANGES_DIFFS_DIR} script: - - tooling/bin/qa/check_if_qa_only_spec_changes ${CHANGES_FILE} ${ONLY_QA_CHANGES_FILE} - - '[ -f $ONLY_QA_CHANGES_FILE ] && export QA_TESTS="`cat $ONLY_QA_CHANGES_FILE`"' - 'echo "QA_TESTS: $QA_TESTS"' - exit_code=0 && tooling/bin/qa/package_and_qa_check ${CHANGES_DIFFS_DIR} || exit_code=$? - echo $exit_code @@ -99,16 +120,13 @@ update-qa-cache: artifacts: false - job: build-assets-image artifacts: false + - job: populate-qa-tests-var - detect-tests artifacts: expire_in: 7d paths: - - ${CHANGES_FILE} - - ${ONLY_QA_CHANGES_FILE} - ${CHANGES_DIFFS_DIR}/* variables: - CHANGES_FILE: tmp/changed_files.txt - ONLY_QA_CHANGES_FILE: tmp/qa_only_changed_files.txt CHANGES_DIFFS_DIR: tmp/diffs ALLURE_JOB_NAME: $CI_JOB_NAME diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 77bdfda3eac..0358fe8ec49 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -395,15 +395,15 @@ db:migrate-from-previous-major-version: USE_BUNDLE_INSTALL: "false" SETUP_DB: "false" PROJECT_TO_CHECKOUT: "gitlab-foss" - TAG_TO_CHECKOUT: "v13.12.9" + TAG_TO_CHECKOUT: "v14.10.2" before_script: - !reference [.default-before_script, before_script] - '[[ -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"' - '[[ -d "ee/" ]] || export TAG_TO_CHECKOUT="${TAG_TO_CHECKOUT}-ee"' - retry 'git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT' - git checkout -f FETCH_HEAD - - SETUP_DB=false USE_BUNDLE_INSTALL=true bash scripts/prepare_build.sh - - run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate db:seed_fu" + - SETUP_DB=false USE_BUNDLE_INSTALL=true ENABLE_BOOTSNAP=false bash scripts/prepare_build.sh + - run_timed_command "ENABLE_BOOTSNAP=false bundle exec rake db:drop db:create db:structure:load db:migrate db:seed_fu" - git checkout -f $CI_COMMIT_SHA - SETUP_DB=false USE_BUNDLE_INSTALL=true bash scripts/prepare_build.sh script: @@ -419,7 +419,7 @@ db:migrate-from-previous-major-version-single-db: extends: - .rails:rules:ee-mr-and-default-branch-only variables: - TAG_TO_CHECKOUT: "v14.4.0" + TAG_TO_CHECKOUT: "v14.7.0" # this version updated grpc to 1.42.0, which supports Ruby 2 & 3 script: - run_timed_command "scripts/db_tasks db:migrate" - scripts/schema_changed.sh @@ -460,7 +460,7 @@ db:migrate-non-superuser: db:gitlabcom-database-testing: extends: .rails:rules:db:gitlabcom-database-testing stage: test - image: ruby:2.7-alpine + image: ruby:${RUBY_VERSION}-alpine needs: [] allow_failure: true script: @@ -976,7 +976,6 @@ rspec system pg13: - .rspec-base-pg13 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-system-parallel - # EE/FOSS: default branch nightly scheduled jobs # ########################################## diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 107f37ed47d..68c71b359c2 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -91,7 +91,7 @@ gemnasium-python-dependency_scanning: yarn-audit-dependency_scanning: extends: .ds-analyzer - image: "registry.gitlab.com/gitlab-org/security-products/analyzers/npm-audit:1.4.1" + image: "${REGISTRY_HOST}/${REGISTRY_GROUP}/security-products/analyzers/npm-audit:1" variables: TOOL: yarn rules: !reference [".reports:rules:yarn-audit-dependency_scanning", rules] @@ -102,7 +102,7 @@ yarn-audit-dependency_scanning: extends: .default-retry stage: test image: - name: registry.gitlab.com/gitlab-org/security-products/package-hunter-cli:v1.3.2@sha256:7529deaef9ea21aab56bfb74ae1abbc121311affdb6ece49ce7b1c360f997ca2 + name: ${REGISTRY_HOST}/${REGISTRY_GROUP}/security-products/package-hunter-cli:v1.3.2@sha256:7529deaef9ea21aab56bfb74ae1abbc121311affdb6ece49ce7b1c360f997ca2 entrypoint: [""] variables: HTR_user: '$PACKAGE_HUNTER_USER' diff --git a/.gitlab/ci/review-apps/dast.gitlab-ci.yml b/.gitlab/ci/review-apps/dast.gitlab-ci.yml index df8ad4c517a..8f0c6b60190 100644 --- a/.gitlab/ci/review-apps/dast.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/dast.gitlab-ci.yml @@ -5,12 +5,12 @@ extends: - .reports:rules:schedule-dast image: - name: "registry.gitlab.com/security-products/dast:$DAST_VERSION" + name: "${REGISTRY_HOST}/security-products/dast:$DAST_VERSION" resource_group: dast_scan variables: DAST_USERNAME_FIELD: "user[login]" DAST_PASSWORD_FIELD: "user[password]" - DAST_SUBMIT_FIELD: "commit" + DAST_SUBMIT_FIELD: "name:button" DAST_FULL_SCAN_ENABLED: "true" DAST_VERSION: 2 GIT_STRATEGY: none @@ -28,7 +28,7 @@ needs: ["review-deploy"] stage: dast # Default job timeout set to 90m and dast rules needs 2h to so that it won't timeout. - timeout: 2h + timeout: 3h # Add retry because of intermittent connection problems. See https://gitlab.com/gitlab-org/gitlab/-/issues/244313 retry: 1 artifacts: @@ -42,149 +42,65 @@ # DAST scan with a subset of Release scan rules. # ZAP rule details can be found at https://www.zaproxy.org/docs/alerts/ -# 10019, 10021 Missing security headers -# 10023, 10024, 10025, 10037 Information Disclosure -# 10040 Secure Pages Include Mixed Content -# 10056 X-Debug-Token Information Leak -# Duration: 14 minutes 20 seconds - -dast:secureHeaders-csp-infoLeak: +dast:anti-clickjacking-header: extends: - .dast_conf variables: DAST_USERNAME: "user1" - DAST_ONLY_INCLUDE_RULES: "10019,10021,10023,10024,10025,10037,10040,10056" + DAST_ONLY_INCLUDE_RULES: "10020" script: - /analyze -# 90023 XML External Entity Attack -# Duration: 41 minutes 20 seconds -# 90019 Server Side Code Injection -# Duration: 34 minutes 31 seconds -dast:XXE-SrvSideInj: +dast:xss-persistant: extends: - .dast_conf variables: DAST_USERNAME: "user2" - DAST_ONLY_INCLUDE_RULES: "90023,90019" + DAST_ONLY_INCLUDE_RULES: "40014" script: - /analyze -# 0 Directory Browsing -# 2 Private IP Disclosure -# 3 Session ID in URL Rewrite -# 7 Remote File Inclusion -# Duration: 63 minutes 43 seconds -# 90034 Cloud Metadata Potentially Exposed -# Duration: 13 minutes 48 seconds -# 90022 Application Error Disclosure -# Duration: 12 minutes 7 seconds -dast:infoLeak-fileInc-DirBrowsing: +dast:insecure-http-method: extends: - .dast_conf variables: DAST_USERNAME: "user3" - DAST_ONLY_INCLUDE_RULES: "0,2,3,7,90034,90022" + DAST_ONLY_INCLUDE_RULES: "90028" script: - /analyze -# 10010 Cookie No HttpOnly Flag -# 10011 Cookie Without Secure Flag -# 10017 Cross-Domain JavaScript Source File Inclusion -# 10029 Cookie Poisoning -# 90033 Loosely Scoped Cookie -# 10054 Cookie Without SameSite Attribute -# Duration: 13 minutes 23 seconds -dast:insecureCookie: +dast:server-side-template-inj: extends: - .dast_conf variables: DAST_USERNAME: "user4" - DAST_ONLY_INCLUDE_RULES: "10010,10011,10017,10029,90033,10054" + DAST_ONLY_INCLUDE_RULES: "90035" script: - /analyze - -# 20012 Anti-CSRF Tokens Check -# 10202 Absence of Anti-CSRF Tokens -# https://gitlab.com/gitlab-com/gl-security/appsec/appsec-team/-/issues/192 - -# Commented because of lot of FP's -# dast:csrfTokenCheck: -# extends: -# - .dast_conf -# variables: -# DAST_USERNAME: "user6" -# DAST_ONLY_INCLUDE_RULES: "20012,10202" -# script: -# - /analyze - -# 10098 Cross-Domain Misconfiguration -# 10105 Weak Authentication Method -# 40003 CRLF Injection -# 40008 Parameter Tampering -# Duration: 71 minutes 15 seconds -dast:corsMisconfig-weakauth-crlfInj: +dast:server-side-template-inj-blind: extends: - .dast_conf variables: DAST_USERNAME: "user5" - DAST_ONLY_INCLUDE_RULES: "10098,10105,40003,40008" + DAST_ONLY_INCLUDE_RULES: "90035" script: - /analyze -# 20019 External Redirect -# 20014 HTTP Parameter Pollution -# Duration: 46 minutes 12 seconds -dast:extRedirect-paramPollution: +dast:session-fixation: extends: - .dast_conf variables: DAST_USERNAME: "user6" - DAST_ONLY_INCLUDE_RULES: "20019,20014" - script: - - /analyze - -# 40022 SQL Injection - PostgreSQL -# Duration: 53 minutes 59 seconds -dast:sqlInjection: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user7" - DAST_ONLY_INCLUDE_RULES: "40022" - script: - - /analyze - -# 40014 Cross Site Scripting (Persistent) -# Duration: 21 minutes 50 seconds -dast:xss-persistent: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user8" - DAST_ONLY_INCLUDE_RULES: "40014" - script: - - /analyze - -# 40012 Cross Site Scripting (Reflected) -# Duration: 73 minutes 15 seconds -dast:xss-reflected: - extends: - - .dast_conf - variables: - DAST_USERNAME: "user9" - DAST_ONLY_INCLUDE_RULES: "40012" + DAST_ONLY_INCLUDE_RULES: "40013" script: - /analyze -# 40013 Session Fixation -# Duration: 44 minutes 25 seconds -dast:sessionFixation: +dast:xss-dombased: extends: - .dast_conf variables: DAST_USERNAME: "user10" - DAST_ONLY_INCLUDE_RULES: "40013" + DAST_ONLY_INCLUDE_RULES: "40026" script: - /analyze diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index 22fdce71243..f3cde5d7318 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -20,7 +20,7 @@ review-build-cng-env: extends: - .default-retry - .review:rules:review-build-cng - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 + image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine3.13 stage: prepare needs: [] before_script: diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml index 47e756eb230..07ad5a31135 100644 --- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml @@ -1,6 +1,6 @@ include: - project: gitlab-org/quality/pipeline-common - ref: 0.6.0 + ref: 0.13.0 file: - /ci/allure-report.yml - /ci/knapsack-report.yml @@ -13,8 +13,8 @@ include: .test_variables: variables: - QA_DEBUG: "true" QA_GENERATE_ALLURE_REPORT: "true" + COLORIZED_LOGS: "true" GITLAB_USERNAME: "root" GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" GITLAB_ADMIN_USERNAME: "root" @@ -28,7 +28,7 @@ include: - .qa-cache - .test_variables - .bundler_variables - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-lfs-2.9-chrome-99-docker-20.10.14-gcloud-383-kubectl-1.23 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23 stage: qa needs: - review-deploy @@ -50,6 +50,9 @@ include: --tag ~orchestrated \ --tag ~transient \ --tag ~skip_signup_disabled \ + --tag ~requires_git_protocol_v2 \ + --tag ~requires_praefect \ + --tag ~sanity_feature_flags \ --force-color \ --order random \ --format documentation \ @@ -79,27 +82,52 @@ include: # Store knapsack report as artifact so the same report is reused across all jobs download-knapsack-report: - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-chrome-99 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-2.7:bundler-2.3-git-2.33-chrome-${CHROME_VERSION} extends: - .qa-cache - .bundler_variables - .review:rules:review-qa-reliable stage: prepare + variables: + QA_KNAPSACK_REPORTS: review-qa-reliable,review-qa-all before_script: - cd qa && bundle install script: - - QA_KNAPSACK_REPORT_NAME=review-qa-reliable bundle exec rake "knapsack:download" - - QA_KNAPSACK_REPORT_NAME=review-qa-all bundle exec rake "knapsack:download" + - bundle exec rake "knapsack:download" allow_failure: true artifacts: paths: - qa/knapsack/review-qa-*.json expire_in: 1 day +review-qa-sanity: + extends: + - .review-qa-base + - .review:rules:review-qa-sanity + retry: 1 + variables: + QA_RUN_TYPE: review-qa-sanity + script: + - qa_run_status=0 + - | + bundle exec rake "knapsack:rspec[\ + --tag sanity_feature_flags \ + --force-color \ + --order random \ + --format documentation \ + --format RspecJunitFormatter --out tmp/rspec.xml \ + ]" || qa_run_status=$? + - if [ ${qa_run_status} -ne 0 ]; then + release_sha=$(echo "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-${CI_COMMIT_SHA}}" | cut -c1-11); + echo "Errors can be found at https://sentry.gitlab.net/gitlab/gitlab-review-apps/releases/${release_sha}/all-events/."; + fi + - exit ${qa_run_status} + review-qa-smoke: extends: - .review-qa-base - .review:rules:review-qa-smoke + retry: 1 variables: QA_RUN_TYPE: review-qa-smoke RSPEC_TAGS: --tag smoke @@ -108,6 +136,7 @@ review-qa-reliable: extends: - .review-qa-base - .review:rules:review-qa-reliable + retry: 1 parallel: 10 variables: QA_RUN_TYPE: review-qa-reliable diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 03223e64b23..26c7306c880 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -5,7 +5,7 @@ review-cleanup: extends: - .default-retry - .review:rules:review-cleanup - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:gitlab-helm3-kubectl1.14 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:gitlab-helm3.5-kubectl1.17 stage: prepare environment: name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} @@ -29,8 +29,6 @@ start-review-app-pipeline: needs: - job: build-assets-image artifacts: false - - job: build-qa-image - artifacts: false # These variables are set in the pipeline schedules. # They need to be explicitly passed on to the child pipeline. # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword @@ -72,5 +70,6 @@ danger-review-local: reviewers-recommender: extends: - .default-retry + - .review:rules:reviewers-recommender stage: test needs: [] diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 37593ffd2fc..ccdc2c1b90a 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -73,12 +73,18 @@ .if-merge-request-labels-skip-undercoverage: &if-merge-request-labels-skip-undercoverage if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:skip-undercoverage/' +.if-merge-request-labels-community-contribution: &if-merge-request-labels-community-contribution + if: '$CI_MERGE_REQUEST_LABELS =~ /Community contribution/' + .if-merge-request-labels-jh-contribution: &if-merge-request-labels-jh-contribution if: '$CI_MERGE_REQUEST_LABELS =~ /JiHu contribution/' .if-merge-request-labels-group-global-search: &if-merge-request-labels-group-global-search if: '$CI_MERGE_REQUEST_LABELS =~ /group::global search/' +.if-merge-request-labels-pipeline-revert: &if-merge-request-labels-pipeline-revert + if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:revert/' + .if-security-merge-request: &if-security-merge-request if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID' @@ -175,16 +181,26 @@ - ".gitlab/ci/workhorse.gitlab-ci.yml" .yaml-lint-patterns: &yaml-lint-patterns + - "*.yml" + - "**/*.yml" + +.lint-pipeline-yaml-patterns: &lint-pipeline-yaml-patterns - ".gitlab-ci.yml" - ".gitlab/ci/**/*.yml" - - "data/**/*.yml" - "lib/gitlab/ci/templates/**/*.yml" + - "data/deprecations/**/*.yml" + - "data/removals/**/*.yml" + - "data/whats_new/**/*.yml" + +.lint-metrics-yaml-patterns: &lint-metrics-yaml-patterns + - "config/metrics/**/*.yml" .docs-patterns: &docs-patterns - ".gitlab/route-map.yml" - "doc/**/*" - ".markdownlint.yml" - "scripts/lint-doc.sh" + - ".gitlab/ci/docs.gitlab-ci.yml" .docs-deprecations-and-removals-patterns: &docs-deprecations-and-removals-patterns - "doc/update/deprecations.md" @@ -896,10 +912,26 @@ - <<: *if-default-refs changes: *qa-patterns +.qa:rules:determine-qa-tests: + rules: + - <<: *if-not-ee + when: never + - <<: *if-merge-request-targeting-stable-branch + allow_failure: true + - <<: *if-dot-com-gitlab-org-and-security-merge-request + changes: *code-backstage-qa-patterns + allow_failure: true + - <<: *if-dot-com-gitlab-org-schedule + allow_failure: true + - <<: *if-force-ci + allow_failure: true + .qa:rules:package-and-qa: rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-pipeline-revert + when: never - <<: *if-merge-request-targeting-stable-branch allow_failure: true - <<: *if-dot-com-gitlab-org-and-security-merge-request @@ -925,6 +957,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-pipeline-revert + when: never - <<: *if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-qa changes: *feature-flag-development-config-patterns when: manual @@ -1060,10 +1094,8 @@ rules: - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request - changes: *core-backend-patterns - - <<: *if-merge-request - changes: *ci-patterns - - changes: ["config/**/*"] + changes: *backend-patterns + - changes: *core-backend-patterns .rails:rules:code-backstage-qa: rules: @@ -1354,6 +1386,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-pipeline-revert + when: never - <<: *if-merge-request-labels-skip-undercoverage when: never - <<: *if-merge-request-labels-run-all-rspec @@ -1558,6 +1592,8 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-labels-pipeline-revert + when: never - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns @@ -1601,6 +1637,10 @@ rules: - when: on_success +.review:rules:review-qa-sanity: + rules: + - when: on_success + .review:rules:review-qa-smoke: rules: - when: on_success @@ -1627,7 +1667,6 @@ .review:rules:review-qa-all: rules: - - <<: *if-merge-request-labels-run-review-app # we explicitly don't allow the job to fail in that case - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns when: manual @@ -1662,6 +1701,14 @@ - <<: *if-merge-request changes: *danger-patterns +.review:rules:reviewers-recommender: + rules: + - <<: *if-not-canonical-namespace + when: never + - <<: *if-merge-request-labels-community-contribution + when: never + - <<: *if-merge-request + ############### # Setup rules # ############### @@ -1769,3 +1816,13 @@ rules: - <<: *if-default-refs changes: *yaml-lint-patterns + +.lint-pipeline-yaml:rules: + rules: + - <<: *if-default-refs + changes: *lint-pipeline-yaml-patterns + +.lint-metrics-yaml:rules: + rules: + - <<: *if-default-refs + changes: *lint-metrics-yaml-patterns diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 4339251897c..505caeec837 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -60,7 +60,7 @@ no-jh-check: verify-tests-yml: extends: - .setup:rules:verify-tests-yml - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 + image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine3.13 stage: test needs: [] script: @@ -96,7 +96,7 @@ generate-frontend-fixtures-mapping: - ${FRONTEND_FIXTURES_MAPPING_PATH} .detect-test-base: - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7 + image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION} needs: [] stage: prepare script: @@ -160,7 +160,7 @@ detect-previous-failed-tests: add-jh-folder: extends: .setup:rules:add-jh-folder - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7 + image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION} stage: prepare before_script: - source ./scripts/utils.sh @@ -171,7 +171,6 @@ add-jh-folder: - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-org/gitlab-jh-mirrors/gitlab/-/archive/${JH_BRANCH}/gitlab-${JH_BRANCH}.tar.gz?path=jh" - tar -xf "jh-folder.tar.gz" - mv "gitlab-${JH_BRANCH}-jh/jh/" ./ - - cp Gemfile.lock jh/ - ls -l jh/ artifacts: expire_in: 2d diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 79fea15690c..f4fa39300b6 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,5 +1,5 @@ .tests-metadata-state: - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7 + image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION} before_script: - source scripts/utils.sh artifacts: diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml index 01e059b8a60..6db3582bdab 100644 --- a/.gitlab/ci/workhorse.gitlab-ci.yml +++ b/.gitlab/ci/workhorse.gitlab-ci.yml @@ -1,6 +1,6 @@ workhorse:verify: extends: .workhorse:rules:workhorse - image: ${GITLAB_DEPENDENCY_PROXY}golang:1.16 + image: ${GITLAB_DEPENDENCY_PROXY}golang:1.17 stage: test needs: [] script: @@ -20,10 +20,6 @@ workhorse:verify: - scripts/gitaly-test-build - make -C workhorse test -workhorse:test using go 1.16: - extends: .workhorse:test - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7-golang-1.16-git-2.31 - workhorse:test using go 1.17: extends: .workhorse:test - image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-2.7-golang-1.17-git-2.31 + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-1.17-git-2.31 diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index ac32e4226e2..0420f158bbb 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -1,4 +1,5 @@ -# Yamllint of CI-related yaml. +# Yamllint of yaml files. + # This uses rules from project root `.yamllint`. lint-yaml: extends: @@ -7,15 +8,29 @@ lint-yaml: image: pipelinecomponents/yamllint:latest stage: lint needs: [] + script: + - yamllint --strict -f colored . + +# The jobs below will not use the configuration present in `.yamllint` (it's because of the -d option) +# +# Docs: https://yamllint.readthedocs.io/en/stable/configuration.html#custom-configuration-without-a-config-file + +lint-pipeline-yaml: + extends: + - .default-retry + - .lint-pipeline-yaml:rules + image: pipelinecomponents/yamllint:latest + stage: lint + needs: [] variables: LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates data/deprecations data/removals data/whats_new script: - - yamllint --strict -f colored $LINT_PATHS + - 'yamllint -d "{extends: default, rules: {line-length: disable, document-start: disable}}" $LINT_PATHS' lint-metrics-yaml: extends: - .default-retry - - .yaml-lint:rules + - .lint-metrics-yaml:rules image: pipelinecomponents/yamllint:latest stage: lint needs: [] |