From edaa33dee2ff2f7ea3fac488d41558eb5f86d68c Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 20 Jan 2022 09:16:11 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-7-stable-ee --- .gitlab/ci/docs.gitlab-ci.yml | 9 ++-- .gitlab/ci/frontend.gitlab-ci.yml | 5 +++ .gitlab/ci/qa.gitlab-ci.yml | 9 ++++ .gitlab/ci/rails.gitlab-ci.yml | 14 ++++-- .gitlab/ci/review-apps/dast.gitlab-ci.yml | 3 +- .gitlab/ci/review-apps/qa.gitlab-ci.yml | 75 ++++++++++++------------------- .gitlab/ci/rules.gitlab-ci.yml | 75 ++++++++++++++++--------------- .gitlab/ci/setup.gitlab-ci.yml | 11 +++-- .gitlab/ci/workhorse.gitlab-ci.yml | 2 +- .gitlab/ci/yaml.gitlab-ci.yml | 2 +- 10 files changed, 106 insertions(+), 99 deletions(-) (limited to '.gitlab/ci') diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index ae36c0cea70..c439e9a7c80 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -44,7 +44,7 @@ docs-lint markdown: - .default-retry - .docs:rules:docs-lint # When updating the image version here, update it in /scripts/lint-doc.sh too. - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.12.0-markdownlint-0.29.0 + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.15-vale-2.14.0-markdownlint-0.30.0 stage: lint needs: [] script: @@ -53,7 +53,7 @@ docs-lint markdown: docs-lint links: extends: - .docs:rules:docs-lint - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.5-08847baa + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.15-ruby-2.7.5-cee62c13 stage: lint needs: [] script: @@ -77,15 +77,16 @@ ui-docs-links lint: script: - bundle exec haml-lint -i DocumentationLinks -docs-lint deprecations: +docs-lint deprecations-and-removals: variables: SETUP_DB: "false" extends: - .default-retry - .rails-cache - .default-before_script - - .docs:rules:deprecations + - .docs:rules:deprecations-and-removals stage: lint needs: [] script: - bundle exec rake gitlab:docs:check_deprecations + - bundle exec rake gitlab:docs:check_removals diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index ea4ae3b0492..1dd5285e0ae 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -306,6 +306,11 @@ qa-frontend-node:latest: - .qa-frontend-node - .frontend:rules:qa-frontend-node-latest image: ${GITLAB_DEPENDENCY_PROXY}node:latest + # This is a workaround for https://github.com/webpack/webpack/issues/14532 until + # we can upgrade to Webpack 5 and switch to SHA256: https://gitlab.com/gitlab-org/gitlab/-/issues/350120 + script: + - *yarn-install + - run_timed_command "retry yarn run webpack-prod-node-latest" webpack-dev-server: extends: diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 4fec223e66d..b12f76f2823 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -33,6 +33,15 @@ qa:selectors: script: - bundle exec bin/qa Test::Sanity::Selectors +qa:auto_quarantine: + extends: + - .qa-job-base + rules: + - if: '$QA_TRIGGER_AUTO_QUARANTINE =~ /true|yes|1/i' + script: + - bundle exec confiner -r .confiner/quarantine.yml + allow_failure: true + qa:selectors-as-if-foss: extends: - qa:selectors diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index d676dc2f331..1d2f94b616d 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -346,7 +346,7 @@ rspec fast_spec_helper minimal: db:rollback: extends: .db-job-base script: - - scripts/db_tasks db:migrate VERSION=20181228175414 + - scripts/db_tasks db:migrate VERSION=20210301200959 - scripts/db_tasks db:migrate SKIP_SCHEMA_VERSION_CHECK=true db:rollback decomposed: @@ -360,6 +360,12 @@ db:migrate:reset: script: - bundle exec rake db:migrate:reset +db:migrate:reset decomposed: + extends: + - db:migrate:reset + - .decomposed-database + - .rails:rules:decomposed-databases + db:migrate-from-previous-major-version: extends: .db-job-base variables: @@ -457,7 +463,7 @@ db:backup_and_restore: script: - . scripts/prepare_build.sh - bundle exec rake db:drop db:create db:structure:load db:seed_fu - - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry} + - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry,packages} - bundle exec rake gitlab:backup:create - date - bundle exec rake gitlab:backup:restore @@ -592,8 +598,10 @@ rspec:undercoverage: else echo "Using \$CI_COMMIT_SHA ($CI_COMMIT_SHA) for this non-merge result pipeline."; fi; + - UNDERCOVERAGE_COMPARE="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-$(git merge-base origin/master HEAD)}" + - echo "Undercoverage comparing with ${UNDERCOVERAGE_COMPARE}" - if [ -f scripts/undercoverage ]; then - run_timed_command "scripts/undercoverage"; + run_timed_command "scripts/undercoverage ${UNDERCOVERAGE_COMPARE}"; fi; rspec:feature-flags: diff --git a/.gitlab/ci/review-apps/dast.gitlab-ci.yml b/.gitlab/ci/review-apps/dast.gitlab-ci.yml index 512c850b7da..d0ad4d23a82 100644 --- a/.gitlab/ci/review-apps/dast.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/dast.gitlab-ci.yml @@ -45,7 +45,6 @@ # 10019, 10021 Missing security headers # 10023, 10024, 10025, 10037 Information Disclosure # 10040 Secure Pages Include Mixed Content -# 10055 CSP # 10056 X-Debug-Token Information Leak # Duration: 14 minutes 20 seconds @@ -54,7 +53,7 @@ dast:secureHeaders-csp-infoLeak: - .dast_conf variables: DAST_USERNAME: "user1" - DAST_ONLY_INCLUDE_RULES: "10019,10021,10023,10024,10025,10037,10040,10055,10056" + DAST_ONLY_INCLUDE_RULES: "10019,10021,10023,10024,10025,10037,10040,10056" script: - /analyze diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml index af4674b802b..4ef6efa2604 100644 --- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml @@ -26,35 +26,22 @@ - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - cd qa - - if [ -n "$KNAPSACK_REPORT_PATH" ]; then - bundle exec rake knapsack:download; - fi - artifacts: - paths: - - qa/tmp - expire_in: 7 days - when: always - -.parallel-qa-base: - parallel: 5 - variables: - KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb" script: - | bin/test "${QA_SCENARIO}" "${CI_ENVIRONMENT_URL}" \ -- \ --color --format documentation \ --format RspecJunitFormatter --out tmp/rspec.xml - after_script: - - if [ -n "$KNAPSACK_GENERATE_REPORT" ]; then - mv qa/${KNAPSACK_REPORT_PATH} qa/knapsack/gcs/regenerated-${CI_NODE_INDEX}.json; - fi artifacts: paths: - - qa/tmp # we can't merge list so need to include explicitly once more - - qa/knapsack/gcs/regenerated-*.json + - qa/tmp reports: junit: qa/tmp/rspec.xml + expire_in: 7 days + when: always + +.parallel-qa-base: + parallel: 5 .allure-report-base: image: @@ -79,16 +66,6 @@ --ignore-missing-results \ --color -.knapsack-upload-base: - image: - name: ${QA_IMAGE} - entrypoint: [""] - stage: post-qa - before_script: - - cd qa - script: - - bundle exec rake 'knapsack:upload[knapsack/gcs/regenerated-*.json]' - review-qa-smoke: extends: - .review-qa-base @@ -96,8 +73,8 @@ review-qa-smoke: retry: 1 # This is confusing but this means "2 runs at max". variables: QA_RUN_TYPE: review-qa-smoke - script: - - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}" + QA_SCENARIO: Test::Instance::Smoke + review-qa-reliable: extends: @@ -108,7 +85,6 @@ review-qa-reliable: variables: QA_RUN_TYPE: review-qa-reliable QA_SCENARIO: Test::Instance::Reliable - KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json review-qa-all: extends: @@ -118,7 +94,6 @@ review-qa-all: variables: QA_RUN_TYPE: review-qa-all QA_SCENARIO: Test::Instance::All - KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json review-performance: extends: @@ -155,6 +130,15 @@ allure-report-qa-smoke: ALLURE_REPORT_PATH_PREFIX: gitlab-review-smoke ALLURE_JOB_NAME: review-qa-smoke +allure-report-qa-reliable: + extends: + - .allure-report-base + - .review:rules:review-qa-reliable-report + needs: ["review-qa-reliable"] + variables: + ALLURE_REPORT_PATH_PREFIX: gitlab-review-reliable + ALLURE_JOB_NAME: review-qa-reliable + allure-report-qa-all: extends: - .allure-report-base @@ -164,18 +148,15 @@ allure-report-qa-all: ALLURE_REPORT_PATH_PREFIX: gitlab-review-all ALLURE_JOB_NAME: review-qa-all -knapsack-report-qa-all: +knapsack-report: extends: - - .knapsack-upload-base - - .review:rules:knapsack-report-qa-all - needs: ["review-qa-all"] - variables: - KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-all.json - -knapsack-report-qa-reliable: - extends: - - .knapsack-upload-base - - .review:rules:knapsack-report-qa-reliable - needs: ["review-qa-reliable"] - variables: - KNAPSACK_REPORT_PATH: knapsack/gcs/review-qa-reliable.json + - .review:rules:knapsack-report + image: + name: ${QA_IMAGE} + entrypoint: [""] + stage: post-qa + allow_failure: true + before_script: + - cd qa + script: + - bundle exec rake 'knapsack:upload[tmp/knapsack/*/*.json]' diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index e62de4bc6dc..008b62f6a0f 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -171,12 +171,13 @@ - ".markdownlint.yml" - "scripts/lint-doc.sh" -.docs-deprecations-patterns: &docs-deprecations-patterns +.docs-deprecations-and-removals-patterns: &docs-deprecations-and-removals-patterns - "doc/update/deprecations.md" - - "data/deprecations/*.yml" - - "data/deprecations/templates/_deprecation_template.md.erb" + - "doc/update/removals.md" + - "data/deprecations/**/*" + - "data/removals/**/*" + - "tooling/docs/**/*" - "lib/tasks/gitlab/docs/compile_deprecations.rake" - - "tooling/deprecations/docs.rb" .bundler-patterns: &bundler-patterns - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' @@ -228,6 +229,9 @@ - "vendor/assets/**/*" - "{,ee/,jh/}{app/assets,app/helpers,app/presenters,app/views,locale,public,symbol}/**/*" +.controllers-patterns: &controllers-patterns + - "{,ee/,jh/}{app/controllers}/**/*" + .startup-css-patterns: &startup-css-patterns - "{,ee/,jh/}app/assets/stylesheets/startup/**/*" @@ -256,7 +260,7 @@ - "lib/gitlab/markdown_cache/active_record/**/*" - "config/prometheus/common_metrics.yml" # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer - "{,ee/,jh/}app/models/project_statistics.rb" # Used to calculate sizes in migration specs - - "GITALY_SERVER_VERSION" # Has interactions with background migrations:https://gitlab.com/gitlab-org/gitlab/-/issues/336538 + - "GITALY_SERVER_VERSION" # Has interactions with background migrations:https://gitlab.com/gitlab-org/gitlab/-/issues/336538 # CI changes - ".gitlab-ci.yml" - ".gitlab/ci/**/*" @@ -279,7 +283,7 @@ - ".dockerignore" - "qa/**/*" -# Code patterns + .ci-patterns + .workhorse-patterns +# Code patterns + .ci-patterns .code-patterns: &code-patterns - "{package.json,yarn.lock}" - ".browserslistrc" @@ -541,10 +545,10 @@ changes: *docs-patterns when: on_success -.docs:rules:deprecations: +.docs:rules:deprecations-and-removals: rules: - <<: *if-default-refs - changes: *docs-deprecations-patterns + changes: *docs-deprecations-and-removals-patterns ################## # GraphQL rules # @@ -1612,12 +1616,14 @@ changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *controllers-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: *qa-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns when: manual allow_failure: true - - <<: *if-dot-com-gitlab-org-merge-request - changes: *qa-patterns - <<: *if-dot-com-gitlab-org-schedule variables: KNAPSACK_GENERATE_REPORT: "true" @@ -1648,56 +1654,51 @@ rules: - when: on_success -# The rule needs to be duplicated between `on_success` and `on_failure` -# because the jobs `needs` the previous job to complete. -# With `when: always`, and the `review-qa-*` jobs are manual, the `allure-report-qa-*` jobs -# would start running before the qa jobs have started. -# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 +# If the needed job isn't allowed to fail, we need to use `when: always` in +# order to keep the job always running after it. +# +# If the needed job is allowed to fail, we need to use both +# `when: on_success` and `when: on_failure` in order to keep +# the job always running after it. +# Not that if the needed job has `when: on_success` we can use `when: always` +# for the depending job. +# +# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76756 + +# Since `review-qa-smoke` isn't allowed to fail, we need to use `when: always` for `review-qa-smoke-report`. .review:rules:review-qa-smoke-report: rules: - - when: on_success - - when: on_failure + - when: always .review:rules:review-qa-reliable: rules: - when: on_success - allow_failure: true + +# Since `review-qa-reliable` isn't allowed to fail, we need to use `when: always`for `review-qa-reliable-report`. +.review:rules:review-qa-reliable-report: + rules: + - when: always .review:rules:review-qa-all: rules: - - <<: *if-merge-request-labels-run-review-app # we explicitely don't allow the job to fail in that case + - <<: *if-merge-request-labels-run-review-app # we explicitly don't allow the job to fail in that case - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns when: manual - allow_failure: true # manual jobs needs to be allowd to fail, otherwise they block the pipeline + allow_failure: true # manual jobs needs to be allowed to fail, otherwise they block the pipeline - when: on_success allow_failure: true -# The rule needs to be duplicated between `on_success` and `on_failure` -# because the jobs `needs` the previous job to complete. -# With `when: always`, and the `review-qa-*` jobs are manual, the `allure-report-qa-*` jobs -# would start running before the qa jobs have started. -# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 +# Since `review-qa-all` is allowed to fail (and potentially manual), we need to use `when: on_success` and `when: on_failure` for `review-qa-all-report`. .review:rules:review-qa-all-report: rules: - when: on_success - allow_failure: true - when: on_failure - allow_failure: true -# Generate knapsack report on successful runs only -# Reliable suite will pass most of the time so this should yield best distribution -.review:rules:knapsack-report-qa-reliable: - rules: - - if: '$KNAPSACK_GENERATE_REPORT == "true"' - when: on_success - allow_failure: true - -.review:rules:knapsack-report-qa-all: +.review:rules:knapsack-report: rules: - if: '$KNAPSACK_GENERATE_REPORT == "true"' when: always - allow_failure: true .review:rules:review-cleanup: rules: diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 1eb3bd2ea41..13108ba289a 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -151,14 +151,17 @@ detect-previous-failed-tests: add-jh-folder: extends: .setup:rules:add-jh-folder - image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge + image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7 stage: prepare before_script: - - apk add --no-cache --update curl bash + - source ./scripts/utils.sh + - install_gitlab_gem script: - - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-jh/gitlab/-/archive/main-jh/gitlab-main-jh.tar.gz?path=jh" + - JH_BRANCH=$(./scripts/setup/find-jh-branch.rb) + - 'echo "JH_BRANCH: ${JH_BRANCH}"' + - curl --location -o "jh-folder.tar.gz" "https://gitlab.com/gitlab-jh/gitlab/-/archive/${JH_BRANCH}/gitlab-${JH_BRANCH}.tar.gz?path=jh" - tar -xf "jh-folder.tar.gz" - - mv gitlab-main-jh-jh/jh/ ./ + - mv "gitlab-${JH_BRANCH}-jh/jh/" ./ - cp Gemfile.lock jh/ - ls -l jh/ artifacts: diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml index cd53adc6d4b..aab077e575b 100644 --- a/.gitlab/ci/workhorse.gitlab-ci.yml +++ b/.gitlab/ci/workhorse.gitlab-ci.yml @@ -4,7 +4,7 @@ workhorse:verify: stage: test needs: [] script: - - make -C workhorse # test build + - make -C workhorse # test build - make -C workhorse verify .workhorse:test: diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index 590593b9d75..218dc0a7859 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -10,4 +10,4 @@ lint-yaml: variables: LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates script: - - yamllint -f colored $LINT_PATHS + - yamllint --strict -f colored $LINT_PATHS -- cgit v1.2.3