diff options
Diffstat (limited to '.gitlab/ci')
| -rw-r--r-- | .gitlab/ci/as-if-jh.gitlab-ci.yml | 9 | ||||
| -rw-r--r-- | .gitlab/ci/cng/main.gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 10 | ||||
| -rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | .gitlab/ci/gitlab-gems.gitlab-ci.yml | 3 | ||||
| -rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 5 | ||||
| -rw-r--r-- | .gitlab/ci/package-and-test/main.gitlab-ci.yml | 4 | ||||
| -rw-r--r-- | .gitlab/ci/qa-common/main.gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | .gitlab/ci/qa-common/rules.gitlab-ci.yml | 24 | ||||
| -rw-r--r-- | .gitlab/ci/qa-common/variables.gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 68 | ||||
| -rw-r--r-- | .gitlab/ci/rails/shared.gitlab-ci.yml | 3 | ||||
| -rw-r--r-- | .gitlab/ci/review-apps/main.gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 4 | ||||
| -rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 81 | ||||
| -rw-r--r-- | .gitlab/ci/test-on-gdk/main.gitlab-ci.yml | 45 | ||||
| -rw-r--r-- | .gitlab/ci/workhorse.gitlab-ci.yml | 9 |
18 files changed, 165 insertions, 107 deletions
diff --git a/.gitlab/ci/as-if-jh.gitlab-ci.yml b/.gitlab/ci/as-if-jh.gitlab-ci.yml index ec9acdb5d4d..6c1c3357089 100644 --- a/.gitlab/ci/as-if-jh.gitlab-ci.yml +++ b/.gitlab/ci/as-if-jh.gitlab-ci.yml @@ -1,6 +1,7 @@ .as-if-jh-sandbox-variables: variables: AS_IF_JH_BRANCH: "as-if-jh/${CI_COMMIT_REF_NAME}" + JH_MIRROR_REPOSITORY: "https://dummy:${ADD_JH_FILES_TOKEN}@gitlab.com/gitlab-org/gitlab-jh-mirrors/gitlab.git" SANDBOX_REPOSITORY: "https://dummy:${AS_IF_JH_TOKEN}@gitlab.com/gitlab-org-sandbox/gitlab-jh-validation.git" .shared-as-if-jh: @@ -22,18 +23,22 @@ add-jh-files: - source ./scripts/setup/as-if-jh.sh - install_gitlab_gem script: - - prepare_jh_branch - - download_jh_path ${JH_FILES_TO_COMMIT} + - set_jh_branch_env_variable + - download_jh_files ${JH_FILES_TO_COMMIT} - echoinfo "Changes after downloading JiHu files:" - git diff - git status artifacts: expire_in: 2d + when: always # We also want the artifacts when we exit with error 3 (detached pipeline in that case) paths: # This should match JH_FILES_TO_COMMIT - jh/ - package.json - yarn.lock + allow_failure: + exit_codes: + - 3 # Set in the download_jh_files_from_git_clone function prepare-as-if-jh-branch: extends: diff --git a/.gitlab/ci/cng/main.gitlab-ci.yml b/.gitlab/ci/cng/main.gitlab-ci.yml index 208567f569b..e7593b8f208 100644 --- a/.gitlab/ci/cng/main.gitlab-ci.yml +++ b/.gitlab/ci/cng/main.gitlab-ci.yml @@ -49,7 +49,6 @@ include: EE_PIPELINE: "${EE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$EE_PIPELINE'` will evaluate to `false` when this variable is empty GITLAB_ELASTICSEARCH_INDEXER_VERSION: "${GITLAB_ELASTICSEARCH_INDEXER_VERSION}" GITLAB_KAS_VERSION: "${GITLAB_KAS_VERSION}" - GITLAB_METRICS_EXPORTER_VERSION: "${GITLAB_METRICS_EXPORTER_VERSION}" GITLAB_PAGES_VERSION: "${GITLAB_PAGES_VERSION}" GITLAB_SHELL_VERSION: "${GITLAB_SHELL_VERSION}" GITLAB_WORKHORSE_VERSION: "${GITLAB_WORKHORSE_VERSION}" diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 6aad4de64bd..25d974b1580 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -123,3 +123,13 @@ docs-lint deprecations-and-removals: needs: [] script: - bundle exec rake gitlab:docs:check_deprecations + +docs-lint redirects: + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-alpine + stage: lint + extends: + - .default-retry + - .docs:rules:redirect-check + needs: [] + script: + - ./scripts/lint-docs-redirects.rb diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index f103032ee69..a1c209abd98 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -67,6 +67,7 @@ compile-test-assets: paths: - public/assets/ - node_modules/@gitlab/svgs/dist/icons.json # app/helpers/icons_helper.rb uses this file + - node_modules/@gitlab/svgs/dist/file_icons/file_icons.json # app/helpers/icons_helper.rb uses this file - "${WEBPACK_COMPILE_LOG_PATH}" when: always diff --git a/.gitlab/ci/gitlab-gems.gitlab-ci.yml b/.gitlab/ci/gitlab-gems.gitlab-ci.yml index 1ee08c4ab85..a773e9c7f90 100644 --- a/.gitlab/ci/gitlab-gems.gitlab-ci.yml +++ b/.gitlab/ci/gitlab-gems.gitlab-ci.yml @@ -26,3 +26,6 @@ include: - local: .gitlab/ci/templates/gem.gitlab-ci.yml inputs: gem_name: "csv_builder" + - local: .gitlab/ci/templates/gem.gitlab-ci.yml + inputs: + gem_name: "gitlab-http" diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index dd615fe5e9d..51e23dce320 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -179,11 +179,6 @@ cache: - *ruby-coverage-gems-cache-push -# This cache should eventually be replaced by .ruby-gems-coverage-cache. -.coverage-cache: - cache: - - *ruby-gems-cache - .ruby-node-cache: cache: - *ruby-gems-cache diff --git a/.gitlab/ci/package-and-test/main.gitlab-ci.yml b/.gitlab/ci/package-and-test/main.gitlab-ci.yml index 9e11a6606f7..c616fe3de82 100644 --- a/.gitlab/ci/package-and-test/main.gitlab-ci.yml +++ b/.gitlab/ci/package-and-test/main.gitlab-ci.yml @@ -104,6 +104,7 @@ instance: variables: QA_SCENARIO: Test::Instance::Image rules: + - !reference [.rules:test:smoke-for-omnibus-mr, rules] - !reference [.rules:test:feature-flags-set, rules] # always run instance to validate ff change - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ @@ -140,6 +141,7 @@ praefect: QA_CAN_TEST_PRAEFECT: "true" KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/3_create/**/*_spec.rb" rules: + - !reference [.rules:test:smoke-for-omnibus-mr, rules] - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ @@ -177,6 +179,7 @@ decomposition-single-db: QA_SCENARIO: Test::Instance::Image GITLAB_QA_OPTS: --omnibus-config decomposition_single_db $EXTRA_GITLAB_QA_OPTS rules: + - !reference [.rules:test:smoke-for-omnibus-mr, rules] - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ @@ -213,6 +216,7 @@ decomposition-multiple-db: GITLAB_ALLOW_SEPARATE_CI_DATABASE: "true" GITLAB_QA_OPTS: --omnibus-config decomposition_multiple_db $EXTRA_GITLAB_QA_OPTS rules: + - !reference [.rules:test:smoke-for-omnibus-mr, rules] - !reference [.rules:test:qa-parallel, rules] - if: $QA_SUITES =~ /Test::Instance::All/ diff --git a/.gitlab/ci/qa-common/main.gitlab-ci.yml b/.gitlab/ci/qa-common/main.gitlab-ci.yml index 5c9043f8694..bdb5e776808 100644 --- a/.gitlab/ci/qa-common/main.gitlab-ci.yml +++ b/.gitlab/ci/qa-common/main.gitlab-ci.yml @@ -6,7 +6,7 @@ workflow: include: - project: gitlab-org/quality/pipeline-common - ref: 7.2.3 + ref: 7.5.1 file: - /ci/base.gitlab-ci.yml - /ci/knapsack-report.yml diff --git a/.gitlab/ci/qa-common/rules.gitlab-ci.yml b/.gitlab/ci/qa-common/rules.gitlab-ci.yml index 7518f08398f..c593ec4ccfb 100644 --- a/.gitlab/ci/qa-common/rules.gitlab-ci.yml +++ b/.gitlab/ci/qa-common/rules.gitlab-ci.yml @@ -43,6 +43,10 @@ .not-canonical-project: ¬-canonical-project if: '$CI_PROJECT_PATH != "gitlab-org/gitlab" && $CI_PROJECT_PATH != "gitlab-cn/gitlab"' +# If Schedule pipeline +.if-schedule-pipeline: &if-schedule-pipeline + if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $SCHEDULE_TYPE == "maintenance"' + # Selective test execution against omnibus instance have following execution scenarios: # * only e2e spec files changed - runs only changed specs # * qa framework changes - runs full test suite @@ -149,12 +153,32 @@ when: never - !reference [.rules:test:qa, rules] +.rules:test:never-schedule-pipeline: + rules: + - <<: *if-schedule-pipeline + when: never + +.rules:test:gdk-load-balancer-changes: + rules: + - changes: + - ".gitlab/ci/test-on-gdk/**" + - "lib/gitlab/database/load_balancing/**/*" + .rules:test:qa-default-branch: rules: - *qa-run-all-e2e-label - *default-branch - *feature-flags-set-manual +.rules:test:smoke-for-omnibus-mr: + rules: + - if: '$CI_PROJECT_NAME == "omnibus-gitlab" && $PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/ && $QA_OMNIBUS_MR_TESTS == "only-smoke-reliable"' + variables: + QA_RSPEC_TAGS: "--tag smoke --tag reliable --tag ~orchestrated --tag ~skip_live_env" + - if: '$CI_PROJECT_NAME == "omnibus-gitlab" && $PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/ && $QA_OMNIBUS_MR_TESTS == "except-smoke-reliable"' + variables: + QA_RSPEC_TAGS: "--tag ~smoke --tag ~reliable --tag ~orchestrated --tag ~skip_live_env --tag ~transient" + # ------------------------------------------ # Report # ------------------------------------------ diff --git a/.gitlab/ci/qa-common/variables.gitlab-ci.yml b/.gitlab/ci/qa-common/variables.gitlab-ci.yml index 9498df47ecc..a449d960cff 100644 --- a/.gitlab/ci/qa-common/variables.gitlab-ci.yml +++ b/.gitlab/ci/qa-common/variables.gitlab-ci.yml @@ -17,3 +17,4 @@ variables: RSPEC_FAST_QUARANTINE_FILE: "fast_quarantine-gitlab.txt" # This path is relative to /home/gitlab/qa/ in the QA container RSPEC_FAST_QUARANTINE_PATH: "rspec/${RSPEC_FAST_QUARANTINE_FILE}" + QA_OMNIBUS_MR_TESTS: "only-smoke-reliable" diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 5cdaf6ada82..0a06418e7c6 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -84,7 +84,6 @@ trigger-omnibus: GITALY_SERVER_VERSION: $GITALY_SERVER_VERSION GITLAB_ELASTICSEARCH_INDEXER_VERSION: $GITLAB_ELASTICSEARCH_INDEXER_VERSION GITLAB_KAS_VERSION: $GITLAB_KAS_VERSION - GITLAB_METRICS_EXPORTER_VERSION: $GITLAB_METRICS_EXPORTER_VERSION GITLAB_PAGES_VERSION: $GITLAB_PAGES_VERSION GITLAB_SHELL_VERSION: $GITLAB_SHELL_VERSION GITLAB_WORKHORSE_VERSION: $GITLAB_WORKHORSE_VERSION diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 326d23be5a4..419c659e745 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -68,19 +68,17 @@ update-ruby-gems-coverage-cache-push: - bundle_install_script # Used in: +# - rspec:coverage # - rspec:undercoverage -# - rspec:feature-flags -# - rspec:merge-auto-explain-logs -# -# TODO: Consider making rspec:coverage reuse this base job, or split into two base jobs. .coverage-base: extends: - .default-retry - # TODO: If applicable for all children jobs, delete/replace this cache with .ruby-gems-coverage-cache, as it is much smaller. - - .coverage-cache + - .ruby-gems-coverage-cache + variables: + BUNDLE_WITHOUT: "" # This is to override the variable defined in .gitlab-ci.yml + BUNDLE_ONLY: "coverage" before_script: - source scripts/utils.sh - - export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:kerberos:metrics:omnibus:ed25519" - bundle_install_script rspec migration pg14: @@ -289,6 +287,11 @@ rspec unit clickhouse: - .rspec-base-pg14-clickhouse23 - .rails:rules:clickhouse-changes +rspec-ee unit clickhouse: + extends: + - .rspec-base-pg14-clickhouse23 + - .rails:rules:clickhouse-changes + gitlab:setup: extends: .db-job-base variables: @@ -306,35 +309,6 @@ gitlab:setup: paths: - log/*.log -rspec:deprecations: - extends: - - .default-retry - - .default-before_script - - .static-analysis-cache - - .rails:rules:deprecations - stage: post-test - allow_failure: true - needs: - - job: rspec:artifact-collector unit - optional: true - - job: rspec:artifact-collector system - optional: true - - job: rspec:artifact-collector remainder - optional: true - - job: rspec:artifact-collector ee - optional: true - variables: - SETUP_DB: "false" - script: - - grep -h -R "keyword" deprecations/ | awk '{$1=$1};1' | sort | uniq -c | sort - - grep -R "keyword" deprecations/ | wc - - run_timed_command "fail_on_warnings bundle exec rubocop --only Lint/LastKeywordArgument --parallel" - artifacts: - expire_in: 31d - when: always - paths: - - deprecations/ - # The jobs built upon `.artifact-collector` are to work around the # needs: [] limit of a maximum of 50 dependencies. # These intermediate jobs allow us to collect the artifacts of @@ -362,6 +336,8 @@ rspec:artifact-collector unit: - .rails:rules:ee-and-foss-unit needs: - rspec unit pg14 # 28 jobs + - job: rspec unit clickhouse # 1 job + optional: true rspec:artifact-collector system: extends: @@ -451,6 +427,8 @@ rspec:artifact-collector ee: optional: true - job: rspec-ee unit pg14 # 18 jobs optional: true + - job: rspec-ee unit clickhouse # 1 job + optional: true - job: rspec-ee integration pg14 # 6 jobs optional: true - job: rspec-ee system pg14 # 10 jobs @@ -463,8 +441,9 @@ rspec:artifact-collector ee: - !reference ['.rails:rules:ee-only-system', rules] rspec:coverage: + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-slim-ruby-${RUBY_VERSION} extends: - - .ruby-gems-coverage-cache + - .coverage-base - .rails:rules:rspec-coverage stage: post-test needs: @@ -497,12 +476,6 @@ rspec:coverage: - job: memory-on-boot optional: true artifacts: false - variables: - BUNDLE_WITHOUT: "" # This is to override the variable defined in .gitlab-ci.yml - BUNDLE_ONLY: "coverage" - before_script: - - source scripts/utils.sh - - bundle_install_script script: - run_timed_command "bundle exec scripts/merge-simplecov" coverage: '/LOC \((\d+\.\d+%)\) covered.$/' @@ -519,6 +492,7 @@ rspec:coverage: path: coverage/coverage.xml rspec:undercoverage: + image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-slim-ruby-${RUBY_VERSION} extends: - .coverage-base - .rails:rules:rspec-undercoverage @@ -539,7 +513,6 @@ rspec:undercoverage: rspec:feature-flags: extends: - - .coverage-base - .rails:rules:rspec-feature-flags stage: post-test needs: @@ -547,11 +520,13 @@ rspec:feature-flags: - job: "haml-lint" - job: "haml-lint ee" optional: true + before_script: + - source scripts/utils.sh script: - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then - run_timed_command "bundle exec scripts/used-feature-flags" || (scripts/slack master-broken "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL}" ci_failing "GitLab Bot" && exit 1); + run_timed_command "scripts/used-feature-flags" || (scripts/slack master-broken "☠️ \`${CI_JOB_NAME}\` failed! ☠️ See ${CI_JOB_URL}" ci_failing "GitLab Bot" && exit 1); else - run_timed_command "bundle exec scripts/used-feature-flags"; + run_timed_command "scripts/used-feature-flags"; fi rspec:flaky-tests-report: @@ -572,7 +547,6 @@ rspec:flaky-tests-report: rspec:merge-auto-explain-logs: extends: - - .coverage-base - .rails:rules:rspec-merge-auto-explain-logs stage: post-test needs: !reference ["rspec:coverage", "needs"] diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml index ce89e7ef689..6803bdd3386 100644 --- a/.gitlab/ci/rails/shared.gitlab-ci.yml +++ b/.gitlab/ci/rails/shared.gitlab-ci.yml @@ -88,6 +88,9 @@ include: if [ "$CREATE_RAILS_SLOW_TEST_ISSUES" == "true" ]; then bundle exec slow-test-issues --input-files "rspec/rspec-*.json" --project "gitlab-org/gitlab" --token "${TEST_FAILURES_PROJECT_TOKEN}"; fi + if [ "$ADD_SLOW_TEST_NOTE_TO_MERGE_REQUEST" == "true" ]; then + bundle exec slow-test-merge-request-report-note --input-files "rspec/rspec-*.json" --project "gitlab-org/gitlab" --merge_request_iid "$CI_MERGE_REQUEST_IID" --token "${TEST_SLOW_NOTE_PROJECT_TOKEN}"; + fi - echo -e "\e[0Ksection_end:`date +%s`:report_results_section\r\e[0K" allow_failure: diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index a3ced427ea1..2df67713ecc 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -40,7 +40,6 @@ review-build-cng-env: GITALY_SERVER_VERSION GITLAB_ELASTICSEARCH_INDEXER_VERSION GITLAB_KAS_VERSION - GITLAB_METRICS_EXPORTER_VERSION GITLAB_PAGES_VERSION GITLAB_SHELL_VERSION scripts/trigger-build.rb diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 224312bd8ee..d4b199a9a81 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -75,6 +75,10 @@ start-review-app-pipeline: stage: review needs: - job: e2e-test-pipeline-generate + - job: rails-production-server-boot-puma-example + optional: true + - job: rails-production-server-boot-puma-cng + optional: true - job: build-assets-image artifacts: false # We do not want to have ALL global variables passed as trigger variables, diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 304544468ac..d8cc67a966a 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -378,9 +378,12 @@ - "spec/support/database_cleaner.rb" - "config/prometheus/common_metrics.yml" # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer - "{,ee/,jh/}app/models/project_statistics.rb" # Used to calculate sizes in migration specs + - "{,ee/,jh/}app/{,models/,services/,workers/}loose_foreign_keys/**/*" # Gitaly has interactions with background migrations: https://gitlab.com/gitlab-org/gitlab/-/issues/336538 - "GITALY_SERVER_VERSION" - "lib/gitlab/setup_helper.rb" + # Test-gap from https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/issues/275 + - "{,ee/,jh/}lib/tasks/gitlab/seed/*.rake" # DB backup patterns .db-backup-patterns: &db-backup-patterns @@ -503,7 +506,9 @@ - "tests.yml" - "config.ru" - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,storybook,symbol,vendor}/**/*" - - "doc/api/graphql/reference/*" # Files in this folder are auto-generated + # Auto-generated files + - "doc/api/graphql/reference/*" + - "doc/administration/audit_event_streaming/audit_event_types.md" # CI changes - ".gitlab-ci.yml" - ".gitlab/ci/**/*" @@ -609,10 +614,13 @@ - qa/gdk/**/* - Gemfile.lock - yarn.lock + - scripts/build_gdk_image + - scripts/frontend/postinstall.js - workhorse/**/* + - vendor/gems/**/* + - gems/**/* - VERSION - GITLAB_WORKHORSE_VERSION - - GITLAB_METRICS_EXPORTER_VERSION - GITLAB_SHELL_VERSION - GITALY_SERVER_VERSION @@ -756,6 +764,8 @@ changes: *backend-patterns - <<: *if-merge-request changes: *backstage-patterns + - <<: *if-merge-request + changes: ["**/*click_house*"] .rails:rules:ee-and-foss-integration:predictive: rules: @@ -905,9 +915,12 @@ # It's better to fail early and avoid wasting resources running test jobs that would just fail anyway. .build-images:rules:build-gdk-image: rules: + # Allows to force rebuild base image in case something goes wrong and this had to be disabled - if: '$QA_RUN_TESTS_ON_GDK !~ /true|yes|1/i' when: manual allow_failure: true + variables: + BUILD_GDK_BASE: "true" - !reference [".qa:rules:package-and-test-never-run", rules] - <<: *if-default-branch-schedule-nightly # already executed in the 2-hourly schedule when: never @@ -982,6 +995,8 @@ ################# .caching:rules:cache-workhorse: rules: + - <<: *if-not-ee + when: never # That would run for any project that has a "maintenance" pipeline schedule # but in fact, the cache package is only uploaded for gitlab.com/gitlab-org/gitlab and jihulab.com/gitlab-cn/gitlab - <<: *if-schedule-maintenance @@ -999,6 +1014,8 @@ # The new strategy to cache assets as generic packages is experimental and can be disabled by removing the `CACHE_ASSETS_AS_PACKAGE` variable - if: '$CACHE_ASSETS_AS_PACKAGE != "true"' when: never + - <<: *if-not-ee + when: never # That would run for any project that has a "maintenance" pipeline schedule # but in fact, the cache package is only uploaded for gitlab.com/gitlab-org/gitlab and jihulab.com/gitlab-cn/gitlab - <<: *if-schedule-maintenance @@ -1014,8 +1031,7 @@ .caching:rules:cache-assets-as-if-foss: rules: - - <<: *if-jh - when: never + - !reference [".strict-ee-only-rules", rules] - !reference [".caching:rules:cache-assets", "rules"] .caching:rules:packages-cleanup: @@ -1023,6 +1039,8 @@ # The new strategy to cache assets as generic packages is experimental and can be disabled by removing the `CACHE_ASSETS_AS_PACKAGE` variable - if: '$CACHE_ASSETS_AS_PACKAGE != "true"' when: never + - <<: *if-not-ee + when: never # That would run for any project that has a "maintenance" pipeline schedule # but in fact, the cache package is only uploaded for gitlab.com/gitlab-org/gitlab and jihulab.com/gitlab-cn/gitlab - <<: *if-schedule-maintenance @@ -1071,6 +1089,11 @@ when: manual allow_failure: true +.docs:rules:redirect-check: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *docs-patterns + .docs:rules:docs-lint: rules: - <<: *if-default-refs @@ -1182,8 +1205,7 @@ rules: - <<: *if-not-canonical-namespace when: never - - <<: *if-not-ee - when: never + - !reference [".strict-ee-only-rules", rules] - !reference [.frontend:rules:compile-production-assets, rules] .frontend:rules:compile-test-assets: @@ -1240,12 +1262,12 @@ # The new strategy to upload fixtures as generic packages is experimental and can be disabled by removing the `REUSE_FRONTEND_FIXTURES_ENABLED` variable - if: '$REUSE_FRONTEND_FIXTURES_ENABLED != "true"' when: never + - <<: *if-not-ee + when: never - <<: *if-merge-request-labels-pipeline-expedite when: never - <<: *if-dot-com-gitlab-org-default-branch changes: *code-backstage-patterns - - <<: *if-foss-default-branch - changes: *code-backstage-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: - ".gitlab/ci/frontend.gitlab-ci.yml" @@ -1603,14 +1625,12 @@ .qa:rules:package-and-test-schedule: rules: - - <<: *if-dot-com-gitlab-org-schedule + - <<: [*if-dot-com-gitlab-org-schedule, *qa-e2e-test-schedule-variables] allow_failure: true - <<: *qa-e2e-test-schedule-variables .qa:rules:e2e-schedule-blocking: rules: - - <<: *if-dot-com-gitlab-org-schedule - <<: *qa-e2e-test-schedule-variables + - <<: [*if-dot-com-gitlab-org-schedule, *qa-e2e-test-schedule-variables] # Note: If any changes are made to this rule, the following should also be updated: # 1) .qa:rules:manual-omnibus-and-follow-up-e2e @@ -1653,6 +1673,7 @@ .qa:rules:package-and-test-nightly: rules: + - !reference [".qa:rules:package-and-test-never-run", rules] - <<: *if-default-branch-schedule-nightly allow_failure: true variables: @@ -1800,6 +1821,9 @@ # From .qa:rules:package-and-test-schedule - <<: *if-dot-com-gitlab-org-schedule when: never + # Do not run on unapproved MR + - <<: *if-merge-request-not-approved + when: never # From .qa:rules:code-merge-request-manual - <<: *if-merge-request changes: *code-patterns @@ -1840,8 +1864,7 @@ .rails:rules:single-db-as-if-foss: rules: - - <<: *if-jh - when: never + - !reference [".strict-ee-only-rules", rules] - !reference [".rails:rules:single-db", "rules"] .rails:rules:db:check-migrations-single-db: @@ -1867,8 +1890,7 @@ .rails:rules:single-db-ci-connection-as-if-foss: rules: - - <<: *if-jh - when: never + - !reference [".strict-ee-only-rules", rules] - !reference [".rails:rules:single-db-ci-connection", "rules"] .rails:rules:db:check-migrations-single-db-ci-connection: @@ -1973,6 +1995,8 @@ changes: *backend-patterns - <<: *if-default-refs changes: *backstage-patterns + - <<: *if-merge-request + changes: ["**/*click_house*"] .rails:rules:ee-and-foss-integration: rules: @@ -2192,16 +2216,6 @@ changes: *code-backstage-patterns when: on_failure -.rails:rules:deprecations: - rules: - - <<: *if-not-ee - when: never - - <<: *if-merge-request-labels-pipeline-expedite - when: never - - <<: *if-default-branch-schedule-nightly - - <<: *if-ruby3_1-branch-schedule-nightly - - <<: *if-merge-request-labels-run-all-rspec - .rails:rules:rspec-coverage: rules: - <<: *if-not-ee @@ -2221,11 +2235,11 @@ when: never - <<: *if-merge-request-labels-skip-undercoverage when: never + - <<: *if-merge-request-labels-run-all-rspec # We cannot get the coverage data from child pipeline so we only run undercoverage on full pipelines for now # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/113410#note_1335422806 - <<: *if-merge-request-not-approved when: never - - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *backend-patterns @@ -2242,6 +2256,8 @@ when: never - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request-labels-record-queries + - <<: *if-default-branch-refs + changes: *code-patterns .rails:rules:default-branch-schedule-nightly--code-backstage-default-rules: rules: @@ -2287,6 +2303,7 @@ rules: - <<: *if-merge-request changes: ["**/*click_house*"] + - <<: *if-merge-request-labels-run-all-rspec ######################### # Static analysis rules # @@ -2530,7 +2547,7 @@ # The following rules needs to be the same as the one for .review:rules:review-cleanup # except that: -# - most rules re automatic here (i.e. no `when: manual`) and not allowed to fail (i.e. no `allow_failure: true`) here +# - we start review apps automatically for scheduled pipelines and when the `pipeline:run-review-app` label is set # - several rules have `variables: *review-change-pattern` here .review:rules:start-review-app-pipeline: rules: @@ -2538,12 +2555,18 @@ when: never - <<: *if-merge-request-labels-pipeline-expedite when: never + - if: '$CI_REVIEW_APPS_ENABLED != "true"' + when: never - <<: *if-merge-request-labels-run-review-app - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns + when: manual + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-build-patterns variables: *review-change-pattern + when: manual + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *controllers-patterns variables: *review-change-pattern @@ -2561,6 +2584,8 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns + when: manual + allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns when: manual diff --git a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml index 41f85c492d9..9e179fec458 100644 --- a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml +++ b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml @@ -26,6 +26,7 @@ variables: image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-2.3-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23 extends: - .qa-cache + - .default-retry stage: test services: - docker:${DOCKER_VERSION}-dind @@ -41,15 +42,16 @@ variables: FF_NETWORK_PER_BUILD: 1 GDK_URL: http://gdk.test:3000 KNAPSACK_TEST_FILE_PATTERN: "qa/specs/features/**/*_spec.rb" + QA_SUITE_STATUS_ENV_FILE: "$CI_PROJECT_DIR/suite_status.env" before_script: - - echo "SUITE_RAN=true" > suite_status.env - - echo -e "\e[0Ksection_start:`date +%s`:pull_image\r\e[0KPull GDK QA image" + - echo "SUITE_RAN=true" > "$QA_SUITE_STATUS_ENV_FILE" + - echo -e "\e[0Ksection_start:`date +%s`:pull_image[collapsed=true]\r\e[0KPull GDK QA image" - docker pull ${GDK_IMAGE} - echo -e "\e[0Ksection_end:`date +%s`:pull_image\r\e[0K" # Despite `incremental: false` and `static: true`, GDK sometimes fails to start without increasing max user watches # This is why we're not running the GDK container as a service - sysctl -n -w fs.inotify.max_user_watches=524288 - - echo -e "\e[0Ksection_start:`date +%s`:launch_gdk\r\e[0KLaunch GDK" + - echo -e "\e[0Ksection_start:`date +%s`:launch_gdk[collapsed=true]\r\e[0KLaunch GDK" - mkdir -p $CI_PROJECT_DIR/log/gdk $CI_PROJECT_DIR/log/gitlab # This command matches the permissions of the user that runs GDK inside the container. - chown -R 1000:1000 $CI_PROJECT_DIR/log @@ -61,29 +63,28 @@ variables: # With `FF_NETWORK_PER_BUILD=1` and `--network host` the IP of the gdk container should be 172.18.0.2, but we get it # dynamically just in case - echo "$(docker exec gdk bash -c "getent hosts \$HOSTNAME" | awk '{print $1}') gdk.test" >> /etc/hosts + - echo -e "\e[0Ksection_end:`date +%s`:launch_gdk\r\e[0K" + - echo -e "\e[0Ksection_start:`date +%s`:install_gems[collapsed=true]\r\e[0KInstall gems" - source scripts/utils.sh - cd qa && bundle install + - echo -e "\e[0Ksection_end:`date +%s`:install_gems\r\e[0K" script: - - retry_exponential test_url $GDK_URL/users/sign_in - - echo -e "\e[0Ksection_end:`date +%s`:launch_gdk\r\e[0K" + - echo -e "\e[0Ksection_start:`date +%s`:healthcheck[collapsed=true]\r\e[0KWait for gdk to start" + - retry_times_sleep 100 3 test_url $GDK_URL/users/sign_in + - echo -e "\e[0Ksection_end:`date +%s`:healthcheck\r\e[0K" - echo -e "\e[0Ksection_start:`date +%s`:run_tests\r\e[0KRun E2E tests" - export QA_COMMAND="bundle exec bin/qa ${QA_SCENARIO:=Test::Instance::All} $GDK_URL $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS" - echo "Running - '$QA_COMMAND'" - eval "$QA_COMMAND" - echo -e "\e[0Ksection_end:`date +%s`:run_tests\r\e[0K" - after_script: - - | - if [ "$CI_JOB_STATUS" == "failed" ]; then - echo "SUITE_FAILED=true" >> suite_status.env - fi - - docker stop gdk artifacts: paths: - qa/tmp - - log + - log/gitlab + - log/gdk/*/current reports: junit: qa/tmp/rspec-*.xml - dotenv: suite_status.env + dotenv: "$QA_SUITE_STATUS_ENV_FILE" expire_in: 7 days when: always @@ -125,6 +126,7 @@ download-fast-quarantine-report: gdk-qa-smoke: extends: - .gdk-qa-base + - .gitlab-qa-report variables: QA_SCENARIO: Test::Instance::Smoke QA_RUN_TYPE: gdk-qa-smoke @@ -144,21 +146,23 @@ gdk-qa-smoke-with-load-balancer: reports: dotenv: "" rules: - - changes: - - ".gitlab/ci/test-on-gdk/**" - - "lib/gitlab/database/load_balancing/**/*" + - !reference [".rules:test:never-schedule-pipeline", rules] + - !reference [".rules:test:gdk-load-balancer-changes", rules] allow_failure: true gdk-qa-reliable: extends: - .gdk-qa-base + - .gitlab-qa-report - .parallel variables: QA_SCENARIO: Test::Instance::Blocking QA_RUN_TYPE: gdk-qa-blocking + parallel: 10 rules: - - when: always - allow_failure: true + - if: '$CI_MERGE_REQUEST_LABELS =~ /devops::govern/' + - when: on_success + allow_failure: true gdk-qa-reliable-with-load-balancer: extends: @@ -174,9 +178,8 @@ gdk-qa-reliable-with-load-balancer: reports: dotenv: "" rules: - - changes: - - ".gitlab/ci/test-on-gdk/**" - - "lib/gitlab/database/load_balancing/**/*" + - !reference [".rules:test:never-schedule-pipeline", rules] + - !reference [".rules:test:gdk-load-balancer-changes", rules] allow_failure: true gdk-qa-non-blocking: diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml index 5b128ef6170..cedcde27b7e 100644 --- a/.gitlab/ci/workhorse.gitlab-ci.yml +++ b/.gitlab/ci/workhorse.gitlab-ci.yml @@ -11,6 +11,8 @@ workhorse:verify: .workhorse:test: extends: .workhorse:rules:workhorse image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-exiftool-12.60 + services: + - name: redis:${REDIS_VERSION}-alpine variables: GITALY_ADDRESS: "tcp://127.0.0.1:8075" stage: test @@ -22,6 +24,8 @@ workhorse:verify: - bundle_install_script - go version - scripts/gitaly-test-build + - cp workhorse/config.toml.example workhorse/config.toml + - sed -i 's|URL.*$|URL = "redis://redis:6379"|g' workhorse/config.toml script: - make -C workhorse test @@ -30,6 +34,7 @@ workhorse:test go: parallel: matrix: - GO_VERSION: ["1.18", "1.19", "1.20"] + REDIS_VERSION: ["7.0", "6.2"] script: - make -C workhorse test-coverage coverage: '/\d+.\d+%/' @@ -43,11 +48,15 @@ workhorse:test fips: parallel: matrix: - GO_VERSION: ["1.18", "1.19", "1.20"] + REDIS_VERSION: ["7.0", "6.2"] image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/ubi-${UBI_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-exiftool-12.60 variables: FIPS_MODE: 1 workhorse:test race: extends: .workhorse:test + parallel: + matrix: + - REDIS_VERSION: ["7.0", "6.2"] script: - make -C workhorse test-race |