diff options
Diffstat (limited to '.gitlab/ci')
26 files changed, 752 insertions, 381 deletions
diff --git a/.gitlab/ci/_skip.yml b/.gitlab/ci/_skip.yml index 27a3ff5b836..9d3745cf2f1 100644 --- a/.gitlab/ci/_skip.yml +++ b/.gitlab/ci/_skip.yml @@ -1,7 +1,7 @@ # no-op pipeline template for skipping whole child pipeline execution no-op: - image: ${GITLAB_DEPENDENCY_PROXY}alpine:latest + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:latest stage: test variables: GIT_STRATEGY: none diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index 1b041c9af38..3c7056a92c1 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -15,27 +15,7 @@ build-qa-image: stage: build-images needs: [] script: - # Tag with commit SHA by default - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_SHA}" - # For branches, tag with slugified branch name. For tags, use the tag directly - - export QA_IMAGE_BRANCH="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_TAG:-$CI_COMMIT_REF_SLUG}" - # Auto-deploy tag format uses first 12 letters of commit SHA. Tag with that - # reference also - - export QA_IMAGE_FOR_AUTO_DEPLOY="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_SHA:0:11}" - - echo $QA_IMAGE - - echo $QA_IMAGE_BRANCH - - echo $QA_IMAGE_FOR_AUTO_DEPLOY - - | - /kaniko/executor \ - --context=${CI_PROJECT_DIR} \ - --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile \ - --destination=${QA_IMAGE} \ - --destination=${QA_IMAGE_BRANCH} \ - --destination=${QA_IMAGE_FOR_AUTO_DEPLOY} \ - --build-arg=CHROME_VERSION=${CHROME_VERSION} \ - --build-arg=DOCKER_VERSION=${DOCKER_VERSION} \ - --build-arg=QA_BUILD_TARGET=${QA_BUILD_TARGET:-qa} \ - --cache=true + - ./scripts/build_qa_image # This image is used by: # - The `CNG` pipelines (via the `review-build-cng` job): https://gitlab.com/gitlab-org/build/CNG/-/blob/cfc67136d711e1c8c409bf8e57427a644393da2f/.gitlab-ci.yml#L335 diff --git a/.gitlab/ci/caching.gitlab-ci.yml b/.gitlab/ci/caching.gitlab-ci.yml new file mode 100644 index 00000000000..6a13fc3c56f --- /dev/null +++ b/.gitlab/ci/caching.gitlab-ci.yml @@ -0,0 +1,65 @@ +cache-workhorse: + extends: + - .default-retry + - .default-before_script + - .rails-cache + - .setup-test-env-cache + - .caching:rules:cache-workhorse + stage: prepare + variables: + SETUP_DB: "false" + script: + - source scripts/gitlab_component_helpers.sh + - 'gitlab_workhorse_archive_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }' + - run_timed_command "scripts/setup-test-env" + - run_timed_command "select_gitlab_workhorse_essentials" + - run_timed_command "create_gitlab_workhorse_package" + - run_timed_command "upload_gitlab_workhorse_package" + artifacts: + expire_in: 7d + paths: + - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/ + +.cache-assets-base: + extends: + - .compile-assets-base + - .assets-compile-cache + - .caching:rules:cache-assets + stage: prepare + variables: + WEBPACK_REPORT: "false" + script: + - yarn_install_script + - export GITLAB_ASSETS_HASH=$(bundle exec rake gitlab:assets:hash_sum) + - source scripts/gitlab_component_helpers.sh + - 'gitlab_assets_archive_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }' + - assets_compile_script + - echo -n "${GITLAB_ASSETS_HASH}" > "cached-assets-hash.txt" + - run_timed_command "create_gitlab_assets_package" + - run_timed_command "upload_gitlab_assets_package" + +cache-assets:test: + extends: .cache-assets-base + +cache-assets:test as-if-foss: + extends: + - .cache-assets-base + - .as-if-foss + +cache-assets:production: + extends: .cache-assets-base + variables: + NODE_ENV: "production" + RAILS_ENV: "production" + +packages-cleanup: + extends: + - .default-retry + - .caching:rules:packages-cleanup + image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION} + stage: prepare + before_script: + - source scripts/utils.sh + - install_gitlab_gem + script: + - scripts/packages/automated_cleanup.rb diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 7e157171183..022f1c17a93 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -2,7 +2,7 @@ extends: - .default-retry - .docs:rules:review-docs - image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-alpine stage: review needs: [] variables: diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 3bd65b565e4..085c0aa890d 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -1,11 +1,3 @@ -.yarn-install: &yarn-install - - source scripts/utils.sh - - run_timed_command "retry yarn install --frozen-lockfile" - -.storybook-yarn-install: &storybook-yarn-install - - source scripts/utils.sh - - run_timed_command "retry yarn run storybook:install --frozen-lockfile" - .compile-assets-base: extends: - .default-retry @@ -21,9 +13,16 @@ WEBPACK_COMPILE_LOG_PATH: "tmp/webpack-output.log" stage: prepare script: - - *yarn-install - - run_timed_command "bin/rake gitlab:assets:compile" - - run_timed_command "scripts/clean-old-cached-assets" + - yarn_install_script + - export GITLAB_ASSETS_HASH=$(bin/rake gitlab:assets:hash_sum) + - 'echo "CACHE_ASSETS_AS_PACKAGE: ${CACHE_ASSETS_AS_PACKAGE}"' + # The new strategy to cache assets as generic packages is experimental and can be disabled by removing the `CACHE_ASSETS_AS_PACKAGE` variable + - | + if [[ "${CACHE_ASSETS_AS_PACKAGE}" == "true" ]]; then + source scripts/gitlab_component_helpers.sh + gitlab_assets_archive_doesnt_exist || run_timed_command "download_and_extract_gitlab_assets" + fi + - assets_compile_script compile-production-assets: extends: @@ -42,8 +41,6 @@ compile-production-assets: - public/assets/ - "${WEBPACK_COMPILE_LOG_PATH}" when: always - before_script: - - !reference [.default-before_script, before_script] after_script: - rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here @@ -71,6 +68,9 @@ update-assets-compile-production-cache: - .assets-compile-cache-push - .shared:rules:update-cache stage: prepare + script: + - !reference [compile-production-assets, script] + - echo -n "${GITLAB_ASSETS_HASH}" > "cached-assets-hash.txt" artifacts: {} # This job's purpose is only to update the cache. update-assets-compile-test-cache: @@ -79,25 +79,31 @@ update-assets-compile-test-cache: - .assets-compile-cache-push - .shared:rules:update-cache stage: prepare + script: + - !reference [compile-test-assets, script] + - echo -n "${GITLAB_ASSETS_HASH}" > "cached-assets-hash.txt" artifacts: {} # This job's purpose is only to update the cache. +# TODO: Remove this as it's duplicating update-assets-compile-*-cache update-yarn-cache: extends: - .default-retry + - .default-utils-before_script - .yarn-cache-push - .shared:rules:update-cache stage: prepare script: - - *yarn-install + - yarn_install_script update-storybook-yarn-cache: extends: - .default-retry + - .default-utils-before_script - .storybook-yarn-cache-push - .shared:rules:update-cache stage: prepare script: - - *storybook-yarn-install + - yarn_install_script .frontend-fixtures-base: extends: @@ -180,7 +186,7 @@ graphql-schema-dump as-if-foss: SETUP_DB: "false" before_script: - !reference [.default-before_script, before_script] - - *yarn-install + - yarn_install_script stage: test .jest-base: @@ -247,6 +253,7 @@ jest-integration: coverage-frontend: extends: - .default-retry + - .default-utils-before_script - .yarn-cache - .frontend:rules:coverage-frontend needs: @@ -255,9 +262,8 @@ coverage-frontend: - job: "jest minimal" optional: true stage: post-test - before_script: - - *yarn-install script: + - yarn_install_script - run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js" # Removing the individual coverage results, as we just merged them. - if ls coverage-frontend/jest-* > /dev/null 2>&1; then @@ -277,31 +283,33 @@ coverage-frontend: .qa-frontend-node: extends: - .default-retry - - .yarn-cache + - .default-utils-before_script + - .qa-frontend-node-cache - .frontend:rules:qa-frontend-node stage: test - dependencies: [] + needs: [] script: - - *yarn-install + - yarn_install_script - run_timed_command "retry yarn run webpack-prod" qa-frontend-node:14: extends: .qa-frontend-node - image: ${GITLAB_DEPENDENCY_PROXY}node:14 + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:14 qa-frontend-node:16: extends: .qa-frontend-node - image: ${GITLAB_DEPENDENCY_PROXY}node:16 + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:16 qa-frontend-node:latest: extends: - .qa-frontend-node - .frontend:rules:qa-frontend-node-latest - image: ${GITLAB_DEPENDENCY_PROXY}node:latest + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:latest webpack-dev-server: extends: - .default-retry + - .default-utils-before_script - .yarn-cache - .frontend:rules:default-frontend-jobs stage: test @@ -310,7 +318,7 @@ webpack-dev-server: WEBPACK_MEMORY_TEST: "true" WEBPACK_VENDOR_DLL: "true" script: - - *yarn-install + - yarn_install_script - run_timed_command "retry yarn webpack-vendor" - run_timed_command "node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js" artifacts: @@ -322,13 +330,14 @@ webpack-dev-server: bundle-size-review: extends: - .default-retry + - .default-utils-before_script - .assets-compile-cache - .frontend:rules:bundle-size-review image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images:danger stage: test needs: [] script: - - *yarn-install + - yarn_install_script - scripts/bundle_size_review artifacts: when: always @@ -366,8 +375,8 @@ startup-css-check as-if-foss: - .frontend-test-base - .storybook-yarn-cache script: - - *storybook-yarn-install - - yarn run storybook:build + - run_timed_command "retry yarn run storybook:install --frozen-lockfile" + - run_timed_command "yarn run storybook:build" needs: ["graphql-schema-dump"] compile-storybook: diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 7bdca62b68e..ed59a0dd8fe 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -8,13 +8,17 @@ - job_execution_timeout - stuck_or_timeout_failure -.default-before_script: +.default-utils-before_script: before_script: - echo $FOSS_ONLY - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' - export GOPATH=$CI_PROJECT_DIR/.go - mkdir -p $GOPATH - source scripts/utils.sh + +.default-before_script: + before_script: + - !reference [.default-utils-before_script, before_script] - source scripts/prepare_build.sh .ruby-gems-cache: &ruby-gems-cache @@ -79,19 +83,30 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .assets-cache: &assets-cache - key: "assets-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}-v2" + key: "assets-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}-v5" + # This list should match GITLAB_ASSETS_PATHS_LIST from scripts/gitlab_component_helpers.sh paths: - - assets-hash.txt - - public/assets/webpack/ - - tmp/cache/assets/sprockets/ - - tmp/cache/babel-loader/ - - tmp/cache/vue-loader/ + - cached-assets-hash.txt + - app/assets/javascripts/locale/**/app.js + - public/assets/ policy: pull .assets-cache-push: &assets-cache-push <<: *assets-cache policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. +.assets-tmp-cache: &assets-tmp-cache + key: "assets-tmp-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}-v1" + paths: + - tmp/cache/assets/sprockets/ + - tmp/cache/babel-loader/ + - tmp/cache/vue-loader/ + policy: pull + +.assets-tmp-cache-push: &assets-tmp-cache-push + <<: *assets-tmp-cache + policy: push # We want to rebuild the cache from scratch to ensure we don't pile up outdated cache files. + .storybook-node-modules-cache: &storybook-node-modules-cache key: "storybook-node-modules-${DEBIAN_VERSION}-${NODE_ENV}" paths: @@ -174,7 +189,7 @@ cache: - *ruby-gems-cache -.danger-review-cache: +.ruby-node-cache: cache: - *ruby-gems-cache - *node-modules-cache @@ -199,6 +214,12 @@ cache: - *node-modules-cache +.qa-frontend-node-cache: + cache: + - *node-modules-cache + - *assets-tmp-cache + +# TODO: Remove this as it's duplicating .assets-compile-cache-push .yarn-cache-push: cache: - *node-modules-cache-push @@ -208,12 +229,14 @@ - *ruby-gems-cache - *node-modules-cache - *assets-cache + - *assets-tmp-cache .assets-compile-cache-push: cache: - *ruby-gems-cache # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache` - *node-modules-cache-push - *assets-cache-push + - *assets-tmp-cache-push .storybook-yarn-cache: cache: @@ -323,7 +346,7 @@ FOSS_ONLY: '1' .use-docker-in-docker: - image: ${GITLAB_DEPENDENCY_PROXY}docker:${DOCKER_VERSION} + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}docker:${DOCKER_VERSION} services: - docker:${DOCKER_VERSION}-dind variables: diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml index 95318d5ce08..51b0f4071eb 100644 --- a/.gitlab/ci/notify.gitlab-ci.yml +++ b/.gitlab/ci/notify.gitlab-ci.yml @@ -1,12 +1,12 @@ .notify-slack: - image: ${GITLAB_DEPENDENCY_PROXY}alpine + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine/curl stage: notify dependencies: [] cache: {} variables: MERGE_REQUEST_URL: ${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID} before_script: - - apk update && apk add git curl bash + - apk update && apk add git bash - echo "NOTIFY_CHANNEL is ${NOTIFY_CHANNEL}" - echo "CI_PIPELINE_URL is ${CI_PIPELINE_URL}" @@ -34,13 +34,28 @@ notify-security-pipeline: - scripts/slack ${NOTIFY_CHANNEL} "<!subteam^S0127FU8PDE> ☠️ Pipeline for merged result failed! ☠️ See ${CI_PIPELINE_URL} (triggered from ${MERGE_REQUEST_URL})" ci_failing "GitLab Release Tools Bot" notify-pipeline-failure: - extends: - - .notify-slack + extends: .notify-slack + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION} rules: - - if: '$NOTIFY_PIPELINE_FAILURE_CHANNEL' + # Don't report child pipeline failures + - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"' + when: never + - if: '$CI_SLACK_WEBHOOK_URL && $NOTIFY_PIPELINE_FAILURE_CHANNEL' when: on_failure allow_failure: true variables: - NOTIFY_CHANNEL: "${NOTIFY_PIPELINE_FAILURE_CHANNEL}" + SLACK_CHANNEL: "${NOTIFY_PIPELINE_FAILURE_CHANNEL}" + FAILED_PIPELINE_REPORT_FILE: "failed_pipeline_report.json" + before_script: + - source scripts/utils.sh + - apt-get update && apt-get install -y jq + - install_gitlab_gem script: - - scripts/slack ${NOTIFY_CHANNEL} "❌ \`${CI_COMMIT_REF_NAME}\` pipeline failed! See ${CI_PIPELINE_URL}" ci_failing "notify-pipeline-failure" + - scripts/generate-failed-pipeline-slack-message.rb + - | + curl -X POST -H 'Content-Type: application/json' --data @${FAILED_PIPELINE_REPORT_FILE} "$CI_SLACK_WEBHOOK_URL" + artifacts: + paths: + - ${FAILED_PIPELINE_REPORT_FILE} + when: always + expire_in: 2 days diff --git a/.gitlab/ci/package-and-test/main.gitlab-ci.yml b/.gitlab/ci/package-and-test/main.gitlab-ci.yml index d96da7744ab..1a1c67bf572 100644 --- a/.gitlab/ci/package-and-test/main.gitlab-ci.yml +++ b/.gitlab/ci/package-and-test/main.gitlab-ci.yml @@ -1,11 +1,13 @@ # E2E tests pipeline loaded dynamically by script: scripts/generate-e2e-pipeline +default: + interruptible: true + include: - - local: .gitlab/ci/global.gitlab-ci.yml - local: .gitlab/ci/package-and-test/rules.gitlab-ci.yml - local: .gitlab/ci/package-and-test/variables.gitlab-ci.yml - project: gitlab-org/quality/pipeline-common - ref: 1.2.1 + ref: 1.3.0 file: - /ci/base.gitlab-ci.yml - /ci/allure-report.yml @@ -27,13 +29,12 @@ stages: .ruby-image: image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.3 -.bundle-install: +.qa-install: + variables: + BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES: "true" + BUNDLE_SILENCE_ROOT_WARNING: "true" extends: - .gitlab-qa-install - - .qa-cache - variables: - RUN_WITH_BUNDLE: "true" # installs and runs gitlab-qa via bundler - QA_PATH: qa .omnibus-env: variables: @@ -44,6 +45,8 @@ stages: echo "SECURITY_SOURCES=${SECURITY_SOURCES:-false}" > $BUILD_ENV echo "OMNIBUS_GITLAB_CACHE_UPDATE=${OMNIBUS_GITLAB_CACHE_UPDATE:-false}" >> $BUILD_ENV for version_file in *_VERSION; do echo "$version_file=$(cat $version_file)" >> $BUILD_ENV; done + echo "OMNIBUS_GITLAB_RUBY3_BUILD=${OMNIBUS_GITLAB_RUBY3_BUILD:-false}" >> $BUILD_ENV + echo "OMNIBUS_GITLAB_CACHE_EDITION=${OMNIBUS_GITLAB_CACHE_EDITION:-GITLAB}" >> $BUILD_ENV echo "Built environment file for omnibus build:" cat $BUILD_ENV artifacts: @@ -59,14 +62,16 @@ stages: .qa: extends: - .qa-base - - .bundle-install + - .qa-install - .gitlab-qa-report stage: test tags: - e2e needs: - - trigger-omnibus - - download-knapsack-report + - job: trigger-omnibus + optional: true + - job: download-knapsack-report + artifacts: true variables: QA_GENERATE_ALLURE_REPORT: "true" QA_CAN_TEST_PRAEFECT: "false" @@ -80,14 +85,33 @@ stages: # ========================================== # Prepare stage # ========================================== +check-release-set: + extends: .rules:prepare + stage: .pre + script: + - | + if [ -z "$RELEASE" ]; then + echo "E2E test pipeline requires omnibus installation docker image to be set via $RELEASE environment variable" + exit 1 + else + echo "Omnibus installation image is set to '$RELEASE'" + fi + +dont-interrupt-me: + extends: .rules:dont-interrupt + stage: .pre + interruptible: false + script: + - echo "This jobs makes sure this pipeline won't be interrupted! See https://docs.gitlab.com/ee/ci/yaml/#interruptible." + trigger-omnibus-env: extends: - .omnibus-env - - .rules:prepare + - .rules:omnibus-build stage: .pre trigger-omnibus: - extends: .rules:prepare + extends: .rules:omnibus-build stage: .pre needs: - trigger-omnibus-env @@ -106,6 +130,8 @@ trigger-omnibus: TOP_UPSTREAM_SOURCE_PROJECT: $CI_PROJECT_PATH SECURITY_SOURCES: $SECURITY_SOURCES CACHE_UPDATE: $OMNIBUS_GITLAB_CACHE_UPDATE + RUBY3_BUILD: $OMNIBUS_GITLAB_RUBY3_BUILD + CACHE_EDITION: $OMNIBUS_GITLAB_CACHE_EDITION SKIP_QA_DOCKER: "true" SKIP_QA_TEST: "true" ee: "true" @@ -115,30 +141,34 @@ trigger-omnibus: download-knapsack-report: extends: - - .bundle-install - - .ruby-image + - .gitlab-qa-image - .rules:prepare stage: .pre + variables: + KNAPSACK_DIR: ${CI_PROJECT_DIR}/qa/knapsack + GIT_STRATEGY: none script: + # when using qa-image, code runs in /home/gitlab/qa folder - bundle exec rake "knapsack:download[test]" + - mkdir -p "$KNAPSACK_DIR" && cp knapsack/*.json "${KNAPSACK_DIR}/" allow_failure: true artifacts: paths: - - qa/knapsack/ee-*.json + - qa/knapsack/*.json expire_in: 1 day -# e2e test jobs run on separate runner which has separate cache setup cache-gems: extends: - - .bundle-install + - .qa-install - .ruby-image - - .qa-cache-push - .rules:prepare stage: .pre tags: - e2e script: - echo "Populated qa cache" + cache: + policy: pull-push # ========================================== # Test stage @@ -154,11 +184,11 @@ cache-gems: _ee:quarantine: extends: - .qa - - .rules:test:quarantine + - .rules:test:manual needs: - - trigger-omnibus + - job: trigger-omnibus + optional: true stage: test - allow_failure: true variables: QA_RSPEC_TAGS: --tag quarantine @@ -332,8 +362,9 @@ ee:update-minor: UPDATE_TYPE: minor QA_RSPEC_TAGS: --tag smoke rules: - - !reference [.rules:test:qa, rules] + - !reference [.rules:test:update, rules] - if: $QA_SUITES =~ /Test::Instance::Smoke/ + - !reference [.rules:test:manual, rules] ee:update-major: extends: @@ -343,8 +374,9 @@ ee:update-major: UPDATE_TYPE: major QA_RSPEC_TAGS: --tag smoke rules: - - !reference [.rules:test:qa, rules] + - !reference [.rules:test:update, rules] - if: $QA_SUITES =~ /Test::Instance::Smoke/ + - !reference [.rules:test:manual, rules] ee:gitaly-cluster: extends: .qa @@ -353,6 +385,7 @@ ee:gitaly-cluster: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::GitalyCluster/ + - !reference [.rules:test:manual, rules] ee:group-saml: extends: .qa @@ -361,6 +394,7 @@ ee:group-saml: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::GroupSAML/ + - !reference [.rules:test:manual, rules] ee:instance-saml: extends: .qa @@ -369,6 +403,7 @@ ee:instance-saml: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::InstanceSAML/ + - !reference [.rules:test:manual, rules] ee:jira: extends: .qa @@ -379,6 +414,7 @@ ee:jira: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Jira/ + - !reference [.rules:test:manual, rules] ee:ldap-no-server: extends: .qa @@ -387,6 +423,7 @@ ee:ldap-no-server: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::LDAPNoServer/ + - !reference [.rules:test:manual, rules] ee:ldap-tls: extends: .qa @@ -395,6 +432,7 @@ ee:ldap-tls: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::LDAPTLS/ + - !reference [.rules:test:manual, rules] ee:ldap-no-tls: extends: .qa @@ -403,6 +441,7 @@ ee:ldap-no-tls: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::LDAPNoTLS/ + - !reference [.rules:test:manual, rules] ee:mtls: extends: .qa @@ -411,6 +450,7 @@ ee:mtls: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::MTLS/ + - !reference [.rules:test:manual, rules] ee:mattermost: extends: .qa @@ -419,6 +459,7 @@ ee:mattermost: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Mattermost/ + - !reference [.rules:test:manual, rules] ee:registry: extends: .qa @@ -427,6 +468,7 @@ ee:registry: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Registry/ + - !reference [.rules:test:manual, rules] ee:registry-with-cdn: extends: .qa @@ -443,6 +485,7 @@ ee:registry-with-cdn: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::RegistryWithCDN/ + - !reference [.rules:test:manual, rules] ee:repository-storage: extends: .qa @@ -451,6 +494,7 @@ ee:repository-storage: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Instance::RepositoryStorage/ + - !reference [.rules:test:manual, rules] ee:service-ping-disabled: extends: .qa @@ -459,6 +503,7 @@ ee:service-ping-disabled: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::ServicePingDisabled/ + - !reference [.rules:test:manual, rules] ee:smtp: extends: .qa @@ -467,6 +512,7 @@ ee:smtp: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::SMTP/ + - !reference [.rules:test:manual, rules] ee:cloud-activation: extends: .qa @@ -476,6 +522,7 @@ ee:cloud-activation: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::CloudActivation/ + - !reference [.rules:test:manual, rules] ee:large-setup: extends: .qa @@ -485,6 +532,7 @@ ee:large-setup: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Instance::LargeSetup/ + - !reference [.rules:test:manual, rules] ee:metrics: extends: .qa @@ -493,6 +541,7 @@ ee:metrics: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Instance::Metrics/ + - !reference [.rules:test:manual, rules] ee:elasticsearch: extends: .qa @@ -504,6 +553,7 @@ ee:elasticsearch: rules: - !reference [.rules:test:qa, rules] - if: $QA_SUITES =~ /Test::Integration::Elasticsearch/ + - !reference [.rules:test:manual, rules] ee:registry-object-storage-tls: extends: ee:object-storage-aws @@ -536,7 +586,7 @@ e2e-test-report: upload-knapsack-report: extends: - .generate-knapsack-report-base - - .bundle-install + - .qa-install - .ruby-image - .rules:report:process-results stage: report @@ -544,7 +594,7 @@ upload-knapsack-report: relate-test-failures: extends: - - .bundle-install + - .qa-install - .ruby-image - .rules:report:process-results stage: report @@ -562,7 +612,7 @@ relate-test-failures: generate-test-session: extends: - - .bundle-install + - .qa-install - .ruby-image - .rules:report:process-results stage: report @@ -585,7 +635,7 @@ generate-test-session: notify-slack: extends: - .notify-slack-qa - - .bundle-install + - .qa-install - .ruby-image - .rules:report:process-results stage: notify @@ -594,6 +644,7 @@ notify-slack: SLACK_ICON_EMOJI: ci_failing STATUS_SYM: ☠️ STATUS: failed + TYPE: "(package-and-test) " when: on_failure script: - bundle exec gitlab-qa-report --prepare-stage-reports "$CI_PROJECT_DIR/gitlab-qa-run-*/**/rspec-*.xml" # generate summary diff --git a/.gitlab/ci/package-and-test/rules.gitlab-ci.yml b/.gitlab/ci/package-and-test/rules.gitlab-ci.yml index d866dec1cd0..47625340a3a 100644 --- a/.gitlab/ci/package-and-test/rules.gitlab-ci.yml +++ b/.gitlab/ci/package-and-test/rules.gitlab-ci.yml @@ -41,12 +41,27 @@ rules: - when: always +.rules:dont-interrupt: + rules: + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + allow_failure: true + - if: $CI_MERGE_REQUEST_IID + when: manual + allow_failure: true + +.rules:omnibus-build: + rules: + - if: $SKIP_OMNIBUS_TRIGGER == "true" + when: never + - when: always + # ------------------------------------------ # Test # ------------------------------------------ -.rules:test:quarantine: +.rules:test:manual: rules: - when: manual + allow_failure: true variables: QA_TESTS: "" @@ -78,7 +93,10 @@ rules: - *qa-framework-changes - <<: *specific-specs - when: never + when: manual + allow_failure: true + variables: + QA_TESTS: "" - *feature-flags-set-manual # general qa job rule for jobs without the need to run in parallel @@ -87,11 +105,21 @@ - *qa-framework-changes - *feature-flags-set-manual +.rules:test:update: + rules: + # skip upgrade jobs if gitlab version is not provided + # these jobs need gitlab version because we can't reliably detect it from just the image + - if: $GITLAB_VERSION == null + when: never + - !reference [.rules:test:qa, rules] + # ------------------------------------------ # Report # ------------------------------------------ .rules:report:allure-report: rules: + - if: $SKIP_ALLURE_REPORT == "true" + when: never - when: always .rules:report:process-results: diff --git a/.gitlab/ci/package-and-test/variables.gitlab-ci.yml b/.gitlab/ci/package-and-test/variables.gitlab-ci.yml index 324092c80f4..cd22fa0e6e4 100644 --- a/.gitlab/ci/package-and-test/variables.gitlab-ci.yml +++ b/.gitlab/ci/package-and-test/variables.gitlab-ci.yml @@ -1,9 +1,10 @@ # Default variables for package-and-test variables: - RELEASE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/build/omnibus-gitlab-mirror/gitlab-ee:${CI_COMMIT_SHA}" SKIP_REPORT_IN_ISSUES: "true" OMNIBUS_GITLAB_CACHE_UPDATE: "false" + OMNIBUS_GITLAB_RUBY3_BUILD: "false" + OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB" QA_LOG_LEVEL: "info" QA_TESTS: "" QA_FEATURE_FLAGS: "" diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index c5a182b055a..bd587cb4418 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -76,8 +76,15 @@ e2e:package-and-test: - e2e-test-pipeline-generate variables: SKIP_MESSAGE: Skipping package-and-test due to mr containing only quarantine changes! + RELEASE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/build/omnibus-gitlab-mirror/gitlab-ee:${CI_COMMIT_SHA}" + GITLAB_QA_IMAGE: "${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_COMMIT_SHA}" + RUN_WITH_BUNDLE: "true" # instructs pipeline to install and run gitlab-qa gem via bundler + QA_PATH: qa # sets the optional path for bundler to run from trigger: strategy: depend + forward: + yaml_variables: true + pipeline_variables: true include: - artifact: package-and-test-pipeline.yml job: e2e-test-pipeline-generate diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 0f524f03188..c60f85634b6 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,160 +1,5 @@ -####################### -# rspec job base specs -.rails-job-base: - extends: - - .default-retry - - .default-before_script - - .rails-cache - -.base-script: - script: - - source ./scripts/rspec_helpers.sh - # Only install knapsack after bundle install! Otherwise oddly some native - # gems could not be found under some circumstance. No idea why, hours wasted. - - run_timed_command "gem install knapsack --no-document" - - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-spawn[collapsed=true]\r\e[0KStarting Gitaly" - - run_timed_command "scripts/gitaly-test-spawn" # Do not use 'bundle exec' here - - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-spawn\r\e[0K" - -.minimal-rspec-tests: - variables: - RSPEC_TESTS_MAPPING_ENABLED: "true" - -.single-db: - variables: - DECOMPOSED_DB: "false" - -.single-db-rspec: - extends: .single-db - -.praefect-with-db: - variables: - GITALY_PRAEFECT_WITH_DB: '1' - -.rspec-base: - extends: - - .rails-job-base - - .base-artifacts - stage: test - variables: - RUBY_GC_MALLOC_LIMIT: 67108864 - RUBY_GC_MALLOC_LIMIT_MAX: 134217728 - RECORD_DEPRECATIONS: "true" - GEO_SECONDARY_PROXY: 0 - needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] - script: - - !reference [.base-script, script] - - rspec_paralellized_job "--tag ~quarantine --tag ~level:migration" - -.base-artifacts: - artifacts: - expire_in: 31d - when: always - paths: - - coverage/ - - crystalball/ - - deprecations/ - - knapsack/ - - rspec/ - - tmp/capybara/ - - log/*.log - reports: - junit: ${JUNIT_RESULT_FILE} - -.rspec-base-migration: - extends: - - .base-artifacts - - .rails:rules:ee-and-foss-migration - script: - - !reference [.base-script, script] - - rspec_paralellized_job "--tag ~quarantine --tag level:migration" - -.rspec-base-pg11: - extends: - - .rspec-base - - .use-pg11 - -.rspec-base-pg12: - extends: - - .rspec-base - - .use-pg12 - -.rspec-base-pg12-as-if-foss: - extends: - - .rspec-base - - .as-if-foss - - .use-pg12 - needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"] - -.rspec-base-pg13: - extends: - - .rspec-base - - .use-pg13 - -.rspec-ee-base-pg11: - extends: - - .rspec-base - - .use-pg11-ee - -.rspec-ee-base-pg12: - extends: - - .rspec-base - - .use-pg12-ee - -.rspec-ee-base-pg12-es8: - extends: - - .rspec-base - - .use-pg12-es8-ee - - .rails:rules:run-search-tests - -.rspec-ee-base-pg12-opensearch1: - extends: - - .rspec-base - - .use-pg12-opensearch1-ee - - .rails:rules:run-search-tests - -.rspec-ee-base-pg13: - extends: - - .rspec-base - - .use-pg13-ee - -.db-job-base: - extends: - - .rails-job-base - - .rails:rules:ee-and-foss-migration - - .use-pg12 - stage: test - needs: ["setup-test-env"] -# rspec job base specs -###################### - -############################ -# rspec job parallel configs -.rspec-migration-parallel: - parallel: 12 - -.rspec-ee-migration-parallel: - parallel: 4 - -.rspec-unit-parallel: - parallel: 28 - -.rspec-ee-unit-parallel: - parallel: 18 - -.rspec-integration-parallel: - parallel: 12 - -.rspec-ee-integration-parallel: - parallel: 6 - -.rspec-system-parallel: - parallel: 28 - -.rspec-ee-system-parallel: - parallel: 10 -# rspec job parallel configs -############################ +include: + - local: .gitlab/ci/rails/shared.gitlab-ci.yml ############################################################### # EE/FOSS: default refs (MRs, default branch, schedules) jobs # @@ -168,7 +13,7 @@ setup-test-env: SETUP_DB: "false" script: - echo $CI_MERGE_REQUEST_APPROVED - - source scripts/gitlab_workhorse_component_helpers.sh + - source scripts/gitlab_component_helpers.sh - run_timed_command "download_and_extract_gitlab_workhorse_package" || true - run_timed_command "scripts/setup-test-env" - run_timed_command "select_gitlab_workhorse_essentials" @@ -197,22 +42,6 @@ setup-test-env: - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/ when: always -build-components: - extends: - - setup-test-env - - .rails:rules:build-components - script: - - source scripts/gitlab_workhorse_component_helpers.sh - - 'gitlab_workhorse_archive_doesnt_exist || { echoinfo "INFO: Exiting early as package exists."; exit 0; }' - - run_timed_command "scripts/setup-test-env" - - run_timed_command "select_gitlab_workhorse_essentials" - - run_timed_command "create_gitlab_workhorse_package" - - run_timed_command "upload_gitlab_workhorse_package" - artifacts: - expire_in: 7d - paths: - - ${TMP_TEST_GITLAB_WORKHORSE_PATH}/ - update-setup-test-env-cache: extends: - setup-test-env @@ -407,6 +236,21 @@ db:check-migrations-single-db: - .single-db - .rails:rules:single-db +db:post_deployment_migrations_validator: + extends: + - .db-job-base + - .rails:rules:ee-and-foss-mr-with-migration + script: + - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME --depth 20 + - scripts/post_deployment_migrations_validator + allow_failure: true + +db:post_deployment_migrations_validator-single-db: + extends: + - db:post_deployment_migrations_validator + - .single-db + - .rails:rules:single-db + db:migrate-non-superuser: extends: - .db-job-base @@ -943,18 +787,41 @@ rspec fail-fast: paths: - tmp/capybara/ -rspec foss-impact: +rspec-foss-impact:pipeline-generate: extends: - - .rspec-base-pg12-as-if-foss - .rails:rules:rspec-foss-impact - needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"] + stage: prepare + needs: ["detect-tests"] script: - - !reference [.base-script, script] - - rspec_matched_foss_tests "${RSPEC_MATCHING_TESTS_PATH}" "--tag ~quarantine" + - scripts/generate-rspec-foss-impact-pipeline "${RSPEC_MATCHING_TESTS_FOSS_PATH}" "${RSPEC_FOSS_IMPACT_PIPELINE_YML}" artifacts: - expire_in: 7d + expire_in: 1 day paths: - - tmp/capybara/ + - $RSPEC_FOSS_IMPACT_PIPELINE_YML + +rspec-foss-impact:trigger: + extends: + - .rails:rules:rspec-foss-impact + stage: test + needs: + - job: "setup-test-env" + artifacts: false + - job: "retrieve-tests-metadata" + artifacts: false + - job: "compile-test-assets as-if-foss" + artifacts: false + - job: "rspec-foss-impact:pipeline-generate" + artifacts: true + variables: + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + trigger: + strategy: depend + forward: + yaml_variables: true + pipeline_variables: true + include: + - artifact: $RSPEC_FOSS_IMPACT_PIPELINE_YML + job: rspec-foss-impact:pipeline-generate fail-pipeline-early: extends: diff --git a/.gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb b/.gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb new file mode 100644 index 00000000000..4ae4cb75a25 --- /dev/null +++ b/.gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb @@ -0,0 +1,50 @@ +# RSpec FOSS impact pipeline loaded dynamically by script: scripts/generate-rspec-foss-impact-pipeline + +include: + - local: .gitlab/ci/rails/shared.gitlab-ci.yml + +default: + image: $DEFAULT_CI_IMAGE + tags: + - gitlab-org + # Default job timeout set to 90m https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/10520 + timeout: 90m + interruptible: true + +stages: + - test + +dont-interrupt-me: + extends: .rules:dont-interrupt + stage: .pre + interruptible: false + script: + - echo "This jobs makes sure this pipeline won't be interrupted! See https://docs.gitlab.com/ee/ci/yaml/#interruptible." + +rspec foss-impact: + extends: .rspec-base-pg12-as-if-foss + needs: + - pipeline: $PARENT_PIPELINE_ID + job: detect-tests + - pipeline: $PARENT_PIPELINE_ID + job: setup-test-env + - pipeline: $PARENT_PIPELINE_ID + job: retrieve-tests-metadata + - pipeline: $PARENT_PIPELINE_ID + job: compile-test-assets as-if-foss + rules: + - when: always + variables: + RSPEC_TESTS_FILTER_FILE: "${RSPEC_MATCHING_TESTS_FOSS_PATH}" + RSPEC_TESTS_MAPPING_ENABLED: "true" +<% if Integer(parallel_value) > 1 %> + parallel: <%= parallel_value %> +<% end %> + script: + - !reference [.base-script, script] + - rspec_paralellized_job "--tag ~quarantine --tag ~level:migration" + artifacts: + expire_in: 7d + paths: + - "${RSPEC_MATCHING_TESTS_FOSS_PATH}" + - tmp/capybara/ diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml new file mode 100644 index 00000000000..60c9826abfe --- /dev/null +++ b/.gitlab/ci/rails/shared.gitlab-ci.yml @@ -0,0 +1,172 @@ +include: + - local: .gitlab/ci/global.gitlab-ci.yml + - local: .gitlab/ci/rules.gitlab-ci.yml + +.rules:dont-interrupt: + rules: + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + allow_failure: true + - if: $CI_MERGE_REQUEST_IID + when: manual + allow_failure: true + +####################### +# rspec job base specs +.rails-job-base: + extends: + - .default-retry + - .default-before_script + - .rails-cache + +.base-script: + script: + - source ./scripts/rspec_helpers.sh + # Only install knapsack after bundle install! Otherwise oddly some native + # gems could not be found under some circumstance. No idea why, hours wasted. + - run_timed_command "gem install knapsack --no-document" + - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-spawn[collapsed=true]\r\e[0KStarting Gitaly" + - run_timed_command "scripts/gitaly-test-spawn" # Do not use 'bundle exec' here + - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-spawn\r\e[0K" + +.minimal-rspec-tests: + variables: + RSPEC_TESTS_MAPPING_ENABLED: "true" + +.single-db: + variables: + DECOMPOSED_DB: "false" + +.single-db-rspec: + extends: .single-db + +.praefect-with-db: + variables: + GITALY_PRAEFECT_WITH_DB: '1' + +.rspec-base: + extends: + - .rails-job-base + - .base-artifacts + stage: test + variables: + RUBY_GC_MALLOC_LIMIT: 67108864 + RUBY_GC_MALLOC_LIMIT_MAX: 134217728 + RECORD_DEPRECATIONS: "true" + GEO_SECONDARY_PROXY: 0 + RSPEC_TESTS_FILTER_FILE: "${RSPEC_MATCHING_TESTS_PATH}" + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] + script: + - !reference [.base-script, script] + - rspec_paralellized_job "--tag ~quarantine --tag ~level:migration" + +.base-artifacts: + artifacts: + expire_in: 31d + when: always + paths: + - coverage/ + - crystalball/ + - deprecations/ + - knapsack/ + - rspec/ + - tmp/capybara/ + - log/*.log + reports: + junit: ${JUNIT_RESULT_FILE} + +.rspec-base-migration: + extends: + - .base-artifacts + - .rails:rules:ee-and-foss-migration + variables: + RSPEC_TESTS_FILTER_FILE: "${RSPEC_MATCHING_TESTS_PATH}" + script: + - !reference [.base-script, script] + - rspec_paralellized_job "--tag ~quarantine --tag level:migration" + +.rspec-base-pg11: + extends: + - .rspec-base + - .use-pg11 + +.rspec-base-pg12: + extends: + - .rspec-base + - .use-pg12 + +.rspec-base-pg12-as-if-foss: + extends: + - .rspec-base + - .as-if-foss + - .use-pg12 + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"] + +.rspec-base-pg13: + extends: + - .rspec-base + - .use-pg13 + +.rspec-ee-base-pg11: + extends: + - .rspec-base + - .use-pg11-ee + +.rspec-ee-base-pg12: + extends: + - .rspec-base + - .use-pg12-ee + +.rspec-ee-base-pg12-es8: + extends: + - .rspec-base + - .use-pg12-es8-ee + - .rails:rules:run-search-tests + +.rspec-ee-base-pg12-opensearch1: + extends: + - .rspec-base + - .use-pg12-opensearch1-ee + - .rails:rules:run-search-tests + +.rspec-ee-base-pg13: + extends: + - .rspec-base + - .use-pg13-ee + +.db-job-base: + extends: + - .rails-job-base + - .rails:rules:ee-and-foss-migration + - .use-pg12 + stage: test + needs: ["setup-test-env"] +# rspec job base specs +###################### + +############################ +# rspec job parallel configs +.rspec-migration-parallel: + parallel: 12 + +.rspec-ee-migration-parallel: + parallel: 4 + +.rspec-unit-parallel: + parallel: 28 + +.rspec-ee-unit-parallel: + parallel: 18 + +.rspec-integration-parallel: + parallel: 12 + +.rspec-ee-integration-parallel: + parallel: 6 + +.rspec-system-parallel: + parallel: 28 + +.rspec-ee-system-parallel: + parallel: 10 +# rspec job parallel configs +############################ diff --git a/.gitlab/ci/releases.gitlab-ci.yml b/.gitlab/ci/releases.gitlab-ci.yml index 77f23814f3c..df7b07f5545 100644 --- a/.gitlab/ci/releases.gitlab-ci.yml +++ b/.gitlab/ci/releases.gitlab-ci.yml @@ -4,7 +4,7 @@ .merge-train-sync: # We don't need/want any global before/after commands, so we overwrite these # settings. - image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:edge stage: sync before_script: - apk add --no-cache --update curl bash jq diff --git a/.gitlab/ci/review-apps/dast-api.gitlab-ci.yml b/.gitlab/ci/review-apps/dast-api.gitlab-ci.yml new file mode 100644 index 00000000000..e2f32f120af --- /dev/null +++ b/.gitlab/ci/review-apps/dast-api.gitlab-ci.yml @@ -0,0 +1,14 @@ +include: + - template: DAST-API.gitlab-ci.yml + +dast_api: + variables: + DAST_API_PROFILE: Passive + DAST_API_GRAPHQL: /api/graphql + DAST_API_TARGET_URL: ${CI_ENVIRONMENT_URL} + DAST_API_OVERRIDES_ENV: "{\"headers\":{\"Authorization\":\"Bearer $REVIEW_APPS_ROOT_TOKEN\"}}" + needs: ["review-deploy"] + # Uncomment resource_group if DAST_API_PROFILE is changed to an active scan + # resource_group: dast_api_scan + rules: + - !reference [".reports:rules:schedule-dast", rules] diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index e28ffc82811..d3f5d014464 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -1,3 +1,6 @@ +default: + interruptible: true + stages: - prepare - deploy @@ -11,17 +14,25 @@ include: - local: .gitlab/ci/review-apps/rules.gitlab-ci.yml - local: .gitlab/ci/review-apps/qa.gitlab-ci.yml - local: .gitlab/ci/review-apps/dast.gitlab-ci.yml + - local: .gitlab/ci/review-apps/dast-api.gitlab-ci.yml .base-before_script: &base-before_script - source ./scripts/utils.sh - source ./scripts/review_apps/review-apps.sh - apt-get update && apt-get install -y jq +dont-interrupt-me: + extends: .rules:dont-interrupt + stage: prepare + interruptible: false + script: + - echo "This jobs makes sure this pipeline won't be interrupted! See https://docs.gitlab.com/ee/ci/yaml/#interruptible." + review-build-cng-env: extends: - .default-retry - .review:rules:review-build-cng - image: ${GITLAB_DEPENDENCY_PROXY}ruby:3.0-alpine3.13 + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:3.0-alpine3.13 stage: prepare needs: [] before_script: @@ -90,6 +101,10 @@ review-deploy: - .review:rules:review-deploy stage: deploy needs: ["review-build-cng"] + cache: + key: "review-deploy-dependencies-charts-${GITLAB_HELM_CHART_REF}-v1" + paths: + - "gitlab-${GITLAB_HELM_CHART_REF}" before_script: - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) @@ -100,7 +115,6 @@ review-deploy: script: - check_kube_domain - download_chart - - date - deploy || (display_deployment_debug && exit 1) - verify_deploy || exit 1 - disable_sign_ups || (delete_release && exit 1) diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml index 21e6a8e42fb..0214f5ef3f2 100644 --- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml @@ -1,7 +1,8 @@ include: - project: gitlab-org/quality/pipeline-common - ref: 1.2.2 + ref: 1.3.0 file: + - /ci/base.gitlab-ci.yml - /ci/allure-report.yml - /ci/knapsack-report.yml @@ -58,7 +59,7 @@ include: download-knapsack-report: extends: - .bundle-base - - .rules:app-or-qa-framework-changes-or-review-scenarios + - .rules:prepare-report stage: prepare script: - bundle exec rake "knapsack:download[qa]" @@ -134,7 +135,7 @@ review-performance: e2e-test-report: extends: - .generate-allure-report-base - - .rules:app-or-qa-framework-changes-or-review-scenarios + - .rules:prepare-report stage: post-qa variables: ALLURE_JOB_NAME: e2e-review-qa @@ -162,7 +163,7 @@ upload-knapsack-report: delete-test-resources: extends: - .bundle-base - - .rules:app-or-qa-framework-changes-or-review-scenarios + - .rules:prepare-report stage: post-qa variables: QA_TEST_RESOURCES_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-resources-*.json @@ -172,3 +173,21 @@ delete-test-resources: - bundle exec rake "test_resources:delete[$QA_TEST_RESOURCES_FILE_PATTERN]" allow_failure: true when: always + +notify-slack: + extends: + - .notify-slack-qa + - .qa-cache + - .rules:notify-slack + stage: post-qa + variables: + RUN_WITH_BUNDLE: "true" + QA_PATH: qa + ALLURE_JOB_NAME: e2e-review-qa + SLACK_ICON_EMOJI: ci_failing + STATUS_SYM: ☠️ + STATUS: failed + TYPE: "(review-app) " + script: + - bundle exec gitlab-qa-report --prepare-stage-reports "$CI_PROJECT_DIR/qa/tmp/rspec-*.xml" # generate summary + - !reference [.notify-slack-qa, script] diff --git a/.gitlab/ci/review-apps/rules.gitlab-ci.yml b/.gitlab/ci/review-apps/rules.gitlab-ci.yml index 56d3731bb56..4e07f381bc9 100644 --- a/.gitlab/ci/review-apps/rules.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/rules.gitlab-ci.yml @@ -19,6 +19,15 @@ .qa-framework-changes: &qa-framework-changes if: $QA_FRAMEWORK_CHANGES == "true" +.default-branch: &default-branch + if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + +.qa-manual: &qa-manual + when: manual + allow_failure: true + variables: + QA_TESTS: "" + .never-when-qa-framework-changes-or-no-specific-specs: - <<: *qa-framework-changes when: never @@ -27,10 +36,24 @@ .never-when-specific-specs-always-when-qa-framework-changes: - <<: *specific-specs - when: never + when: manual + allow_failure: true + variables: + QA_TESTS: "" - *qa-framework-changes # ------------------------------------------ +# Prepare +# ------------------------------------------ +.rules:dont-interrupt: + rules: + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + allow_failure: true + - if: $CI_MERGE_REQUEST_IID + when: manual + allow_failure: true + +# ------------------------------------------ # Test # ------------------------------------------ .rules:qa-smoke: @@ -41,6 +64,7 @@ QA_TESTS: "" # unset QA_TESTS even if specific tests were inferred from stage label - *qa-framework-changes - if: $QA_SUITES =~ /Test::Instance::Smoke/ + - *qa-manual .rules:qa-blocking: rules: @@ -70,12 +94,11 @@ # ------------------------------------------ # Prepare/Report # ------------------------------------------ -# if no rules for test execution are matched, pipeline will not have e2e test jobs -# so we need to skip knapsack, allure and test resource deletion jobs as well -.rules:app-or-qa-framework-changes-or-review-scenarios: +.rules:prepare-report: rules: - - *app-changes - - *qa-framework-changes - - if: $QA_SUITES =~ /Test::Instance::Smoke/ - - if: $QA_SUITES =~ /Test::Instance::ReviewBlocking/ - - if: $QA_SUITES =~ /Test::Instance::ReviewNonBlocking/ + - when: always + +.rules:notify-slack: + rules: + - <<: *default-branch + when: on_failure diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 46e62829394..aefa96da159 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,6 +1,3 @@ -include: - - remote: 'https://gitlab.com/gitlab-org/modelops/applied-ml/review-recommender/ci-templates/-/raw/v0.2.1/recommender/Reviewers.gitlab-ci.yml' - review-cleanup: extends: - .default-retry @@ -20,7 +17,7 @@ review-cleanup: script: - delete_release - delete_namespace - - ruby -rrubygems scripts/review_apps/automated_cleanup.rb + - scripts/review_apps/automated_cleanup.rb - gcp_cleanup start-review-app-pipeline: @@ -72,14 +69,14 @@ review-app-test-results: danger-review: extends: - .default-retry - - .danger-review-cache + - .ruby-node-cache - .review:rules:danger stage: test needs: [] before_script: - source scripts/utils.sh - bundle_install_script "--with danger" - - run_timed_command "retry yarn install --frozen-lockfile" + - yarn_install_script script: # ${DANGER_DANGERFILE} is used by Jihulab for customizing danger support: https://jihulab.com/gitlab-cn/gitlab/-/blob/main-jh/jh/.gitlab-ci.yml - > @@ -96,12 +93,3 @@ danger-review-local: - .review:rules:danger-local script: - run_timed_command danger_as_local - -reviewers-recommender: - extends: - - .default-retry - # extends generated values cannot overwrite values from included files - # Use !reference as a workaround here - rules: !reference [".review:rules:reviewers-recommender", rules] - stage: test - needs: [] diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 9dc2f5eff23..afe900f39a6 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -31,11 +31,14 @@ .if-merge-request: &if-merge-request if: '$CI_MERGE_REQUEST_IID' +# Once https://gitlab.com/gitlab-org/gitlab/-/issues/373904 is implemented, we should be able to change this back to +# if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVALS_COUNT > 0' +# or any similar condition to check that the MR has *any* approval (not just required approval). .if-merge-request-approved: &if-merge-request-approved - if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVED' + if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-full-rspec/' .if-merge-request-not-approved: &if-merge-request-not-approved - if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_APPROVED != "true"' + if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_LABELS !~ /pipeline:run-full-rspec/' .if-automated-merge-request: &if-automated-merge-request if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME == "release-tools/update-gitaly" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /stable-ee$/' @@ -70,9 +73,6 @@ .if-merge-request-labels-skip-undercoverage: &if-merge-request-labels-skip-undercoverage if: '$CI_MERGE_REQUEST_LABELS =~ /pipeline:skip-undercoverage/' -.if-merge-request-labels-community-contribution: &if-merge-request-labels-community-contribution - if: '$CI_MERGE_REQUEST_LABELS =~ /Community contribution/' - .if-merge-request-labels-jh-contribution: &if-merge-request-labels-jh-contribution if: '$CI_MERGE_REQUEST_LABELS =~ /JiHu contribution/' @@ -130,6 +130,9 @@ .if-dot-com-gitlab-org-and-security-tag: &if-dot-com-gitlab-org-and-security-tag if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_COMMIT_TAG' +.if-ruby3-branch: &if-ruby3-branch + if: '$CI_COMMIT_BRANCH == "ruby3"' + # For Security merge requests, the gitlab-release-tools-bot triggers a new # pipeline for the "Pipelines for merged results" feature. If the pipeline # fails, we notify release managers. @@ -179,6 +182,7 @@ - "GITLAB_WORKHORSE_VERSION" - "workhorse/**/*" - ".gitlab/ci/workhorse.gitlab-ci.yml" + - "spec/support/gitlab-git-test.git/**/*" .yaml-lint-patterns: &yaml-lint-patterns - "*.yml" @@ -246,8 +250,21 @@ - "babel.config.js" - "config/webpack.config.js" - "config/**/*.js" + - "{,ee/,jh/}app/assets/**/*" - "vendor/assets/**/*" + +# This list should match the list in Tasks::Gitlab::Assets.assets_impacting_compilation +.assets-compilation-patterns: &assets-compilation-patterns + - "{package.json,yarn.lock}" + - ".browserslistrc" + - "babel.config.js" + - "config/webpack.config.js" + - "*.js" + - "config/**/*.js" + - "locale/**/gitlab.po" - "{,ee/,jh/}app/assets/**/*" + - "fixtures/emojis/**/*" + - "vendor/assets/**/*" .frontend-patterns-for-as-if-foss: &frontend-patterns-for-as-if-foss - "{package.json,yarn.lock}" @@ -516,7 +533,7 @@ - "vendor/assets/javascripts/**/*" .feature-flag-development-config-patterns: &feature-flag-development-config-patterns - - "{,ee/}config/feature_flags/{development,ops}/*.yml" + - "{,ee/,jh/}config/feature_flags/{development,ops}/*.yml" ################## # Conditions set # @@ -532,9 +549,6 @@ rules: - <<: *if-merge-request-approved when: never - # Temporarily disabled minimal rspec jobs before and after approval because of https://gitlab.com/gitlab-org/gitlab/-/issues/373064. - - <<: *if-merge-request-not-approved - when: never - <<: *if-automated-merge-request when: never - <<: *if-security-merge-request @@ -550,12 +564,12 @@ - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *core-backend-patterns - - <<: *if-merge-request - changes: *ci-patterns - <<: *if-automated-merge-request changes: *backend-patterns - <<: *if-security-merge-request changes: *backend-patterns + - <<: *if-merge-request-not-approved + when: never .rails:rules:as-if-foss-migration-unit-integration:minimal-default-rules: rules: @@ -579,12 +593,12 @@ changes: *core-backend-patterns - <<: *if-merge-request changes: *workhorse-patterns - - <<: *if-merge-request - changes: *ci-patterns - <<: *if-automated-merge-request changes: *code-backstage-patterns - <<: *if-security-merge-request changes: *code-backstage-patterns + - <<: *if-merge-request-not-approved + when: never .rails:rules:system:minimal-default-rules: rules: @@ -629,8 +643,6 @@ ###################### .build-images:rules:build-qa-image: rules: - - <<: *if-not-ee - when: never - <<: *if-not-canonical-namespace when: never - <<: *if-merge-request-targeting-stable-branch @@ -643,6 +655,7 @@ - <<: *if-default-branch-or-tag - <<: *if-dot-com-gitlab-org-schedule - <<: *if-force-ci + - <<: *if-ruby3-branch .build-images:rules:build-assets-image: rules: @@ -651,9 +664,60 @@ - <<: *if-merge-request-targeting-stable-branch - <<: *if-merge-request-labels-run-review-app - <<: *if-auto-deploy-branches + - <<: *if-ruby3-branch - changes: *ci-build-images-patterns - changes: *code-qa-patterns +################# +# Caching rules # +################# +.caching:rules:cache-workhorse: + rules: + # That would run for any project that has a "maintenance" pipeline schedule + # but in fact, the cache package is only uploaded for gitlab.com/gitlab-org/gitlab and jihulab.com/gitlab-cn/gitlab + - <<: *if-default-branch-schedule-maintenance + - <<: *if-dot-com-gitlab-org-default-branch + changes: ["workhorse/**/*"] + - <<: *if-dot-com-gitlab-org-merge-request + changes: + - ".gitlab/ci/caching.gitlab-ci.yml" + - "scripts/gitlab_component_helpers.sh" + when: manual + allow_failure: true + +.caching:rules:cache-assets: + rules: + # The new strategy to cache assets as generic packages is experimental and can be disabled by removing the `CACHE_ASSETS_AS_PACKAGE` variable + - if: '$CACHE_ASSETS_AS_PACKAGE != "true"' + when: never + # That would run for any project that has a "maintenance" pipeline schedule + # but in fact, the cache package is only uploaded for gitlab.com/gitlab-org/gitlab and jihulab.com/gitlab-cn/gitlab + - <<: *if-default-branch-schedule-maintenance + - <<: *if-dot-com-gitlab-org-default-branch + changes: *assets-compilation-patterns + - <<: *if-dot-com-gitlab-org-merge-request + changes: + - ".gitlab/ci/caching.gitlab-ci.yml" + - "scripts/gitlab_component_helpers.sh" + - "lib/tasks/gitlab/assets.rake" + when: manual + allow_failure: true + +.caching:rules:packages-cleanup: + rules: + # The new strategy to cache assets as generic packages is experimental and can be disabled by removing the `CACHE_ASSETS_AS_PACKAGE` variable + - if: '$CACHE_ASSETS_AS_PACKAGE != "true"' + when: never + # That would run for any project that has a "maintenance" pipeline schedule + # but in fact, the cache package is only uploaded for gitlab.com/gitlab-org/gitlab and jihulab.com/gitlab-cn/gitlab + - <<: *if-default-branch-schedule-maintenance + - <<: *if-dot-com-gitlab-org-merge-request + changes: + - ".gitlab/ci/caching.gitlab-ci.yml" + - "scripts/packages/automated_cleanup.rb" + when: manual + allow_failure: true + ###################### # CI Templates Rules # ###################### @@ -768,8 +832,6 @@ - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *startup-css-patterns - - <<: *if-merge-request - changes: *ci-patterns .frontend:rules:frontend_fixture-as-if-foss: rules: @@ -906,8 +968,6 @@ changes: *code-qa-patterns - <<: *if-merge-request-labels-as-if-foss - <<: *if-merge-request-labels-run-all-rspec - - <<: *if-merge-request - changes: *ci-patterns .qa:rules:internal-as-if-foss: rules: @@ -928,6 +988,7 @@ allow_failure: true - <<: *if-force-ci allow_failure: true + - <<: *if-ruby3-branch .qa:rules:package-and-test: rules: @@ -939,6 +1000,8 @@ when: never - <<: *if-merge-request-targeting-stable-branch allow_failure: true + - <<: *if-ruby3-branch + allow_failure: true - <<: *if-dot-com-gitlab-org-and-security-merge-request-manual-ff-package-and-e2e changes: *feature-flag-development-config-patterns when: manual @@ -975,16 +1038,6 @@ ############### # Rails rules # ############### -.rails:rules:build-components: - rules: - - <<: *if-dot-com-ee-schedule - - <<: *if-dot-com-gitlab-org-default-branch - changes: - - "workhorse/**/*" - - <<: *if-dot-com-gitlab-org-merge-request - when: manual - allow_failure: true - .rails:rules:setup-test-env: rules: - changes: *setup-test-env-patterns @@ -1009,8 +1062,6 @@ - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *core-backend-patterns - - <<: *if-merge-request - changes: *ci-patterns # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request @@ -1019,6 +1070,8 @@ changes: *db-patterns - <<: *if-security-merge-request changes: *db-patterns + - <<: *if-merge-request-not-approved + when: never - changes: *db-patterns .rails:rules:ee-and-foss-migration:minimal: @@ -1112,8 +1165,6 @@ - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *core-backend-patterns - - <<: *if-merge-request - changes: *ci-patterns # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request @@ -1122,6 +1173,8 @@ changes: *db-patterns - <<: *if-security-merge-request changes: *db-patterns + - <<: *if-merge-request-not-approved + when: never - changes: *db-patterns .rails:rules:ee-only-migration:minimal: @@ -1201,8 +1254,6 @@ - <<: *if-merge-request-labels-run-all-rspec - <<: *if-merge-request changes: *core-backend-patterns - - <<: *if-merge-request - changes: *ci-patterns # When DB schema changes, many migrations spec may be affected. However, the test mapping from Crystalball does not map db change to a specific migration spec well. # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68840. - <<: *if-merge-request-labels-as-if-foss @@ -1211,6 +1262,8 @@ changes: *db-patterns - <<: *if-security-merge-request changes: *db-patterns + - <<: *if-merge-request-not-approved + when: never .rails:rules:as-if-foss-migration:minimal: rules: @@ -1416,9 +1469,6 @@ - if: '$SKIP_FLAKY_TESTS_AUTOMATICALLY == "true" || $RETRY_FAILED_TESTS_IN_NEW_PROCESS == "true"' changes: *code-backstage-patterns when: always - - if: '$SKIP_FLAKY_TESTS_AUTOMATICALLY == "true" || $RETRY_FAILED_TESTS_IN_NEW_PROCESS == "true"' - changes: *ci-patterns - when: always ######################### # Static analysis rules # @@ -1482,6 +1532,12 @@ changes: ["vendor/gems/mail-smtp_pool/**/*"] - <<: *if-merge-request-labels-run-all-rspec +.vendor:rules:attr_encrypted: + rules: + - <<: *if-merge-request + changes: ["vendor/gems/attr_encrypted/**/*"] + - <<: *if-merge-request-labels-run-all-rspec + .vendor:rules:microsoft_graph_mailer: rules: - <<: *if-merge-request @@ -1518,12 +1574,6 @@ changes: ["vendor/gems/omniauth-gitlab/**/*"] - <<: *if-merge-request-labels-run-all-rspec -.vendor:rules:omniauth-google-oauth2: - rules: - - <<: *if-merge-request - changes: ["vendor/gems/omniauth-google-oauth2/**/*"] - - <<: *if-merge-request-labels-run-all-rspec - .vendor:rules:omniauth-salesforce: rules: - <<: *if-merge-request @@ -1660,7 +1710,7 @@ rules: - if: '$LICENSE_MANAGEMENT_DISABLED || $GITLAB_FEATURES !~ /\blicense_scanning\b/' when: never - - changes: *code-backstage-qa-patterns + - changes: *dependency-patterns ################ # Review rules # @@ -1783,14 +1833,6 @@ - <<: *if-merge-request changes: *danger-patterns -.review:rules:reviewers-recommender: - rules: - - <<: *if-not-canonical-namespace - when: never - - <<: *if-merge-request-labels-community-contribution - when: never - - <<: *if-merge-request - ############### # Setup rules # ############### diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 4f3111de2bf..e417b054cd6 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -30,7 +30,7 @@ cache gems: .absolutely-minimal-job: extends: - .minimal-job - image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:edge variables: GIT_STRATEGY: none @@ -79,7 +79,7 @@ verify-ruby-2.7: verify-tests-yml: extends: - .setup:rules:verify-tests-yml - image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine3.13 + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-alpine3.13 stage: test needs: [] script: @@ -116,7 +116,7 @@ generate-frontend-fixtures-mapping: detect-tests: extends: .rails:rules:detect-tests - image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION} + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION} needs: [] stage: prepare variables: @@ -134,14 +134,17 @@ detect-tests: tooling/bin/find_changes ${RSPEC_CHANGED_FILES_PATH}; tooling/bin/find_tests ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_TESTS_PATH}; tooling/bin/find_changes ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_TESTS_PATH} ${FRONTEND_FIXTURES_MAPPING_PATH}; - echo "Changed files: $(cat $RSPEC_CHANGED_FILES_PATH)"; - echo "Related rspec tests: $(cat $RSPEC_MATCHING_TESTS_PATH)"; + filter_rspec_matched_foss_tests ${RSPEC_MATCHING_TESTS_PATH} ${RSPEC_MATCHING_TESTS_FOSS_PATH}; + echoinfo "Changed files: $(cat $RSPEC_CHANGED_FILES_PATH)"; + echoinfo "Related RSpec tests: $(cat $RSPEC_MATCHING_TESTS_PATH)"; + echoinfo "Related FOSS RSpec tests: $(cat $RSPEC_MATCHING_TESTS_FOSS_PATH)"; fi artifacts: expire_in: 7d paths: - ${RSPEC_CHANGED_FILES_PATH} - ${RSPEC_MATCHING_TESTS_PATH} + - ${RSPEC_MATCHING_TESTS_FOSS_PATH} - ${FRONTEND_FIXTURES_MAPPING_PATH} detect-previous-failed-tests: diff --git a/.gitlab/ci/static-analysis.gitlab-ci.yml b/.gitlab/ci/static-analysis.gitlab-ci.yml index e845e7eb88b..59ea665ae07 100644 --- a/.gitlab/ci/static-analysis.gitlab-ci.yml +++ b/.gitlab/ci/static-analysis.gitlab-ci.yml @@ -31,7 +31,7 @@ static-analysis: - .static-analysis:rules:static-analysis parallel: 2 script: - - run_timed_command "retry yarn install --frozen-lockfile" + - yarn_install_script - scripts/static-analysis static-analysis as-if-foss: @@ -84,7 +84,7 @@ eslint: variables: USE_BUNDLE_INSTALL: "false" script: - - run_timed_command "retry yarn install --frozen-lockfile" + - yarn_install_script - run_timed_command "yarn run lint:eslint:all" eslint as-if-foss: diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index f4fa39300b6..e147305e25a 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,5 +1,5 @@ .tests-metadata-state: - image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION} + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION} before_script: - source scripts/utils.sh artifacts: diff --git a/.gitlab/ci/vendored-gems.gitlab-ci.yml b/.gitlab/ci/vendored-gems.gitlab-ci.yml index 577bd37ca9e..a22ac5337f6 100644 --- a/.gitlab/ci/vendored-gems.gitlab-ci.yml +++ b/.gitlab/ci/vendored-gems.gitlab-ci.yml @@ -6,6 +6,14 @@ vendor mail-smtp_pool: include: vendor/gems/mail-smtp_pool/.gitlab-ci.yml strategy: depend +vendor attr_encrypted: + extends: + - .vendor:rules:attr_encrypted + needs: [] + trigger: + include: vendor/gems/attr_encrypted/.gitlab-ci.yml + strategy: depend + vendor microsoft_graph_mailer: extends: - .vendor:rules:microsoft_graph_mailer @@ -54,14 +62,6 @@ vendor omniauth-gitlab: include: vendor/gems/omniauth-gitlab/.gitlab-ci.yml strategy: depend -vendor omniauth-google-oauth2: - extends: - - .vendor:rules:omniauth-google-oauth2 - needs: [] - trigger: - include: vendor/gems/omniauth-google-oauth2/.gitlab-ci.yml - strategy: depend - vendor omniauth-salesforce: extends: - .vendor:rules:omniauth-salesforce diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml index efd37b2247b..a11d5f000cf 100644 --- a/.gitlab/ci/workhorse.gitlab-ci.yml +++ b/.gitlab/ci/workhorse.gitlab-ci.yml @@ -1,9 +1,10 @@ workhorse:verify: extends: .workhorse:rules:workhorse - image: ${GITLAB_DEPENDENCY_PROXY}golang:1.17 + image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}golang:${GO_VERSION} stage: test needs: [] script: + - go version - make -C workhorse # test build - make -C workhorse verify @@ -12,7 +13,6 @@ workhorse:verify: image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}:git-2.36 variables: GITALY_ADDRESS: "tcp://127.0.0.1:8075" - GO_VERSION: "1.17" stage: test needs: - setup-test-env @@ -27,7 +27,7 @@ workhorse:test go: extends: .workhorse:test parallel: matrix: - - GO_VERSION: ["1.17", "1.18"] + - GO_VERSION: ["1.17", "1.18", "1.19"] script: - make -C workhorse test-coverage coverage: '/\d+.\d+%/' |