diff options
Diffstat (limited to '.gitlab/ci')
| -rw-r--r-- | .gitlab/ci/build-images.gitlab-ci.yml | 12 | ||||
| -rw-r--r-- | .gitlab/ci/cache-repo.gitlab-ci.yml | 8 | ||||
| -rw-r--r-- | .gitlab/ci/cng.gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | .gitlab/ci/dast.gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | .gitlab/ci/dev-fixtures.gitlab-ci.yml | 4 | ||||
| -rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 7 | ||||
| -rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 35 | ||||
| -rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 166 | ||||
| -rw-r--r-- | .gitlab/ci/memory.gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | .gitlab/ci/pages.gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 11 | ||||
| -rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 304 | ||||
| -rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 146 | ||||
| -rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 35 | ||||
| -rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 51 | ||||
| -rw-r--r-- | .gitlab/ci/test-metadata.gitlab-ci.yml | 22 |
16 files changed, 417 insertions, 392 deletions
diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index 4e352472047..ed1f71e27bb 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -9,8 +9,18 @@ build-qa-image: - .build-images:rules:build-qa-image stage: build-images needs: [] + variables: + QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" script: - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" + # With .git/hooks/post-checkout in place, Git tries to pull LFS objects, but the image doesn't have Git LFS, and we actually don't care about it for this specific so we just remove the file. + # Without removing the file, the error is as follows: "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout." + - rm .git/hooks/post-checkout + # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync. + # This falls back to $CI_COMMIT_SHA (the default checked out commit) for the non-merged result pipelines. + # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results. + - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then + git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; + fi - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --cache=true retry: 2 diff --git a/.gitlab/ci/cache-repo.gitlab-ci.yml b/.gitlab/ci/cache-repo.gitlab-ci.yml index 475cbca3156..98c8c72ae3a 100644 --- a/.gitlab/ci/cache-repo.gitlab-ci.yml +++ b/.gitlab/ci/cache-repo.gitlab-ci.yml @@ -28,9 +28,9 @@ cache-repo: before_script: - '[ -z "$CI_REPO_CACHE_CREDENTIALS" ] || gcloud auth activate-service-account --key-file=$CI_REPO_CACHE_CREDENTIALS' script: - # Enable shallow repo caching only if the $ENABLE_SHALLOW_REPO_CACHING variable exists + # Enable shallow repo caching unless the $DISABLE_SHALLOW_REPO_CACHING variable exists (in the case the shallow clone caching isn't working well) # The `git repack` call works around a Git bug with shallow clones: https://gitlab.com/gitlab-org/git/-/issues/86 - - if [ -n "$ENABLE_SHALLOW_REPO_CACHING" ]; then + - if [ -z "$DISABLE_SHALLOW_REPO_CACHING" ]; then cd .. && rm -rf $CI_PROJECT_NAME; today=$(date +%Y-%m-%d); year=$(date +%Y); @@ -47,8 +47,8 @@ cache-repo: time gzip /tmp/$SHALLOW_CLONE_TAR_FILENAME; [ -z "$CI_REPO_CACHE_CREDENTIALS" ] || (echo "Uploading /tmp/$SHALLOW_CLONE_TAR_FILENAME.gz to GCloud." && time gsutil cp /tmp/$SHALLOW_CLONE_TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/$SHALLOW_CLONE_TAR_FILENAME.gz); fi - # By default, we want to cache the full repo, unless the $DISABLE_FULL_REPO_CACHING variable exists (in the case the shallow clone caching is working well) - - if [ -z "$DISABLE_FULL_REPO_CACHING" ]; then + # Disable the full repo caching unless the $DISABLE_SHALLOW_REPO_CACHING variable exists (in the case the shallow clone caching isn't working well) + - if [ -n "$DISABLE_SHALLOW_REPO_CACHING" ]; then cd .. && rm -rf $CI_PROJECT_NAME; echo "Cloning $CI_REPOSITORY_URL into $CI_PROJECT_NAME."; time git clone --progress $CI_REPOSITORY_URL $CI_PROJECT_NAME; diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index af735d3212a..f75497c1a3f 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -7,4 +7,4 @@ cloud-native-image: GIT_DEPTH: "1" script: - install_gitlab_gem - - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng + - CNG_PROJECT_PATH="gitlab-org/build/CNG" ./scripts/trigger-build cng diff --git a/.gitlab/ci/dast.gitlab-ci.yml b/.gitlab/ci/dast.gitlab-ci.yml index a8a201bd1fa..309714f8739 100644 --- a/.gitlab/ci/dast.gitlab-ci.yml +++ b/.gitlab/ci/dast.gitlab-ci.yml @@ -3,7 +3,7 @@ - prm # For scheduling dast job extends: - - .reports:schedule-dast + - .reports:rules:schedule-dast image: name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" resource_group: dast_scan diff --git a/.gitlab/ci/dev-fixtures.gitlab-ci.yml b/.gitlab/ci/dev-fixtures.gitlab-ci.yml index 1848283f921..21eae3f23e9 100644 --- a/.gitlab/ci/dev-fixtures.gitlab-ci.yml +++ b/.gitlab/ci/dev-fixtures.gitlab-ci.yml @@ -3,7 +3,7 @@ - .default-retry - .rails-cache - .default-before_script - - .use-pg11 + - .use-pg12 stage: test needs: ["setup-test-env"] variables: @@ -29,7 +29,7 @@ run-dev-fixtures-ee: extends: - .run-dev-fixtures - .dev-fixtures:rules:ee-only - - .use-pg11-ee + - .use-pg12-ee script: - cp ee/db/fixtures/development/* $FIXTURE_PATH - *run-dev-fixtures-script diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 8f70127be65..35d462584ee 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -44,7 +44,7 @@ docs-lint markdown: - .default-retry - .docs:rules:docs-lint # When updating the image version here, update it in /scripts/lint-doc.sh too. - image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.12-vale-2.8.0-markdownlint-0.26.0" + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.13-vale-2.10.2-markdownlint-0.26.0 stage: test needs: [] script: @@ -52,9 +52,10 @@ docs-lint markdown: docs-lint links: extends: - - .default-retry - .docs:rules:docs-lint - image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.12-ruby-2.7.2" + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.13-ruby-2.7.2 + # TODO: revert to .default-retry when https://gitlab.com/gitlab-org/gitlab/-/issues/331002 is fixed. + retry: 2 stage: test needs: [] script: diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index c39f5cdce9c..33aab8554e7 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -58,38 +58,34 @@ compile-test-assets as-if-foss: update-assets-compile-production-cache: extends: - compile-production-assets + - .assets-compile-cache-push - .shared:rules:update-cache stage: prepare artifacts: {} # This job's purpose is only to update the cache. - cache: - policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. update-assets-compile-test-cache: extends: - compile-test-assets + - .assets-compile-cache-push - .shared:rules:update-cache stage: prepare artifacts: {} # This job's purpose is only to update the cache. - cache: - policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. update-yarn-cache: extends: - .default-retry - - .yarn-cache + - .yarn-cache-push - .shared:rules:update-cache stage: prepare script: - *yarn-install - cache: - policy: push .frontend-fixtures-base: extends: - .default-retry - .default-before_script - .rails-cache - - .use-pg11 + - .use-pg12 stage: fixtures needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"] variables: @@ -121,7 +117,7 @@ rspec frontend_fixture as-if-foss: rspec-ee frontend_fixture: extends: - .frontend-fixtures-base - - .frontend:rules:default-frontend-jobs + - .frontend:rules:default-frontend-jobs-ee parallel: 2 graphql-schema-dump: @@ -156,7 +152,7 @@ eslint-as-if-foss: needs: [] script: - *yarn-install - - run_timed_command "yarn run eslint" + - run_timed_command "yarn run lint:eslint:all" .karma-base: extends: .frontend-test-base @@ -169,8 +165,10 @@ karma: extends: - .karma-base - .frontend:rules:default-frontend-jobs - # Don't use `needs` since `rspec-ee frontend_fixture` doesn't exist in `gitlab-foss` pipelines. - dependencies: ["rspec frontend_fixture", "rspec-ee frontend_fixture"] + needs: + - job: "rspec frontend_fixture" + - job: "rspec-ee frontend_fixture" + optional: true coverage: '/^Statements *: (\d+\.\d+%)/' artifacts: name: coverage-javascript @@ -201,8 +199,10 @@ jest: extends: - .jest-base - .frontend:rules:default-frontend-jobs - # Don't use `needs` since `rspec-ee frontend_fixture` doesn't exist in `gitlab-foss` pipelines. - dependencies: ["rspec frontend_fixture", "rspec-ee frontend_fixture"] + needs: + - job: "rspec frontend_fixture" + - job: "rspec-ee frontend_fixture" + optional: true artifacts: name: coverage-frontend expire_in: 31d @@ -222,8 +222,11 @@ jest-integration: script: - *yarn-install - run_timed_command "yarn jest:integration --ci" - # Don't use `needs` since `rspec-ee frontend_fixture` doesn't exist in `gitlab-foss` pipelines. - dependencies: ["rspec frontend_fixture", "rspec-ee frontend_fixture", "graphql-schema-dump"] + needs: + - job: "rspec frontend_fixture" + - job: "rspec-ee frontend_fixture" + optional: true + - job: "graphql-schema-dump" jest-as-if-foss: extends: diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 891457afe6e..489b02fe2f6 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -16,75 +16,147 @@ - source scripts/utils.sh - source scripts/prepare_build.sh +.ruby-gems-cache: &ruby-gems-cache + key: "ruby-gems-v1" + paths: + - vendor/ruby/ + policy: pull + +.ruby-gems-cache-push: &ruby-gems-cache-push + <<: *ruby-gems-cache + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + +.gitaly-ruby-gems-cache: &gitaly-ruby-gems-cache + key: "gitaly-ruby-gems-v1" + paths: + - vendor/gitaly-ruby/ + policy: pull + +.gitaly-ruby-gems-cache-push: &gitaly-ruby-gems-cache-push + <<: *gitaly-ruby-gems-cache + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + +.go-pkg-cache: &go-pkg-cache + key: "go-pkg-v1" + paths: + - .go/pkg/mod/ + policy: pull + +.go-pkg-cache-push: &go-pkg-cache-push + <<: *go-pkg-cache + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + +.node-modules-cache: &node-modules-cache + key: "node-modules-${NODE_ENV}-v1" + paths: + - node_modules/ + - tmp/cache/webpack-dlls/ + policy: pull + +.node-modules-cache-push: &node-modules-cache-push + <<: *node-modules-cache + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + +.assets-cache: &assets-cache + key: "assets-${NODE_ENV}-v1" + paths: + - assets-hash.txt + - public/assets/webpack/ + - tmp/cache/assets/sprockets/ + - tmp/cache/babel-loader/ + - tmp/cache/vue-loader/ + policy: pull + +.assets-cache-push: &assets-cache-push + <<: *assets-cache + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + +.rubocop-cache: &rubocop-cache + key: "rubocop-v1" + paths: + - tmp/rubocop_cache/ + policy: pull + +.rubocop-cache-push: &rubocop-cache-push + <<: *rubocop-cache + # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism + # for keeping only the N latest cache files, so we take advantage of it with `pull-push`. + policy: pull-push + +.qa-ruby-gems-cache: &qa-ruby-gems-cache + key: "qa-ruby-gems-v1" + paths: + - qa/vendor/ruby/ + policy: pull + +.qa-ruby-gems-cache-push: &qa-ruby-gems-cache-push + <<: *qa-ruby-gems-cache + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + .setup-test-env-cache: cache: - key: "setup-test-env-v1" - paths: - - vendor/ruby/ - - vendor/gitaly-ruby/ - - .go/pkg/mod/ - policy: pull + - *ruby-gems-cache + - *gitaly-ruby-gems-cache + - *go-pkg-cache + +.setup-test-env-cache-push: + cache: + - *ruby-gems-cache-push + - *gitaly-ruby-gems-cache-push + - *go-pkg-cache-push .rails-cache: cache: - key: "rails-v5" - paths: - - vendor/ruby/ - - vendor/gitaly-ruby/ - policy: pull + - *ruby-gems-cache + - *gitaly-ruby-gems-cache .static-analysis-cache: cache: - key: "static-analysis-v2" - paths: - - vendor/ruby/ - - node_modules/ - - tmp/rubocop_cache/ - policy: pull + - *ruby-gems-cache + - *node-modules-cache + - *rubocop-cache + +.static-analysis-cache-push: + cache: + - *ruby-gems-cache # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache` + - *rubocop-cache-push .coverage-cache: cache: - key: "coverage-cache-v1" - paths: - - vendor/ruby/ - policy: pull + - *ruby-gems-cache .danger-review-cache: cache: - key: "danger-review-v1" - paths: - - vendor/ruby/ - - node_modules/ - policy: pull + - *ruby-gems-cache + - *node-modules-cache .qa-cache: cache: - key: "qa-v2" - paths: - - qa/vendor/ruby/ - policy: pull + - *qa-ruby-gems-cache + +.qa-cache-push: + cache: + - *qa-ruby-gems-cache-push .yarn-cache: cache: - key: "yarn-v1" - paths: - - node_modules/ - - tmp/cache/webpack-dlls/ - policy: pull + - *node-modules-cache + +.yarn-cache-push: + cache: + - *node-modules-cache-push .assets-compile-cache: cache: - key: "assets-compile-${NODE_ENV}-v1" - paths: - - vendor/ruby/ - - node_modules/ - - assets-hash.txt - - public/assets/webpack/ - - tmp/cache/assets/sprockets/ - - tmp/cache/babel-loader/ - - tmp/cache/vue-loader/ - - tmp/cache/webpack-dlls/ - policy: pull + - *ruby-gems-cache + - *node-modules-cache + - *assets-cache + +.assets-compile-cache-push: + cache: + - *ruby-gems-cache # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache` + - *node-modules-cache-push + - *assets-cache-push .use-pg11: image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" @@ -128,7 +200,7 @@ .use-kaniko: image: - name: gcr.io/kaniko-project/executor:debug-v1.3.0 + name: registry.gitlab.com/gitlab-org/gitlab-build-images:kaniko entrypoint: [""] before_script: - source scripts/utils.sh diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index 3e5639e4d69..f3ad8f81da5 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -37,7 +37,7 @@ memory-static: memory-on-boot: extends: - .only-code-memory-job-base - - .use-pg11 + - .use-pg12 stage: test needs: ["setup-test-env", "compile-test-assets"] variables: diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index 4961bd508d3..b6d4b0ef11d 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -3,7 +3,7 @@ pages: - .default-retry - .pages:rules stage: pages - dependencies: + needs: - rspec:coverage - coverage-frontend - karma diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 788b482f0a6..8bbd7dbf075 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -4,11 +4,13 @@ - .qa-cache stage: test needs: [] + variables: + USE_BUNDLE_INSTALL: "false" + SETUP_DB: "false" before_script: - - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' + - !reference [.default-before_script, before_script] - cd qa/ - - bundle install --clean --jobs=$(nproc) --path=vendor --retry=3 --without=development --quiet - - bundle check + - bundle_install_script qa:internal: extends: @@ -39,12 +41,11 @@ qa:selectors-as-if-foss: update-qa-cache: extends: - .qa-job-base + - .qa-cache-push - .shared:rules:update-cache stage: prepare script: - echo "Cache has been updated and ready to be uploaded." - cache: - policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .package-and-qa-base: image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 064aa5a8351..5cd64baf4d3 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -8,7 +8,8 @@ .minimal-bundle-install: script: - - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --without default development test production puma unicorn kerberos metrics omnibus ed25519" + - export BUNDLE_WITHOUT="${BUNDLE_WITHOUT}:default:test:puma:unicorn:kerberos:metrics:omnibus:ed25519" + - bundle_install_script .base-script: script: @@ -67,11 +68,11 @@ - .rspec-base - .use-pg12 -.rspec-base-pg11-as-if-foss: +.rspec-base-pg12-as-if-foss: extends: - .rspec-base - .as-if-foss - - .use-pg11 + - .use-pg12 needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"] .rspec-ee-base-pg11: @@ -104,7 +105,7 @@ extends: - .rails-job-base - .rails:rules:ee-and-foss-migration - - .use-pg11 + - .use-pg12 stage: test needs: ["setup-test-env"] # rspec job base specs @@ -113,10 +114,10 @@ ############################ # rspec job parallel configs .rspec-migration-parallel: - parallel: 5 + parallel: 7 .rspec-ee-migration-parallel: - parallel: 2 + parallel: 3 .rspec-unit-parallel: parallel: 20 @@ -148,7 +149,7 @@ setup-test-env: - .rails-job-base - .setup-test-env-cache - .rails:rules:code-backstage-qa - - .use-pg11 + - .use-pg12 stage: prepare variables: GITLAB_TEST_EAGER_LOAD: "0" @@ -159,18 +160,13 @@ setup-test-env: expire_in: 7d paths: - config/secrets.yml + - tmp/tests/gitaly/_build/bin/ - tmp/tests/gitaly/config.toml - - tmp/tests/gitaly/gitaly - tmp/tests/gitaly/gitaly2.config.toml - - tmp/tests/gitaly/gitaly-git2go - - tmp/tests/gitaly/gitaly-hooks - - tmp/tests/gitaly/gitaly-lfs-smudge - - tmp/tests/gitaly/gitaly-ssh - tmp/tests/gitaly/internal/ - tmp/tests/gitaly/internal_gitaly2/ - tmp/tests/gitaly/internal_sockets/ - tmp/tests/gitaly/Makefile - - tmp/tests/gitaly/praefect - tmp/tests/gitaly/praefect.config.toml - tmp/tests/gitaly/ruby/ - tmp/tests/gitlab-elasticsearch-indexer/bin/gitlab-elasticsearch-indexer @@ -191,18 +187,10 @@ setup-test-env: update-setup-test-env-cache: extends: - setup-test-env + - .setup-test-env-cache-push - .shared:rules:update-cache artifacts: paths: [] # This job's purpose is only to update the cache. - cache: - policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. - -update-rails-cache: - extends: - - update-setup-test-env-cache - - .rails-cache - cache: - policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .coverage-base: extends: @@ -213,16 +201,6 @@ update-rails-cache: SETUP_DB: "false" USE_BUNDLE_INSTALL: "false" -update-coverage-cache: - extends: - - .coverage-base - - .shared:rules:update-cache - stage: prepare - script: - - !reference [.minimal-bundle-install, script] - cache: - policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. - .static-analysis-base: extends: - .default-retry @@ -236,80 +214,81 @@ update-coverage-cache: update-static-analysis-cache: extends: - .static-analysis-base + - .static-analysis-cache-push - .shared:rules:update-cache stage: prepare script: - - rm -rf ./node_modules # We remove node_modules because there's no mechanism to remove stall entries. - - run_timed_command "retry yarn install --frozen-lockfile" - - run_timed_command "bundle exec rubocop --parallel" # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks, - cache: - # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism - # for keeping only the N latest cache files, so we take advantage of it with `pull-push` and removing `node_modules` at the start of the job. - policy: pull-push + - run_timed_command "bundle exec rubocop --parallel" # For the moment we only cache `tmp/rubocop_cache` so we don't need to run all the tasks. static-analysis: extends: - .static-analysis-base - - .rails:rules:code-backstage-qa + - .static-analysis:rules:ee-and-foss stage: test parallel: 4 script: - run_timed_command "retry yarn install --frozen-lockfile" - scripts/static-analysis -rspec migration pg11: +static-analysis as-if-foss: extends: - - .rspec-base-pg11 + - static-analysis + - .static-analysis:rules:as-if-foss + - .as-if-foss + +rspec migration pg12: + extends: + - .rspec-base-pg12 - .rspec-base-migration - .rspec-migration-parallel -rspec migration pg11 minimal: +rspec migration pg12 minimal: extends: - - rspec migration pg11 + - rspec migration pg12 - .minimal-rspec-tests - .rails:rules:ee-and-foss-migration:minimal -rspec unit pg11: +rspec unit pg12: extends: - - .rspec-base-pg11 + - .rspec-base-pg12 - .rails:rules:ee-and-foss-unit - .rspec-unit-parallel -rspec unit pg11 minimal: +rspec unit pg12 minimal: extends: - - rspec unit pg11 + - rspec unit pg12 - .minimal-rspec-tests - .rails:rules:ee-and-foss-unit:minimal -rspec integration pg11: +rspec integration pg12: extends: - - .rspec-base-pg11 + - .rspec-base-pg12 - .rails:rules:ee-and-foss-integration - .rspec-integration-parallel -rspec integration pg11 minimal: +rspec integration pg12 minimal: extends: - - rspec integration pg11 + - rspec integration pg12 - .minimal-rspec-tests - .rails:rules:ee-and-foss-integration:minimal -rspec system pg11: +rspec system pg12: extends: - - .rspec-base-pg11 + - .rspec-base-pg12 - .rails:rules:ee-and-foss-system - .rspec-system-parallel -rspec system pg11 minimal: +rspec system pg12 minimal: extends: - - rspec system pg11 + - rspec system pg12 - .minimal-rspec-tests - .rails:rules:ee-and-foss-system:minimal -# Dedicated job to test DB library code against PG12. -# Note that these are already tested against PG11 in the `rspec unit pg11` / `rspec-ee unit pg11` jobs. -rspec db-library-code pg12: +# Dedicated job to test DB library code against PG11. +# Note that these are already tested against PG12 in the `rspec unit pg12` / `rspec-ee unit pg12` jobs. +rspec db-library-code pg11: extends: - - .rspec-base-pg12 + - .rspec-base-pg11 - .rails:rules:ee-and-foss-db-library-code script: - !reference [.base-script, script] @@ -317,7 +296,7 @@ rspec db-library-code pg12: rspec fast_spec_helper: extends: - - .rspec-base-pg11 + - .rspec-base-pg12 - .rails:rules:ee-and-foss-fast_spec_helper script: - bin/rspec spec/fast_spec_helper.rb @@ -339,6 +318,7 @@ db:check-schema: - .rails:rules:ee-mr-and-default-branch-only script: - source scripts/schema_changed.sh + - scripts/validate_migration_timestamps db:check-migrations: extends: @@ -434,17 +414,17 @@ rspec:deprecations: # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) # so we use `dependencies` here. dependencies: - - rspec migration pg11 - - rspec unit pg11 - - rspec integration pg11 - - rspec system pg11 - - rspec-ee migration pg11 - - rspec-ee unit pg11 - - rspec-ee integration pg11 - - rspec-ee system pg11 - - rspec-ee unit pg11 geo - - rspec-ee integration pg11 geo - - rspec-ee system pg11 geo + - rspec migration pg12 + - rspec unit pg12 + - rspec integration pg12 + - rspec system pg12 + - rspec-ee migration pg12 + - rspec-ee unit pg12 + - rspec-ee integration pg12 + - rspec-ee system pg12 + - rspec-ee unit pg12 geo + - rspec-ee integration pg12 geo + - rspec-ee system pg12 geo variables: SETUP_DB: "false" script: @@ -466,17 +446,17 @@ rspec:coverage: # so we use `dependencies` here. dependencies: - setup-test-env - - rspec migration pg11 - - rspec unit pg11 - - rspec integration pg11 - - rspec system pg11 - - rspec-ee migration pg11 - - rspec-ee unit pg11 - - rspec-ee integration pg11 - - rspec-ee system pg11 - - rspec-ee unit pg11 geo - - rspec-ee integration pg11 geo - - rspec-ee system pg11 geo + - rspec migration pg12 + - rspec unit pg12 + - rspec integration pg12 + - rspec system pg12 + - rspec-ee migration pg12 + - rspec-ee unit pg12 + - rspec-ee integration pg12 + - rspec-ee system pg12 + - rspec-ee unit pg12 geo + - rspec-ee integration pg12 geo + - rspec-ee system pg12 geo - memory-static - memory-on-boot script: @@ -503,17 +483,17 @@ rspec:feature-flags: # so we use `dependencies` here. dependencies: - setup-test-env - - rspec migration pg11 - - rspec unit pg11 - - rspec integration pg11 - - rspec system pg11 - - rspec-ee migration pg11 - - rspec-ee unit pg11 - - rspec-ee integration pg11 - - rspec-ee system pg11 - - rspec-ee unit pg11 geo - - rspec-ee integration pg11 geo - - rspec-ee system pg11 geo + - rspec migration pg12 + - rspec unit pg12 + - rspec integration pg12 + - rspec system pg12 + - rspec-ee migration pg12 + - rspec-ee unit pg12 + - rspec-ee integration pg12 + - rspec-ee system pg12 + - rspec-ee unit pg12 geo + - rspec-ee integration pg12 geo + - rspec-ee system pg12 geo - memory-static - memory-on-boot script: @@ -529,138 +509,138 @@ rspec:feature-flags: ################################################## # EE: default refs (MRs, default branch, schedules) jobs # -rspec migration pg11-as-if-foss: +rspec migration pg12-as-if-foss: extends: - - .rspec-base-pg11-as-if-foss + - .rspec-base-pg12-as-if-foss - .rspec-base-migration - .rails:rules:as-if-foss-migration - .rspec-migration-parallel -rspec migration pg11-as-if-foss minimal: +rspec migration pg12-as-if-foss minimal: extends: - - rspec migration pg11-as-if-foss + - rspec migration pg12-as-if-foss - .minimal-rspec-tests - .rails:rules:as-if-foss-migration:minimal -rspec unit pg11-as-if-foss: +rspec unit pg12-as-if-foss: extends: - - .rspec-base-pg11-as-if-foss + - .rspec-base-pg12-as-if-foss - .rails:rules:as-if-foss-unit - .rspec-unit-parallel -rspec unit pg11-as-if-foss minimal: +rspec unit pg12-as-if-foss minimal: extends: - - rspec unit pg11-as-if-foss + - rspec unit pg12-as-if-foss - .minimal-rspec-tests - .rails:rules:as-if-foss-unit:minimal -rspec integration pg11-as-if-foss: +rspec integration pg12-as-if-foss: extends: - - .rspec-base-pg11-as-if-foss + - .rspec-base-pg12-as-if-foss - .rails:rules:as-if-foss-integration - .rspec-integration-parallel -rspec integration pg11-as-if-foss minimal: +rspec integration pg12-as-if-foss minimal: extends: - - rspec integration pg11-as-if-foss + - rspec integration pg12-as-if-foss - .minimal-rspec-tests - .rails:rules:as-if-foss-integration:minimal -rspec system pg11-as-if-foss: +rspec system pg12-as-if-foss: extends: - - .rspec-base-pg11-as-if-foss + - .rspec-base-pg12-as-if-foss - .rails:rules:as-if-foss-system - .rspec-system-parallel -rspec system pg11-as-if-foss minimal: +rspec system pg12-as-if-foss minimal: extends: - - rspec system pg11-as-if-foss + - rspec system pg12-as-if-foss - .minimal-rspec-tests - .rails:rules:as-if-foss-system:minimal -rspec-ee migration pg11: +rspec-ee migration pg12: extends: - - .rspec-ee-base-pg11 + - .rspec-ee-base-pg12 - .rspec-base-migration - .rails:rules:ee-only-migration - .rspec-ee-migration-parallel -rspec-ee migration pg11 minimal: +rspec-ee migration pg12 minimal: extends: - - rspec-ee migration pg11 + - rspec-ee migration pg12 - .minimal-rspec-tests - .rails:rules:ee-only-migration:minimal -rspec-ee unit pg11: +rspec-ee unit pg12: extends: - - .rspec-ee-base-pg11 + - .rspec-ee-base-pg12 - .rails:rules:ee-only-unit - .rspec-ee-unit-parallel -rspec-ee unit pg11 minimal: +rspec-ee unit pg12 minimal: extends: - - rspec-ee unit pg11 + - rspec-ee unit pg12 - .minimal-rspec-tests - .rails:rules:ee-only-unit:minimal -rspec-ee integration pg11: +rspec-ee integration pg12: extends: - - .rspec-ee-base-pg11 + - .rspec-ee-base-pg12 - .rails:rules:ee-only-integration - .rspec-ee-integration-parallel -rspec-ee integration pg11 minimal: +rspec-ee integration pg12 minimal: extends: - - rspec-ee integration pg11 + - rspec-ee integration pg12 - .minimal-rspec-tests - .rails:rules:ee-only-integration:minimal -rspec-ee system pg11: +rspec-ee system pg12: extends: - - .rspec-ee-base-pg11 + - .rspec-ee-base-pg12 - .rails:rules:ee-only-system - .rspec-ee-system-parallel -rspec-ee system pg11 minimal: +rspec-ee system pg12 minimal: extends: - - rspec-ee system pg11 + - rspec-ee system pg12 - .minimal-rspec-tests - .rails:rules:ee-only-system:minimal -rspec-ee unit pg11 geo: +rspec-ee unit pg12 geo: extends: - - .rspec-ee-base-geo-pg11 + - .rspec-ee-base-geo-pg12 - .rails:rules:ee-only-unit - .rspec-ee-unit-geo-parallel # FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212 -#rspec-ee unit pg11 geo minimal: +#rspec-ee unit pg12 geo minimal: # extends: -# - rspec-ee unit pg11 geo +# - rspec-ee unit pg12 geo # - .minimal-rspec-tests # - .rails:rules:ee-only-unit:minimal -rspec-ee integration pg11 geo: +rspec-ee integration pg12 geo: extends: - - .rspec-ee-base-geo-pg11 + - .rspec-ee-base-geo-pg12 - .rails:rules:ee-only-integration # FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212 -#rspec-ee integration pg11 geo minimal: +#rspec-ee integration pg12 geo minimal: # extends: -# - rspec-ee integration pg11 geo +# - rspec-ee integration pg12 geo # - .minimal-rspec-tests # - .rails:rules:ee-only-integration:minimal -rspec-ee system pg11 geo: +rspec-ee system pg12 geo: extends: - - .rspec-ee-base-geo-pg11 + - .rspec-ee-base-geo-pg12 - .rails:rules:ee-only-system # FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212 -#rspec-ee system pg11 geo minimal: +#rspec-ee system pg12 geo minimal: # extends: -# - rspec-ee system pg11 geo +# - rspec-ee system pg12 geo # - .minimal-rspec-tests # - .rails:rules:ee-only-system:minimal @@ -676,28 +656,28 @@ db:rollback geo: ########################################## # EE/FOSS: default branch nightly scheduled jobs # -rspec migration pg12: +rspec migration pg11: extends: - - .rspec-base-pg12 + - .rspec-base-pg11 - .rspec-base-migration - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-migration-parallel -rspec unit pg12: +rspec unit pg11: extends: - - .rspec-base-pg12 + - .rspec-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-unit-parallel -rspec integration pg12: +rspec integration pg11: extends: - - .rspec-base-pg12 + - .rspec-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-integration-parallel -rspec system pg12: +rspec system pg11: extends: - - .rspec-base-pg12 + - .rspec-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-system-parallel # EE/FOSS: default branch nightly scheduled jobs # @@ -705,45 +685,45 @@ rspec system pg12: ##################################### # EE: default branch nightly scheduled jobs # -rspec-ee migration pg12: +rspec-ee migration pg11: extends: - - .rspec-ee-base-pg12 + - .rspec-ee-base-pg11 - .rspec-base-migration - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-migration-parallel -rspec-ee unit pg12: +rspec-ee unit pg11: extends: - - .rspec-ee-base-pg12 + - .rspec-ee-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-unit-parallel -rspec-ee integration pg12: +rspec-ee integration pg11: extends: - - .rspec-ee-base-pg12 + - .rspec-ee-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-integration-parallel -rspec-ee system pg12: +rspec-ee system pg11: extends: - - .rspec-ee-base-pg12 + - .rspec-ee-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-system-parallel -rspec-ee unit pg12 geo: +rspec-ee unit pg11 geo: extends: - - .rspec-ee-base-geo-pg12 + - .rspec-ee-base-geo-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only - .rspec-ee-unit-geo-parallel -rspec-ee integration pg12 geo: +rspec-ee integration pg11 geo: extends: - - .rspec-ee-base-geo-pg12 + - .rspec-ee-base-geo-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only -rspec-ee system pg12 geo: +rspec-ee system pg11 geo: extends: - - .rspec-ee-base-geo-pg12 + - .rspec-ee-base-geo-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only # EE: default branch nightly scheduled jobs # ##################################### @@ -752,7 +732,7 @@ rspec-ee system pg12 geo: # EE: Canonical MR pipelines rspec fail-fast: extends: - - .rspec-ee-base-pg11 # This job also runs EE spec which needs elasticsearch + - .rspec-ee-base-pg12 # This job also runs EE spec which needs elasticsearch - .rails:rules:rspec fail-fast stage: test needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"] @@ -766,7 +746,7 @@ rspec fail-fast: rspec foss-impact: extends: - - .rspec-base-pg11-as-if-foss + - .rspec-base-pg12-as-if-foss - .rails:rules:rspec-foss-impact needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests as-if-foss"] script: diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 4d54380cefe..0162996e3a9 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -1,120 +1,71 @@ -# include: -# - template: Jobs/Code-Quality.gitlab-ci.yml -# - template: Security/SAST.gitlab-ci.yml -# - template: Security/Dependency-Scanning.gitlab-ci.yml -# - template: Security/DAST.gitlab-ci.yml +include: + - template: Jobs/Code-Quality.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Scanning.gitlab-ci.yml -# We need to duplicate this job's definition because the rules -# defined in the extended jobs rely on local YAML anchors -# (`*if-default-refs`) code_quality: extends: - .default-retry - - .reports:rules:code_quality - .use-docker-in-docker - stage: test - needs: [] - variables: - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.23" - script: - - | - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - fi - fi - - docker pull --quiet "$CODE_QUALITY_IMAGE" - - docker run - --env SOURCE_CODE="$PWD" - --volume "$PWD":/code - --volume /var/run/docker.sock:/var/run/docker.sock - "$CODE_QUALITY_IMAGE" /code artifacts: - reports: - codequality: gl-code-quality-report.json paths: - gl-code-quality-report.json # GitLab-specific - expire_in: 1 week # GitLab-specific + rules: !reference [".reports:rules:code_quality", rules] -# We need to duplicate this job's definition because the rules -# defined in the extended jobs rely on local YAML anchors -# (`*if-default-refs`) -.sast: +.sast-analyzer: + # We need to re-`extends` from `sast` as the `extends` here overrides the one from the template. extends: - .default-retry - - .reports:rules:sast - stage: test - # `needs: []` starts the job immediately in the pipeline - # https://docs.gitlab.com/ee/ci/yaml/README.html#needs + - sast needs: [] artifacts: paths: - gl-sast-report.json # GitLab-specific - reports: - sast: gl-sast-report.json expire_in: 1 week # GitLab-specific variables: - DOCKER_TLS_CERTDIR: "" - SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_ANALYZER_IMAGE_TAG: 2 SAST_BRAKEMAN_LEVEL: 2 # GitLab-specific - SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec,config/gitlab.yml.example # GitLab-specific + SAST_EXCLUDED_PATHS: "qa, spec, doc, ee/spec, config/gitlab.yml.example, tmp" # GitLab-specific SAST_DISABLE_BABEL: "true" - script: - - /analyzer run brakeman-sast: - extends: .sast - image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" + rules: !reference [".reports:rules:sast", rules] eslint-sast: - extends: .sast - image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + rules: !reference [".reports:rules:sast", rules] nodejs-scan-sast: - extends: .sast - image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" + rules: !reference [".reports:rules:sast", rules] -secrets-sast: - extends: .sast - image: - name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:3" +semgrep-sast: + rules: !reference [".reports:rules:sast", rules] + +.secret-analyzer: + extends: .default-retry + needs: [] artifacts: paths: - gl-secret-detection-report.json # GitLab-specific - reports: - sast: gl-secret-detection-report.json expire_in: 1 week # GitLab-specific -# We need to duplicate this job's definition because the rules -# defined in the extended jobs rely on local YAML anchors -# (`*if-default-refs`) -.dependency_scanning: +secret_detection: + rules: !reference [".reports:rules:secret_detection", rules] + +.ds-analyzer: + # We need to re-`extends` from `dependency_scanning` as the `extends` here overrides the one from the template. extends: - .default-retry - - .reports:rules:dependency_scanning - stage: test + - dependency_scanning needs: [] variables: - DS_MAJOR_VERSION: 2 - DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec" # GitLab-specific - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec, tmp" # GitLab-specific artifacts: paths: - gl-dependency-scanning-report.json # GitLab-specific - reports: - dependency_scanning: gl-dependency-scanning-report.json expire_in: 1 week # GitLab-specific - script: - - /analyzer run -dependency_scanning gemnasium: - extends: .dependency_scanning - image: - name: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" +gemnasium-dependency_scanning: before_script: # git-lfs is needed for auto-remediation - apk add git-lfs @@ -123,56 +74,43 @@ dependency_scanning gemnasium: - apk add jq # Lower execa severity based on https://gitlab.com/gitlab-org/gitlab/-/issues/223859#note_452922390 - jq '(.vulnerabilities[] | select (.cve == "yarn.lock:execa:gemnasium:05cfa2e8-2d0c-42c1-8894-638e2f12ff3d")).severity = "Medium"' gl-dependency-scanning-report.json > temp.json && mv temp.json gl-dependency-scanning-report.json + rules: !reference [".reports:rules:dependency_scanning", rules] -dependency_scanning bundler-audit: - extends: .dependency_scanning - image: - name: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" +bundler-audit-dependency_scanning: + rules: !reference [".reports:rules:dependency_scanning", rules] -dependency_scanning retire-js: - extends: .dependency_scanning - image: - name: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" +retire-js-dependency_scanning: + rules: !reference [".reports:rules:dependency_scanning", rules] -dependency_scanning gemnasium-python: - extends: .dependency_scanning - image: - name: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" +gemnasium-python-dependency_scanning: + rules: !reference [".reports:rules:dependency_scanning", rules] # Analyze dependencies for malicious behavior # See https://gitlab.com/gitlab-com/gl-security/security-research/package-hunter package_hunter: extends: - - .reports:schedule-dast + - .default-retry + - .reports:rules:package_hunter stage: test image: name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:latest entrypoint: [""] needs: [] + allow_failure: true script: - rm -r spec locale .git app/assets/images doc/ - cd .. && tar -I "gzip --best" -cf gitlab.tgz gitlab/ - DEBUG=* HTR_user=$PACKAGE_HUNTER_USER HTR_pass=$PACKAGE_HUNTER_PASS node /usr/src/app/cli.js analyze --format gitlab gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json artifacts: paths: - - gl-dependency-scanning-report.json # GitLab-specific + - gl-dependency-scanning-report.json reports: dependency_scanning: gl-dependency-scanning-report.json - expire_in: 1 week # GitLab-specific + expire_in: 1 week license_scanning: - extends: - - .default-retry - - .reports:rules:license_scanning - stage: test - image: - name: "registry.gitlab.com/gitlab-org/security-products/analyzers/license-finder:3" - entrypoint: [""] + extends: .default-retry needs: [] - script: - - /run.sh analyze . artifacts: - reports: - license_scanning: gl-license-scanning-report.json expire_in: 1 week # GitLab-specific - dependencies: [] + rules: !reference [".reports:rules:license_scanning", rules] diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index e1037edc325..5decc83da2b 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -34,10 +34,7 @@ review-build-cng: - job: compile-production-assets artifacts: false script: - - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng - # When the job is manual, review-deploy is also manual and we don't want people - # to have to manually start the jobs in sequence, so we do it for them. - - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-deploy"' + - ./scripts/trigger-build cng .review-workflow-base: extends: @@ -45,7 +42,6 @@ review-build-cng: image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 variables: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" - REVIEW_APPS_DOMAIN: "temp.gitlab-review.app" # FIXME: using temporary domain DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" GITLAB_HELM_CHART_REF: "v4.6.3" environment: @@ -59,7 +55,7 @@ review-deploy: - .review-workflow-base - .review:rules:review-deploy stage: review - dependencies: [] + needs: ["review-build-cng"] resource_group: "review/${CI_COMMIT_REF_NAME}" before_script: - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) @@ -75,10 +71,6 @@ review-deploy: - date - deploy || (display_deployment_debug && exit 1) - disable_sign_ups || (delete_release && exit 1) - # When the job is manual, review-qa-smoke is also manual and we don't want people - # to have to manually start the jobs in sequence, so we do it for them. - - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-qa-smoke"' - - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-performance"' after_script: # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. # Set DAST_RUN to true when jobs are manually scheduled. @@ -123,9 +115,7 @@ review-stop: - .use-docker-in-docker image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7 stage: qa - # This is needed so that manual jobs with needs don't block the pipeline. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. - dependencies: ["review-deploy"] + needs: ["review-deploy"] variables: QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" QA_CAN_TEST_GIT_PROTOCOL_V2: "false" @@ -175,9 +165,7 @@ review-performance: name: sitespeedio/sitespeed.io entrypoint: [""] stage: qa - # This is needed so that manual jobs with needs don't block the pipeline. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. - dependencies: ["review-deploy"] + needs: ["review-deploy"] before_script: - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" @@ -200,7 +188,7 @@ parallel-spec-reports: - .review:rules:review-qa-all image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine stage: post-qa - dependencies: ["review-qa-all"] + needs: ["review-qa-all"] variables: NEW_PARALLEL_SPECS_REPORT: qa/report-new.html BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" @@ -229,8 +217,8 @@ danger-review: stage: test needs: [] before_script: - - source ./scripts/utils.sh - - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --with danger" + - source scripts/utils.sh + - bundle_install_script "--with danger" - run_timed_command "retry yarn install --frozen-lockfile" script: - > @@ -242,12 +230,3 @@ danger-review: else run_timed_command "bundle exec danger --fail-on-errors=true --verbose" fi - -update-danger-review-cache: - extends: - - danger-review - - .shared:rules:update-cache - stage: prepare - script: echo 'Cache is fresh!' - cache: - policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index c2d16582a68..5c9be5b1e9f 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -124,7 +124,7 @@ .docs-patterns: &docs-patterns - ".gitlab/route-map.yml" - "doc/**/*" - - ".markdownlint.json" + - ".markdownlint.yml" - "scripts/lint-doc.sh" .frontend-dependency-patterns: &frontend-dependency-patterns @@ -424,6 +424,13 @@ - <<: *if-default-refs changes: *code-backstage-patterns +.frontend:rules:default-frontend-jobs-ee: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-backstage-patterns + .frontend:rules:default-frontend-jobs-as-if-foss: rules: - <<: *if-not-ee @@ -518,6 +525,8 @@ .qa:rules:package-and-qa: rules: + - <<: *if-not-ee + when: never - <<: *if-dot-com-gitlab-org-and-security-merge-request changes: *ci-qa-patterns allow_failure: true @@ -929,6 +938,25 @@ - <<: *if-merge-request changes: [".gitlab/ci/rails.gitlab-ci.yml"] +######################### +# Static analysis rules # +######################### + +.static-analysis:rules:ee-and-foss: + rules: + - changes: *code-backstage-qa-patterns + +.static-analysis:rules:as-if-foss: + rules: + - <<: *if-not-ee + when: never + - <<: *if-merge-request-title-as-if-foss + changes: *code-backstage-qa-patterns + - <<: *if-security-merge-request + changes: *code-backstage-qa-patterns + - <<: *if-merge-request + changes: *ci-patterns + ####################### # Vendored gems rules # ####################### @@ -975,6 +1003,16 @@ changes: *code-backstage-qa-patterns allow_failure: true +.reports:rules:secret_detection: + rules: + - if: '$SECRET_DETECTION_DISABLED' + when: never + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # The Secret-Detection template already has a `secret_detection_default_branch` job + when: never + # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 + - changes: *code-backstage-qa-patterns + allow_failure: true + .reports:rules:dependency_scanning: rules: - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/' @@ -996,13 +1034,19 @@ when: manual allow_failure: true -.reports:schedule-dast: +.reports:rules:schedule-dast: rules: - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' when: never - <<: *if-default-branch-schedule-nightly allow_failure: true +.reports:rules:package_hunter: + rules: + - <<: *if-default-branch-schedule-2-hourly + - <<: *if-merge-request + changes: ["yarn.lock"] + .reports:rules:license_scanning: rules: - if: '$LICENSE_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\blicense_scanning\b/' @@ -1042,7 +1086,6 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-patterns - when: manual allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns @@ -1063,7 +1106,6 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns - when: manual allow_failure: true - <<: *if-dot-com-gitlab-org-schedule allow_failure: true @@ -1086,7 +1128,6 @@ allow_failure: true - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns - when: manual allow_failure: true .review:rules:review-qa-all: diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index b90c02c08ef..08c5a7267c2 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -27,19 +27,19 @@ update-tests-metadata: stage: post-test dependencies: - setup-test-env - - rspec migration pg11 + - rspec migration pg12 - rspec frontend_fixture - rspec-ee frontend_fixture - - rspec unit pg11 - - rspec integration pg11 - - rspec system pg11 - - rspec-ee migration pg11 - - rspec-ee unit pg11 - - rspec-ee integration pg11 - - rspec-ee system pg11 - - rspec-ee unit pg11 geo - - rspec-ee integration pg11 geo - - rspec-ee system pg11 geo + - rspec unit pg12 + - rspec integration pg12 + - rspec system pg12 + - rspec-ee migration pg12 + - rspec-ee unit pg12 + - rspec-ee integration pg12 + - rspec-ee system pg12 + - rspec-ee unit pg12 geo + - rspec-ee integration pg12 geo + - rspec-ee system pg12 geo script: - run_timed_command "retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document" - source ./scripts/rspec_helpers.sh |