diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 12:55:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-07-20 12:55:51 +0300 |
commit | e8d2c2579383897a1dd7f9debd359abe8ae8373d (patch) | |
tree | c42be41678c2586d49a75cabce89322082698334 /.gitlab/ci | |
parent | fc845b37ec3a90aaa719975f607740c22ba6a113 (diff) |
Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42
Diffstat (limited to '.gitlab/ci')
-rw-r--r-- | .gitlab/ci/build-images.gitlab-ci.yml | 43 | ||||
-rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 50 | ||||
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 40 | ||||
-rw-r--r-- | .gitlab/ci/pages.gitlab-ci.yml | 10 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 144 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 34 | ||||
-rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 88 | ||||
-rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 140 | ||||
-rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 2 |
10 files changed, 417 insertions, 136 deletions
diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml index ed1f71e27bb..853f92ed98d 100644 --- a/.gitlab/ci/build-images.gitlab-ci.yml +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -1,43 +1,48 @@ -# This image is used by the `review-qa-*` jobs. The image name is also passed to the downstream `omnibus-gitlab-mirror` pipeline -# triggered by `package-and-qa` so that it doesn't have to rebuild it a second time. The downstream `omnibus-gitlab-mirror` pipeline -# itself passes the image name to the `gitlab-qa-mirror` pipeline so that it can use it instead of inferring an end-to-end image -# from the GitLab image built by the downstream `omnibus-gitlab-mirror` pipeline. +.base-image-build: + extends: .use-kaniko + script: + # With .git/hooks/post-checkout in place, Git tries to pull LFS objects, but the image doesn't have Git LFS, and we actually don't care about it for this specific so we just remove the file. + # Without removing the file, the error is as follows: "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout." + - rm -f .git/hooks/post-checkout + - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then + echo "Checking out \$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ($CI_MERGE_REQUEST_SOURCE_BRANCH_SHA) instead of \$CI_COMMIT_SHA (merge result commit $CI_COMMIT_SHA) so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync."; + git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; + else + echo "Building the image from \$CI_COMMIT_SHA ($CI_COMMIT_SHA) for this non-merge result pipeline."; + fi; + - echo "See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results for more details."; + retry: 2 + +# This image is used by: +# - The `review-qa-*` jobs +# - The downstream `omnibus-gitlab-mirror` pipeline triggered by `package-and-qa` so that it doesn't have to rebuild it again. +# The downstream `omnibus-gitlab-mirror` pipeline itself passes the image name to the `gitlab-qa-mirror` pipeline so that +# it can use it instead of inferring an end-to-end imag from the GitLab image built by the downstream `omnibus-gitlab-mirror` pipeline. # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#testing-code-in-merge-requests for more details. build-qa-image: extends: - - .use-kaniko + - .base-image-build - .build-images:rules:build-qa-image stage: build-images needs: [] variables: QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" script: - # With .git/hooks/post-checkout in place, Git tries to pull LFS objects, but the image doesn't have Git LFS, and we actually don't care about it for this specific so we just remove the file. - # Without removing the file, the error is as follows: "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout." - - rm .git/hooks/post-checkout - # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync. - # This falls back to $CI_COMMIT_SHA (the default checked out commit) for the non-merged result pipelines. - # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results. - - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then - git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; - fi + - !reference [.base-image-build, script] - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --cache=true - retry: 2 # This image is used by: # - The `CNG` pipelines (via the `review-build-cng` job): https://gitlab.com/gitlab-org/build/CNG/-/blob/cfc67136d711e1c8c409bf8e57427a644393da2f/.gitlab-ci.yml#L335 # - The `omnibus-gitlab` pipelines (via the `package-and-qa` job): https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/dfd1ad475868fc84e91ab7b5706aa03e46dc3a86/.gitlab-ci.yml#L130 build-assets-image: extends: - - .use-kaniko + - .base-image-build - .build-images:rules:build-assets-image stage: build-images needs: ["compile-production-assets"] - variables: - GIT_DEPTH: "1" script: + - !reference [.base-image-build, script] # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines # https://gitlab.com/gitlab-org/gitlab/issues/208389 - run_timed_command "scripts/build_assets_image" - retry: 2 diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index a4317c72413..60025c244f1 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -53,7 +53,7 @@ docs-lint markdown: docs-lint links: extends: - .docs:rules:docs-lint - image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.13-ruby-2.7.2 + image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.13-ruby-2.7.2-d2b92621 stage: test needs: [] script: diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index dfd595c2696..3aa8ab1591a 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -2,6 +2,10 @@ - source scripts/utils.sh - run_timed_command "retry yarn install --frozen-lockfile" +.storybook-yarn-install: &storybook-yarn-install + - source scripts/utils.sh + - run_timed_command "retry yarn run storybook:install --frozen-lockfile" + .compile-assets-base: extends: - .default-retry @@ -35,6 +39,15 @@ compile-production-assets: - public/assets/ - webpack-report/ when: always + before_script: + - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then + echo "Checking out \$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ($CI_MERGE_REQUEST_SOURCE_BRANCH_SHA) instead of \$CI_COMMIT_SHA (merge result commit $CI_COMMIT_SHA) so that GitLab assets image tag actually reflect the commit for which assets were compiled."; + git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; + else + echo "Building the image from \$CI_COMMIT_SHA ($CI_COMMIT_SHA) for this non-merge result pipeline."; + fi; + - echo "See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results for more details."; + - !reference [.default-before_script, before_script] after_script: - rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here @@ -80,6 +93,15 @@ update-yarn-cache: script: - *yarn-install +update-storybook-yarn-cache: + extends: + - .default-retry + - .storybook-yarn-cache-push + - .shared:rules:update-cache + stage: prepare + script: + - *storybook-yarn-install + .frontend-fixtures-base: extends: - .default-retry @@ -94,7 +116,7 @@ update-yarn-cache: - run_timed_command "gem install knapsack --no-document" - run_timed_command "scripts/gitaly-test-spawn" - source ./scripts/rspec_helpers.sh - - rspec_paralellized_job "--tag frontend_fixture" + - rspec_paralellized_job artifacts: name: frontend-fixtures expire_in: 31d @@ -344,3 +366,29 @@ startup-css-check as-if-foss: needs: - job: "compile-test-assets as-if-foss" - job: "rspec frontend_fixture as-if-foss" + +.compile-storybook-base: + extends: + - .frontend-test-base + - .storybook-yarn-cache + script: + - *yarn-install # storybook depends on the global webpack config, so we must install global deps. + - *storybook-yarn-install + - yarn run storybook:build + +compile-storybook: + extends: + - .compile-storybook-base + - .frontend:rules:default-frontend-jobs + artifacts: + name: storybook + expire_in: 31d + when: always + paths: + - storybook/public + +compile-storybook as-if-foss: + extends: + - .compile-storybook-base + - .as-if-foss + - .frontend:rules:default-frontend-jobs-as-if-foss diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index ba49ddfce9d..860397b9437 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -75,7 +75,7 @@ policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. .assets-cache: &assets-cache - key: "assets-${NODE_ENV}-v1" + key: "assets-${NODE_ENV}-v2" paths: - assets-hash.txt - public/assets/webpack/ @@ -88,6 +88,16 @@ <<: *assets-cache policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. +.storybook-node-modules-cache: &storybook-node-modules-cache + key: "storybook-node-modules-${NODE_ENV}-v1" + paths: + - storybook/node_modules/ + policy: pull + +.storybook-node-modules-cache-push: &storybook-node-modules-cache-push + <<: *storybook-node-modules-cache + policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up. + .rubocop-cache: &rubocop-cache key: "rubocop-v1" paths: @@ -181,6 +191,14 @@ - *node-modules-cache-push - *assets-cache-push +.storybook-yarn-cache: + cache: + - *storybook-node-modules-cache + +.storybook-yarn-cache-push: + cache: + - *storybook-node-modules-cache-push + .use-pg11: image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: @@ -199,6 +217,15 @@ variables: POSTGRES_HOST_AUTH_METHOD: trust +.use-pg13: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" + services: + - name: postgres:13 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:5.0-alpine + variables: + POSTGRES_HOST_AUTH_METHOD: trust + .use-pg11-ee: image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36" services: @@ -221,6 +248,17 @@ variables: POSTGRES_HOST_AUTH_METHOD: trust +.use-pg13-ee: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36" + services: + - name: postgres:13 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:5.0-alpine + - name: elasticsearch:7.11.1 + command: ["elasticsearch", "-E", "discovery.type=single-node"] + variables: + POSTGRES_HOST_AUTH_METHOD: trust + .use-kaniko: image: name: registry.gitlab.com/gitlab-org/gitlab-build-images:kaniko diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index b6d4b0ef11d..e75b0db9071 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -1,3 +1,7 @@ +.compress-public: &compress-public + - find public -type f -regex '.*\.\(htm\|html\|txt\|text\|js\|json\|css\|svg\|xml\)$' -exec gzip -f -k {} \; + - find public -type f -regex '.*\.\(htm\|html\|txt\|text\|js\|json\|css\|svg\|xml\)$' -exec brotli -f -k {} \; + pages: extends: - .default-retry @@ -8,14 +12,18 @@ pages: - coverage-frontend - karma - compile-production-assets + - compile-storybook + before_script: + - apt-get update && apt-get -y install brotli gzip script: - mv public/ .public/ - mkdir public/ - mv coverage/ public/coverage-ruby/ || true - mv coverage-frontend/ public/coverage-frontend/ || true - mv coverage-javascript/ public/coverage-javascript/ || true + - mv storybook/public public/storybook || true - cp .public/assets/application-*.css public/application.css || true - - cp .public/assets/application-*.css.gz public/application.css.gz || true + - *compress-public artifacts: paths: - public diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index b74c8ebf575..0dfc353820f 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -47,7 +47,6 @@ - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ - - tmp/feature_flags/ - log/*.log reports: junit: junit_rspec.xml @@ -75,6 +74,11 @@ - .use-pg12 needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss", "detect-tests"] +.rspec-base-pg13: + extends: + - .rspec-base + - .use-pg13 + .rspec-ee-base-pg11: extends: - .rspec-base @@ -85,6 +89,11 @@ - .rspec-base - .use-pg12-ee +.rspec-ee-base-pg13: + extends: + - .rspec-base + - .use-pg13-ee + .rspec-ee-base-geo: extends: .rspec-base script: @@ -101,6 +110,11 @@ - .rspec-ee-base-geo - .use-pg12-ee +.rspec-ee-base-geo-pg13: + extends: + - .rspec-ee-base-geo + - .use-pg13-ee + .db-job-base: extends: - .rails-job-base @@ -114,7 +128,7 @@ ############################ # rspec job parallel configs .rspec-migration-parallel: - parallel: 7 + parallel: 9 .rspec-ee-migration-parallel: parallel: 3 @@ -123,7 +137,7 @@ parallel: 20 .rspec-ee-unit-parallel: - parallel: 10 + parallel: 12 .rspec-ee-unit-geo-parallel: parallel: 2 @@ -149,13 +163,16 @@ setup-test-env: - .rails-job-base - .setup-test-env-cache - .rails:rules:code-backstage-qa - - .use-pg12 stage: prepare variables: GITLAB_TEST_EAGER_LOAD: "0" + SETUP_DB: "false" script: - - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'" + - run_timed_command "scripts/setup-test-env" + - echo -e "\e[0Ksection_start:`date +%s`:gitaly-test-build[collapsed=true]\r\e[0KCompiling Gitaly binaries" - run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here + - echo -e "\e[0Ksection_end:`date +%s`:gitaly-test-build\r\e[0K" + artifacts: expire_in: 7d paths: @@ -237,6 +254,11 @@ static-analysis: script: - run_timed_command "retry yarn install --frozen-lockfile" - scripts/static-analysis + artifacts: + expire_in: 31d + when: always + paths: + - tmp/feature_flags/ static-analysis as-if-foss: extends: @@ -244,6 +266,22 @@ static-analysis as-if-foss: - .static-analysis:rules:as-if-foss - .as-if-foss +zeitwerk-check: + extends: + - .rails-cache + - .default-before_script + - .rails:rules:ee-and-foss-unit + variables: + BUNDLE_WITHOUT: "" + SETUP_DB: "false" + needs: [] + stage: test + script: + - sed -i -e "s/config\.autoloader = :classic/config\.autoloader = :zeitwerk/" config/application.rb + - RAILS_ENV=test bundle exec rake zeitwerk:check + - RAILS_ENV=development bundle exec rake zeitwerk:check + - RAILS_ENV=production bundle exec rake zeitwerk:check + rspec migration pg12: extends: - .rspec-base-pg12 @@ -355,11 +393,10 @@ db:migrate-from-previous-major-version: - sed -i -e "s/gem 'mimemagic', '~> 0.3.2'/gem 'ruby-magic', '~> 0.4.0'/" Gemfile - run_timed_command "gem install bundler:1.17.3" - run_timed_command "bundle update google-protobuf nokogiri grpc mimemagic bootsnap" - - run_timed_command "bundle install ${BUNDLE_INSTALL_FLAGS}" - - cp config/gitlab.yml.example config/gitlab.yml + - SETUP_DB=false USE_BUNDLE_INSTALL=true bash scripts/prepare_build.sh - run_timed_command "bundle exec rake db:drop db:create db:structure:load db:migrate db:seed_fu" - git checkout -f $CI_COMMIT_SHA - - run_timed_command "bundle install ${BUNDLE_INSTALL_FLAGS}" + - SETUP_DB=false USE_BUNDLE_INSTALL=true bash scripts/prepare_build.sh - run_timed_command "bundle exec rake db:migrate" db:rollback: @@ -487,23 +524,7 @@ rspec:feature-flags: - .coverage-base - .rails:rules:rspec-feature-flags stage: post-test - # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) - # so we use `dependencies` here. - dependencies: - - setup-test-env - - rspec migration pg12 - - rspec unit pg12 - - rspec integration pg12 - - rspec system pg12 - - rspec-ee migration pg12 - - rspec-ee unit pg12 - - rspec-ee integration pg12 - - rspec-ee system pg12 - - rspec-ee unit pg12 geo - - rspec-ee integration pg12 geo - - rspec-ee system pg12 geo - - memory-static - - memory-on-boot + needs: ["static-analysis"] script: - !reference [.minimal-bundle-install, script] - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then @@ -664,6 +685,8 @@ db:rollback geo: ########################################## # EE/FOSS: default branch nightly scheduled jobs # + +# PG11 rspec migration pg11: extends: - .rspec-base-pg11 @@ -688,11 +711,40 @@ rspec system pg11: - .rspec-base-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage - .rspec-system-parallel + +# PG13 +rspec migration pg13: + extends: + - .rspec-base-pg13 + - .rspec-base-migration + - .rails:rules:default-branch-schedule-nightly--code-backstage + - .rspec-migration-parallel + +rspec unit pg13: + extends: + - .rspec-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage + - .rspec-unit-parallel + +rspec integration pg13: + extends: + - .rspec-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage + - .rspec-integration-parallel + +rspec system pg13: + extends: + - .rspec-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage + - .rspec-system-parallel + # EE/FOSS: default branch nightly scheduled jobs # ########################################## ##################################### # EE: default branch nightly scheduled jobs # + +# PG11 rspec-ee migration pg11: extends: - .rspec-ee-base-pg11 @@ -733,6 +785,48 @@ rspec-ee system pg11 geo: extends: - .rspec-ee-base-geo-pg11 - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + +# PG13 +rspec-ee migration pg13: + extends: + - .rspec-ee-base-pg13 + - .rspec-base-migration + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-migration-parallel + +rspec-ee unit pg13: + extends: + - .rspec-ee-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-unit-parallel + +rspec-ee integration pg13: + extends: + - .rspec-ee-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-integration-parallel + +rspec-ee system pg13: + extends: + - .rspec-ee-base-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-system-parallel + +rspec-ee unit pg13 geo: + extends: + - .rspec-ee-base-geo-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + - .rspec-ee-unit-geo-parallel + +rspec-ee integration pg13 geo: + extends: + - .rspec-ee-base-geo-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only + +rspec-ee system pg13 geo: + extends: + - .rspec-ee-base-geo-pg13 + - .rails:rules:default-branch-schedule-nightly--code-backstage-ee-only # EE: default branch nightly scheduled jobs # ##################################### diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 4cc03fdb1a4..690a971927c 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -74,33 +74,35 @@ gemnasium-dependency_scanning: - apk add jq # Lower execa severity based on https://gitlab.com/gitlab-org/gitlab/-/issues/223859#note_452922390 - jq '(.vulnerabilities[] | select (.cve == "yarn.lock:execa:gemnasium:05cfa2e8-2d0c-42c1-8894-638e2f12ff3d")).severity = "Medium"' gl-dependency-scanning-report.json > temp.json && mv temp.json gl-dependency-scanning-report.json - rules: !reference [".reports:rules:dependency_scanning", rules] + rules: !reference [".reports:rules:gemnasium-dependency_scanning", rules] bundler-audit-dependency_scanning: - rules: !reference [".reports:rules:dependency_scanning", rules] + rules: !reference [".reports:rules:bundler-audit-dependency_scanning", rules] retire-js-dependency_scanning: - rules: !reference [".reports:rules:dependency_scanning", rules] + rules: !reference [".reports:rules:retire-js-dependency_scanning", rules] gemnasium-python-dependency_scanning: - rules: !reference [".reports:rules:dependency_scanning", rules] + rules: !reference [".reports:rules:gemnasium-python-dependency_scanning", rules] # Analyze dependencies for malicious behavior # See https://gitlab.com/gitlab-com/gl-security/security-research/package-hunter -package_hunter: +.package_hunter-base: extends: - .default-retry - - .reports:rules:package_hunter stage: test image: - name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:latest + name: registry.gitlab.com/gitlab-com/gl-security/security-research/package-hunter-cli:1.1.0 entrypoint: [""] + variables: + DEBUG: '*' + HTR_user: '$PACKAGE_HUNTER_USER' + HTR_pass: '$PACKAGE_HUNTER_PASS' needs: [] allow_failure: true - script: + before_script: - rm -r spec locale .git app/assets/images doc/ - cd .. && tar -I "gzip --best" -cf gitlab.tgz gitlab/ - - DEBUG=* HTR_user=$PACKAGE_HUNTER_USER HTR_pass=$PACKAGE_HUNTER_PASS node /usr/src/app/cli.js analyze --format gitlab gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json artifacts: paths: - gl-dependency-scanning-report.json @@ -108,6 +110,20 @@ package_hunter: dependency_scanning: gl-dependency-scanning-report.json expire_in: 1 week +package_hunter-yarn: + extends: + - .package_hunter-base + - .reports:rules:package_hunter-yarn + script: + - node /usr/src/app/cli.js analyze --format gitlab --manager yarn gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json + +package_hunter-bundler: + extends: + - .package_hunter-base + - .reports:rules:package_hunter-bundler + script: + - node /usr/src/app/cli.js analyze --format gitlab --manager bundler gitlab.tgz | tee $CI_PROJECT_DIR/gl-dependency-scanning-report.json + license_scanning: extends: .default-retry needs: [] diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index cabeb0cc1ac..38d59af5aed 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -2,7 +2,7 @@ review-cleanup: extends: - .default-retry - .review:rules:review-cleanup - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3.5-kubectl1.17 stage: prepare environment: name: review/auto-cleanup @@ -25,7 +25,7 @@ review-build-cng: extends: - .default-retry - .review:rules:review-build-cng - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine + image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 stage: review-prepare before_script: - source ./scripts/utils.sh @@ -54,7 +54,6 @@ review-deploy: extends: - .review-workflow-base - .review:rules:review-deploy - retry: 2 stage: review needs: ["review-build-cng"] resource_group: "review/${CI_COMMIT_REF_NAME}" @@ -66,8 +65,6 @@ review-deploy: - *base-before_script script: - check_kube_domain - - "ensure_namespace ${KUBE_NAMESPACE}" - - install_external_dns - download_chart - date - deploy || (display_deployment_debug && exit 1) @@ -97,13 +94,13 @@ review-deploy: before_script: - *base-before_script -review-stop-failed-deployment: +review-delete-deployment: extends: - .review-stop-base - - .review:rules:review-stop-failed-deployment + - .review:rules:review-delete-deployment stage: prepare script: - - delete_failed_release + - delete_release review-stop: extends: @@ -111,18 +108,19 @@ review-stop: - .review:rules:review-stop stage: post-qa script: - - delete_release + - delete_k8s_release_namespace .review-qa-base: extends: - .use-docker-in-docker - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.7 + image: + name: ${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG} + entrypoint: [""] stage: qa - needs: ["review-deploy"] + needs: ["build-qa-image", "review-deploy"] variables: - QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" - QA_CAN_TEST_GIT_PROTOCOL_V2: "false" QA_DEBUG: "true" + QA_CAN_TEST_GIT_PROTOCOL_V2: "false" QA_GENERATE_ALLURE_REPORT: "true" GITLAB_USERNAME: "root" GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" @@ -132,21 +130,22 @@ review-stop: EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" SIGNUP_DISABLED: "true" before_script: - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" + # Use $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA so that GitLab image built in omnibus-gitlab-mirror and QA image are in sync. + - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA" ]; then + git checkout -f ${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}; + fi - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - - echo "${QA_IMAGE}" - - *base-before_script - - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} + - cd qa artifacts: paths: - - ./qa/gitlab-qa-run-* + - qa/tmp expire_in: 7 days when: always .allure-report-base: image: - name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.2 + name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.4 entrypoint: [""] stage: post-qa variables: @@ -157,7 +156,7 @@ review-stop: script: - | allure-report-publisher upload gcs \ - --results-glob="qa/gitlab-qa-run-*/**/allure-results/*" \ + --results-glob="qa/tmp/allure-results/*" \ --bucket="gitlab-qa-allure-reports" \ --prefix="$ALLURE_REPORT_PATH_PREFIX/$CI_COMMIT_REF_SLUG" \ --update-pr="comment" \ @@ -171,7 +170,7 @@ review-qa-smoke: - .review:rules:review-qa-smoke retry: 1 # This is confusing but this means "2 runs at max". script: - - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" + - bin/test Test::Instance::Smoke "${CI_ENVIRONMENT_URL}" review-qa-all: extends: @@ -181,7 +180,14 @@ review-qa-all: script: - export KNAPSACK_REPORT_PATH=knapsack/master_report.json - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb - - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation + - | + bin/test Test::Instance::All "${CI_ENVIRONMENT_URL}" \ + -- \ + --color --format documentation \ + --format RspecJunitFormatter --out tmp/rspec.xml + artifacts: + reports: + junit: qa/tmp/rspec.xml review-performance: extends: @@ -209,32 +215,6 @@ review-performance: performance: performance.json expire_in: 31d -parallel-spec-reports: - extends: - - .review:rules:review-qa-all - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine - stage: post-qa - needs: ["review-qa-all"] - variables: - NEW_PARALLEL_SPECS_REPORT: qa/report-new.html - BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" - script: - - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/* - - gem install nokogiri --no-document - - cd qa/gitlab-qa-run-*/gitlab-* - - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_) - - cd - - - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}' - - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm - artifacts: - when: always - paths: - - qa/report-new.html - - qa/gitlab-qa-run-* - reports: - junit: qa/gitlab-qa-run-*/**/rspec-*.xml - expire_in: 31d - allure-report-qa-smoke: extends: - .allure-report-base @@ -267,10 +247,14 @@ danger-review: script: - > if [ -z "$DANGER_GITLAB_API_TOKEN" ]; then - # Force danger to skip CI source GitLab and fallback to "local only git repo". - unset GITLAB_CI - # We need to base SHA to help danger determine the base commit for this shallow clone. - run_timed_command "bundle exec danger dry_run --fail-on-errors=true --verbose --base='$CI_MERGE_REQUEST_DIFF_BASE_SHA'" + run_timed_command danger_as_local else run_timed_command "bundle exec danger --fail-on-errors=true --verbose" fi + +danger-review-local: + extends: + - danger-review + - .review:rules:danger-local + script: + - run_timed_command danger_as_local diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 8dd97c1fe69..c97ef4fc868 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -131,6 +131,30 @@ - ".markdownlint.yml" - "scripts/lint-doc.sh" +.bundler-patterns: &bundler-patterns + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + +.nodejs-patterns: &nodejs-patterns + - '{package.json,*/package.json,*/*/package.json}' + +.python-patterns: &python-patterns + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + +.dependency-patterns: &dependency-patterns + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' + .frontend-dependency-patterns: &frontend-dependency-patterns - "{package.json,yarn.lock}" - "config/webpack.config.js" @@ -138,6 +162,7 @@ .frontend-build-patterns: &frontend-build-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "config/webpack.config.js" - "config/**/*.js" @@ -146,6 +171,7 @@ .frontend-patterns: &frontend-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -163,7 +189,7 @@ - "config.ru" # List explicitly all the app/ dirs that are backend (i.e. all except app/assets). - "{,ee/,jh/}{app/channels,app/controllers,app/finders,app/graphql,app/helpers,app/mailers,app/models,app/policies,app/presenters,app/serializers,app/services,app/uploaders,app/validators,app/views,app/workers}/**/*" - - "{,ee/,jh/}{bin,cable,config,db,lib}/**/*" + - "{,ee/,jh/}{bin,cable,config,db,generator_templates,lib}/**/*" - "{,ee/,jh/}spec/**/*.rb" # CI changes - ".gitlab-ci.yml" @@ -198,6 +224,7 @@ .code-patterns: &code-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -213,7 +240,7 @@ - "Rakefile" - "tests.yml" - "config.ru" - - "{,ee/,jh/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" @@ -223,6 +250,7 @@ .code-backstage-patterns: &code-backstage-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -238,7 +266,7 @@ - "Rakefile" - "tests.yml" - "config.ru" - - "{,ee/,jh/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" # Backstage changes @@ -251,6 +279,7 @@ .code-qa-patterns: &code-qa-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -266,7 +295,7 @@ - "Rakefile" - "tests.yml" - "config.ru" - - "{,ee/,jh/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" # QA changes @@ -275,6 +304,7 @@ .code-backstage-qa-patterns: &code-backstage-qa-patterns - "{package.json,yarn.lock}" + - ".browserslistrc" - "babel.config.js" - "jest.config.{base,integration,unit}.js" - ".csscomb.json" @@ -290,7 +320,7 @@ - "Rakefile" - "tests.yml" - "config.ru" - - "{,ee/,jh/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "{,ee/,jh/}{app,bin,config,db,generator_templates,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - "data/whats_new/*.yml" # Backstage changes @@ -304,6 +334,12 @@ - ".dockerignore" - "qa/**/*" +.code-backstage-danger-patterns: &code-backstage-danger-patterns + # Backstage changes + - "Dangerfile" + - "danger/**/*" + - "tooling/danger/**/*" + ################ # Shared rules # ################ @@ -932,14 +968,6 @@ - <<: *if-merge-request-title-run-all-rspec when: always -.rails:rules:rspec-feature-flags: - rules: - - <<: *if-not-ee - when: never - - <<: *if-default-branch-schedule-2-hourly - allow_failure: true - - <<: *if-merge-request-title-run-all-rspec - .rails:rules:default-branch-schedule-nightly--code-backstage: rules: - <<: *if-default-branch-schedule-nightly @@ -954,6 +982,12 @@ - <<: *if-merge-request changes: [".gitlab/ci/rails.gitlab-ci.yml"] +.rails:rules:rspec-feature-flags: + rules: + - <<: *if-not-ee + when: never + - changes: *code-backstage-patterns + ######################### # Static analysis rules # ######################### @@ -1005,7 +1039,6 @@ rules: - if: '$CODE_QUALITY_DISABLED' when: never - # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-patterns allow_failure: true @@ -1014,7 +1047,6 @@ rules: - if: '$SAST_DISABLED || $GITLAB_FEATURES !~ /\bsast\b/' when: never - # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs changes: *code-backstage-qa-patterns allow_failure: true @@ -1025,17 +1057,39 @@ when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # The Secret-Detection template already has a `secret_detection_default_branch` job when: never - # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - changes: *code-backstage-qa-patterns allow_failure: true -.reports:rules:dependency_scanning: +.reports:rules:gemnasium-dependency_scanning: rules: - - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/' + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/' when: never - # - <<: *if-default-branch-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 - <<: *if-default-refs - changes: *code-backstage-qa-patterns + changes: *dependency-patterns + allow_failure: true + +.reports:rules:bundler-audit-dependency_scanning: + rules: + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /bundler-audit/' + when: never + - <<: *if-default-refs + changes: *bundler-patterns + allow_failure: true + +.reports:rules:retire-js-dependency_scanning: + rules: + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /retire.js/' + when: never + - <<: *if-default-refs + changes: *nodejs-patterns + allow_failure: true + +.reports:rules:gemnasium-python-dependency_scanning: + rules: + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/ || $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/' + when: never + - <<: *if-default-refs + changes: *python-patterns allow_failure: true .reports:rules:dast: @@ -1057,12 +1111,22 @@ - <<: *if-default-branch-schedule-nightly allow_failure: true -.reports:rules:package_hunter: +.reports:rules:package_hunter-yarn: rules: + - if: "$PACKAGE_HUNTER_USER == null || $PACKAGE_HUNTER_USER == ''" + when: never - <<: *if-default-branch-schedule-2-hourly - <<: *if-merge-request changes: ["yarn.lock"] +.reports:rules:package_hunter-bundler: + rules: + - if: "$PACKAGE_HUNTER_USER == null || $PACKAGE_HUNTER_USER == ''" + when: never + - <<: *if-default-branch-schedule-2-hourly + - <<: *if-merge-request + changes: ["Gemfile.lock"] + .reports:rules:license_scanning: rules: - if: '$LICENSE_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\blicense_scanning\b/' @@ -1126,7 +1190,7 @@ - <<: *if-dot-com-gitlab-org-schedule allow_failure: true -.review:rules:review-stop-failed-deployment: +.review:rules:review-delete-deployment: rules: - <<: *if-not-ee when: never @@ -1146,19 +1210,31 @@ changes: *code-qa-patterns allow_failure: true +# The rule needs to be duplicated between `on_success` and `on_failure` +# because the jobs `needs` the previous job to complete. +# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 .review:rules:review-qa-smoke-report: rules: - <<: *if-not-ee when: never - <<: *if-dot-com-gitlab-org-merge-request changes: *ci-review-patterns - when: always + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *ci-review-patterns + when: on_failure - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - when: always + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *frontend-patterns + when: on_failure - <<: *if-dot-com-gitlab-org-merge-request changes: *code-qa-patterns - when: always + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: on_failure .review:rules:review-qa-all: rules: @@ -1172,6 +1248,9 @@ changes: *qa-patterns allow_failure: true +# The rule needs to be duplicated between `on_success` and `on_failure` +# because the jobs `needs` the previous job to complete. +# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63844#note_599012559 .review:rules:review-qa-all-report: rules: - <<: *if-not-ee @@ -1183,7 +1262,11 @@ - <<: *if-dot-com-gitlab-org-merge-request changes: *qa-patterns allow_failure: true - when: always + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *qa-patterns + allow_failure: true + when: on_failure .review:rules:review-cleanup: rules: @@ -1211,6 +1294,11 @@ rules: - if: '$CI_MERGE_REQUEST_IID' +.review:rules:danger-local: + rules: + - if: '$CI_MERGE_REQUEST_IID' + changes: *code-backstage-danger-patterns + ############### # Setup rules # ############### diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 27b68115edc..f2d5d872d64 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -52,7 +52,7 @@ no_ee_check: verify-tests-yml: extends: - .setup:rules:verify-tests-yml - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine + image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 stage: test needs: [] script: |