diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 15:26:25 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 15:26:25 +0300 |
commit | a09983ae35713f5a2bbb100981116d31ce99826e (patch) | |
tree | 2ee2af7bd104d57086db360a7e6d8c9d5d43667a /.gitlab | |
parent | 18c5ab32b738c0b6ecb4d0df3994000482f34bd8 (diff) |
Add latest changes from gitlab-org/gitlab@13-2-stable-ee
Diffstat (limited to '.gitlab')
-rw-r--r-- | .gitlab/CODEOWNERS | 14 | ||||
-rw-r--r-- | .gitlab/ci/build-images.gitlab-ci.yml | 31 | ||||
-rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 9 | ||||
-rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 49 | ||||
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 21 | ||||
-rw-r--r-- | .gitlab/ci/qa.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 362 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 13 | ||||
-rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 321 | ||||
-rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 199 | ||||
-rw-r--r-- | .gitlab/ci/setup.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | .gitlab/ci/yaml.gitlab-ci.yml | 4 | ||||
-rw-r--r-- | .gitlab/issue_templates/Feature proposal.md | 9 | ||||
-rw-r--r-- | .gitlab/issue_templates/Security developer workflow.md | 12 | ||||
-rw-r--r-- | .gitlab/merge_request_templates/Documentation.md | 6 | ||||
-rw-r--r-- | .gitlab/merge_request_templates/Security Release.md | 22 |
16 files changed, 711 insertions, 363 deletions
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 6cd316349c8..4e2c4aa5c76 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -13,6 +13,7 @@ /doc/development/ @marcia @mjang1 /doc/development/documentation/ @mikelewis /doc/ci @marcel.amirault @sselhorn +/doc/operations @aqualls @eread /doc/user/clusters @aqualls /doc/user/infrastructure @aqualls /doc/user/project/clusters @aqualls @@ -43,17 +44,12 @@ # Feature specific owners /ee/lib/ee/gitlab/auth/ldap/ @dblessing @mkozono /lib/gitlab/auth/ldap/ @dblessing @mkozono -/lib/gitlab/ci/templates/ @nolith @zj +/lib/gitlab/ci/templates/ @nolith @dosuken123 /lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @DylanGriffith @mayra-cabrera @tkuah /lib/gitlab/ci/templates/Security/ @plafoucriere @gonzoyumo @twoodham @sethgitlab /ee/app/models/project_alias.rb @patrickbajao /ee/lib/api/project_aliases.rb @patrickbajao -# Code Owners -# -/ee/lib/gitlab/code_owners/ @reprazent @kerrizor @garyh -/doc/user/project/code_owners.md @reprazent @kerrizor @garyh - # Quality owned files /qa/ @gl-quality @@ -77,3 +73,9 @@ Dangerfile @gl-quality/eng-prod /lib/gitlab/usage_data.rb @gitlab-org/growth/telemetry /lib/gitlab/cycle_analytics/usage_data.rb @gitlab-org/growth/telemetry /lib/gitlab/usage_data_counters/ @gitlab-org/growth/telemetry + +[Code Owners] +/ee/lib/gitlab/code_owners.rb @reprazent @kerrizor @garyh +/ee/lib/gitlab/code_owners/ @reprazent @kerrizor @garyh +/ee/spec/lib/gitlab/code_owners/ @reprazent @kerrizor @garyh +/doc/user/project/code_owners.md @reprazent @kerrizor @garyh diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml new file mode 100644 index 00000000000..e6c3e7598d3 --- /dev/null +++ b/.gitlab/ci/build-images.gitlab-ci.yml @@ -0,0 +1,31 @@ +# This image is used by the `review-qa-*` jobs. Not currently used by the `omnibus-gitlab` pipelines which rebuild this +# image, e.g. https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/jobs/587107399, which we could probably avoid. +# See https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5429. +build-qa-image: + extends: + - .use-kaniko + - .build-images:rules:build-qa-image + stage: build-images + needs: [] + script: + - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" + - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --cache=true + retry: 2 + +# This image is used by: +# - The `CNG` pipelines (via the `review-build-cng` job): https://gitlab.com/gitlab-org/build/CNG/-/blob/cfc67136d711e1c8c409bf8e57427a644393da2f/.gitlab-ci.yml#L335 +# - The `omnibus-gitlab` pipelines (via the `package-and-qa` job): https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/dfd1ad475868fc84e91ab7b5706aa03e46dc3a86/.gitlab-ci.yml#L130 +build-assets-image: + extends: + - .use-kaniko + - .build-images:rules:build-assets-image + stage: build-images + needs: ["compile-production-assets"] + variables: + GIT_DEPTH: "1" + script: + # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists + # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines + # https://gitlab.com/gitlab-org/gitlab/issues/208389 + - run_timed_command "scripts/build_assets_image" + retry: 2 diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 5a6f2aacf93..8745e7d8e9e 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -59,6 +59,15 @@ docs lint: # Check the internal anchor links - bundle exec nanoc check internal_anchors +ui-docs-links lint: + extends: + - .docs:rules:docs-lint + - .static-analysis-base + stage: test + needs: [] + script: + - bundle exec haml-lint -i DocumentationLinks + graphql-reference-verify: extends: - .default-retry diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 4403187d422..084a48a7fc6 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -2,16 +2,18 @@ extends: - .default-retry - .default-before_script - - .assets-compile-cache variables: SETUP_DB: "false" # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 - WEBPACK_VENDOR_DLL: "true" .compile-assets-base: - extends: .frontend-base + extends: + - .frontend-base + - .assets-compile-cache image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.27-lfs-2.9-node-12.x-yarn-1.21-graphicsmagick-1.3.34 + variables: + WEBPACK_VENDOR_DLL: "true" stage: prepare script: - node --version @@ -90,21 +92,6 @@ update-yarn-cache: cache: policy: push -build-assets-image: - extends: - - .use-kaniko - - .frontend:rules:compile-production-assets - stage: build-images - needs: ["compile-production-assets"] - variables: - GIT_DEPTH: "1" - script: - # TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists - # We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines - # https://gitlab.com/gitlab-org/gitlab/issues/208389 - - run_timed_command "scripts/build_assets_image" - retry: 2 - .frontend-fixtures-base: extends: - .frontend-base @@ -114,6 +101,7 @@ build-assets-image: needs: ["setup-test-env", "compile-test-assets"] variables: SETUP_DB: "true" + WEBPACK_VENDOR_DLL: "true" script: - run_timed_command "scripts/gitaly-test-build" - run_timed_command "scripts/gitaly-test-spawn" @@ -138,22 +126,25 @@ frontend-fixtures-as-if-foss: .frontend-test-base: extends: - - .default-retry + - .frontend-base - .yarn-cache variables: USE_BUNDLE_INSTALL: "false" - SETUP_DB: "false" stage: test - before_script: - - source scripts/utils.sh + +eslint-as-if-foss: + extends: + - .frontend-test-base + - .frontend:rules:eslint-as-if-foss + - .as-if-foss + needs: [] + script: + - run_timed_command "retry yarn install --frozen-lockfile" + - yarn run eslint .karma-base: extends: .frontend-test-base - variables: - # we override the max_old_space_size to prevent OOM errors - NODE_OPTIONS: --max_old_space_size=3584 script: - - source scripts/utils.sh - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log - run_timed_command "retry yarn install --frozen-lockfile" - run_timed_command "yarn karma" @@ -174,6 +165,7 @@ karma: - tmp/tests/frontend/ reports: junit: junit_karma.xml + cobertura: coverage-javascript/cobertura-coverage.xml karma-as-if-foss: extends: @@ -185,7 +177,6 @@ karma-as-if-foss: .jest-base: extends: .frontend-test-base script: - - source scripts/utils.sh - run_timed_command "retry yarn install --frozen-lockfile" - run_timed_command "yarn jest --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js" @@ -211,7 +202,6 @@ jest-integration: - .frontend-test-base - .frontend:rules:default-frontend-jobs script: - - source scripts/utils.sh - run_timed_command "retry yarn install --frozen-lockfile" - run_timed_command "yarn jest:integration --ci" needs: ["frontend-fixtures"] @@ -236,11 +226,14 @@ coverage-frontend: - run_timed_command "retry yarn install --frozen-lockfile" script: - run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js" + coverage: '/^Statements\s*:\s*?(\d+(?:\.\d+)?)%/' artifacts: name: coverage-frontend expire_in: 31d paths: - coverage-frontend/ + reports: + cobertura: coverage-frontend/cobertura-coverage.xml .qa-frontend-node: extends: diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 30e3abf13be..3101a42c058 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -18,7 +18,7 @@ .rails-cache: cache: - key: "rails-v1" + key: "rails-v2" paths: - vendor/ruby/ - vendor/gitaly-ruby/ @@ -72,6 +72,15 @@ variables: POSTGRES_HOST_AUTH_METHOD: trust +.use-pg12: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34" + services: + - name: postgres:12 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + variables: + POSTGRES_HOST_AUTH_METHOD: trust + .use-pg11-ee: image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34" services: @@ -82,6 +91,16 @@ variables: POSTGRES_HOST_AUTH_METHOD: trust +.use-pg12-ee: + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34" + services: + - name: postgres:12 + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + - name: redis:alpine + - name: elasticsearch:6.4.2 + variables: + POSTGRES_HOST_AUTH_METHOD: trust + .use-kaniko: image: name: gcr.io/kaniko-project/executor:debug-v0.20.0 diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 20527b690a7..9a81ea513b7 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -49,7 +49,6 @@ update-qa-cache: .package-and-qa-base: image: ruby:2.6-alpine stage: qa - dependencies: [] retry: 0 script: - source scripts/utils.sh diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index f73e0c1d503..4cef4ee26ff 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,9 +1,129 @@ +###################### +# rspec job base specs .rails-job-base: extends: - .default-retry - .default-before_script - .rails-cache +.rspec-base: + extends: .rails-job-base + stage: test + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"] + script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - source scripts/rspec_helpers.sh + - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration" + artifacts: + expire_in: 31d + when: always + paths: + - coverage/ + - knapsack/ + - rspec_flaky/ + - rspec_profiling/ + - tmp/capybara/ + - tmp/memory_test/ + - log/*.log + reports: + junit: junit_rspec.xml + +.rspec-base-migration: + extends: .rails:rules:ee-and-foss-migration + script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - source scripts/rspec_helpers.sh + - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration" + +.rspec-base-pg11: + extends: + - .rspec-base + - .use-pg11 + +.rspec-base-pg12: + extends: + - .rspec-base + - .use-pg12 + +.rspec-base-pg11-as-if-foss: + extends: + - .rspec-base + - .as-if-foss + - .use-pg11 + needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"] + +.rspec-ee-base-pg11: + extends: + - .rspec-base + - .use-pg11-ee + +.rspec-ee-base-pg12: + extends: + - .rspec-base + - .use-pg12-ee + +.rspec-ee-base-geo: + extends: .rspec-base + script: + - run_timed_command "scripts/gitaly-test-build" + - run_timed_command "scripts/gitaly-test-spawn" + - source scripts/rspec_helpers.sh + - scripts/prepare_postgres_fdw.sh + - rspec_paralellized_job "--tag ~quarantine --tag geo" + +.rspec-ee-base-geo-pg11: + extends: + - .rspec-ee-base-geo + - .use-pg11-ee + +.rspec-ee-base-geo-pg12: + extends: + - .rspec-ee-base-geo + - .use-pg12-ee + +.db-job-base: + extends: + - .rails-job-base + - .rails:rules:ee-and-foss-migration + - .use-pg11 + stage: test + needs: ["setup-test-env"] +# rspec job base specs +###################### + +############################ +# rspec job parallel configs +.rspec-migration-parallel: + parallel: 5 + +.rspec-ee-migration-parallel: + parallel: 2 + +.rspec-unit-parallel: + parallel: 20 + +.rspec-ee-unit-parallel: + parallel: 10 + +.rspec-ee-unit-geo-parallel: + parallel: 2 + +.rspec-integration-parallel: + parallel: 8 + +.rspec-ee-integration-parallel: + parallel: 4 + +.rspec-system-parallel: + parallel: 24 + +.rspec-ee-system-parallel: + parallel: 6 +# rspec job parallel configs +############################ + ####################################################### # EE/FOSS: default refs (MRs, master, schedules) jobs # setup-test-env: @@ -86,73 +206,37 @@ downtime_check: script: - bundle exec rake downtime_check -.rspec-base: - extends: .rails-job-base - stage: test - needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"] - script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh - - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration" - artifacts: - expire_in: 31d - when: always - paths: - - coverage/ - - knapsack/ - - rspec_flaky/ - - rspec_profiling/ - - tmp/capybara/ - - tmp/memory_test/ - - log/*.log - reports: - junit: junit_rspec.xml - -.rspec-base-pg11: - extends: - - .rspec-base - - .rails:rules:ee-and-foss - - .use-pg11 - -.rspec-base-migration: - script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh - - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration" - rspec migration pg11: extends: - .rspec-base-pg11 - .rspec-base-migration - parallel: 5 + - .rspec-migration-parallel rspec unit pg11: - extends: .rspec-base-pg11 - parallel: 20 + extends: + - .rspec-base-pg11 + - .rails:rules:ee-and-foss-unit + - .rspec-unit-parallel rspec integration pg11: - extends: .rspec-base-pg11 - parallel: 8 + extends: + - .rspec-base-pg11 + - .rails:rules:ee-and-foss-integration + - .rspec-integration-parallel rspec system pg11: - extends: .rspec-base-pg11 - parallel: 24 + extends: + - .rspec-base-pg11 + - .rails:rules:ee-and-foss-system + - .rspec-system-parallel rspec fast_spec_helper: - extends: .rspec-base-pg11 + extends: + - .rspec-base-pg11 + - .rails:rules:ee-and-foss-fast_spec_helper script: - bin/rspec spec/fast_spec_helper.rb -.db-job-base: - extends: - - .rails-job-base - - .rails:rules:ee-and-foss - - .use-pg11 - stage: test - needs: ["setup-test-env"] - db:migrate:reset: extends: .db-job-base script: @@ -216,7 +300,7 @@ gitlab:setup: rspec:coverage: extends: - .rails-job-base - - .rails:rules:ee-mr-and-master-only + - .rails:rules:rspec-coverage stage: post-test # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) # so we use `dependencies` here. @@ -248,118 +332,180 @@ rspec:coverage: - coverage/index.html - coverage/assets/ - tmp/memory_test/ + reports: + cobertura: coverage/coverage.xml # EE/FOSS: default refs (MRs, master, schedules) jobs # ####################################################### ################################################## # EE: default refs (MRs, master, schedules) jobs # -.rspec-base-ee: - extends: - - .rspec-base - - .rails:rules:ee-only - -.rspec-base-pg11-as-if-foss: - extends: - - .rspec-base - - .rails:rules:as-if-foss - - .as-if-foss - - .use-pg11 - needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"] - -.rspec-ee-base-pg11: - extends: - - .rspec-base-ee - - .use-pg11-ee - rspec migration pg11-as-if-foss: extends: - .rspec-base-pg11-as-if-foss - .rspec-base-migration - parallel: 5 + - .rails:rules:as-if-foss-migration + - .rspec-migration-parallel rspec unit pg11-as-if-foss: - extends: .rspec-base-pg11-as-if-foss - parallel: 20 + extends: + - .rspec-base-pg11-as-if-foss + - .rails:rules:as-if-foss-unit + - .rspec-unit-parallel rspec integration pg11-as-if-foss: - extends: .rspec-base-pg11-as-if-foss - parallel: 8 + extends: + - .rspec-base-pg11-as-if-foss + - .rails:rules:as-if-foss-integration + - .rspec-integration-parallel rspec system pg11-as-if-foss: - extends: .rspec-base-pg11-as-if-foss - parallel: 24 + extends: + - .rspec-base-pg11-as-if-foss + - .rails:rules:as-if-foss-system + - .rspec-system-parallel rspec-ee migration pg11: extends: - .rspec-ee-base-pg11 - .rspec-base-migration - parallel: 2 + - .rails:rules:ee-only-migration + - .rspec-ee-migration-parallel rspec-ee unit pg11: - extends: .rspec-ee-base-pg11 - parallel: 10 + extends: + - .rspec-ee-base-pg11 + - .rails:rules:ee-only-unit + - .rspec-ee-unit-parallel rspec-ee integration pg11: - extends: .rspec-ee-base-pg11 - parallel: 4 + extends: + - .rspec-ee-base-pg11 + - .rails:rules:ee-only-integration + - .rspec-ee-integration-parallel rspec-ee system pg11: - extends: .rspec-ee-base-pg11 - parallel: 6 - -.rspec-ee-base-geo: - extends: .rspec-base-ee - script: - - run_timed_command "scripts/gitaly-test-build" - - run_timed_command "scripts/gitaly-test-spawn" - - source scripts/rspec_helpers.sh - - scripts/prepare_postgres_fdw.sh - - rspec_paralellized_job "--tag ~quarantine --tag geo" - -.rspec-ee-base-geo-pg11: extends: - - .rspec-ee-base-geo - - .use-pg11-ee + - .rspec-ee-base-pg11 + - .rails:rules:ee-only-system + - .rspec-ee-system-parallel rspec-ee unit pg11 geo: - extends: .rspec-ee-base-geo-pg11 - parallel: 2 + extends: + - .rspec-ee-base-geo-pg11 + - .rails:rules:ee-only-unit + - .rspec-ee-unit-geo-parallel rspec-ee integration pg11 geo: - extends: .rspec-ee-base-geo-pg11 + extends: + - .rspec-ee-base-geo-pg11 + - .rails:rules:ee-only-integration rspec-ee system pg11 geo: - extends: .rspec-ee-base-geo-pg11 + extends: + - .rspec-ee-base-geo-pg11 + - .rails:rules:ee-only-system db:rollback geo: extends: - db:rollback - - .rails:rules:ee-only + - .rails:rules:ee-only-migration script: - bundle exec rake geo:db:migrate VERSION=20170627195211 - bundle exec rake geo:db:migrate # EE: default refs (MRs, master, schedules) jobs # ################################################## +########################################## +# EE/FOSS: master nightly scheduled jobs # +rspec migration pg12: + extends: + - .rspec-base-pg12 + - .rspec-base-migration + - .rails:rules:master-schedule-nightly--code-backstage + - .rspec-migration-parallel + +rspec unit pg12: + extends: + - .rspec-base-pg12 + - .rails:rules:master-schedule-nightly--code-backstage + - .rspec-unit-parallel + +rspec integration pg12: + extends: + - .rspec-base-pg12 + - .rails:rules:master-schedule-nightly--code-backstage + - .rspec-integration-parallel + +rspec system pg12: + extends: + - .rspec-base-pg12 + - .rails:rules:master-schedule-nightly--code-backstage + - .rspec-system-parallel +# EE/FOSS: master nightly scheduled jobs # +########################################## + +##################################### +# EE: master nightly scheduled jobs # +rspec-ee migration pg12: + extends: + - .rspec-ee-base-pg12 + - .rspec-base-migration + - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rspec-ee-migration-parallel + +rspec-ee unit pg12: + extends: + - .rspec-ee-base-pg12 + - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rspec-ee-unit-parallel + +rspec-ee integration pg12: + extends: + - .rspec-ee-base-pg12 + - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rspec-ee-integration-parallel + +rspec-ee system pg12: + extends: + - .rspec-ee-base-pg12 + - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rspec-ee-system-parallel + +rspec-ee unit pg12 geo: + extends: + - .rspec-ee-base-geo-pg12 + - .rails:rules:master-schedule-nightly--code-backstage-ee-only + - .rspec-ee-unit-geo-parallel + +rspec-ee integration pg12 geo: + extends: + - .rspec-ee-base-geo-pg12 + - .rails:rules:master-schedule-nightly--code-backstage-ee-only + +rspec-ee system pg12 geo: + extends: + - .rspec-ee-base-geo-pg12 + - .rails:rules:master-schedule-nightly--code-backstage-ee-only +# EE: master nightly scheduled jobs # +##################################### + ################################################## # EE: Canonical MR pipelines rspec foss-impact: extends: - - .rspec-base - - .as-if-foss + - .rspec-base-pg11-as-if-foss - .rails:rules:ee-mr-only - - .use-pg11 script: - install_gitlab_gem - run_timed_command "scripts/gitaly-test-build" - run_timed_command "scripts/gitaly-test-spawn" - source scripts/rspec_helpers.sh - tooling/bin/find_foss_tests tmp/matching_foss_tests.txt - - rspec_matched_tests tmp/matching_foss_tests.txt "--tag ~quarantine --tag ~geo --tag ~level:migration" + - rspec_matched_tests tmp/matching_foss_tests.txt "--tag ~quarantine" artifacts: expire_in: 7d paths: - tmp/matching_foss_tests.txt - tmp/capybara/ -# EE: Merge Request pipelines +# EE: Canonical MR pipelines ################################################## diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 65abb6c5cba..228747ae8d3 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -15,7 +15,7 @@ code_quality: stage: test needs: [] variables: - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.9" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10" script: - | if ! docker info &>/dev/null; then @@ -59,6 +59,7 @@ code_quality: SAST_ANALYZER_IMAGE_TAG: 2 SAST_BRAKEMAN_LEVEL: 2 # GitLab-specific SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec # GitLab-specific + SAST_DISABLE_BABEL: "true" script: - /analyzer run @@ -72,11 +73,10 @@ eslint-sast: image: name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" -# Temporary disabled as it's constantly failing. See https://gitlab.com/gitlab-org/gitlab/-/issues/213769. -# nodejs-scan-sast: -# extends: .sast -# image: -# name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" +nodejs-scan-sast: + extends: .sast + image: + name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" secrets-sast: extends: .sast @@ -172,6 +172,7 @@ dependency_scanning: # # - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' # # - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' # - /analyze -t $DAST_WEBSITE +# timeout: 4h # artifacts: # paths: # - gl-dast-report.json # GitLab-specific diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index 6898da95c15..4e3a80372a6 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,14 +1,3 @@ -build-qa-image: - extends: - - .use-kaniko - - .review:rules:build-qa-image - stage: build-images - needs: [] - script: - - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" - - /kaniko/executor --context=${CI_PROJECT_DIR} --dockerfile=${CI_PROJECT_DIR}/qa/Dockerfile --destination=${QA_IMAGE} --cache=true - retry: 2 - review-cleanup: extends: - .default-retry @@ -27,25 +16,24 @@ review-cleanup: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb - gcp_cleanup -# Temporarily disabling review apps -#review-build-cng: -# extends: -# - .default-retry -# - .review:rules:review-build-cng -# image: ruby:2.6-alpine -# stage: review-prepare -# before_script: -# - source scripts/utils.sh -# - install_api_client_dependencies_with_apk -# - install_gitlab_gem -# needs: -# - job: compile-production-assets -# artifacts: false -# script: -# - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng -# # When the job is manual, review-deploy is also manual and we don't want people -# # to have to manually start the jobs in sequence, so we do it for them. -# - '[ -z $CI_JOB_MANUAL ] || play_job "review-deploy"' +review-build-cng: + extends: + - .default-retry + - .review:rules:review-build-cng + image: ruby:2.6-alpine + stage: review-prepare + before_script: + - source scripts/utils.sh + - install_api_client_dependencies_with_apk + - install_gitlab_gem + needs: + - job: compile-production-assets + artifacts: false + script: + - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng + # When the job is manual, review-deploy is also manual and we don't want people + # to have to manually start the jobs in sequence, so we do it for them. + - '[ -z $CI_JOB_MANUAL ] || play_job "review-deploy"' .review-workflow-base: extends: @@ -53,45 +41,46 @@ review-cleanup: image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14 variables: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" + REVIEW_APPS_DOMAIN: "temp.gitlab-review.app" # FIXME: using temporary domain DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "master" + GITLAB_HELM_CHART_REF: "v4.1.3" environment: name: review/${CI_COMMIT_REF_NAME} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop auto_stop_in: 48 hours -# Temporarily disabling review apps -#review-deploy: -# extends: -# - .review-workflow-base -# - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise -# stage: review -# dependencies: [] -# resource_group: "review/${CI_COMMIT_REF_NAME}" -# before_script: -# - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) -# - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) -# - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) -# - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt -# - source ./scripts/utils.sh -# - install_api_client_dependencies_with_apk -# - source scripts/review_apps/review-apps.sh -# script: -# - check_kube_domain -# - ensure_namespace -# - install_external_dns -# - download_chart -# - date -# - deploy || (display_deployment_debug && exit 1) -# # When the job is manual, review-qa-smoke is also manual and we don't want people -# # to have to manually start the jobs in sequence, so we do it for them. -# - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"' -# - '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"' -# artifacts: -# paths: [environment_url.txt] -# expire_in: 2 days -# when: always +review-deploy: + extends: + - .review-workflow-base + - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise + stage: review + dependencies: [] + resource_group: "review/${CI_COMMIT_REF_NAME}" + before_script: + - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) + - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) + - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) + - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt + - source ./scripts/utils.sh + - install_api_client_dependencies_with_apk + - source scripts/review_apps/review-apps.sh + script: + - check_kube_domain + - ensure_namespace + - install_external_dns + - download_chart + - date + - deploy || (display_deployment_debug && exit 1) + - disable_sign_ups + # When the job is manual, review-qa-smoke is also manual and we don't want people + # to have to manually start the jobs in sequence, so we do it for them. + - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"' + - '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"' + artifacts: + paths: [environment_url.txt] + expire_in: 2 days + when: always .review-stop-base: extends: .review-workflow-base @@ -124,110 +113,110 @@ review-stop: script: - delete_release -# Temporarily disabling review apps -#.review-qa-base: -# extends: -# - .default-retry -# - .use-docker-in-docker -# image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6 -# stage: qa -# # This is needed so that manual jobs with needs don't block the pipeline. -# # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. -# dependencies: ["review-deploy"] -# variables: -# QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" -# QA_CAN_TEST_GIT_PROTOCOL_V2: "false" -# QA_DEBUG: "true" -# GITLAB_USERNAME: "root" -# GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" -# GITLAB_ADMIN_USERNAME: "root" -# GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" -# GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" -# EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" -# before_script: -# - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" -# - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" -# - echo "${CI_ENVIRONMENT_URL}" -# - echo "${QA_IMAGE}" -# - source scripts/utils.sh -# - install_api_client_dependencies_with_apk -# - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} -# artifacts: -# paths: -# - ./qa/gitlab-qa-run-* -# expire_in: 7 days -# when: always -# -#review-qa-smoke: -# extends: -# - .review-qa-base -# - .review:rules:review-qa-smoke -# script: -# - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -# -#review-qa-all: -# extends: -# - .review-qa-base -# - .review:rules:mr-only-manual -# parallel: 5 -# script: -# - export KNAPSACK_REPORT_PATH=knapsack/master_report.json -# - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb -# - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation -# -#review-performance: -# extends: -# - .default-retry -# - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise -# image: -# name: sitespeedio/sitespeed.io:6.3.1 -# entrypoint: [""] -# stage: qa -# # This is needed so that manual jobs with needs don't block the pipeline. -# # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. -# dependencies: ["review-deploy"] -# before_script: -# - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" -# - echo "${CI_ENVIRONMENT_URL}" -# - mkdir -p gitlab-exporter -# - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js -# - mkdir -p sitespeed-results -# script: -# - /start.sh --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}" -# after_script: -# - mv sitespeed-results/data/performance.json performance.json -# artifacts: -# paths: -# - sitespeed-results/ -# reports: -# performance: performance.json -# expire_in: 31d -# -#parallel-spec-reports: -# extends: -# - .review:rules:mr-only-manual -# image: ruby:2.6-alpine -# stage: post-qa -# dependencies: ["review-qa-all"] -# variables: -# NEW_PARALLEL_SPECS_REPORT: qa/report-new.html -# BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" -# script: -# - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/* -# - gem install nokogiri --no-document -# - cd qa/gitlab-qa-run-*/gitlab-* -# - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_) -# - cd - -# - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}' -# - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm -# artifacts: -# when: always -# paths: -# - qa/report-new.html -# - qa/gitlab-qa-run-* -# reports: -# junit: qa/gitlab-qa-run-*/**/rspec-*.xml -# expire_in: 31d +.review-qa-base: + extends: + - .default-retry + - .use-docker-in-docker + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6 + stage: qa + # This is needed so that manual jobs with needs don't block the pipeline. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. + dependencies: ["review-deploy"] + variables: + QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" + QA_CAN_TEST_GIT_PROTOCOL_V2: "false" + QA_DEBUG: "true" + GITLAB_USERNAME: "root" + GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITLAB_ADMIN_USERNAME: "root" + GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}" + GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}" + EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}" + SIGNUP_DISABLED: "true" + before_script: + - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}" + - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" + - echo "${CI_ENVIRONMENT_URL}" + - echo "${QA_IMAGE}" + - source scripts/utils.sh + - install_api_client_dependencies_with_apk + - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}} + artifacts: + paths: + - ./qa/gitlab-qa-run-* + expire_in: 7 days + when: always + +review-qa-smoke: + extends: + - .review-qa-base + - .review:rules:review-qa-smoke + script: + - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" + +review-qa-all: + extends: + - .review-qa-base + - .review:rules:mr-only-manual + parallel: 5 + script: + - export KNAPSACK_REPORT_PATH=knapsack/master_report.json + - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb + - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation + +review-performance: + extends: + - .default-retry + - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise + image: + name: sitespeedio/sitespeed.io:6.3.1 + entrypoint: [""] + stage: qa + # This is needed so that manual jobs with needs don't block the pipeline. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979. + dependencies: ["review-deploy"] + before_script: + - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" + - echo "${CI_ENVIRONMENT_URL}" + - mkdir -p gitlab-exporter + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js + - mkdir -p sitespeed-results + script: + - /start.sh --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}" + after_script: + - mv sitespeed-results/data/performance.json performance.json + artifacts: + paths: + - sitespeed-results/ + reports: + performance: performance.json + expire_in: 31d + +parallel-spec-reports: + extends: + - .review:rules:mr-only-manual + image: ruby:2.6-alpine + stage: post-qa + dependencies: ["review-qa-all"] + variables: + NEW_PARALLEL_SPECS_REPORT: qa/report-new.html + BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" + script: + - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/* + - gem install nokogiri --no-document + - cd qa/gitlab-qa-run-*/gitlab-* + - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_) + - cd - + - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}' + - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm + artifacts: + when: always + paths: + - qa/report-new.html + - qa/gitlab-qa-run-* + reports: + junit: qa/gitlab-qa-run-*/**/rspec-*.xml + expire_in: 31d danger-review: extends: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index fbbb0391ec5..f508bfa1465 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -11,7 +11,7 @@ if: '$CI_PROJECT_NAME != "gitlab-foss" && $CI_PROJECT_NAME != "gitlab-ce" && $CI_PROJECT_NAME != "gitlabhq"' .if-default-refs: &if-default-refs - if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG' + if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG || $FORCE_GITLAB_CI' .if-master-refs: &if-master-refs if: '$CI_COMMIT_REF_NAME == "master"' @@ -40,6 +40,9 @@ .if-merge-request-title-update-caches: &if-merge-request-title-update-caches if: '$CI_MERGE_REQUEST_TITLE =~ /UPDATE CACHE/' +.if-merge-request-title-run-all-rspec: &if-merge-request-title-run-all-rspec + if: '$CI_MERGE_REQUEST_TITLE =~ /RUN ALL RSPEC/' + .if-security-merge-request: &if-security-merge-request if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID' @@ -71,6 +74,22 @@ - ".gitlab-ci.yml" - ".gitlab/ci/**/*" +.ci-build-images-patterns: &ci-build-images-patterns + - ".gitlab-ci.yml" + - ".gitlab/ci/build-images.gitlab-ci.yml" + +.ci-review-patterns: &ci-review-patterns + - ".gitlab-ci.yml" + - ".gitlab/ci/frontend.gitlab-ci.yml" + - ".gitlab/ci/build-images.gitlab-ci.yml" + - ".gitlab/ci/review.gitlab-ci.yml" + +.ci-qa-patterns: &ci-qa-patterns + - ".gitlab-ci.yml" + - ".gitlab/ci/frontend.gitlab-ci.yml" + - ".gitlab/ci/build-images.gitlab-ci.yml" + - ".gitlab/ci/qa.gitlab-ci.yml" + .yaml-patterns: &yaml-patterns - "**/*.yml" @@ -92,6 +111,21 @@ - "vendor/assets/**/*" - "{,ee/}{app/assets,app/helpers,app/presenters,app/views,locale,public,symbol}/**/*" +.backend-patterns: &backend-patterns + - "Gemfile{,.lock}" + - "Rakefile" + - "config.ru" + # List explicitly all the app/ dirs that are backend (i.e. all except app/assets). + - "{,ee/}{app/channels,app/controllers,app/finders,app/graphql,app/helpers,app/mailers,app/models,app/policies,app/presenters,app/serializers,app/services,app/uploaders,app/validators,app/views,app/workers}/**/*" + - "{,ee/}{bin,cable,config,db,lib}/**/*" + - "{,ee/}spec/**/*.rb" + - ".gitlab-ci.yml" + - ".gitlab/ci/**/*" + +.db-patterns: &db-patterns + - "{,ee/}{,spec/}{db,migrations}/**/*" + - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration/**/*" + .backstage-patterns: &backstage-patterns - "Dangerfile" - "danger/**/*" @@ -197,6 +231,26 @@ - <<: *if-master-schedule-2-hourly - <<: *if-merge-request-title-update-caches +###################### +# Build images rules # +###################### +.build-images:rules:build-qa-image: + rules: + - <<: *if-not-ee + when: never + - <<: *if-dot-com-gitlab-org-and-security-merge-request + changes: *ci-build-images-patterns + - <<: *if-dot-com-gitlab-org-and-security-merge-request + changes: *code-qa-patterns + - <<: *if-dot-com-gitlab-org-schedule + +.build-images:rules:build-assets-image: + rules: + - <<: *if-not-canonical-namespace + when: never + - changes: *ci-build-images-patterns + - changes: *code-qa-patterns + #################### # Cache repo rules # #################### @@ -263,7 +317,7 @@ - <<: *if-not-canonical-namespace when: never - <<: *if-default-refs - changes: *code-backstage-qa-patterns + changes: *code-qa-patterns .frontend:rules:compile-test-assets: rules: @@ -273,11 +327,8 @@ rules: - <<: *if-not-ee when: never - - <<: *if-security-merge-request + - <<: *if-merge-request # Always run for MRs since `compile-test-assets as-if-foss` is either needed by `rspec foss-impact` or the `rspec * as-if-foss` jobs. changes: *code-backstage-qa-patterns - - <<: *if-merge-request-title-as-if-foss - - <<: *if-merge-request - changes: *ci-patterns .frontend:rules:default-frontend-jobs: rules: @@ -294,6 +345,15 @@ - <<: *if-merge-request changes: *ci-patterns +.frontend:rules:eslint-as-if-foss: + rules: + - <<: *if-not-ee + when: never + - <<: *if-merge-request-title-as-if-foss + when: never + - <<: *if-merge-request + changes: *frontend-patterns + .frontend:rules:ee-mr-and-master-only: rules: - <<: *if-not-ee @@ -341,9 +401,7 @@ rules: - <<: *if-not-ee when: never - - <<: *if-dot-com-gitlab-org-master - changes: *code-backstage-qa-patterns - when: on_success + - <<: *if-master-schedule-2-hourly ############ # QA rules # @@ -367,7 +425,7 @@ .qa:rules:package-and-qa: rules: - <<: *if-dot-com-gitlab-org-and-security-merge-request - changes: *ci-patterns + changes: *ci-qa-patterns allow_failure: true - <<: *if-dot-com-gitlab-org-and-security-merge-request changes: *qa-patterns @@ -382,24 +440,95 @@ ############### # Rails rules # ############### -.rails:rules:ee-and-foss: +.rails:rules:ee-and-foss-migration: rules: - - <<: *if-default-refs - changes: *code-backstage-patterns + - changes: *db-patterns + - <<: *if-merge-request-title-run-all-rspec + +.rails:rules:ee-and-foss-unit: + rules: + - changes: *backend-patterns + - <<: *if-merge-request-title-run-all-rspec + +.rails:rules:ee-and-foss-integration: + rules: + - changes: *backend-patterns + - <<: *if-merge-request-title-run-all-rspec + +.rails:rules:ee-and-foss-system: + rules: + - changes: *code-backstage-patterns + - <<: *if-merge-request-title-run-all-rspec + +.rails:rules:ee-and-foss-fast_spec_helper: + rules: + - changes: ["config/**/*"] + - <<: *if-merge-request-title-run-all-rspec .rails:rules:default-refs-code-backstage-qa: rules: - <<: *if-default-refs changes: *code-backstage-qa-patterns -.rails:rules:ee-only: +.rails:rules:ee-only-migration: rules: - <<: *if-not-ee when: never - - <<: *if-default-refs - changes: *code-backstage-patterns + - changes: *db-patterns + - <<: *if-merge-request-title-run-all-rspec -.rails:rules:as-if-foss: +.rails:rules:ee-only-unit: + rules: + - <<: *if-not-ee + when: never + - changes: *backend-patterns + - <<: *if-merge-request-title-run-all-rspec + +.rails:rules:ee-only-integration: + rules: + - <<: *if-not-ee + when: never + - changes: *backend-patterns + - <<: *if-merge-request-title-run-all-rspec + +.rails:rules:ee-only-system: + rules: + - <<: *if-not-ee + when: never + - changes: *code-backstage-patterns + - <<: *if-merge-request-title-run-all-rspec + +.rails:rules:as-if-foss-migration: + rules: + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *db-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns + +.rails:rules:as-if-foss-unit: + rules: + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *backend-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns + +.rails:rules:as-if-foss-integration: + rules: + - <<: *if-not-ee + when: never + - <<: *if-security-merge-request + changes: *backend-patterns + - <<: *if-merge-request-title-as-if-foss + - <<: *if-merge-request + changes: *ci-patterns + +.rails:rules:as-if-foss-system: rules: - <<: *if-not-ee when: never @@ -413,6 +542,7 @@ rules: - <<: *if-not-ee when: never + - <<: *if-merge-request-title-run-all-rspec - <<: *if-merge-request changes: *code-backstage-patterns - <<: *if-master-refs @@ -434,6 +564,27 @@ - <<: *if-merge-request changes: *code-backstage-patterns +.rails:rules:rspec-coverage: + rules: + - <<: *if-not-ee + when: never + - <<: *if-master-schedule-2-hourly + - <<: *if-merge-request-title-run-all-rspec + +.rails:rules:master-schedule-nightly--code-backstage: + rules: + - <<: *if-master-schedule-nightly + - <<: *if-merge-request + changes: [".gitlab/ci/rails.gitlab-ci.yml"] + +.rails:rules:master-schedule-nightly--code-backstage-ee-only: + rules: + - <<: *if-not-ee + when: never + - <<: *if-master-schedule-nightly + - <<: *if-merge-request + changes: [".gitlab/ci/rails.gitlab-ci.yml"] + ################## # Releases rules # ################## @@ -496,18 +647,12 @@ ################ # Review rules # ################ -.review:rules:build-qa-image: +.review:rules:review-build-cng: rules: - <<: *if-not-ee when: never - - <<: *if-dot-com-gitlab-org-and-security-merge-request - changes: *code-qa-patterns - - <<: *if-dot-com-gitlab-org-schedule - -.review:rules:review-build-cng: - rules: - <<: *if-dot-com-gitlab-org-merge-request - changes: *ci-patterns + changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns - <<: *if-dot-com-gitlab-org-merge-request @@ -521,7 +666,7 @@ - <<: *if-not-ee when: never - <<: *if-dot-com-gitlab-org-merge-request - changes: *ci-patterns + changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns allow_failure: true @@ -544,7 +689,7 @@ - <<: *if-not-ee when: never - <<: *if-dot-com-gitlab-org-merge-request - changes: *ci-patterns + changes: *ci-review-patterns - <<: *if-dot-com-gitlab-org-merge-request changes: *frontend-patterns allow_failure: true diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index b878bec3751..26c7a2194cc 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -9,6 +9,7 @@ cache gems: stage: test needs: ["setup-test-env"] variables: + BUNDLE_INSTALL_FLAGS: --with=production --with=development --with=test --jobs=2 --path=vendor --retry=3 --quiet SETUP_DB: "false" script: - bundle package --all --all-platforms diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index ab31dd59299..a650ee7e4b4 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -4,11 +4,11 @@ lint-ci-gitlab: extends: - .default-retry - .yaml:rules - image: sdesbure/yamllint:latest + image: pipelinecomponents/yamllint:latest stage: test needs: [] variables: LINT_PATHS: .gitlab-ci.yml .gitlab/ci lib/gitlab/ci/templates changelogs script: - '[[ ! -d "ee/" ]] || export LINT_PATHS="$LINT_PATHS ee/changelogs"' - - yamllint $LINT_PATHS + - yamllint -f colored $LINT_PATHS diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index 45b5fc85cd1..589310b4cef 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -43,7 +43,14 @@ https://about.gitlab.com/handbook/engineering/ux/ux-research-training/user-story ### Permissions and Security -<!-- What permissions are required to perform the described actions? Are they consistent with the existing permissions as documented for users, groups, and projects as appropriate? Is the proposed behavior consistent between the UI, API, and other access methods (e.g. email replies)?--> +<!-- What permissions are required to perform the described actions? Are they consistent with the existing permissions as documented for users, groups, and projects as appropriate? Is the proposed behavior consistent between the UI, API, and other access methods (e.g. email replies)? +Consider adding checkboxes and expectations of users with certain levels of membership https://docs.gitlab.com/ee/user/permissions.html +* [ ] Add expected impact to members with no access (0) +* [ ] Add expected impact to Guest (10) members +* [ ] Add expected impact to Reporter (20) members +* [ ] Add expected impact to Developer (30) members +* [ ] Add expected impact to Maintainer (40) members +* [ ] Add expected impact to Owner (50) members --> ### Documentation diff --git a/.gitlab/issue_templates/Security developer workflow.md b/.gitlab/issue_templates/Security developer workflow.md index 695f0167ad4..7de137bd2e2 100644 --- a/.gitlab/issue_templates/Security developer workflow.md +++ b/.gitlab/issue_templates/Security developer workflow.md @@ -9,19 +9,17 @@ Set the title to: `Description of the original issue` ## Prior to starting the security release work - [ ] Read the [security process for developers] if you are not familiar with it. -- [ ] Mark this [issue as related] to the Security Release tracking issue. You can find it on the topic of the `#releases` Slack channel. -- [ ] Run `scripts/security-harness` in your local repository to prevent accidentally pushing to any remote besides `gitlab.com/gitlab-org/security`. +- [ ] Mark this [issue as related] to the Security Release Tracking Issue. You can find it on the topic of the `#releases` Slack channel. - Fill out the [Links section](#links): - [ ] Next to **Issue on GitLab**, add a link to the `gitlab-org/gitlab` issue that describes the security vulnerability. - - [ ] Next to **Security Release tracking issue**, add a link to the security release issue that will include this security issue. ## Development +- [ ] Run `scripts/security-harness` in your local repository to prevent accidentally pushing to any remote besides `gitlab.com/gitlab-org/security`. - [ ] Create a new branch prefixing it with `security-`. - [ ] Create a merge request targeting `master` on `gitlab.com/gitlab-org/security` and use the [Security Release merge request template]. -- [ ] Follow the same [code review process]: Assign to a reviewer, then to a maintainer. -After your merge request has been approved according to our [approval guidelines], you're ready to prepare the backports +After your merge request has been approved according to our [approval guidelines] and by a team member of the AppSec team, you're ready to prepare the backports ## Backports @@ -41,7 +39,6 @@ After your merge request has been approved according to our [approval guidelines - [ ] Fill in any upgrade notes that users may need to take into account in the [details section](#details) - [ ] Add Yes/No and further details if needed to the migration and settings columns in the [details section](#details) - [ ] Add the nickname of the external user who found the issue (and/or HackerOne profile) to the Thanks row in the [details section](#details) -- [ ] Once your `master` MR is merged, comment on the original security issue with a link to that MR indicating the issue is fixed. ## Summary @@ -50,7 +47,6 @@ After your merge request has been approved according to our [approval guidelines | Description | Link | | -------- | -------- | | Issue on [GitLab](https://gitlab.com/gitlab-org/gitlab/issues) | #TODO | -| Security Release tracking issue | #TODO | ### Details @@ -64,7 +60,7 @@ After your merge request has been approved according to our [approval guidelines | Thanks | | | [security process for developers]: https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md -[secpick documentation]: https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#secpick-script +[secpick documentation]: https://gitlab.com/gitlab-org/release/docs/-/blob/master/general/security/utilities/secpick_script.md [security Release merge request template]: https://gitlab.com/gitlab-org/security/gitlab/blob/master/.gitlab/merge_request_templates/Security%20Release.md [code review process]: https://docs.gitlab.com/ee/development/code_review.html [approval guidelines]: https://docs.gitlab.com/ee/development/code_review.html#approval-guidelines diff --git a/.gitlab/merge_request_templates/Documentation.md b/.gitlab/merge_request_templates/Documentation.md index 282e80d700a..fb828b995b1 100644 --- a/.gitlab/merge_request_templates/Documentation.md +++ b/.gitlab/merge_request_templates/Documentation.md @@ -45,9 +45,11 @@ All reviewers can help ensure accuracy, clarity, completeness, and adherence to **2. Technical Writer** -- [ ] Optional: Technical writer review. If not requested for this MR, must be scheduled post-merge. To request for this MR, assign the writer listed for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/categories/#devops-stages). - - [ ] Add ~"Technical Writing" and `docs::` workflow label. +- [ ] Technical writer review. If not requested for this MR, must be scheduled post-merge. To request for this MR, assign the writer listed for the applicable [DevOps stage](https://about.gitlab.com/handbook/product/product-categories/#devops-stages). + - [ ] Ensure ~"Technical Writing", ~"documentation", and a `docs::` scoped label are added. - [ ] Add ~docs-only when the only files changed are under `doc/*`. + - [ ] Add ~"tw::doing" when starting work on the MR. + - [ ] Add ~"tw::finished" if Technical Writing team work on the MR is complete but it remains open. **3. Maintainer** diff --git a/.gitlab/merge_request_templates/Security Release.md b/.gitlab/merge_request_templates/Security Release.md index f852bebae95..bdf26041e62 100644 --- a/.gitlab/merge_request_templates/Security Release.md +++ b/.gitlab/merge_request_templates/Security Release.md @@ -13,25 +13,33 @@ See [the general developer security release guidelines](https://gitlab.com/gitla ## Developer checklist - [ ] **On "Related issues" section, write down the [GitLab Security] issue it belongs to (i.e. `Related to <issue_id>`).** -- [ ] Merge request targets `master`, or `X-Y-stable` for backports. +- [ ] Merge request targets `master`, or a versioned stable branch (`X-Y-stable-ee`). - [ ] Milestone is set for the version this merge request applies to. A closed milestone can be assigned via [quick actions]. - [ ] Title of this merge request is the same as for all backports. -- [ ] A [CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html) is added without a `merge_request` value, with `type` set to `security` -- [ ] Assign to a reviewer and maintainer, per our [Code Review process]. +- [ ] A [CHANGELOG entry] is added without a `merge_request` value, with `type` set to `security` - [ ] For the MR targeting `master`: - - [ ] Ask for a non-blocking review from the AppSec team member associated to the issue in the [Canonical repository](https://gitlab.com/gitlab-org/gitlab). If you're unsure who to ping, ask on `#sec-appsec` Slack channel. + - [ ] Assign to a reviewer and maintainer, per our [Code Review process]. - [ ] Ensure it's approved according to our [Approval Guidelines]. -- [ ] Merge request _must not_ close the corresponding security issue, _unless_ it targets `master`. + - [ ] Ensure it's approved by an AppSec engineer. + - If you're unsure who should approve, find the AppSec engineer associated to the issue in the [Canonical repository], or ask #sec-appsec on Slack. + - Trigger the [`package-and-qa` build]. The docker image generated will be used by the AppSec engineer to validate the security vulnerability has been remediated. + - [ ] Merge request _must_ close the corresponding security issue. +- [ ] For a backport MR targeting a versioned stable branch (`X-Y-stable-ee`) + - [ ] Ensure it's approved by a maintainer. **Note:** Reviewer/maintainer should not be a Release Manager ## Maintainer checklist + - [ ] Correct milestone is applied and the title is matching across all backports - [ ] Assigned to `@gitlab-release-tools-bot` with passing CI pipelines and **when all backports including the MR targeting master are ready.** /label ~security [GitLab Security]: https://gitlab.com/gitlab-org/security/gitlab -[approval guidelines]: https://docs.gitlab.com/ee/development/code_review.html#approval-guidelines -[Code Review process]: https://docs.gitlab.com/ee/development/code_review.html [quick actions]: https://docs.gitlab.com/ee/user/project/quick_actions.html#quick-actions-for-issues-merge-requests-and-epics +[CHANGELOG entry]: https://docs.gitlab.com/ee/development/changelog.html +[Code Review process]: https://docs.gitlab.com/ee/development/code_review.html +[Approval Guidelines]: https://docs.gitlab.com/ee/development/code_review.html#approval-guidelines +[Canonical repository]: https://gitlab.com/gitlab-org/gitlab +[`package-and-qa` build]: https://docs.gitlab.com/ee/development/testing_guide/end_to_end/#using-the-package-and-qa-job |