diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-20 15:52:10 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-20 15:52:10 +0300 |
commit | dba864470fbcbb6bdd5b94eb510acdce62c962d8 (patch) | |
tree | e8ead0b84e7b814f5891d2c8cd3db2d6b635fb64 /.gitlab | |
parent | b7d29500f28ff59c8898cdf889a40d3da908f162 (diff) |
Add latest changes from gitlab-org/gitlab@12-8-stable-ee
Diffstat (limited to '.gitlab')
23 files changed, 1105 insertions, 922 deletions
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 19aa96701d8..0b817fe14b5 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -1,10 +1,13 @@ +# When adding a group as a code owner, make sure to invite the group to the +# project here: https://gitlab.com/gitlab-org/gitlab/-/project_members +# As described in https://docs.gitlab.com/ee/user/project/code_owners.html + # Backend Maintainers are the default for all ruby files *.rb @gitlab-org/maintainers/rails-backend *.rake @gitlab-org/maintainers/rails-backend -# Technical writing team are the default reviewers for everything in `doc/` +# Technical writing team are the default reviewers for all markdown docs *.md @gl-docsteam -doc/ @gl-docsteam # Frontend maintainers should see everything in `app/assets/` app/assets/ @gitlab-org/maintainers/frontend @@ -29,10 +32,17 @@ lib/gitlab/github_import/ @gitlab-org/maintainers/database /ee/app/models/project_alias.rb @patrickbajao /ee/lib/api/project_aliases.rb @patrickbajao +# Quality owned files +/qa/ @gl-quality + # Engineering Productivity owned files /.gitlab-ci.yml @gl-quality/eng-prod /.gitlab/ci/ @gl-quality/eng-prod +/.gitlab/CODEOWNERS @gl-quality/eng-prod Dangerfile @gl-quality/eng-prod /danger/ @gl-quality/eng-prod /lib/gitlab/danger/ @gl-quality/eng-prod /scripts/ @gl-quality/eng-prod + +# Delivery owner files +/.gitlab/ci/releases.gitlab-ci.yml @gitlab-org/delivery diff --git a/.gitlab/ci/cache-repo.gitlab-ci.yml b/.gitlab/ci/cache-repo.gitlab-ci.yml index 1162e98e246..ecbed0ed6c8 100644 --- a/.gitlab/ci/cache-repo.gitlab-ci.yml +++ b/.gitlab/ci/cache-repo.gitlab-ci.yml @@ -18,6 +18,7 @@ # runner, or network egress charges will apply: # https://cloud.google.com/storage/pricing cache-repo: + extends: .cache-repo:rules image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine stage: sync allow_failure: true @@ -30,11 +31,7 @@ cache-repo: - git clone --progress $CI_REPOSITORY_URL $CI_PROJECT_NAME - cd $CI_PROJECT_NAME - gcloud auth activate-service-account --key-file=$CI_REPO_CACHE_CREDENTIALS + - git remote rm origin - tar cf $TAR_FILENAME . - gzip $TAR_FILENAME - gsutil cp $TAR_FILENAME.gz gs://gitlab-ci-git-repo-cache/project-$CI_PROJECT_ID/gitlab-master.tar.gz - only: - variables: - - $CI_REPO_CACHE_CREDENTIALS - refs: - - schedules diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index bd11042eb11..2450e346569 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -1,15 +1,11 @@ cloud-native-image: - extends: .only:variables-canonical-dot-com + extends: .cng:rules image: ruby:2.6-alpine dependencies: [] stage: post-test allow_failure: true variables: GIT_DEPTH: "1" - when: manual script: - install_gitlab_gem - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng - only: - refs: - - tags diff --git a/.gitlab/ci/dev-fixtures.gitlab-ci.yml b/.gitlab/ci/dev-fixtures.gitlab-ci.yml index a5dab5d8708..e08f7e969cb 100644 --- a/.gitlab/ci/dev-fixtures.gitlab-ci.yml +++ b/.gitlab/ci/dev-fixtures.gitlab-ci.yml @@ -1,6 +1,9 @@ .run-dev-fixtures: extends: - - .only-code-rails-job-base + - .default-tags + - .default-retry + - .default-cache + - .default-before_script - .use-pg9 stage: test needs: ["setup-test-env"] @@ -13,17 +16,19 @@ SIZE: 0 # number of external projects to fork, requires network connection # SEED_NESTED_GROUPS: "false" # requires network connection -run-dev-fixtures-foss: - extends: .run-dev-fixtures +run-dev-fixtures: + extends: + - .run-dev-fixtures + - .dev-fixtures:rules:ee-and-foss script: - scripts/gitaly-test-spawn - RAILS_ENV=test bundle exec rake db:seed_fu run-dev-fixtures-ee: extends: - - .only-ee - - .use-pg9-ee - .run-dev-fixtures + - .dev-fixtures:rules:ee-only + - .use-pg9-ee script: - scripts/gitaly-test-spawn - cp ee/db/fixtures/development/* $FIXTURE_PATH diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 4acc3c7d1fe..59f1773da2e 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -2,11 +2,8 @@ extends: - .default-tags - .default-retry - - .only:variables-canonical-dot-com - - .only:changes-docs - only: - refs: - - merge_requests + - .docs:rules:review-docs + allow_failure: true image: ruby:2.6-alpine stage: review dependencies: [] @@ -32,7 +29,6 @@ review-docs-deploy: extends: .review-docs script: - ./scripts/trigger-build-docs deploy - when: manual # Cleanup remote environment of gitlab-docs review-docs-cleanup: @@ -42,14 +38,12 @@ review-docs-cleanup: action: stop script: - ./scripts/trigger-build-docs cleanup - when: manual docs lint: extends: - .default-tags - .default-retry - - .default-only - - .only:changes-docs + - .docs:rules:docs-lint image: "registry.gitlab.com/gitlab-org/gitlab-docs:docs-lint" stage: test dependencies: [] @@ -69,13 +63,11 @@ docs lint: graphql-reference-verify: extends: - - .only-ee - .default-tags - .default-retry - .default-cache - - .default-only - .default-before_script - - .only:changes-code-backstage-qa + - .docs:rules:graphql-reference-verify - .use-pg9 stage: test needs: ["setup-test-env"] diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 076de55014e..3a72c941b89 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -1,38 +1,3 @@ -# Make sure to update all the similar conditions in other CI config files if you modify these conditions -.if-default: &if-default - if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG' - -# Make sure to update all the similar conditions in other CI config files if you modify these conditions -.if-default-ee: &if-default-ee - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG) && $CI_PROJECT_NAME =~ /^gitlab(-ee)?$/' - -# Make sure to update all the similar conditions in other CI config files if you modify these conditions -.if-master: &if-master - if: '$CI_COMMIT_REF_NAME == "master"' - -# Make sure to update all the similar patterns in other CI config files if you modify these patterns -.code-backstage-patterns: &code-backstage-patterns - - ".gitlab/ci/**/*" - - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - - ".csscomb.json" - - "Dockerfile.assets" - - "*_VERSION" - - "Gemfile{,.lock}" - - "Rakefile" - - "{babel.config,jest.config}.js" - - "config.ru" - - "{package.json,yarn.lock}" - - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - # Backstage changes - - "Dangerfile" - - "danger/**/*" - - "{,ee/}fixtures/**/*" - - "{,ee/}rubocop/**/*" - - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file - .assets-compile-cache: cache: paths: @@ -46,11 +11,9 @@ extends: - .default-tags - .default-retry - - .default-only - .default-before_script - .assets-compile-cache - - .only:changes-code-backstage-qa - image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-19.03.1 + image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1 stage: prepare services: - docker:19.03.0-dind @@ -65,7 +28,7 @@ DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 cache: - key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:tmp_cache_webpack:v7" + key: "assets-compile:production:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:tmp_cache_webpack:v9" artifacts: name: webpack-report expire_in: 31d @@ -80,24 +43,21 @@ - time scripts/build_assets_image - scripts/clean-old-cached-assets - rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here - only: - variables: - - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/)/ # Matches the gitlab-org group and its subgroups - - $CI_SERVER_HOST == "dev.gitlab.org" tags: - gitlab-org - docker gitlab:assets:compile pull-push-cache: - extends: .gitlab:assets:compile-metadata - only: - refs: - - master + extends: + - .gitlab:assets:compile-metadata + - .frontend:rules:gitlab-assets-compile-pull-push-cache cache: policy: pull-push gitlab:assets:compile pull-cache: - extends: .gitlab:assets:compile-metadata + extends: + - .gitlab:assets:compile-metadata + - .frontend:rules:gitlab-assets-compile-pull-cache cache: policy: pull @@ -105,10 +65,8 @@ gitlab:assets:compile pull-cache: extends: - .default-tags - .default-retry - - .default-only - .default-before_script - .assets-compile-cache - - .only:changes-code-backstage-qa stage: prepare script: - node --version @@ -121,7 +79,7 @@ gitlab:assets:compile pull-cache: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 cache: - key: "assets-compile:v8" + key: "assets-compile:v9" artifacts: expire_in: 7d paths: @@ -129,47 +87,49 @@ gitlab:assets:compile pull-cache: - public/assets compile-assets pull-push-cache: - extends: .compile-assets-metadata - only: - refs: - - master + extends: + - .compile-assets-metadata + - .frontend:rules:compile-assets-pull-push-cache cache: policy: pull-push -compile-assets pull-push-cache foss: - extends: [".compile-assets-metadata", ".only-ee-as-if-foss"] - only: - refs: - - master +compile-assets pull-push-cache as-if-foss: + extends: + - .compile-assets-metadata + - .frontend:rules:compile-assets-pull-push-cache-as-if-foss + - .as-if-foss cache: policy: pull-push - key: "assets-compile:v8:foss" + key: "assets-compile:v9:foss" compile-assets pull-cache: - extends: .compile-assets-metadata + extends: + - .compile-assets-metadata + - .frontend:rules:compile-assets-pull-cache cache: policy: pull -compile-assets pull-cache foss: - extends: [".compile-assets-metadata", ".only-ee-as-if-foss"] +compile-assets pull-cache as-if-foss: + extends: + - .compile-assets-metadata + - .frontend:rules:compile-assets-pull-cache-as-if-foss + - .as-if-foss cache: policy: pull - key: "assets-compile:v8:foss" + key: "assets-compile:v9:foss" -.only-code-frontend-job-base: +.frontend-job-base: extends: - .default-tags - .default-retry - .default-cache - - .default-only - .default-before_script - - .only:changes-code-backstage - .use-pg9 stage: test needs: ["setup-test-env", "compile-assets pull-cache"] .karma-base: - extends: .only-code-frontend-job-base + extends: .frontend-job-base variables: # we override the max_old_space_size to prevent OOM errors NODE_OPTIONS: --max_old_space_size=3584 @@ -181,7 +141,9 @@ compile-assets pull-cache foss: - bundle exec rake karma karma: - extends: .karma-base + extends: + - .karma-base + - .frontend:rules:default-frontend-jobs coverage: '/^Statements *: (\d+\.\d+%)/' artifacts: name: coverage-javascript @@ -194,13 +156,14 @@ karma: reports: junit: junit_karma.xml -karma-foss: +karma-as-if-foss: extends: - .karma-base - - .only-ee-as-if-foss + - .frontend:rules:default-frontend-jobs-as-if-foss + - .as-if-foss .jest-base: - extends: .only-code-frontend-job-base + extends: .frontend-job-base script: - scripts/gitaly-test-spawn - date @@ -214,7 +177,9 @@ karma-foss: policy: pull-push jest: - extends: .jest-base + extends: + - .jest-base + - .frontend:rules:default-frontend-jobs artifacts: name: coverage-frontend expire_in: 31d @@ -226,10 +191,11 @@ jest: reports: junit: junit_jest.xml -jest-foss: +jest-as-if-foss: extends: - .jest-base - - .only-ee-as-if-foss + - .frontend:rules:default-frontend-jobs-as-if-foss + - .as-if-foss cache: policy: pull @@ -238,10 +204,8 @@ jest-foss: - .default-tags - .default-retry - .default-cache + - .frontend:rules:qa-frontend-node stage: test - rules: - - <<: *if-master - when: on_success dependencies: [] cache: key: "$CI_JOB_NAME" @@ -272,11 +236,8 @@ webpack-dev-server: - .default-tags - .default-retry - .default-cache + - .frontend:rules:default-frontend-jobs stage: test - rules: - - <<: *if-default - changes: *code-backstage-patterns - when: on_success needs: ["setup-test-env", "compile-assets pull-cache"] variables: WEBPACK_MEMORY_TEST: "true" diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index b044aaf3821..ffdc115cff7 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -9,10 +9,13 @@ - unknown_failure - api_failure - runner_system_failure + - job_execution_timeout + - stuck_or_timeout_failure .default-before_script: before_script: - date + - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' - export GOPATH=$CI_PROJECT_DIR/.go - mkdir -p $GOPATH - source scripts/utils.sh @@ -22,7 +25,7 @@ # Jobs that only need to pull cache .default-cache: cache: - key: "debian-stretch-ruby-2.6.5-node-12.x" + key: "debian-stretch-ruby-2.6.5-pg9.6-node-12.x" paths: - .go/pkg/mod - vendor/ruby @@ -30,183 +33,18 @@ - vendor/gitaly-ruby policy: pull -.default-only: - only: - refs: - - master - - /^[\d-]+-stable(-ee)?$/ - - /^\d+-\d+-auto-deploy-\d+$/ - - /^security\// - - merge_requests - - tags - -.only:variables-canonical-dot-com: - only: - variables: - - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/)/ # Matches the gitlab-org group or its subgroups - -.only:variables_refs-canonical-dot-com-schedules: - extends: .only:variables-canonical-dot-com - only: - refs: - - schedules - -.except:refs-deploy: - except: - refs: - - /^\d+-\d+-auto-deploy-\d+$/ - -.except:refs-master-tags-stable-deploy: - except: - refs: - - master - - tags - - /^[\d-]+-stable(-ee)?$/ - - /^\d+-\d+-auto-deploy-\d+$/ - -.only:kubernetes: - only: - kubernetes: active - -.only-review: - extends: - - .only:variables-canonical-dot-com - - .only:kubernetes - - .except:refs-master-tags-stable-deploy - -.only-review-schedules: - extends: - - .only:variables_refs-canonical-dot-com-schedules - - .only:kubernetes - - .except:refs-deploy - -.code-patterns: &code-patterns - - ".gitlab/ci/**/*" - - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - - ".csscomb.json" - - "Dockerfile.assets" - - "*_VERSION" - - "Gemfile{,.lock}" - - "Rakefile" - - "{babel.config,jest.config}.js" - - "config.ru" - - "{package.json,yarn.lock}" - - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - -.backstage-patterns: &backstage-patterns - - "Dangerfile" - - "danger/**/*" - - "{,ee/}fixtures/**/*" - - "{,ee/}rubocop/**/*" - - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file - -.qa-patterns: &qa-patterns - - ".dockerignore" - - "qa/**/*" - -.docs-patterns: &docs-patterns - - ".gitlab/route-map.yml" - - "doc/**/*" - - ".markdownlint.json" - -.only:changes-code: - only: - changes: *code-patterns - -.only:changes-qa: - only: - changes: *qa-patterns - -.only:changes-docs: - only: - changes: *docs-patterns - -.only:changes-code-backstage: - only: - changes: - - ".gitlab/ci/**/*" - - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - - ".csscomb.json" - - "Dockerfile.assets" - - "*_VERSION" - - "Gemfile{,.lock}" - - "Rakefile" - - "{babel.config,jest.config}.js" - - "config.ru" - - "{package.json,yarn.lock}" - - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - # Backstage changes - - "Dangerfile" - - "danger/**/*" - - "{,ee/}fixtures/**/*" - - "{,ee/}rubocop/**/*" - - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file - -.only:changes-code-qa: - only: - changes: - - ".gitlab/ci/**/*" - - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - - ".csscomb.json" - - "Dockerfile.assets" - - "*_VERSION" - - "Gemfile{,.lock}" - - "Rakefile" - - "{babel.config,jest.config}.js" - - "config.ru" - - "{package.json,yarn.lock}" - - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - # QA changes - - ".dockerignore" - - "qa/**/*" - -.only:changes-code-backstage-qa: - only: - changes: - - ".gitlab/ci/**/*" - - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - - ".csscomb.json" - - "Dockerfile.assets" - - "*_VERSION" - - "Gemfile{,.lock}" - - "Rakefile" - - "{babel.config,jest.config}.js" - - "config.ru" - - "{package.json,yarn.lock}" - - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - # Backstage changes - - "Dangerfile" - - "danger/**/*" - - "{,ee/}fixtures/**/*" - - "{,ee/}rubocop/**/*" - - "{,ee/}spec/**/*" - - "doc/README.md" # Some RSpec test rely on this file - # QA changes - - ".dockerignore" - - "qa/**/*" - .use-pg9: services: - - name: postgres:9.6 + - name: postgres:9.6.17 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine variables: POSTGRES_HOST_AUTH_METHOD: trust .use-pg10: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.12-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.12-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" services: - - name: postgres:10.9 + - name: postgres:10.12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine variables: @@ -214,7 +52,7 @@ .use-pg9-ee: services: - - name: postgres:9.6 + - name: postgres:9.6.17 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine - name: elasticsearch:6.4.2 @@ -222,22 +60,15 @@ POSTGRES_HOST_AUTH_METHOD: trust .use-pg10-ee: - image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.12-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.16-postgresql-10-graphicsmagick-1.3.33" + image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.12-git-2.24-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-10-graphicsmagick-1.3.34" services: - - name: postgres:10.9 + - name: postgres:10.12 command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - name: redis:alpine - name: elasticsearch:6.4.2 variables: POSTGRES_HOST_AUTH_METHOD: trust -.only-ee: - only: - variables: - - $CI_PROJECT_NAME == "gitlab" - - $CI_PROJECT_NAME == "gitlab-ee" # Support former project name for forks/mirrors - -.only-ee-as-if-foss: - extends: .only-ee +.as-if-foss: variables: FOSS_ONLY: '1' diff --git a/.gitlab/ci/memory.gitlab-ci.yml b/.gitlab/ci/memory.gitlab-ci.yml index ba14024df34..a8e2d90a4ee 100644 --- a/.gitlab/ci/memory.gitlab-ci.yml +++ b/.gitlab/ci/memory.gitlab-ci.yml @@ -3,9 +3,8 @@ - .default-tags - .default-retry - .default-cache - - .default-only - .default-before_script - - .only:changes-code + - .memory:rules memory-static: extends: .only-code-memory-job-base diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index 6a2d3702bdd..993ed21e39d 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -3,12 +3,7 @@ pages: - .default-tags - .default-retry - .default-cache - - .default-only - - .only:variables-canonical-dot-com - - .only:changes-code-backstage-qa - only: - refs: - - master + - .pages:rules stage: pages dependencies: ["coverage", "karma", "gitlab:assets:compile pull-cache"] script: diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index 5a58c3f9416..0e9d7abb3ac 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -1,38 +1,7 @@ -# Make sure to update all the similar conditions in other CI config files if you modify these conditions -.if-canonical-gitlab-schedule: &if-canonical-gitlab-schedule - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"' - -# Make sure to update all the similar conditions in other CI config files if you modify these conditions -.if-canonical-gitlab-merge-request: &if-canonical-gitlab-merge-request - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_MERGE_REQUEST_IID' - -# Make sure to update all the similar patterns in other CI config files if you modify these patterns -.code-patterns: &code-patterns - - ".gitlab/ci/**/*" - - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - - ".csscomb.json" - - "Dockerfile.assets" - - "*_VERSION" - - "Gemfile{,.lock}" - - "Rakefile" - - "{babel.config,jest.config}.js" - - "config.ru" - - "{package.json,yarn.lock}" - - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - -# Make sure to update all the similar patterns in other CI config files if you modify these patterns -.qa-patterns: &qa-patterns - - ".dockerignore" - - "qa/**/*" - .qa-job-base: extends: - .default-tags - .default-retry - - .default-only - - .only:changes-code-qa stage: test dependencies: [] cache: @@ -40,24 +9,38 @@ paths: - vendor/ruby before_script: + - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb' - cd qa/ - bundle install --clean --jobs=$(nproc) --path=vendor --retry=3 --quiet - bundle check qa:internal: - extends: .qa-job-base + extends: + - .qa-job-base + - .qa:rules:ee-and-foss + script: + - bundle exec rspec + +qa:internal-as-if-foss: + extends: + - .qa-job-base + - .qa:rules:ee-only + - .as-if-foss script: - bundle exec rspec qa:selectors: - extends: .qa-job-base + extends: + - .qa-job-base + - .qa:rules:ee-and-foss script: - bundle exec bin/qa Test::Sanity::Selectors -qa:selectors-foss: +qa:selectors-as-if-foss: extends: - qa:selectors - - .only-ee-as-if-foss + - .qa:rules:ee-only + - .as-if-foss .package-and-qa-base: image: ruby:2.6-alpine @@ -70,15 +53,8 @@ qa:selectors-foss: - ./scripts/trigger-build omnibus package-and-qa: - extends: .package-and-qa-base - rules: - - <<: *if-canonical-gitlab-merge-request - changes: *qa-patterns - when: on_success - - <<: *if-canonical-gitlab-merge-request - changes: *code-patterns - when: manual - - <<: *if-canonical-gitlab-schedule - when: on_success + extends: + - .package-and-qa-base + - .qa:rules:package-and-qa needs: ["build-qa-image", "gitlab:assets:compile pull-cache"] allow_failure: true diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 8c3df170f6d..58c4ef0e02a 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -1,41 +1,23 @@ -.only-master: - only: - refs: - - master - -.rake-exec: +.rails:needs:setup-and-assets: + needs: + - job: setup-test-env + artifacts: true + - job: compile-assets pull-cache + artifacts: true + +.rails-job-base: extends: - .default-tags - .default-retry - .default-cache - - .default-only - .default-before_script - variables: - SETUP_DB: "false" - script: - - bundle exec rake $CI_JOB_NAME - -.only-code-rails-job-base: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-only - - .default-before_script - - .only:changes-code-backstage - -.only-code-qa-rails-job-base: - extends: - - .default-tags - - .default-retry - - .default-cache - - .default-only - - .default-before_script - - .only:changes-code-backstage-qa +#################### +# ee and foss jobs # setup-test-env: extends: - - .only-code-qa-rails-job-base + - .rails-job-base + - .rails:rules:default-refs-code-backstage-qa - .use-pg9 stage: prepare script: @@ -50,11 +32,45 @@ setup-test-env: cache: policy: pull-push +static-analysis: + extends: + - .rails-job-base + - .rails:rules:default-refs-code-backstage-qa + - .rails:needs:setup-and-assets + stage: test + variables: + SETUP_DB: "false" + parallel: 2 + script: + - scripts/static-analysis + cache: + key: "debian-stretch-ruby-2.6-pg9.6-rubocop" + paths: + - vendor/ruby + - tmp/rubocop_cache + policy: pull-push + +downtime_check: + extends: + - .rails-job-base + - .rails:needs:setup-and-assets + - .rails:rules:downtime_check + stage: test + variables: + SETUP_DB: "false" + script: + - bundle exec rake downtime_check + .rspec-base: - extends: .only-code-rails-job-base + extends: .rails-job-base stage: test - needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"] - dependencies: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"] + needs: + - job: setup-test-env + artifacts: true + - job: retrieve-tests-metadata + artifacts: true + - job: compile-assets pull-cache + artifacts: true script: - source scripts/rspec_helpers.sh - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration" @@ -68,29 +84,26 @@ setup-test-env: - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ + - junit_rspec.xml reports: junit: junit_rspec.xml -.rspec-base-foss: - extends: [".rspec-base", ".only-ee-as-if-foss"] - needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache foss"] - dependencies: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache foss"] - -.rspec-base-pg9: +.rspec-base-quarantine: extends: - .rspec-base - .use-pg9 + variables: + RSPEC_OPTS: "--tag quarantine -- spec/" + script: + - source scripts/rspec_helpers.sh + - rspec_simple_job "${RSPEC_OPTS}" + allow_failure: true -.rspec-base-pg9-foss: - extends: - - .rspec-base-foss - - .use-pg9 - -.rspec-base-pg10: +.rspec-base-pg9: extends: - .rspec-base - - .use-pg10 - - .only-master + - .rails:rules:ee-and-foss + - .use-pg9 .rspec-base-migration: script: @@ -101,202 +114,35 @@ rspec migration pg9: extends: - .rspec-base-pg9 - .rspec-base-migration - parallel: 4 - -rspec migration pg9-foss: - extends: - - .rspec-base-pg9-foss - - .rspec-base-migration - parallel: 4 + parallel: 5 rspec unit pg9: extends: .rspec-base-pg9 parallel: 20 -rspec unit pg9-foss: - extends: .rspec-base-pg9-foss - parallel: 20 - rspec integration pg9: extends: .rspec-base-pg9 - parallel: 6 - -rspec integration pg9-foss: - extends: .rspec-base-pg9-foss - parallel: 6 + parallel: 8 rspec system pg9: extends: .rspec-base-pg9 parallel: 24 -rspec system pg9-foss: - extends: .rspec-base-pg9-foss - parallel: 24 - -rspec unit pg10: - extends: .rspec-base-pg10 - parallel: 20 - -rspec integration pg10: - extends: .rspec-base-pg10 - parallel: 6 - -rspec system pg10: - extends: .rspec-base-pg10 - parallel: 24 - -.rspec-ee-base-pg9: - extends: - - .rspec-base - - .only-ee - - .use-pg9-ee - -.rspec-ee-base-pg10: - extends: - - .rspec-base - - .only-ee - - .use-pg10-ee - -rspec-ee migration pg9: - extends: - - .rspec-ee-base-pg9 - - .rspec-base-migration - parallel: 2 - -rspec-ee unit pg9: - extends: .rspec-ee-base-pg9 - parallel: 5 - -rspec-ee integration pg9: - extends: .rspec-ee-base-pg9 - parallel: 3 - -rspec-ee system pg9: - extends: .rspec-ee-base-pg9 - parallel: 5 - -rspec-ee migration pg10: - extends: - - .rspec-ee-base-pg10 - - .rspec-base-migration - - .only-master - parallel: 2 - -rspec-ee unit pg10: - extends: - - .rspec-ee-base-pg10 - - .only-master - parallel: 5 - -rspec-ee integration pg10: - extends: - - .rspec-ee-base-pg10 - - .only-master - parallel: 3 - -rspec-ee system pg10: - extends: - - .rspec-ee-base-pg10 - - .only-master - parallel: 5 - -.rspec-ee-base-geo: - extends: - - .rspec-base - - .only-ee - script: - - source scripts/rspec_helpers.sh - - scripts/prepare_postgres_fdw.sh - - rspec_paralellized_job "--tag ~quarantine --tag geo" - -.rspec-ee-base-geo-pg9: - extends: - - .rspec-ee-base-geo - - .use-pg9-ee - -.rspec-ee-base-geo-pg10: - extends: - - .rspec-ee-base-geo - - .use-pg10-ee - -rspec-ee unit pg9 geo: - extends: .rspec-ee-base-geo-pg9 - parallel: 2 - -rspec-ee integration pg9 geo: - extends: .rspec-ee-base-geo-pg9 - -rspec-ee system pg9 geo: - extends: .rspec-ee-base-geo-pg9 - -rspec-ee unit pg10 geo: - extends: .rspec-ee-base-geo-pg10 - parallel: 2 - -rspec-ee integration pg10 geo: - extends: .rspec-ee-base-geo-pg10 - -rspec-ee system pg10 geo: - extends: .rspec-ee-base-geo-pg10 - -rspec quarantine pg9: - extends: - - .rspec-base-pg9 - - .only-master - variables: - RSPEC_OPTS: "--tag quarantine -- spec/" - script: - - source scripts/rspec_helpers.sh - - rspec_simple_job "${RSPEC_OPTS}" - allow_failure: true - -rspec-ee quarantine pg9: - extends: - - rspec quarantine pg9 - - .only-ee - variables: - RSPEC_OPTS: "--tag quarantine -- ee/spec/" - rspec fast_spec_helper: extends: .rspec-base-pg9 script: - bin/rspec spec/fast_spec_helper.rb -static-analysis: - extends: .only-code-qa-rails-job-base - stage: test - needs: ["setup-test-env", "compile-assets pull-cache"] - dependencies: ["setup-test-env", "compile-assets pull-cache"] - variables: - SETUP_DB: "false" - parallel: 2 - script: - - scripts/static-analysis - cache: - key: "debian-stretch-ruby-2.6-and-rubocop" - paths: - - vendor/ruby - - tmp/rubocop_cache - policy: pull-push - -downtime_check: - extends: - - .rake-exec - - .only:changes-code-backstage - - .except:refs-master-tags-stable-deploy - stage: test - needs: ["setup-test-env"] - dependencies: ["setup-test-env"] - .db-job-base: extends: - - .only-code-rails-job-base + - .rails-job-base + - .rails:rules:ee-and-foss - .use-pg9 stage: test - needs: ["setup-test-env"] - dependencies: ["setup-test-env"] + needs: + - job: setup-test-env + artifacts: true -# DB migration, rollback, and seed jobs db:migrate:reset: extends: .db-job-base script: @@ -357,12 +203,14 @@ gitlab:setup: - log/development.log coverage: - extends: .only-code-rails-job-base - cache: - policy: pull + extends: + - .rails-job-base + - .rails:rules:ee-and-foss + stage: post-test variables: SETUP_DB: "false" - stage: post-test + cache: + policy: pull script: - bundle exec scripts/merge-simplecov - bundle exec scripts/gather-test-memory-data @@ -374,11 +222,181 @@ coverage: - coverage/index.html - coverage/assets/ - tmp/memory_test/ +# ee and foss jobs # +#################### + +#################### +# master-only jobs # +rspec quarantine pg9: + extends: + - .rspec-base-quarantine + - .rails:rules:master-refs-code-backstage + +.rspec-base-pg10: + extends: + - .rspec-base + - .rails:rules:master-refs-code-backstage + - .use-pg10 + +rspec unit pg10: + extends: .rspec-base-pg10 + parallel: 20 + +rspec integration pg10: + extends: .rspec-base-pg10 + parallel: 8 + +rspec system pg10: + extends: .rspec-base-pg10 + parallel: 24 +# master-only jobs # +#################### + +######################### +# ee + master-only jobs # +rspec-ee quarantine pg9: + extends: + - .rspec-base-quarantine + - .rails:rules:master-refs-code-backstage-ee-only + variables: + RSPEC_OPTS: "--tag quarantine -- ee/spec/" + +rspec-ee migration pg10: + extends: + - .rspec-ee-base-pg10 + - .rspec-base-migration + - .rails:rules:master-refs-code-backstage + parallel: 2 + +rspec-ee unit pg10: + extends: + - .rspec-ee-base-pg10 + - .rails:rules:master-refs-code-backstage + parallel: 10 + +rspec-ee integration pg10: + extends: + - .rspec-ee-base-pg10 + - .rails:rules:master-refs-code-backstage + parallel: 4 + +rspec-ee system pg10: + extends: + - .rspec-ee-base-pg10 + - .rails:rules:master-refs-code-backstage + parallel: 6 +# ee + master-only jobs # +######################### + +################# +# ee-only jobs # +.rspec-base-ee: + extends: + - .rspec-base + - .rails:rules:ee-only + +.rspec-base-pg9-as-if-foss: + extends: + - .rspec-base-ee + - .as-if-foss + - .use-pg9 + needs: + - job: setup-test-env + artifacts: true + - job: retrieve-tests-metadata + artifacts: true + - job: compile-assets pull-cache as-if-foss + artifacts: true + +.rspec-ee-base-pg9: + extends: + - .rspec-base-ee + - .use-pg9-ee + +.rspec-ee-base-pg10: + extends: + - .rspec-base-ee + - .use-pg10-ee + +rspec migration pg9-as-if-foss: + extends: + - .rspec-base-pg9-as-if-foss + - .rspec-base-migration + parallel: 5 + +rspec unit pg9-as-if-foss: + extends: .rspec-base-pg9-as-if-foss + parallel: 20 + +rspec integration pg9-as-if-foss: + extends: .rspec-base-pg9-as-if-foss + parallel: 8 + +rspec system pg9-as-if-foss: + extends: .rspec-base-pg9-as-if-foss + parallel: 24 + +rspec-ee migration pg9: + extends: + - .rspec-ee-base-pg9 + - .rspec-base-migration + parallel: 2 + +rspec-ee unit pg9: + extends: .rspec-ee-base-pg9 + parallel: 10 + +rspec-ee integration pg9: + extends: .rspec-ee-base-pg9 + parallel: 4 + +rspec-ee system pg9: + extends: .rspec-ee-base-pg9 + parallel: 6 + +.rspec-ee-base-geo: + extends: .rspec-base-ee + script: + - source scripts/rspec_helpers.sh + - scripts/prepare_postgres_fdw.sh + - rspec_paralellized_job "--tag ~quarantine --tag geo" + +.rspec-ee-base-geo-pg9: + extends: + - .rspec-ee-base-geo + - .use-pg9-ee + +.rspec-ee-base-geo-pg10: + extends: + - .rspec-ee-base-geo + - .use-pg10-ee + +rspec-ee unit pg9 geo: + extends: .rspec-ee-base-geo-pg9 + parallel: 2 + +rspec-ee integration pg9 geo: + extends: .rspec-ee-base-geo-pg9 + +rspec-ee system pg9 geo: + extends: .rspec-ee-base-geo-pg9 + +rspec-ee unit pg10 geo: + extends: .rspec-ee-base-geo-pg10 + parallel: 2 + +rspec-ee integration pg10 geo: + extends: .rspec-ee-base-geo-pg10 + +rspec-ee system pg10 geo: + extends: .rspec-ee-base-geo-pg10 db:rollback geo: extends: - db:rollback - - .only-ee + - .rails:rules:ee-only script: - bundle exec rake geo:db:migrate VERSION=20170627195211 - bundle exec rake geo:db:migrate +# ee-only jobs # +################ diff --git a/.gitlab/ci/releases.gitlab-ci.yml b/.gitlab/ci/releases.gitlab-ci.yml index 8ca4041e6be..b3f961afe62 100644 --- a/.gitlab/ci/releases.gitlab-ci.yml +++ b/.gitlab/ci/releases.gitlab-ci.yml @@ -1,5 +1,3 @@ ---- - # Syncs any changes pushed to a stable branch to the corresponding # gitlab-foss/CE stable branch. We run this prior to any tests so that random # failures don't prevent a sync. @@ -10,27 +8,21 @@ stage: sync before_script: - apk add --no-cache --update curl bash jq - after_script: [] script: - bash scripts/sync-stable-branch.sh - only: - variables: - - $CI_SERVER_HOST == "gitlab.com" sync-stable-branch: - extends: .merge-train-sync + extends: + - .releases:rules:canonical-dot-com-gitlab-stable-branch-only + - .merge-train-sync variables: SOURCE_PROJECT: gitlab-org/gitlab TARGET_PROJECT: gitlab-org/gitlab-foss - only: - refs: - - /^[\d-]+-stable-ee$/@gitlab-org/gitlab sync-security-branch: - extends: .merge-train-sync + extends: + - .releases:rules:canonical-dot-com-security-gitlab-stable-branch-only + - .merge-train-sync variables: SOURCE_PROJECT: gitlab-org/security/gitlab TARGET_PROJECT: gitlab-org/security/gitlab-foss - only: - refs: - - /^[\d-]+-stable-ee$/@gitlab-org/security/gitlab diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 81cc3e7dd2f..e7ee47fbe0a 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -1,5 +1,5 @@ # include: -# - template: Code-Quality.gitlab-ci.yml +# - template: Jobs/Code-Quality.gitlab-ci.yml # - template: Security/SAST.gitlab-ci.yml # - template: Security/Dependency-Scanning.gitlab-ci.yml # - template: Security/DAST.gitlab-ci.yml @@ -10,8 +10,7 @@ code_quality: extends: - .default-retry - - .default-only - - .only:changes-code-backstage + - .reports:rules:code_quality stage: test image: docker:stable allow_failure: true @@ -38,12 +37,9 @@ code_quality: reports: codequality: gl-code-quality-report.json paths: - - gl-code-quality-report.json - expire_in: 1 week + - gl-code-quality-report.json # GitLab-specific + expire_in: 1 week # GitLab-specific dependencies: [] - except: - variables: - - $CODE_QUALITY_DISABLED # We need to duplicate this job's definition because it seems it's impossible to # override an included `only.refs`. @@ -53,16 +49,22 @@ code_quality: sast: extends: - .default-retry - - .default-only - - .only:changes-code-backstage-qa + - .reports:rules:sast stage: test + allow_failure: true + dependencies: [] # GitLab-specific + artifacts: + paths: + - gl-sast-report.json # GitLab-specific + reports: + sast: gl-sast-report.json + expire_in: 1 week # GitLab-specific image: docker:stable variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - SAST_BRAKEMAN_LEVEL: 2 - SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec - allow_failure: true + SAST_BRAKEMAN_LEVEL: 2 # GitLab-specific + SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec # GitLab-specific services: - docker:stable-dind script: @@ -73,61 +75,12 @@ sast: export DOCKER_HOST='tcp://localhost:2375' fi fi - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } - | - docker run \ - $(propagate_env_vars \ - SAST_BANDIT_EXCLUDED_PATHS \ - SAST_ANALYZER_IMAGES \ - SAST_ANALYZER_IMAGE_PREFIX \ - SAST_ANALYZER_IMAGE_TAG \ - SAST_DEFAULT_ANALYZERS \ - SAST_PULL_ANALYZER_IMAGES \ - SAST_BRAKEMAN_LEVEL \ - SAST_FLAWFINDER_LEVEL \ - SAST_GITLEAKS_ENTROPY_LEVEL \ - SAST_GOSEC_LEVEL \ - SAST_EXCLUDED_PATHS \ - SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - SAST_PULL_ANALYZER_IMAGE_TIMEOUT \ - SAST_RUN_ANALYZER_TIMEOUT \ - SAST_JAVA_VERSION \ - ANT_HOME \ - ANT_PATH \ - GRADLE_PATH \ - JAVA_OPTS \ - JAVA_PATH \ - JAVA_8_VERSION \ - JAVA_11_VERSION \ - MAVEN_CLI_OPTS \ - MAVEN_PATH \ - MAVEN_REPO_PATH \ - SBT_PATH \ - FAIL_NEVER \ - ) \ + ENVS=`printenv | grep -vE '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | sed -n '/^[^\t]/s/=.*//p' | sed '/^$/d' | sed 's/^/-e /g' | tr '\n' ' '` + docker run "$ENVS" \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code - artifacts: - expire_in: 7 days - paths: - - gl-sast-report.json - reports: - sast: gl-sast-report.json - dependencies: [] - only: - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ - except: - variables: - - $SAST_DISABLED # We need to duplicate this job's definition because it seems it's impossible to # override an included `only.refs`. @@ -135,13 +88,13 @@ sast: dependency_scanning: extends: - .default-retry - - .default-only - - .only:changes-code-backstage-qa + - .reports:rules:dependency_scanning stage: test image: docker:stable variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" + DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec" # GitLab-specific allow_failure: true services: - docker:stable-dind @@ -173,23 +126,29 @@ dependency_scanning: DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ DS_PYTHON_VERSION \ + DS_PIP_VERSION \ DS_PIP_DEPENDENCY_PATH \ + GEMNASIUM_DB_LOCAL_PATH \ + GEMNASIUM_DB_REMOTE_URL \ + GEMNASIUM_DB_REF_NAME \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ + PIP_REQUIREMENTS_FILE \ + MAVEN_CLI_OPTS \ + BUNDLER_AUDIT_UPDATE_DISABLED \ + BUNDLER_AUDIT_ADVISORY_DB_URL \ + BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \ ) \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code artifacts: + paths: + - gl-dependency-scanning-report.json # GitLab-specific reports: dependency_scanning: gl-dependency-scanning-report.json + expire_in: 1 week # GitLab-specific dependencies: [] - only: - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ - except: - variables: - - $DEPENDENCY_SCANNING_DISABLED # We need to duplicate this job's definition because it seems it's impossible to # override an included `only.refs`. @@ -197,40 +156,36 @@ dependency_scanning: dast: extends: - .default-retry - - .default-only - - .only:changes-code-qa - - .only-review - stage: qa - needs: ["review-deploy"] - dependencies: ["review-deploy"] - before_script: - - export DAST_WEBSITE="$(cat review_app_url.txt)" + - .reports:rules:dast + needs: + - job: review-deploy + artifacts: true + stage: qa # GitLab-specific image: - name: "registry.gitlab.com/gitlab-org/security-products/dast:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" + name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" variables: - # URL to scan: - # DAST_WEBSITE: https://example.com/ - # - # Time limit for target availability (scan is attempted even when timeout): - # DAST_TARGET_AVAILABILITY_TIMEOUT: 60 - # - # Set these variables to scan with an authenticated user: - # DAST_AUTH_URL: https://example.com/sign-in - # DAST_USERNAME: john.doe@example.com - # DAST_PASSWORD: john-doe-password - # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form - # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form - # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between - # - # Perform ZAP Full Scan, which includes both passive and active scanning: - # DAST_FULL_SCAN_ENABLED: "true" + # To be done in a later iteration + # DAST_USERNAME: "root" + # DAST_USERNAME_FIELD: "user[login]" + # DAST_PASSWORD_FIELD: "user[passowrd]" allow_failure: true script: - - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} + - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"' + # To be done in a later iteration + # - 'export DAST_AUTH_URL="${DAST_WEBSITE}/users/sign_in"' + # - 'export DAST_PASSWORD="${REVIEW_APPS_ROOT_PASSWORD}"' - /analyze -t $DAST_WEBSITE artifacts: - expire_in: 7 days paths: - - gl-dast-report.json + - gl-dast-report.json # GitLab-specific reports: dast: gl-dast-report.json + expire_in: 1 week # GitLab-specific + +# To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 +# schedule:dast: +# extends: +# - dast +# - .reports:schedule-dast +# variables: +# DAST_FULL_SCAN_ENABLED: "true" diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index ad045d6c974..e23708337a4 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -1,35 +1,8 @@ -# Make sure to update all the similar conditions in other CI config files if you modify these conditions -.if-canonical-gitlab-schedule: &if-canonical-gitlab-schedule - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"' - -# Make sure to update all the similar conditions in other CI config files if you modify these conditions -.if-canonical-gitlab-merge-request: &if-canonical-gitlab-merge-request - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_MERGE_REQUEST_IID' - -# Make sure to update all the similar patterns in other CI config files if you modify these patterns -.code-qa-patterns: &code-qa-patterns - - ".gitlab/ci/**/*" - - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" - - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" - - ".csscomb.json" - - "Dockerfile.assets" - - "*_VERSION" - - "Gemfile{,.lock}" - - "Rakefile" - - "{babel.config,jest.config}.js" - - "config.ru" - - "{package.json,yarn.lock}" - - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" - - "doc/api/graphql/reference/*" # Files in this folder are auto-generated - # QA changes - - ".dockerignore" - - "qa/**/*" - .review-docker: extends: - .default-tags - .default-retry - image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine + image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6 services: - docker:19.03.0-dind tags: @@ -41,14 +14,10 @@ GITLAB_EDITION: "ce" build-qa-image: - extends: .review-docker + extends: + - .review-docker + - .review:rules:mr-and-schedule stage: prepare - rules: - - <<: *if-canonical-gitlab-merge-request - changes: *code-qa-patterns - when: on_success - - <<: *if-canonical-gitlab-schedule - when: on_success script: - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - export QA_MASTER_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:master" @@ -58,11 +27,11 @@ build-qa-image: - time docker build --cache-from "${QA_MASTER_IMAGE}" --tag ${QA_IMAGE} --file ./qa/Dockerfile ./ - time docker push ${QA_IMAGE} -.base-review-cleanup: +review-cleanup: extends: - .default-tags - .default-retry - - .default-only + - .review:rules:review-cleanup stage: prepare image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base allow_failure: true @@ -75,51 +44,28 @@ build-qa-image: script: - ruby -rrubygems scripts/review_apps/automated_cleanup.rb -schedule:review-cleanup: - extends: - - .base-review-cleanup - - .only-review-schedules - -manual:review-cleanup: - extends: - - .base-review-cleanup - - .only:changes-code-qa - when: manual - -.review-build-cng-base: +review-build-cng: extends: - .default-tags - .default-retry - - .default-only + - .review:rules:mr-and-schedule image: ruby:2.6-alpine stage: review-prepare before_script: - source scripts/utils.sh - install_api_client_dependencies_with_apk - install_gitlab_gem - dependencies: [] + needs: + - job: gitlab:assets:compile pull-cache + artifacts: false script: - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng -review-build-cng: - extends: - - .review-build-cng-base - - .only-review - - .only:changes-code-qa - needs: ["gitlab:assets:compile pull-cache"] - -schedule:review-build-cng: - extends: - - .review-build-cng-base - - .only-review-schedules - needs: ["gitlab:assets:compile pull-cache"] - .review-workflow-base: extends: - .default-tags - .default-retry image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base - dependencies: [] variables: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" @@ -130,17 +76,22 @@ schedule:review-build-cng: url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} on_stop: review-stop -.review-deploy-base: - extends: .review-workflow-base +review-deploy: + extends: + - .review-workflow-base + - .review:rules:mr-and-schedule stage: review + needs: + - job: review-build-cng + artifacts: false allow_failure: true before_script: - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION) - export GITALY_VERSION=$(<GITALY_SERVER_VERSION) - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION) - - echo "${CI_ENVIRONMENT_URL}" > review_app_url.txt - - source scripts/utils.sh + - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt + - source ./scripts/utils.sh - install_api_client_dependencies_with_apk - source scripts/review_apps/review-apps.sh script: @@ -152,26 +103,15 @@ schedule:review-build-cng: - date - deploy || (display_deployment_debug && exit 1) artifacts: - paths: [review_app_url.txt] + paths: [environment_url.txt] expire_in: 2 days when: always -review-deploy: - extends: .review-deploy-base - rules: - - <<: *if-canonical-gitlab-merge-request - changes: *code-qa-patterns - when: on_success - -schedule:review-deploy: - extends: - - .review-deploy-base - - .only-review-schedules - -.base-review-stop: +.review-stop-base: extends: .review-workflow-base environment: action: stop + dependencies: [] variables: # We're cloning the repo instead of downloading the script for now # because some repos are private and CI_JOB_TOKEN cannot access files. @@ -183,22 +123,18 @@ schedule:review-deploy: - source ./scripts/review_apps/review-apps.sh review-stop-failed-deployment: - extends: .base-review-stop + extends: + - .review-stop-base + - .review:rules:mr-only-auto stage: prepare - rules: - - <<: *if-canonical-gitlab-merge-request - changes: *code-qa-patterns - when: on_success script: - delete_failed_release review-stop: - extends: .base-review-stop + extends: + - .review-stop-base + - .review:rules:mr-only-manual stage: review - rules: - - <<: *if-canonical-gitlab-merge-request - changes: *code-qa-patterns - when: manual allow_failure: true script: - delete_release @@ -206,8 +142,9 @@ review-stop: .review-qa-base: extends: .review-docker stage: qa - needs: ["review-deploy"] - dependencies: ["review-deploy"] + needs: + - job: review-deploy + artifacts: true allow_failure: true variables: QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa" @@ -222,7 +159,7 @@ review-stop: before_script: - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"' - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}" - - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" + - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - echo "${QA_IMAGE}" - source scripts/utils.sh @@ -235,32 +172,33 @@ review-stop: when: always review-qa-smoke: - extends: .review-qa-base - rules: - - <<: *if-canonical-gitlab-merge-request - changes: *code-qa-patterns - when: on_success + extends: + - .review-qa-base + - .review:rules:mr-only-auto script: - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" review-qa-all: - extends: .review-qa-base - rules: - - <<: *if-canonical-gitlab-merge-request - changes: *code-qa-patterns - when: manual + extends: + - .review-qa-base + - .review:rules:mr-only-manual parallel: 5 script: - export KNAPSACK_REPORT_PATH=knapsack/master_report.json - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation -.review-performance-base: - extends: .review-docker +review-performance: + extends: + - .review-docker + - .review:rules:mr-and-schedule stage: qa + needs: + - job: review-deploy + artifacts: true allow_failure: true before_script: - - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" + - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)" - echo "${CI_ENVIRONMENT_URL}" - mkdir -p gitlab-exporter - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js @@ -275,58 +213,17 @@ review-qa-all: reports: performance: performance.json -review-performance: - extends: .review-performance-base - rules: - - <<: *if-canonical-gitlab-merge-request - changes: *code-qa-patterns - when: on_success - needs: ["review-deploy"] - dependencies: ["review-deploy"] - before_script: - - export CI_ENVIRONMENT_URL="$(cat review_app_url.txt)" - - echo "${CI_ENVIRONMENT_URL}" - - mkdir -p gitlab-exporter - - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js - - mkdir -p sitespeed-results - script: - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}" - after_script: - - mv sitespeed-results/data/performance.json performance.json - artifacts: - paths: - - sitespeed-results/ - reports: - performance: performance.json - -schedule:review-performance: - extends: - - .review-performance-base - - .only-review-schedules - needs: ["schedule:review-deploy"] - dependencies: ["schedule:review-deploy"] - parallel-spec-reports: extends: - .default-tags - - .default-only - - .only-review - - .only:changes-code-qa + - .review:rules:mr-only-manual image: ruby:2.6-alpine stage: post-qa dependencies: ["review-qa-all"] + allow_failure: true variables: NEW_PARALLEL_SPECS_REPORT: qa/report-new.html BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/" - allow_failure: true - when: manual - artifacts: - when: always - paths: - - qa/report-new.html - - qa/gitlab-qa-run-* - reports: - junit: qa/gitlab-qa-run-*/**/rspec-*.xml script: - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/* - gem install nokogiri --no-document @@ -335,20 +232,23 @@ parallel-spec-reports: - cd - - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}' - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm + artifacts: + when: always + paths: + - qa/report-new.html + - qa/gitlab-qa-run-* + reports: + junit: qa/gitlab-qa-run-*/**/rspec-*.xml danger-review: extends: - .default-tags - .default-retry - .default-cache - - .default-only - - .except:refs-master-tags-stable-deploy + - .review:rules:danger image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger stage: test dependencies: [] - only: - variables: - - $DANGER_GITLAB_API_TOKEN script: - git version - node --version diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml new file mode 100644 index 00000000000..8cf652d51e0 --- /dev/null +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -0,0 +1,500 @@ +############## +# Conditions # +############## +.if-not-canonical-namespace: &if-not-canonical-namespace + if: '$CI_PROJECT_NAMESPACE !~ /^gitlab(-org)?($|\/)/' + +.if-not-ee: &if-not-ee + if: '$CI_PROJECT_NAME !~ /^gitlab(-ee)?$/' + +.if-not-foss: &if-not-foss + if: '$CI_PROJECT_NAME != "gitlab-foss" && $CI_PROJECT_NAME != "gitlab-ce" && $CI_PROJECT_NAME != "gitlabhq"' + +.if-default-refs: &if-default-refs + if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/ || $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/ || $CI_COMMIT_REF_NAME =~ /^security\// || $CI_MERGE_REQUEST_IID || $CI_COMMIT_TAG' + +.if-master-refs: &if-master-refs + if: '$CI_COMMIT_REF_NAME == "master"' + +.if-master-or-tag: &if-master-or-tag + if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_TAG' + +.if-merge-request: &if-merge-request + if: '$CI_MERGE_REQUEST_IID' + +.if-dot-com-gitlab-org-schedule: &if-dot-com-gitlab-org-schedule + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_PIPELINE_SOURCE == "schedule"' + +.if-dot-com-gitlab-org-master: &if-dot-com-gitlab-org-master + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_COMMIT_REF_NAME == "master"' + +.if-dot-com-gitlab-org-merge-request: &if-dot-com-gitlab-org-merge-request + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org" && $CI_MERGE_REQUEST_IID' + +.if-dot-com-gitlab-org-and-security-merge-request: &if-dot-com-gitlab-org-and-security-merge-request + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_MERGE_REQUEST_IID' + +.if-dot-com-gitlab-org-and-security-tag: &if-dot-com-gitlab-org-and-security-tag + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE =~ /^gitlab-org($|\/security$)/ && $CI_COMMIT_TAG' + +.if-dot-com-ee-schedule: &if-dot-com-ee-schedule + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_PIPELINE_SOURCE == "schedule"' + +.if-cache-credentials-schedule: &if-cache-credentials-schedule + if: '$CI_REPO_CACHE_CREDENTIALS && $CI_PIPELINE_SOURCE == "schedule"' + +#################### +# Changes patterns # +#################### +.yaml-patterns: &yaml-patterns + - "**/*.yml" + +.docs-patterns: &docs-patterns + - ".gitlab/route-map.yml" + - "doc/**/*" + - ".markdownlint.json" + +.backstage-patterns: &backstage-patterns + - "Dangerfile" + - "danger/**/*" + - "{,ee/}fixtures/**/*" + - "{,ee/}rubocop/**/*" + - "{,ee/}spec/**/*" + - "doc/README.md" # Some RSpec test rely on this file + +.code-patterns: &code-patterns + - ".gitlab/ci/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" + - ".csscomb.json" + - "Dockerfile.assets" + - "*_VERSION" + - "Gemfile{,.lock}" + - "Rakefile" + - "{babel.config,jest.config}.js" + - "config.ru" + - "{package.json,yarn.lock}" + - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "doc/api/graphql/reference/*" # Files in this folder are auto-generated + +.qa-patterns: &qa-patterns + - ".dockerignore" + - "qa/**/*" + +.code-backstage-patterns: &code-backstage-patterns + - ".gitlab/ci/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" + - ".csscomb.json" + - "Dockerfile.assets" + - "*_VERSION" + - "Gemfile{,.lock}" + - "Rakefile" + - "{babel.config,jest.config}.js" + - "config.ru" + - "{package.json,yarn.lock}" + - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "doc/api/graphql/reference/*" # Files in this folder are auto-generated + # Backstage changes + - "Dangerfile" + - "danger/**/*" + - "{,ee/}fixtures/**/*" + - "{,ee/}rubocop/**/*" + - "{,ee/}spec/**/*" + - "doc/README.md" # Some RSpec test rely on this file + +.code-qa-patterns: &code-qa-patterns + - ".gitlab/ci/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" + - ".csscomb.json" + - "Dockerfile.assets" + - "*_VERSION" + - "Gemfile{,.lock}" + - "Rakefile" + - "{babel.config,jest.config}.js" + - "config.ru" + - "{package.json,yarn.lock}" + - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "doc/api/graphql/reference/*" # Files in this folder are auto-generated + # QA changes + - ".dockerignore" + - "qa/**/*" + +.code-backstage-qa-patterns: &code-backstage-qa-patterns + - ".gitlab/ci/**/*" + - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}" + - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml" + - ".csscomb.json" + - "Dockerfile.assets" + - "*_VERSION" + - "Gemfile{,.lock}" + - "Rakefile" + - "{babel.config,jest.config}.js" + - "config.ru" + - "{package.json,yarn.lock}" + - "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*" + - "doc/api/graphql/reference/*" # Files in this folder are auto-generated + # Backstage changes + - "Dangerfile" + - "danger/**/*" + - "{,ee/}fixtures/**/*" + - "{,ee/}rubocop/**/*" + - "{,ee/}spec/**/*" + - "doc/README.md" # Some RSpec test rely on this file + # QA changes + - ".dockerignore" + - "qa/**/*" + +#################### +# Cache repo rules # +#################### +.cache-repo:rules: + rules: + - <<: *if-cache-credentials-schedule + when: on_success + +############# +# CNG rules # +############# +.cng:rules: + rules: + - <<: *if-dot-com-gitlab-org-and-security-tag + when: manual + +###################### +# Dev fixtures rules # +###################### +.dev-fixtures:rules:ee-and-foss: + rules: + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +.dev-fixtures:rules:ee-only: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +############## +# Docs rules # +############## +.docs:rules:review-docs: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *docs-patterns + when: manual + +.docs:rules:docs-lint: + rules: + - <<: *if-default-refs + changes: *docs-patterns + when: on_success + +.docs:rules:graphql-reference-verify: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-backstage-qa-patterns + when: on_success + +################## +# Frontend rules # +################## +.frontend:rules:gitlab-assets-compile-pull-push-cache: + rules: + - <<: *if-not-canonical-namespace + when: never + - <<: *if-master-refs + changes: *code-backstage-qa-patterns + when: on_success + +.frontend:rules:gitlab-assets-compile-pull-cache: + rules: + - <<: *if-not-canonical-namespace + when: never + - <<: *if-default-refs + changes: *code-backstage-qa-patterns + when: on_success + +.frontend:rules:compile-assets-pull-push-cache: + rules: + - <<: *if-master-refs + changes: *code-backstage-qa-patterns + when: on_success + +.frontend:rules:compile-assets-pull-push-cache-as-if-foss: + rules: + - <<: *if-not-ee + when: never + - <<: *if-master-refs + changes: *code-backstage-qa-patterns + when: on_success + +.frontend:rules:compile-assets-pull-cache: + rules: + - <<: *if-default-refs + changes: *code-backstage-qa-patterns + when: on_success + +.frontend:rules:compile-assets-pull-cache-as-if-foss: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-backstage-qa-patterns + when: on_success + +.frontend:rules:default-frontend-jobs: + rules: + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +.frontend:rules:default-frontend-jobs-as-if-foss: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +.frontend:rules:qa-frontend-node: + rules: + - <<: *if-master-refs + when: on_success + +################ +# Memory rules # +################ +.memory:rules: + rules: + - <<: *if-default-refs + changes: *code-patterns + when: on_success + +############### +# Pages rules # +############### +.pages:rules: + rules: + - <<: *if-dot-com-gitlab-org-master + changes: *code-backstage-qa-patterns + when: on_success + +############ +# QA rules # +############ +.qa:rules:ee-and-foss: + rules: + - <<: *if-default-refs + changes: *code-qa-patterns + when: on_success + +.qa:rules:ee-only: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-qa-patterns + when: on_success + +.qa:rules:package-and-qa: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *qa-patterns + when: on_success + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-patterns + when: manual + - <<: *if-dot-com-gitlab-org-schedule + when: on_success + +############### +# Rails rules # +############### +.rails:rules:ee-and-foss: + rules: + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +.rails:rules:default-refs-code-backstage-qa: + rules: + - <<: *if-default-refs + changes: *code-backstage-qa-patterns + when: on_success + +.rails:rules:master-refs-code-backstage: + rules: + - <<: *if-master-refs + changes: *code-backstage-patterns + when: on_success + +.rails:rules:master-refs-code-backstage-ee-only: + rules: + - <<: *if-not-ee + when: never + - <<: *if-master-refs + changes: *code-backstage-patterns + when: on_success + +.rails:rules:ee-only: + rules: + - <<: *if-not-ee + when: never + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +.rails:rules:downtime_check: + rules: + - <<: *if-merge-request + changes: *code-backstage-patterns + when: on_success + +################## +# Releases rules # +################## +.releases:rules:canonical-dot-com-gitlab-stable-branch-only: + rules: + - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/' + +.releases:rules:canonical-dot-com-security-gitlab-stable-branch-only: + rules: + - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/' + +################# +# Reports rules # +################# +.reports:rules:code_quality: + rules: + - if: '$CODE_QUALITY_DISABLED' + when: never + # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 + - <<: *if-default-refs + changes: *code-backstage-patterns + +.reports:rules:sast: + rules: + - if: '$SAST_DISABLED || $GITLAB_FEATURES !~ /\bsast\b/' + when: never + # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 + - <<: *if-default-refs + changes: *code-backstage-qa-patterns + +.reports:rules:dependency_scanning: + rules: + - if: '$DEPENDENCY_SCANNING_DISABLED || $GITLAB_FEATURES !~ /\bdependency_scanning\b/' + when: never + # - <<: *if-master-refs # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255 + - <<: *if-default-refs + changes: *code-backstage-qa-patterns + +.reports:rules:dast: + rules: + - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' + when: never + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + +.reports:schedule-dast: + rules: + - if: '$DAST_DISABLED || $GITLAB_FEATURES !~ /\bdast\b/' + when: never + - <<: *if-dot-com-gitlab-org-schedule + +################ +# Review rules # +################ +.review:rules:mr-and-schedule: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: on_success + - <<: *if-dot-com-gitlab-org-schedule + when: on_success + +.review:rules:mr-only-auto: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: on_success + +.review:rules:mr-only-manual: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: manual + +.review:rules:review-cleanup: + rules: + - <<: *if-dot-com-gitlab-org-merge-request + changes: *code-qa-patterns + when: manual + - <<: *if-dot-com-gitlab-org-schedule + when: on_success + +.review:rules:danger: + rules: + - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID' + when: on_success + +############### +# Setup rules # +############### +.setup:rules:cache-gems: + rules: + - <<: *if-not-canonical-namespace + when: never + - <<: *if-master-or-tag + changes: *code-backstage-qa-patterns + when: on_success + +.setup:rules:gitlab_git_test: + rules: + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +.setup:rules:no_ee_check: + rules: + - <<: *if-not-foss + when: never + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +####################### +# Test metadata rules # +####################### +.test-metadata:rules:retrieve-tests-metadata: + rules: + - <<: *if-default-refs + changes: *code-backstage-patterns + when: on_success + +.test-metadata:rules:update-tests-metadata: + rules: + - <<: *if-dot-com-ee-schedule + changes: *code-backstage-patterns + when: on_success + +.test-metadata:rules:flaky-examples-check: + rules: + - <<: *if-merge-request + changes: *code-backstage-patterns + when: on_success + + +############## +# YAML rules # +############## +.yaml:rules: + rules: + - <<: *if-default-refs + changes: *yaml-patterns diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml index 24267584393..fb203db1478 100644 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ b/.gitlab/ci/setup.gitlab-ci.yml @@ -6,11 +6,11 @@ cache gems: - .default-retry - .default-cache - .default-before_script - - .only:variables-canonical-dot-com - - .only:changes-code-backstage-qa + - .setup:rules:cache-gems stage: test - dependencies: ["setup-test-env"] - needs: ["setup-test-env"] + needs: + - job: setup-test-env + artifacts: true variables: SETUP_DB: "false" script: @@ -18,30 +18,23 @@ cache gems: artifacts: paths: - vendor/cache - only: - refs: - - master - - tags .minimal-job: extends: - .default-tags - .default-retry - - .default-only - - .only:changes-code-backstage dependencies: [] gitlab_git_test: - extends: .minimal-job + extends: + - .minimal-job + - .setup:rules:gitlab_git_test script: - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes no_ee_check: - extends: .minimal-job + extends: + - .minimal-job + - .setup:rules:no_ee_check script: - scripts/no-ee-check - only: - variables: - - $CI_PROJECT_NAME == "gitlab-foss" - - $CI_PROJECT_NAME == "gitlab-ce" # Support former project name for forks/mirrors - - $CI_PROJECT_NAME == "gitlabhq" # Support former project name for dev diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml index 21af0d373bc..719e4e821c9 100644 --- a/.gitlab/ci/test-metadata.gitlab-ci.yml +++ b/.gitlab/ci/test-metadata.gitlab-ci.yml @@ -1,7 +1,4 @@ .tests-metadata-state: - extends: - - .default-only - - .only:changes-code-backstage variables: TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache" before_script: @@ -19,7 +16,9 @@ - rspec_profiling/ retrieve-tests-metadata: - extends: .tests-metadata-state + extends: + - .tests-metadata-state + - .test-metadata:rules:retrieve-tests-metadata stage: prepare cache: policy: pull @@ -28,7 +27,9 @@ retrieve-tests-metadata: - retrieve_tests_metadata update-tests-metadata: - extends: .tests-metadata-state + extends: + - .tests-metadata-state + - .test-metadata:rules:update-tests-metadata stage: post-test cache: policy: push @@ -36,27 +37,17 @@ update-tests-metadata: - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document - source scripts/rspec_helpers.sh - update_tests_metadata - only: - refs: - - schedules - variables: - # Only update the Knapsack metadata on GitLab.com/gitlab-org/gitlab - - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" flaky-examples-check: extends: - .default-tags - .default-retry - - .default-only - - .only:changes-code-backstage + - .test-metadata:rules:flaky-examples-check image: ruby:2.6-alpine stage: post-test variables: NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json allow_failure: true - only: - refs: - - merge_requests artifacts: expire_in: 30d paths: @@ -64,4 +55,4 @@ flaky-examples-check: script: - '[[ -f $NEW_FLAKY_SPECS_REPORT ]] || echo "{}" > ${NEW_FLAKY_SPECS_REPORT}' - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json - - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT + - scripts/flaky_examples/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT diff --git a/.gitlab/ci/yaml.gitlab-ci.yml b/.gitlab/ci/yaml.gitlab-ci.yml index 4fcf940974b..cdc3aa6e577 100644 --- a/.gitlab/ci/yaml.gitlab-ci.yml +++ b/.gitlab/ci/yaml.gitlab-ci.yml @@ -4,10 +4,7 @@ lint-ci-gitlab: extends: - .default-tags - .default-retry - - .default-only - only: - changes: - - "**/*.yml" + - .yaml:rules image: sdesbure/yamllint:latest dependencies: [] variables: diff --git a/.gitlab/issue_templates/Coding style proposal.md b/.gitlab/issue_templates/Coding style proposal.md deleted file mode 100644 index 95f0fb5f366..00000000000 --- a/.gitlab/issue_templates/Coding style proposal.md +++ /dev/null @@ -1,16 +0,0 @@ -## Description of the proposal - -<!-- -Please describe the proposal and add a link to the source (for example, http://www.betterspecs.org/). ---> - -- [ ] Mention the proposal in the next backend weekly call and the #backend channel to encourage contribution -- [ ] Proceed with the proposal once 50% of the maintainers have weighed in, and 80% of their votes are :+1: -- [ ] Once approved, mention it again in the next backend weekly call and the #backend channel - - -/label ~"development guidelines" -/label ~"Style decision" -/label ~documentation - -/cc @gitlab-org/maintainers/rails-backend diff --git a/.gitlab/issue_templates/Feature proposal.md b/.gitlab/issue_templates/Feature proposal.md index 45e9c58205f..fa989e45281 100644 --- a/.gitlab/issue_templates/Feature proposal.md +++ b/.gitlab/issue_templates/Feature proposal.md @@ -36,15 +36,24 @@ Personas are described at https://about.gitlab.com/handbook/marketing/product-ma Add all known Documentation Requirements here, per https://docs.gitlab.com/ee/development/documentation/feature-change-workflow.html#documentation-requirements If this feature requires changing permissions, this document https://docs.gitlab.com/ee/user/permissions.html must be updated accordingly. --> -### Testing +### Availability & Testing -<!-- What risks does this change pose? How might it affect the quality of the product? What additional test coverage or changes to tests will be needed? Will it require cross-browser testing? See the test engineering process for further help: https://about.gitlab.com/handbook/engineering/quality/test-engineering/ --> +<!-- This section needs to be retained and filled in during the workflow planning breakdown phase of this feature proposal, if not earlier. + + What risks does this change pose to our availability? How might it affect the quality of the product? What additional test coverage or changes to tests will be needed? Will it require cross-browser testing? + +Please list the test areas (unit, integration and end-to-end) that needs to be added or updated to ensure that this feature will work as intended. Please use the list below as guidance. +* Unit test changes +* Integration test changes +* End-to-end test change + +See the test engineering planning process and reach out to your counterpart Software Engineer in Test for assistance: https://about.gitlab.com/handbook/engineering/quality/test-engineering/#test-planning --> ### What does success look like, and how can we measure that? <!-- Define both the success metrics and acceptance criteria. Note that success metrics indicate the desired business outcomes, while acceptance criteria indicate when the solution is working correctly. If there is no way to measure success, link to an issue that will implement a way to measure this. --> -### What is the type of buyer? +### What is the type of buyer? <!-- Which leads to: in which enterprise tier should this feature go? See https://about.gitlab.com/handbook/product/pricing/#four-tiers --> diff --git a/.gitlab/issue_templates/QA failure.md b/.gitlab/issue_templates/QA failure.md new file mode 100644 index 00000000000..13b5d7bf92c --- /dev/null +++ b/.gitlab/issue_templates/QA failure.md @@ -0,0 +1,65 @@ +<!--- +Before opening a new QA failure issue, make sure to first search for it in the +QA failures board: https://gitlab.com/groups/gitlab-org/-/boards/1385578 + +The issue should have the following: + +- The relative path of the failing spec file in the title, e.g. if the login + test fails, include `qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb` in the title. + This is required so that existing issues can easily be found by searching for the spec file. +- If the issue is about multiple test failures, include the path for each failing spec file in the description. +- A link to the failing job. +- The stack trace from the job's logs in the "Stack trace" section below. +- A screenshot (if available), and HTML capture (if available), in the "Screenshot / HTML page" section below. +---> + +### Summary + + + +### Stack trace + +``` +PUT STACK TRACE HERE +``` + +### Screenshot / HTML page + +<!-- +Attach the screenshot and HTML snapshot of the page from the job's artifacts: +1. Download the job's artifacts and unarchive them. +1. Open the `gitlab-qa-run-2020-*/gitlab-{ce,ee}-qa-*/{,ee}/{api,browser_ui}/<path to failed test>` folder. +1. Select the `.png` and `.html` files that appears in the job logs (look for `HTML screenshot: /path/to/html/page.html` / `Image screenshot: `/path/to/html/page.png`). +1. Drag and drop them here. +--> + +### Possible fixes + + +<!-- Default due date. --> +/due in 2 weeks + +<!-- Base labels. --> +/label ~Quality ~QA ~bug ~S1 + +<!-- +Choose the stage that appears in the test path, e.g. ~"devops::create" for +`qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb`. +--> +/label ~devops:: + +<!-- +Select a label for where the failure was found, e.g. if the failure occurred in +a nightly pipeline, select ~"found:nightly". +--> +/label ~found: + +<!-- +https://about.gitlab.com/handbook/engineering/quality/guidelines/#priorities: +- ~P1: Tests that are needed to verify fundamental GitLab functionality. +- ~P2: Tests that deal with external integrations which may take a longer time to debug and fix. +--> +/label ~P + +<!-- Select the current milestone if ~P1 or the next milestone if ~P2. --> +/milestone % diff --git a/.gitlab/issue_templates/Security Release.md b/.gitlab/issue_templates/Security Release.md index e6e5d731d96..e5b26bc6fc6 100644 --- a/.gitlab/issue_templates/Security Release.md +++ b/.gitlab/issue_templates/Security Release.md @@ -16,37 +16,27 @@ Set the title to: `Security Release: 12.2.X, 12.1.X, and 12.0.X` * 12.1.X: {release task link} * 12.0.X: {release task link} -## Security Issues: +## Issues in GitLab Security -* {https://gitlab.com/gitlab-org/gitlab/issues link} - -## Security Issues in dev.gitlab.org: - -### CE - -- {https://dev.gitlab.org/gitlab/gitlabhq/issues link} +* {https://gitlab.com/gitlab-org/security/gitlab/issues/ link} | Version | MR | |---------|----| -| 12.2 | {https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/ link} | -| 12.1 | {https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/ link} | -| 12.0 | {https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/ link} | -| master | {https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/ link} | - +| 12.2 | {https://gitlab.com/gitlab-org/security/gitlab/-/merge_requests link} | +| 12.1 | {https://gitlab.com/gitlab-org/security/gitlab/-/merge_requests link} | +| 12.0 | {https://gitlab.com/gitlab-org/security/gitlab/-/merge_requests link} | +| master | {https://gitlab.com/gitlab-org/security/gitlab/-/merge_requests link} | +## Issues in Omnibus-GitLab -### EE - -* {https://dev.gitlab.org/gitlab/gitlabhq/issues/ link} - +* {https://gitlab.com/gitlab-org/security/gitlab/issues/ link} | Version | MR | |---------|----| -| 12.2 | {https://dev.gitlab.org/gitlab/gitlab-ee/merge_requests/ link} | -| 12.1 | {https://dev.gitlab.org/gitlab/gitlab-ee/merge_requests/ link} | -| 12.0 | {https://dev.gitlab.org/gitlab/gitlab-ee/merge_requests/ link} | -| master | {https://dev.gitlab.org/gitlab/gitlab-ee/merge_requests/ link} | - +| 12.2 | {https://dev.gitlab.org/gitlab/omnibus-gitlab/merge_requests/ link} | +| 12.1 | {https://dev.gitlab.org/gitlab/omnibus-gitlab/merge_requests/ link} | +| 12.0 | {https://dev.gitlab.org/gitlab/omnibus-gitlab/merge_requests/ link} | +| master | {https://dev.gitlab.org/gitlab/omnibus-gitlab/merge_requests/ link} | ## QA {QA issue link} @@ -54,7 +44,7 @@ Set the title to: `Security Release: 12.2.X, 12.1.X, and 12.0.X` ## Blog post Dev: {https://dev.gitlab.org/gitlab/www-gitlab-com/merge_requests/ link}<br/> -gitlab.com: {https://gitlab.com/gitlab-com/www-gitlab-com/merge_requests/ link} +GitLab.com: {https://gitlab.com/gitlab-com/www-gitlab-com/merge_requests/ link} ## Email notification {https://gitlab.com/gitlab-com/marketing/general/issues/ link} diff --git a/.gitlab/merge_request_templates/New static analysis check.md b/.gitlab/merge_request_templates/New static analysis check.md new file mode 100644 index 00000000000..b89b6f7dbc4 --- /dev/null +++ b/.gitlab/merge_request_templates/New static analysis check.md @@ -0,0 +1,27 @@ +## Description of the proposal + +<!-- +Please describe the proposal and add a link to the source (for example, http://www.betterspecs.org/). +--> + +### Check-list + +- [ ] Make sure this MR enables a static analysis check rule for new usage but + ignores current offenses +- [ ] Create a follow-up issue to fix the current offenses as a separate iteration: ISSUE_LINK +- [ ] Mention this proposal in the relevant Slack channels (e.g. `#development`, `#backend`, `#frontend`) +- [ ] If there is a choice to make between two potential styles, set up an emoji vote in the MR: + - CHOICE_A: :a: + - CHOICE_B: :b: + - Vote yourself for both choices so that people know these are the choices +- [ ] The MR doesn't have significant objections, and is getting a majority of :+1: vs :-1: (remember that [we don't need to reach a consensus](https://about.gitlab.com/handbook/values/#collaboration-is-not-consensus)) +- [ ] (If applicable) One style is getting a majority of vote (compared to the other choice) +- [ ] (If applicable) Update the MR with the chosen style +- [ ] Follow the [review process](https://docs.gitlab.com/ee/development/code_review.html) as usual +- [ ] Once approved and merged by a maintainer, mention it again: + - [ ] In the relevant Slack channels (e.g. `#development`, `#backend`, `#frontend`) + - [ ] (Optional depending on the impact of the change) In the Engineering Week in Review + +/label ~"Engineering Productivity" ~"Style decision" ~"development guidelines" ~"static analysis" + +/cc @gitlab-org/maintainers/rails-backend |