diff options
author | Robert Speicher <rspeicher@gmail.com> | 2021-01-20 22:34:23 +0300 |
---|---|---|
committer | Robert Speicher <rspeicher@gmail.com> | 2021-01-20 22:34:23 +0300 |
commit | 6438df3a1e0fb944485cebf07976160184697d72 (patch) | |
tree | 00b09bfd170e77ae9391b1a2f5a93ef6839f2597 /.gitlab/ci | |
parent | 42bcd54d971da7ef2854b896a7b34f4ef8601067 (diff) |
Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42
Diffstat (limited to '.gitlab/ci')
-rw-r--r-- | .gitlab/ci/dev-fixtures.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | .gitlab/ci/docs.gitlab-ci.yml | 15 | ||||
-rw-r--r-- | .gitlab/ci/frontend.gitlab-ci.yml | 3 | ||||
-rw-r--r-- | .gitlab/ci/global.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | .gitlab/ci/graphql.gitlab-ci.yml | 14 | ||||
-rw-r--r-- | .gitlab/ci/pages.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | .gitlab/ci/rails.gitlab-ci.yml | 119 | ||||
-rw-r--r-- | .gitlab/ci/reports.gitlab-ci.yml | 104 | ||||
-rw-r--r-- | .gitlab/ci/review.gitlab-ci.yml | 8 | ||||
-rw-r--r-- | .gitlab/ci/rules.gitlab-ci.yml | 30 |
10 files changed, 188 insertions, 109 deletions
diff --git a/.gitlab/ci/dev-fixtures.gitlab-ci.yml b/.gitlab/ci/dev-fixtures.gitlab-ci.yml index c19dce7e4a9..1848283f921 100644 --- a/.gitlab/ci/dev-fixtures.gitlab-ci.yml +++ b/.gitlab/ci/dev-fixtures.gitlab-ci.yml @@ -15,7 +15,6 @@ # SEED_NESTED_GROUPS: "false" # requires network connection .run-dev-fixtures-script: &run-dev-fixtures-script - - run_timed_command "scripts/gitaly-test-build" - run_timed_command "scripts/gitaly-test-spawn" - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu" diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index d6dc709a11a..955f44c6216 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -43,7 +43,7 @@ docs-lint markdown: - .default-retry - .docs:rules:docs-lint # When updating the image version here, update it in /scripts/lint-doc.sh too. - image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.12-vale-2.6.1-markdownlint-0.24.0" + image: "registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.12-vale-2.8.0-markdownlint-0.26.0" stage: test needs: [] script: @@ -84,16 +84,3 @@ ui-docs-links lint: needs: [] script: - bundle exec haml-lint -i DocumentationLinks - -graphql-reference-verify: - extends: - - .default-retry - - .rails-cache - - .default-before_script - - .docs:rules:graphql-reference-verify - - .use-pg11 - stage: test - needs: ["setup-test-env"] - script: - - bundle exec rake gitlab:graphql:check_docs - - bundle exec rake gitlab:graphql:check_schema diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 0b921309ced..c87305cab18 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -103,7 +103,6 @@ update-yarn-cache: WEBPACK_VENDOR_DLL: "true" script: - run_timed_command "gem install knapsack --no-document" - - run_timed_command "scripts/gitaly-test-build" - run_timed_command "scripts/gitaly-test-spawn" - source ./scripts/rspec_helpers.sh - rspec_paralellized_job "--tag frontend_fixture" @@ -236,6 +235,8 @@ coverage-frontend: - *yarn-install script: - run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js" + # Removing the individual coverage results, as we just merged them. + - rm -r coverage-frontend/jest-* coverage: '/^Statements\s*:\s*?(\d+(?:\.\d+)?)%/' artifacts: name: coverage-frontend diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml index 0fafd5869d9..355607c17ac 100644 --- a/.gitlab/ci/global.gitlab-ci.yml +++ b/.gitlab/ci/global.gitlab-ci.yml @@ -112,7 +112,7 @@ .use-kaniko: image: - name: gcr.io/kaniko-project/executor:debug-v0.20.0 + name: gcr.io/kaniko-project/executor:debug-v1.3.0 entrypoint: [""] before_script: - source scripts/utils.sh diff --git a/.gitlab/ci/graphql.gitlab-ci.yml b/.gitlab/ci/graphql.gitlab-ci.yml new file mode 100644 index 00000000000..4aff0ef6306 --- /dev/null +++ b/.gitlab/ci/graphql.gitlab-ci.yml @@ -0,0 +1,14 @@ +graphql-verify: + variables: + SETUP_DB: "false" + extends: + - .default-retry + - .rails-cache + - .default-before_script + - .graphql:rules:graphql-verify + stage: test + needs: [] + script: + - bundle exec rake gitlab:graphql:validate + - bundle exec rake gitlab:graphql:check_docs + - bundle exec rake gitlab:graphql:check_schema diff --git a/.gitlab/ci/pages.gitlab-ci.yml b/.gitlab/ci/pages.gitlab-ci.yml index a66e0d88db3..4961bd508d3 100644 --- a/.gitlab/ci/pages.gitlab-ci.yml +++ b/.gitlab/ci/pages.gitlab-ci.yml @@ -14,7 +14,6 @@ pages: - mv coverage/ public/coverage-ruby/ || true - mv coverage-frontend/ public/coverage-frontend/ || true - mv coverage-javascript/ public/coverage-javascript/ || true - - mv webpack-report/ public/webpack-report/ || true - cp .public/assets/application-*.css public/application.css || true - cp .public/assets/application-*.css.gz public/application.css.gz || true artifacts: diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 2818b6be176..7f8dfa900ca 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -10,7 +10,6 @@ # Only install knapsack after bundle install! Otherwise oddly some native # gems could not be found under some circumstance. No idea why, hours wasted. - run_timed_command "gem install knapsack --no-document" - - run_timed_command "scripts/gitaly-test-build" - run_timed_command "scripts/gitaly-test-spawn" - source ./scripts/rspec_helpers.sh @@ -150,20 +149,35 @@ setup-test-env: script: - run_timed_command "bundle exec ruby -I. -e 'require \"config/environment\"; TestEnv.init'" - run_timed_command "scripts/gitaly-test-build" # Do not use 'bundle exec' here - - rm tmp/tests/gitaly/.ruby-bundle # This file prevents gems from being installed even if vendor/gitaly-ruby is missing artifacts: expire_in: 7d paths: - config/secrets.yml - - tmp/tests/gitaly - - tmp/tests/gitlab-elasticsearch-indexer - - tmp/tests/gitlab-shell - - tmp/tests/gitlab-test-fork - - tmp/tests/gitlab-test-fork_bare - - tmp/tests/gitlab-test - - tmp/tests/gitlab-workhorse - - tmp/tests/repositories - - tmp/tests/second_storage + - tmp/tests/gitaly/config.toml + - tmp/tests/gitaly/gitaly + - tmp/tests/gitaly/gitaly2.config.toml + - tmp/tests/gitaly/gitaly-git2go + - tmp/tests/gitaly/gitaly-hooks + - tmp/tests/gitaly/gitaly-lfs-smudge + - tmp/tests/gitaly/gitaly-ssh + - tmp/tests/gitaly/internal/ + - tmp/tests/gitaly/internal_sockets/ + - tmp/tests/gitaly/Makefile + - tmp/tests/gitaly/praefect + - tmp/tests/gitaly/praefect.config.toml + - tmp/tests/gitaly/ruby/ + - tmp/tests/gitlab-elasticsearch-indexer/bin/gitlab-elasticsearch-indexer + - tmp/tests/gitlab-shell/ + - tmp/tests/gitlab-test-fork/ + - tmp/tests/gitlab-test-fork_bare/ + - tmp/tests/gitlab-test/ + - tmp/tests/gitlab-workhorse/gitlab-zip-metadata + - tmp/tests/gitlab-workhorse/gitlab-zip-cat + - tmp/tests/gitlab-workhorse/gitlab-workhorse + - tmp/tests/gitlab-workhorse/gitlab-resize-image + - tmp/tests/gitlab-workhorse/config.toml + - tmp/tests/repositories/ + - tmp/tests/second_storage/ when: always update-rails-cache: @@ -286,6 +300,16 @@ rspec system pg11 minimal: - .minimal-rspec-tests - .rails:rules:ee-and-foss-system:minimal +# Dedicated job to test DB library code against PG12. +# Note that these are already tested against PG11 in the `rspec unit pg11` / `rspec-ee unit pg11` jobs. +rspec db-library-code pg12: + extends: + - .rspec-base-pg12 + - .rails:rules:ee-and-foss-db-library-code + script: + - *base-script + - rspec_simple_job "-- spec/lib/gitlab/database/ spec/support/helpers/database/ ee/spec/lib/gitlab/database/ ee/spec/lib/ee/gitlab/database_spec.rb" + rspec fast_spec_helper: extends: - .rspec-base-pg11 @@ -311,6 +335,14 @@ db:check-schema: script: - source scripts/schema_changed.sh +db:check-migrations: + extends: + - .db-job-base + - .rails:rules:ee-and-foss-mr-with-migration + script: + - scripts/validate_migration_schema + allow_failure: true + db:migrate-from-v12.10.0: extends: .db-job-base variables: @@ -376,6 +408,38 @@ db:backup_and_restore: rules: - changes: ["lib/backup/**/*"] +rspec:deprecations: + extends: + - .default-retry + - .default-before_script + - .static-analysis-cache + - .rails:rules:deprecations + stage: post-test + allow_failure: true + # We cannot use needs since it would mean needing 84 jobs (since most are parallelized) + # so we use `dependencies` here. + dependencies: + - rspec migration pg11 + - rspec unit pg11 + - rspec integration pg11 + - rspec system pg11 + - rspec-ee migration pg11 + - rspec-ee unit pg11 + - rspec-ee integration pg11 + - rspec-ee system pg11 + - rspec-ee unit pg11 geo + - rspec-ee integration pg11 geo + - rspec-ee system pg11 geo + variables: + SETUP_DB: "false" + script: + - run_timed_command "bundle exec rubocop --only Lint/LastKeywordArgument --parallel" + artifacts: + expire_in: 31d + when: always + paths: + - deprecations/ + rspec:coverage: extends: - .coverage-base @@ -549,33 +613,36 @@ rspec-ee unit pg11 geo: - .rails:rules:ee-only-unit - .rspec-ee-unit-geo-parallel -rspec-ee unit pg11 geo minimal: - extends: - - rspec-ee unit pg11 geo - - .minimal-rspec-tests - - .rails:rules:ee-only-unit:minimal +# FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212 +#rspec-ee unit pg11 geo minimal: +# extends: +# - rspec-ee unit pg11 geo +# - .minimal-rspec-tests +# - .rails:rules:ee-only-unit:minimal rspec-ee integration pg11 geo: extends: - .rspec-ee-base-geo-pg11 - .rails:rules:ee-only-integration -rspec-ee integration pg11 geo minimal: - extends: - - rspec-ee integration pg11 geo - - .minimal-rspec-tests - - .rails:rules:ee-only-integration:minimal +# FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212 +#rspec-ee integration pg11 geo minimal: +# extends: +# - rspec-ee integration pg11 geo +# - .minimal-rspec-tests +# - .rails:rules:ee-only-integration:minimal rspec-ee system pg11 geo: extends: - .rspec-ee-base-geo-pg11 - .rails:rules:ee-only-system -rspec-ee system pg11 geo minimal: - extends: - - rspec-ee system pg11 geo - - .minimal-rspec-tests - - .rails:rules:ee-only-system:minimal +# FIXME: Temporarily disable geo minimal rspec jobs https://gitlab.com/gitlab-org/gitlab/-/issues/294212 +#rspec-ee system pg11 geo minimal: +# extends: +# - rspec-ee system pg11 geo +# - .minimal-rspec-tests +# - .rails:rules:ee-only-system:minimal db:rollback geo: extends: diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 85aec070557..77ada89aa6a 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -4,9 +4,9 @@ # - template: Security/Dependency-Scanning.gitlab-ci.yml # - template: Security/DAST.gitlab-ci.yml -# We need to duplicate this job's definition because it seems it's impossible to -# override an included `only.refs`. -# See https://gitlab.com/gitlab-org/gitlab/issues/31371. +# We need to duplicate this job's definition because the rules +# defined in the extended jobs rely on local YAML anchors +# (`*if-default-refs`) code_quality: extends: - .default-retry @@ -36,9 +36,9 @@ code_quality: - gl-code-quality-report.json # GitLab-specific expire_in: 1 week # GitLab-specific -# We need to duplicate this job's definition because it seems it's impossible to -# override an included `only.refs`. -# See https://gitlab.com/gitlab-org/gitlab/issues/31371. +# We need to duplicate this job's definition because the rules +# defined in the extended jobs rely on local YAML anchors +# (`*if-default-refs`) .sast: extends: - .default-retry @@ -89,74 +89,58 @@ secrets-sast: sast: gl-secret-detection-report.json expire_in: 1 week # GitLab-specific -# We need to duplicate this job's definition because it seems it's impossible to -# override an included `only.refs`. -# See https://gitlab.com/gitlab-org/gitlab/issues/31371. -dependency_scanning: +# We need to duplicate this job's definition because the rules +# defined in the extended jobs rely on local YAML anchors +# (`*if-default-refs`) +.dependency_scanning: extends: - .default-retry - .reports:rules:dependency_scanning - - .use-docker-in-docker stage: test needs: [] variables: DS_MAJOR_VERSION: 2 - DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec" # GitLab-specific - script: - - | - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - fi - fi - - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage - function propagate_env_vars() { - CURRENT_ENV=$(printenv) - - for VAR_NAME; do - echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " - done - } - - | - docker run \ - $(propagate_env_vars \ - DS_ANALYZER_IMAGES \ - DS_ANALYZER_IMAGE_PREFIX \ - DS_ANALYZER_IMAGE_TAG \ - DS_DEFAULT_ANALYZERS \ - DS_EXCLUDED_PATHS \ - DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ - DS_PULL_ANALYZER_IMAGE_TIMEOUT \ - DS_RUN_ANALYZER_TIMEOUT \ - DS_PYTHON_VERSION \ - DS_PIP_VERSION \ - DS_PIP_DEPENDENCY_PATH \ - GEMNASIUM_DB_LOCAL_PATH \ - GEMNASIUM_DB_REMOTE_URL \ - GEMNASIUM_DB_REF_NAME \ - PIP_INDEX_URL \ - PIP_EXTRA_INDEX_URL \ - PIP_REQUIREMENTS_FILE \ - MAVEN_CLI_OPTS \ - BUNDLER_AUDIT_UPDATE_DISABLED \ - BUNDLER_AUDIT_ADVISORY_DB_URL \ - BUNDLER_AUDIT_ADVISORY_DB_REF_NAME \ - ) \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_MAJOR_VERSION" /code - # Post-processing: This will be an after_script once this job will use the Dependency Scanning CI template - - apk add jq - # Lower execa severity based on https://gitlab.com/gitlab-org/gitlab/-/issues/223859#note_452922390 - - jq '(.vulnerabilities[] | select (.cve == "yarn.lock:execa:gemnasium:05cfa2e8-2d0c-42c1-8894-638e2f12ff3d")).severity = "Medium"' gl-dependency-scanning-report.json > temp.json && mv temp.json gl-dependency-scanning-report.json + DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports, spec, ee/spec" # GitLab-specific + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" artifacts: paths: - gl-dependency-scanning-report.json # GitLab-specific reports: dependency_scanning: gl-dependency-scanning-report.json expire_in: 1 week # GitLab-specific + script: + - /analyzer run + +dependency_scanning gemnasium: + extends: .dependency_scanning + image: + name: "$SECURE_ANALYZERS_PREFIX/gemnasium:$DS_MAJOR_VERSION" + before_script: + # git-lfs is needed for auto-remediation + - apk add git-lfs + after_script: + # Post-processing + - apk add jq + # Lower execa severity based on https://gitlab.com/gitlab-org/gitlab/-/issues/223859#note_452922390 + - jq '(.vulnerabilities[] | select (.cve == "yarn.lock:execa:gemnasium:05cfa2e8-2d0c-42c1-8894-638e2f12ff3d")).severity = "Medium"' gl-dependency-scanning-report.json > temp.json && mv temp.json gl-dependency-scanning-report.json + +dependency_scanning bundler-audit: + extends: .dependency_scanning + image: + name: "$SECURE_ANALYZERS_PREFIX/bundler-audit:$DS_MAJOR_VERSION" + +dependency_scanning retire-js: + extends: .dependency_scanning + image: + name: "$SECURE_ANALYZERS_PREFIX/retire.js:$DS_MAJOR_VERSION" + +dependency_scanning gemnasium-python: + extends: .dependency_scanning + image: + name: "$SECURE_ANALYZERS_PREFIX/gemnasium-python:$DS_MAJOR_VERSION" -# The job below analysis dependencies for malicous behavior +# Analyze dependencies for malicious behavior +# See https://gitlab.com/gitlab-com/gl-security/security-research/package-hunter package_hunter: extends: - .reports:schedule-dast diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml index f1bd173ff6d..b7d9f18dcb4 100644 --- a/.gitlab/ci/review.gitlab-ci.yml +++ b/.gitlab/ci/review.gitlab-ci.yml @@ -38,7 +38,7 @@ review-build-cng: - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng # When the job is manual, review-deploy is also manual and we don't want people # to have to manually start the jobs in sequence, so we do it for them. - - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job --job-name "review-deploy"' + - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-deploy"' .review-workflow-base: extends: @@ -48,7 +48,7 @@ review-build-cng: HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}" REVIEW_APPS_DOMAIN: "temp.gitlab-review.app" # FIXME: using temporary domain DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}" - GITLAB_HELM_CHART_REF: "v4.3.0" + GITLAB_HELM_CHART_REF: "v4.6.3" environment: name: review/${CI_COMMIT_REF_SLUG}${FREQUENCY} url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN} @@ -78,8 +78,8 @@ review-deploy: - disable_sign_ups || (delete_release && exit 1) # When the job is manual, review-qa-smoke is also manual and we don't want people # to have to manually start the jobs in sequence, so we do it for them. - - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job --job-name "review-qa-smoke"' - - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job --job-name "review-performance"' + - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-qa-smoke"' + - '[ -z $CI_JOB_MANUAL ] || scripts/api/play_job.rb --job-name "review-performance"' after_script: # Run seed-dast-test-data.sh only when DAST_RUN is set to true. This is to pupulate review app with data for DAST scan. # Set DAST_RUN to true when jobs are manually scheduled. diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 159defc83c3..5e8cdf0daaf 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -155,9 +155,15 @@ - "{,ee/}{,spec/}lib/{,ee/}gitlab/database{,_spec}.rb" - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration/**/*" - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration{,_spec}.rb" + - "{,ee/}spec/support/helpers/database/**/*" - "config/prometheus/common_metrics.yml" # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer - "{,ee/}app/models/project_statistics.rb" # Used to calculate sizes in migration specs +.db-library-patterns: &db-library-patterns + - "{,ee/}{,spec/}lib/{,ee/}gitlab/database/**/*" + - "{,ee/}{,spec/}lib/{,ee/}gitlab/database{,_spec}.rb" + - "{,ee/}spec/support/helpers/database/**/*" + .backstage-patterns: &backstage-patterns - "Dangerfile" - "danger/**/*" @@ -349,7 +355,11 @@ changes: *docs-patterns when: on_success -.docs:rules:graphql-reference-verify: +################## +# GraphQL rules # +################## + +.graphql:rules:graphql-verify: rules: - <<: *if-not-ee when: never @@ -507,6 +517,12 @@ - <<: *if-merge-request changes: *db-patterns +.rails:rules:ee-and-foss-mr-with-migration: + rules: + - <<: *if-merge-request + changes: *db-patterns + - <<: *if-merge-request-title-run-all-rspec + .rails:rules:ee-and-foss-unit: rules: - changes: *backend-patterns @@ -765,6 +781,11 @@ - <<: *if-merge-request-title-as-if-foss changes: *code-backstage-patterns +.rails:rules:ee-and-foss-db-library-code: + rules: + - changes: *db-library-patterns + - <<: *if-merge-request-title-run-all-rspec + .rails:rules:ee-mr-and-master-only: rules: - <<: *if-not-ee @@ -825,6 +846,13 @@ - <<: *if-merge-request changes: *code-backstage-patterns +.rails:rules:deprecations: + rules: + - <<: *if-not-ee + when: never + - <<: *if-master-schedule-nightly + - <<: *if-merge-request-title-run-all-rspec + .rails:rules:rspec-coverage: rules: - <<: *if-not-ee |