diff options
Diffstat (limited to 'lib/gitlab/ci')
32 files changed, 257 insertions, 52 deletions
diff --git a/lib/gitlab/ci/build/releaser.rb b/lib/gitlab/ci/build/releaser.rb new file mode 100644 index 00000000000..ba6c7857e96 --- /dev/null +++ b/lib/gitlab/ci/build/releaser.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Releaser + BASE_COMMAND = 'release-cli create' + + attr_reader :config + + def initialize(config:) + @config = config + end + + def script + command = BASE_COMMAND.dup + config.each { |k, v| command.concat(" --#{k.to_s.dasherize} \"#{v}\"") } + + command + end + end + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 48111ae5717..f8550b50905 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -20,6 +20,19 @@ module Gitlab end end + def from_release(job) + return unless Gitlab::Ci::Features.release_generation_enabled? + + release = job.options[:release] + return unless release + + self.new(:release).tap do |step| + step.script = Gitlab::Ci::Build::Releaser.new(config: job.options[:release]).script + step.timeout = job.metadata_timeout + step.when = WHEN_ON_SUCCESS + end + end + def from_after_script(job) after_script = job.options[:after_script] return unless after_script diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 1ea59491378..66050a7bbe0 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -28,7 +28,7 @@ module Gitlab in: %i[release], message: 'release features are not enabled' }, - unless: -> { Feature.enabled?(:ci_release_generation, default_enabled: false) } + unless: -> { Gitlab::Ci::Features.release_generation_enabled? } with_options allow_nil: true do validates :allow_failure, boolean: true diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 1a871e043a6..74736b24d73 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -12,9 +12,10 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = - %i[junit codequality sast dependency_scanning container_scanning + %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif - dotenv cobertura terraform accessibility cluster_applications].freeze + dotenv cobertura terraform accessibility cluster_applications + requirements].freeze attributes ALLOWED_KEYS @@ -26,6 +27,7 @@ module Gitlab validates :junit, array_of_strings_or_string: true validates :codequality, array_of_strings_or_string: true validates :sast, array_of_strings_or_string: true + validates :secret_detection, array_of_strings_or_string: true validates :dependency_scanning, array_of_strings_or_string: true validates :container_scanning, array_of_strings_or_string: true validates :dast, array_of_strings_or_string: true @@ -39,6 +41,7 @@ module Gitlab validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true validates :cluster_applications, array_of_strings_or_string: true + validates :requirements, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 48f3d4fdd2f..a2eb31369c7 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -7,12 +7,40 @@ module Gitlab # module Features def self.artifacts_exclude_enabled? - ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: false) + ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end def self.ensure_scheduling_type_enabled? ::Feature.enabled?(:ci_ensure_scheduling_type, default_enabled: true) end + + def self.job_heartbeats_runner?(project) + ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) + end + + def self.instance_level_variables_limit_enabled? + ::Feature.enabled?(:ci_instance_level_variables_limit, default_enabled: true) + end + + def self.pipeline_fixed_notifications? + ::Feature.enabled?(:ci_pipeline_fixed_notifications) + end + + def self.instance_variables_ui_enabled? + ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) + end + + def self.composite_status?(project) + ::Feature.enabled?(:ci_composite_status, project, default_enabled: true) + end + + def self.atomic_processing?(project) + ::Feature.enabled?(:ci_atomic_processing, project, default_enabled: true) + end + + def self.release_generation_enabled? + ::Feature.enabled?(:ci_release_generation) + end end end end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb index 26a18c6603e..19f724b79af 100644 --- a/lib/gitlab/ci/parsers/terraform/tfplan.rb +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -8,15 +8,11 @@ module Gitlab TfplanParserError = Class.new(Gitlab::Ci::Parsers::ParserError) def parse!(json_data, terraform_reports, artifact:) - tfplan = Gitlab::Json.parse(json_data).tap do |parsed_data| - parsed_data['job_path'] = Gitlab::Routing.url_helpers.project_job_path( - artifact.job.project, artifact.job - ) - end + plan_data = Gitlab::Json.parse(json_data) - raise TfplanParserError, 'Tfplan missing required key' unless valid_supported_keys?(tfplan) + raise TfplanParserError, 'Tfplan missing required key' unless has_required_keys?(plan_data) - terraform_reports.add_plan(artifact.filename, tfplan) + terraform_reports.add_plan(artifact.job.id.to_s, tfplan(plan_data, artifact.job)) rescue JSON::ParserError raise TfplanParserError, 'JSON parsing failed' rescue @@ -25,8 +21,18 @@ module Gitlab private - def valid_supported_keys?(tfplan) - tfplan.keys == %w[create update delete job_path] + def has_required_keys?(plan_data) + (%w[create update delete] - plan_data.keys).empty? + end + + def tfplan(plan_data, artifact_job) + { + 'create' => plan_data['create'].to_i, + 'delete' => plan_data['delete'].to_i, + 'job_name' => artifact_job.options.dig(:artifacts, :name).to_s, + 'job_path' => Gitlab::Routing.url_helpers.project_job_path(artifact_job.project, artifact_job), + 'update' => plan_data['update'].to_i + } end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 73187401903..8118e7b2487 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -77,19 +77,18 @@ module Gitlab bridge&.parent_pipeline end - def duration_histogram - strong_memoize(:duration_histogram) do - name = :gitlab_ci_pipeline_creation_duration_seconds - comment = 'Pipeline creation duration' - labels = {} - buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] - - Gitlab::Metrics.histogram(name, comment, labels, buckets) - end + def metrics + @metrics ||= Chain::Metrics.new end def observe_creation_duration(duration) - duration_histogram.observe({}, duration.seconds) + metrics.pipeline_creation_duration_histogram + .observe({}, duration.seconds) + end + + def observe_pipeline_size(pipeline) + metrics.pipeline_size_histogram + .observe({ source: pipeline.source.to_s }, pipeline.total_size) end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb new file mode 100644 index 00000000000..980ab2de9b0 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Metrics + include Gitlab::Utils::StrongMemoize + + def pipeline_creation_duration_histogram + strong_memoize(:pipeline_creation_duration_histogram) do + name = :gitlab_ci_pipeline_creation_duration_seconds + comment = 'Pipeline creation duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def pipeline_size_histogram + strong_memoize(:pipeline_size_histogram) do + name = :gitlab_ci_pipeline_size_builds + comment = 'Pipeline size' + labels = { source: nil } + buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 2e177cfec7e..e48e79d561b 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,6 +13,7 @@ module Gitlab # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! + pipeline.ensure_ci_ref! # Protect the pipeline. This is assigned in Populate instead of # Build to prevent erroring out on ambiguous refs. diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index a7c671e76d3..204c7725214 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -27,6 +27,7 @@ module Gitlab yield @pipeline, self if block_given? @command.observe_creation_duration(Time.now - @start) + @command.observe_pipeline_size(@pipeline) end end diff --git a/lib/gitlab/ci/reports/terraform_reports.rb b/lib/gitlab/ci/reports/terraform_reports.rb index f955d007daf..4b52c25d724 100644 --- a/lib/gitlab/ci/reports/terraform_reports.rb +++ b/lib/gitlab/ci/reports/terraform_reports.rb @@ -10,14 +10,6 @@ module Gitlab @plans = {} end - def pick(keys) - terraform_plans = plans.select do |key| - keys.include?(key) - end - - { plans: terraform_plans } - end - def add_plan(name, plan) plans[name] = plan end diff --git a/lib/gitlab/ci/status/bridge/failed.rb b/lib/gitlab/ci/status/bridge/failed.rb index de7446c238c..b0ab0992594 100644 --- a/lib/gitlab/ci/status/bridge/failed.rb +++ b/lib/gitlab/ci/status/bridge/failed.rb @@ -5,6 +5,14 @@ module Gitlab module Status module Bridge class Failed < Status::Build::Failed + private + + def failure_reason_message + [ + self.class.reasons.fetch(subject.failure_reason.to_sym), + subject.options[:downstream_errors] + ].flatten.compact.join(', ') + end end end end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index ea773ee9944..4779c8d3d53 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -3,7 +3,7 @@ module Gitlab module Ci module Status - # Base abstract class fore core status + # Base abstract class for core status # class Core include Gitlab::Routing diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml new file mode 100644 index 00000000000..82b2f5c035e --- /dev/null +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -0,0 +1,13 @@ +stages: + - build + - test + - review + - deploy + - production + +variables: + AUTO_DEVOPS_PLATFORM_TARGET: ECS + +include: + - template: Jobs/Build.gitlab-ci.yml + - template: Jobs/Deploy/ECS.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 5017037fb5a..e37cd14d1d1 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -13,6 +13,7 @@ # * license_management: LICENSE_MANAGEMENT_DISABLED # * performance: PERFORMANCE_DISABLED # * sast: SAST_DISABLED +# * secret_detection: SECRET_DETECTION_DISABLED # * dependency_scanning: DEPENDENCY_SCANNING_DISABLED # * container_scanning: CONTAINER_SCANNING_DISABLED # * dast: DAST_DISABLED @@ -160,3 +161,4 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml index 82b2f5c035e..5f4bd631db6 100644 --- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml @@ -1,3 +1,18 @@ +# This template is deprecated and will be removed as part of GitLab 13.2! +# +# If you have referenced this template in your CI pipeline, please +# update your CI configuration by replacing the following occurrence(s): +# +# template: Deploy-ECS.gitlab-ci.yml +# +# with +# +# template: AWS/Deploy-ECS.gitlab-ci.yml +# +# -------------------- +# +# Documentation: https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs + stages: - build - test @@ -5,6 +20,9 @@ stages: - deploy - production +before_script: + - printf '\nWARNING!\nThis job includes "Deploy-ECS.gitlab-ci.yml". Please rename this to "AWS/Deploy-ECS.gitlab-ci.yml".\n' + variables: AUTO_DEVOPS_PLATFORM_TARGET: ECS diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index adbf9731e43..9a34f8cb113 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -1,11 +1,11 @@ performance: stage: performance - image: docker:19.03.8 + image: docker:19.03.11 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.8-dind + - docker:19.03.11-dind script: - | if ! docker info &>/dev/null; then diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 787f07521e0..b5550461482 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.3" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.8-dind + - docker:19.03.11-dind script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 24e75c56a75..bde6f185d3a 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -1,9 +1,9 @@ code_quality: stage: test - image: docker:19.03.8 + image: docker:19.03.11 allow_failure: true services: - - docker:19.03.8-dind + - docker:19.03.11-dind variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 5174aed04ba..bab4fae67f0 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.0" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 3fbae496896..97b5f3fd7f5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.0" include: - template: Jobs/Deploy/ECS.gitlab-ci.yml @@ -177,6 +177,7 @@ production_manual: .manual_rollout_template: &manual_rollout_template <<: *rollout_template stage: production + resource_group: production allow_failure: true rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 642f0ebeaf7..bb3d5526f3a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -1,3 +1,13 @@ +# WARNING (post-GitLab 13.0): +# +# This CI template should NOT be included in your own CI configuration files: +# 'review_ecs' and 'production_ecs' are two temporary names given to the jobs below. +# +# Should this template be included in your CI configuration, the upcoming name changes could +# then result in potentially breaking your future pipelines. +# +# More about including CI templates: https://docs.gitlab.com/ee/ci/yaml/#includetemplate + .deploy_to_ecs: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' script: @@ -15,7 +25,9 @@ review_ecs: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' production_ecs: extends: .deploy_to_ecs @@ -27,4 +39,6 @@ production_ecs: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 54a29b04d39..316647b5921 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.15.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.20.0" environment: name: production variables: @@ -19,12 +19,17 @@ apply: CROSSPLANE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/crossplane/values.yaml FLUENTD_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/fluentd/values.yaml KNATIVE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/knative/values.yaml + POSTHOG_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/posthog/values.yaml + FALCO_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/falco/values.yaml + APPARMOR_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/apparmor/values.yaml script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: refs: - master artifacts: + reports: + cluster_applications: gl-cluster-applications.json when: on_failure paths: - tiller.log diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index a25dc38e4e7..f35470367cc 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -20,4 +20,4 @@ image: "rust:latest" test:cargo: script: - rustc --version && cargo --version # Print version info for debugging - - cargo test --all --verbose + - cargo test --workspace --verbose diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 616966b4f04..fa8ccb7cf93 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -13,6 +13,7 @@ variables: DS_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX" DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" + DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 2 DS_DISABLE_DIND: "true" @@ -125,6 +126,7 @@ gemnasium-maven-dependency_scanning: $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ exists: - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' - '{build.sbt,*/build.sbt,*/*/build.sbt}' - '{pom.xml,*/pom.xml,*/*/pom.xml}' diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index b86014c1ebc..b0c75b0aab0 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -19,6 +19,7 @@ license_scanning: entrypoint: [""] variables: LM_REPORT_FILE: gl-license-scanning-report.json + LM_REPORT_VERSION: '2.1' SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD allow_failure: true script: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 47f68118ee0..ec7b34d17b5 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -13,6 +13,7 @@ variables: SAST_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX" SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 SAST_DISABLE_DIND: "true" SCAN_KUBERNETES_MANIFESTS: "false" @@ -80,10 +81,9 @@ brakeman-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ exists: - - '**/*.rb' + - 'config/routes.rb' eslint-sast: extends: .sast-analyzer @@ -149,7 +149,7 @@ nodejs-scan-sast: $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ exists: - - '**/*.js' + - 'package.json' phpcs-security-audit-sast: extends: .sast-analyzer @@ -213,8 +213,7 @@ sobelow-sast: $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ exists: - - '**/*.ex' - - '**/*.exs' + - 'mix.exs' spotbugs-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml new file mode 100644 index 00000000000..e18f89cadd7 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -0,0 +1,24 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://gitlab.com/gitlab-org/security-products/secret_detection#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECRETS_ANALYZER_VERSION: "3" + +secret_detection: + stage: test + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" + services: [] + rules: + - if: $SECRET_DETECTION_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bsecret_detection\b/ + when: on_success + artifacts: + reports: + secret_detection: gl-secret-detection-report.json + script: + - /analyzer run diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index a0832718214..377c72e8031 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -40,7 +40,6 @@ plan: - terraform plan -out=$PLAN - "terraform show --json $PLAN | convert_report > $JSON_PLAN_FILE" artifacts: - name: plan paths: - $PLAN reports: diff --git a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml new file mode 100644 index 00000000000..77a1b57d92f --- /dev/null +++ b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml @@ -0,0 +1,17 @@ +rspec-rails-modified-path-specs: + stage: .pre + rules: + - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train" + changes: ["**/*.rb"] + script: + - gem install test_file_finder + - spec_files=$(tff $(git diff --name-only "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA..$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA")) + - | + if [ -n "$spec_files" ] + then + bundle install + bundle exec rspec -- $spec_files + else + echo "No relevant spec files found by tff" + exit 0 + fi diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 4e83826b249..f76aacc2d19 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -147,7 +147,7 @@ module Gitlab raise AlreadyArchivedError, 'Could not write to the archived trace' elsif current_path File.open(current_path, mode) - elsif Feature.enabled?('ci_enable_live_trace', job.project) + elsif Feature.enabled?(:ci_enable_live_trace, job.project) Gitlab::Ci::Trace::ChunkedIO.new(job) else File.open(ensure_path, mode) diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 5816ac3bc54..6a9b7b2fc85 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -88,7 +88,7 @@ module Gitlab end def release(job) - job[:release] if Feature.enabled?(:ci_release_generation, default_enabled: false) + job[:release] if Gitlab::Ci::Features.release_generation_enabled? end def stage_builds_attributes(stage) |