diff options
Diffstat (limited to 'lib/gitlab/ci')
19 files changed, 268 insertions, 69 deletions
diff --git a/lib/gitlab/ci/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb index 28863a0703b..78b51dbdaf0 100644 --- a/lib/gitlab/ci/badge/coverage/report.rb +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -15,7 +15,10 @@ module Gitlab::Ci @job = opts[:job] @customization = { key_width: opts[:key_width].to_i, - key_text: opts[:key_text] + key_text: opts[:key_text], + min_good: opts[:min_good].to_i, + min_acceptable: opts[:min_acceptable].to_i, + min_medium: opts[:min_medium].to_i } end diff --git a/lib/gitlab/ci/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index 96702420e9d..f12b4f2dbfb 100644 --- a/lib/gitlab/ci/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -16,12 +16,20 @@ module Gitlab::Ci low: '#e05d44', unknown: '#9f9f9f' }.freeze + COVERAGE_MAX = 100 + COVERAGE_MIN = 0 + MIN_GOOD_DEFAULT = 95 + MIN_ACCEPTABLE_DEFAULT = 90 + MIN_MEDIUM_DEFAULT = 75 def initialize(badge) @entity = badge.entity @status = badge.status @key_text = badge.customization.dig(:key_text) @key_width = badge.customization.dig(:key_width) + @min_good = badge.customization.dig(:min_good) + @min_acceptable = badge.customization.dig(:min_acceptable) + @min_medium = badge.customization.dig(:min_medium) end def value_text @@ -32,12 +40,36 @@ module Gitlab::Ci @status ? 54 : 58 end + def min_good_value + if @min_good && @min_good.between?(3, COVERAGE_MAX) + @min_good + else + MIN_GOOD_DEFAULT + end + end + + def min_acceptable_value + if @min_acceptable && @min_acceptable.between?(2, min_good_value - 1) + @min_acceptable + else + [MIN_ACCEPTABLE_DEFAULT, (min_good_value - 1)].min + end + end + + def min_medium_value + if @min_medium && @min_medium.between?(1, min_acceptable_value - 1) + @min_medium + else + [MIN_MEDIUM_DEFAULT, (min_acceptable_value - 1)].min + end + end + def value_color case @status - when 95..100 then STATUS_COLOR[:good] - when 90..95 then STATUS_COLOR[:acceptable] - when 75..90 then STATUS_COLOR[:medium] - when 0..75 then STATUS_COLOR[:low] + when min_good_value..COVERAGE_MAX then STATUS_COLOR[:good] + when min_acceptable_value..min_good_value then STATUS_COLOR[:acceptable] + when min_medium_value..min_acceptable_value then STATUS_COLOR[:medium] + when COVERAGE_MIN..min_medium_value then STATUS_COLOR[:low] else STATUS_COLOR[:unknown] end diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb index b98d1d7b330..6ab567dff7c 100644 --- a/lib/gitlab/ci/build/auto_retry.rb +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -9,7 +9,8 @@ class Gitlab::Ci::Build::AutoRetry RETRY_OVERRIDES = { ci_quota_exceeded: 0, - no_matching_runner: 0 + no_matching_runner: 0, + missing_dependency_failure: 0 }.freeze def initialize(build) diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 97e4922b2a1..95f1a842c50 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -58,9 +58,6 @@ module Gitlab end def verify_rules(location) - # Behaves like there is no `rules` - return location unless ::Feature.enabled?(:ci_include_rules, context.project, default_enabled: :yaml) - return unless Rules.new(location[:rules]).evaluate(context).pass? location diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 5a788427172..95470537de3 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -5,7 +5,13 @@ module Gitlab class Config module External class Rules + ALLOWED_KEYS = Entry::Include::Rules::Rule::ALLOWED_KEYS + + InvalidIncludeRulesError = Class.new(Mapper::Error) + def initialize(rule_hashes) + validate(rule_hashes) + @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) end @@ -19,6 +25,16 @@ module Gitlab @rule_list.find { |rule| rule.matches?(nil, context) } end + def validate(rule_hashes) + return unless rule_hashes.is_a?(Array) + + rule_hashes.each do |rule_hash| + next if (rule_hash.keys - ALLOWED_KEYS).empty? + + raise InvalidIncludeRulesError, "invalid include rule: #{rule_hash}" + end + end + Result = Struct.new(:result) do def pass? !!result diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 27bb7fdc05a..28ba1cd4d47 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -91,7 +91,8 @@ module Gitlab email: current_user.email, created_at: current_user.created_at&.iso8601, current_sign_in_ip: current_user.current_sign_in_ip, - last_sign_in_ip: current_user.last_sign_in_ip + last_sign_in_ip: current_user.last_sign_in_ip, + sign_in_count: current_user.sign_in_count }, pipeline: { sha: pipeline.sha, diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 28df9f5386c..321efa7854f 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -65,13 +65,6 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end - def self.legacy_update_jobs_counter - name = :ci_legacy_update_jobs_as_retried_total - comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' - - Gitlab::Metrics.counter(name, comment) - end - def self.pipeline_failure_reason_counter name = :gitlab_ci_pipeline_failure_reasons comment = 'Counter of pipeline failure reasons' @@ -92,14 +85,6 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end - - def self.gitlab_ci_difference_live_vs_actual_minutes - name = :gitlab_ci_difference_live_vs_actual_minutes - comment = 'Comparison between CI minutes consumption from live tracking vs actual consumption' - labels = {} - buckets = [-120.0, -60.0, -30.0, -10.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 10.0, 30.0, 60.0, 120.0] - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 934bf22d8ad..9ad5d6538b7 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -106,10 +106,15 @@ module Gitlab environment = Seed::Environment.new(build).to_resource - # If there is a validation error on environment creation, such as - # the name contains invalid character, the build falls back to a - # non-environment job. unless environment.persisted? + if Feature.enabled?(:surface_environment_creation_failure, build.project, default_enabled: :yaml) && + Feature.disabled?(:surface_environment_creation_failure_override, build.project) + return { status: :failed, failure_reason: :environment_creation_failure } + end + + # If there is a validation error on environment creation, such as + # the name contains invalid character, the build falls back to a + # non-environment job. Gitlab::ErrorTracking.track_exception( EnvironmentCreationFailure.new, project_id: build.project_id, diff --git a/lib/gitlab/ci/reports/security/flag.rb b/lib/gitlab/ci/reports/security/flag.rb index 7e6cc758864..8370dd60418 100644 --- a/lib/gitlab/ci/reports/security/flag.rb +++ b/lib/gitlab/ci/reports/security/flag.rb @@ -20,7 +20,7 @@ module Gitlab @description = description end - def to_hash + def to_h { flag_type: flag_type, origin: origin, diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb index 6cb2e0ddb33..4be4cf62e7b 100644 --- a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb +++ b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb @@ -80,6 +80,8 @@ module Gitlab matcher = FindingMatcher.new(head_findings) base_findings.each do |base_finding| + next if base_finding.requires_manual_resolution? + matched_head_finding = matcher.find_and_remove_match!(base_finding) @fixed_findings << base_finding if matched_head_finding.nil? diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index ee210e51232..b0f12ff7517 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -33,7 +33,8 @@ module Gitlab ci_quota_exceeded: 'no more CI minutes available', no_matching_runner: 'no matching runner available', trace_size_exceeded: 'log size limit exceeded', - builds_disabled: 'project builds are disabled' + builds_disabled: 'project builds are disabled', + environment_creation_failure: 'environment creation failure' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index e0627b85aba..65a58130962 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.14.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 2df985cfbb5..58f13746a1f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.14.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 917a28bb1ee..37a746a223c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -253,6 +253,7 @@ semgrep-sast: - '**/*.ts' - '**/*.tsx' - '**/*.c' + - '**/*.go' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index bfea437b8f1..64c784f43cb 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -11,7 +11,7 @@ publish: changes: - package.json script: - # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry + # If no .npmrc is included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry - | if [[ ! -f .npmrc ]]; then echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 72a94dcd412..25075cc8f90 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -25,7 +25,7 @@ module Gitlab delegate :old_trace, to: :job delegate :can_attempt_archival_now?, :increment_archival_attempts!, - :archival_attempts_message, to: :trace_metadata + :archival_attempts_message, :archival_attempts_available?, to: :trace_metadata def initialize(job) @job = job @@ -122,6 +122,10 @@ module Gitlab end end + def attempt_archive_cleanup! + destroy_any_orphan_trace_data! + end + def update_interval if being_watched? UPDATE_FREQUENCY_WHEN_BEING_WATCHED @@ -191,7 +195,10 @@ module Gitlab def unsafe_archive! raise ArchiveError, 'Job is not finished yet' unless job.complete? - unsafe_trace_conditionally_cleanup_before_retry! + already_archived?.tap do |archived| + destroy_any_orphan_trace_data! + raise AlreadyArchivedError, 'Could not archive again' if archived + end if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| @@ -214,16 +221,15 @@ module Gitlab def already_archived? # TODO check checksum to ensure archive completed successfully # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619 - trace_artifact.archived_trace_exists? + trace_artifact&.archived_trace_exists? end - def unsafe_trace_conditionally_cleanup_before_retry! + def destroy_any_orphan_trace_data! return unless trace_artifact if already_archived? # An archive already exists, so make sure to remove the trace chunks erase_trace_chunks! - raise AlreadyArchivedError, 'Could not archive again' else # An archive already exists, but its associated file does not, so remove it trace_artifact.destroy! @@ -236,35 +242,7 @@ module Gitlab end def archive_stream!(stream) - clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| - create_build_trace!(job, clone_path) - end - end - - def clone_file!(src_stream, temp_dir) - FileUtils.mkdir_p(temp_dir) - Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| - temp_path = File.join(dir_path, "job.log") - FileUtils.touch(temp_path) - size = IO.copy_stream(src_stream, temp_path) - raise ArchiveError, 'Failed to copy stream' unless size == src_stream.size - - yield(temp_path) - end - end - - def create_build_trace!(job, path) - File.open(path) do |stream| - # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration - # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307 - trace_artifact = job.create_job_artifacts_trace!( - project: job.project, - file_type: :trace, - file: stream, - file_sha256: self.class.hexdigest(path)) - - trace_metadata.track_archival!(trace_artifact.id) - end + ::Gitlab::Ci::Trace::Archive.new(job, trace_metadata).execute!(stream) end def trace_metadata @@ -314,7 +292,8 @@ module Gitlab def destroy_stream(build) if consistent_archived_trace?(build) - ::Gitlab::Database::LoadBalancing::Sticking + ::Ci::Build + .sticking .stick(LOAD_BALANCING_STICKING_NAMESPACE, build.id) end @@ -323,7 +302,8 @@ module Gitlab def read_trace_artifact(build) if consistent_archived_trace?(build) - ::Gitlab::Database::LoadBalancing::Sticking + ::Ci::Build + .sticking .unstick_or_continue_sticking(LOAD_BALANCING_STICKING_NAMESPACE, build.id) end diff --git a/lib/gitlab/ci/trace/archive.rb b/lib/gitlab/ci/trace/archive.rb new file mode 100644 index 00000000000..5047cf04562 --- /dev/null +++ b/lib/gitlab/ci/trace/archive.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + class Archive + include ::Gitlab::Utils::StrongMemoize + include Checksummable + + def initialize(job, trace_metadata, metrics = ::Gitlab::Ci::Trace::Metrics.new) + @job = job + @trace_metadata = trace_metadata + @metrics = metrics + end + + def execute!(stream) + clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| + md5_checksum = self.class.md5_hexdigest(clone_path) + sha256_checksum = self.class.sha256_hexdigest(clone_path) + + job.transaction do + self.trace_artifact = create_build_trace!(clone_path, sha256_checksum) + trace_metadata.track_archival!(trace_artifact.id, md5_checksum) + end + end + + validate_archived_trace + end + + private + + attr_reader :job, :trace_metadata, :metrics + attr_accessor :trace_artifact + + def clone_file!(src_stream, temp_dir) + FileUtils.mkdir_p(temp_dir) + Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| + temp_path = File.join(dir_path, "job.log") + FileUtils.touch(temp_path) + size = IO.copy_stream(src_stream, temp_path) + raise ::Gitlab::Ci::Trace::ArchiveError, 'Failed to copy stream' unless size == src_stream.size + + yield(temp_path) + end + end + + def create_build_trace!(path, file_sha256) + File.open(path) do |stream| + # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration + # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307 + job.create_job_artifacts_trace!( + project: job.project, + file_type: :trace, + file: stream, + file_sha256: file_sha256) + end + end + + def validate_archived_trace + return unless remote_checksum + + trace_metadata.update!(remote_checksum: remote_checksum) + + unless trace_metadata.remote_checksum_valid? + metrics.increment_error_counter(type: :archive_invalid_checksum) + end + end + + def remote_checksum + strong_memoize(:remote_checksum) do + ::Gitlab::Ci::Trace::RemoteChecksum.new(trace_artifact).md5_checksum + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index fcd70634630..174a5f184ff 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -21,6 +21,12 @@ module Gitlab :corrupted # malformed trace found after comparing CRC32 and size ].freeze + TRACE_ERROR_TYPES = [ + :chunks_invalid_size, # used to be :corrupted + :chunks_invalid_checksum, # used to be :invalid + :archive_invalid_checksum # malformed trace found into object store after comparing MD5 + ].freeze + def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) raise ArgumentError, "unknown trace operation: #{operation}" @@ -33,6 +39,14 @@ module Gitlab self.class.trace_bytes.increment({}, size.to_i) end + def increment_error_counter(type: :unknown) + unless TRACE_ERROR_TYPES.include?(type) + raise ArgumentError, "unknown error type: #{type}" + end + + self.class.trace_errors_counter.increment(type: type) + end + def observe_migration_duration(seconds) self.class.finalize_histogram.observe({}, seconds.to_f) end @@ -65,6 +79,15 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end end + + def self.trace_errors_counter + strong_memoize(:trace_errors_counter) do + name = :gitlab_ci_build_trace_errors_total + comment = 'Total amount of different error types on a build trace' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb new file mode 100644 index 00000000000..d57f3888ec0 --- /dev/null +++ b/lib/gitlab/ci/trace/remote_checksum.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # RemoteChecksum class is responsible for fetching the MD5 checksum of + # an uploaded build trace. + # + class RemoteChecksum + include Gitlab::Utils::StrongMemoize + + def initialize(trace_artifact) + @trace_artifact = trace_artifact + end + + def md5_checksum + strong_memoize(:md5_checksum) do + fetch_md5_checksum + end + end + + private + + attr_reader :trace_artifact + delegate :aws?, :google?, to: :object_store_config, prefix: :provider + + def fetch_md5_checksum + return unless Feature.enabled?(:ci_archived_build_trace_checksum, trace_artifact.project, default_enabled: :yaml) + return unless object_store_config.enabled? + return if trace_artifact.local_store? + + remote_checksum_value + end + + def remote_checksum_value + strong_memoize(:remote_checksum_value) do + if provider_google? + checksum_from_google + elsif provider_aws? + checksum_from_aws + end + end + end + + def object_store_config + strong_memoize(:object_store_config) do + trace_artifact.file.class.object_store_config + end + end + + def checksum_from_google + content_md5 = upload_attributes.fetch(:content_md5) + + Base64 + .decode64(content_md5) + .unpack1('H*') + end + + def checksum_from_aws + upload_attributes.fetch(:etag) + end + + # Carrierwave caches attributes for the local file and does not replace + # them with the ones from object store after the upload completes. + # We need to force it to fetch them directly from the object store. + def upload_attributes + strong_memoize(:upload_attributes) do + ::Ci::JobArtifact.find(trace_artifact.id).file.file.attributes + end + end + end + end + end +end |