diff options
Diffstat (limited to 'lib/gitlab/ci/pipeline')
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/build.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/command.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/config/content.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/config/content/parameter.rb | 30 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/config/content/source.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/config/process.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/create.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/helpers.rb | 9 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/metrics.rb | 35 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/validate/abilities.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/validate/repository.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/metrics.rb | 42 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/preloader.rb | 18 |
13 files changed, 125 insertions, 46 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 9662209f88e..4190c40eb66 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -20,7 +20,11 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - variables_attributes: Array(@command.variables_attributes) + variables_attributes: Array(@command.variables_attributes), + # This should be removed and set on the database column default + # level when the keep_latest_artifacts_for_ref feature flag is + # removed. + locked: ::Gitlab::Ci::Features.keep_latest_artifacts_for_ref_enabled?(@command.project) ? :artifacts_locked : :unlocked ) end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 8118e7b2487..74b28b181bc 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, + :chat_data, :allow_mirror_update, :bridge, :content, # These attributes are set by Chains during processing: :config_content, :config_processor, :stage_seeds ) do @@ -78,7 +78,7 @@ module Gitlab end def metrics - @metrics ||= Chain::Metrics.new + @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new end def observe_creation_duration(duration) @@ -90,6 +90,10 @@ module Gitlab metrics.pipeline_size_histogram .observe({ source: pipeline.source.to_s }, pipeline.total_size) end + + def dangling_build? + %i[ondemand_dast_scan webide].include?(source) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index 2008010b523..5314fd471c3 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -9,6 +9,7 @@ module Gitlab include Chain::Helpers SOURCES = [ + Gitlab::Ci::Pipeline::Chain::Config::Content::Parameter, Gitlab::Ci::Pipeline::Chain::Config::Content::Bridge, Gitlab::Ci::Pipeline::Chain::Config::Content::Repository, Gitlab::Ci::Pipeline::Chain::Config::Content::ExternalProject, diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb new file mode 100644 index 00000000000..3dd216b33d1 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Config + class Content + class Parameter < Source + UnsupportedSourceError = Class.new(StandardError) + + def content + strong_memoize(:content) do + next unless command.content.present? + raise UnsupportedSourceError, "#{command.source} not a dangling build" unless command.dangling_build? + + command.content + end + end + + def source + :parameter_source + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb index 3389187473b..8bc172f93d3 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/source.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/source.rb @@ -11,6 +11,8 @@ module Gitlab DEFAULT_YAML_FILE = '.gitlab-ci.yml' + attr_reader :command + def initialize(pipeline, command) @pipeline = pipeline @command = command diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 1e47be21b93..2cfcb295407 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -19,7 +19,11 @@ module Gitlab parent_pipeline: parent_pipeline } ) + + add_warnings_to_pipeline(@command.config_processor.warnings) rescue Gitlab::Ci::YamlProcessor::ValidationError => ex + add_warnings_to_pipeline(ex.warnings) + error(ex.message, config_error: true) rescue => ex Gitlab::ErrorTracking.track_exception(ex, @@ -34,6 +38,14 @@ module Gitlab def break? @pipeline.errors.any? || @pipeline.persisted? end + + private + + def add_warnings_to_pipeline(warnings) + return unless warnings.present? + + warnings.each { |message| warning(message) } + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index aa627bdb009..34649fe16f3 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -8,7 +8,9 @@ module Gitlab include Chain::Helpers def perform! - pipeline.save! + BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do + pipeline.save! + end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 982ecc0ff51..aba7dab508d 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -11,9 +11,18 @@ module Gitlab pipeline.yaml_errors = message end + pipeline.add_error_message(message) pipeline.drop!(drop_reason) if drop_reason + + # TODO: consider not to rely on AR errors directly as they can be + # polluted with other unrelated errors (e.g. state machine) + # https://gitlab.com/gitlab-org/gitlab/-/issues/220823 pipeline.errors.add(:base, message) end + + def warning(message) + pipeline.add_warning_message(message) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb deleted file mode 100644 index 980ab2de9b0..00000000000 --- a/lib/gitlab/ci/pipeline/chain/metrics.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - class Metrics - include Gitlab::Utils::StrongMemoize - - def pipeline_creation_duration_histogram - strong_memoize(:pipeline_creation_duration_histogram) do - name = :gitlab_ci_pipeline_creation_duration_seconds - comment = 'Pipeline creation duration' - labels = {} - buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] - - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end - end - - def pipeline_size_histogram - strong_memoize(:pipeline_size_histogram) do - name = :gitlab_ci_pipeline_size_builds - comment = 'Pipeline size' - labels = { source: nil } - buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] - - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index a30b6c6ef0e..769d0dffd0b 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -19,7 +19,7 @@ module Gitlab end unless allowed_to_write_ref? - return error("Insufficient permissions for protected ref '#{command.ref}'") + error("Insufficient permissions for protected ref '#{command.ref}'") end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb index 8f5445850d7..7977ce90443 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -18,7 +18,7 @@ module Gitlab end if @command.ambiguous_ref? - return error('Ref is ambiguous') + error('Ref is ambiguous') end end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb new file mode 100644 index 00000000000..649da745eea --- /dev/null +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + class Metrics + include Gitlab::Utils::StrongMemoize + + def pipeline_creation_duration_histogram + strong_memoize(:pipeline_creation_duration_histogram) do + name = :gitlab_ci_pipeline_creation_duration_seconds + comment = 'Pipeline creation duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def pipeline_size_histogram + strong_memoize(:pipeline_size_histogram) do + name = :gitlab_ci_pipeline_size_builds + comment = 'Pipeline size' + labels = { source: nil } + buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def pipeline_processing_events_counter + strong_memoize(:pipeline_processing_events_counter) do + name = :gitlab_ci_pipeline_processing_events_total + comment = 'Total amount of pipeline processing events' + + Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/preloader.rb b/lib/gitlab/ci/pipeline/preloader.rb index db0a1ea4dab..7befc126ca9 100644 --- a/lib/gitlab/ci/pipeline/preloader.rb +++ b/lib/gitlab/ci/pipeline/preloader.rb @@ -17,6 +17,7 @@ module Gitlab pipelines.each do |pipeline| self.new(pipeline).tap do |preloader| preloader.preload_commit_authors + preloader.preload_ref_commits preloader.preload_pipeline_warnings preloader.preload_stages_warnings end @@ -27,12 +28,19 @@ module Gitlab @pipeline = pipeline end + # This also preloads the author of every commit. We're using "lazy_author" + # here since "author" immediately loads the data on the first call. def preload_commit_authors - # This also preloads the author of every commit. We're using "lazy_author" - # here since "author" immediately loads the data on the first call. @pipeline.commit.try(:lazy_author) end + # This preloads latest commits for given refs and therefore makes it + # much less expensive to check if a pipeline is a latest one for + # given branch. + def preload_ref_commits + @pipeline.lazy_ref_commit + end + def preload_pipeline_warnings # This preloads the number of warnings for every pipeline, ensuring # that Ci::Pipeline#has_warnings? doesn't execute any additional @@ -40,10 +48,10 @@ module Gitlab @pipeline.number_of_warnings end + # This preloads the number of warnings for every stage, ensuring + # that Ci::Stage#has_warnings? doesn't execute any additional + # queries. def preload_stages_warnings - # This preloads the number of warnings for every stage, ensuring - # that Ci::Stage#has_warnings? doesn't execute any additional - # queries. @pipeline.stages.each { |stage| stage.number_of_warnings } end end |