diff options
Diffstat (limited to 'lib/gitlab/ci/pipeline/chain')
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/base.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/build.rb | 4 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/command.rb | 13 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/config/process.rb | 25 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/create.rb | 43 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/create_deployments.rb | 44 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/ensure_environments.rb | 36 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb | 34 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/seed.rb | 29 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/sequence.rb | 14 | ||||
| -rw-r--r-- | lib/gitlab/ci/pipeline/chain/validate/external.rb | 10 |
11 files changed, 226 insertions, 28 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index 9b494f3a7ec..28567437719 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -7,7 +7,7 @@ module Gitlab class Base attr_reader :pipeline, :command, :config - delegate :project, :current_user, :parent_pipeline, to: :command + delegate :project, :current_user, :parent_pipeline, :logger, to: :command def initialize(pipeline, command) @pipeline = pipeline diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 6feb693221b..bbdc6b65b96 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -21,6 +21,10 @@ module Gitlab merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, locked: @command.project.default_pipeline_lock) + + # Initialize the feature flag at the beginning of the pipeline creation process + # so that the flag references in the latter chains return the same value. + @pipeline.create_deployment_in_separate_transaction? end def break? diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index beb8801096b..c466b8b36d0 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -11,7 +11,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, # These attributes are set by Chains during processing: :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed ) do @@ -88,7 +88,14 @@ module Gitlab @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end + def logger + self[:logger] ||= ::Gitlab::Ci::Pipeline::Logger.new(project: project) + end + def observe_step_duration(step_class, duration) + step = step_class.name.underscore.parameterize(separator: '_') + logger.observe("pipeline_step_#{step}_duration_s", duration) + if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml) metrics.pipeline_creation_step_duration_histogram .observe({ step: step_class.name }, duration.seconds) @@ -96,11 +103,15 @@ module Gitlab end def observe_creation_duration(duration) + logger.observe(:pipeline_creation_duration_s, duration) + metrics.pipeline_creation_duration_histogram .observe({}, duration.seconds) end def observe_pipeline_size(pipeline) + logger.observe(:pipeline_size_count, pipeline.total_size) + metrics.pipeline_size_histogram .observe({ source: pipeline.source.to_s }, pipeline.total_size) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index f3c937ddd28..64d1b001e3c 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -11,16 +11,21 @@ module Gitlab def perform! raise ArgumentError, 'missing config content' unless @command.config_content - result = ::Gitlab::Ci::YamlProcessor.new( - @command.config_content, { - project: project, - pipeline: @pipeline, - sha: @pipeline.sha, - source: @pipeline.source, - user: current_user, - parent_pipeline: parent_pipeline - } - ).execute + result = logger.instrument(:pipeline_config_process) do + processor = ::Gitlab::Ci::YamlProcessor.new( + @command.config_content, { + project: project, + pipeline: @pipeline, + sha: @pipeline.sha, + source: @pipeline.source, + user: current_user, + parent_pipeline: parent_pipeline, + logger: logger + } + ) + + processor.execute + end add_warnings_to_pipeline(result.warnings) diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 81ef3bb074d..15b0ff3c04d 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -6,10 +6,18 @@ module Gitlab module Chain class Create < Chain::Base include Chain::Helpers + include Gitlab::Utils::StrongMemoize def perform! - BulkInsertableAssociations.with_bulk_insert do - pipeline.save! + logger.instrument(:pipeline_save) do + BulkInsertableAssociations.with_bulk_insert do + tags = extract_tag_list_by_status + + pipeline.transaction do + pipeline.save! + CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags? + end + end end rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") @@ -18,6 +26,37 @@ module Gitlab def break? !pipeline.persisted? end + + private + + def statuses + strong_memoize(:statuses) do + pipeline.stages.flat_map(&:statuses) + end + end + + # We call `job.tag_list=` to assign tags to the jobs from the + # Chain::Seed step which uses the `@tag_list` instance variable to + # store them on the record. We remove them here because we want to + # bulk insert them, otherwise they would be inserted and assigned one + # by one with callbacks. We must use `remove_instance_variable` + # because having the instance variable defined would still run the callbacks + def extract_tag_list_by_status + return {} unless bulk_insert_tags? + + statuses.each.with_object({}) do |job, acc| + tag_list = job.clear_memoization(:tag_list) + next unless tag_list + + acc[job.name] = tag_list + end + end + + def bulk_insert_tags? + strong_memoize(:bulk_insert_tags) do + ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb new file mode 100644 index 00000000000..b92aa89d62d --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CreateDeployments < Chain::Base + DeploymentCreationError = Class.new(StandardError) + + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + create_deployments! + end + + def break? + false + end + + private + + def create_deployments! + pipeline.stages.map(&:statuses).flatten.map(&method(:create_deployment)) + end + + def create_deployment(build) + return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? + + deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment + .new(build, build.persisted_environment).to_resource + + return unless deployment + + deployment.deployable = build + deployment.save! + rescue ActiveRecord::RecordInvalid => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + DeploymentCreationError.new(e.message), build_id: build.id) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb new file mode 100644 index 00000000000..424e1d87fb4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EnsureEnvironments < Chain::Base + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_environment)) + end + + def break? + false + end + + private + + def ensure_environment(build) + return unless build.instance_of?(::Ci::Build) && build.has_environment? + + environment = ::Gitlab::Ci::Pipeline::Seed::Environment.new(build).to_resource + + if environment.persisted? + build.persisted_environment = environment + build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name }) + else + build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb new file mode 100644 index 00000000000..f4e5e6e467a --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class EnsureResourceGroups < Chain::Base + def perform! + return unless pipeline.create_deployment_in_separate_transaction? + + pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_resource_group)) + end + + def break? + false + end + + private + + def ensure_resource_group(processable) + return unless processable.is_a?(::Ci::Processable) + + key = processable.options.delete(:resource_group_key) + + resource_group = ::Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup + .new(processable, key).to_resource + + processable.resource_group = resource_group + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index ef7447fa83d..356eeb76908 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,8 +13,10 @@ module Gitlab raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result # Allocate next IID. This operation must be outside of transactions of pipeline creations. - pipeline.ensure_project_iid! - pipeline.ensure_ci_ref! + logger.instrument(:pipeline_allocate_seed_attributes) do + pipeline.ensure_project_iid! + pipeline.ensure_ci_ref! + end # Protect the pipeline. This is assigned in Populate instead of # Build to prevent erroring out on ambiguous refs. @@ -23,8 +25,12 @@ module Gitlab ## # Gather all runtime build/stage errors # - if pipeline_seed.errors - return error(pipeline_seed.errors.join("\n"), config_error: true) + seed_errors = logger.instrument(:pipeline_seed_evaluation) do + pipeline_seed.errors + end + + if seed_errors + return error(seed_errors.join("\n"), config_error: true) end @command.pipeline_seed = pipeline_seed @@ -38,8 +44,11 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do - stages_attributes = @command.yaml_processor_result.stages_attributes - Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + logger.instrument(:pipeline_seed_initialization) do + stages_attributes = @command.yaml_processor_result.stages_attributes + + Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + end end end @@ -48,9 +57,11 @@ module Gitlab end def root_variables - ::Gitlab::Ci::Variables::Helpers.merge_variables( - @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables - ) + logger.instrument(:pipeline_seed_merge_variables) do + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 845eb6c7a42..de147914850 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -9,30 +9,36 @@ module Gitlab @pipeline = pipeline @command = command @sequence = sequence - @start = Time.now + @start = current_monotonic_time end def build! @sequence.each do |step_class| - step_start = ::Gitlab::Metrics::System.monotonic_time + step_start = current_monotonic_time step = step_class.new(@pipeline, @command) step.perform! @command.observe_step_duration( step_class, - ::Gitlab::Metrics::System.monotonic_time - step_start + current_monotonic_time - step_start ) break if step.break? end - @command.observe_creation_duration(Time.now - @start) + @command.observe_creation_duration(current_monotonic_time - @start) @command.observe_pipeline_size(@pipeline) @command.observe_jobs_count_in_alive_pipelines @pipeline end + + private + + def current_monotonic_time + ::Gitlab::Metrics::System.monotonic_time + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 28ba1cd4d47..85bd5f0a7c1 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -113,7 +113,7 @@ module Gitlab name: build[:name], stage: build[:stage], image: build.dig(:options, :image, :name), - services: build.dig(:options, :services)&.map { |service| service[:name] }, + services: service_names(build), script: [ build.dig(:options, :before_script), build.dig(:options, :script), @@ -122,6 +122,14 @@ module Gitlab } end + def service_names(build) + services = build.dig(:options, :services) + + return unless services + + services.compact.map { |service| service[:name] } + end + def stages_attributes command.yaml_processor_result.stages_attributes end |