diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 16:18:24 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 16:18:24 +0300 |
commit | 0653e08efd039a5905f3fa4f6e9cef9f5d2f799c (patch) | |
tree | 4dcc884cf6d81db44adae4aa99f8ec1233a41f55 /app/models/ci | |
parent | 744144d28e3e7fddc117924fef88de5d9674fe4c (diff) |
Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42
Diffstat (limited to 'app/models/ci')
-rw-r--r-- | app/models/ci/application_record.rb | 1 | ||||
-rw-r--r-- | app/models/ci/bridge.rb | 18 | ||||
-rw-r--r-- | app/models/ci/build.rb | 43 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/fog.rb | 10 | ||||
-rw-r--r-- | app/models/ci/build_trace_metadata.rb | 45 | ||||
-rw-r--r-- | app/models/ci/job_artifact.rb | 15 | ||||
-rw-r--r-- | app/models/ci/pending_build.rb | 82 | ||||
-rw-r--r-- | app/models/ci/pipeline.rb | 29 | ||||
-rw-r--r-- | app/models/ci/pipeline_variable.rb | 2 | ||||
-rw-r--r-- | app/models/ci/runner.rb | 40 | ||||
-rw-r--r-- | app/models/ci/sources/pipeline.rb | 3 |
11 files changed, 189 insertions, 99 deletions
diff --git a/app/models/ci/application_record.rb b/app/models/ci/application_record.rb index 9d4a8f0648e..913e7a62c66 100644 --- a/app/models/ci/application_record.rb +++ b/app/models/ci/application_record.rb @@ -2,6 +2,7 @@ module Ci class ApplicationRecord < ::ApplicationRecord + self.gitlab_schema = :gitlab_ci self.abstract_class = true def self.table_name_prefix diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 577bca282ef..97fb8233d34 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -28,10 +28,10 @@ module Ci state_machine :status do after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge| - next unless bridge.downstream_project + next unless bridge.triggers_downstream_pipeline? bridge.run_after_commit do - bridge.schedule_downstream_pipeline! + ::Ci::CreateCrossProjectPipelineWorker.perform_async(bridge.id) end end @@ -64,12 +64,6 @@ module Ci ) end - def schedule_downstream_pipeline! - raise InvalidBridgeTypeError unless downstream_project - - ::Ci::CreateCrossProjectPipelineWorker.perform_async(self.id) - end - def inherit_status_from_downstream!(pipeline) case pipeline.status when 'success' @@ -112,10 +106,18 @@ module Ci pipeline if triggers_child_pipeline? end + def triggers_downstream_pipeline? + triggers_child_pipeline? || triggers_cross_project_pipeline? + end + def triggers_child_pipeline? yaml_for_downstream.present? end + def triggers_cross_project_pipeline? + downstream_project_path.present? + end + def tags [:bridge] end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1ca291a659b..e2e24247679 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -90,6 +90,10 @@ module Ci end end + def persisted_environment=(environment) + strong_memoize(:persisted_environment) { environment } + end + serialize :options # rubocop:disable Cop/ActiveRecordSerialize serialize :yaml_variables, Gitlab::Serializer::Ci::Variables # rubocop:disable Cop/ActiveRecordSerialize @@ -166,8 +170,6 @@ module Ci scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) } scope :finished_before, -> (date) { finished.where('finished_at < ?', date) } - scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") } - scope :with_secure_reports_from_config_options, -> (job_types) do joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) end @@ -306,7 +308,9 @@ module Ci end after_transition pending: :running do |build| - build.deployment&.run + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment&.run + end build.run_after_commit do build.pipeline.persistent_ref.create @@ -328,7 +332,9 @@ module Ci end after_transition any => [:success] do |build| - build.deployment&.succeed + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment&.succeed + end build.run_after_commit do BuildSuccessWorker.perform_async(id) @@ -341,7 +347,9 @@ module Ci next unless build.deployment begin - build.deployment.drop! + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment.drop! + end rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, build_id: build.id) end @@ -362,10 +370,12 @@ module Ci end after_transition any => [:skipped, :canceled] do |build, transition| - if transition.to_name == :skipped - build.deployment&.skip - else - build.deployment&.cancel + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + if transition.to_name == :skipped + build.deployment&.skip + else + build.deployment&.cancel + end end end end @@ -712,6 +722,10 @@ module Ci update_column(:trace, nil) end + def ensure_trace_metadata! + Ci::BuildTraceMetadata.find_or_upsert_for!(id) + end + def artifacts_expose_as options.dig(:artifacts, :expose_as) end @@ -748,7 +762,9 @@ module Ci def any_runners_available? cache_for_available_runners do - project.active_runners.exists? + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do + project.active_runners.exists? + end end end @@ -1013,9 +1029,10 @@ module Ci # Consider this object to have a structural integrity problems def doom! - update_columns( - status: :failed, - failure_reason: :data_integrity_failure) + transaction do + update_columns(status: :failed, failure_reason: :data_integrity_failure) + all_queuing_entries.delete_all + end end def degradation_threshold diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb index 3bfac2b33c0..1cae2279434 100644 --- a/app/models/ci/build_trace_chunks/fog.rb +++ b/app/models/ci/build_trace_chunks/fog.rb @@ -80,12 +80,10 @@ module Ci private def append_strings(old_data, new_data) - if Feature.enabled?(:ci_job_trace_force_encode, default_enabled: :yaml) - # When object storage is in use, old_data may be retrieved in UTF-8. - old_data = old_data.force_encoding(Encoding::ASCII_8BIT) - # new_data should already be in ASCII-8BIT, but just in case it isn't, do this. - new_data = new_data.force_encoding(Encoding::ASCII_8BIT) - end + # When object storage is in use, old_data may be retrieved in UTF-8. + old_data = old_data.force_encoding(Encoding::ASCII_8BIT) + # new_data should already be in ASCII-8BIT, but just in case it isn't, do this. + new_data = new_data.force_encoding(Encoding::ASCII_8BIT) old_data + new_data end diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb index 05bdb3d8b7b..901b84ceec6 100644 --- a/app/models/ci/build_trace_metadata.rb +++ b/app/models/ci/build_trace_metadata.rb @@ -2,6 +2,7 @@ module Ci class BuildTraceMetadata < Ci::ApplicationRecord + MAX_ATTEMPTS = 5 self.table_name = 'ci_build_trace_metadata' self.primary_key = :build_id @@ -9,5 +10,49 @@ module Ci belongs_to :trace_artifact, class_name: 'Ci::JobArtifact' validates :build, presence: true + validates :archival_attempts, presence: true + + def self.find_or_upsert_for!(build_id) + record = find_by(build_id: build_id) + return record if record + + upsert({ build_id: build_id }, unique_by: :build_id) + find_by!(build_id: build_id) + end + + # The job is retried around 5 times during the 7 days retention period for + # trace chunks as defined in `Ci::BuildTraceChunks::RedisBase::CHUNK_REDIS_TTL` + def can_attempt_archival_now? + return false unless archival_attempts_available? + return true unless last_archival_attempt_at + + last_archival_attempt_at + backoff < Time.current + end + + def archival_attempts_available? + archival_attempts <= MAX_ATTEMPTS + end + + def increment_archival_attempts! + increment!(:archival_attempts, touch: :last_archival_attempt_at) + end + + def track_archival!(trace_artifact_id) + update!(trace_artifact_id: trace_artifact_id, archived_at: Time.current) + end + + def archival_attempts_message + if archival_attempts_available? + 'The job can not be archived right now.' + else + 'The job is out of archival attempts.' + end + end + + private + + def backoff + ::Gitlab::Ci::Trace::Backoff.new(archival_attempts).value_with_jitter + end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 1f0da4345f2..ad3e867f9d5 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -10,6 +10,9 @@ module Ci include Artifactable include FileStoreMounter include EachBatch + include IgnorableColumns + + ignore_columns %i[id_convert_to_bigint job_id_convert_to_bigint], remove_with: '14.5', remove_after: '2021-11-22' TEST_REPORT_FILE_TYPES = %w[junit].freeze COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze @@ -182,7 +185,6 @@ module Ci scope :order_expired_desc, -> { order(expire_at: :desc) } scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) } - scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') } scope :for_project, ->(project) { where(project_id: project) } scope :created_in_time_range, ->(from: nil, to: nil) { where(created_at: from..to) } @@ -232,6 +234,17 @@ module Ci hashed_path: 2 } + # `locked` will be populated from the source of truth on Ci::Pipeline + # in order to clean up expired job artifacts in a performant way. + # The values should be the same as `Ci::Pipeline.lockeds` with the + # additional value of `unknown` to indicate rows that have not + # yet been populated from the parent Ci::Pipeline + enum locked: { + unlocked: 0, + artifacts_locked: 1, + unknown: 2 + }, _prefix: :artifact + def validate_file_format! unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym errors.add(:base, _('Invalid file format with specified file type')) diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index 7cf3a387516..ccad6290fac 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -2,6 +2,8 @@ module Ci class PendingBuild < Ci::ApplicationRecord + include EachBatch + belongs_to :project belongs_to :build, class_name: 'Ci::Build' belongs_to :namespace, inverse_of: :pending_builds, class_name: 'Namespace' @@ -11,52 +13,62 @@ module Ci scope :ref_protected, -> { where(protected: true) } scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) } scope :with_instance_runners, -> { where(instance_runners_enabled: true) } + scope :for_tags, ->(tag_ids) do + if tag_ids.present? + where('ci_pending_builds.tag_ids <@ ARRAY[?]::int[]', Array.wrap(tag_ids)) + else + where("ci_pending_builds.tag_ids = '{}'") + end + end - def self.upsert_from_build!(build) - entry = self.new(args_from_build(build)) + class << self + def upsert_from_build!(build) + entry = self.new(args_from_build(build)) - entry.validate! + entry.validate! - self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) - end + self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) + end - def self.args_from_build(build) - args = { - build: build, - project: build.project, - protected: build.protected?, - namespace: build.project.namespace - } + private + + def args_from_build(build) + project = build.project + + args = { + build: build, + project: project, + protected: build.protected?, + namespace: project.namespace + } + + if Feature.enabled?(:ci_pending_builds_maintain_tags_data, type: :development, default_enabled: :yaml) + args.store(:tag_ids, build.tags_ids) + end + + if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) + args.store(:instance_runners_enabled, shared_runners_enabled?(project)) + end + + if Feature.enabled?(:ci_pending_builds_maintain_namespace_traversal_ids, type: :development, default_enabled: :yaml) + args.store(:namespace_traversal_ids, project.namespace.traversal_ids) if group_runners_enabled?(project) + end - if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) - args.merge(instance_runners_enabled: shareable?(build)) - else args end - end - private_class_method :args_from_build - - def self.shareable?(build) - shared_runner_enabled?(build) && - builds_access_level?(build) && - project_not_removed?(build) - end - private_class_method :shareable? - def self.shared_runner_enabled?(build) - build.project.shared_runners.exists? - end - private_class_method :shared_runner_enabled? + def shared_runners_enabled?(project) + builds_enabled?(project) && project.shared_runners_enabled? + end - def self.project_not_removed?(build) - !build.project.pending_delete? - end - private_class_method :project_not_removed? + def group_runners_enabled?(project) + builds_enabled?(project) && project.group_runners_enabled? + end - def self.builds_access_level?(build) - build.project.project_feature.builds_access_level.nil? || build.project.project_feature.builds_access_level > 0 + def builds_enabled?(project) + project.builds_enabled? && !project.pending_delete? + end end - private_class_method :builds_access_level? end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 70e67953e31..1a0cec3c935 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -66,6 +66,7 @@ module Ci has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline has_many :bridges, class_name: 'Ci::Bridge', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline + has_many :generic_commit_statuses, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'GenericCommitStatus' has_many :job_artifacts, through: :builds has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' @@ -307,6 +308,7 @@ module Ci scope :ci_and_parent_sources, -> { where(source: Enums::Ci::Pipeline.ci_and_parent_sources.values) } scope :for_user, -> (user) { where(user: user) } scope :for_sha, -> (sha) { where(sha: sha) } + scope :where_not_sha, -> (sha) { where.not(sha: sha) } scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) } scope :for_sha_or_source_sha, -> (sha) { for_sha(sha).or(for_source_sha(sha)) } scope :for_ref, -> (ref) { where(ref: ref) } @@ -317,7 +319,6 @@ module Ci scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) } scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } - scope :eager_load_project, -> { eager_load(project: [:route, { namespace: :route }]) } scope :with_pipeline_source, -> (source) { where(source: source)} scope :outside_pipeline_family, ->(pipeline) do @@ -588,13 +589,11 @@ module Ci end def cancel_running(retries: 1) - commit_status_relations = [:project, :pipeline] - ci_build_relations = [:deployment, :taggings] + preloaded_relations = [:project, :pipeline, :deployment, :taggings] retry_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables| cancelables.find_in_batches do |batch| - ActiveRecord::Associations::Preloader.new.preload(batch, commit_status_relations) - ActiveRecord::Associations::Preloader.new.preload(batch.select { |job| job.is_a?(Ci::Build) }, ci_build_relations) + Preloaders::CommitStatusPreloader.new(batch).execute(preloaded_relations) batch.each do |job| yield(job) if block_given? @@ -1108,7 +1107,7 @@ module Ci merge_request.modified_paths elsif branch_updated? push_details.modified_paths - elsif external_pull_request? && ::Feature.enabled?(:ci_modified_paths_of_external_prs, project, default_enabled: :yaml) + elsif external_pull_request? external_pull_request.modified_paths end end @@ -1220,24 +1219,12 @@ module Ci self.ci_ref = Ci::Ref.ensure_for(self) end - # We need `base_and_ancestors` in a specific order to "break" when needed. - # If we use `find_each`, then the order is broken. - # rubocop:disable Rails/FindEach def reset_source_bridge!(current_user) - if ::Feature.enabled?(:ci_reset_bridge_with_subsequent_jobs, project, default_enabled: :yaml) - return unless bridge_waiting? + return unless bridge_waiting? - source_bridge.pending! - Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass - else - self_and_upstreams.includes(:source_bridge).each do |pipeline| - break unless pipeline.bridge_waiting? - - pipeline.source_bridge.pending! - end - end + source_bridge.pending! + Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass end - # rubocop:enable Rails/FindEach # EE-only def merge_train_pipeline? diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index a0e8886414b..3dca77af051 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -8,7 +8,7 @@ module Ci alias_attribute :secret_value, :value - validates :key, uniqueness: { scope: :pipeline_id } + validates :key, presence: true def hook_attrs { key: key, value: value } diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 432c3a408a9..4aa232ad26b 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -208,16 +208,18 @@ module Ci Arel.sql("(#{arel_tag_names_array.to_sql})") ] - group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| - Gitlab::Ci::Matching::RunnerMatcher.new({ - runner_ids: values[0], - runner_type: values[1], - public_projects_minutes_cost_factor: values[2], - private_projects_minutes_cost_factor: values[3], - run_untagged: values[4], - access_level: values[5], - tag_list: values[6] - }) + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339621') do + group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| + Gitlab::Ci::Matching::RunnerMatcher.new({ + runner_ids: values[0], + runner_type: values[1], + public_projects_minutes_cost_factor: values[2], + private_projects_minutes_cost_factor: values[3], + run_untagged: values[4], + access_level: values[5], + tag_list: values[6] + }) + end end end @@ -385,6 +387,12 @@ module Ci read_attribute(:contacted_at) end + def namespace_ids + strong_memoize(:namespace_ids) do + runner_namespaces.pluck(:namespace_id).compact + end + end + private def cleanup_runner_queue @@ -420,14 +428,18 @@ module Ci end def no_projects - if projects.any? - errors.add(:runner, 'cannot have projects assigned') + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do + if projects.any? + errors.add(:runner, 'cannot have projects assigned') + end end end def no_groups - if groups.any? - errors.add(:runner, 'cannot have groups assigned') + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do + if groups.any? + errors.add(:runner, 'cannot have groups assigned') + end end end diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index f78caf710a6..95842d944f9 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -4,6 +4,9 @@ module Ci module Sources class Pipeline < Ci::ApplicationRecord include Ci::NamespacedModelName + include IgnorableColumns + + ignore_columns 'source_job_id_convert_to_bigint', remove_with: '14.5', remove_after: '2021-11-22' self.table_name = "ci_sources_pipelines" |