diff options
Diffstat (limited to 'app/models/ci')
-rw-r--r-- | app/models/ci/build.rb | 53 | ||||
-rw-r--r-- | app/models/ci/build_dependencies.rb | 3 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunk.rb | 4 | ||||
-rw-r--r-- | app/models/ci/build_trace_chunks/redis.rb | 2 | ||||
-rw-r--r-- | app/models/ci/group.rb | 7 | ||||
-rw-r--r-- | app/models/ci/job_artifact.rb | 8 | ||||
-rw-r--r-- | app/models/ci/pipeline.rb | 61 | ||||
-rw-r--r-- | app/models/ci/pipeline_artifact.rb | 2 | ||||
-rw-r--r-- | app/models/ci/pipeline_schedule.rb | 2 | ||||
-rw-r--r-- | app/models/ci/processable.rb | 8 | ||||
-rw-r--r-- | app/models/ci/runner.rb | 2 | ||||
-rw-r--r-- | app/models/ci/stage.rb | 11 | ||||
-rw-r--r-- | app/models/ci/test_case.rb | 35 | ||||
-rw-r--r-- | app/models/ci/test_case_failure.rb | 29 | ||||
-rw-r--r-- | app/models/ci/unit_test.rb | 46 | ||||
-rw-r--r-- | app/models/ci/unit_test_failure.rb | 29 |
16 files changed, 192 insertions, 110 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 824e35a6480..3d8e9f4c126 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -14,8 +14,6 @@ module Ci BuildArchivedError = Class.new(StandardError) - ignore_columns :artifacts_file, :artifacts_file_store, :artifacts_metadata, :artifacts_metadata_store, :artifacts_size, :commands, remove_after: '2019-12-15', remove_with: '12.7' - belongs_to :project, inverse_of: :builds belongs_to :runner belongs_to :trigger_request @@ -35,6 +33,7 @@ module Ci }.freeze DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD' + RUNNERS_STATUS_CACHE_EXPIRATION = 1.minute has_one :deployment, as: :deployable, class_name: 'Deployment' has_one :pending_state, class_name: 'Ci::BuildPendingState', inverse_of: :build @@ -75,7 +74,14 @@ module Ci return unless has_environment? strong_memoize(:persisted_environment) do - Environment.find_by(name: expanded_environment_name, project: project) + # This code path has caused N+1s in the past, since environments are only indirectly + # associated to builds and pipelines; see https://gitlab.com/gitlab-org/gitlab/-/issues/326445 + # We therefore batch-load them to prevent dormant N+1s until we found a proper solution. + BatchLoader.for(expanded_environment_name).batch(key: project_id) do |names, loader, args| + Environment.where(name: names, project: args[:key]).find_each do |environment| + loader.call(environment.name, environment) + end + end end end @@ -88,8 +94,7 @@ module Ci validates :ref, presence: true scope :not_interruptible, -> do - joins(:metadata).where('ci_builds_metadata.id NOT IN (?)', - Ci::BuildMetadata.scoped_build.with_interruptible.select(:id)) + joins(:metadata).where.not('ci_builds_metadata.id' => Ci::BuildMetadata.scoped_build.with_interruptible.select(:id)) end scope :unstarted, -> { where(runner_id: nil) } @@ -319,7 +324,7 @@ module Ci end end - before_transition any => [:failed] do |build| + after_transition any => [:failed] do |build| next unless build.project next unless build.deployment @@ -372,11 +377,11 @@ module Ci end def other_manual_actions - pipeline.manual_actions.where.not(name: name) + pipeline.manual_actions.reject { |action| action.name == self.name } end def other_scheduled_actions - pipeline.scheduled_actions.where.not(name: name) + pipeline.scheduled_actions.reject { |action| action.name == self.name } end def pages_generator? @@ -698,7 +703,23 @@ module Ci end def any_runners_online? - project.any_active_runners? { |runner| runner.match_build_if_online?(self) } + if Feature.enabled?(:runners_cached_states, project, default_enabled: :yaml) + cache_for_online_runners do + project.any_online_runners? { |runner| runner.match_build_if_online?(self) } + end + else + project.any_active_runners? { |runner| runner.match_build_if_online?(self) } + end + end + + def any_runners_available? + if Feature.enabled?(:runners_cached_states, project, default_enabled: :yaml) + cache_for_available_runners do + project.active_runners.exists? + end + else + project.any_active_runners? + end end def stuck? @@ -1103,6 +1124,20 @@ module Ci .to_a .include?(exit_code) end + + def cache_for_online_runners(&block) + Rails.cache.fetch( + ['has-online-runners', id], + expires_in: RUNNERS_STATUS_CACHE_EXPIRATION + ) { yield } + end + + def cache_for_available_runners(&block) + Rails.cache.fetch( + ['has-available-runners', project.id], + expires_in: RUNNERS_STATUS_CACHE_EXPIRATION + ) { yield } + end end end diff --git a/app/models/ci/build_dependencies.rb b/app/models/ci/build_dependencies.rb index b50ecf99439..8ae921f1416 100644 --- a/app/models/ci/build_dependencies.rb +++ b/app/models/ci/build_dependencies.rb @@ -21,8 +21,7 @@ module Ci deps = model_class.where(pipeline_id: processable.pipeline_id).latest deps = from_previous_stages(deps) deps = from_needs(deps) - deps = from_dependencies(deps) - deps + from_dependencies(deps) end # Dependencies from the same parent-pipeline hierarchy excluding diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index d4f9f78a1ac..7e03d709f24 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -30,9 +30,9 @@ module Ci fog: 3 }.freeze - STORE_TYPES = DATA_STORES.keys.map do |store| + STORE_TYPES = DATA_STORES.keys.to_h do |store| [store, "Ci::BuildTraceChunks::#{store.capitalize}".constantize] - end.to_h.freeze + end.freeze enum data_store: DATA_STORES diff --git a/app/models/ci/build_trace_chunks/redis.rb b/app/models/ci/build_trace_chunks/redis.rb index 58d50b39c11..003ec107895 100644 --- a/app/models/ci/build_trace_chunks/redis.rb +++ b/app/models/ci/build_trace_chunks/redis.rb @@ -4,7 +4,7 @@ module Ci module BuildTraceChunks class Redis CHUNK_REDIS_TTL = 1.week - LUA_APPEND_CHUNK = <<~EOS.freeze + LUA_APPEND_CHUNK = <<~EOS local key, new_data, offset = KEYS[1], ARGV[1], ARGV[2] local length = new_data:len() local expire = #{CHUNK_REDIS_TTL.seconds} diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb index 4ba09fd8152..47b91fcf2ce 100644 --- a/app/models/ci/group.rb +++ b/app/models/ci/group.rb @@ -22,6 +22,13 @@ module Ci @jobs = jobs end + def ==(other) + other.present? && other.is_a?(self.class) && + project == other.project && + stage == other.stage && + name == other.name + end + def status strong_memoize(:status) do status_struct.status diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index d5e88f2be5b..50e21a1c323 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -131,8 +131,6 @@ module Ci update_project_statistics project_statistics_name: :build_artifacts_size scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } - scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) } - scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } @@ -292,8 +290,12 @@ module Ci end end + def archived_trace_exists? + file&.file&.exists? + end + def self.archived_trace_exists_for?(job_id) - where(job_id: job_id).trace.take&.file&.file&.exists? + where(job_id: job_id).trace.take&.archived_trace_exists? end def self.max_artifact_size(type:, project:) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index b63ec0c8a97..c9ab69317e1 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -286,9 +286,11 @@ module Ci end after_transition any => [:failed] do |pipeline| - next unless pipeline.auto_devops_source? + pipeline.run_after_commit do + ::Gitlab::Ci::Pipeline::Metrics.pipeline_failure_reason_counter.increment(reason: pipeline.failure_reason) - pipeline.run_after_commit { AutoDevops::DisableWorker.perform_async(pipeline.id) } + AutoDevops::DisableWorker.perform_async(pipeline.id) if pipeline.auto_devops_source? + end end end @@ -309,6 +311,7 @@ module Ci scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) } scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } + scope :eager_load_project, -> { eager_load(project: [:route, { namespace: :route }]) } scope :outside_pipeline_family, ->(pipeline) do where.not(id: pipeline.same_family_pipeline_ids) @@ -393,26 +396,13 @@ module Ci # given we simply get the latest pipelines for the commits, regardless # of what refs the pipelines belong to. def self.latest_pipeline_per_commit(commits, ref = nil) - p1 = arel_table - p2 = arel_table.alias - - # This LEFT JOIN will filter out all but the newest row for every - # combination of (project_id, sha) or (project_id, sha, ref) if a ref is - # given. - cond = p1[:sha].eq(p2[:sha]) - .and(p1[:project_id].eq(p2[:project_id])) - .and(p1[:id].lt(p2[:id])) - - cond = cond.and(p1[:ref].eq(p2[:ref])) if ref - join = p1.join(p2, Arel::Nodes::OuterJoin).on(cond) + sql = select('DISTINCT ON (sha) *') + .where(sha: commits) + .order(:sha, id: :desc) - relation = where(sha: commits) - .where(p2[:id].eq(nil)) - .joins(join.join_sources) + sql = sql.where(ref: ref) if ref - relation = relation.where(ref: ref) if ref - - relation.each_with_object({}) do |pipeline, hash| + sql.each_with_object({}) do |pipeline, hash| hash[pipeline.sha] = pipeline end end @@ -445,6 +435,10 @@ module Ci @auto_devops_pipelines_completed_total ||= Gitlab::Metrics.counter(:auto_devops_pipelines_completed_total, 'Number of completed auto devops pipelines') end + def uses_needs? + builds.where(scheduling_type: :dag).any? + end + def stages_count statuses.select(:stage).distinct.count end @@ -510,6 +504,12 @@ module Ci end end + def git_author_full_text + strong_memoize(:git_author_full_text) do + commit.try(:author_full_text) + end + end + def git_commit_message strong_memoize(:git_commit_message) do commit.try(:message) @@ -573,10 +573,18 @@ module Ci end def cancel_running(retries: nil) - retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelable| - cancelable.find_each do |job| - yield(job) if block_given? - job.cancel + commit_status_relations = [:project, :pipeline] + ci_build_relations = [:deployment, :taggings] + + retry_optimistic_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables| + cancelables.find_in_batches do |batch| + ActiveRecord::Associations::Preloader.new.preload(batch, commit_status_relations) + ActiveRecord::Associations::Preloader.new.preload(batch.select { |job| job.is_a?(Ci::Build) }, ci_build_relations) + + batch.each do |job| + yield(job) if block_given? + job.cancel + end end end end @@ -664,7 +672,9 @@ module Ci end def has_kubernetes_active? - project.deployment_platform&.active? + strong_memoize(:has_kubernetes_active) do + project.deployment_platform&.active? + end end def freeze_period? @@ -822,6 +832,7 @@ module Ci variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!protected_ref?).to_s) variables.append(key: 'CI_COMMIT_TIMESTAMP', value: git_commit_timestamp.to_s) + variables.append(key: 'CI_COMMIT_AUTHOR', value: git_author_full_text.to_s) # legacy variables variables.append(key: 'CI_BUILD_REF', value: sha) diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb index f538a4cd808..9dfe4252e95 100644 --- a/app/models/ci/pipeline_artifact.rb +++ b/app/models/ci/pipeline_artifact.rb @@ -57,3 +57,5 @@ module Ci end end end + +Ci::PipelineArtifact.prepend_ee_mod diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 2fae077dd87..3c17246bc34 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -7,6 +7,7 @@ module Ci include StripAttribute include Schedulable include Limitable + include EachBatch self.limit_name = 'ci_pipeline_schedules' self.limit_scope = :project @@ -28,6 +29,7 @@ module Ci scope :active, -> { where(active: true) } scope :inactive, -> { where(active: false) } scope :preloaded, -> { preload(:owner, project: [:route]) } + scope :owned_by, ->(user) { where(owner: user) } accepts_nested_attributes_for :variables, allow_destroy: true diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 0ad1ed2fce8..3b61840805a 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -165,7 +165,13 @@ module Ci end def all_dependencies - dependencies.all + if Feature.enabled?(:preload_associations_jobs_request_api_endpoint, project, default_enabled: :yaml) + strong_memoize(:all_dependencies) do + dependencies.all + end + else + dependencies.all + end end private diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index d1a20bc93c3..05126853e0f 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -45,8 +45,6 @@ module Ci FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze MINUTES_COST_FACTOR_FIELDS = %i[public_projects_minutes_cost_factor private_projects_minutes_cost_factor].freeze - ignore_column :is_shared, remove_after: '2019-12-15', remove_with: '12.6' - has_many :builds has_many :runner_projects, inverse_of: :runner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 03a97355574..9dd75150ac7 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -14,11 +14,20 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id has_many :latest_statuses, -> { ordered.latest }, class_name: 'CommitStatus', foreign_key: :stage_id + has_many :retried_statuses, -> { ordered.retried }, class_name: 'CommitStatus', foreign_key: :stage_id has_many :processables, class_name: 'Ci::Processable', foreign_key: :stage_id has_many :builds, foreign_key: :stage_id has_many :bridges, foreign_key: :stage_id scope :ordered, -> { order(position: :asc) } + scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } + scope :by_name, ->(names) { where(name: names) } + scope :with_latest_and_retried_statuses, -> do + includes( + latest_statuses: [:pipeline, project: :namespace], + retried_statuses: [:pipeline, project: :namespace] + ) + end with_options unless: :importing? do validates :project, presence: true @@ -35,7 +44,7 @@ module Ci next if position.present? self.position = statuses.select(:stage_idx) - .where('stage_idx IS NOT NULL') + .where.not(stage_idx: nil) .group(:stage_idx) .order('COUNT(*) DESC') .first&.stage_idx.to_i diff --git a/app/models/ci/test_case.rb b/app/models/ci/test_case.rb deleted file mode 100644 index 19ecc177436..00000000000 --- a/app/models/ci/test_case.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Ci - class TestCase < ApplicationRecord - extend Gitlab::Ci::Model - - validates :project, :key_hash, presence: true - - has_many :test_case_failures, class_name: 'Ci::TestCaseFailure' - - belongs_to :project - - scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) } - - class << self - def find_or_create_by_batch(project, test_case_keys) - # Insert records first. Existing ones will be skipped. - insert_all(test_case_attrs(project, test_case_keys)) - - # Find all matching records now that we are sure they all are persisted. - by_project_and_keys(project, test_case_keys) - end - - private - - def test_case_attrs(project, test_case_keys) - # NOTE: Rails 6.1 will add support for insert_all on relation so that - # we will be able to do project.test_cases.insert_all. - test_case_keys.map do |hashed_key| - { project_id: project.id, key_hash: hashed_key } - end - end - end - end -end diff --git a/app/models/ci/test_case_failure.rb b/app/models/ci/test_case_failure.rb deleted file mode 100644 index 8867b954240..00000000000 --- a/app/models/ci/test_case_failure.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Ci - class TestCaseFailure < ApplicationRecord - extend Gitlab::Ci::Model - - REPORT_WINDOW = 14.days - - validates :test_case, :build, :failed_at, presence: true - - belongs_to :test_case, class_name: "Ci::TestCase", foreign_key: :test_case_id - belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id - - def self.recent_failures_count(project:, test_case_keys:, date_range: REPORT_WINDOW.ago..Time.current) - joins(:test_case) - .where( - ci_test_cases: { - project_id: project.id, - key_hash: test_case_keys - }, - ci_test_case_failures: { - failed_at: date_range - } - ) - .group(:key_hash) - .count('ci_test_case_failures.id') - end - end -end diff --git a/app/models/ci/unit_test.rb b/app/models/ci/unit_test.rb new file mode 100644 index 00000000000..81623b4f6ad --- /dev/null +++ b/app/models/ci/unit_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Ci + class UnitTest < ApplicationRecord + extend Gitlab::Ci::Model + + MAX_NAME_SIZE = 255 + MAX_SUITE_NAME_SIZE = 255 + + validates :project, :key_hash, :name, :suite_name, presence: true + + has_many :unit_test_failures, class_name: 'Ci::UnitTestFailure' + + belongs_to :project + + scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) } + + class << self + def find_or_create_by_batch(project, unit_test_attrs) + # Insert records first. Existing ones will be skipped. + insert_all(build_insert_attrs(project, unit_test_attrs)) + + # Find all matching records now that we are sure they all are persisted. + by_project_and_keys(project, gather_keys(unit_test_attrs)) + end + + private + + def build_insert_attrs(project, unit_test_attrs) + # NOTE: Rails 6.1 will add support for insert_all on relation so that + # we will be able to do project.test_cases.insert_all. + unit_test_attrs.map do |attrs| + attrs.merge( + project_id: project.id, + name: attrs[:name].truncate(MAX_NAME_SIZE), + suite_name: attrs[:suite_name].truncate(MAX_SUITE_NAME_SIZE) + ) + end + end + + def gather_keys(unit_test_attrs) + unit_test_attrs.map { |attrs| attrs[:key_hash] } + end + end + end +end diff --git a/app/models/ci/unit_test_failure.rb b/app/models/ci/unit_test_failure.rb new file mode 100644 index 00000000000..653a56bd2b3 --- /dev/null +++ b/app/models/ci/unit_test_failure.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Ci + class UnitTestFailure < ApplicationRecord + extend Gitlab::Ci::Model + + REPORT_WINDOW = 14.days + + validates :unit_test, :build, :failed_at, presence: true + + belongs_to :unit_test, class_name: "Ci::UnitTest", foreign_key: :unit_test_id + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id + + def self.recent_failures_count(project:, unit_test_keys:, date_range: REPORT_WINDOW.ago..Time.current) + joins(:unit_test) + .where( + ci_unit_tests: { + project_id: project.id, + key_hash: unit_test_keys + }, + ci_unit_test_failures: { + failed_at: date_range + } + ) + .group(:key_hash) + .count('ci_unit_test_failures.id') + end + end +end |