diff options
Diffstat (limited to 'app/models/ci')
-rw-r--r-- | app/models/ci/build.rb | 25 | ||||
-rw-r--r-- | app/models/ci/catalog/resources/version.rb | 9 | ||||
-rw-r--r-- | app/models/ci/instance_variable.rb | 1 | ||||
-rw-r--r-- | app/models/ci/namespace_mirror.rb | 1 | ||||
-rw-r--r-- | app/models/ci/pipeline.rb | 11 | ||||
-rw-r--r-- | app/models/ci/pipeline_artifact.rb | 3 | ||||
-rw-r--r-- | app/models/ci/pipeline_chat_data.rb | 7 | ||||
-rw-r--r-- | app/models/ci/pipeline_config.rb | 4 | ||||
-rw-r--r-- | app/models/ci/pipeline_metadata.rb | 7 | ||||
-rw-r--r-- | app/models/ci/pipeline_variable.rb | 3 | ||||
-rw-r--r-- | app/models/ci/processable.rb | 4 | ||||
-rw-r--r-- | app/models/ci/project_mirror.rb | 2 | ||||
-rw-r--r-- | app/models/ci/runner.rb | 52 | ||||
-rw-r--r-- | app/models/ci/runner_manager.rb | 37 | ||||
-rw-r--r-- | app/models/ci/stage.rb | 3 |
15 files changed, 111 insertions, 58 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index e56f3d2536c..d4c70a294ff 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -27,6 +27,7 @@ module Ci foreign_key: :commit_id, partition_foreign_key: :partition_id, inverse_of: :builds + belongs_to :project_mirror, primary_key: :project_id, foreign_key: :project_id, inverse_of: :builds RUNNER_FEATURES = { upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? }, @@ -42,6 +43,8 @@ module Ci DEPLOYMENT_NAMES = %w[deploy release rollout].freeze + TOKEN_PREFIX = 'glcbt-' + has_one :pending_state, class_name: 'Ci::BuildPendingState', foreign_key: :build_id, inverse_of: :build has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id, inverse_of: :build has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id, inverse_of: :build @@ -98,6 +101,7 @@ module Ci delegate :harbor_integration, to: :project delegate :apple_app_store_integration, to: :project delegate :google_play_integration, to: :project + delegate :diffblue_cover_integration, to: :project delegate :trigger_short_token, to: :trigger_request, allow_nil: true delegate :ensure_persistent_ref, to: :pipeline delegate :enable_debug_trace!, to: :metadata @@ -188,6 +192,10 @@ module Ci # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123131 scope :with_runner_type, -> (runner_type) { joins(:runner).where(runner: { runner_type: runner_type }) } + scope :belonging_to_runner_manager, -> (runner_machine_id) { + joins(:runner_manager_build).where(p_ci_runner_machine_builds: { runner_machine_id: runner_machine_id }) + } + scope :with_secure_reports_from_config_options, -> (job_types) do joins(:metadata).where("#{Ci::BuildMetadata.quoted_table_name}.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) end @@ -204,7 +212,7 @@ module Ci add_authentication_token_field :token, encrypted: :required, - format_with_prefix: :partition_id_prefix_in_16_bit_encode + format_with_prefix: :prefix_and_partition_for_token after_save :stick_build_if_status_changed @@ -516,6 +524,7 @@ module Ci .concat(harbor_variables) .concat(apple_app_store_variables) .concat(google_play_variables) + .concat(diffblue_cover_variables) end end @@ -568,6 +577,12 @@ module Ci Gitlab::Ci::Variables::Collection.new(google_play_integration.ci_variables(protected_ref: pipeline.protected_ref?)) end + def diffblue_cover_variables + return [] unless diffblue_cover_integration.try(:activated?) + + Gitlab::Ci::Variables::Collection.new(diffblue_cover_integration.ci_variables) + end + def features { trace_sections: true, @@ -1232,6 +1247,14 @@ module Ci def partition_id_prefix_in_16_bit_encode "#{partition_id.to_s(16)}_" end + + def prefix_and_partition_for_token + if Feature.enabled?(:prefix_ci_build_tokens, project, type: :beta) + TOKEN_PREFIX + partition_id_prefix_in_16_bit_encode + else + partition_id_prefix_in_16_bit_encode + end + end end end diff --git a/app/models/ci/catalog/resources/version.rb b/app/models/ci/catalog/resources/version.rb index 4273c4515bc..0ea2735b030 100644 --- a/app/models/ci/catalog/resources/version.rb +++ b/app/models/ci/catalog/resources/version.rb @@ -19,6 +19,7 @@ module Ci scope :for_catalog_resources, ->(catalog_resources) { where(catalog_resource_id: catalog_resources) } scope :preloaded, -> { includes(:catalog_resource, project: [:route, { namespace: :route }], release: :author) } + scope :by_name, ->(name) { joins(:release).merge(Release.where(tag: name)) } scope :order_by_created_at_asc, -> { reorder(created_at: :asc) } scope :order_by_created_at_desc, -> { reorder(created_at: :desc) } @@ -122,6 +123,14 @@ module Ci project.commit_by(oid: sha) end + def path + Gitlab::Routing.url_helpers.project_tag_path(project, name) + end + + def readme + project.repository.tree(sha).readme + end + private def update_catalog_resource diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb index 179befb8469..6a2fb1132c0 100644 --- a/app/models/ci/instance_variable.rb +++ b/app/models/ci/instance_variable.rb @@ -13,6 +13,7 @@ module Ci alias_attribute :secret_value, :value + validates :description, length: { maximum: 255 }, allow_blank: true validates :key, uniqueness: { message: -> (object, data) { _("(%{value}) has already been taken") } } diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb index ff7e681217a..5f55713b436 100644 --- a/app/models/ci/namespace_mirror.rb +++ b/app/models/ci/namespace_mirror.rb @@ -7,6 +7,7 @@ module Ci include FromUnion belongs_to :namespace + has_many :project_mirrors, primary_key: :namespace_id, foreign_key: :namespace_id, inverse_of: :namespace_mirror scope :by_group_and_descendants, -> (id) do where('traversal_ids @> ARRAY[?]::int[]', id) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 9d5b2e5a0b1..1bf4d585e1c 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -20,7 +20,6 @@ module Ci include IgnorableColumns ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' - ignore_column :auto_canceled_by_id_convert_to_bigint, remove_with: '16.6', remove_after: '2023-10-22' MAX_OPEN_MERGE_REQUESTS_REFS = 4 @@ -439,7 +438,7 @@ module Ci where_exists(Ci::Build.latest.scoped_pipeline.with_artifacts(reports_scope)) end - scope :with_only_interruptible_builds, -> do + scope :conservative_interruptible, -> do where_not_exists( Ci::Build.scoped_pipeline.with_status(STARTED_STATUSES).not_interruptible ) @@ -621,7 +620,7 @@ module Ci end def valid_commit_sha - if self.sha == Gitlab::Git::BLANK_SHA + if self.sha == Gitlab::Git::SHA1_BLANK_SHA self.errors.add(:sha, " cant be 00000000 (branch removal)") end end @@ -675,7 +674,7 @@ module Ci end def before_sha - super || Gitlab::Git::BLANK_SHA + super || Gitlab::Git::SHA1_BLANK_SHA end def short_sha @@ -1394,6 +1393,10 @@ module Ci merge_request.merge_request_diff_for(merge_request_diff_sha) end + def auto_cancel_on_new_commit + pipeline_metadata&.auto_cancel_on_new_commit || 'conservative' + end + private def add_message(severity, content) diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb index 6d22a875aab..e0e6906f211 100644 --- a/app/models/ci/pipeline_artifact.rb +++ b/app/models/ci/pipeline_artifact.rb @@ -4,6 +4,7 @@ module Ci class PipelineArtifact < Ci::ApplicationRecord + include Ci::Partitionable include UpdateProjectStatistics include Artifactable include FileStoreMounter @@ -31,6 +32,8 @@ module Ci validates :size, presence: true, numericality: { less_than_or_equal_to: FILE_SIZE_LIMIT } validates :file_type, presence: true + partitionable scope: :pipeline + mount_file_store_uploader Ci::PipelineArtifactUploader update_project_statistics project_statistics_name: :pipeline_artifacts_size diff --git a/app/models/ci/pipeline_chat_data.rb b/app/models/ci/pipeline_chat_data.rb index ba20c993e36..1a2bc37d17d 100644 --- a/app/models/ci/pipeline_chat_data.rb +++ b/app/models/ci/pipeline_chat_data.rb @@ -2,14 +2,21 @@ module Ci class PipelineChatData < Ci::ApplicationRecord + include Ci::Partitionable include Ci::NamespacedModelName + include SafelyChangeColumnDefault + + columns_changing_default :partition_id self.table_name = 'ci_pipeline_chat_data' belongs_to :chat_name + belongs_to :pipeline validates :pipeline_id, presence: true validates :chat_name_id, presence: true validates :response_url, presence: true + + partitionable scope: :pipeline end end diff --git a/app/models/ci/pipeline_config.rb b/app/models/ci/pipeline_config.rb index e2dcad653d7..11decd3fc66 100644 --- a/app/models/ci/pipeline_config.rb +++ b/app/models/ci/pipeline_config.rb @@ -2,11 +2,15 @@ module Ci class PipelineConfig < Ci::ApplicationRecord + include Ci::Partitionable + self.table_name = 'ci_pipelines_config' self.primary_key = :pipeline_id belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_config validates :pipeline, presence: true validates :content, presence: true + + partitionable scope: :pipeline end end diff --git a/app/models/ci/pipeline_metadata.rb b/app/models/ci/pipeline_metadata.rb index 37fa3e32ad8..21d102374f0 100644 --- a/app/models/ci/pipeline_metadata.rb +++ b/app/models/ci/pipeline_metadata.rb @@ -2,12 +2,15 @@ module Ci class PipelineMetadata < Ci::ApplicationRecord + include Ci::Partitionable + include Importable + self.primary_key = :pipeline_id enum auto_cancel_on_new_commit: { conservative: 0, interruptible: 1, - disabled: 2 + none: 2 }, _prefix: true enum auto_cancel_on_job_failure: { @@ -21,5 +24,7 @@ module Ci validates :pipeline, presence: true validates :project, presence: true validates :name, length: { minimum: 1, maximum: 255 }, allow_nil: true + + partitionable scope: :pipeline end end diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index b1831e365b1..4fddb3e053e 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -5,9 +5,6 @@ module Ci include Ci::Partitionable include Ci::HasVariable include Ci::RawVariable - include IgnorableColumns - - ignore_column :pipeline_id_convert_to_bigint, remove_with: '16.5', remove_after: '2023-10-22' belongs_to :pipeline diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 414d36da7c3..989d6337ab7 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -33,6 +33,10 @@ module Ci where('NOT EXISTS (?)', needs) end + scope :interruptible, -> do + joins(:metadata).merge(Ci::BuildMetadata.with_interruptible) + end + scope :not_interruptible, -> do joins(:metadata).where.not( Ci::BuildMetadata.table_name => { id: Ci::BuildMetadata.scoped_build.with_interruptible.select(:id) } diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb index 23cd5d92730..c6828f827b5 100644 --- a/app/models/ci/project_mirror.rb +++ b/app/models/ci/project_mirror.rb @@ -7,6 +7,8 @@ module Ci include FromUnion belongs_to :project + belongs_to :namespace_mirror, primary_key: :namespace_id, foreign_key: :namespace_id, inverse_of: :project_mirrors + has_many :builds, primary_key: :project_id, foreign_key: :project_id, inverse_of: :project_mirror scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) } scope :by_project_id, -> (project_id) { where(project_id: project_id) } diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 9c30beeeb59..5fb982ee21e 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -14,6 +14,7 @@ module Ci include Presentable include EachBatch include Ci::HasRunnerExecutor + include Ci::HasRunnerStatus extend ::Gitlab::Utils::Override @@ -85,22 +86,22 @@ module Ci before_save :ensure_token - scope :active, -> (value = true) { where(active: value) } + scope :active, ->(value = true) { where(active: value) } scope :paused, -> { active(false) } - scope :online, -> { where(arel_table[:contacted_at].gt(online_contact_time_deadline)) } scope :recent, -> do timestamp = stale_deadline where(arel_table[:created_at].gteq(timestamp).or(arel_table[:contacted_at].gteq(timestamp))) end scope :stale, -> do - timestamp = stale_deadline + stale_timestamp = stale_deadline + + created_before_stale_deadline = arel_table[:created_at].lteq(stale_timestamp) + contacted_before_stale_deadline = arel_table[:contacted_at].lteq(stale_timestamp) + never_contacted = arel_table[:contacted_at].eq(nil) - where(arel_table[:created_at].lteq(timestamp)) - .where(arel_table[:contacted_at].eq(nil).or(arel_table[:contacted_at].lteq(timestamp))) + where(created_before_stale_deadline).where(never_contacted.or(contacted_before_stale_deadline)) end - scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) } - scope :never_contacted, -> { where(contacted_at: nil) } scope :ordered, -> { order(id: :desc) } scope :with_recent_runner_queue, -> { where(arel_table[:contacted_at].gt(recent_queue_deadline)) } @@ -220,6 +221,11 @@ module Ci validate :exactly_one_group, if: :group_type? scope :with_version_prefix, ->(value) { joins(:runner_managers).merge(RunnerManager.with_version_prefix(value)) } + scope :with_runner_type, ->(runner_type) do + return all if AVAILABLE_TYPES.exclude?(runner_type.to_s) + + where(runner_type: runner_type) + end acts_as_taggable @@ -348,23 +354,6 @@ module Ci description end - def online? - contacted_at && contacted_at > self.class.online_contact_time_deadline - end - - def stale? - return false unless created_at - - [created_at, contacted_at].compact.max <= self.class.stale_deadline - end - - def status - return :stale if stale? - return :never_contacted unless contacted_at - - online? ? :online : :offline - end - # DEPRECATED # TODO Remove in v5 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 def deprecated_rest_status @@ -475,6 +464,21 @@ module Ci end end + def clear_heartbeat + cleared_attributes = { + version: nil, + revision: nil, + platform: nil, + architecture: nil, + ip_address: nil, + executor_type: nil, + config: {}, + contacted_at: nil + } + merge_cache_attributes(cleared_attributes) + update_columns(cleared_attributes) + end + def pick_build!(build) tick_runner_queue if matches_build?(build) end diff --git a/app/models/ci/runner_manager.rb b/app/models/ci/runner_manager.rb index e6576859827..44fe1bdd67d 100644 --- a/app/models/ci/runner_manager.rb +++ b/app/models/ci/runner_manager.rb @@ -5,10 +5,13 @@ module Ci include FromUnion include RedisCacheable include Ci::HasRunnerExecutor + include Ci::HasRunnerStatus # For legacy reasons, the table name is ci_runner_machines in the database self.table_name = 'ci_runner_machines' + AVAILABLE_STATUSES = %w[online offline never_contacted stale].freeze + # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner Machine DB entry can be updated UPDATE_CONTACT_COLUMN_EVERY = (40.minutes)..(55.minutes) @@ -36,19 +39,26 @@ module Ci STALE_TIMEOUT = 7.days scope :stale, -> do - created_some_time_ago = arel_table[:created_at].lteq(STALE_TIMEOUT.ago) - contacted_some_time_ago = arel_table[:contacted_at].lteq(STALE_TIMEOUT.ago) + stale_timestamp = stale_deadline + + created_before_stale_deadline = arel_table[:created_at].lteq(stale_timestamp) + contacted_before_stale_deadline = arel_table[:contacted_at].lteq(stale_timestamp) from_union( - where(contacted_at: nil), - where(contacted_some_time_ago), - remove_duplicates: false).where(created_some_time_ago) + never_contacted, + where(contacted_before_stale_deadline), + remove_duplicates: false + ).where(created_before_stale_deadline) end scope :for_runner, ->(runner_id) do where(runner_id: runner_id) end + scope :with_system_xid, ->(system_xid) do + where(system_xid: system_xid) + end + scope :with_running_builds, -> do where('EXISTS(?)', Ci::Build.select(1) @@ -114,25 +124,8 @@ module Ci end end - def status - return :stale if stale? - return :never_contacted unless contacted_at - - online? ? :online : :offline - end - private - def online? - contacted_at && contacted_at > self.class.online_contact_time_deadline - end - - def stale? - return false unless created_at - - [created_at, contacted_at].compact.max <= self.class.stale_deadline - end - def persist_cached_data? # Use a random threshold to prevent beating DB updates. contacted_at_max_age = Random.rand(UPDATE_CONTACT_COLUMN_EVERY) diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index becb8f204bf..ba1a0a46247 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -7,9 +7,6 @@ module Ci include Ci::HasStatus include Gitlab::OptimisticLocking include Presentable - include IgnorableColumns - - ignore_column :pipeline_id_convert_to_bigint, remove_with: '16.6', remove_after: '2023-10-22' partitionable scope: :pipeline |