Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/app/models/ci
diff options
context:
space:
mode:
Diffstat (limited to 'app/models/ci')
-rw-r--r--app/models/ci/bridge.rb3
-rw-r--r--app/models/ci/build.rb49
-rw-r--r--app/models/ci/build_dependencies.rb2
-rw-r--r--app/models/ci/build_report_result.rb45
-rw-r--r--app/models/ci/build_runner_session.rb15
-rw-r--r--app/models/ci/daily_build_group_report_result.rb2
-rw-r--r--app/models/ci/freeze_period.rb2
-rw-r--r--app/models/ci/group.rb2
-rw-r--r--app/models/ci/instance_variable.rb53
-rw-r--r--app/models/ci/job_artifact.rb29
-rw-r--r--app/models/ci/pipeline.rb50
-rw-r--r--app/models/ci/pipeline_enums.rb5
-rw-r--r--app/models/ci/processable.rb4
-rw-r--r--app/models/ci/ref.rb71
-rw-r--r--app/models/ci/runner.rb32
15 files changed, 251 insertions, 113 deletions
diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index 1e92a47ab49..58c26e8c806 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -16,6 +16,9 @@ module Ci
has_many :sourced_pipelines, class_name: "::Ci::Sources::Pipeline",
foreign_key: :source_job_id
+ has_one :sourced_pipeline, class_name: "::Ci::Sources::Pipeline", foreign_key: :source_job_id
+ has_one :downstream_pipeline, through: :sourced_pipeline, source: :pipeline
+
validates :ref, presence: true
# rubocop:disable Cop/ActiveRecordSerialize
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 7f64ea7dd97..b5e68b55f72 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -26,7 +26,8 @@ module Ci
RUNNER_FEATURES = {
upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? },
refspecs: -> (build) { build.merge_request_ref? },
- artifacts_exclude: -> (build) { build.supports_artifacts_exclude? }
+ artifacts_exclude: -> (build) { build.supports_artifacts_exclude? },
+ release_steps: -> (build) { build.release_steps? }
}.freeze
DEFAULT_RETRIES = {
@@ -39,6 +40,7 @@ module Ci
has_one :resource, class_name: 'Ci::Resource', inverse_of: :build
has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id
+ has_many :report_results, class_name: 'Ci::BuildReportResult', inverse_of: :build
has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent
has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id
@@ -55,6 +57,7 @@ module Ci
delegate :url, to: :runner_session, prefix: true, allow_nil: true
delegate :terminal_specification, to: :runner_session, allow_nil: true
+ delegate :service_specification, to: :runner_session, allow_nil: true
delegate :gitlab_deploy_token, to: :project
delegate :trigger_short_token, to: :trigger_request, allow_nil: true
@@ -137,8 +140,8 @@ module Ci
.includes(:metadata, :job_artifacts_metadata)
end
- scope :with_artifacts_not_expired, ->() { with_downloadable_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
- scope :with_expired_artifacts, ->() { with_downloadable_artifacts.where('artifacts_expire_at < ?', Time.now) }
+ scope :with_artifacts_not_expired, ->() { with_downloadable_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.current) }
+ scope :with_expired_artifacts, ->() { with_downloadable_artifacts.where('artifacts_expire_at < ?', Time.current) }
scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) }
scope :scheduled_actions, ->() { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) }
@@ -259,7 +262,7 @@ module Ci
end
before_transition any => :waiting_for_resource do |build|
- build.waiting_for_resource_at = Time.now
+ build.waiting_for_resource_at = Time.current
end
before_transition on: :enqueue_waiting_for_resource do |build|
@@ -352,7 +355,7 @@ module Ci
begin
Ci::Build.retry(build, build.user)
rescue Gitlab::Access::AccessDeniedError => ex
- Rails.logger.error "Unable to auto-retry job #{build.id}: #{ex}" # rubocop:disable Gitlab/RailsLogger
+ Gitlab::AppLogger.error "Unable to auto-retry job #{build.id}: #{ex}"
end
end
end
@@ -576,7 +579,7 @@ module Ci
def environment_changed_page_variables
Gitlab::Ci::Variables::Collection.new.tap do |variables|
- break variables unless environment_status
+ break variables unless environment_status && Feature.enabled?(:modifed_path_ci_variables, project)
variables.append(key: 'CI_MERGE_REQUEST_CHANGED_PAGE_PATHS', value: environment_status.changed_paths.join(','))
variables.append(key: 'CI_MERGE_REQUEST_CHANGED_PAGE_URLS', value: environment_status.changed_urls.join(','))
@@ -686,6 +689,10 @@ module Ci
job_artifacts.any?
end
+ def has_test_reports?
+ job_artifacts.test_reports.exists?
+ end
+
def has_old_trace?
old_trace.present?
end
@@ -713,7 +720,7 @@ module Ci
end
def needs_touch?
- Time.now - updated_at > 15.minutes.to_i
+ Time.current - updated_at > 15.minutes.to_i
end
def valid_token?(token)
@@ -756,13 +763,13 @@ module Ci
# and use that for `ExpireBuildInstanceArtifactsWorker`?
def erase_erasable_artifacts!
- job_artifacts.erasable.destroy_all # rubocop: disable DestroyAll
+ job_artifacts.erasable.destroy_all # rubocop: disable Cop/DestroyAll
end
def erase(opts = {})
return false unless erasable?
- job_artifacts.destroy_all # rubocop: disable DestroyAll
+ job_artifacts.destroy_all # rubocop: disable Cop/DestroyAll
erase_trace!
update_erased!(opts[:erased_by])
end
@@ -776,11 +783,11 @@ module Ci
end
def artifacts_expired?
- artifacts_expire_at && artifacts_expire_at < Time.now
+ artifacts_expire_at && artifacts_expire_at < Time.current
end
def artifacts_expire_in
- artifacts_expire_at - Time.now if artifacts_expire_at
+ artifacts_expire_at - Time.current if artifacts_expire_at
end
def artifacts_expire_in=(value)
@@ -809,6 +816,7 @@ module Ci
def steps
[Gitlab::Ci::Build::Step.from_commands(self),
+ Gitlab::Ci::Build::Step.from_release(self),
Gitlab::Ci::Build::Step.from_after_script(self)].compact
end
@@ -872,6 +880,16 @@ module Ci
options&.dig(:artifacts, :reports)&.any?
end
+ def supports_artifacts_exclude?
+ options&.dig(:artifacts, :exclude)&.any? &&
+ Gitlab::Ci::Features.artifacts_exclude_enabled?
+ end
+
+ def release_steps?
+ options.dig(:release)&.any? &&
+ Gitlab::Ci::Features.release_generation_enabled?
+ end
+
def hide_secrets(trace)
return unless trace
@@ -945,11 +963,6 @@ module Ci
failure_reason: :data_integrity_failure)
end
- def supports_artifacts_exclude?
- options&.dig(:artifacts, :exclude)&.any? &&
- Gitlab::Ci::Features.artifacts_exclude_enabled?
- end
-
def degradation_threshold
var = yaml_variables.find { |v| v[:key] == DEGRADATION_THRESHOLD_VARIABLE_NAME }
var[:value]&.to_i if var
@@ -993,7 +1006,7 @@ module Ci
end
def update_erased!(user = nil)
- self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil)
+ self.update(erased_by: user, erased_at: Time.current, artifacts_expire_at: nil)
end
def unscoped_project
@@ -1026,7 +1039,7 @@ module Ci
end
def has_expiring_artifacts?
- artifacts_expire_at.present? && artifacts_expire_at > Time.now
+ artifacts_expire_at.present? && artifacts_expire_at > Time.current
end
def job_jwt_variables
diff --git a/app/models/ci/build_dependencies.rb b/app/models/ci/build_dependencies.rb
index d3ff870e36a..2fcd1708cf4 100644
--- a/app/models/ci/build_dependencies.rb
+++ b/app/models/ci/build_dependencies.rb
@@ -45,7 +45,7 @@ module Ci
end
def valid_local?
- return true if Feature.enabled?('ci_disable_validates_dependencies')
+ return true if Feature.enabled?(:ci_disable_validates_dependencies)
local.all?(&:valid_dependency?)
end
diff --git a/app/models/ci/build_report_result.rb b/app/models/ci/build_report_result.rb
new file mode 100644
index 00000000000..530233ad5c0
--- /dev/null
+++ b/app/models/ci/build_report_result.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Ci
+ class BuildReportResult < ApplicationRecord
+ extend Gitlab::Ci::Model
+
+ self.primary_key = :build_id
+
+ belongs_to :build, class_name: "Ci::Build", inverse_of: :report_results
+ belongs_to :project, class_name: "Project", inverse_of: :build_report_results
+
+ validates :build, :project, presence: true
+ validates :data, json_schema: { filename: "build_report_result_data" }
+
+ store_accessor :data, :tests
+
+ def tests_name
+ tests.dig("name")
+ end
+
+ def tests_duration
+ tests.dig("duration")
+ end
+
+ def tests_success
+ tests.dig("success").to_i
+ end
+
+ def tests_failed
+ tests.dig("failed").to_i
+ end
+
+ def tests_errored
+ tests.dig("errored").to_i
+ end
+
+ def tests_skipped
+ tests.dig("skipped").to_i
+ end
+
+ def tests_total
+ [tests_success, tests_failed, tests_errored, tests_skipped].sum
+ end
+ end
+end
diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb
index b46bbe69c7c..bc7f17f046c 100644
--- a/app/models/ci/build_runner_session.rb
+++ b/app/models/ci/build_runner_session.rb
@@ -7,6 +7,8 @@ module Ci
extend Gitlab::Ci::Model
TERMINAL_SUBPROTOCOL = 'terminal.gitlab.com'
+ DEFAULT_SERVICE_NAME = 'build'.freeze
+ DEFAULT_PORT_NAME = 'default_port'.freeze
self.table_name = 'ci_builds_runner_session'
@@ -23,6 +25,17 @@ module Ci
channel_specification(wss_url, TERMINAL_SUBPROTOCOL)
end
+ def service_specification(service: nil, path: nil, port: nil, subprotocols: nil)
+ return {} unless url.present?
+
+ port = port.presence || DEFAULT_PORT_NAME
+ service = service.presence || DEFAULT_SERVICE_NAME
+ url = "#{self.url}/proxy/#{service}/#{port}/#{path}"
+ subprotocols = subprotocols.presence || ::Ci::BuildRunnerSession::TERMINAL_SUBPROTOCOL
+
+ channel_specification(url, subprotocols)
+ end
+
private
def channel_specification(url, subprotocol)
@@ -37,5 +50,3 @@ module Ci
end
end
end
-
-Ci::BuildRunnerSession.prepend_if_ee('EE::Ci::BuildRunnerSession')
diff --git a/app/models/ci/daily_build_group_report_result.rb b/app/models/ci/daily_build_group_report_result.rb
index 3506b27e974..d6617b8c2eb 100644
--- a/app/models/ci/daily_build_group_report_result.rb
+++ b/app/models/ci/daily_build_group_report_result.rb
@@ -9,6 +9,8 @@ module Ci
belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id
belongs_to :project
+ validates :data, json_schema: { filename: "daily_build_group_report_result_data" }
+
def self.upsert_reports(data)
upsert_all(data, unique_by: :index_daily_build_group_report_results_unique_columns) if data.any?
end
diff --git a/app/models/ci/freeze_period.rb b/app/models/ci/freeze_period.rb
index bf03b92259a..d215372bb45 100644
--- a/app/models/ci/freeze_period.rb
+++ b/app/models/ci/freeze_period.rb
@@ -5,7 +5,7 @@ module Ci
include StripAttribute
self.table_name = 'ci_freeze_periods'
- default_scope { order(created_at: :asc) }
+ default_scope { order(created_at: :asc) } # rubocop:disable Cop/DefaultScope
belongs_to :project, inverse_of: :freeze_periods
diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb
index 4b2081f2977..779c6c0396f 100644
--- a/app/models/ci/group.rb
+++ b/app/models/ci/group.rb
@@ -24,7 +24,7 @@ module Ci
def status
strong_memoize(:status) do
- if Feature.enabled?(:ci_composite_status, project, default_enabled: false)
+ if ::Gitlab::Ci::Features.composite_status?(project)
Gitlab::Ci::Status::Composite
.new(@jobs)
.status
diff --git a/app/models/ci/instance_variable.rb b/app/models/ci/instance_variable.rb
index c674f76d229..8245729a884 100644
--- a/app/models/ci/instance_variable.rb
+++ b/app/models/ci/instance_variable.rb
@@ -3,8 +3,13 @@
module Ci
class InstanceVariable < ApplicationRecord
extend Gitlab::Ci::Model
+ extend Gitlab::ProcessMemoryCache::Helper
include Ci::NewHasVariable
include Ci::Maskable
+ include Limitable
+
+ self.limit_name = 'ci_instance_level_variables'
+ self.limit_scope = Limitable::GLOBAL_SCOPE
alias_attribute :secret_value, :value
@@ -12,8 +17,14 @@ module Ci
message: "(%{value}) has already been taken"
}
+ validates :encrypted_value, length: {
+ maximum: 1024,
+ too_long: 'The encrypted value of the provided variable exceeds %{count} bytes. Variables over 700 characters risk exceeding the limit.'
+ }
+
scope :unprotected, -> { where(protected: false) }
- after_commit { self.class.touch_redis_cache_timestamp }
+
+ after_commit { self.class.invalidate_memory_cache(:ci_instance_variable_data) }
class << self
def all_cached
@@ -24,10 +35,6 @@ module Ci
cached_data[:unprotected]
end
- def touch_redis_cache_timestamp(time = Time.current.to_f)
- shared_backend.write(:ci_instance_variable_changed_at, time)
- end
-
private
def cached_data
@@ -37,39 +44,13 @@ module Ci
{ all: all_records, unprotected: all_records.reject(&:protected?) }
end
end
+ end
- def fetch_memory_cache(key, &payload)
- cache = process_backend.read(key)
-
- if cache && !stale_cache?(cache)
- cache[:data]
- else
- store_cache(key, &payload)
- end
- end
-
- def stale_cache?(cache_info)
- shared_timestamp = shared_backend.read(:ci_instance_variable_changed_at)
- return true unless shared_timestamp
-
- shared_timestamp.to_f > cache_info[:cached_at].to_f
- end
-
- def store_cache(key)
- data = yield
- time = Time.current.to_f
-
- process_backend.write(key, data: data, cached_at: time)
- touch_redis_cache_timestamp(time)
- data
- end
-
- def shared_backend
- Rails.cache
- end
+ private
- def process_backend
- Gitlab::ProcessMemoryCache.cache_backend
+ def validate_plan_limit_not_exceeded
+ if Gitlab::Ci::Features.instance_level_variables_limit_enabled?
+ super
end
end
end
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
index d931428dccd..8aba9356949 100644
--- a/app/models/ci/job_artifact.rb
+++ b/app/models/ci/job_artifact.rb
@@ -5,6 +5,7 @@ module Ci
include AfterCommitQueue
include ObjectStorage::BackgroundMove
include UpdateProjectStatistics
+ include UsageStatistics
include Sortable
extend Gitlab::Ci::Model
@@ -26,6 +27,7 @@ module Ci
accessibility: 'gl-accessibility.json',
codequality: 'gl-code-quality-report.json',
sast: 'gl-sast-report.json',
+ secret_detection: 'gl-secret-detection-report.json',
dependency_scanning: 'gl-dependency-scanning-report.json',
container_scanning: 'gl-container-scanning-report.json',
dast: 'gl-dast-report.json',
@@ -37,7 +39,8 @@ module Ci
dotenv: '.env',
cobertura: 'cobertura-coverage.xml',
terraform: 'tfplan.json',
- cluster_applications: 'gl-cluster-applications.json'
+ cluster_applications: 'gl-cluster-applications.json',
+ requirements: 'requirements.json'
}.freeze
INTERNAL_TYPES = {
@@ -62,13 +65,15 @@ module Ci
accessibility: :raw,
codequality: :raw,
sast: :raw,
+ secret_detection: :raw,
dependency_scanning: :raw,
container_scanning: :raw,
dast: :raw,
license_management: :raw,
license_scanning: :raw,
performance: :raw,
- terraform: :raw
+ terraform: :raw,
+ requirements: :raw
}.freeze
DOWNLOADABLE_TYPES = %w[
@@ -87,6 +92,8 @@ module Ci
metrics
performance
sast
+ secret_detection
+ requirements
].freeze
TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze
@@ -109,6 +116,7 @@ module Ci
after_save :update_file_store, if: :saved_change_to_file?
+ scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
scope :with_files_stored_locally, -> { where(file_store: [nil, ::JobArtifactUploader::Store::LOCAL]) }
scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) }
@@ -147,7 +155,8 @@ module Ci
where(file_type: types)
end
- scope :expired, -> (limit) { where('expire_at < ?', Time.now).limit(limit) }
+ scope :expired, -> (limit) { where('expire_at < ?', Time.current).limit(limit) }
+ scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) }
scope :locked, -> { where(locked: true) }
scope :unlocked, -> { where(locked: [false, nil]) }
@@ -176,7 +185,9 @@ module Ci
cobertura: 17,
terraform: 18, # Transformed json
accessibility: 19,
- cluster_applications: 20
+ cluster_applications: 20,
+ secret_detection: 21, ## EE-specific
+ requirements: 22 ## EE-specific
}
enum file_format: {
@@ -242,8 +253,16 @@ module Ci
super || self.file_location.nil?
end
+ def expired?
+ expire_at.present? && expire_at < Time.current
+ end
+
+ def expiring?
+ expire_at.present? && expire_at > Time.current
+ end
+
def expire_in
- expire_at - Time.now if expire_at
+ expire_at - Time.current if expire_at
end
def expire_in=(value)
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 5db1635f64d..497e1a4d74a 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -31,6 +31,7 @@ module Ci
belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
belongs_to :merge_request, class_name: 'MergeRequest'
belongs_to :external_pull_request
+ belongs_to :ci_ref, class_name: 'Ci::Ref', foreign_key: :ci_ref_id, inverse_of: :pipelines
has_internal_id :iid, scope: :project, presence: false, track_if: -> { !importing? }, ensure_if: -> { !importing? }, init: ->(s) do
s&.project&.all_pipelines&.maximum(:iid) || s&.project&.all_pipelines&.count
@@ -40,11 +41,15 @@ module Ci
has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline
has_many :latest_statuses_ordered_by_stage, -> { latest.order(:stage_idx, :stage) }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline
has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline
+ has_many :bridges, class_name: 'Ci::Bridge', foreign_key: :commit_id, inverse_of: :pipeline
has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline
+ has_many :job_artifacts, through: :builds
has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent
has_many :variables, class_name: 'Ci::PipelineVariable'
has_many :deployments, through: :builds
has_many :environments, -> { distinct }, through: :deployments
+ has_many :latest_builds, -> { latest }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build'
+ has_many :downloadable_artifacts, -> { not_expired.downloadable }, through: :latest_builds, source: :job_artifacts
# Merge requests for which the current pipeline is running against
# the merge request's latest commit.
@@ -56,7 +61,6 @@ module Ci
has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
- has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
@@ -64,13 +68,6 @@ module Ci
has_one :source_pipeline, class_name: 'Ci::Sources::Pipeline', inverse_of: :pipeline
- has_one :ref_status, ->(pipeline) {
- # We use .read_attribute to save 1 extra unneeded query to load the :project.
- unscope(:where)
- .where(project_id: pipeline.read_attribute(:project_id), ref: pipeline.ref, tag: pipeline.tag)
- # Sadly :inverse_of is not supported (yet) by Rails for composite PKs.
- }, class_name: 'Ci::Ref', inverse_of: :pipelines
-
has_one :chat_data, class_name: 'Ci::PipelineChatData'
has_many :triggered_pipelines, through: :sourced_pipelines, source: :pipeline
@@ -163,11 +160,11 @@ module Ci
# Create a separate worker for each new operation
before_transition [:created, :waiting_for_resource, :preparing, :pending] => :running do |pipeline|
- pipeline.started_at = Time.now
+ pipeline.started_at = Time.current
end
before_transition any => [:success, :failed, :canceled] do |pipeline|
- pipeline.finished_at = Time.now
+ pipeline.finished_at = Time.current
pipeline.update_duration
end
@@ -235,12 +232,10 @@ module Ci
end
after_transition any => [:success, :failed] do |pipeline|
+ ref_status = pipeline.ci_ref&.update_status_by!(pipeline)
+
pipeline.run_after_commit do
- if Feature.enabled?(:ci_pipeline_fixed_notifications)
- PipelineUpdateCiRefStatusWorker.perform_async(pipeline.id)
- else
- PipelineNotificationWorker.perform_async(pipeline.id)
- end
+ PipelineNotificationWorker.perform_async(pipeline.id, ref_status: ref_status)
end
end
@@ -260,6 +255,7 @@ module Ci
scope :for_sha_or_source_sha, -> (sha) { for_sha(sha).or(for_source_sha(sha)) }
scope :for_ref, -> (ref) { where(ref: ref) }
scope :for_id, -> (id) { where(id: id) }
+ scope :for_iid, -> (iid) { where(iid: iid) }
scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) }
scope :with_reports, -> (reports_scope) do
@@ -397,11 +393,11 @@ module Ci
end
def ordered_stages
- if Feature.enabled?(:ci_atomic_processing, project, default_enabled: false)
+ if ::Gitlab::Ci::Features.atomic_processing?(project)
# The `Ci::Stage` contains all up-to date data
# as atomic processing updates all data in-bulk
stages
- elsif Feature.enabled?(:ci_pipeline_persisted_stages, default_enabled: true) && complete?
+ elsif complete?
# The `Ci::Stage` contains up-to date data only for `completed` pipelines
# this is due to asynchronous processing of pipeline, and stages possibly
# not updated inline with processing of pipeline
@@ -445,7 +441,7 @@ module Ci
end
def legacy_stages
- if Feature.enabled?(:ci_composite_status, project, default_enabled: false)
+ if ::Gitlab::Ci::Features.composite_status?(project)
legacy_stages_using_composite_status
else
legacy_stages_using_sql
@@ -798,13 +794,17 @@ module Ci
@latest_builds_with_artifacts ||= builds.latest.with_artifacts_not_expired.to_a
end
+ def latest_report_builds(reports_scope = ::Ci::JobArtifact.with_reports)
+ builds.latest.with_reports(reports_scope)
+ end
+
def has_reports?(reports_scope)
- complete? && builds.latest.with_reports(reports_scope).exists?
+ complete? && latest_report_builds(reports_scope).exists?
end
def test_reports
Gitlab::Ci::Reports::TestReports.new.tap do |test_reports|
- builds.latest.with_reports(Ci::JobArtifact.test_reports).preload(:project).find_each do |build|
+ latest_report_builds(Ci::JobArtifact.test_reports).preload(:project).find_each do |build|
build.collect_test_reports!(test_reports)
end
end
@@ -826,7 +826,7 @@ module Ci
def coverage_reports
Gitlab::Ci::Reports::CoverageReports.new.tap do |coverage_reports|
- builds.latest.with_reports(Ci::JobArtifact.coverage_reports).each do |build|
+ latest_report_builds(Ci::JobArtifact.coverage_reports).each do |build|
build.collect_coverage_reports!(coverage_reports)
end
end
@@ -834,7 +834,7 @@ module Ci
def terraform_reports
::Gitlab::Ci::Reports::TerraformReports.new.tap do |terraform_reports|
- builds.latest.with_reports(::Ci::JobArtifact.terraform_reports).each do |build|
+ latest_report_builds(::Ci::JobArtifact.terraform_reports).each do |build|
build.collect_terraform_reports!(terraform_reports)
end
end
@@ -969,6 +969,12 @@ module Ci
processables.populate_scheduling_type!
end
+ def ensure_ci_ref!
+ return unless Gitlab::Ci::Features.pipeline_fixed_notifications?
+
+ self.ci_ref = Ci::Ref.ensure_for(self)
+ end
+
private
def pipeline_data
diff --git a/app/models/ci/pipeline_enums.rb b/app/models/ci/pipeline_enums.rb
index 7e203cb67c4..2ccd8445aa8 100644
--- a/app/models/ci/pipeline_enums.rb
+++ b/app/models/ci/pipeline_enums.rb
@@ -27,9 +27,11 @@ module Ci
# https://gitlab.com/gitlab-org/gitlab/issues/195991
pipeline: 7,
chat: 8,
+ webide: 9,
merge_request_event: 10,
external_pull_request_event: 11,
- parent_pipeline: 12
+ parent_pipeline: 12,
+ ondemand_scan: 13
}
end
@@ -40,6 +42,7 @@ module Ci
unknown_source: nil,
repository_source: 1,
auto_devops_source: 2,
+ webide_source: 3,
remote_source: 4,
external_project_source: 5,
bridge_source: 6
diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb
index cc00500662d..ac5785d9c91 100644
--- a/app/models/ci/processable.rb
+++ b/app/models/ci/processable.rb
@@ -4,12 +4,8 @@ module Ci
class Processable < ::CommitStatus
include Gitlab::Utils::StrongMemoize
- has_many :needs, class_name: 'Ci::BuildNeed', foreign_key: :build_id, inverse_of: :build
-
accepts_nested_attributes_for :needs
- enum scheduling_type: { stage: 0, dag: 1 }, _prefix: true
-
scope :preload_needs, -> { preload(:needs) }
scope :with_needs, -> (names = nil) do
diff --git a/app/models/ci/ref.rb b/app/models/ci/ref.rb
index a0782bc0444..be6062b6e6e 100644
--- a/app/models/ci/ref.rb
+++ b/app/models/ci/ref.rb
@@ -3,21 +3,62 @@
module Ci
class Ref < ApplicationRecord
extend Gitlab::Ci::Model
+ include Gitlab::OptimisticLocking
- STATUSES = %w[success failed fixed].freeze
-
- belongs_to :project
- belongs_to :last_updated_by_pipeline, foreign_key: :last_updated_by_pipeline_id, class_name: 'Ci::Pipeline'
- # ActiveRecord doesn't support composite FKs for this reason we have to do the 'unscope(:where)'
- # hack.
- has_many :pipelines, ->(ref) {
- # We use .read_attribute to save 1 extra unneeded query to load the :project.
- unscope(:where)
- .where(ref: ref.ref, project_id: ref.read_attribute(:project_id), tag: ref.tag)
- # Sadly :inverse_of is not supported (yet) by Rails for composite PKs.
- }, inverse_of: :ref_status
-
- validates :status, inclusion: { in: STATUSES }
- validates :last_updated_by_pipeline, presence: true
+ FAILING_STATUSES = %w[failed broken still_failing].freeze
+
+ belongs_to :project, inverse_of: :ci_refs
+ has_many :pipelines, class_name: 'Ci::Pipeline', foreign_key: :ci_ref_id, inverse_of: :ci_ref
+
+ state_machine :status, initial: :unknown do
+ event :succeed do
+ transition unknown: :success
+ transition fixed: :success
+ transition %i[failed broken still_failing] => :fixed
+ end
+
+ event :do_fail do
+ transition unknown: :failed
+ transition %i[failed broken] => :still_failing
+ transition %i[success fixed] => :broken
+ end
+
+ state :unknown, value: 0
+ state :success, value: 1
+ state :failed, value: 2
+ state :fixed, value: 3
+ state :broken, value: 4
+ state :still_failing, value: 5
+ end
+
+ class << self
+ def ensure_for(pipeline)
+ safe_find_or_create_by(project_id: pipeline.project_id,
+ ref_path: pipeline.source_ref_path)
+ end
+
+ def failing_state?(status_name)
+ FAILING_STATUSES.include?(status_name)
+ end
+ end
+
+ def last_finished_pipeline_id
+ Ci::Pipeline.where(ci_ref_id: self.id).finished.order(id: :desc).select(:id).take&.id
+ end
+
+ def update_status_by!(pipeline)
+ return unless Gitlab::Ci::Features.pipeline_fixed_notifications?
+
+ retry_lock(self) do
+ next unless last_finished_pipeline_id == pipeline.id
+
+ case pipeline.status
+ when 'success' then self.succeed
+ when 'failed' then self.do_fail
+ end
+
+ self.status_name
+ end
+ end
end
end
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index d4e9217ff9f..8fc273556f0 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -23,10 +23,17 @@ module Ci
project_type: 3
}
- ONLINE_CONTACT_TIMEOUT = 1.hour
+ # This `ONLINE_CONTACT_TIMEOUT` needs to be larger than
+ # `RUNNER_QUEUE_EXPIRY_TIME+UPDATE_CONTACT_COLUMN_EVERY`
+ #
+ ONLINE_CONTACT_TIMEOUT = 2.hours
+
+ # The `RUNNER_QUEUE_EXPIRY_TIME` indicates the longest interval that
+ # Runner request needs to be refreshed by Rails instead of being handled
+ # by Workhorse
RUNNER_QUEUE_EXPIRY_TIME = 1.hour
- # This needs to be less than `ONLINE_CONTACT_TIMEOUT`
+ # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner DB entry can be updated
UPDATE_CONTACT_COLUMN_EVERY = (40.minutes..55.minutes).freeze
AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze
@@ -81,6 +88,17 @@ module Ci
joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups })
}
+ scope :belonging_to_group_or_project, -> (group_id, project_id) {
+ groups = ::Group.where(id: group_id)
+
+ group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups })
+ project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_id })
+
+ union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql
+
+ from("(#{union_sql}) #{table_name}")
+ }
+
scope :belonging_to_parent_group_of_project, -> (project_id) {
project_groups = ::Group.joins(:projects).where(projects: { id: project_id })
hierarchy_groups = Gitlab::ObjectHierarchy.new(project_groups).base_and_ancestors
@@ -145,14 +163,14 @@ module Ci
# Searches for runners matching the given query.
#
- # This method uses ILIKE on PostgreSQL and LIKE on MySQL.
+ # This method uses ILIKE on PostgreSQL.
#
# This method performs a *partial* match on tokens, thus a query for "a"
# will match any runner where the token contains the letter "a". As a result
# you should *not* use this method for non-admin purposes as otherwise users
# might be able to query a list of all runners.
#
- # query - The search query as a String
+ # query - The search query as a String.
#
# Returns an ActiveRecord::Relation.
def self.search(query)
@@ -271,9 +289,9 @@ module Ci
ensure_runner_queue_value == value if value.present?
end
- def update_cached_info(values)
+ def heartbeat(values)
values = values&.slice(:version, :revision, :platform, :architecture, :ip_address) || {}
- values[:contacted_at] = Time.now
+ values[:contacted_at] = Time.current
cache_attributes(values)
@@ -309,7 +327,7 @@ module Ci
real_contacted_at = read_attribute(:contacted_at)
real_contacted_at.nil? ||
- (Time.now - real_contacted_at) >= contacted_at_max_age
+ (Time.current - real_contacted_at) >= contacted_at_max_age
end
def tag_constraints
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-15 17:59:43 +0300