diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 12:16:11 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 12:16:11 +0300 |
commit | edaa33dee2ff2f7ea3fac488d41558eb5f86d68c (patch) | |
tree | 11f143effbfeba52329fb7afbd05e6e2a3790241 /app/models/ci | |
parent | d8a5691316400a0f7ec4f83832698f1988eb27c1 (diff) |
Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42
Diffstat (limited to 'app/models/ci')
-rw-r--r-- | app/models/ci/build.rb | 64 | ||||
-rw-r--r-- | app/models/ci/job_artifact.rb | 16 | ||||
-rw-r--r-- | app/models/ci/namespace_mirror.rb | 6 | ||||
-rw-r--r-- | app/models/ci/pipeline.rb | 22 | ||||
-rw-r--r-- | app/models/ci/project_mirror.rb | 3 | ||||
-rw-r--r-- | app/models/ci/runner.rb | 66 | ||||
-rw-r--r-- | app/models/ci/secure_file.rb | 33 |
7 files changed, 154 insertions, 56 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 428e440afba..c4d1a2c740b 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -268,6 +268,10 @@ module Ci !build.any_unmet_prerequisites? # If false is returned, it stops the transition end + before_transition on: :enqueue do |build| + !build.waiting_for_deployment_approval? # If false is returned, it stops the transition + end + after_transition created: :scheduled do |build| build.run_after_commit do Ci::BuildScheduleWorker.perform_at(build.scheduled_at, build.id) @@ -393,6 +397,10 @@ module Ci auto_retry.allowed? end + def auto_retry_expected? + failed? && auto_retry_allowed? + end + def detailed_status(current_user) Gitlab::Ci::Status::Build::Factory .new(self.present, current_user) @@ -416,6 +424,10 @@ module Ci true end + def save_tags + super unless Thread.current['ci_bulk_insert_tags'] + end + def archived? return true if degenerated? @@ -424,7 +436,11 @@ module Ci end def playable? - action? && !archived? && (manual? || scheduled? || retryable?) + action? && !archived? && (manual? || scheduled? || retryable?) && !waiting_for_deployment_approval? + end + + def waiting_for_deployment_approval? + manual? && starts_environment? && deployment&.blocked? end def schedulable? @@ -751,9 +767,7 @@ module Ci def any_runners_available? cache_for_available_runners do - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do - project.active_runners.exists? - end + project.active_runners.exists? end end @@ -1021,7 +1035,15 @@ module Ci transaction do update_columns(status: :failed, failure_reason: :data_integrity_failure) all_queuing_entries.delete_all + all_runtime_metadata.delete_all end + + Gitlab::AppLogger.info( + message: 'Build doomed', + class: self.class.name, + build_id: id, + pipeline_id: pipeline_id, + project_id: project_id) end def degradation_threshold @@ -1051,10 +1073,7 @@ module Ci end def drop_with_exit_code!(failure_reason, exit_code) - transaction do - conditionally_allow_failure!(exit_code) - drop!(failure_reason) - end + drop!(::Gitlab::Ci::Build::Status::Reason.new(self, failure_reason, exit_code)) end def exit_codes_defined? @@ -1065,6 +1084,10 @@ module Ci ::Ci::PendingBuild.upsert_from_build!(self) end + def create_runtime_metadata! + ::Ci::RunningBuild.upsert_shared_runner_build!(self) + end + ## # We can have only one queuing entry or running build tracking entry, # because there is a unique index on `build_id` in each table, but we need @@ -1093,6 +1116,13 @@ module Ci end end + def allowed_to_fail_with_code?(exit_code) + options + .dig(:allow_failure_criteria, :exit_codes) + .to_a + .include?(exit_code) + end + protected def run_status_commit_hooks! @@ -1174,27 +1204,15 @@ module Ci break variables unless Feature.enabled?(:ci_job_jwt, project, default_enabled: true) jwt = Gitlab::Ci::Jwt.for_build(self) + jwt_v2 = Gitlab::Ci::JwtV2.for_build(self) variables.append(key: 'CI_JOB_JWT', value: jwt, public: false, masked: true) + variables.append(key: 'CI_JOB_JWT_V1', value: jwt, public: false, masked: true) + variables.append(key: 'CI_JOB_JWT_V2', value: jwt_v2, public: false, masked: true) rescue OpenSSL::PKey::RSAError, Gitlab::Ci::Jwt::NoSigningKeyError => e Gitlab::ErrorTracking.track_exception(e) end end - def conditionally_allow_failure!(exit_code) - return unless exit_code - - if allowed_to_fail_with_code?(exit_code) - update_columns(allow_failure: true) - end - end - - def allowed_to_fail_with_code?(exit_code) - options - .dig(:allow_failure_criteria, :exit_codes) - .to_a - .include?(exit_code) - end - def cache_for_online_runners(&block) Rails.cache.fetch( ['has-online-runners', id], diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index e6dd62fab34..3426c4d5248 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -181,9 +181,7 @@ module Ci end scope :erasable, -> do - types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values - - where(file_type: types) + where(file_type: self.erasable_file_types) end scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } @@ -263,6 +261,10 @@ module Ci [file_type] end + def self.erasable_file_types + self.file_types.keys - NON_ERASABLE_FILE_TYPES + end + def self.total_size self.sum(:size) end @@ -271,10 +273,6 @@ module Ci self.where(project: project).sum(:size) end - def self.distinct_job_ids - distinct.pluck(:job_id) - end - ## # FastDestroyAll concerns # rubocop: disable CodeReuse/ServiceClass @@ -350,9 +348,7 @@ module Ci def store_after_commit? strong_memoize(:store_after_commit) do - trace? && - JobArtifactUploader.direct_upload_enabled? && - Feature.enabled?(:ci_store_trace_outside_transaction, project, default_enabled: :yaml) + trace? && JobArtifactUploader.direct_upload_enabled? end end diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb index 8a4be3139e8..ce3faf3546b 100644 --- a/app/models/ci/namespace_mirror.rb +++ b/app/models/ci/namespace_mirror.rb @@ -10,6 +10,12 @@ module Ci where('traversal_ids @> ARRAY[?]::int[]', id) end + scope :contains_any_of_namespaces, -> (ids) do + where('traversal_ids && ARRAY[?]::int[]', ids) + end + + scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) } + class << self def sync!(event) namespace = event.namespace diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index a90bd739741..00d331df4c3 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -69,6 +69,7 @@ module Ci has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline has_many :generic_commit_statuses, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'GenericCommitStatus' has_many :job_artifacts, through: :builds + has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' has_many :deployments, through: :builds @@ -130,6 +131,7 @@ module Ci after_create :keep_around_commits, unless: :importing? use_fast_destroy :job_artifacts + use_fast_destroy :build_trace_chunks # We use `Enums::Ci::Pipeline.sources` here so that EE can more easily extend # this `Hash` with new values. @@ -242,11 +244,7 @@ module Ci ::JiraConnect::SyncBuildsWorker.perform_async(pipeline.id, seq_id) end - if Feature.enabled?(:expire_job_and_pipeline_cache_synchronously, pipeline.project, default_enabled: :yaml) - Ci::ExpirePipelineCacheService.new.execute(pipeline) # rubocop: disable CodeReuse/ServiceClass - else - ExpirePipelineCacheWorker.perform_async(pipeline.id) - end + Ci::ExpirePipelineCacheService.new.execute(pipeline) # rubocop: disable CodeReuse/ServiceClass end end @@ -466,6 +464,14 @@ module Ci statuses.count(:id) end + def tags_count + ActsAsTaggableOn::Tagging.where(taggable: builds).count + end + + def distinct_tags_count + ActsAsTaggableOn::Tagging.where(taggable: builds).count('distinct(tag_id)') + end + def stages_names statuses.order(:stage_idx).distinct .pluck(:stage, :stage_idx).map(&:first) @@ -1284,6 +1290,12 @@ module Ci end end + def use_variables_builder_definitions? + strong_memoize(:use_variables_builder_definitions) do + ::Feature.enabled?(:ci_use_variables_builder_definitions, project, default_enabled: :yaml) + end + end + private def add_message(severity, content) diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb index d6aaa3f50c1..9000d1791a6 100644 --- a/app/models/ci/project_mirror.rb +++ b/app/models/ci/project_mirror.rb @@ -6,6 +6,9 @@ module Ci class ProjectMirror < ApplicationRecord belongs_to :project + scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) } + scope :by_project_id, -> (project_id) { where(project_id: project_id) } + class << self def sync!(event) upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id }, diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index a80fd02080f..809c245d2b9 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -95,7 +95,33 @@ module Ci joins(:runner_projects).where(ci_runner_projects: { project_id: project_id }) } - scope :belonging_to_group, -> (group_id, include_ancestors: false) { + scope :belonging_to_group, -> (group_id) { + joins(:runner_namespaces) + .where(ci_runner_namespaces: { namespace_id: group_id }) + } + + scope :belonging_to_group_or_project_descendants, -> (group_id) { + group_ids = Ci::NamespaceMirror.contains_namespace(group_id).select(:namespace_id) + project_ids = Ci::ProjectMirror.by_namespace_id(group_ids).select(:project_id) + + group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_ids }) + project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_ids }) + + union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql + + from("(#{union_sql}) #{table_name}") + } + + scope :belonging_to_group_and_ancestors, -> (group_id) { + group_self_and_ancestors_ids = ::Group.find_by(id: group_id)&.self_and_ancestor_ids + + joins(:runner_namespaces) + .where(ci_runner_namespaces: { namespace_id: group_self_and_ancestors_ids }) + } + + # deprecated + # split this into: belonging_to_group & belonging_to_group_and_ancestors + scope :legacy_belonging_to_group, -> (group_id, include_ancestors: false) { groups = ::Group.where(id: group_id) groups = groups.self_and_ancestors if include_ancestors @@ -104,7 +130,8 @@ module Ci .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') } - scope :belonging_to_group_or_project, -> (group_id, project_id) { + # deprecated + scope :legacy_belonging_to_group_or_project, -> (group_id, project_id) { groups = ::Group.where(id: group_id) group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: groups }) @@ -117,11 +144,11 @@ module Ci } scope :belonging_to_parent_group_of_project, -> (project_id) { + raise ArgumentError, "only 1 project_id allowed for performance reasons" unless project_id.is_a?(Integer) + project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) - joins(:groups) - .where(namespaces: { id: project_groups.self_and_ancestors.as_ids }) - .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') + belonging_to_group(project_groups.self_and_ancestors.pluck(:id)) } scope :owned_or_instance_wide, -> (project_id) do @@ -132,7 +159,7 @@ module Ci instance_type ], remove_duplicates: false - ).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') + ) end scope :assignable_for, ->(project) do @@ -183,6 +210,8 @@ module Ci validates :config, json_schema: { filename: 'ci_runner_config' } + validates :maintainer_note, length: { maximum: 255 } + # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL for the description field and performs a full match on tokens. @@ -233,18 +262,16 @@ module Ci Arel.sql("(#{arel_tag_names_array.to_sql})") ] - ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339621') do - group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| - Gitlab::Ci::Matching::RunnerMatcher.new({ - runner_ids: values[0], - runner_type: values[1], - public_projects_minutes_cost_factor: values[2], - private_projects_minutes_cost_factor: values[3], - run_untagged: values[4], - access_level: values[5], - tag_list: values[6] - }) - end + group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| + Gitlab::Ci::Matching::RunnerMatcher.new({ + runner_ids: values[0], + runner_type: values[1], + public_projects_minutes_cost_factor: values[2], + private_projects_minutes_cost_factor: values[3], + run_untagged: values[4], + access_level: values[5], + tag_list: values[6] + }) end end @@ -441,6 +468,7 @@ module Ci private EXECUTOR_NAME_TO_TYPES = { + 'unknown' => :unknown, 'custom' => :custom, 'shell' => :shell, 'docker' => :docker, @@ -454,6 +482,8 @@ module Ci 'kubernetes' => :kubernetes }.freeze + EXECUTOR_TYPE_TO_NAMES = EXECUTOR_NAME_TO_TYPES.invert.freeze + def cleanup_runner_queue Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb new file mode 100644 index 00000000000..56f632b6232 --- /dev/null +++ b/app/models/ci/secure_file.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Ci + class SecureFile < Ci::ApplicationRecord + include FileStoreMounter + + FILE_SIZE_LIMIT = 5.megabytes.freeze + CHECKSUM_ALGORITHM = 'sha256' + + belongs_to :project, optional: false + + validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT } + validates :checksum, :file_store, :name, :permissions, :project_id, presence: true + + before_validation :assign_checksum + + enum permissions: { read_only: 0, read_write: 1, execute: 2 } + + default_value_for(:file_store) { Ci::SecureFileUploader.default_store } + + mount_file_store_uploader Ci::SecureFileUploader + + def checksum_algorithm + CHECKSUM_ALGORITHM + end + + private + + def assign_checksum + self.checksum = file.checksum if file.present? && file_changed? + end + end +end |