diff options
Diffstat (limited to 'app/models/ci')
| -rw-r--r-- | app/models/ci/build.rb | 49 | ||||
| -rw-r--r-- | app/models/ci/build_metadata.rb | 5 | ||||
| -rw-r--r-- | app/models/ci/job_token/project_scope_link.rb | 5 | ||||
| -rw-r--r-- | app/models/ci/job_token/scope.rb | 2 | ||||
| -rw-r--r-- | app/models/ci/pipeline.rb | 54 | ||||
| -rw-r--r-- | app/models/ci/pipeline_metadata.rb | 14 | ||||
| -rw-r--r-- | app/models/ci/runner.rb | 38 | ||||
| -rw-r--r-- | app/models/ci/secure_file.rb | 39 |
8 files changed, 166 insertions, 40 deletions
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 4e58f877217..b8511536e32 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -108,10 +108,12 @@ module Ci validates :ref, presence: true scope :not_interruptible, -> do - joins(:metadata).where.not('ci_builds_metadata.id' => Ci::BuildMetadata.scoped_build.with_interruptible.select(:id)) + joins(:metadata) + .where.not(Ci::BuildMetadata.table_name => { id: Ci::BuildMetadata.scoped_build.with_interruptible.select(:id) }) end scope :unstarted, -> { where(runner_id: nil) } + scope :with_downloadable_artifacts, -> do where('EXISTS (?)', Ci::JobArtifact.select(1) @@ -120,6 +122,14 @@ module Ci ) end + scope :with_erasable_artifacts, -> do + where('EXISTS (?)', + Ci::JobArtifact.select(1) + .where('ci_builds.id = ci_job_artifacts.job_id') + .where(file_type: Ci::JobArtifact.erasable_file_types) + ) + end + scope :in_pipelines, ->(pipelines) do where(pipeline: pipelines) end @@ -178,7 +188,7 @@ module Ci scope :license_management_jobs, -> { where(name: %i(license_management license_scanning)) } # handle license rename https://gitlab.com/gitlab-org/gitlab/issues/8911 scope :with_secure_reports_from_config_options, -> (job_types) do - joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) + joins(:metadata).where("#{Ci::BuildMetadata.quoted_table_name}.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) end scope :with_coverage, -> { where.not(coverage: nil) } @@ -218,7 +228,7 @@ module Ci yaml_variables when environment coverage_regex description tag_list protected needs_attributes job_variables_attributes resource_group scheduling_type - ci_stage partition_id].freeze + ci_stage partition_id id_tokens].freeze end end @@ -407,18 +417,10 @@ module Ci pipeline.manual_actions.reject { |action| action.name == self.name } end - def environment_manual_actions - pipeline.manual_actions.filter { |action| action.expanded_environment_name == self.expanded_environment_name } - end - def other_scheduled_actions pipeline.scheduled_actions.reject { |action| action.name == self.name } end - def environment_scheduled_actions - pipeline.scheduled_actions.filter { |action| action.expanded_environment_name == self.expanded_environment_name } - end - def pages_generator? Gitlab.config.pages.enabled && self.name == 'pages' @@ -445,8 +447,7 @@ module Ci def prevent_rollback_deployment? strong_memoize(:prevent_rollback_deployment) do - Feature.enabled?(:prevent_outdated_deployment_jobs, project) && - starts_environment? && + starts_environment? && project.ci_forward_deployment_enabled? && deployment&.older_than_last_successful_deployment? end @@ -1195,6 +1196,14 @@ module Ci end def job_jwt_variables + if project.ci_cd_settings.opt_in_jwt? + id_tokens_variables + else + legacy_jwt_variables.concat(id_tokens_variables) + end + end + + def legacy_jwt_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless Feature.enabled?(:ci_job_jwt, project) @@ -1208,6 +1217,20 @@ module Ci end end + def id_tokens_variables + return [] unless id_tokens? + + Gitlab::Ci::Variables::Collection.new.tap do |variables| + id_tokens.each do |var_name, token_data| + token = Gitlab::Ci::JwtV2.for_build(self, aud: token_data['id_token']['aud']) + + variables.append(key: var_name, value: token, public: false, masked: true) + end + rescue OpenSSL::PKey::RSAError, Gitlab::Ci::Jwt::NoSigningKeyError => e + Gitlab::ErrorTracking.track_exception(e) + end + end + def cache_for_online_runners(&block) Rails.cache.fetch( ['has-online-runners', id], diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index 3bdf2f90acb..33092e881f0 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -6,11 +6,14 @@ module Ci class BuildMetadata < Ci::ApplicationRecord BuildTimeout = Struct.new(:value, :source) + include Ci::Partitionable include Presentable include ChronicDurationAttribute include Gitlab::Utils::StrongMemoize self.table_name = 'ci_builds_metadata' + self.primary_key = 'id' + partitionable scope: :build belongs_to :build, class_name: 'CommitStatus' belongs_to :project @@ -27,7 +30,7 @@ module Ci chronic_duration_attr_reader :timeout_human_readable, :timeout - scope :scoped_build, -> { where('ci_builds_metadata.build_id = ci_builds.id') } + scope :scoped_build, -> { where("#{quoted_table_name}.build_id = #{Ci::Build.quoted_table_name}.id") } scope :with_interruptible, -> { where(interruptible: true) } scope :with_exposed_artifacts, -> { where(has_exposed_artifacts: true) } diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb index c2ab8ca0929..3fdf07123e6 100644 --- a/app/models/ci/job_token/project_scope_link.rb +++ b/app/models/ci/job_token/project_scope_link.rb @@ -19,6 +19,11 @@ module Ci validates :target_project, presence: true validate :not_self_referential_link + enum direction: { + outbound: 0, + inbound: 1 + } + def self.for_source_and_target(source_project, target_project) self.find_by(source_project: source_project, target_project: target_project) end diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb index 26a49d6a730..1aa49b95201 100644 --- a/app/models/ci/job_token/scope.rb +++ b/app/models/ci/job_token/scope.rb @@ -23,7 +23,7 @@ module Ci def includes?(target_project) # if the setting is disabled any project is considered to be in scope. - return true unless source_project.ci_job_token_scope_enabled? + return true unless source_project.ci_outbound_job_token_scope_enabled? target_project.id == source_project.id || Ci::JobToken::ProjectScopeLink.from_project(source_project).to_project(target_project).exists? diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 1e328c3c573..950e0a583bc 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -112,6 +112,8 @@ module Ci has_one :pipeline_config, class_name: 'Ci::PipelineConfig', inverse_of: :pipeline + has_one :pipeline_metadata, class_name: 'Ci::PipelineMetadata', inverse_of: :pipeline + has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id has_many :latest_builds_report_results, through: :latest_builds, source: :report_results has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :pipeline, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -119,6 +121,7 @@ module Ci accepts_nested_attributes_for :variables, reject_if: :persisted? delegate :full_path, to: :project, prefix: true + delegate :title, to: :pipeline_metadata, allow_nil: true validates :sha, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } @@ -614,6 +617,15 @@ module Ci # auto_canceled_by_pipeline_id - store the pipeline_id of the pipeline that triggered cancellation # execute_async - if true cancel the children asyncronously def cancel_running(retries: 1, cascade_to_children: true, auto_canceled_by_pipeline_id: nil, execute_async: true) + Gitlab::AppJsonLogger.info( + event: 'pipeline_cancel_running', + pipeline_id: id, + auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id, + cascade_to_children: cascade_to_children, + execute_async: execute_async, + **Gitlab::ApplicationContext.current + ) + update(auto_canceled_by_id: auto_canceled_by_pipeline_id) if auto_canceled_by_pipeline_id cancel_jobs(cancelable_statuses, retries: retries, auto_canceled_by_pipeline_id: auto_canceled_by_pipeline_id) @@ -760,8 +772,14 @@ module Ci # There is no ActiveRecord relation between Ci::Pipeline and notes # as they are related to a commit sha. This method helps importing # them using the +Gitlab::ImportExport::Project::RelationFactory+ class. - def notes=(notes) - notes.each do |note| + def notes=(notes_to_save) + notes_to_save.reject! do |note_to_save| + notes.any? do |note| + [note_to_save.note, note_to_save.created_at.to_i] == [note.note, note.created_at.to_i] + end + end + + notes_to_save.each do |note| note[:id] = nil note[:commit_id] = sha note[:noteable_id] = self['id'] @@ -850,7 +868,6 @@ module Ci variables.append(key: 'CI_COMMIT_REF_NAME', value: source_ref) variables.append(key: 'CI_COMMIT_REF_SLUG', value: source_ref_slug) variables.append(key: 'CI_COMMIT_BRANCH', value: ref) if branch? - variables.append(key: 'CI_COMMIT_TAG', value: ref) if tag? variables.append(key: 'CI_COMMIT_MESSAGE', value: git_commit_message.to_s) variables.append(key: 'CI_COMMIT_TITLE', value: git_commit_full_title.to_s) variables.append(key: 'CI_COMMIT_DESCRIPTION', value: git_commit_description.to_s) @@ -863,7 +880,8 @@ module Ci variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha) variables.append(key: 'CI_BUILD_REF_NAME', value: source_ref) variables.append(key: 'CI_BUILD_REF_SLUG', value: source_ref_slug) - variables.append(key: 'CI_BUILD_TAG', value: ref) if tag? + + variables.concat(predefined_commit_tag_variables) end end end @@ -888,6 +906,20 @@ module Ci end end + def predefined_commit_tag_variables + strong_memoize(:predefined_commit_ref_variables) do + Gitlab::Ci::Variables::Collection.new.tap do |variables| + next variables unless tag? + + variables.append(key: 'CI_COMMIT_TAG', value: ref) + variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: project.repository.find_tag(ref).message) + + # legacy variable + variables.append(key: 'CI_BUILD_TAG', value: ref) + end + end + end + def queued_duration return unless started_at @@ -972,8 +1004,8 @@ module Ci # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 expanded_environment_names = builds_in_self_and_project_descendants.joins(:metadata) - .where.not('ci_builds_metadata.expanded_environment_name' => nil) - .distinct('ci_builds_metadata.expanded_environment_name') + .where.not(Ci::BuildMetadata.table_name => { expanded_environment_name: nil }) + .distinct("#{Ci::BuildMetadata.quoted_table_name}.expanded_environment_name") .limit(100) .pluck(:expanded_environment_name) @@ -1162,6 +1194,10 @@ module Ci complete? && builds.latest.with_exposed_artifacts.exists? end + def has_erasable_artifacts? + complete? && builds.latest.with_erasable_artifacts.exists? + end + def branch_updated? strong_memoize(:branch_updated) do push_details.branch_updated? @@ -1328,9 +1364,9 @@ module Ci self.builds.latest.build_matchers(project) end - def authorized_cluster_agents - strong_memoize(:authorized_cluster_agents) do - ::Clusters::AgentAuthorizationsFinder.new(project).execute.map(&:agent) + def cluster_agent_authorizations + strong_memoize(:cluster_agent_authorizations) do + ::Clusters::AgentAuthorizationsFinder.new(project).execute end end diff --git a/app/models/ci/pipeline_metadata.rb b/app/models/ci/pipeline_metadata.rb new file mode 100644 index 00000000000..c96b395b45f --- /dev/null +++ b/app/models/ci/pipeline_metadata.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Ci + class PipelineMetadata < Ci::ApplicationRecord + self.primary_key = :pipeline_id + + belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_metadata + belongs_to :project, class_name: "Project", inverse_of: :pipeline_metadata + + validates :pipeline, presence: true + validates :project, presence: true + validates :title, presence: true, length: { minimum: 1, maximum: 255 } + end +end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 28d9edcc135..3be627989b1 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -14,7 +14,7 @@ module Ci include Presentable include EachBatch - add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration, expiration_enforced?: :token_expiration_enforced? + add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration enum access_level: { not_protected: 0, @@ -99,27 +99,26 @@ module Ci } scope :belonging_to_group, -> (group_id) { - joins(:runner_namespaces) - .where(ci_runner_namespaces: { namespace_id: group_id }) + joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_id }) } scope :belonging_to_group_or_project_descendants, -> (group_id) { group_ids = Ci::NamespaceMirror.by_group_and_descendants(group_id).select(:namespace_id) project_ids = Ci::ProjectMirror.by_namespace_id(group_ids).select(:project_id) - group_runners = joins(:runner_namespaces).where(ci_runner_namespaces: { namespace_id: group_ids }) - project_runners = joins(:runner_projects).where(ci_runner_projects: { project_id: project_ids }) + group_runners = belonging_to_group(group_ids) + project_runners = belonging_to_project(project_ids).distinct - union_sql = ::Gitlab::SQL::Union.new([group_runners, project_runners]).to_sql - - from("(#{union_sql}) #{table_name}") + from_union( + [group_runners, project_runners], + remove_duplicates: false + ) } scope :belonging_to_group_and_ancestors, -> (group_id) { group_self_and_ancestors_ids = ::Group.find_by(id: group_id)&.self_and_ancestor_ids - joins(:runner_namespaces) - .where(ci_runner_namespaces: { namespace_id: group_self_and_ancestors_ids }) + belonging_to_group(group_self_and_ancestors_ids) } scope :belonging_to_parent_group_of_project, -> (project_id) { @@ -153,6 +152,17 @@ module Ci ) end + scope :usable_from_scope, -> (group) do + from_union( + [ + belonging_to_group(group.ancestor_ids), + belonging_to_group_or_project_descendants(group.id), + group.shared_runners + ], + remove_duplicates: false + ) + end + scope :assignable_for, ->(project) do # FIXME: That `to_sql` is needed to workaround a weird Rails bug. # Without that, placeholders would miss one and couldn't match. @@ -205,7 +215,7 @@ module Ci validates :maintenance_note, length: { maximum: 1024 } - alias_attribute :maintenance_note, :maintainer_note + alias_attribute :maintenance_note, :maintainer_note # NOTE: Need to keep until REST v5 is implemented # Searches for runners matching the given query. # @@ -335,7 +345,7 @@ module Ci end # DEPRECATED - # TODO Remove in %16.0 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 + # TODO Remove in v5 in favor of `status` for REST calls, see https://gitlab.com/gitlab-org/gitlab/-/issues/344648 def deprecated_rest_status return :stale if stale? @@ -470,10 +480,6 @@ module Ci end end - def self.token_expiration_enforced? - Feature.enabled?(:enforce_runner_token_expires_at) - end - private scope :with_upgrade_status, ->(upgrade_status) do diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 9a35f1876c9..ffff7eebbee 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -7,6 +7,7 @@ module Ci FILE_SIZE_LIMIT = 5.megabytes.freeze CHECKSUM_ALGORITHM = 'sha256' + PARSABLE_EXTENSIONS = %w[cer p12 mobileprovision].freeze self.limit_scope = :project self.limit_name = 'project_ci_secure_files' @@ -16,6 +17,7 @@ module Ci validates :file, presence: true, file_size: { maximum: FILE_SIZE_LIMIT } validates :checksum, :file_store, :name, :project_id, presence: true validates :name, uniqueness: { scope: :project } + validates :metadata, json_schema: { filename: "ci_secure_file_metadata" }, allow_nil: true after_initialize :generate_key_data before_validation :assign_checksum @@ -23,6 +25,8 @@ module Ci scope :order_by_created_at, -> { order(created_at: :desc) } scope :project_id_in, ->(ids) { where(project_id: ids) } + serialize :metadata, Serializers::Json # rubocop:disable Cop/ActiveRecordSerialize + default_value_for(:file_store) { Ci::SecureFileUploader.default_store } mount_file_store_uploader Ci::SecureFileUploader @@ -31,6 +35,41 @@ module Ci CHECKSUM_ALGORITHM end + def file_extension + File.extname(name).delete_prefix('.') + end + + def metadata_parsable? + PARSABLE_EXTENSIONS.include?(file_extension) + end + + def metadata_parser + return unless metadata_parsable? + + case file_extension + when 'cer' + Gitlab::Ci::SecureFiles::Cer.new(file.read) + when 'p12' + Gitlab::Ci::SecureFiles::P12.new(file.read) + when 'mobileprovision' + Gitlab::Ci::SecureFiles::MobileProvision.new(file.read) + end + end + + def update_metadata! + return unless metadata_parser + + begin + parser = metadata_parser + self.metadata = parser.metadata + self.expires_at = parser.metadata[:expires_at] + save! + rescue StandardError => err + Gitlab::AppLogger.error("Secure File Parser Failure (#{id}): #{err.message} - #{parser.error}.") + nil + end + end + private def assign_checksum |