diff options
Diffstat (limited to 'app/models')
158 files changed, 1851 insertions, 1292 deletions
diff --git a/app/models/ability.rb b/app/models/ability.rb index eb645bcd653..4da4d113a7f 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -77,6 +77,8 @@ class Ability policy = policy_for(user, subject) + before_check(policy, ability.to_sym, user, subject, opts) + case opts[:scope] when :user DeclarativePolicy.user_scope { policy.allowed?(ability) } @@ -92,6 +94,11 @@ class Ability forget_runner_result(policy.runner(ability)) if policy && ability_forgetting? end + # Hook call right before ability check. + def before_check(policy, ability, user, subject, opts) + # See Support::AbilityCheck and Support::PermissionsCheck. + end + def policy_for(user, subject = :global) DeclarativePolicy.policy_for(user, subject, cache: ::Gitlab::SafeRequestStore.storage) end diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb index dbcdfa5e946..5ae5367ca5a 100644 --- a/app/models/abuse_report.rb +++ b/app/models/abuse_report.rb @@ -42,7 +42,9 @@ class AbuseReport < ApplicationRecord before_validation :filter_empty_strings_from_links_to_spam validate :links_to_spam_contains_valid_urls - scope :by_user, ->(user) { where(user_id: user) } + scope :by_user_id, ->(id) { where(user_id: id) } + scope :by_reporter_id, ->(id) { where(reporter_id: id) } + scope :by_category, ->(category) { where(category: category) } scope :with_users, -> { includes(:reporter, :user) } enum category: { @@ -56,6 +58,11 @@ class AbuseReport < ApplicationRecord other: 8 } + enum status: { + open: 1, + closed: 2 + } + # For CacheMarkdownField alias_method :author, :reporter diff --git a/app/models/achievements/user_achievement.rb b/app/models/achievements/user_achievement.rb index 885ec660cc9..bc5d10923d7 100644 --- a/app/models/achievements/user_achievement.rb +++ b/app/models/achievements/user_achievement.rb @@ -8,10 +8,16 @@ module Achievements belongs_to :awarded_by_user, class_name: 'User', inverse_of: :awarded_user_achievements, - optional: true + optional: false belongs_to :revoked_by_user, class_name: 'User', inverse_of: :revoked_user_achievements, optional: true + + scope :not_revoked, -> { where(revoked_by_user_id: nil) } + + def revoked? + revoked_by_user_id.present? + end end end diff --git a/app/models/airflow/dags.rb b/app/models/airflow/dags.rb deleted file mode 100644 index d17d4a4f3db..00000000000 --- a/app/models/airflow/dags.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Airflow - class Dags < ApplicationRecord - belongs_to :project - - validates :project, presence: true - validates :dag_name, length: { maximum: 255 }, presence: true - validates :schedule, length: { maximum: 255 } - validates :fileloc, length: { maximum: 255 } - - scope :by_project_id, ->(project_id) { where(project_id: project_id).order(id: :asc) } - end -end diff --git a/app/models/alert_management/alert.rb b/app/models/alert_management/alert.rb index a5a539eae75..74edcf12ac2 100644 --- a/app/models/alert_management/alert.rb +++ b/app/models/alert_management/alert.rb @@ -25,8 +25,9 @@ module AlertManagement has_many :assignees, through: :alert_assignees has_many :notes, as: :noteable, inverse_of: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - has_many :ordered_notes, -> { fresh }, as: :noteable, class_name: 'Note' - has_many :user_mentions, class_name: 'AlertManagement::AlertUserMention', foreign_key: :alert_management_alert_id + has_many :ordered_notes, -> { fresh }, as: :noteable, class_name: 'Note', inverse_of: :noteable + has_many :user_mentions, class_name: 'AlertManagement::AlertUserMention', foreign_key: :alert_management_alert_id, + inverse_of: :alert has_many :metric_images, class_name: '::AlertManagement::MetricImage' has_internal_id :iid, scope: :project @@ -139,7 +140,7 @@ module AlertManagement end def self.link_reference_pattern - @link_reference_pattern ||= super("alert_management", %r{(?<alert>\d+)/details(\#)?}) + @link_reference_pattern ||= compose_link_reference_pattern('alert_management', %r{(?<alert>\d+)/details(\#)?}) end def self.reference_valid?(reference) diff --git a/app/models/alert_management/alert_assignee.rb b/app/models/alert_management/alert_assignee.rb index c74b2699182..27e720c3262 100644 --- a/app/models/alert_management/alert_assignee.rb +++ b/app/models/alert_management/alert_assignee.rb @@ -3,7 +3,7 @@ module AlertManagement class AlertAssignee < ApplicationRecord belongs_to :alert, inverse_of: :alert_assignees - belongs_to :assignee, class_name: 'User', foreign_key: :user_id + belongs_to :assignee, class_name: 'User', foreign_key: :user_id, inverse_of: :alert_assignees validates :alert, presence: true validates :assignee, presence: true, uniqueness: { scope: :alert_id } diff --git a/app/models/alert_management/alert_user_mention.rb b/app/models/alert_management/alert_user_mention.rb index d36aa80ee05..1ab71127677 100644 --- a/app/models/alert_management/alert_user_mention.rb +++ b/app/models/alert_management/alert_user_mention.rb @@ -2,7 +2,10 @@ module AlertManagement class AlertUserMention < UserMention - belongs_to :alert_management_alert, class_name: '::AlertManagement::Alert' + belongs_to :alert, class_name: '::AlertManagement::Alert', + foreign_key: :alert_management_alert_id, + inverse_of: :user_mentions + belongs_to :note end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 98adbd3ab06..71434931d8c 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -22,6 +22,10 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord KROKI_URL_ERROR_MESSAGE = 'Please check your Kroki URL setting in ' \ 'Admin Area > Settings > General > Kroki' + # Validate URIs in this model according to the current value of the `deny_all_requests_except_allowed` property, + # rather than the persisted value. + ADDRESSABLE_URL_VALIDATION_OPTIONS = { deny_all_requests_except_allowed: ->(settings) { settings.deny_all_requests_except_allowed } }.freeze + enum whats_new_variant: { all_tiers: 0, current_tier: 1, disabled: 2 }, _prefix: true enum email_confirmation_setting: { off: 0, soft: 1, hard: 2 }, _prefix: true @@ -30,11 +34,13 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord add_authentication_token_field :static_objects_external_storage_auth_token, encrypted: :required add_authentication_token_field :error_tracking_access_token, encrypted: :required - belongs_to :self_monitoring_project, class_name: "Project", foreign_key: 'instance_administration_project_id' + belongs_to :self_monitoring_project, class_name: "Project", foreign_key: :instance_administration_project_id, + inverse_of: :application_setting belongs_to :push_rule alias_attribute :self_monitoring_project_id, :instance_administration_project_id - belongs_to :instance_group, class_name: "Group", foreign_key: 'instance_administrators_group_id' + belongs_to :instance_group, class_name: "Group", foreign_key: :instance_administrators_group_id, + inverse_of: :application_setting alias_attribute :instance_group_id, :instance_administrators_group_id alias_attribute :instance_administrators_group, :instance_group alias_attribute :housekeeping_optimize_repository_period, :housekeeping_incremental_repack_period @@ -90,9 +96,9 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord chronic_duration_attr :project_runner_token_expiration_interval_human_readable, :project_runner_token_expiration_interval validates :grafana_url, - system_hook_url: { + system_hook_url: ADDRESSABLE_URL_VALIDATION_OPTIONS.merge({ blocked_message: "is blocked: %{exception_message}. #{GRAFANA_URL_ERROR_MESSAGE}" - }, + }), if: :grafana_url_absolute? validate :validate_grafana_url @@ -116,22 +122,22 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :home_page_url, allow_blank: true, - addressable_url: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, if: :home_page_url_column_exists? validates :help_page_support_url, allow_blank: true, - addressable_url: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, if: :help_page_support_url_column_exists? validates :help_page_documentation_base_url, length: { maximum: 255, message: N_("is too long (maximum is %{count} characters)") }, allow_blank: true, - addressable_url: true + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS validates :after_sign_out_path, allow_blank: true, - addressable_url: true + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS validates :abuse_notification_email, devise_email: true, @@ -188,7 +194,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :gitpod_url, presence: true, - addressable_url: { enforce_sanitization: true }, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS.merge({ enforce_sanitization: true }), if: :gitpod_enabled validates :mailgun_signing_key, @@ -348,7 +354,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord if: :asset_proxy_enabled? validates :static_objects_external_storage_url, - addressable_url: true, allow_blank: true + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, allow_blank: true validates :static_objects_external_storage_auth_token, presence: true, @@ -421,6 +427,10 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord allow_nil: false, inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :deny_all_requests_except_allowed, + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + Gitlab::SSHPublicKey.supported_types.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end @@ -452,7 +462,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord if: :external_authorization_service_enabled validates :external_authorization_service_url, - addressable_url: true, allow_blank: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, allow_blank: true, if: :external_authorization_service_enabled validates :external_authorization_service_timeout, @@ -460,7 +470,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord if: :external_authorization_service_enabled validates :spam_check_endpoint_url, - addressable_url: { schemes: %w(tls grpc) }, allow_blank: true + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS.merge({ schemes: %w(tls grpc) }), allow_blank: true validates :spam_check_endpoint_url, presence: true, @@ -534,7 +544,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :jira_connect_proxy_url, length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, allow_blank: true, - public_url: true + public_url: ADDRESSABLE_URL_VALIDATION_OPTIONS with_options(presence: true, numericality: { only_integer: true, greater_than: 0 }) do validates :throttle_unauthenticated_api_requests_per_period @@ -563,14 +573,12 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :throttle_protected_paths_period_in_seconds end - validates :notes_create_limit, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } - - validates :search_rate_limit, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } - - validates :search_rate_limit_unauthenticated, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + with_options(numericality: { only_integer: true, greater_than_or_equal_to: 0 }) do + validates :notes_create_limit + validates :search_rate_limit + validates :search_rate_limit_unauthenticated + validates :projects_api_rate_limit_unauthenticated + end validates :notes_create_limit_allowlist, length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, @@ -580,7 +588,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :external_pipeline_validation_service_url, - addressable_url: true, allow_blank: true + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, allow_blank: true validates :external_pipeline_validation_service_timeout, allow_nil: true, @@ -607,10 +615,10 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :sentry_enabled, inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :sentry_dsn, - addressable_url: true, presence: true, length: { maximum: 255 }, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, presence: true, length: { maximum: 255 }, if: :sentry_enabled? validates :sentry_clientside_dsn, - addressable_url: true, allow_blank: true, length: { maximum: 255 }, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, allow_blank: true, length: { maximum: 255 }, if: :sentry_enabled? validates :sentry_environment, presence: true, length: { maximum: 255 }, @@ -620,7 +628,7 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :error_tracking_api_url, presence: true, - addressable_url: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, length: { maximum: 255 }, if: :error_tracking_enabled? @@ -630,7 +638,12 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, allow_nil: false - validates :public_runner_releases_url, addressable_url: true, presence: true + validates :update_runner_versions_enabled, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :public_runner_releases_url, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, + presence: true, + if: :update_runner_versions_enabled? validates :inactive_projects_min_size_mb, numericality: { only_integer: true, greater_than_or_equal_to: 0 } @@ -698,6 +711,15 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord allow_nil: false, inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :default_syntax_highlighting_theme, + allow_nil: false, + numericality: { only_integer: true, greater_than: 0 }, + inclusion: { in: Gitlab::ColorSchemes.valid_ids, message: N_('must be a valid syntax highlighting theme ID') } + + validates :gitlab_dedicated_instance, + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + before_validation :ensure_uuid! before_validation :coerce_repository_storages_weighted, if: :repository_storages_weighted_changed? before_validation :normalize_default_branch_name @@ -822,6 +844,33 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord false end + # Overriding the enum check for `email_confirmation_setting` as the feature flag is being removed and is taking a + # release M, M.N+1 strategy as noted in: + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107302#note_1286005956 + def email_confirmation_setting_off? + if Feature.enabled?(:soft_email_confirmation) + false + else + super + end + end + + def email_confirmation_setting_soft? + if Feature.enabled?(:soft_email_confirmation) + true + else + super + end + end + + def email_confirmation_setting_hard? + if Feature.enabled?(:soft_email_confirmation) + false + else + super + end + end + private def parsed_grafana_url diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index a5f262f2e1e..b8d6434d9c9 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -60,6 +60,7 @@ module ApplicationSettingImplementation default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_projects_limit: Settings.gitlab['default_projects_limit'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], + deny_all_requests_except_allowed: false, diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, diff_max_files: Commit::DEFAULT_MAX_DIFF_FILES_SETTING, diff_max_lines: Commit::DEFAULT_MAX_DIFF_LINES_SETTING, @@ -249,7 +250,9 @@ module ApplicationSettingImplementation can_create_group: true, bulk_import_enabled: false, allow_runner_registration_token: true, - user_defaults_to_private_profile: false + user_defaults_to_private_profile: false, + projects_api_rate_limit_unauthenticated: 400, + gitlab_dedicated_instance: false }.tap do |hsh| hsh.merge!(non_production_defaults) unless Rails.env.production? end diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 3312216932b..163e741d990 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -21,7 +21,7 @@ class AuditEvent < ApplicationRecord serialize :details, Hash # rubocop:disable Cop/ActiveRecordSerialize - belongs_to :user, foreign_key: :author_id + belongs_to :user, foreign_key: :author_id, inverse_of: :audit_events validates :author_id, presence: true validates :entity_id, presence: true diff --git a/app/models/badge.rb b/app/models/badge.rb index 0676de10d02..23e6f305c32 100644 --- a/app/models/badge.rb +++ b/app/models/badge.rb @@ -42,7 +42,7 @@ class Badge < ApplicationRecord private def build_rendered_url(url, project = nil) - return url unless valid? && project + return url unless project Gitlab::StringPlaceholderReplacer.replace_string_placeholders(url, PLACEHOLDERS_REGEX) do |arg| replace_placeholder_action(PLACEHOLDERS[arg], project) diff --git a/app/models/board.rb b/app/models/board.rb index 2181b2f0545..702ae0cc9f5 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -6,8 +6,8 @@ class Board < ApplicationRecord belongs_to :group belongs_to :project - has_many :lists, -> { ordered }, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - has_many :destroyable_lists, -> { destroyable.ordered }, class_name: "List" + has_many :lists, -> { ordered }, dependent: :delete_all, inverse_of: :board # rubocop:disable Cop/ActiveRecordDependent + has_many :destroyable_lists, -> { destroyable.ordered }, class_name: "List", inverse_of: :board validates :name, presence: true validates :project, presence: true, if: :project_needed? diff --git a/app/models/bulk_import.rb b/app/models/bulk_import.rb index 2565ad5f2b8..c2d7529f468 100644 --- a/app/models/bulk_import.rb +++ b/app/models/bulk_import.rb @@ -42,6 +42,12 @@ class BulkImport < ApplicationRecord event :fail_op do transition any => :failed end + + # rubocop:disable Style/SymbolProc + after_transition any => [:finished, :failed, :timeout] do |bulk_import| + bulk_import.update_has_failures + end + # rubocop:enable Style/SymbolProc end def source_version_info @@ -55,4 +61,11 @@ class BulkImport < ApplicationRecord def self.all_human_statuses state_machine.states.map(&:human_name) end + + def update_has_failures + return if has_failures + return unless entities.any?(&:has_failures) + + update!(has_failures: true) + end end diff --git a/app/models/bulk_imports/batch_tracker.rb b/app/models/bulk_imports/batch_tracker.rb new file mode 100644 index 00000000000..df1fab89ee6 --- /dev/null +++ b/app/models/bulk_imports/batch_tracker.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module BulkImports + class BatchTracker < ApplicationRecord + self.table_name = 'bulk_import_batch_trackers' + + belongs_to :tracker, class_name: 'BulkImports::Tracker' + + validates :batch_number, presence: true, uniqueness: { scope: :tracker_id } + + state_machine :status, initial: :created do + state :created, value: 0 + state :started, value: 1 + state :finished, value: 2 + state :timeout, value: 3 + state :failed, value: -1 + state :skipped, value: -2 + + event :start do + transition created: :started + end + + event :retry do + transition started: :created + end + + event :finish do + transition started: :finished + transition failed: :failed + transition skipped: :skipped + end + + event :skip do + transition any => :skipped + end + + event :fail_op do + transition any => :failed + end + + event :cleanup_stale do + transition [:created, :started] => :timeout + end + end + end +end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 6fc24c77f1d..ae2d3758110 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -26,10 +26,11 @@ class BulkImports::Entity < ApplicationRecord belongs_to :parent, class_name: 'BulkImports::Entity', optional: true belongs_to :project, optional: true - belongs_to :group, foreign_key: :namespace_id, optional: true + belongs_to :group, foreign_key: :namespace_id, optional: true, inverse_of: :bulk_import_entities has_many :trackers, class_name: 'BulkImports::Tracker', + inverse_of: :entity, foreign_key: :bulk_import_entity_id has_many :failures, @@ -104,6 +105,12 @@ class BulkImports::Entity < ApplicationRecord transition created: :timeout transition started: :timeout end + + # rubocop:disable Style/SymbolProc + after_transition any => [:finished, :failed, :timeout] do |entity| + entity.update_has_failures + end + # rubocop:enable Style/SymbolProc end def self.all_human_statuses @@ -185,6 +192,13 @@ class BulkImports::Entity < ApplicationRecord default_project_visibility end + def update_has_failures + return if has_failures + return unless failures.any? + + update!(has_failures: true) + end + private def validate_parent_is_a_group @@ -194,13 +208,6 @@ class BulkImports::Entity < ApplicationRecord end def validate_imported_entity_type - if project_entity? && !BulkImports::Features.project_migration_enabled?(destination_namespace) - errors.add( - :base, - s_('BulkImport|invalid entity source type') - ) - end - if group.present? && project_entity? errors.add( :group, diff --git a/app/models/bulk_imports/export.rb b/app/models/bulk_imports/export.rb index 8d4d31ee92d..1ea317a100a 100644 --- a/app/models/bulk_imports/export.rb +++ b/app/models/bulk_imports/export.rb @@ -14,6 +14,7 @@ module BulkImports belongs_to :group, optional: true has_one :upload, class_name: 'BulkImports::ExportUpload' + has_many :batches, class_name: 'BulkImports::ExportBatch' validates :project, presence: true, unless: :group validates :group, presence: true, unless: :project diff --git a/app/models/bulk_imports/export_batch.rb b/app/models/bulk_imports/export_batch.rb new file mode 100644 index 00000000000..9d34dae12d0 --- /dev/null +++ b/app/models/bulk_imports/export_batch.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module BulkImports + class ExportBatch < ApplicationRecord + self.table_name = 'bulk_import_export_batches' + + BATCH_SIZE = 1000 + + belongs_to :export, class_name: 'BulkImports::Export' + has_one :upload, class_name: 'BulkImports::ExportUpload', foreign_key: :batch_id, inverse_of: :batch + + validates :batch_number, presence: true, uniqueness: { scope: :export_id } + + state_machine :status, initial: :started do + state :started, value: 0 + state :finished, value: 1 + state :failed, value: -1 + + event :start do + transition any => :started + end + + event :finish do + transition started: :finished + transition failed: :failed + end + + event :fail_op do + transition any => :failed + end + end + end +end diff --git a/app/models/bulk_imports/export_upload.rb b/app/models/bulk_imports/export_upload.rb index 4304032b28c..00f8e8f1304 100644 --- a/app/models/bulk_imports/export_upload.rb +++ b/app/models/bulk_imports/export_upload.rb @@ -7,6 +7,7 @@ module BulkImports self.table_name = 'bulk_import_export_uploads' belongs_to :export, class_name: 'BulkImports::Export' + belongs_to :batch, class_name: 'BulkImports::ExportBatch', optional: true mount_uploader :export_file, ExportUploader diff --git a/app/models/bulk_imports/file_transfer.rb b/app/models/bulk_imports/file_transfer.rb index 5be954b98da..c6af4e0c833 100644 --- a/app/models/bulk_imports/file_transfer.rb +++ b/app/models/bulk_imports/file_transfer.rb @@ -9,9 +9,9 @@ module BulkImports def config_for(portable) case portable when ::Project - FileTransfer::ProjectConfig.new(portable) + ::BulkImports::FileTransfer::ProjectConfig.new(portable) when ::Group - FileTransfer::GroupConfig.new(portable) + ::BulkImports::FileTransfer::GroupConfig.new(portable) else raise(UnsupportedObjectType, "Unsupported object type: #{portable.class}") end diff --git a/app/models/bulk_imports/file_transfer/base_config.rb b/app/models/bulk_imports/file_transfer/base_config.rb index 036d511bc59..67c4e7400b3 100644 --- a/app/models/bulk_imports/file_transfer/base_config.rb +++ b/app/models/bulk_imports/file_transfer/base_config.rb @@ -51,7 +51,8 @@ module BulkImports end def portable_relations_tree - @portable_relations_tree ||= attributes_finder.find_relations_tree(portable_class_sym).deep_stringify_keys + @portable_relations_tree ||= attributes_finder + .find_relations_tree(portable_class_sym, include_import_only_tree: true).deep_stringify_keys end private diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index b04ef1cb7ae..55502721a76 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -7,9 +7,12 @@ class BulkImports::Tracker < ApplicationRecord belongs_to :entity, class_name: 'BulkImports::Entity', + inverse_of: :trackers, foreign_key: :bulk_import_entity_id, optional: false + has_many :batches, class_name: 'BulkImports::BatchTracker', inverse_of: :tracker + validates :relation, presence: true, uniqueness: { scope: :bulk_import_entity_id } diff --git a/app/models/chat_name.rb b/app/models/chat_name.rb index 9bd618c1008..cda19273f52 100644 --- a/app/models/chat_name.rb +++ b/app/models/chat_name.rb @@ -3,7 +3,9 @@ class ChatName < ApplicationRecord LAST_USED_AT_INTERVAL = 1.hour - belongs_to :integration + include IgnorableColumns + ignore_column :integration_id, remove_with: '16.0', remove_after: '2023-04-22' + belongs_to :user validates :user, presence: true diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1e70dd171ed..627604ec26c 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -18,7 +18,7 @@ module Ci belongs_to :runner belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' - belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id + belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, inverse_of: :builds RUNNER_FEATURES = { upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? }, @@ -35,8 +35,8 @@ module Ci has_one :deployment, as: :deployable, class_name: 'Deployment', inverse_of: :deployable has_one :pending_state, class_name: 'Ci::BuildPendingState', foreign_key: :build_id, inverse_of: :build - has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id - has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id + has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id, inverse_of: :build + has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id, inverse_of: :build has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build has_many :report_results, class_name: 'Ci::BuildReportResult', foreign_key: :build_id, inverse_of: :build has_one :namespace, through: :project @@ -47,7 +47,7 @@ module Ci # Details: https://gitlab.com/gitlab-org/gitlab/-/issues/24644#note_689472685 has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id, inverse_of: :job - has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id, inverse_of: :build has_many :pages_deployments, foreign_key: :ci_build_id, inverse_of: :ci_build @@ -55,7 +55,9 @@ module Ci has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', foreign_key: :job_id, inverse_of: :job end - has_one :runner_machine, through: :metadata, class_name: 'Ci::RunnerMachine' + has_one :runner_machine_build, class_name: 'Ci::RunnerMachineBuild', foreign_key: :build_id, inverse_of: :build, + autosave: true + has_one :runner_machine, through: :runner_machine_build, class_name: 'Ci::RunnerMachine' has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, foreign_key: :build_id, inverse_of: :build has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', foreign_key: :build_id, inverse_of: :build @@ -71,6 +73,7 @@ module Ci delegate :gitlab_deploy_token, to: :project delegate :harbor_integration, to: :project delegate :apple_app_store_integration, to: :project + delegate :google_play_integration, to: :project delegate :trigger_short_token, to: :trigger_request, allow_nil: true delegate :ensure_persistent_ref, to: :pipeline delegate :enable_debug_trace!, to: :metadata @@ -132,7 +135,7 @@ module Ci scope :eager_load_job_artifacts, -> { includes(:job_artifacts) } scope :eager_load_tags, -> { includes(:tags) } - scope :eager_load_for_archiving_trace, -> { includes(:project, :pending_state) } + scope :eager_load_for_archiving_trace, -> { preload(:project, :pending_state) } scope :eager_load_everything, -> do includes( @@ -180,7 +183,9 @@ module Ci acts_as_taggable - add_authentication_token_field :token, encrypted: :required + add_authentication_token_field :token, + encrypted: :required, + format_with_prefix: :partition_id_prefix_in_16_bit_encode after_save :stick_build_if_status_changed @@ -600,6 +605,7 @@ module Ci .concat(deploy_token_variables) .concat(harbor_variables) .concat(apple_app_store_variables) + .concat(google_play_variables) end end @@ -650,6 +656,13 @@ module Ci Gitlab::Ci::Variables::Collection.new(apple_app_store_integration.ci_variables) end + def google_play_variables + return [] unless google_play_integration.try(:activated?) + return [] unless pipeline.protected_ref? + + Gitlab::Ci::Variables::Collection.new(google_play_integration.ci_variables) + end + def features { trace_sections: true, @@ -757,9 +770,7 @@ module Ci end def remove_token! - if Feature.enabled?(:remove_job_token_on_completion, project) - update!(token_encrypted: nil) - end + update!(token_encrypted: nil) end # acts_as_taggable uses this method create/remove tags with contexts @@ -802,7 +813,7 @@ module Ci return unless project return if user&.blocked? - ActiveRecord::Associations::Preloader.new.preload([self], { runner: :tags }) + ActiveRecord::Associations::Preloader.new(records: [self], associations: { runner: :tags }).call project.execute_hooks(build_data.dup, :job_hooks) if project.has_active_hooks?(:job_hooks) project.execute_integrations(build_data.dup, :job_hooks) if project.has_active_integrations?(:job_hooks) @@ -1091,10 +1102,6 @@ module Ci ::Ci::PendingBuild.upsert_from_build!(self) end - def create_runtime_metadata! - ::Ci::RunningBuild.upsert_shared_runner_build!(self) - end - ## # We can have only one queuing entry or running build tracking entry, # because there is a unique index on `build_id` in each table, but we need @@ -1161,11 +1168,6 @@ module Ci end end - override :format_token - def format_token(token) - "#{partition_id.to_s(16)}_#{token}" - end - protected def run_status_commit_hooks! @@ -1308,6 +1310,10 @@ module Ci ).to_context] ) end + + def partition_id_prefix_in_16_bit_encode + "#{partition_id.to_s(16)}_" + end end end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index b294afd405d..4b2be446fe3 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -10,15 +10,17 @@ module Ci include Presentable include ChronicDurationAttribute include Gitlab::Utils::StrongMemoize + include IgnorableColumns self.table_name = 'p_ci_builds_metadata' self.primary_key = 'id' partitionable scope: :build + ignore_column :runner_machine_id, remove_with: '16.0', remove_after: '2023-04-22' + belongs_to :build, class_name: 'CommitStatus' belongs_to :project - belongs_to :runner_machine, class_name: 'Ci::RunnerMachine' before_create :set_build_project diff --git a/app/models/ci/build_pending_state.rb b/app/models/ci/build_pending_state.rb index 3684dac06c7..966884ae158 100644 --- a/app/models/ci/build_pending_state.rb +++ b/app/models/ci/build_pending_state.rb @@ -3,7 +3,7 @@ class Ci::BuildPendingState < Ci::ApplicationRecord include Ci::Partitionable - belongs_to :build, class_name: 'Ci::Build', foreign_key: :build_id + belongs_to :build, class_name: 'Ci::Build', foreign_key: :build_id, inverse_of: :pending_state partitionable scope: :build diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 541a8b5bffa..03b59b19ef1 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -9,7 +9,7 @@ module Ci include ::Gitlab::ExclusiveLeaseHelpers include ::Gitlab::OptimisticLocking - belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :trace_chunks partitionable scope: :build diff --git a/app/models/ci/catalog/listing.rb b/app/models/ci/catalog/listing.rb new file mode 100644 index 00000000000..92464cb645f --- /dev/null +++ b/app/models/ci/catalog/listing.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Ci + module Catalog + class Listing + # This class is the SSoT to displaying the list of resources in the + # CI/CD Catalog given a namespace as a scope. + # This model is not directly backed by a table and joins catalog resources + # with projects to return relevant data. + def initialize(namespace, current_user) + raise ArgumentError, 'Namespace is not a root namespace' unless namespace.root? + + @namespace = namespace + @current_user = current_user + end + + def resources + Ci::Catalog::Resource + .joins(:project).includes(:project) + .merge(projects_in_namespace_visible_to_user) + end + + private + + attr_reader :namespace, :current_user + + def projects_in_namespace_visible_to_user + Project + .in_namespace(namespace.self_and_descendant_ids) + .public_or_visible_to_user(current_user) + end + end + end +end diff --git a/app/models/ci/catalog/resource.rb b/app/models/ci/catalog/resource.rb new file mode 100644 index 00000000000..1b3dec5f54d --- /dev/null +++ b/app/models/ci/catalog/resource.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Ci + module Catalog + # This class represents a CI/CD Catalog resource. + # A Catalog resource is normally associated to a project. + # This model connects to the `main` database because of its + # dependency on the Project model and its need to join with that table + # in order to generate the CI/CD catalog. + class Resource < ::ApplicationRecord + self.table_name = 'catalog_resources' + + belongs_to :project + end + end +end diff --git a/app/models/ci/daily_build_group_report_result.rb b/app/models/ci/daily_build_group_report_result.rb index 598d1456a48..5ec54ee2983 100644 --- a/app/models/ci/daily_build_group_report_result.rb +++ b/app/models/ci/daily_build_group_report_result.rb @@ -4,9 +4,10 @@ module Ci class DailyBuildGroupReportResult < Ci::ApplicationRecord PARAM_TYPES = %w[coverage].freeze - belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id + belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id, + inverse_of: :daily_build_group_report_results belongs_to :project - belongs_to :group + belongs_to :group, class_name: '::Group' validates :data, json_schema: { filename: "daily_build_group_report_result_data" } diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 89a3d269a43..5a7860174ff 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -132,7 +132,7 @@ module Ci PLAN_LIMIT_PREFIX = 'ci_max_artifact_size_' belongs_to :project - belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id, inverse_of: :job_artifacts mount_file_store_uploader JobArtifactUploader, skip_store_file: true @@ -177,6 +177,8 @@ module Ci where(file_type: self.erasable_file_types) end + scope :non_trace, -> { where.not(file_type: [:trace]) } + scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) } scope :order_expired_asc, -> { order(expire_at: :asc) } diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb index 20775077bd8..f389c642fd8 100644 --- a/app/models/ci/job_token/scope.rb +++ b/app/models/ci/job_token/scope.rb @@ -58,8 +58,7 @@ module Ci end def inbound_accessible?(accessed_project) - # if the flag or setting is disabled any project is considered to be in scope. - return true unless Feature.enabled?(:ci_inbound_job_token_scope, accessed_project) + # if the setting is disabled any project is considered to be in scope. return true unless accessed_project.ci_inbound_job_token_scope_enabled? inbound_linked_as_accessible?(accessed_project) diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb index 998f0647ad5..573999995bc 100644 --- a/app/models/ci/job_variable.rb +++ b/app/models/ci/job_variable.rb @@ -7,7 +7,7 @@ module Ci include Ci::RawVariable include BulkInsertSafe - belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id, inverse_of: :job_variables partitionable scope: :job diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index bd426e02b9c..2b0c79aab87 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -11,7 +11,6 @@ module Ci include Gitlab::OptimisticLocking include Gitlab::Utils::StrongMemoize include AtomicInternalId - include EnumWithNil include Ci::HasRef include ShaAttribute include FromUnion @@ -46,7 +45,7 @@ module Ci belongs_to :project, inverse_of: :all_pipelines belongs_to :user - belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' + belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline', inverse_of: :auto_canceled_pipelines belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' belongs_to :merge_request, class_name: 'MergeRequest' belongs_to :external_pull_request @@ -67,14 +66,15 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :latest_statuses_ordered_by_stage, -> { latest.order(:stage_idx, :stage) }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :latest_statuses, -> { latest }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline - has_many :statuses_order_id_desc, -> { order_id_desc }, class_name: 'CommitStatus', foreign_key: :commit_id + has_many :statuses_order_id_desc, -> { order_id_desc }, class_name: 'CommitStatus', foreign_key: :commit_id, + inverse_of: :pipeline has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline has_many :bridges, class_name: 'Ci::Bridge', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline has_many :generic_commit_statuses, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'GenericCommitStatus' has_many :job_artifacts, through: :builds has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks - has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent + has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' has_many :latest_builds, -> { latest.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' has_many :downloadable_artifacts, -> do @@ -86,17 +86,24 @@ module Ci # Merge requests for which the current pipeline is running against # the merge request's latest commit. - has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest' + has_many :merge_requests_as_head_pipeline, foreign_key: :head_pipeline_id, class_name: 'MergeRequest', + inverse_of: :head_pipeline + has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline - has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline + has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', + inverse_of: :pipeline has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline - has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus' + has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus', + inverse_of: :pipeline has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline - has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id' - has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' - has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_pipeline_id + has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: :auto_canceled_by_id, + inverse_of: :auto_canceled_by + has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: :auto_canceled_by_id, + inverse_of: :auto_canceled_by + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_pipeline_id, + inverse_of: :source_pipeline has_one :source_pipeline, class_name: 'Ci::Sources::Pipeline', inverse_of: :pipeline @@ -114,7 +121,9 @@ module Ci has_one :pipeline_metadata, class_name: 'Ci::PipelineMetadata', inverse_of: :pipeline - has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id + has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', + foreign_key: :last_pipeline_id, inverse_of: :last_pipeline + has_many :latest_builds_report_results, through: :latest_builds, source: :report_results has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :pipeline, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -143,9 +152,9 @@ module Ci # We use `Enums::Ci::Pipeline.sources` here so that EE can more easily extend # this `Hash` with new values. - enum_with_nil source: Enums::Ci::Pipeline.sources + enum source: Enums::Ci::Pipeline.sources - enum_with_nil config_source: Enums::Ci::Pipeline.config_sources + enum config_source: Enums::Ci::Pipeline.config_sources # We use `Enums::Ci::Pipeline.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. @@ -336,6 +345,22 @@ module Ci AutoDevops::DisableWorker.perform_async(pipeline.id) if pipeline.auto_devops_source? end end + + after_transition any => [:running, *::Ci::Pipeline.completed_statuses] do |pipeline| + project = pipeline&.project + + next unless project + next unless Feature.enabled?(:pipeline_trigger_merge_status, project) + + pipeline.run_after_commit do + next if pipeline.child? + next unless project.only_allow_merge_if_pipeline_succeeds?(inherit_group_setting: true) + + pipeline.all_merge_requests.opened.each do |merge_request| + GraphqlTriggers.merge_request_merge_status_updated(merge_request) + end + end + end end scope :internal, -> { where(source: internal_sources) } @@ -1282,7 +1307,7 @@ module Ci types_to_collect = report_types.empty? ? ::Ci::JobArtifact::SECURITY_REPORT_FILE_TYPES : report_types ::Gitlab::Ci::Reports::Security::Reports.new(self).tap do |security_reports| - latest_report_builds(reports_scope).each do |build| + latest_report_builds_in_self_and_project_descendants(reports_scope).includes(pipeline: { project: :route }).each do |build| # rubocop:disable Rails/FindEach build.collect_security_reports!(security_reports, report_types: types_to_collect) end end diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 20ff07e88ba..83e6fa2f862 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -8,14 +8,15 @@ module Ci include CronSchedulable include Limitable include EachBatch + include BatchNullifyDependentAssociations self.limit_name = 'ci_pipeline_schedules' self.limit_scope = :project belongs_to :project belongs_to :owner, class_name: 'User' - has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline' - has_many :pipelines + has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline', inverse_of: :pipeline_schedule + has_many :pipelines, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineScheduleVariable' validates :cron, unless: :importing?, cron: true, presence: { unless: :importing? } @@ -81,6 +82,12 @@ module Ci def worker_cron_expression Settings.cron_jobs['pipeline_schedule_worker']['cron'] end + + def destroy + nullify_dependent_associations_in_batches + + super + end end end diff --git a/app/models/ci/resource_group.rb b/app/models/ci/resource_group.rb index b788e4f58c1..a220aa7bb18 100644 --- a/app/models/ci/resource_group.rb +++ b/app/models/ci/resource_group.rb @@ -29,13 +29,19 @@ module Ci partition_id: processable.partition_id } - resources.free.limit(1).update_all(attrs) > 0 + success = resources.free.limit(1).update_all(attrs) > 0 + log_event(success: success, processable: processable, action: "assign resource to processable") + + success end def release_resource_from(processable) attrs = { build_id: nil, partition_id: nil } - resources.retained_by(processable).update_all(attrs) > 0 + success = resources.retained_by(processable).update_all(attrs) > 0 + log_event(success: success, processable: processable, action: "release resource from processable") + + success end def upcoming_processables @@ -72,5 +78,14 @@ module Ci # belong to the same resource group are executed once at time. self.resources.build if self.resources.empty? end + + def log_event(success:, processable:, action:) + Gitlab::Ci::ResourceGroups::Logger.build.info({ + resource_group_id: self.id, + processable_id: processable.id, + message: "attempted to #{action}", + success: success + }) + end end end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 09ac0fa69e7..6fefe95769b 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -17,7 +17,10 @@ module Ci extend ::Gitlab::Utils::Override - add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration + add_authentication_token_field :token, + encrypted: :optional, + expires_at: :compute_token_expiration, + format_with_prefix: :prefix_for_new_and_legacy_runner enum access_level: { not_protected: 0, @@ -54,6 +57,9 @@ module Ci # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner will be considered stale STALE_TIMEOUT = 3.months + # Only allow authentication token to be visible for a short while + REGISTRATION_AVAILABILITY_TIME = 1.hour + AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze AVAILABLE_TYPES = runner_types.keys.freeze AVAILABLE_STATUSES = %w[active paused online offline never_contacted stale].freeze # TODO: Remove in %16.0: active, paused. Relevant issue: https://gitlab.com/gitlab-org/gitlab/-/issues/344648 @@ -81,8 +87,13 @@ module Ci scope :active, -> (value = true) { where(active: value) } scope :paused, -> { active(false) } scope :online, -> { where('contacted_at > ?', online_contact_time_deadline) } - scope :recent, -> { where('ci_runners.created_at >= :date OR ci_runners.contacted_at >= :date', date: stale_deadline) } - scope :stale, -> { where('ci_runners.created_at < :date AND (ci_runners.contacted_at IS NULL OR ci_runners.contacted_at < :date)', date: stale_deadline) } + scope :recent, -> do + where('ci_runners.created_at >= :datetime OR ci_runners.contacted_at >= :datetime', datetime: stale_deadline) + end + scope :stale, -> do + where('ci_runners.created_at <= :datetime AND ' \ + '(ci_runners.contacted_at IS NULL OR ci_runners.contacted_at <= :datetime)', datetime: stale_deadline) + end scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) } scope :never_contacted, -> { where(contacted_at: nil) } scope :ordered, -> { order(id: :desc) } @@ -185,6 +196,7 @@ module Ci scope :order_token_expires_at_asc, -> { order(token_expires_at: :asc) } scope :order_token_expires_at_desc, -> { order(token_expires_at: :desc) } scope :with_tags, -> { preload(:tags) } + scope :with_creator, -> { preload(:creator) } validate :tag_constraints validates :access_level, presence: true @@ -332,7 +344,7 @@ module Ci def stale? return false unless created_at - [created_at, contacted_at].compact.max < self.class.stale_deadline + [created_at, contacted_at].compact.max <= self.class.stale_deadline end def status(legacy_mode = nil) @@ -434,7 +446,7 @@ module Ci ensure_runner_queue_value == value if value.present? end - def heartbeat(values) + def heartbeat(values, update_contacted_at: true) ## # We can safely ignore writes performed by a runner heartbeat. We do # not want to upgrade database connection proxy to use the primary @@ -442,20 +454,18 @@ module Ci # ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} - values[:contacted_at] = Time.current + values[:contacted_at] = Time.current if update_contacted_at if values.include?(:executor) values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) end - cache_attributes(values) + new_version = values[:version] + schedule_runner_version_update(new_version) if new_version && values[:version] != version - # We save data without validation, it will always change due to `contacted_at` - if persist_cached_data? - version_updated = values.include?(:version) && values[:version] != version + merge_cache_attributes(values) - update_columns(values) - schedule_runner_version_update if version_updated - end + # We save data without validation, it will always change due to `contacted_at` + update_columns(values) if persist_cached_data? end end @@ -488,17 +498,16 @@ module Ci end end - override :format_token - def format_token(token) - return token if registration_token_registration_type? - - "#{CREATED_RUNNER_TOKEN_PREFIX}#{token}" - end - def ensure_machine(system_xid, &blk) RunnerMachine.safe_find_or_create_by!(runner_id: id, system_xid: system_xid.to_s, &blk) # rubocop: disable Performance/ActiveRecordSubtransactionMethods end + def registration_available? + authenticated_user_registration_type? && + created_at > REGISTRATION_AVAILABILITY_TIME.ago && + !runner_machines.any? + end + private scope :with_upgrade_status, ->(upgrade_status) do @@ -594,10 +603,16 @@ module Ci # TODO Remove in 16.0 when runners are known to send a system_id # For now, heartbeats with version updates might result in two Sidekiq jobs being queued if a runner has a system_id # This is not a problem since the jobs are deduplicated on the version - def schedule_runner_version_update - return unless version + def schedule_runner_version_update(new_version) + return unless new_version && Gitlab::Ci::RunnerReleases.instance.enabled? + + Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(new_version) + end + + def prefix_for_new_and_legacy_runner + return if registration_token_registration_type? - Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(version) + CREATED_RUNNER_TOKEN_PREFIX end end end diff --git a/app/models/ci/runner_machine.rb b/app/models/ci/runner_machine.rb index e52659a011f..8cf395aadb4 100644 --- a/app/models/ci/runner_machine.rb +++ b/app/models/ci/runner_machine.rb @@ -5,17 +5,14 @@ module Ci include FromUnion include RedisCacheable include Ci::HasRunnerExecutor - include IgnorableColumns - - ignore_column :machine_xid, remove_with: '15.11', remove_after: '2022-03-22' # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner Machine DB entry can be updated - UPDATE_CONTACT_COLUMN_EVERY = 40.minutes..55.minutes + UPDATE_CONTACT_COLUMN_EVERY = (40.minutes)..(55.minutes) belongs_to :runner - has_many :build_metadata, class_name: 'Ci::BuildMetadata' - has_many :builds, through: :build_metadata, class_name: 'Ci::Build' + has_many :runner_machine_builds, inverse_of: :runner_machine, class_name: 'Ci::RunnerMachineBuild' + has_many :builds, through: :runner_machine_builds, class_name: 'Ci::Build' belongs_to :runner_version, inverse_of: :runner_machines, primary_key: :version, foreign_key: :version, class_name: 'Ci::RunnerVersion' @@ -44,7 +41,15 @@ module Ci remove_duplicates: false).where(created_some_time_ago) end - def heartbeat(values) + def self.online_contact_time_deadline + Ci::Runner.online_contact_time_deadline + end + + def self.stale_deadline + STALE_TIMEOUT.ago + end + + def heartbeat(values, update_contacted_at: true) ## # We can safely ignore writes performed by a runner heartbeat. We do # not want to upgrade database connection proxy to use the primary @@ -52,24 +57,40 @@ module Ci # ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} - values[:contacted_at] = Time.current + values[:contacted_at] = Time.current if update_contacted_at if values.include?(:executor) values[:executor_type] = Ci::Runner::EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) end - version_changed = values.include?(:version) && values[:version] != version - - cache_attributes(values) + new_version = values[:version] + schedule_runner_version_update(new_version) if new_version && values[:version] != version - schedule_runner_version_update if version_changed + merge_cache_attributes(values) # We save data without validation, it will always change due to `contacted_at` update_columns(values) if persist_cached_data? end end + def status + return :stale if stale? + return :never_contacted unless contacted_at + + online? ? :online : :offline + end + private + def online? + contacted_at && contacted_at > self.class.online_contact_time_deadline + end + + def stale? + return false unless created_at + + [created_at, contacted_at].compact.max <= self.class.stale_deadline + end + def persist_cached_data? # Use a random threshold to prevent beating DB updates. contacted_at_max_age = Random.rand(UPDATE_CONTACT_COLUMN_EVERY) @@ -79,10 +100,10 @@ module Ci (Time.current - real_contacted_at) >= contacted_at_max_age end - def schedule_runner_version_update - return unless version + def schedule_runner_version_update(new_version) + return unless new_version && Gitlab::Ci::RunnerReleases.instance.enabled? - Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(version) + Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(new_version) end end end diff --git a/app/models/ci/runner_machine_build.rb b/app/models/ci/runner_machine_build.rb new file mode 100644 index 00000000000..d4f2c403337 --- /dev/null +++ b/app/models/ci/runner_machine_build.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Ci + class RunnerMachineBuild < Ci::ApplicationRecord + include Ci::Partitionable + + self.table_name = :p_ci_runner_machine_builds + self.primary_key = :build_id + + partitionable scope: :build, partitioned: true + + belongs_to :build, inverse_of: :runner_machine_build, class_name: 'Ci::Build' + belongs_to :runner_machine, inverse_of: :runner_machine_builds, class_name: 'Ci::RunnerMachine' + + validates :build, presence: true + validates :runner_machine, presence: true + + scope :for_build, ->(build_id) { where(build_id: build_id) } + + def self.pluck_build_id_and_runner_machine_id + select(:build_id, :runner_machine_id) + .pluck(:build_id, :runner_machine_id) + .to_h + end + end +end diff --git a/app/models/ci/runner_version.rb b/app/models/ci/runner_version.rb index ec42f46b165..41e7a2b8e8a 100644 --- a/app/models/ci/runner_version.rb +++ b/app/models/ci/runner_version.rb @@ -3,9 +3,8 @@ module Ci class RunnerVersion < Ci::ApplicationRecord include EachBatch - include EnumWithNil - enum_with_nil status: { + enum status: { not_processed: nil, invalid_version: -1, unavailable: 1, diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index 855e68d1db1..719d19f4169 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -10,6 +10,7 @@ module Ci belongs_to :project, class_name: "::Project" belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :source_pipeline + belongs_to :build, class_name: "Ci::Build", foreign_key: :source_job_id, inverse_of: :sourced_pipelines belongs_to :source_project, class_name: "::Project", foreign_key: :source_project_id belongs_to :source_job, class_name: "CommitStatus", foreign_key: :source_job_id diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 46a9e3f6494..02093bdf153 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -27,6 +27,7 @@ module Ci has_many :processables, class_name: 'Ci::Processable', foreign_key: :stage_id, inverse_of: :ci_stage has_many :builds, foreign_key: :stage_id, inverse_of: :ci_stage has_many :bridges, foreign_key: :stage_id, inverse_of: :ci_stage + has_many :generic_commit_statuses, foreign_key: :stage_id, inverse_of: :ci_stage scope :ordered, -> { order(position: :asc) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } @@ -117,6 +118,7 @@ module Ci end end + # This will be removed with ci_remove_ensure_stage_service def update_legacy_status set_status(latest_stage_status.to_s) end @@ -150,6 +152,7 @@ module Ci blocked? || skipped? end + # This will be removed with ci_remove_ensure_stage_service def latest_stage_status statuses.latest.composite_status || 'skipped' end diff --git a/app/models/clusters/applications/crossplane.rb b/app/models/clusters/applications/crossplane.rb deleted file mode 100644 index a7b4fb57149..00000000000 --- a/app/models/clusters/applications/crossplane.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - # DEPRECATED for removal in %14.0 - # See https://gitlab.com/groups/gitlab-org/-/epics/4280 - class Crossplane < ApplicationRecord - VERSION = '0.4.1' - - self.table_name = 'clusters_applications_crossplane' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - - attribute :version, default: VERSION - attribute :stack, default: "" - - validates :stack, presence: true - - def chart - 'crossplane/crossplane' - end - - def repository - 'https://charts.crossplane.io/alpha' - end - - def install_command - helm_command_module::InstallCommand.new( - name: 'crossplane', - repository: repository, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files - ) - end - - def values - crossplane_values.to_yaml - end - - private - - def crossplane_values - { - "clusterStacks" => { - self.stack => { - "deploy" => true - } - } - } - end - end - end -end diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb index 64366594583..c8c043f3312 100644 --- a/app/models/clusters/applications/knative.rb +++ b/app/models/clusters/applications/knative.rb @@ -13,8 +13,6 @@ module Clusters self.table_name = 'clusters_applications_knative' - has_one :serverless_domain_cluster, class_name: '::Serverless::DomainCluster', foreign_key: 'clusters_applications_knative_id', inverse_of: :knative - include ::Clusters::Concerns::ApplicationCore include ::Clusters::Concerns::ApplicationStatus include ::Clusters::Concerns::ApplicationVersion @@ -49,8 +47,6 @@ module Clusters scope :for_cluster, -> (cluster) { where(cluster: cluster) } - has_one :pages_domain, through: :serverless_domain_cluster - def chart 'knative/knative' end @@ -140,16 +136,14 @@ module Clusters @api_groups ||= YAML.safe_load(File.read(Rails.root.join(API_GROUPS_PATH))) end + # Relied on application_prometheus which is now removed def install_knative_metrics - return [] unless cluster.application_prometheus&.available? - - [Gitlab::Kubernetes::KubectlCmd.apply_file(METRICS_CONFIG)] + [] end + # Relied on application_prometheus which is now removed def delete_knative_istio_metrics - return [] unless cluster.application_prometheus&.available? - - [Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "-f", METRICS_CONFIG)] + [] end end end diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb deleted file mode 100644 index a076c871824..00000000000 --- a/app/models/clusters/applications/prometheus.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - class Prometheus < ApplicationRecord - include ::Clusters::Concerns::PrometheusClient - - VERSION = '10.4.1' - - self.table_name = 'clusters_applications_prometheus' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - include AfterCommitQueue - - attribute :version, default: VERSION - - scope :preload_cluster_platform, -> { preload(cluster: [:platform_kubernetes]) } - - attr_encrypted :alert_manager_token, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm' - - after_initialize :set_alert_manager_token, if: :new_record? - - after_destroy do - cluster.find_or_build_integration_prometheus.destroy - end - - state_machine :status do - after_transition any => [:installed, :externally_installed] do |application| - application.cluster.find_or_build_integration_prometheus.update(enabled: true, alert_manager_token: application.alert_manager_token) - end - - after_transition any => :updating do |application| - application.update(last_update_started_at: Time.current) - end - end - - def managed_prometheus? - !externally_installed? && !uninstalled? - end - - def updated_since?(timestamp) - last_update_started_at && - last_update_started_at > timestamp && - !update_errored? - end - - def chart - "#{name}/prometheus" - end - - def repository - 'https://gitlab-org.gitlab.io/cluster-integration/helm-stable-archive' - end - - def install_command - helm_command_module::InstallCommand.new( - name: name, - repository: repository, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files, - postinstall: install_knative_metrics - ) - end - - # Deprecated, to be removed in %14.0 as part of https://gitlab.com/groups/gitlab-org/-/epics/4280 - def patch_command(values) - helm_command_module::PatchCommand.new( - name: name, - repository: repository, - version: version, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files_with_replaced_values(values) - ) - end - - def uninstall_command - helm_command_module::DeleteCommand.new( - name: name, - rbac: cluster.platform_kubernetes_rbac?, - files: files, - predelete: delete_knative_istio_metrics - ) - end - - # Returns a copy of files where the values of 'values.yaml' - # are replaced by the argument. - # - # See #values for the data format required - def files_with_replaced_values(replaced_values) - files.merge('values.yaml': replaced_values) - end - - private - - def set_alert_manager_token - self.alert_manager_token = SecureRandom.hex - end - - def install_knative_metrics - return [] unless cluster.application_knative_available? - - [Gitlab::Kubernetes::KubectlCmd.apply_file(Clusters::Applications::Knative::METRICS_CONFIG)] - end - - def delete_knative_istio_metrics - return [] unless cluster.application_knative_available? - - [ - Gitlab::Kubernetes::KubectlCmd.delete( - "-f", Clusters::Applications::Knative::METRICS_CONFIG, - "--ignore-not-found" - ) - ] - end - end - end -end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index a35ea6ddb46..5cd11265808 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -14,8 +14,6 @@ module Clusters APPLICATIONS = { Clusters::Applications::Helm.application_name => Clusters::Applications::Helm, Clusters::Applications::Ingress.application_name => Clusters::Applications::Ingress, - Clusters::Applications::Crossplane.application_name => Clusters::Applications::Crossplane, - Clusters::Applications::Prometheus.application_name => Clusters::Applications::Prometheus, Clusters::Applications::Runner.application_name => Clusters::Applications::Runner, Clusters::Applications::Jupyter.application_name => Clusters::Applications::Jupyter, Clusters::Applications::Knative.application_name => Clusters::Applications::Knative @@ -56,8 +54,6 @@ module Clusters has_one_cluster_application :helm has_one_cluster_application :ingress - has_one_cluster_application :crossplane - has_one_cluster_application :prometheus has_one_cluster_application :runner has_one_cluster_application :jupyter has_one_cluster_application :knative @@ -365,12 +361,6 @@ module Clusters end end - def serverless_domain - strong_memoize(:serverless_domain) do - self.application_knative&.serverless_domain_cluster - end - end - def prometheus_adapter integration_prometheus end diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 165285b34b2..123ad0ebfaf 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -4,7 +4,6 @@ module Clusters module Platforms class Kubernetes < ApplicationRecord include Gitlab::Kubernetes - include EnumWithNil include AfterCommitQueue include ReactiveCaching include NullifyIfBlank @@ -63,7 +62,7 @@ module Clusters alias_attribute :ca_pem, :ca_cert - enum_with_nil authorization_type: { + enum authorization_type: { unknown_authorization: nil, rbac: 1, abac: 2 diff --git a/app/models/commit.rb b/app/models/commit.rb index 4517b3ef216..ea90b4e4dda 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -206,7 +206,8 @@ class Commit def self.link_reference_pattern @link_reference_pattern ||= - super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})?(\.(?<extension>#{LINK_EXTENSION_PATTERN}))?/o) + compose_link_reference_pattern('commit', + /(?<commit>#{COMMIT_SHA_PATTERN})?(\.(?<extension>#{LINK_EXTENSION_PATTERN}))?/o) end def to_reference(from = nil, full: false) diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb index 47ecdfa8574..eb7db0fc9b4 100644 --- a/app/models/commit_collection.rb +++ b/app/models/commit_collection.rb @@ -118,4 +118,21 @@ class CommitCollection def next_page @pagination.next_page end + + def load_tags + oids = commits.map(&:id) + references = repository.list_refs([Gitlab::Git::TAG_REF_PREFIX], pointing_at_oids: oids, peel_tags: true) + oid_to_references = references.group_by { |reference| reference.peeled_target.presence || reference.target } + + return self if oid_to_references.empty? + + commits.each do |commit| + grouped_references = oid_to_references[commit.id] + next unless grouped_references + + commit.referenced_by = grouped_references.map(&:name) + end + + self + end end diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index 87029cb2033..90cdd267cbd 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -50,7 +50,7 @@ class CommitRange end def self.link_reference_pattern - @link_reference_pattern ||= super("compare", /(?<commit_range>#{PATTERN})/o) + @link_reference_pattern ||= compose_link_reference_pattern('compare', /(?<commit_range>#{PATTERN})/o) end # Initialize a CommitRange diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 333a176b8f3..716be080851 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -6,17 +6,17 @@ class CommitStatus < Ci::ApplicationRecord include Importable include AfterCommitQueue include Presentable - include EnumWithNil include BulkInsertableAssociations include TaggableQueries self.table_name = 'ci_builds' + self.primary_key = :id partitionable scope: :pipeline belongs_to :user belongs_to :project - belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id - belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' + belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, inverse_of: :statuses + belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline', inverse_of: :auto_canceled_jobs belongs_to :ci_stage, class_name: 'Ci::Stage', foreign_key: :stage_id has_many :needs, class_name: 'Ci::BuildNeed', foreign_key: :build_id, inverse_of: :build @@ -26,7 +26,7 @@ class CommitStatus < Ci::ApplicationRecord enum scheduling_type: { stage: 0, dag: 1 }, _prefix: true # We use `Enums::Ci::CommitStatus.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. - enum_with_nil failure_reason: Enums::Ci::CommitStatus.failure_reasons + enum failure_reason: Enums::Ci::CommitStatus.failure_reasons delegate :commit, to: :pipeline delegate :sha, :short_sha, :before_sha, to: :pipeline @@ -43,14 +43,6 @@ class CommitStatus < Ci::ApplicationRecord scope :order_id_desc, -> { order(id: :desc) } - scope :exclude_ignored, -> do - # We want to ignore failed but allowed to fail jobs. - # - # TODO, we also skip ignored optional manual actions. - where("allow_failure = ? OR status IN (?)", - false, all_state_names - [:failed, :canceled, :manual]) - end - scope :latest, -> { where(retried: [false, nil]) } scope :retried, -> { where(retried: true) } scope :ordered, -> { order(:name) } @@ -239,10 +231,6 @@ class CommitStatus < Ci::ApplicationRecord name.to_s.sub(regex, '').strip end - def failed_but_allowed? - allow_failure? && (failed? || canceled?) - end - # Time spent running. def duration calculate_duration(started_at, finished_at) diff --git a/app/models/concerns/analytics/cycle_analytics/stageable.rb b/app/models/concerns/analytics/cycle_analytics/stageable.rb index caac4f31e1a..d1dd46883e3 100644 --- a/app/models/concerns/analytics/cycle_analytics/stageable.rb +++ b/app/models/concerns/analytics/cycle_analytics/stageable.rb @@ -7,8 +7,8 @@ module Analytics include Gitlab::Utils::StrongMemoize included do - belongs_to :start_event_label, class_name: 'GroupLabel', optional: true - belongs_to :end_event_label, class_name: 'GroupLabel', optional: true + belongs_to :start_event_label, class_name: 'Label', optional: true + belongs_to :end_event_label, class_name: 'Label', optional: true belongs_to :stage_event_hash, class_name: 'Analytics::CycleAnalytics::StageEventHash', optional: true validates :name, presence: true @@ -119,10 +119,11 @@ module Analytics end def label_available_for_namespace?(label_id) - subject = is_a?(::Analytics::CycleAnalytics::Stage) ? namespace : project.group + subject = namespace.is_a?(Namespaces::ProjectNamespace) ? namespace.project.group : namespace return unless subject - LabelsFinder.new(nil, { group_id: subject.id, include_ancestor_groups: true, only_group_labels: true }) + LabelsFinder.new(nil, + { group_id: subject.id, include_ancestor_groups: true, only_group_labels: namespace.is_a?(Group) }) .execute(skip_authorization: true) .id_in(label_id) .exists? diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 14be924f9da..ec4ee7985fe 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -61,6 +61,8 @@ module AtomicInternalId AtomicInternalId.project_init(self) when :group AtomicInternalId.group_init(self) + when :namespace + AtomicInternalId.namespace_init(self) else # We require init here to retain the ability to recalculate in the absence of a # InternalId record (we may delete records in `internal_ids` for example). @@ -241,6 +243,16 @@ module AtomicInternalId end end + def self.namespace_init(klass, column_name = :iid) + ->(instance, scope) do + if instance + klass.where(namespace_id: instance.namespace_id).maximum(column_name) + elsif scope.present? + klass.where(**scope).maximum(column_name) + end + end + end + def internal_id_read_scope(scope) association(scope).reader end diff --git a/app/models/concerns/cached_commit.rb b/app/models/concerns/cached_commit.rb index 0fb72552dd5..8a53fec0612 100644 --- a/app/models/concerns/cached_commit.rb +++ b/app/models/concerns/cached_commit.rb @@ -14,4 +14,9 @@ module CachedCommit def parent_ids [] end + + # These are not saved + def referenced_by + [] + end end diff --git a/app/models/concerns/cascading_namespace_setting_attribute.rb b/app/models/concerns/cascading_namespace_setting_attribute.rb index 731729a1ed5..d0ee4f33ce6 100644 --- a/app/models/concerns/cascading_namespace_setting_attribute.rb +++ b/app/models/concerns/cascading_namespace_setting_attribute.rb @@ -57,11 +57,13 @@ module CascadingNamespaceSettingAttribute # private methods define_validator_methods(attribute) + define_attr_before_save(attribute) define_after_update(attribute) validate :"#{attribute}_changeable?" validate :"lock_#{attribute}_changeable?" + before_save :"before_save_#{attribute}", if: -> { will_save_change_to_attribute?(attribute) } after_update :"clear_descendant_#{attribute}_locks", if: -> { saved_change_to_attribute?("lock_#{attribute}", to: true) } end end @@ -92,13 +94,26 @@ module CascadingNamespaceSettingAttribute def define_attr_writer(attribute) define_method("#{attribute}=") do |value| - return value if value == cascaded_ancestor_value(attribute) + return value if read_attribute(attribute).nil? && to_bool(value) == cascaded_ancestor_value(attribute) clear_memoization(attribute) super(value) end end + def define_attr_before_save(attribute) + # rubocop:disable GitlabSecurity/PublicSend + define_method("before_save_#{attribute}") do + new_value = public_send(attribute) + if public_send("#{attribute}_was").nil? && new_value == cascaded_ancestor_value(attribute) + write_attribute(attribute, nil) + end + end + # rubocop:enable GitlabSecurity/PublicSend + + private :"before_save_#{attribute}" + end + def define_lock_attr_writer(attribute) define_method("lock_#{attribute}=") do |value| attr_value = public_send(attribute) # rubocop:disable GitlabSecurity/PublicSend @@ -239,4 +254,8 @@ module CascadingNamespaceSettingAttribute namespace.descendants.pluck(:id) end end + + def to_bool(value) + ActiveModel::Type::Boolean.new.cast(value) + end end diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index 9a04776f1c6..2971ecb04b8 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -13,7 +13,7 @@ module Ci STOPPED_STATUSES = COMPLETED_STATUSES + BLOCKED_STATUS ORDERED_STATUSES = %w[failed preparing pending running waiting_for_resource manual scheduled canceled success skipped created].freeze PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze - EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze + IGNORED_STATUSES = %w[manual].to_set.freeze ALIVE_STATUSES = (ACTIVE_STATUSES + ['created']).freeze CANCELABLE_STATUSES = (ALIVE_STATUSES + ['scheduled']).freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, @@ -23,6 +23,7 @@ module Ci UnknownStatusError = Class.new(StandardError) class_methods do + # This will be removed with ci_remove_ensure_stage_service def composite_status Gitlab::Ci::Status::Composite .new(all, with_allow_failure: columns_hash.key?('allow_failure')) diff --git a/app/models/concerns/ci/partitionable.rb b/app/models/concerns/ci/partitionable.rb index d6ba0f4488f..28cc17432bc 100644 --- a/app/models/concerns/ci/partitionable.rb +++ b/app/models/concerns/ci/partitionable.rb @@ -36,6 +36,7 @@ module Ci Ci::Pipeline Ci::PendingBuild Ci::RunningBuild + Ci::RunnerMachineBuild Ci::PipelineVariable Ci::Sources::Pipeline Ci::Stage @@ -70,8 +71,8 @@ module Ci class_methods do def partitionable(scope:, through: nil, partitioned: false) handle_partitionable_through(through) - handle_partitionable_dml(partitioned) handle_partitionable_scope(scope) + handle_partitionable_ddl(partitioned) end private @@ -85,13 +86,6 @@ module Ci include Partitionable::Switch end - def handle_partitionable_dml(partitioned) - define_singleton_method(:partitioned?) { partitioned } - return unless partitioned - - include Partitionable::PartitionedFilter - end - def handle_partitionable_scope(scope) define_method(:partition_scope_value) do strong_memoize(:partition_scope_value) do @@ -102,6 +96,17 @@ module Ci end end end + + def handle_partitionable_ddl(partitioned) + return unless partitioned + + include ::PartitionedTable + + partitioned_by :partition_id, + strategy: :ci_sliding_list, + next_partition_if: proc { false }, + detach_partition_if: proc { false } + end end end end diff --git a/app/models/concerns/ci/partitionable/partitioned_filter.rb b/app/models/concerns/ci/partitionable/partitioned_filter.rb deleted file mode 100644 index 4adae3be26a..00000000000 --- a/app/models/concerns/ci/partitionable/partitioned_filter.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Ci - module Partitionable - # Used to patch the save, update, delete, destroy methods to use the - # partition_id attributes for their SQL queries. - module PartitionedFilter - extend ActiveSupport::Concern - - if Rails::VERSION::MAJOR >= 7 - # These methods are updated in Rails 7 to use `_primary_key_constraints_hash` - # by default, so this patch will no longer be required. - # - # rubocop:disable Gitlab/NoCodeCoverageComment - # :nocov: - raise "`#{__FILE__}` should be double checked" if Rails.env.test? - - warn "Update `#{__FILE__}`. Patches Rails internals for partitioning" - # :nocov: - # rubocop:enable Gitlab/NoCodeCoverageComment - else - def _update_row(attribute_names, attempted_action = "update") - self.class._update_record( - attributes_with_values(attribute_names), - _primary_key_constraints_hash - ) - end - - def _delete_row - self.class._delete_record(_primary_key_constraints_hash) - end - end - - # Introduced in Rails 7, but updated to include `partition_id` filter. - # https://github.com/rails/rails/blob/a4dbb153fd390ac31bb9808809e7ac4d3a2c5116/activerecord/lib/active_record/persistence.rb#L1031-L1033 - def _primary_key_constraints_hash - { @primary_key => id_in_database, partition_id: partition_id } # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - end - end -end diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index 58ea57962c5..d7ee533b53c 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -5,7 +5,7 @@ # after a period of time (10 minutes). # When an attribute is incremented by a value, the increment is added # to a Redis key. Then, FlushCounterIncrementsWorker will execute -# `flush_increments_to_database!` which removes increments from Redis for a +# `commit_increment!` which removes increments from Redis for a # given model attribute and updates the values in the database. # # @example: @@ -29,8 +29,24 @@ # counter_attribute :conditional_one, if: -> { |object| object.use_counter_attribute? } # end # +# The `counter_attribute` by default will return last persisted value. +# It's possible to always return accurate (real) value instead by using `returns_current: true`. +# While doing this the `counter_attribute` will overwrite attribute accessor to fetch +# the buffered information added to the last persisted value. This will incur cost a Redis call per attribute fetched. +# +# @example: +# +# class ProjectStatistics +# include CounterAttribute +# +# counter_attribute :commit_count, returns_current: true +# end +# +# in that case +# model.commit_count => persisted value + buffered amount to be added +# # To increment the counter we can use the method: -# increment_counter(:commit_count, 3) +# increment_amount(:commit_count, 3) # # This method would determine whether it would increment the counter using Redis, # or fallback to legacy increment on ActiveRecord counters. @@ -50,11 +66,22 @@ module CounterAttribute include Gitlab::Utils::StrongMemoize class_methods do - def counter_attribute(attribute, if: nil) + def counter_attribute(attribute, if: nil, returns_current: false) counter_attributes << { attribute: attribute, - if_proc: binding.local_variable_get(:if) # can't read `if` directly + if_proc: binding.local_variable_get(:if), # can't read `if` directly + returns_current: returns_current } + + if returns_current + define_method(attribute) do + current_counter(attribute) + end + end + + define_method("increment_#{attribute}") do |amount| + increment_amount(attribute, amount) + end end def counter_attributes @@ -87,6 +114,15 @@ module CounterAttribute end end + def increment_amount(attribute, amount) + counter = Gitlab::Counters::Increment.new(amount: amount) + increment_counter(attribute, counter) + end + + def current_counter(attribute) + read_attribute(attribute) + counter(attribute).get + end + def increment_counter(attribute, increment) return if increment.amount == 0 @@ -172,7 +208,8 @@ module CounterAttribute Gitlab::AppLogger.info( message: 'Acquiring lease for project statistics update', - project_statistics_id: id, + model: self.class.name, + model_id: id, project_id: project.id, **log_fields, **Gitlab::ApplicationContext.current @@ -184,7 +221,8 @@ module CounterAttribute rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError Gitlab::AppLogger.warn( message: 'Concurrent project statistics update detected', - project_statistics_id: id, + model: self.class.name, + model_id: id, project_id: project.id, **log_fields, **Gitlab::ApplicationContext.current diff --git a/app/models/concerns/each_batch.rb b/app/models/concerns/each_batch.rb index dbc0887dc97..79fb81e7820 100644 --- a/app/models/concerns/each_batch.rb +++ b/app/models/concerns/each_batch.rb @@ -161,5 +161,81 @@ module EachBatch break unless stop end end + + # Iterates over the relation and counts the rows. The counting + # logic is combined with the iteration query which saves one query + # compared to a standard each_batch approach. + # + # Basic usage: + # count, _last_value = Project.each_batch_count + # + # The counting can be stopped by passing a block and making the last statement true. + # Example: + # + # query_count = 0 + # count, last_value = Project.each_batch_count do + # query_count += 1 + # query_count == 5 # stop counting after 5 loops + # end + # + # Resume where the previous counting has stopped: + # + # count, last_value = Project.each_batch_count(last_count: count, last_value: last_value) + # + # Another example, counting issues in project: + # + # project = Project.find(1) + # count, _ = project.issues.each_batch_count(column: :iid) + def each_batch_count(of: 1000, column: :id, last_count: 0, last_value: nil) + arel_table = self.arel_table + window = Arel::Nodes::Window.new.order(arel_table[column]) + last_value_column = Arel::Nodes::NamedFunction + .new('LAST_VALUE', [arel_table[column]]) + .over(window) + .as(column.to_s) + + loop do + count_column = Arel::Nodes::Addition + .new(Arel::Nodes::NamedFunction.new('ROW_NUMBER', []).over(window), last_count) + .as('count') + + projections = [count_column, last_value_column] + scope = limit(1).offset(of - 1) + scope = scope.where(arel_table[column].gt(last_value)) if last_value + new_count, last_value = scope.pick(*projections) + + # When reaching the last batch the offset query might return no data, to address this + # problem, we invoke a specialized query that takes the last row out of the resultset. + # We could do this for each batch, however it would add unnecessary overhead to all + # queries. + if new_count.nil? + inner_query = scope + .select(*projections) + .limit(nil) + .offset(nil) + .arel + .as(quoted_table_name) + + new_count, last_value = + unscoped + .from(inner_query) + .order(count: :desc) + .limit(1) + .pick(:count, column) + + last_count = new_count if new_count + last_value = nil + break + end + + last_count = new_count + + if block_given? + should_break = yield(last_count, last_value) + break if should_break + end + end + [last_count, last_value] + end end end diff --git a/app/models/concerns/enum_with_nil.rb b/app/models/concerns/enum_with_nil.rb deleted file mode 100644 index c66942025d7..00000000000 --- a/app/models/concerns/enum_with_nil.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module EnumWithNil - extend ActiveSupport::Concern - - included do - def self.enum_with_nil(definitions) - # use original `enum` to auto-define all methods - enum(definitions) - - # override auto-defined methods only for the - # key which uses nil value - definitions.each do |name, values| - # E.g. for enum_with_nil failure_reason: { unknown_failure: nil } - # this overrides auto-generated method `failure_reason` - define_method(name) do - orig = super() - - return orig unless orig.nil? - - self.class.public_send(name.to_s.pluralize).key(nil) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end -end diff --git a/app/models/concerns/has_unique_internal_users.rb b/app/models/concerns/has_unique_internal_users.rb index 4d60cfa03b0..25b56f6d70f 100644 --- a/app/models/concerns/has_unique_internal_users.rb +++ b/app/models/concerns/has_unique_internal_users.rb @@ -28,7 +28,7 @@ module HasUniqueInternalUsers existing_user = uncached { scope.first } return existing_user if existing_user.present? - uniquify = Uniquify.new + uniquify = Gitlab::Utils::Uniquify.new username = uniquify.string(username) { |s| User.find_by_username(s) } diff --git a/app/models/concerns/has_user_type.rb b/app/models/concerns/has_user_type.rb index b02c95c9662..0b1c6780db8 100644 --- a/app/models/concerns/has_user_type.rb +++ b/app/models/concerns/has_user_type.rb @@ -14,8 +14,10 @@ module HasUserType migration_bot: 7, security_bot: 8, automation_bot: 9, + security_policy_bot: 10, # Currently not in use. See https://gitlab.com/gitlab-org/gitlab/-/issues/384174 admin_bot: 11, - suggested_reviewers_bot: 12 + suggested_reviewers_bot: 12, + service_account: 13 }.with_indifferent_access.freeze BOT_USER_TYPES = %w[ @@ -26,11 +28,15 @@ module HasUserType migration_bot security_bot automation_bot + security_policy_bot admin_bot suggested_reviewers_bot + service_account ].freeze - NON_INTERNAL_USER_TYPES = %w[human project_bot service_user].freeze + # `service_account` allows instance/namespaces to configure a user for external integrations/automations + # `service_user` is an internal, `gitlab-com`-specific user type for integrations like suggested reviewers + NON_INTERNAL_USER_TYPES = %w[human project_bot service_user service_account].freeze INTERNAL_USER_TYPES = (USER_TYPES.keys - NON_INTERNAL_USER_TYPES).freeze included do @@ -53,10 +59,8 @@ module HasUserType BOT_USER_TYPES.include?(user_type) end - # The explicit check for project_bot will be removed with Bot Categorization - # Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/213945 def internal? - ghost? || (bot? && !project_bot?) + INTERNAL_USER_TYPES.include?(user_type) end def redacted_name(viewing_user) diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 50696c7b5e1..c1c1691e424 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -640,10 +640,6 @@ module Issuable false end - def ensure_metrics - self.metrics || create_metrics - end - ## # Overridden in MergeRequest # @@ -658,6 +654,10 @@ module Issuable { name: name, subject: self } end + + def supports_health_status? + false + end end Issuable.prepend_mod_with('Issuable') diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index 7addcf9e2ec..0333cfc5f9e 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -169,6 +169,7 @@ module Noteable def expire_note_etag_cache return unless discussions_rendered_on_frontend? return unless etag_caching_enabled? + return unless project.present? Gitlab::EtagCaching::Store.new.touch(note_etag_key) end diff --git a/app/models/concerns/packages/debian/component_file.rb b/app/models/concerns/packages/debian/component_file.rb index 77409549e85..5905670227c 100644 --- a/app/models/concerns/packages/debian/component_file.rb +++ b/app/models/concerns/packages/debian/component_file.rb @@ -88,6 +88,10 @@ module Packages end end + def empty? + size == 0 + end + private def extension diff --git a/app/models/concerns/partitioned_table.rb b/app/models/concerns/partitioned_table.rb index f95f9dd8ad7..c322a736e79 100644 --- a/app/models/concerns/partitioned_table.rb +++ b/app/models/concerns/partitioned_table.rb @@ -8,7 +8,8 @@ module PartitionedTable PARTITIONING_STRATEGIES = { monthly: Gitlab::Database::Partitioning::MonthlyStrategy, - sliding_list: Gitlab::Database::Partitioning::SlidingListStrategy + sliding_list: Gitlab::Database::Partitioning::SlidingListStrategy, + ci_sliding_list: Gitlab::Database::Partitioning::CiSlidingListStrategy }.freeze def partitioned_by(partitioning_key, strategy:, **kwargs) diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index f1d29ad5a90..460cb529715 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -33,6 +33,14 @@ module RedisCacheable clear_memoization(:cached_attributes) end + def merge_cache_attributes(values) + existing_attributes = Hash(cached_attributes) + merged_attributes = existing_attributes.merge(values.symbolize_keys) + return if merged_attributes == existing_attributes + + cache_attributes(merged_attributes) + end + private def cache_attribute_key diff --git a/app/models/concerns/referable.rb b/app/models/concerns/referable.rb index 9a17131c91c..5303d110078 100644 --- a/app/models/concerns/referable.rb +++ b/app/models/concerns/referable.rb @@ -76,7 +76,11 @@ module Referable true end - def link_reference_pattern(route, pattern) + def link_reference_pattern + raise NotImplementedError, "#{self} does not implement #{__method__}" + end + + def compose_link_reference_pattern(route, pattern) %r{ (?<url> #{Regexp.escape(Gitlab.config.gitlab.url)} diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 262839a3fa6..d70aad4e9ae 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -99,39 +99,11 @@ module Routable end def full_name - # We have to test for persistence as the cache key uses #updated_at - return (route&.name || build_full_name) unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops) - - # Return the name as-is if the parent is missing - return name if route.nil? && parent.nil? && name.present? - - # If the route is already preloaded, return directly, preventing an extra load - return route.name if route_loaded? && route.present? - - # Similarly, we can allow the build if the parent is loaded - return build_full_name if parent_loaded? - - Gitlab::Cache.fetch_once([cache_key, :full_name]) do - route&.name || build_full_name - end + full_attribute(:name) end def full_path - # We have to test for persistence as the cache key uses #updated_at - return (route&.path || build_full_path) unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops) - - # Return the path as-is if the parent is missing - return path if route.nil? && parent.nil? && path.present? - - # If the route is already preloaded, return directly, preventing an extra load - return route.path if route_loaded? && route.present? - - # Similarly, we can allow the build if the parent is loaded - return build_full_path if parent_loaded? - - Gitlab::Cache.fetch_once([cache_key, :full_path]) do - route&.path || build_full_path - end + full_attribute(:path) end # Overriden in the Project model @@ -163,6 +135,31 @@ module Routable private + # rubocop: disable GitlabSecurity/PublicSend + def full_attribute(attribute) + attribute_from_route_or_self = ->(attribute) do + route&.public_send(attribute) || send("build_full_#{attribute}") + end + + unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops) + return attribute_from_route_or_self.call(attribute) + end + + # Return the attribute as-is if the parent is missing + return public_send(attribute) if route.nil? && parent.nil? && public_send(attribute).present? + + # If the route is already preloaded, return directly, preventing an extra load + return route.public_send(attribute) if route_loaded? && route.present? && route.public_send(attribute) + + # Similarly, we can allow the build if the parent is loaded + return send("build_full_#{attribute}") if parent_loaded? + + Gitlab::Cache.fetch_once([cache_key, :"full_#{attribute}"]) do + attribute_from_route_or_self.call(attribute) + end + end + # rubocop: enable GitlabSecurity/PublicSend + def set_path_errors route_path_errors = self.errors.delete(:"route.path") route_path_errors&.each do |msg| diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb index 5a10ea7a248..fe47393c554 100644 --- a/app/models/concerns/subscribable.rb +++ b/app/models/concerns/subscribable.rb @@ -27,8 +27,6 @@ module Subscribable def lazy_subscription(user, project = nil) return unless user - # handle project and group labels as well as issuable subscriptions - subscribable_type = self.class.ancestors.include?(Label) ? 'Label' : self.class.name BatchLoader.for(id: id, subscribable_type: subscribable_type, project_id: project&.id).batch do |items, loader| values = items.each_with_object({ ids: Set.new, subscribable_types: Set.new, project_ids: Set.new }) do |item, result| result[:ids] << item[:id] @@ -121,4 +119,15 @@ module Subscribable subscriptions .where(t[:project_id].eq(nil).or(t[:project_id].eq(project.try(:id)))) end + + def subscribable_type + # handle project and group labels as well as issuable subscriptions + if self.class.ancestors.include?(Label) + 'Label' + elsif self.class.ancestors.include?(Issue) + 'Issue' + else + self.class.name + end + end end diff --git a/app/models/concerns/token_authenticatable_strategies/base.rb b/app/models/concerns/token_authenticatable_strategies/base.rb index 2b677f37c89..d0085b60d98 100644 --- a/app/models/concerns/token_authenticatable_strategies/base.rb +++ b/app/models/concerns/token_authenticatable_strategies/base.rb @@ -31,9 +31,13 @@ module TokenAuthenticatableStrategies result end - # Default implementation returns the token as-is + # If a `format_with_prefix` option is provided, it applies and returns the formatted token. + # Otherwise, default implementation returns the token as-is def format_token(instance, token) - instance.send("format_#{@token_field}", token) # rubocop:disable GitlabSecurity/PublicSend + prefix = prefix_for(instance) + prefixed_token = prefix ? "#{prefix}#{token}" : token + + instance.send("format_#{@token_field}", prefixed_token) # rubocop:disable GitlabSecurity/PublicSend end def ensure_token(instance) @@ -88,6 +92,17 @@ module TokenAuthenticatableStrategies protected + def prefix_for(instance) + case prefix_option = options[:format_with_prefix] + when nil + nil + when Symbol + instance.send(prefix_option) # rubocop:disable GitlabSecurity/PublicSend + else + raise NotImplementedError + end + end + def write_new_token(instance) new_token = generate_available_token formatted_token = format_token(instance, new_token) diff --git a/app/models/concerns/token_authenticatable_strategies/encrypted.rb b/app/models/concerns/token_authenticatable_strategies/encrypted.rb index 1db88c27181..4b3b80437db 100644 --- a/app/models/concerns/token_authenticatable_strategies/encrypted.rb +++ b/app/models/concerns/token_authenticatable_strategies/encrypted.rb @@ -106,11 +106,7 @@ module TokenAuthenticatableStrategies end def matches_prefix?(instance, token) - prefix = options[:prefix] - prefix = prefix.call(instance) if prefix.is_a?(Proc) - prefix = '' unless prefix.is_a?(String) - - token.start_with?(prefix) + !options[:require_prefix_for_validation] || token.start_with?(prefix_for(instance)) end def token_set?(instance) diff --git a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb index 447521ad8c1..5e77dfde397 100644 --- a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb +++ b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb @@ -20,8 +20,6 @@ module TokenAuthenticatableStrategies end def self.encrypt_token(plaintext_token) - return Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token) unless Feature.enabled?(:dynamic_nonce, type: :ops) - iv = ::Digest::SHA256.hexdigest(plaintext_token).bytes.take(NONCE_SIZE).pack('c*') token = Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token, nonce: iv) "#{DYNAMIC_NONCE_IDENTIFIER}#{token}#{iv}" diff --git a/app/models/concerns/uniquify.rb b/app/models/concerns/uniquify.rb deleted file mode 100644 index 382e826ec58..00000000000 --- a/app/models/concerns/uniquify.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -# Uniquify -# -# Return a version of the given 'base' string that is unique -# by appending a counter to it. Uniqueness is determined by -# repeated calls to the passed block. -# -# You can pass an initial value for the counter, if not given -# counting starts from 1. -# -# If `base` is a function/proc, we expect that calling it with a -# candidate counter returns a string to test/return. -class Uniquify - def initialize(counter = nil) - @counter = counter - end - - def string(base) - @base = base - - increment_counter! while yield(base_string) - base_string - end - - private - - def base_string - if @base.respond_to?(:call) - @base.call(@counter) - else - "#{@base}#{@counter}" - end - end - - def increment_counter! - @counter ||= 0 - @counter += 1 - end -end diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index 2cc17a6f185..05aaca32f35 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -4,7 +4,32 @@ module WebHooks module AutoDisabling extend ActiveSupport::Concern + ENABLED_HOOK_TYPES = %w[ProjectHook].freeze + MAX_FAILURES = 100 + FAILURE_THRESHOLD = 3 + EXCEEDED_FAILURE_THRESHOLD = FAILURE_THRESHOLD + 1 + INITIAL_BACKOFF = 1.minute.freeze + MAX_BACKOFF = 1.day.freeze + BACKOFF_GROWTH_FACTOR = 2.0 + + class_methods do + def auto_disabling_enabled? + enabled_hook_types.include?(name) && + Gitlab::SafeRequestStore.fetch(:auto_disabling_web_hooks) do + Feature.enabled?(:auto_disabling_web_hooks, type: :ops) + end + end + + private + + def enabled_hook_types + ENABLED_HOOK_TYPES + end + end + included do + delegate :auto_disabling_enabled?, to: :class, private: true + # A hook is disabled if: # # - we are no longer in the grace-perod (recent_failures > ?) @@ -12,8 +37,10 @@ module WebHooks # - disabled_until is nil (i.e. this was set by WebHook#fail!) # - or disabled_until is in the future (i.e. this was set by WebHook#backoff!) scope :disabled, -> do + return none unless auto_disabling_enabled? + where('recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)', - WebHook::FAILURE_THRESHOLD, Time.current) + FAILURE_THRESHOLD, Time.current) end # A hook is executable if: @@ -23,40 +50,81 @@ module WebHooks # - disabled_until is nil (i.e. this was set by WebHook#fail!) # - disabled_until is in the future (i.e. this was set by WebHook#backoff!) scope :executable, -> do + return all unless auto_disabling_enabled? + where('recent_failures <= ? OR (recent_failures > ? AND (disabled_until IS NOT NULL) AND (disabled_until < ?))', - WebHook::FAILURE_THRESHOLD, WebHook::FAILURE_THRESHOLD, Time.current) + FAILURE_THRESHOLD, FAILURE_THRESHOLD, Time.current) end end def executable? + return true unless auto_disabling_enabled? + !temporarily_disabled? && !permanently_disabled? end def temporarily_disabled? - return false if recent_failures <= WebHook::FAILURE_THRESHOLD + return false unless auto_disabling_enabled? - disabled_until.present? && disabled_until >= Time.current + disabled_until.present? && disabled_until >= Time.current && recent_failures > FAILURE_THRESHOLD end def permanently_disabled? - return false if disabled_until.present? + return false unless auto_disabling_enabled? - recent_failures > WebHook::FAILURE_THRESHOLD + recent_failures > FAILURE_THRESHOLD && disabled_until.blank? end def disable! - return if permanently_disabled? + return if !auto_disabling_enabled? || permanently_disabled? - super + update_attribute(:recent_failures, EXCEEDED_FAILURE_THRESHOLD) end + def enable! + return unless auto_disabling_enabled? + return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0 + + assign_attributes(recent_failures: 0, disabled_until: nil, backoff_count: 0) + save(validate: false) + end + + # Don't actually back-off until FAILURE_THRESHOLD failures have been seen + # we mark the grace-period using the recent_failures counter def backoff! - return if permanently_disabled? || (backoff_count >= WebHook::MAX_FAILURES && temporarily_disabled?) + return unless auto_disabling_enabled? + return if permanently_disabled? || (backoff_count >= MAX_FAILURES && temporarily_disabled?) + + attrs = { recent_failures: next_failure_count } - super + if recent_failures >= FAILURE_THRESHOLD + attrs[:backoff_count] = next_backoff_count + attrs[:disabled_until] = next_backoff.from_now + end + + assign_attributes(attrs) + save(validate: false) if changed? + end + + def failed! + return unless auto_disabling_enabled? + return unless recent_failures < MAX_FAILURES + + assign_attributes(disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count) + save(validate: false) + end + + def next_backoff + return MAX_BACKOFF if backoff_count >= 8 # optimization to prevent expensive exponentiation and possible overflows + + (INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**backoff_count)) + .clamp(INITIAL_BACKOFF, MAX_BACKOFF) + .seconds end def alert_status + return :executable unless auto_disabling_enabled? + if temporarily_disabled? :temporarily_disabled elsif permanently_disabled? @@ -65,5 +133,18 @@ module WebHooks :executable end end + + private + + def next_failure_count + recent_failures.succ.clamp(1, MAX_FAILURES) + end + + def next_backoff_count + backoff_count.succ.clamp(1, MAX_FAILURES) + end end end + +WebHooks::AutoDisabling.prepend_mod +WebHooks::AutoDisabling::ClassMethods.prepend_mod diff --git a/app/models/concerns/web_hooks/has_web_hooks.rb b/app/models/concerns/web_hooks/has_web_hooks.rb index 161ce106b9b..2183cc3c44b 100644 --- a/app/models/concerns/web_hooks/has_web_hooks.rb +++ b/app/models/concerns/web_hooks/has_web_hooks.rb @@ -2,8 +2,6 @@ module WebHooks module HasWebHooks - extend ActiveSupport::Concern - WEB_HOOK_CACHE_EXPIRY = 1.hour def any_hook_failed? @@ -15,7 +13,7 @@ module WebHooks end def last_failure_redis_key - "web_hooks:last_failure:project-#{id}" + "web_hooks:last_failure:#{self.class.name.underscore}-#{id}" end def get_web_hook_failure @@ -42,5 +40,13 @@ module WebHooks state end end + + def last_webhook_failure + last_failure = Gitlab::Redis::SharedState.with do |redis| + redis.get(last_failure_redis_key) + end + + DateTime.parse(last_failure) if last_failure + end end end diff --git a/app/models/concerns/web_hooks/unstoppable.rb b/app/models/concerns/web_hooks/unstoppable.rb deleted file mode 100644 index 26284fe3c36..00000000000 --- a/app/models/concerns/web_hooks/unstoppable.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module WebHooks - module Unstoppable - extend ActiveSupport::Concern - - included do - scope :executable, -> { all } - - scope :disabled, -> { none } - end - - def executable? - true - end - - def temporarily_disabled? - false - end - - def permanently_disabled? - false - end - - def alert_status - :executable - end - end -end diff --git a/app/models/container_registry/data_repair_detail.rb b/app/models/container_registry/data_repair_detail.rb new file mode 100644 index 00000000000..09e617e69f5 --- /dev/null +++ b/app/models/container_registry/data_repair_detail.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module ContainerRegistry + class DataRepairDetail < ApplicationRecord + self.table_name = 'container_registry_data_repair_details' + self.primary_key = :project_id + + belongs_to :project, optional: false + end +end diff --git a/app/models/container_registry/event.rb b/app/models/container_registry/event.rb index c4d06be8841..dd2675e17d8 100644 --- a/app/models/container_registry/event.rb +++ b/app/models/container_registry/event.rb @@ -8,7 +8,7 @@ module ContainerRegistry PUSH_ACTION = 'push' DELETE_ACTION = 'delete' EVENT_TRACKING_CATEGORY = 'container_registry:notification' - EVENT_PREFIX = "i_container_registry" + EVENT_PREFIX = 'i_container_registry' ALLOWED_ACTOR_TYPES = %w( personal_access_token @@ -48,8 +48,12 @@ module ContainerRegistry ::Gitlab::Tracking.event(EVENT_TRACKING_CATEGORY, tracking_action) - event = usage_data_event_for(tracking_action) - ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: originator.id) if event + if manifest_delete_event? + ::Gitlab::UsageDataCounters::ContainerRegistryEventCounter.count("#{EVENT_PREFIX}_delete_manifest") + else + event = usage_data_event_for(tracking_action) + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: originator.id) if event + end end private @@ -122,9 +126,13 @@ module ContainerRegistry end end + def manifest_delete_event? + action_delete? && target_digest? + end + def update_project_statistics return unless supported? - return unless target_tag? || (action_delete? && target_digest?) + return unless target_tag? || manifest_delete_event? return unless project Rails.cache.delete(project.root_ancestor.container_repositories_size_cache_key) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 98ce981ad8e..b3cbe498551 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -69,7 +69,7 @@ class ContainerRepository < ApplicationRecord scope :with_migration_import_started_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_import_started_at, '01-01-1970') < ?", timestamp) } scope :with_migration_pre_import_started_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_pre_import_started_at, '01-01-1970') < ?", timestamp) } scope :with_migration_pre_import_done_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_pre_import_done_at, '01-01-1970') < ?", timestamp) } - scope :with_stale_ongoing_cleanup, ->(threshold) { cleanup_ongoing.where('expiration_policy_started_at < ?', threshold) } + scope :with_stale_ongoing_cleanup, ->(threshold) { cleanup_ongoing.expiration_policy_started_at_nil_or_before(threshold) } scope :with_stale_delete_at, ->(threshold) { where('delete_started_at < ?', threshold) } scope :import_in_process, -> { where(migration_state: %w[pre_importing pre_import_done importing]) } @@ -395,7 +395,7 @@ class ContainerRepository < ApplicationRecord end def migrated? - (self.created_at && MIGRATION_PHASE_1_ENDED_AT < self.created_at) || import_done? + Gitlab.com? end def last_import_step_done_at @@ -509,7 +509,11 @@ class ContainerRepository < ApplicationRecord end def start_expiration_policy! - update!(expiration_policy_started_at: Time.zone.now, last_cleanup_deleted_tags_count: nil) + update!( + expiration_policy_started_at: Time.zone.now, + last_cleanup_deleted_tags_count: nil, + expiration_policy_cleanup_status: :cleanup_ongoing + ) end def size diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index 5ad746e4cd1..11fe0503f50 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -12,6 +12,11 @@ class DependencyProxy::Manifest < ApplicationRecord MAX_FILE_SIZE = 10.megabytes.freeze DIGEST_HEADER = 'Docker-Content-Digest' + ACCEPTED_TYPES = [ + ContainerRegistry::BaseClient::DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, + ContainerRegistry::BaseClient::OCI_MANIFEST_V1_TYPE, + ContainerRegistry::BaseClient::OCI_DISTRIBUTION_INDEX_TYPE + ].freeze validates :group, presence: true validates :file, presence: true diff --git a/app/models/dependency_proxy/registry.rb b/app/models/dependency_proxy/registry.rb index 6492acf325a..3073dd59c7b 100644 --- a/app/models/dependency_proxy/registry.rb +++ b/app/models/dependency_proxy/registry.rb @@ -33,3 +33,5 @@ class DependencyProxy::Registry end end end + +::DependencyProxy::Registry.prepend_mod diff --git a/app/models/design_management/design.rb b/app/models/design_management/design.rb index 317399e780a..cb6d4e72c80 100644 --- a/app/models/design_management/design.rb +++ b/app/models/design_management/design.rb @@ -13,6 +13,9 @@ module DesignManagement include RelativePositioning include Todoable include Participable + include CacheMarkdownField + + cache_markdown_field :description belongs_to :project, inverse_of: :designs belongs_to :issue @@ -34,6 +37,7 @@ module DesignManagement validates :project, :filename, presence: true validates :issue, presence: true, unless: :importing? validates :filename, uniqueness: { scope: :issue_id }, length: { maximum: 255 } + validates :description, length: { maximum: Gitlab::Database::MAX_TEXT_SIZE_LIMIT } validate :validate_file_is_image alias_attribute :title, :filename @@ -43,7 +47,7 @@ module DesignManagement # Pre-fetching scope to include the data necessary to construct a # reference using `to_reference`. - scope :for_reference, -> { includes(issue: [{ project: [:route, :namespace] }]) } + scope :for_reference, -> { includes(issue: [{ namespace: :project }, { project: [:route, :namespace] }]) } # A design can be uniquely identified by issue_id and filename # Takes one or more sets of composite IDs of the form: @@ -174,7 +178,7 @@ module DesignManagement (?<url_filename> #{valid_char}+ \. #{ext}) }x - super(path_segment, filename_pattern) + compose_link_reference_pattern(path_segment, filename_pattern) end end @@ -182,10 +186,6 @@ module DesignManagement File.join(DesignManagement.designs_directory, "issue-#{issue.iid}", design.filename) end - def description - '' - end - def new_design? strong_memoize(:new_design) { actions.none? } end diff --git a/app/models/draft_note.rb b/app/models/draft_note.rb index 9f7977fce68..ffc04f9bf90 100644 --- a/app/models/draft_note.rb +++ b/app/models/draft_note.rb @@ -108,7 +108,7 @@ class DraftNote < ApplicationRecord end def self.preload_author(draft_notes) - ActiveRecord::Associations::Preloader.new.preload(draft_notes, { author: :status }) + ActiveRecord::Associations::Preloader.new(records: draft_notes, associations: { author: :status }).call end def diff_file diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb index 1c7a8d93e6e..c52f8a58c00 100644 --- a/app/models/error_tracking/project_error_tracking_setting.rb +++ b/app/models/error_tracking/project_error_tracking_setting.rb @@ -145,7 +145,7 @@ module ErrorTracking ensure_issue_belongs_to_project!(issue_to_be_updated.project_id) handle_exceptions do - { updated: sentry_client.update_issue(opts) } + { updated: sentry_client.update_issue(**opts) } end end diff --git a/app/models/group.rb b/app/models/group.rb index 7e09280dfff..01e2c220dbe 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -7,7 +7,6 @@ class Group < Namespace include AfterCommitQueue include AccessRequestable include Avatarable - include Referable include SelectForProjectAuthorization include LoadedInGroupList include GroupDescendant @@ -21,7 +20,6 @@ class Group < Namespace include ChronicDurationAttribute include RunnerTokenExpirationInterval include Todoable - include IssueParent extend ::Gitlab::Utils::Override @@ -110,7 +108,10 @@ class Group < Namespace has_one :import_state, class_name: 'GroupImportState', inverse_of: :group + has_many :application_setting, foreign_key: :instance_administrators_group_id, inverse_of: :instance_group + has_many :bulk_import_exports, class_name: 'BulkImports::Export', inverse_of: :group + has_many :bulk_import_entities, class_name: 'BulkImports::Entity', foreign_key: :namespace_id, inverse_of: :group has_many :group_deploy_keys_groups, inverse_of: :group has_many :group_deploy_keys, through: :group_deploy_keys_groups @@ -162,7 +163,8 @@ class Group < Namespace add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption) ? :optional : :required }, - prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + format_with_prefix: :runners_token_prefix, + require_prefix_for_validation: true after_create :post_create_hook after_create -> { create_or_load_association(:group_feature) } @@ -240,14 +242,6 @@ class Group < Namespace end end - def reference_prefix - User.reference_prefix - end - - def reference_pattern - User.reference_pattern - end - # WARNING: This method should never be used on its own # please do make sure the number of rows you are filtering is small # enough for this query @@ -364,10 +358,6 @@ class Group < Namespace notification_settings.find { |n| n.notification_email.present? }&.notification_email end - def to_reference(_from = nil, target_project: nil, full: nil) - "#{self.class.reference_prefix}#{full_path}" - end - def web_url(only_path: nil) Gitlab::UrlBuilder.build(self, only_path: only_path) end @@ -762,11 +752,6 @@ class Group < Namespace ensure_runners_token! end - override :format_runners_token - def format_runners_token(token) - "#{RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX}#{token}" - end - def project_creation_level super || ::Gitlab::CurrentSettings.default_project_creation end @@ -814,8 +799,10 @@ class Group < Namespace end def preload_shared_group_links - preloader = ActiveRecord::Associations::Preloader.new - preloader.preload(self, shared_with_group_links: [shared_with_group: :route]) + ActiveRecord::Associations::Preloader.new( + records: [self], + associations: { shared_with_group_links: [shared_with_group: :route] } + ).call end def update_shared_runners_setting!(state) @@ -1095,6 +1082,10 @@ class Group < Namespace def enable_shared_runners! update!(shared_runners_enabled: true) end + + def runners_token_prefix + RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + end end Group.prepend_mod_with('Group') diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index 8e9a74a68d0..695041f0247 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -2,7 +2,6 @@ class ProjectHook < WebHook include TriggerableHooks - include WebHooks::AutoDisabling include Presentable include Limitable extend ::Gitlab::Utils::Override diff --git a/app/models/hooks/service_hook.rb b/app/models/hooks/service_hook.rb index 6af70c249a0..453b986ca4d 100644 --- a/app/models/hooks/service_hook.rb +++ b/app/models/hooks/service_hook.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class ServiceHook < WebHook - include WebHooks::Unstoppable include Presentable extend ::Gitlab::Utils::Override diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb index eaffe83cab3..3c7f0ef9ffc 100644 --- a/app/models/hooks/system_hook.rb +++ b/app/models/hooks/system_hook.rb @@ -2,7 +2,6 @@ class SystemHook < WebHook include TriggerableHooks - include WebHooks::Unstoppable triggerable_hooks [ :repository_update_hooks, diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 819152a38c8..7e55ffe2e5e 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -2,15 +2,10 @@ class WebHook < ApplicationRecord include Sortable + include WebHooks::AutoDisabling InterpolationError = Class.new(StandardError) - MAX_FAILURES = 100 - FAILURE_THRESHOLD = 3 # three strikes - EXCEEDED_FAILURE_THRESHOLD = FAILURE_THRESHOLD + 1 - INITIAL_BACKOFF = 1.minute - MAX_BACKOFF = 1.day - BACKOFF_GROWTH_FACTOR = 2.0 SECRET_MASK = '************' attr_encrypted :token, @@ -78,46 +73,6 @@ class WebHook < ApplicationRecord 'user/project/integrations/webhooks' end - def next_backoff - return MAX_BACKOFF if backoff_count >= 8 # optimization to prevent expensive exponentiation and possible overflows - - (INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**backoff_count)) - .clamp(INITIAL_BACKOFF, MAX_BACKOFF) - .seconds - end - - def disable! - update_attribute(:recent_failures, EXCEEDED_FAILURE_THRESHOLD) - end - - def enable! - return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0 - - assign_attributes(recent_failures: 0, disabled_until: nil, backoff_count: 0) - save(validate: false) - end - - # Don't actually back-off until FAILURE_THRESHOLD failures have been seen - # we mark the grace-period using the recent_failures counter - def backoff! - attrs = { recent_failures: next_failure_count } - - if recent_failures >= FAILURE_THRESHOLD - attrs[:backoff_count] = next_backoff_count - attrs[:disabled_until] = next_backoff.from_now - end - - assign_attributes(attrs) - save(validate: false) if changed? - end - - def failed! - return unless recent_failures < MAX_FAILURES - - assign_attributes(disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count) - save(validate: false) - end - # @return [Boolean] Whether or not the WebHook is currently throttled. def rate_limited? rate_limiter.rate_limited? @@ -179,14 +134,6 @@ class WebHook < ApplicationRecord self.url_variables = {} if url_changed? && !encrypted_url_variables_changed? end - def next_failure_count - recent_failures.succ.clamp(1, MAX_FAILURES) - end - - def next_backoff_count - backoff_count.succ.clamp(1, MAX_FAILURES) - end - def initialize_url_variables self.url_variables = {} if encrypted_url_variables.nil? end diff --git a/app/models/import_failure.rb b/app/models/import_failure.rb index 109c0c82487..0ca99faeb71 100644 --- a/app/models/import_failure.rb +++ b/app/models/import_failure.rb @@ -6,6 +6,7 @@ class ImportFailure < ApplicationRecord validates :project, presence: true, unless: :group validates :group, presence: true, unless: :project + validates :external_identifiers, json_schema: { filename: "import_failure_external_identifiers" } # Returns any `import_failures` for relations that were unrecoverable errors or failed after # several retries. An import can be successful even if some relations failed to import correctly. @@ -13,4 +14,8 @@ class ImportFailure < ApplicationRecord scope :hard_failures_by_correlation_id, ->(correlation_id) { where(correlation_id_value: correlation_id, retry_count: 0).order(created_at: :desc) } + + scope :failures_by_correlation_id, ->(correlation_id) { + where(correlation_id_value: correlation_id).order(created_at: :desc) + } end diff --git a/app/models/integration.rb b/app/models/integration.rb index d3006f00ba1..860739fe5aa 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -21,13 +21,14 @@ class Integration < ApplicationRecord asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord drone_ci emails_on_push ewm external_wiki hangouts_chat harbor irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email - pivotaltracker prometheus pumble pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao + pivotaltracker prometheus pumble pushover redmine slack slack_slash_commands squash_tm teamcity + unify_circuit webex_teams youtrack zentao ].freeze # TODO Shimo is temporary disabled on group and instance-levels. # See: https://gitlab.com/gitlab-org/gitlab/-/issues/345677 PROJECT_SPECIFIC_INTEGRATION_NAMES = %w[ - apple_app_store jenkins shimo + apple_app_store google_play jenkins shimo ].freeze # Fake integrations to help with local development. diff --git a/app/models/integrations/apple_app_store.rb b/app/models/integrations/apple_app_store.rb index 84185542939..34da4c0f4b8 100644 --- a/app/models/integrations/apple_app_store.rb +++ b/app/models/integrations/apple_app_store.rb @@ -7,10 +7,13 @@ module Integrations ISSUER_ID_REGEX = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/.freeze KEY_ID_REGEX = /\A(?=.*[A-Z])(?=.*[0-9])[A-Z0-9]+\z/.freeze + SECTION_TYPE_APPLE_APP_STORE = 'apple_app_store' + with_options if: :activated? do validates :app_store_issuer_id, presence: true, format: { with: ISSUER_ID_REGEX } validates :app_store_key_id, presence: true, format: { with: KEY_ID_REGEX } validates :app_store_private_key, presence: true, certificate_key: true + validates :app_store_private_key_file_name, presence: true end field :app_store_issuer_id, @@ -24,13 +27,12 @@ module Integrations title: -> { s_('AppleAppStore|The Apple App Store Connect Key ID.') }, is_secret: false - field :app_store_private_key, + field :app_store_private_key_file_name, section: SECTION_TYPE_CONNECTION, - required: true, - type: 'textarea', - title: -> { s_('AppleAppStore|The Apple App Store Connect Private Key.') }, is_secret: false + field :app_store_private_key, api_only: true, is_secret: false + def title 'Apple App Store Connect' end @@ -69,7 +71,7 @@ module Integrations def sections [ { - type: SECTION_TYPE_CONNECTION, + type: SECTION_TYPE_APPLE_APP_STORE, title: s_('Integrations|Integration details'), description: help } @@ -99,13 +101,11 @@ module Integrations private def client - config = { + AppStoreConnect::Client.new( issuer_id: app_store_issuer_id, key_id: app_store_key_id, private_key: app_store_private_key - } - - AppStoreConnect::Client.new(config) + ) end end end diff --git a/app/models/integrations/base_slack_notification.rb b/app/models/integrations/base_slack_notification.rb index 7a2a91aa0d2..c83a559e0da 100644 --- a/app/models/integrations/base_slack_notification.rb +++ b/app/models/integrations/base_slack_notification.rb @@ -44,8 +44,6 @@ module Integrations Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user_id) - return unless Feature.enabled?(:route_hll_to_snowplow_phase2) - optional_arguments = { project: project, namespace: group || project&.namespace diff --git a/app/models/integrations/base_slash_commands.rb b/app/models/integrations/base_slash_commands.rb index 619579a543a..7662da933ba 100644 --- a/app/models/integrations/base_slash_commands.rb +++ b/app/models/integrations/base_slash_commands.rb @@ -6,10 +6,6 @@ module Integrations class BaseSlashCommands < Integration attribute :category, default: 'chat' - prop_accessor :token - - has_many :chat_names, foreign_key: :integration_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - def valid_token?(token) self.respond_to?(:token) && self.token.present? && @@ -24,18 +20,6 @@ module Integrations false end - def fields - [ - { - type: 'password', - name: 'token', - non_empty_password_title: s_('ProjectService|Enter new token'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), - placeholder: 'XXxxXXxxXXxxXXxxXXxxXXxx' - } - ] - end - def trigger(params) return unless valid_token?(params[:token]) diff --git a/app/models/integrations/campfire.rb b/app/models/integrations/campfire.rb index 3f7fa1c51b2..9b837faf79b 100644 --- a/app/models/integrations/campfire.rb +++ b/app/models/integrations/campfire.rb @@ -68,7 +68,7 @@ module Integrations def execute(data) return unless supported_events.include?(data[:object_kind]) - message = build_message(data) + message = create_message(data) speak(self.room, message, auth) end @@ -116,7 +116,7 @@ module Integrations res.code == 200 ? res["rooms"] : [] end - def build_message(push) + def create_message(push) ref = Gitlab::Git.ref_name(push[:ref]) before = push[:before] after = push[:after] diff --git a/app/models/integrations/google_play.rb b/app/models/integrations/google_play.rb new file mode 100644 index 00000000000..8f1d2e7e1ec --- /dev/null +++ b/app/models/integrations/google_play.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Integrations + class GooglePlay < Integration + SECTION_TYPE_GOOGLE_PLAY = 'google_play' + + with_options if: :activated? do + validates :service_account_key, presence: true, json_schema: { + filename: "google_service_account_key", parse_json: true + } + validates :service_account_key_file_name, presence: true + end + + field :service_account_key_file_name, + section: SECTION_TYPE_CONNECTION, + required: true, + is_secret: false + + field :service_account_key, api_only: true, is_secret: false + + def title + s_('GooglePlay|Google Play') + end + + def description + s_('GooglePlay|Use GitLab to build and release an app in Google Play.') + end + + def help + variable_list = [ + '<code>SUPPLY_JSON_KEY_DATA</code>' + ] + + # rubocop:disable Layout/LineLength + texts = [ + s_("Use the Google Play integration to connect to Google Play with fastlane in CI/CD pipelines."), + s_("After you enable the integration, the following protected variable is created for CI/CD use:"), + variable_list.join('<br>'), + s_(format("To generate a Google Play service account key and use this integration, see the <a href='%{url}' target='_blank'>integration documentation</a>.", url: "#")).html_safe + ] + # rubocop:enable Layout/LineLength + + texts.join('<br><br>'.html_safe) + end + + def self.to_param + 'google_play' + end + + def self.supported_events + [] + end + + def sections + [ + { + type: SECTION_TYPE_GOOGLE_PLAY, + title: s_('Integrations|Integration details'), + description: help + } + ] + end + + def test(*_args) + client.fetch_access_token! + { success: true } + rescue Signet::AuthorizationError => error + { success: false, message: error } + end + + def ci_variables + return [] unless activated? + + [ + { key: 'SUPPLY_JSON_KEY_DATA', value: service_account_key, masked: true, public: false } + ] + end + + private + + def client + Google::Auth::ServiceAccountCredentials.make_creds( + json_key_io: StringIO.new(service_account_key), + scope: ['https://www.googleapis.com/auth/androidpublisher'] + ) + end + end +end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index d96a848c72e..a1cdd55ceae 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -391,8 +391,6 @@ module Integrations Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user.id) - return unless Feature.enabled?(:route_hll_to_snowplow_phase2) - optional_arguments = { project: project, namespace: group || project&.namespace diff --git a/app/models/integrations/mattermost_slash_commands.rb b/app/models/integrations/mattermost_slash_commands.rb index 30a8ba973c1..f5079b9b907 100644 --- a/app/models/integrations/mattermost_slash_commands.rb +++ b/app/models/integrations/mattermost_slash_commands.rb @@ -4,7 +4,11 @@ module Integrations class MattermostSlashCommands < BaseSlashCommands include Ci::TriggersHelper - prop_accessor :token + field :token, + type: 'password', + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, + placeholder: '' def testable? false @@ -37,10 +41,6 @@ module Integrations [[], e.message] end - def chat_responder - ::Gitlab::Chat::Responder::Mattermost - end - private def command(params) diff --git a/app/models/integrations/slack_slash_commands.rb b/app/models/integrations/slack_slash_commands.rb index 72e3c4a8cbc..343c8d68166 100644 --- a/app/models/integrations/slack_slash_commands.rb +++ b/app/models/integrations/slack_slash_commands.rb @@ -4,6 +4,12 @@ module Integrations class SlackSlashCommands < BaseSlashCommands include Ci::TriggersHelper + field :token, + type: 'password', + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, + placeholder: '' + def title 'Slack slash commands' end @@ -23,10 +29,6 @@ module Integrations end end - def chat_responder - ::Gitlab::Chat::Responder::Slack - end - private def format(text) diff --git a/app/models/integrations/squash_tm.rb b/app/models/integrations/squash_tm.rb new file mode 100644 index 00000000000..e0a63b5ae6a --- /dev/null +++ b/app/models/integrations/squash_tm.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Integrations + class SquashTm < Integration + include HasWebHook + + field :url, + placeholder: 'https://your-instance.squashcloud.io/squash/plugin/xsquash4gitlab/webhook/issue', + title: -> { s_('SquashTmIntegration|Squash TM webhook URL') }, + exposes_secrets: true, + required: true + + field :token, + type: 'password', + title: -> { s_('SquashTmIntegration|Secret token (optional)') }, + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, + required: false + + with_options if: :activated? do + validates :url, presence: true, public_url: true + validates :token, length: { maximum: 255 }, allow_blank: true + end + + def title + 'Squash TM' + end + + def description + s_("SquashTmIntegration|Update Squash TM requirements when GitLab issues are modified.") + end + + def help + docs_link = ActionController::Base.helpers.link_to( + _('Learn more.'), + Rails.application.routes.url_helpers.help_page_url('user/project/integrations/squash_tm'), + target: '_blank', + rel: 'noopener noreferrer' + ) + + Kernel.format( + s_('SquashTmIntegration|Update Squash TM requirements when GitLab issues are modified. %{docs_link}'), + { docs_link: docs_link.html_safe } + ).html_safe + end + + def self.supported_events + %w[issue confidential_issue] + end + + def self.to_param + 'squash_tm' + end + + def self.default_test_event + 'issue' + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + execute_web_hook!(data, "#{data[:object_kind]} Hook") + end + + def test(data) + result = execute_web_hook!(data, "Test Configuration Hook") + + { success: result.payload[:http_status] == 200, result: result.message } + rescue StandardError => error + { success: false, result: error.message } + end + + override :hook_url + def hook_url + format("#{url}%s", ('?token={token}' unless token.blank?)) + end + + def url_variables + { 'token' => token }.compact + end + end +end diff --git a/app/models/issue.rb b/app/models/issue.rb index bea86168c8d..a19b5809ff8 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -63,7 +63,24 @@ class Issue < ApplicationRecord belongs_to :moved_to, class_name: 'Issue' has_one :moved_from, class_name: 'Issue', foreign_key: :moved_to_id - has_internal_id :iid, scope: :project, track_if: -> { !importing? } + has_internal_id :iid, scope: :namespace, track_if: -> { !importing? }, init: ->(issue, scope) do + # we need this init for the case where the IID allocation in internal_ids#last_value + # is higher than the actual issues.max(iid) value for a given project. For instance + # in case of an import where a batch of IIDs may be prealocated + # + # TODO: remove this once the UpdateIssuesInternalIdScope migration completes + if issue + [ + InternalId.where(project: issue.project, usage: :issues).pick(:last_value).to_i, + issue.namespace&.issues&.maximum(:iid).to_i + ].max + else + [ + InternalId.where(**scope, usage: :issues).pick(:last_value).to_i, + where(**scope).maximum(:iid).to_i + ].max + end + end has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -104,10 +121,11 @@ class Issue < ApplicationRecord accepts_nested_attributes_for :sentry_issue accepts_nested_attributes_for :incident_management_issuable_escalation_status, update_only: true - validates :project, presence: true + validates :project, presence: true, if: -> { !namespace || namespace.is_a?(Namespaces::ProjectNamespace) } validates :issue_type, presence: true validates :namespace, presence: true validates :work_item_type, presence: true + validates :confidential, inclusion: { in: [true, false], message: 'must be a boolean' } validate :allowed_work_item_type_change, on: :update, if: :work_item_type_id_changed? validate :due_date_after_start_date @@ -136,7 +154,7 @@ class Issue < ApplicationRecord scope :order_due_date_asc, -> { reorder(arel_table[:due_date].asc.nulls_last) } scope :order_due_date_desc, -> { reorder(arel_table[:due_date].desc.nulls_last) } - scope :order_closest_future_date, -> { reorder(Arel.sql('CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC')) } + scope :order_closest_future_date, -> { reorder(Arel.sql("CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC")) } scope :order_created_at_desc, -> { reorder(created_at: :desc) } scope :order_severity_asc, -> do build_keyset_order_on_joined_column( @@ -162,15 +180,15 @@ class Issue < ApplicationRecord scope :order_closed_at_desc, -> { reorder(arel_table[:closed_at].desc.nulls_last) } scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } - scope :with_web_entity_associations, -> { preload(:author, project: [:project_feature, :route, namespace: :route]) } + scope :with_web_entity_associations, -> { preload(:author, :namespace, project: [:project_feature, :route, namespace: :route]) } scope :preload_awardable, -> { preload(:award_emoji) } scope :with_alert_management_alerts, -> { joins(:alert_management_alert) } scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) } scope :with_self_managed_prometheus_alert_events, -> { joins(:issues_self_managed_prometheus_alert_events) } scope :with_api_entity_associations, -> { - preload(:timelogs, :closed_by, :assignees, :author, :labels, :issuable_severity, + preload(:timelogs, :closed_by, :assignees, :author, :labels, :issuable_severity, namespace: [{ parent: :route }, :route], milestone: { project: [:route, { namespace: :route }] }, - project: [:project_feature, :route, { namespace: :route }], + project: [:project_namespace, :project_feature, :route, { group: :route }, { namespace: :route }], duplicated_to: { project: [:project_feature] }) } scope :with_issue_type, ->(types) { where(issue_type: types) } @@ -214,7 +232,7 @@ class Issue < ApplicationRecord before_validation :ensure_namespace_id, :ensure_work_item_type - after_save :ensure_metrics, unless: :importing? + after_save :ensure_metrics!, unless: :importing? after_commit :expire_etag_cache, unless: :importing? after_create_commit :record_create_action, unless: :importing? @@ -345,7 +363,7 @@ class Issue < ApplicationRecord end def self.link_reference_pattern - @link_reference_pattern ||= super(%r{issues(?:\/incident)?}, Gitlab::Regex.issue) + @link_reference_pattern ||= compose_link_reference_pattern(%r{issues(?:\/incident)?}, Gitlab::Regex.issue) end def self.reference_valid?(reference) @@ -450,7 +468,7 @@ class Issue < ApplicationRecord def to_reference(from = nil, full: false) reference = "#{self.class.reference_prefix}#{iid}" - "#{project.to_reference_base(from, full: full)}#{reference}" + "#{namespace.to_reference_base(from, full: full)}#{reference}" end def suggested_branch_name @@ -463,7 +481,7 @@ class Issue < ApplicationRecord "#{to_branch_name}-#{suffix}" end - Uniquify.new(start_counting_from).string(branch_name_generator) do |suggested_branch_name| + Gitlab::Utils::Uniquify.new(start_counting_from).string(branch_name_generator) do |suggested_branch_name| project.repository.branch_exists?(suggested_branch_name) end end @@ -722,8 +740,7 @@ class Issue < ApplicationRecord confidential_changed?(from: true, to: false) end - override :ensure_metrics - def ensure_metrics + def ensure_metrics! Issue::Metrics.record!(self) end diff --git a/app/models/member.rb b/app/models/member.rb index e97c9e929ac..4329b61fc3d 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -320,6 +320,12 @@ class Member < ApplicationRecord end end + def filter_by_user_type(value) + return unless ::User.user_types.key?(value) + + left_join_users.merge(::User.where(user_type: value)) + end + def sort_by_attribute(method) case method.to_s when 'access_level_asc' then reorder(access_level: :asc) diff --git a/app/models/members/member_role.rb b/app/models/members/member_role.rb deleted file mode 100644 index 42ce228c318..00000000000 --- a/app/models/members/member_role.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -class MemberRole < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass - include IgnorableColumns - ignore_column :download_code, remove_with: '15.9', remove_after: '2023-01-22' - - has_many :members - belongs_to :namespace - - validates :namespace, presence: true - validates :base_access_level, presence: true - validate :belongs_to_top_level_namespace - validate :validate_namespace_locked, on: :update - validate :attributes_locked_after_member_associated, on: :update - - validates_associated :members - - before_destroy :prevent_delete_after_member_associated - - private - - def belongs_to_top_level_namespace - return if !namespace || namespace.root? - - errors.add(:namespace, s_("MemberRole|must be top-level namespace")) - end - - def validate_namespace_locked - return unless namespace_id_changed? - - errors.add(:namespace, s_("MemberRole|can't be changed")) - end - - def attributes_locked_after_member_associated - return unless members.present? - - errors.add(:base, s_("MemberRole|cannot be changed because it is already assigned to a user. "\ - "Please create a new Member Role instead")) - end - - def prevent_delete_after_member_associated - return unless members.present? - - errors.add(:base, s_("MemberRole|cannot be deleted because it is already assigned to a user. "\ - "Please disassociate the member role from all users before deletion.")) - - throw :abort # rubocop:disable Cop/BanCatchThrow - end -end diff --git a/app/models/members_preloader.rb b/app/models/members_preloader.rb index ba7e4b39989..f6617fa0888 100644 --- a/app/models/members_preloader.rb +++ b/app/models/members_preloader.rb @@ -8,12 +8,17 @@ class MembersPreloader end def preload_all - ActiveRecord::Associations::Preloader.new.preload(members, :user) - ActiveRecord::Associations::Preloader.new.preload(members, :source) - ActiveRecord::Associations::Preloader.new.preload(members, :created_by) - ActiveRecord::Associations::Preloader.new.preload(members, user: :status) - ActiveRecord::Associations::Preloader.new.preload(members, user: :u2f_registrations) - ActiveRecord::Associations::Preloader.new.preload(members, user: :webauthn_registrations) if Feature.enabled?(:webauthn) + user_associations = [:status] + user_associations << :webauthn_registrations if Feature.enabled?(:webauthn) + + ActiveRecord::Associations::Preloader.new( + records: members, + associations: [ + :source, + :created_by, + { user: user_associations } + ] + ).call end end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 485ca3a3850..85e95a556a8 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -92,7 +92,7 @@ class MergeRequest < ApplicationRecord fallback || super || MergeRequestDiff.new(merge_request_id: id) end - belongs_to :head_pipeline, foreign_key: "head_pipeline_id", class_name: "Ci::Pipeline" + belongs_to :head_pipeline, class_name: "Ci::Pipeline", inverse_of: :merge_requests_as_head_pipeline has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -141,7 +141,7 @@ class MergeRequest < ApplicationRecord after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed after_save :keep_around_commit, unless: :importing? - after_commit :ensure_metrics, on: [:create, :update], unless: :importing? + after_commit :ensure_metrics!, on: [:create, :update], unless: :importing? after_commit :expire_etag_cache, unless: :importing? # When this attribute is true some MR validation is ignored @@ -156,6 +156,9 @@ class MergeRequest < ApplicationRecord # when creating new merge request attr_accessor :can_be_created, :compare_commits, :diff_options, :compare + # Flag to skip triggering mergeRequestMergeStatusUpdated GraphQL subscription. + attr_accessor :skip_merge_status_trigger + participant :reviewers # Keep states definition to be evaluated before the state_machine block to avoid spec failures. @@ -252,6 +255,8 @@ class MergeRequest < ApplicationRecord end after_transition any => [:unchecked, :cannot_be_merged_recheck, :checking, :cannot_be_merged_rechecking, :can_be_merged, :cannot_be_merged] do |merge_request, transition| + next if merge_request.skip_merge_status_trigger + merge_request.run_after_commit do GraphqlTriggers.merge_request_merge_status_updated(merge_request) end @@ -451,7 +456,12 @@ class MergeRequest < ApplicationRecord def self.total_time_to_merge join_metrics - .merge(MergeRequest::Metrics.with_valid_time_to_merge) + .where( + # Replicating the scope MergeRequest::Metrics.with_valid_time_to_merge + MergeRequest::Metrics.arel_table[:merged_at].gt( + MergeRequest::Metrics.arel_table[:created_at] + ) + ) .pick(MergeRequest::Metrics.time_to_merge_expression) end @@ -558,7 +568,7 @@ class MergeRequest < ApplicationRecord end def self.link_reference_pattern - @link_reference_pattern ||= super("merge_requests", Gitlab::Regex.merge_request) + @link_reference_pattern ||= compose_link_reference_pattern('merge_requests', Gitlab::Regex.merge_request) end def self.reference_valid?(reference) @@ -1943,8 +1953,7 @@ class MergeRequest < ApplicationRecord super.merge(label_url_method: :project_merge_requests_url) end - override :ensure_metrics - def ensure_metrics + def ensure_metrics! MergeRequest::Metrics.record!(self) end diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index 7e2efa2049b..fc08dd4d9c8 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -80,7 +80,7 @@ class MergeRequestDiffCommit < ApplicationRecord def self.prepare_commits_for_bulk_insert(commits) user_tuples = Set.new hashes = commits.map do |commit| - hash = commit.to_hash.except(:parent_ids) + hash = commit.to_hash.except(:parent_ids, :referenced_by) TRIM_USER_KEYS.each do |key| hash[key] = MergeRequest::DiffCommitUser.prepare(hash[key]) diff --git a/app/models/milestone.rb b/app/models/milestone.rb index b0676c25f8e..10d70eaa24e 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -8,6 +8,7 @@ class Milestone < ApplicationRecord include FromUnion include Importable include IidRoutes + include UpdatedAtFilterable prepend_mod_with('Milestone') # rubocop: disable Cop/InjectEnterpriseEditionModule @@ -26,6 +27,7 @@ class Milestone < ApplicationRecord has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + scope :by_iid, ->(iid) { where(iid: iid) } scope :active, -> { with_state(:active) } scope :started, -> { active.where('milestones.start_date <= CURRENT_DATE') } scope :not_started, -> { active.where('milestones.start_date > CURRENT_DATE') } @@ -112,7 +114,7 @@ class Milestone < ApplicationRecord end def self.link_reference_pattern - @link_reference_pattern ||= super("milestones", /(?<milestone>\d+)/) + @link_reference_pattern ||= compose_link_reference_pattern('milestones', /(?<milestone>\d+)/) end def self.upcoming_ids(projects, groups) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 9d9b09e3562..b972d6688af 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -16,6 +16,7 @@ class Namespace < ApplicationRecord include EachBatch include BlocksUnsafeSerialization include Ci::NamespaceSettings + include Referable # Tells ActiveRecord not to store the full class name, in order to save some space # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69794 @@ -51,7 +52,8 @@ class Namespace < ApplicationRecord has_one :namespace_statistics has_one :namespace_route, foreign_key: :namespace_id, autosave: false, inverse_of: :namespace, class_name: 'Route' has_many :namespace_members, foreign_key: :member_namespace_id, inverse_of: :member_namespace, class_name: 'Member' - has_many :member_roles + + has_one :namespace_ldap_settings, inverse_of: :namespace, class_name: 'Namespaces::LdapSetting', autosave: true has_many :runner_namespaces, inverse_of: :namespace, class_name: 'Ci::RunnerNamespace' has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner' @@ -97,6 +99,7 @@ class Namespace < ApplicationRecord validates :path, presence: true, length: { maximum: URL_MAX_LENGTH } + validate :container_registry_namespace_path_validation validates :path, namespace_path: true, if: ->(n) { !n.project_namespace? } # Project path validator is used for project namespaces for now to assure @@ -244,27 +247,42 @@ class Namespace < ApplicationRecord def clean_path(path, limited_to: Namespace.all) slug = Gitlab::Slug::Path.new(path).generate path = Namespaces::RandomizedSuffixPath.new(slug) - Uniquify.new.string(path) { |s| limited_to.find_by_path_or_name(s) } + Gitlab::Utils::Uniquify.new.string(path) { |s| limited_to.find_by_path_or_name(s) } end def clean_name(value) value.scan(Gitlab::Regex.group_name_regex_chars).join(' ') end - def find_by_pages_host(host) - gitlab_host = "." + Settings.pages.host.downcase - host = host.downcase - return unless host.ends_with?(gitlab_host) + def top_most + by_parent(nil) + end - name = host.delete_suffix(gitlab_host) - Namespace.top_most.by_path(name) + def reference_prefix + User.reference_prefix end - def top_most - by_parent(nil) + def reference_pattern + User.reference_pattern end end + def to_reference_base(from = nil, full: false) + return full_path if full || cross_namespace_reference?(from) + return path if cross_project_reference?(from) + end + + def to_reference(*) + "#{self.class.reference_prefix}#{full_path}" + end + + def container_registry_namespace_path_validation + return if Feature.disabled?(:restrict_special_characters_in_namespace_path, self) + return if !path_changed? || path.match?(Gitlab::Regex.oci_repository_path_regex) + + errors.add(:path, Gitlab::Regex.oci_repository_path_regex_message) + end + def package_settings package_setting_relation || build_package_setting_relation end @@ -286,11 +304,15 @@ class Namespace < ApplicationRecord end def any_project_has_container_registry_tags? - all_projects.includes(:container_repositories).any?(&:has_container_registry_tags?) + first_project_with_container_registry_tags.present? end def first_project_with_container_registry_tags - all_projects.find(&:has_container_registry_tags?) + if ContainerRegistry::GitlabApiClient.supports_gitlab_api? && Feature.enabled?(:use_sub_repositories_api) + ContainerRegistry::GitlabApiClient.one_project_with_container_registry_tag(full_path) + else + all_projects.includes(:container_repositories).find(&:has_container_registry_tags?) + end end def send_update_instructions @@ -473,18 +495,6 @@ class Namespace < ApplicationRecord ContainerRepository.for_project_id(all_projects) end - def pages_virtual_domain - cache = if Feature.enabled?(:cache_pages_domain_api, root_ancestor) - ::Gitlab::Pages::CacheControl.for_namespace(root_ancestor.id) - end - - Pages::VirtualDomain.new( - projects: all_projects_with_pages.includes(:route, :project_feature, pages_metadatum: :pages_deployment), - trim_prefix: full_path, - cache: cache - ) - end - def any_project_with_pages_deployed? all_projects.with_pages_deployed.any? end @@ -599,8 +609,44 @@ class Namespace < ApplicationRecord namespace_settings&.all_ancestors_have_runner_registration_enabled? end + def all_projects_with_pages + all_projects.with_pages_deployed.includes( + :route, + :project_setting, + :project_feature, + pages_metadatum: :pages_deployment + ) + end + private + def cross_namespace_reference?(from) + return false if from == self + + comparable_namespace_id = project_namespace? ? parent_id : id + + case from + when Project + from.namespace_id != comparable_namespace_id + when Namespaces::ProjectNamespace + from.parent_id != comparable_namespace_id + when Namespace + parent != from + when User + true + end + end + + # Check if a reference is being done cross-project + def cross_project_reference?(from) + case from + when Project + from.project_namespace_id != id + else + from && self != from + end + end + def update_new_emails_created_column return if namespace_settings.nil? return if namespace_settings.emails_enabled == !emails_disabled @@ -630,10 +676,6 @@ class Namespace < ApplicationRecord end end - def all_projects_with_pages - all_projects.with_pages_deployed - end - def parent_changed? parent_id_changed? end diff --git a/app/models/namespaces/ldap_setting.rb b/app/models/namespaces/ldap_setting.rb new file mode 100644 index 00000000000..73125d347cc --- /dev/null +++ b/app/models/namespaces/ldap_setting.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Namespaces + class LdapSetting < ApplicationRecord + belongs_to :namespace, inverse_of: :namespace_ldap_settings + validates :namespace, presence: true + + self.primary_key = :namespace_id + self.table_name = 'namespace_ldap_settings' + end +end diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 0e9760832af..0fae66b18ca 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -127,9 +127,13 @@ module Namespaces return super unless use_traversal_ids_for_root_ancestor? strong_memoize(:root_ancestor) do - if parent_id.nil? + if association(:parent).loaded? && parent.present? + # This case is possible when parent has not been persisted or we're inside a transaction. + parent.root_ancestor + elsif parent_id.nil? + # There is no parent, so we are the root ancestor. self - else + elsif traversal_ids.present? Namespace.find_by(id: traversal_ids.first) end end @@ -215,6 +219,16 @@ module Namespaces hierarchy_order == :desc ? traversal_ids : traversal_ids.reverse end + def parent=(obj) + super(obj) + set_traversal_ids + end + + def parent_id=(id) + super(id) + set_traversal_ids + end + private attr_accessor :transient_traversal_ids @@ -232,11 +246,11 @@ module Namespaces end def set_traversal_ids + return if id.blank? + # This is a temporary guard and will be removed. return if is_a?(Namespaces::ProjectNamespace) - return unless Feature.enabled?(:set_traversal_ids_on_save, root_ancestor) - self.transient_traversal_ids = if parent_id parent.traversal_ids + [id] else @@ -244,7 +258,7 @@ module Namespaces end # Clear root_ancestor memo if changed. - if read_attribute(traversal_ids)&.first != transient_traversal_ids.first + if read_attribute(:traversal_ids)&.first != transient_traversal_ids.first clear_memoization(:root_ancestor) end diff --git a/app/models/note.rb b/app/models/note.rb index a64f7311725..b9b884b88c5 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -60,6 +60,9 @@ class Note < ApplicationRecord # Attribute used to store the attributes that have been changed by quick actions. attr_writer :commands_changes + # Attribute used to store the quick action command names. + attr_accessor :command_names + # Attribute used to determine whether keep_around_commits will be skipped for diff notes. attr_accessor :skip_keep_around_commits @@ -169,7 +172,6 @@ class Note < ApplicationRecord project: [:project_members, :namespace, { group: [:group_members] }]) end scope :with_metadata, -> { includes(:system_note_metadata) } - scope :with_web_entity_associations, -> { preload(:project, :author, :noteable) } scope :for_note_or_capitalized_note, ->(text) { where(note: [text, text.capitalize]) } scope :like_note_or_capitalized_note, ->(text) { where('(note LIKE ? OR note LIKE ?)', text, text.capitalize) } @@ -288,6 +290,10 @@ class Note < ApplicationRecord def cherry_picked_merge_requests(shas) where(noteable_type: 'MergeRequest', commit_id: shas).select(:noteable_id) end + + def with_web_entity_associations + preload(:project, :author, :noteable) + end end # rubocop: disable CodeReuse/ServiceClass @@ -330,6 +336,10 @@ class Note < ApplicationRecord noteable_type == "Issue" end + def for_work_item? + noteable.is_a?(WorkItem) + end + def for_merge_request? noteable_type == "MergeRequest" end diff --git a/app/models/oauth_access_token.rb b/app/models/oauth_access_token.rb index 8e79a750793..601381f1c65 100644 --- a/app/models/oauth_access_token.rb +++ b/app/models/oauth_access_token.rb @@ -4,6 +4,8 @@ class OauthAccessToken < Doorkeeper::AccessToken belongs_to :resource_owner, class_name: 'User' belongs_to :application, class_name: 'Doorkeeper::Application' + validates :expires_in, presence: true + alias_attribute :user, :resource_owner scope :latest_per_application, -> { select('distinct on(application_id) *').order(application_id: :desc, created_at: :desc) } diff --git a/app/models/onboarding/completion.rb b/app/models/onboarding/completion.rb index 269283df826..0966a9f2912 100644 --- a/app/models/onboarding/completion.rb +++ b/app/models/onboarding/completion.rb @@ -5,52 +5,65 @@ module Onboarding include Gitlab::Utils::StrongMemoize include Gitlab::Experiment::Dsl - ACTION_ISSUE_IDS = { - trial_started: 2, - required_mr_approvals_enabled: 11, - code_owners_enabled: 10 - }.freeze - ACTION_PATHS = [ :pipeline_created, + :trial_started, + :required_mr_approvals_enabled, + :code_owners_enabled, :issue_created, :git_write, :merge_request_created, :user_added ].freeze - def initialize(namespace, current_user = nil) - @namespace = namespace + def initialize(project, current_user = nil) + @project = project + @namespace = project.namespace @current_user = current_user end def percentage return 0 unless onboarding_progress - attributes = onboarding_progress.attributes.symbolize_keys - total_actions = action_columns.count - completed_actions = action_columns.count { |column| attributes[column].present? } + completed_actions = action_columns.count { |column| completed?(column) } (completed_actions.to_f / total_actions * 100).round end + def completed?(column) + if column == :code_added + repository.commit_count > 1 || repository.branch_count > 1 + else + attributes[column].present? + end + end + private + def repository + project.repository + end + strong_memoize_attr :repository + + def attributes + onboarding_progress.attributes.symbolize_keys + end + strong_memoize_attr :attributes + def onboarding_progress - strong_memoize(:onboarding_progress) do - ::Onboarding::Progress.find_by(namespace: namespace) - end + ::Onboarding::Progress.find_by(namespace: namespace) end + strong_memoize_attr :onboarding_progress def action_columns - strong_memoize(:action_columns) do + [:code_added] + tracked_actions.map { |action_key| ::Onboarding::Progress.column_name(action_key) } - end end + strong_memoize_attr :action_columns def tracked_actions - ACTION_ISSUE_IDS.keys + ACTION_PATHS + deploy_section_tracked_actions + ACTION_PATHS + deploy_section_tracked_actions end def deploy_section_tracked_actions @@ -65,6 +78,6 @@ module Onboarding end.run end - attr_reader :namespace, :current_user + attr_reader :project, :namespace, :current_user end end diff --git a/app/models/operations/feature_flag.rb b/app/models/operations/feature_flag.rb index 0df8c87f73f..6876af09c2c 100644 --- a/app/models/operations/feature_flag.rb +++ b/app/models/operations/feature_flag.rb @@ -72,7 +72,7 @@ module Operations end def link_reference_pattern - @link_reference_pattern ||= super("feature_flags", %r{(?<feature_flag>\d+)/edit}) + @link_reference_pattern ||= compose_link_reference_pattern('feature_flags', %r{(?<feature_flag>\d+)/edit}) end def reference_postfix diff --git a/app/models/packages/debian.rb b/app/models/packages/debian.rb index 9c615c20250..887a5695530 100644 --- a/app/models/packages/debian.rb +++ b/app/models/packages/debian.rb @@ -10,6 +10,8 @@ module Packages LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze + EMPTY_FILE_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'.freeze + def self.table_name_prefix 'packages_debian_' end diff --git a/app/models/packages/debian/file_metadatum.rb b/app/models/packages/debian/file_metadatum.rb index eb1b03a8e9d..77ce8e265ff 100644 --- a/app/models/packages/debian/file_metadatum.rb +++ b/app/models/packages/debian/file_metadatum.rb @@ -9,14 +9,14 @@ class Packages::Debian::FileMetadatum < ApplicationRecord validate :valid_debian_package_type enum file_type: { - unknown: 1, source: 2, dsc: 3, deb: 4, udeb: 5, buildinfo: 6, changes: 7 + unknown: 1, source: 2, dsc: 3, deb: 4, udeb: 5, buildinfo: 6, changes: 7, ddeb: 8 } validates :file_type, presence: true validates :file_type, inclusion: { in: %w[unknown] }, if: -> { package_file&.package&.debian_incoming? || package_file&.package&.processing? } validates :file_type, - inclusion: { in: %w[source dsc deb udeb buildinfo changes] }, + inclusion: { in: %w[source dsc deb udeb buildinfo changes ddeb] }, if: -> { package_file&.package&.debian_package? && !package_file&.package&.processing? } validates :component, @@ -46,7 +46,7 @@ class Packages::Debian::FileMetadatum < ApplicationRecord end def requires_architecture? - deb? || udeb? + deb? || udeb? || ddeb? end def requires_component? diff --git a/app/models/packages/rpm/repository_file.rb b/app/models/packages/rpm/repository_file.rb index 614ec9b3e56..bbd435691d2 100644 --- a/app/models/packages/rpm/repository_file.rb +++ b/app/models/packages/rpm/repository_file.rb @@ -13,7 +13,7 @@ module Packages enum status: { default: 0, pending_destruction: 1, processing: 2, error: 3 } - belongs_to :project, inverse_of: :repository_files + belongs_to :project, inverse_of: :rpm_repository_files validates :project, presence: true validates :file, presence: true diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index a1ba48f3ab0..222cde19da7 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -49,19 +49,25 @@ module Pages if project.pages_namespace_url == project.pages_url '/' else - project.full_path.delete_prefix(trim_prefix) + '/' + "#{project.full_path.delete_prefix(trim_prefix)}/" end end strong_memoize_attr :prefix + def unique_domain + return unless project.project_setting.pages_unique_domain_enabled? + + project.project_setting.pages_unique_domain + end + strong_memoize_attr :unique_domain + private attr_reader :project, :trim_prefix, :domain def deployment - strong_memoize(:deployment) do - project.pages_metadatum.pages_deployment - end + project.pages_metadatum.pages_deployment end + strong_memoize_attr :deployment end end diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 909658214fd..446c4a6187c 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -15,7 +15,6 @@ class PagesDomain < ApplicationRecord belongs_to :project has_many :acme_orders, class_name: "PagesDomainAcmeOrder" - has_many :serverless_domain_clusters, class_name: 'Serverless::DomainCluster', inverse_of: :pages_domain after_initialize :set_verification_code before_validation :clear_auto_ssl_failure, unless: :auto_ssl_enabled @@ -209,20 +208,6 @@ class PagesDomain < ApplicationRecord self.certificate_source = 'gitlab_provided' if attribute_changed?(:key) end - def pages_virtual_domain - return unless pages_deployed? - - cache = if Feature.enabled?(:cache_pages_domain_api, project.root_namespace) - ::Gitlab::Pages::CacheControl.for_domain(id) - end - - Pages::VirtualDomain.new( - projects: [project], - domain: self, - cache: cache - ) - end - def clear_auto_ssl_failure self.auto_ssl_failed = false end @@ -237,14 +222,14 @@ class PagesDomain < ApplicationRecord end end - private - def pages_deployed? return false unless project project.pages_metadatum&.deployed? end + private + def set_verification_code return if self.verification_code.present? diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index f99c4c6c39d..2e613768873 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -9,7 +9,9 @@ class PersonalAccessToken < ApplicationRecord include Gitlab::SQL::Pattern extend ::Gitlab::Utils::Override - add_authentication_token_field :token, digest: true + add_authentication_token_field :token, + digest: true, + format_with_prefix: :prefix_from_application_current_settings # PATs are 20 characters + optional configurable settings prefix (0..20) TOKEN_LENGTH_RANGE = (20..40).freeze @@ -72,11 +74,6 @@ class PersonalAccessToken < ApplicationRecord fuzzy_search(query, [:name]) end - override :format_token - def format_token(token) - "#{self.class.token_prefix}#{token}" - end - def project_access_token? user&.project_bot? end @@ -107,6 +104,10 @@ class PersonalAccessToken < ApplicationRecord def add_admin_mode_scope self.scopes += [Gitlab::Auth::ADMIN_MODE_SCOPE.to_s] end + + def prefix_from_application_current_settings + self.class.token_prefix + end end PersonalAccessToken.prepend_mod_with('PersonalAccessToken') diff --git a/app/models/preloaders/commit_status_preloader.rb b/app/models/preloaders/commit_status_preloader.rb index 535dd24ba6b..79c2549e371 100644 --- a/app/models/preloaders/commit_status_preloader.rb +++ b/app/models/preloaders/commit_status_preloader.rb @@ -9,10 +9,11 @@ module Preloaders end def execute(relations) - preloader = ActiveRecord::Associations::Preloader.new - CLASSES.each do |klass| - preloader.preload(objects(klass), associations(klass, relations)) + ActiveRecord::Associations::Preloader.new( + records: objects(klass), + associations: associations(klass, relations) + ).call end end diff --git a/app/models/preloaders/labels_preloader.rb b/app/models/preloaders/labels_preloader.rb index b6e73c1cd02..2a3175be420 100644 --- a/app/models/preloaders/labels_preloader.rb +++ b/app/models/preloaders/labels_preloader.rb @@ -19,11 +19,20 @@ module Preloaders end def preload_all - preloader = ActiveRecord::Associations::Preloader.new + ActiveRecord::Associations::Preloader.new( + records: labels, + associations: { parent_container: :route } + ).call - preloader.preload(labels, parent_container: :route) - preloader.preload(labels.select { |l| l.is_a? ProjectLabel }, { project: [:project_feature, namespace: :route] }) - preloader.preload(labels.select { |l| l.is_a? GroupLabel }, { group: :route }) + ActiveRecord::Associations::Preloader.new( + records: labels.select { |l| l.is_a? ProjectLabel }, + associations: { project: [:project_feature, namespace: :route] } + ).call + + ActiveRecord::Associations::Preloader.new( + records: labels.select { |l| l.is_a? GroupLabel }, + associations: { group: :route } + ).call labels.each do |label| label.lazy_subscription(user) diff --git a/app/models/preloaders/project_policy_preloader.rb b/app/models/preloaders/project_policy_preloader.rb index fe9db3464c7..e16eabf40a1 100644 --- a/app/models/preloaders/project_policy_preloader.rb +++ b/app/models/preloaders/project_policy_preloader.rb @@ -10,7 +10,10 @@ module Preloaders def execute return if projects.is_a?(ActiveRecord::NullRelation) - ActiveRecord::Associations::Preloader.new.preload(projects, { group: :route, namespace: :owner }) + ActiveRecord::Associations::Preloader.new( + records: projects, + associations: { group: :route, namespace: :owner } + ).call ::Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects, current_user).execute end diff --git a/app/models/preloaders/project_root_ancestor_preloader.rb b/app/models/preloaders/project_root_ancestor_preloader.rb index 6192f79ce2c..ccb9d2eab98 100644 --- a/app/models/preloaders/project_root_ancestor_preloader.rb +++ b/app/models/preloaders/project_root_ancestor_preloader.rb @@ -19,7 +19,7 @@ module Preloaders root_ancestors_by_id = root_query.group_by(&:source_id) - ActiveRecord::Associations::Preloader.new.preload(@projects, :namespace) + ActiveRecord::Associations::Preloader.new(records: @projects, associations: :namespace).call @projects.each do |project| root_ancestor = root_ancestors_by_id[project.id]&.first project.namespace.root_ancestor = root_ancestor if root_ancestor.present? diff --git a/app/models/preloaders/runner_machine_policy_preloader.rb b/app/models/preloaders/runner_machine_policy_preloader.rb new file mode 100644 index 00000000000..52864eeba8d --- /dev/null +++ b/app/models/preloaders/runner_machine_policy_preloader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Preloaders + class RunnerMachinePolicyPreloader + def initialize(runner_machines, current_user) + @runner_machines = runner_machines + @current_user = current_user + end + + def execute + return if runner_machines.is_a?(ActiveRecord::NullRelation) + + ActiveRecord::Associations::Preloader.new( + records: runner_machines, + associations: [:runner] + ).call + end + + private + + attr_reader :runner_machines, :current_user + end +end diff --git a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb index 0c747ad9c84..16d46facb96 100644 --- a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb +++ b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb @@ -46,14 +46,10 @@ module Preloaders end def all_memberships - if Feature.enabled?(:include_memberships_from_group_shares_in_preloader) - [ - direct_memberships.select(*GroupMember.cached_column_list), - memberships_from_group_shares - ] - else - [direct_memberships] - end + [ + direct_memberships.select(*GroupMember.cached_column_list), + memberships_from_group_shares + ] end def direct_memberships diff --git a/app/models/project.rb b/app/models/project.rb index 43ec26be786..cb218c0a49f 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -19,6 +19,7 @@ class Project < ApplicationRecord include Presentable include HasRepository include HasWiki + include WebHooks::HasWebHooks include CanMoveRepositoryStorage include Routable include GroupDescendant @@ -41,7 +42,6 @@ class Project < ApplicationRecord include BlocksUnsafeSerialization include Subquery include IssueParent - include WebHooks::HasWebHooks extend Gitlab::Cache::RequestCache extend Gitlab::Utils::Override @@ -89,6 +89,14 @@ class Project < ApplicationRecord DEFAULT_SQUASH_COMMIT_TEMPLATE = '%{title}' + PROJECT_FEATURES_DEFAULTS = { + issues: gitlab_config_features.issues, + merge_requests: gitlab_config_features.merge_requests, + builds: gitlab_config_features.builds, + wiki: gitlab_config_features.wiki, + snippets: gitlab_config_features.snippets + }.freeze + cache_markdown_field :description, pipeline: :description attribute :packages_enabled, default: true @@ -101,18 +109,14 @@ class Project < ApplicationRecord attribute :autoclose_referenced_issues, default: true attribute :ci_config_path, default: -> { Gitlab::CurrentSettings.default_ci_config_path } - default_value_for :issues_enabled, gitlab_config_features.issues - default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests - default_value_for :builds_enabled, gitlab_config_features.builds - default_value_for :wiki_enabled, gitlab_config_features.wiki - default_value_for :snippets_enabled, gitlab_config_features.snippets - add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:projects_tokens_optional_encryption) ? :optional : :required }, - prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + format_with_prefix: :runners_token_prefix, + require_prefix_for_validation: true # Storage specific hooks after_initialize :use_hashed_storage + after_initialize :set_project_feature_defaults, if: :new_record? before_validation :mark_remote_mirrors_for_removal, if: -> { RemoteMirror.table_exists? } before_validation :ensure_project_namespace_in_sync @@ -128,7 +132,6 @@ class Project < ApplicationRecord after_create -> { create_or_load_association(:pages_metadatum) } after_create :set_timestamps_for_create after_create :check_repository_absence! - after_update :update_forks_visibility_level before_destroy :remove_private_deploy_keys after_destroy :remove_exports after_save :update_project_statistics, if: :saved_change_to_namespace_id? @@ -168,6 +171,8 @@ class Project < ApplicationRecord has_one :last_event, -> { order 'events.created_at DESC' }, class_name: 'Event' has_many :boards + has_many :application_setting, inverse_of: :self_monitoring_project + def self.integration_association_name(name) "#{name}_integration" end @@ -188,6 +193,7 @@ class Project < ApplicationRecord has_one :emails_on_push_integration, class_name: 'Integrations::EmailsOnPush' has_one :ewm_integration, class_name: 'Integrations::Ewm' has_one :external_wiki_integration, class_name: 'Integrations::ExternalWiki' + has_one :google_play_integration, class_name: 'Integrations::GooglePlay' has_one :hangouts_chat_integration, class_name: 'Integrations::HangoutsChat' has_one :harbor_integration, class_name: 'Integrations::Harbor' has_one :irker_integration, class_name: 'Integrations::Irker' @@ -208,6 +214,7 @@ class Project < ApplicationRecord has_one :shimo_integration, class_name: 'Integrations::Shimo' has_one :slack_integration, class_name: 'Integrations::Slack' has_one :slack_slash_commands_integration, class_name: 'Integrations::SlackSlashCommands' + has_one :squash_tm_integration, class_name: 'Integrations::SquashTm' has_one :teamcity_integration, class_name: 'Integrations::Teamcity' has_one :unify_circuit_integration, class_name: 'Integrations::UnifyCircuit' has_one :webex_teams_integration, class_name: 'Integrations::WebexTeams' @@ -238,14 +245,22 @@ class Project < ApplicationRecord has_many :fork_network_projects, through: :fork_network, source: :projects # Packages - has_many :packages, class_name: 'Packages::Package' - has_many :package_files, through: :packages, class_name: 'Packages::PackageFile' + has_many :packages, + class_name: 'Packages::Package' + has_many :package_files, + through: :packages, class_name: 'Packages::PackageFile' # repository_files must be destroyed by ruby code in order to properly remove carrierwave uploads - has_many :repository_files, inverse_of: :project, class_name: 'Packages::Rpm::RepositoryFile', - dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :rpm_repository_files, + inverse_of: :project, + class_name: 'Packages::Rpm::RepositoryFile', + dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads - has_many :debian_distributions, class_name: 'Packages::Debian::ProjectDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_one :packages_cleanup_policy, class_name: 'Packages::Cleanup::Policy', inverse_of: :project + has_many :debian_distributions, + class_name: 'Packages::Debian::ProjectDistribution', + dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_one :packages_cleanup_policy, + class_name: 'Packages::Cleanup::Policy', + inverse_of: :project has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -259,6 +274,7 @@ class Project < ApplicationRecord has_one :project_setting, inverse_of: :project, autosave: true has_one :alerting_setting, inverse_of: :project, class_name: 'Alerting::ProjectAlertingSetting' has_one :service_desk_setting, class_name: 'ServiceDeskSetting' + has_one :service_desk_custom_email_verification, class_name: 'ServiceDesk::CustomEmailVerification' # Merge requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id', inverse_of: :target_project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -371,7 +387,6 @@ class Project < ApplicationRecord inverse_of: :project has_many :stages, class_name: 'Ci::Stage', inverse_of: :project has_many :ci_refs, class_name: 'Ci::Ref', inverse_of: :project - has_many :pipeline_metadata, class_name: 'Ci::PipelineMetadata', inverse_of: :project has_many :pending_builds, class_name: 'Ci::PendingBuild' has_many :builds, class_name: 'Ci::Build', inverse_of: :project @@ -874,7 +889,7 @@ class Project < ApplicationRecord def reference_pattern %r{ (?<!#{Gitlab::PathRegex::PATH_START_CHAR}) - ((?<namespace>#{Gitlab::PathRegex::FULL_NAMESPACE_FORMAT_REGEX})\/)? + ((?<namespace>#{Gitlab::PathRegex::FULL_NAMESPACE_FORMAT_REGEX})/)? (?<project>#{Gitlab::PathRegex::PROJECT_PATH_FORMAT_REGEX}) }xo end @@ -950,27 +965,44 @@ class Project < ApplicationRecord .where(pending_delete: false) .where(archived: false) end + + def project_features_defaults + PROJECT_FEATURES_DEFAULTS + end + + def by_pages_enabled_unique_domain(domain) + without_deleted + .joins(:project_setting) + .find_by(project_setting: { + pages_unique_domain_enabled: true, + pages_unique_domain: domain + }) + end end def initialize(attributes = nil) - # We can't use default_value_for because the database has a default - # value of 0 for visibility_level. If someone attempts to create a - # private project, default_value_for will assume that the - # visibility_level hasn't changed and will use the application - # setting default, which could be internal or public. For projects - # inside a private group, those levels are invalid. - # - # To fix the problem, we assign the actual default in the application if - # no explicit visibility has been initialized. + # We assign the actual snippet default if no explicit visibility has been initialized. attributes ||= {} unless visibility_attribute_present?(attributes) attributes[:visibility_level] = Gitlab::CurrentSettings.default_project_visibility end + @init_attributes = attributes + super end + # Remove along with ProjectFeaturesCompatibility module + def set_project_feature_defaults + self.class.project_features_defaults.each do |attr, value| + # If the deprecated _enabled or the accepted _access_level attribute is specified, we don't need to set the default + next unless @init_attributes[:"#{attr}_enabled"].nil? && @init_attributes[:"#{attr}_access_level"].nil? + + public_send("#{attr}_enabled=", value) # rubocop:disable GitlabSecurity/PublicSend + end + end + def parent_loaded? association(:namespace).loaded? end @@ -1077,8 +1109,10 @@ class Project < ApplicationRecord end def preload_protected_branches - preloader = ActiveRecord::Associations::Preloader.new - preloader.preload(self, protected_branches: [:push_access_levels, :merge_access_levels]) + ActiveRecord::Associations::Preloader.new( + records: [self], + associations: { protected_branches: [:push_access_levels, :merge_access_levels] } + ).call end # returns all ancestor-groups upto but excluding the given namespace @@ -1089,11 +1123,7 @@ class Project < ApplicationRecord end def ancestors(hierarchy_order: nil) - if Feature.enabled?(:linear_project_ancestors, self) - group&.self_and_ancestors(hierarchy_order: hierarchy_order) || Group.none - else - ancestors_upto(hierarchy_order: hierarchy_order) - end + group&.self_and_ancestors(hierarchy_order: hierarchy_order) || Group.none end def ancestors_upto_ids(...) @@ -1154,10 +1184,6 @@ class Project < ApplicationRecord { scope: :project, status: auto_devops&.enabled || Feature.enabled?(:force_autodevops_on_by_default, self) } end - def unlink_forks_upon_visibility_decrease_enabled? - Feature.enabled?(:unlink_fork_network_upon_visibility_decrease, self) - end - # LFS and hashed repository storage are required for using Design Management. def design_management_enabled? lfs_enabled? && hashed_storage?(:repository) @@ -1177,15 +1203,6 @@ class Project < ApplicationRecord end end - # Because we use default_value_for we need to be sure - # packages_enabled= method does exist even if we rollback migration. - # Otherwise many tests from spec/migrations will fail. - def packages_enabled=(value) - if has_attribute?(:packages_enabled) - write_attribute(:packages_enabled, value) - end - end - def cleanup @repository = nil end @@ -1272,6 +1289,18 @@ class Project < ApplicationRecord import_state&.human_status_name || 'none' end + def beautified_import_status_name + if import_finished? + return 'completed' unless import_checksums.present? + + fetched = import_checksums['fetched'] + imported = import_checksums['imported'] + fetched.keys.any? { |key| fetched[key] != imported[key] } ? 'partially completed' : 'completed' + else + import_status + end + end + def add_import_job job_id = if forked? @@ -1314,6 +1343,11 @@ class Project < ApplicationRecord super(value&.delete("\0")) end + # Used by Import/Export to export commit notes + def commit_notes + notes.where(noteable_type: "Commit") + end + def import_url=(value) if Gitlab::UrlSanitizer.valid?(value) import_url = Gitlab::UrlSanitizer.new(value) @@ -1631,7 +1665,7 @@ class Project < ApplicationRecord def disabled_integrations disabled_integrations = [] - disabled_integrations << 'apple_app_store' unless Feature.enabled?(:apple_app_store_integration, self) + disabled_integrations << 'google_play' unless Feature.enabled?(:google_play_integration, self) disabled_integrations end @@ -1935,19 +1969,6 @@ class Project < ApplicationRecord create_repository(force: true) unless repository_exists? end - # update visibility_level of forks - def update_forks_visibility_level - return if unlink_forks_upon_visibility_decrease_enabled? - return unless visibility_level < visibility_level_before_last_save - - forks.each do |forked_project| - if forked_project.visibility_level > visibility_level - forked_project.visibility_level = visibility_level - forked_project.save! - end - end - end - def allowed_to_share_with_group? !namespace.share_with_group_lock end @@ -2080,7 +2101,11 @@ class Project < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def open_merge_requests_count(_current_user = nil) - Projects::OpenMergeRequestsCountService.new(self).count + BatchLoader.for(self).batch do |projects, loader| + ::Projects::BatchOpenMergeRequestsCountService.new(projects) + .refresh_cache_and_retrieve_data + .each { |project, count| loader.call(project, count) } + end end # rubocop: enable CodeReuse/ServiceClass @@ -2107,23 +2132,13 @@ class Project < ApplicationRecord ensure_runners_token! end - override :format_runners_token - def format_runners_token(token) - "#{RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX}#{token}" - end - def pages_deployed? pages_metadatum&.deployed? end - def pages_namespace_url - # The host in URL always needs to be downcased - Gitlab.config.pages.url.sub(%r{^https?://}) do |prefix| - "#{prefix}#{pages_subdomain}." - end.downcase - end - def pages_url + return pages_unique_url if pages_unique_domain_enabled? + url = pages_namespace_url url_path = full_path.partition('/').last namespace_url = "#{Settings.pages.protocol}://#{url_path}".downcase @@ -2141,6 +2156,14 @@ class Project < ApplicationRecord "#{url}/#{url_path}" end + def pages_unique_url + pages_url_for(project_setting.pages_unique_domain) + end + + def pages_namespace_url + pages_url_for(pages_subdomain) + end + def pages_subdomain full_path.partition('/').first end @@ -2809,7 +2832,7 @@ class Project < ApplicationRecord end def all_protected_branches - if Feature.enabled?(:group_protected_branches) + if Feature.enabled?(:group_protected_branches, group) @all_protected_branches ||= ProtectedBranch.from_union([protected_branches, group_protected_branches]) else protected_branches @@ -2971,7 +2994,7 @@ class Project < ApplicationRecord end def ci_inbound_job_token_scope_enabled? - return false unless ci_cd_settings + return true unless ci_cd_settings ci_cd_settings.inbound_job_token_scope_enabled? end @@ -3121,6 +3144,18 @@ class Project < ApplicationRecord private + def pages_unique_domain_enabled? + Feature.enabled?(:pages_unique_domain) && + project_setting.pages_unique_domain_enabled? + end + + def pages_url_for(domain) + # The host in URL always needs to be downcased + Gitlab.config.pages.url.sub(%r{^https?://}) do |prefix| + "#{prefix}#{domain}." + end.downcase + end + # overridden in EE def project_group_links_with_preload project_group_links @@ -3224,6 +3259,8 @@ class Project < ApplicationRecord case from when Project namespace_id != from.namespace_id + when Namespaces::ProjectNamespace + namespace_id != from.parent_id when Namespace namespace != from when User @@ -3233,9 +3270,14 @@ class Project < ApplicationRecord # Check if a reference is being done cross-project def cross_project_reference?(from) - return true if from.is_a?(Namespace) - - from && self != from + case from + when Namespaces::ProjectNamespace + project_namespace_id != from.id + when Namespace + true + else + from && self != from + end end def update_project_statistics @@ -3401,6 +3443,10 @@ class Project < ApplicationRecord project_setting.emails_enabled = !emails_disabled end end + + def runners_token_prefix + RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + end end Project.prepend_mod_with('Project') diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index 8741a341ad3..cc9003423be 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -20,10 +20,6 @@ class ProjectCiCdSetting < ApplicationRecord attribute :forward_deployment_enabled, default: true attribute :separated_caches, default: true - default_value_for :inbound_job_token_scope_enabled do |settings| - Feature.enabled?(:ci_inbound_job_token_scope, settings.project) - end - chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval def keep_latest_artifacts_available? diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 168646bbe41..053ccfac050 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -162,6 +162,12 @@ class ProjectFeature < ApplicationRecord end end + def public_packages? + return false unless Gitlab.config.packages.enabled + + package_registry_access_level == PUBLIC || project.public? + end + private def set_pages_access_level diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index db86bb5e1fb..379b94b3af5 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -25,6 +25,10 @@ class ProjectSetting < ApplicationRecord validates :target_platforms, inclusion: { in: ALLOWED_TARGET_PLATFORMS } validates :suggested_reviewers_enabled, inclusion: { in: [true, false] } + validates :pages_unique_domain, + uniqueness: { if: -> { pages_unique_domain.present? } }, + presence: { if: :require_unique_domain? } + validate :validates_mr_default_target_self attribute :legacy_open_source_license_available, default: -> do @@ -68,6 +72,11 @@ class ProjectSetting < ApplicationRecord errors.add :mr_default_target_self, _('This setting is allowed for forked projects only') end end + + def require_unique_domain? + pages_unique_domain_enabled || + pages_unique_domain_in_database.present? + end end ProjectSetting.prepend_mod diff --git a/app/models/projects/data_transfer.rb b/app/models/projects/data_transfer.rb index a93aea55781..faab0bb6db2 100644 --- a/app/models/projects/data_transfer.rb +++ b/app/models/projects/data_transfer.rb @@ -4,6 +4,9 @@ # This class ensures that we keep 1 record per project per month. module Projects class DataTransfer < ApplicationRecord + include AfterCommitQueue + include CounterAttribute + self.table_name = 'project_data_transfers' belongs_to :project @@ -11,6 +14,11 @@ module Projects scope :current_month, -> { where(date: beginning_of_month) } + counter_attribute :repository_egress, returns_current: true + counter_attribute :artifacts_egress, returns_current: true + counter_attribute :packages_egress, returns_current: true + counter_attribute :registry_egress, returns_current: true + def self.beginning_of_month(time = Time.current) time.utc.beginning_of_month end diff --git a/app/models/projects/forks/divergence_counts.rb b/app/models/projects/forks/details.rb index 7d630b00083..9e09ef09022 100644 --- a/app/models/projects/forks/divergence_counts.rb +++ b/app/models/projects/forks/details.rb @@ -3,8 +3,11 @@ module Projects module Forks # Class for calculating the divergence of a fork with the source project - class DivergenceCounts + class Details + include Gitlab::Utils::StrongMemoize + LATEST_COMMITS_COUNT = 10 + LEASE_TIMEOUT = 15.minutes.to_i EXPIRATION_TIME = 8.hours def initialize(project, ref) @@ -20,32 +23,55 @@ module Projects { ahead: ahead, behind: behind } end + def exclusive_lease + key = ['project_details', project.id, ref].join(':') + uuid = Gitlab::ExclusiveLease.get_uuid(key) + + Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT) + end + strong_memoize_attr :exclusive_lease + + def syncing? + exclusive_lease.exists? + end + + def has_conflicts? + !(attrs && attrs[:has_conflicts]).nil? + end + + def update!(params) + Rails.cache.write(cache_key, params, expires_in: EXPIRATION_TIME) + + @attrs = nil + end + private attr_reader :project, :fork_repo, :source_repo, :ref def cache_key - @cache_key ||= ['project_forks', project.id, ref, 'divergence_counts'] + @cache_key ||= ['project_fork_details', project.id, ref].join(':') end def divergence_counts - fork_sha = fork_repo.commit(ref).sha - source_sha = source_repo.commit.sha + sha = fork_repo.commit(ref)&.sha + source_sha = source_repo.commit&.sha - cached_source_sha, cached_fork_sha, counts = Rails.cache.read(cache_key) - return counts if cached_source_sha == source_sha && cached_fork_sha == fork_sha + return if sha.blank? || source_sha.blank? - counts = calculate_divergence_counts(fork_sha, source_sha) + return attrs[:counts] if attrs.present? && attrs[:source_sha] == source_sha && attrs[:sha] == sha - Rails.cache.write(cache_key, [source_sha, fork_sha, counts], expires_in: EXPIRATION_TIME) + counts = calculate_divergence_counts(sha, source_sha) + + update!({ sha: sha, source_sha: source_sha, counts: counts }) counts end - def calculate_divergence_counts(fork_sha, source_sha) + def calculate_divergence_counts(sha, source_sha) # If the upstream latest commit exists in the fork repo, then # it's possible to calculate divergence counts within the fork repository. - return fork_repo.diverging_commit_count(fork_sha, source_sha) if fork_repo.commit(source_sha) + return fork_repo.diverging_commit_count(sha, source_sha) if fork_repo.commit(source_sha) # Otherwise, we need to find a commit that exists both in the fork and upstream # in order to use this commit as a base for calculating divergence counts. @@ -67,6 +93,10 @@ module Projects [ahead, behind] end + + def attrs + @attrs ||= Rails.cache.read(cache_key) + end end end end diff --git a/app/models/projects/import_export/relation_export.rb b/app/models/projects/import_export/relation_export.rb index 9bdf10d7c0e..2771c5131b2 100644 --- a/app/models/projects/import_export/relation_export.rb +++ b/app/models/projects/import_export/relation_export.rb @@ -51,12 +51,16 @@ module Projects transition queued: :started end + event :retry do + transition started: :queued + end + event :finish do transition started: :finished end event :fail_op do - transition [:queued, :started] => :failed + transition [:queued, :started, :failed] => :failed end end @@ -65,6 +69,14 @@ module Projects project_tree_relation_names + EXTRA_RELATION_LIST end + + def mark_as_failed(export_error) + sanitized_error = Gitlab::UrlSanitizer.sanitize(export_error) + + fail_op + + update_column(:export_error, sanitized_error) + end end end end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index b3331b99a6b..22eaac94897 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -37,38 +37,13 @@ class ProtectedBranch < ApplicationRecord return true if project.empty_repo? && project.default_branch_protected? return false if ref_name.blank? - dry_run = Feature.disabled?(:rely_on_protected_branches_cache, project) - - new_cache_result = new_cache(project, ref_name, dry_run: dry_run) - - return new_cache_result unless new_cache_result.nil? - - deprecated_cache(project, ref_name) - end - - def self.new_cache(project, ref_name, dry_run: true) - ProtectedBranches::CacheService.new(project).fetch(ref_name, dry_run: dry_run) do # rubocop: disable CodeReuse/ServiceClass - self.matching(ref_name, protected_refs: protected_refs(project)).present? - end - end - - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/370608 - # ---------------------------------------------------------------- - CACHE_EXPIRE_IN = 1.hour - - def self.deprecated_cache(project, ref_name) - Rails.cache.fetch(protected_ref_cache_key(project, ref_name), expires_in: CACHE_EXPIRE_IN) do + ProtectedBranches::CacheService.new(project).fetch(ref_name) do # rubocop: disable CodeReuse/ServiceClass self.matching(ref_name, protected_refs: protected_refs(project)).present? end end - def self.protected_ref_cache_key(project, ref_name) - "protected_ref-#{project.cache_key}-#{Digest::SHA1.hexdigest(ref_name)}" - end - # End of deprecation -------------------------------------------- - def self.allow_force_push?(project, ref_name) - if Feature.enabled?(:group_protected_branches) + if Feature.enabled?(:group_protected_branches, project.group) protected_branches = project.all_protected_branches.matching(ref_name) project_protected_branches, group_protected_branches = protected_branches.partition(&:project_id) @@ -92,11 +67,7 @@ class ProtectedBranch < ApplicationRecord end def self.protected_refs(project) - if Feature.enabled?(:group_protected_branches) - project.all_protected_branches - else - project.protected_branches - end + project.all_protected_branches end # overridden in EE diff --git a/app/models/repository.rb b/app/models/repository.rb index d15f2a430fa..587b71315c2 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -48,7 +48,7 @@ class Repository # For example, for entry `:commit_count` there's a method called `commit_count` which # stores its data in the `commit_count` cache key. CACHED_METHODS = %i(size commit_count readme_path contribution_guide - changelog license_blob license_licensee license_gitaly gitignore + changelog license_blob license_gitaly gitignore gitlab_ci_yml branch_names tag_names branch_count tag_count avatar exists? root_ref merged_branch_names has_visible_content? issue_template_names_hash merge_request_template_names_hash @@ -60,7 +60,7 @@ class Repository METHOD_CACHES_FOR_FILE_TYPES = { readme: %i(readme_path), changelog: :changelog, - license: %i(license_blob license_licensee license_gitaly), + license: %i(license_blob license_gitaly), contributing: :contribution_guide, gitignore: :gitignore, gitlab_ci: :gitlab_ci_yml, @@ -161,7 +161,8 @@ class Repository first_parent: !!opts[:first_parent], order: opts[:order], literal_pathspec: opts.fetch(:literal_pathspec, true), - trailers: opts[:trailers] + trailers: opts[:trailers], + include_referenced_by: opts[:include_referenced_by] } commits = Gitlab::Git::Commit.where(options) @@ -655,24 +656,13 @@ class Repository end def license - if Feature.enabled?(:license_from_gitaly) - license_gitaly - else - license_licensee - end - end - - def license_licensee - return unless exists? - - raw_repository.license(false) + license_gitaly end - cache_method :license_licensee def license_gitaly return unless exists? - raw_repository.license(true) + raw_repository.license end cache_method :license_gitaly @@ -844,6 +834,26 @@ class Repository commit_files(user, **options) end + def move_dir_files(user, path, previous_path, **options) + regex = Regexp.new("^#{Regexp.escape(previous_path + '/')}", 'i') + files = ls_files(options[:branch_name]) + + options[:actions] = files.each_with_object([]) do |item, list| + next unless item =~ regex + + list.push( + action: :move, + file_path: "#{path}/#{item[regex.match(item)[0].size..]}", + previous_path: item, + infer_content: true + ) + end + + return if options[:actions].blank? + + commit_files(user, **options) + end + def delete_file(user, path, **options) options[:actions] = [{ action: :delete, file_path: path }] @@ -948,6 +958,8 @@ class Repository end def merged_to_root_ref?(branch_or_name) + return unless head_commit + branch = Gitlab::Git::Branch.find(self, branch_or_name) if branch @@ -960,7 +972,7 @@ class Repository end def root_ref_sha - @root_ref_sha ||= commit(root_ref).sha + @root_ref_sha ||= head_commit.sha end # If this method is not provided a set of branch names to check merge status, diff --git a/app/models/resource_label_event.rb b/app/models/resource_label_event.rb index efffc1bd6dc..13610d37a74 100644 --- a/app/models/resource_label_event.rb +++ b/app/models/resource_label_event.rb @@ -29,9 +29,8 @@ class ResourceLabelEvent < ResourceEvent labels = events.map(&:label).compact project_labels, group_labels = labels.partition { |label| label.is_a? ProjectLabel } - preloader = ActiveRecord::Associations::Preloader.new - preloader.preload(project_labels, { project: :project_feature }) - preloader.preload(group_labels, :group) + ActiveRecord::Associations::Preloader.new(records: project_labels, associations: { project: :project_feature }).call + ActiveRecord::Associations::Preloader.new(records: group_labels, associations: :group).call end def issuable diff --git a/app/models/resource_milestone_event.rb b/app/models/resource_milestone_event.rb index def7e91af3f..f3301ee2051 100644 --- a/app/models/resource_milestone_event.rb +++ b/app/models/resource_milestone_event.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class ResourceMilestoneEvent < ResourceTimeboxEvent - include IgnorableColumns - belongs_to :milestone scope :include_relations, -> { includes(:user, milestone: [:project, :group]) } @@ -10,8 +8,6 @@ class ResourceMilestoneEvent < ResourceTimeboxEvent # state is used for issue and merge request states. enum state: Issue.available_states.merge(MergeRequest.available_states) - ignore_columns %i[reference reference_html cached_markdown_version], remove_with: '13.1', remove_after: '2020-06-22' - def milestone_title milestone&.title end diff --git a/app/models/serverless/domain.rb b/app/models/serverless/domain.rb deleted file mode 100644 index 164f93afa9a..00000000000 --- a/app/models/serverless/domain.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class Domain - include ActiveModel::Model - - REGEXP = %r{^(?<scheme>https?://)?(?<function_name>[^.]+)-(?<cluster_left>\h{2})a1(?<cluster_middle>\h{10})f2(?<cluster_right>\h{2})(?<environment_id>\h+)-(?<environment_slug>[^.]+)\.(?<pages_domain_name>.+)}.freeze - UUID_LENGTH = 14 - - attr_accessor :function_name, :serverless_domain_cluster, :environment - - validates :function_name, presence: true, allow_blank: false - validates :serverless_domain_cluster, presence: true - validates :environment, presence: true - - def self.generate_uuid - SecureRandom.hex(UUID_LENGTH / 2) - end - - def uri - URI("https://#{function_name}-#{serverless_domain_cluster_uuid}#{"%x" % environment.id}-#{environment.slug}.#{serverless_domain_cluster.domain}") - end - - def knative_uri - URI("http://#{function_name}.#{namespace}.#{serverless_domain_cluster.knative.hostname}") - end - - private - - def namespace - serverless_domain_cluster.cluster.kubernetes_namespace_for(environment) - end - - def serverless_domain_cluster_uuid - [ - serverless_domain_cluster.uuid[0..1], - 'a1', - serverless_domain_cluster.uuid[2..-3], - 'f2', - serverless_domain_cluster.uuid[-2..] - ].join - end - end -end diff --git a/app/models/serverless/domain_cluster.rb b/app/models/serverless/domain_cluster.rb deleted file mode 100644 index 561bfc65b2b..00000000000 --- a/app/models/serverless/domain_cluster.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class DomainCluster < ApplicationRecord - self.table_name = 'serverless_domain_cluster' - - HEX_REGEXP = %r{\A\h+\z}.freeze - - belongs_to :pages_domain - belongs_to :knative, class_name: 'Clusters::Applications::Knative', foreign_key: 'clusters_applications_knative_id' - belongs_to :creator, class_name: 'User', optional: true - - attr_encrypted :key, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm' - - validates :pages_domain, :knative, presence: true - validates :uuid, presence: true, uniqueness: true, length: { is: ::Serverless::Domain::UUID_LENGTH }, - format: { with: HEX_REGEXP, message: 'only allows hex characters' } - - after_initialize :set_uuid, if: :new_record? - - delegate :domain, to: :pages_domain - delegate :cluster, to: :knative - - def self.for_uuid(uuid) - joins(:pages_domain, :knative) - .includes(:pages_domain, :knative) - .find_by(uuid: uuid) - end - - private - - def set_uuid - self.uuid = ::Serverless::Domain.generate_uuid - end - end -end diff --git a/app/models/serverless/function.rb b/app/models/serverless/function.rb deleted file mode 100644 index 5d4f8e0c9e2..00000000000 --- a/app/models/serverless/function.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class Function - attr_accessor :name, :namespace - - def initialize(project, name, namespace) - @project = project - @name = name - @namespace = namespace - end - - def id - @project.id.to_s + "/" + @name + "/" + @namespace - end - - def self.find_by_id(id) - array = id.split("/") - project = Project.find_by_id(array[0]) - name = array[1] - namespace = array[2] - - self.new(project, name, namespace) - end - end -end diff --git a/app/models/serverless/lookup_path.rb b/app/models/serverless/lookup_path.rb deleted file mode 100644 index c09b3718651..00000000000 --- a/app/models/serverless/lookup_path.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class LookupPath - attr_reader :serverless_domain - - delegate :serverless_domain_cluster, to: :serverless_domain - delegate :knative, to: :serverless_domain_cluster - delegate :certificate, to: :serverless_domain_cluster - delegate :key, to: :serverless_domain_cluster - - def initialize(serverless_domain) - @serverless_domain = serverless_domain - end - - def source - { - type: 'serverless', - service: serverless_domain.knative_uri.host, - cluster: { - hostname: knative.hostname, - address: knative.external_ip, - port: 443, - cert: certificate, - key: key - } - } - end - end -end diff --git a/app/models/serverless/virtual_domain.rb b/app/models/serverless/virtual_domain.rb deleted file mode 100644 index d6a23a4c0ce..00000000000 --- a/app/models/serverless/virtual_domain.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class VirtualDomain - attr_reader :serverless_domain - - delegate :serverless_domain_cluster, to: :serverless_domain - delegate :pages_domain, to: :serverless_domain_cluster - delegate :certificate, to: :pages_domain - delegate :key, to: :pages_domain - - def initialize(serverless_domain) - @serverless_domain = serverless_domain - end - - def lookup_paths - [ - ::Serverless::LookupPath.new(serverless_domain) - ] - end - end -end diff --git a/app/models/airflow.rb b/app/models/service_desk.rb index 2e5642a2639..cb9c924c01f 100644 --- a/app/models/airflow.rb +++ b/app/models/service_desk.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true -module Airflow + +module ServiceDesk def self.table_name_prefix - 'airflow_' + 'service_desk_' end end diff --git a/app/models/service_desk/custom_email_verification.rb b/app/models/service_desk/custom_email_verification.rb new file mode 100644 index 00000000000..b3b9390bb82 --- /dev/null +++ b/app/models/service_desk/custom_email_verification.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module ServiceDesk + class CustomEmailVerification < ApplicationRecord + enum state: { + running: 0, + verified: 1, + error: 2 + }, _default: 'running' + + enum error: { + incorrect_token: 0, + incorrect_from: 1, + mail_not_received_within_timeframe: 2, + invalid_credentials: 3, + smtp_host_issue: 4 + } + + TIMEFRAME = 30.minutes + + attr_encrypted :token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_iv: false + + belongs_to :project + belongs_to :triggerer, class_name: 'User', optional: true + + validates :project, presence: true + validates :state, presence: true + + delegate :service_desk_setting, to: :project + + class << self + def generate_token + SecureRandom.alphanumeric(12) + end + end + + def accepted_until + return unless running? + return unless triggered_at.present? + + TIMEFRAME.since(triggered_at) + end + + def in_timeframe? + return false unless running? + + !!accepted_until&.future? + end + end +end diff --git a/app/models/service_desk_setting.rb b/app/models/service_desk_setting.rb index 5152746abb4..69afb445734 100644 --- a/app/models/service_desk_setting.rb +++ b/app/models/service_desk_setting.rb @@ -3,6 +3,8 @@ class ServiceDeskSetting < ApplicationRecord include Gitlab::Utils::StrongMemoize + CUSTOM_EMAIL_VERIFICATION_SUBADDRESS = '+verify' + attribute :custom_email_enabled, default: false attr_encrypted :custom_email_smtp_password, mode: :per_attribute_iv, @@ -12,6 +14,7 @@ class ServiceDeskSetting < ApplicationRecord encode_iv: false belongs_to :project + validates :project_id, presence: true validate :valid_issue_template validate :valid_project_key @@ -32,21 +35,25 @@ class ServiceDeskSetting < ApplicationRecord validates :custom_email, presence: true, devise_email: true, - if: :custom_email_enabled? + if: :needs_custom_email_smtp_credentials? validates :custom_email_smtp_address, presence: true, hostname: { allow_numeric_hostname: true, require_valid_tld: true }, - if: :custom_email_enabled? + if: :needs_custom_email_smtp_credentials? validates :custom_email_smtp_username, presence: true, - if: :custom_email_enabled? + if: :needs_custom_email_smtp_credentials? validates :custom_email_smtp_port, presence: true, numericality: { only_integer: true, greater_than: 0 }, - if: :custom_email_enabled? + if: :needs_custom_email_smtp_credentials? scope :with_project_key, ->(key) { where(project_key: key) } + def custom_email_verification + project&.service_desk_custom_email_verification + end + def custom_email_delivery_options { user_name: custom_email_smtp_username, @@ -57,6 +64,12 @@ class ServiceDeskSetting < ApplicationRecord } end + def custom_email_address_for_verification + return unless custom_email.present? + + custom_email.sub("@", "#{CUSTOM_EMAIL_VERIFICATION_SUBADDRESS}@") + end + def issue_template_content strong_memoize(:issue_template_content) do next unless issue_template_key.present? @@ -102,6 +115,10 @@ class ServiceDeskSetting < ApplicationRecord setting.project.full_path_slug == project_slug end end + + def needs_custom_email_smtp_credentials? + custom_email_enabled? || custom_email_verification.present? + end end ServiceDeskSetting.prepend_mod diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 9ec685c5580..8ed5513aab9 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -183,7 +183,7 @@ class Snippet < ApplicationRecord end def link_reference_pattern - @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/) + @link_reference_pattern ||= compose_link_reference_pattern('snippets', /(?<snippet>\d+)/) end def find_by_id_and_project(id:, project:) @@ -203,14 +203,7 @@ class Snippet < ApplicationRecord end def initialize(attributes = {}) - # We can't use default_value_for because the database has a default - # value of 0 for visibility_level. If someone attempts to create a - # private snippet, default_value_for will assume that the - # visibility_level hasn't changed and will use the application - # setting default, which could be internal or public. - # - # To fix the problem, we assign the actual snippet default if no - # explicit visibility has been initialized. + # We assign the actual snippet default if no explicit visibility has been initialized. attributes ||= {} unless visibility_attribute_present?(attributes) diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index bb8527d8c01..0e0534d45ae 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -26,8 +26,7 @@ class SystemNoteMetadata < ApplicationRecord title time_tracking branch milestone discussion task moved cloned opened closed merged duplicate locked unlocked outdated reviewer tag due_date start_date_or_due_date pinned_embed cherry_pick health_status approved unapproved - status alert_issue_added relate unrelate new_alert_added severity - attention_requested attention_request_removed contact timeline_event + status alert_issue_added relate unrelate new_alert_added severity contact timeline_event issue_type relate_to_child unrelate_from_child relate_to_parent unrelate_from_parent ].freeze diff --git a/app/models/user.rb b/app/models/user.rb index f3e8f14adf5..3bd8a035357 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -28,6 +28,7 @@ class User < ApplicationRecord include UpdateHighestRole include HasUserType include Gitlab::Auth::Otp::Fortinet + include Gitlab::Auth::Otp::DuoAuth include RestrictedSignup include StripAttribute include EachBatch @@ -71,6 +72,7 @@ class User < ApplicationRecord attribute :notified_of_own_activity, default: false attribute :preferred_language, default: -> { Gitlab::CurrentSettings.default_preferred_language } attribute :theme_id, default: -> { gitlab_config.default_theme } + attribute :color_scheme_id, default: -> { Gitlab::CurrentSettings.default_syntax_highlighting_theme } attr_encrypted :otp_secret, key: Gitlab::Application.secrets.otp_key_base, @@ -101,8 +103,6 @@ class User < ApplicationRecord MINIMUM_DAYS_CREATED = 7 - ignore_columns %i[linkedin twitter skype website_url location organization], remove_with: '15.10', remove_after: '2023-02-22' - # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour # rubocop: disable CodeReuse/ServiceClass @@ -227,7 +227,9 @@ class User < ApplicationRecord has_many :notification_settings has_many :award_emoji, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :triggers, class_name: 'Ci::Trigger', foreign_key: :owner_id + has_many :audit_events, foreign_key: :author_id, inverse_of: :user + has_many :alert_assignees, class_name: '::AlertManagement::AlertAssignee', inverse_of: :assignee has_many :issue_assignees, inverse_of: :assignee has_many :merge_request_assignees, inverse_of: :assignee, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :merge_request_reviewers, inverse_of: :reviewer, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -289,7 +291,7 @@ class User < ApplicationRecord validate :check_password_weakness, if: :encrypted_password_changed? validates :namespace, presence: true - validate :namespace_move_dir_allowed, if: :username_changed? + validate :namespace_move_dir_allowed, if: :username_changed?, unless: :new_record? validate :unique_email, if: :email_changed? validate :notification_email_verified, if: :notification_email_changed? @@ -614,13 +616,12 @@ class User < ApplicationRecord def self.with_two_factor where(otp_required_for_login: true) - .or(where_exists(U2fRegistration.where(U2fRegistration.arel_table[:user_id].eq(arel_table[:id])))) .or(where_exists(WebauthnRegistration.where(WebauthnRegistration.arel_table[:user_id].eq(arel_table[:id])))) end def self.without_two_factor where - .missing(:u2f_registrations, :webauthn_registrations) + .missing(:webauthn_registrations) .where(otp_required_for_login: false) end @@ -1062,27 +1063,14 @@ class User < ApplicationRecord end def two_factor_enabled? - two_factor_otp_enabled? || two_factor_webauthn_u2f_enabled? + two_factor_otp_enabled? || two_factor_webauthn_enabled? end def two_factor_otp_enabled? otp_required_for_login? || forti_authenticator_enabled?(self) || - forti_token_cloud_enabled?(self) - end - - def two_factor_u2f_enabled? - return false if Feature.enabled?(:webauthn) - - if u2f_registrations.loaded? - u2f_registrations.any? - else - u2f_registrations.exists? - end - end - - def two_factor_webauthn_u2f_enabled? - two_factor_u2f_enabled? || two_factor_webauthn_enabled? + forti_token_cloud_enabled?(self) || + duo_auth_enabled?(self) end def two_factor_webauthn_enabled? @@ -1725,11 +1713,7 @@ class User < ApplicationRecord end def manageable_groups(include_groups_with_developer_maintainer_access: false) - owned_and_maintainer_group_hierarchy = if Feature.enabled?(:linear_user_manageable_groups, self) - owned_or_maintainers_groups.self_and_descendants - else - Gitlab::ObjectHierarchy.new(owned_or_maintainers_groups).base_and_descendants - end + owned_and_maintainer_group_hierarchy = owned_or_maintainers_groups.self_and_descendants if include_groups_with_developer_maintainer_access union_sql = ::Gitlab::SQL::Union.new( @@ -2136,7 +2120,15 @@ class User < ApplicationRecord end def confirmation_required_on_sign_in? - !confirmed? && !confirmation_period_valid? + return false if confirmed? + + if ::Gitlab::CurrentSettings.email_confirmation_setting_off? + false + elsif ::Gitlab::CurrentSettings.email_confirmation_setting_soft? + !in_confirmation_period? + elsif ::Gitlab::CurrentSettings.email_confirmation_setting_hard? + true + end end def impersonated? @@ -2217,10 +2209,13 @@ class User < ApplicationRecord # override from Devise::Confirmable def confirmation_period_valid? - return false if Feature.disabled?(:soft_email_confirmation) + return super if ::Gitlab::CurrentSettings.email_confirmation_setting_soft? - super + # Following devise logic for method, we want to return `true` + # See: https://github.com/heartcombo/devise/blob/main/lib/devise/models/confirmable.rb#L191-L218 + true end + alias_method :in_confirmation_period?, :confirmation_period_valid? # This is copied from Devise::Models::TwoFactorAuthenticatable#consume_otp! # diff --git a/app/models/user_status.rb b/app/models/user_status.rb index 0c66f465356..da24ef47a2a 100644 --- a/app/models/user_status.rb +++ b/app/models/user_status.rb @@ -17,7 +17,7 @@ class UserStatus < ApplicationRecord '30_days' => 30.days }.freeze - belongs_to :user + belongs_to :user, inverse_of: :status enum availability: { not_set: 0, busy: 1 } diff --git a/app/models/users/banned_user.rb b/app/models/users/banned_user.rb index 615668e2b55..466fc71f83a 100644 --- a/app/models/users/banned_user.rb +++ b/app/models/users/banned_user.rb @@ -10,3 +10,5 @@ module Users validates :user_id, uniqueness: { message: N_("banned user already exists") } end end + +Users::BannedUser.prepend_mod_with('Users::BannedUser') diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 3f9353214ee..70c31f0a8ec 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -65,7 +65,8 @@ module Users new_top_level_group_alert: 61, artifacts_management_page_feedback_banner: 62, vscode_web_ide: 63, - vscode_web_ide_callout: 64 + vscode_web_ide_callout: 64, + branch_rules_info_callout: 65 } validates :feature_name, diff --git a/app/models/users/group_callout.rb b/app/models/users/group_callout.rb index 2552407fa4c..fe04800539c 100644 --- a/app/models/users/group_callout.rb +++ b/app/models/users/group_callout.rb @@ -24,7 +24,9 @@ module Users namespace_storage_limit_banner_error_threshold: 13, # EE-only usage_quota_trial_alert: 14, # EE-only preview_usage_quota_free_plan_alert: 15, # EE-only - enforcement_at_limit_alert: 16 # EE-only + enforcement_at_limit_alert: 16, # EE-only + web_hook_disabled: 17, # EE-only + unlimited_members_during_trial_alert: 18 # EE-only } validates :group, presence: true diff --git a/app/models/wiki.rb b/app/models/wiki.rb index 57488749b76..33b2b3b7c87 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -326,6 +326,11 @@ class Wiki content, previous_path: page.path, **multi_commit_options(:updated, message, title)) + repository.move_dir_files( + user, + sluggified_title(title), + page.url_path, + **multi_commit_options(:moved, message, title)) after_wiki_activity diff --git a/app/models/wiki_directory.rb b/app/models/wiki_directory.rb index 76fe664f23d..e57d186a3e3 100644 --- a/app/models/wiki_directory.rb +++ b/app/models/wiki_directory.rb @@ -7,34 +7,48 @@ class WikiDirectory validates :slug, presence: true alias_method :to_param, :slug - # Groups a list of wiki pages into a nested collection of WikiPage and WikiDirectory objects, - # preserving the order of the passed pages. - # - # Returns an array with all entries for the toplevel directory. - # - # @param [Array<WikiPage>] pages - # @return [Array<WikiPage, WikiDirectory>] - # - def self.group_pages(pages) - # Build a hash to map paths to created WikiDirectory objects, - # and recursively create them for each level of the path. - # For the toplevel directory we use '' as path, as that's what WikiPage#directory returns. - directories = Hash.new do |_, path| - directories[path] = new(path).tap do |directory| - if path.present? - parent = File.dirname(path) - parent = '' if parent == '.' - directories[parent].entries << directory - directories[parent].entries.delete_if { |item| item.is_a?(WikiPage) && item.slug == directory.slug } + class << self + # Groups a list of wiki pages into a nested collection of WikiPage and WikiDirectory objects, + # preserving the order of the passed pages. + # + # Returns an array with all entries for the toplevel directory. + # + # @param [Array<WikiPage>] pages + # @return [Array<WikiPage, WikiDirectory>] + # + def group_pages(pages) + # Build a hash to map paths to created WikiDirectory objects, + # and recursively create them for each level of the path. + # For the toplevel directory we use '' as path, as that's what WikiPage#directory returns. + directories = Hash.new do |_, path| + directories[path] = new(path).tap do |directory| + if path.present? + parent = File.dirname(path) + parent = '' if parent == '.' + directories[parent].entries << directory + directories[parent].entries.delete_if do |item| + item.is_a?(WikiPage) && item.slug.casecmp?(directory.slug) + end + end end end - end - pages.each do |page| - directories[page.directory].entries << page + pages.each do |page| + next unless directory_for_page?(directories[page.directory], page) + + directories[page.directory].entries << page + end + + directories[''].entries end - directories[''].entries + private + + def directory_for_page?(directory, page) + directory.entries.none? do |item| + item.is_a?(WikiDirectory) && item.slug.casecmp?(page.slug) + end + end end def initialize(slug, entries = []) diff --git a/app/models/work_item.rb b/app/models/work_item.rb index 5ae3fb6cf78..a7cd522f023 100644 --- a/app/models/work_item.rb +++ b/app/models/work_item.rb @@ -85,6 +85,26 @@ class WorkItem < Issue COMMON_QUICK_ACTIONS_COMMANDS + commands_for_widgets end + # Widgets have a set of quick action params that they must process. + # Map them to widget_params so they can be picked up by widget services. + def transform_quick_action_params(command_params) + common_params = command_params.deep_dup + widget_params = {} + + work_item_type.widgets + .filter { |widget| widget.respond_to?(:quick_action_params) } + .each do |widget| + widget.quick_action_params + .filter { |param_name| common_params.key?(param_name) } + .each do |param_name| + widget_params[widget.api_symbol] ||= {} + widget_params[widget.api_symbol][param_name] = common_params.delete(param_name) + end + end + + { common: common_params, widgets: widget_params } + end + private override :parent_link_confidentiality diff --git a/app/models/work_items/widget_definition.rb b/app/models/work_items/widget_definition.rb index 5d4414e95d8..9e8c421d740 100644 --- a/app/models/work_items/widget_definition.rb +++ b/app/models/work_items/widget_definition.rb @@ -28,7 +28,8 @@ module WorkItems progress: 10, # EE-only status: 11, # EE-only requirement_legacy: 12, # EE-only - test_reports: 13 # EE-only + test_reports: 13, # EE-only + notifications: 14 } def self.available_widgets diff --git a/app/models/work_items/widgets/notifications.rb b/app/models/work_items/widgets/notifications.rb new file mode 100644 index 00000000000..9a13e5ebbea --- /dev/null +++ b/app/models/work_items/widgets/notifications.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class Notifications < Base + delegate :subscribed?, to: :work_item + end + end +end |