diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
tree | dceebdc68925362117480a5d672bcff122fb625b /app/models | |
parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'app/models')
323 files changed, 5286 insertions, 3691 deletions
diff --git a/app/models/ability.rb b/app/models/ability.rb index eb645bcd653..4da4d113a7f 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -77,6 +77,8 @@ class Ability policy = policy_for(user, subject) + before_check(policy, ability.to_sym, user, subject, opts) + case opts[:scope] when :user DeclarativePolicy.user_scope { policy.allowed?(ability) } @@ -92,6 +94,11 @@ class Ability forget_runner_result(policy.runner(ability)) if policy && ability_forgetting? end + # Hook call right before ability check. + def before_check(policy, ability, user, subject, opts) + # See Support::AbilityCheck and Support::PermissionsCheck. + end + def policy_for(user, subject = :global) DeclarativePolicy.policy_for(user, subject, cache: ::Gitlab::SafeRequestStore.storage) end diff --git a/app/models/abuse/trust_score.rb b/app/models/abuse/trust_score.rb new file mode 100644 index 00000000000..9ad7c9b14b1 --- /dev/null +++ b/app/models/abuse/trust_score.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Abuse + class TrustScore < ApplicationRecord + MAX_EVENTS = 100 + + self.table_name = 'abuse_trust_scores' + + enum source: Enums::Abuse::Source.sources + + belongs_to :user + + validates :user, presence: true + validates :score, presence: true + validates :source, presence: true + + before_create :assign_correlation_id + after_commit :remove_old_scores + + private + + def assign_correlation_id + self.correlation_id_value ||= (Labkit::Correlation::CorrelationId.current_id || '') + end + + def remove_old_scores + count = user.trust_scores_for_source(source).count + return unless count > MAX_EVENTS + + TrustScore.delete( + user.trust_scores_for_source(source) + .order(created_at: :asc) + .limit(count - MAX_EVENTS) + ) + end + end +end diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb index dbcdfa5e946..55b1aff51da 100644 --- a/app/models/abuse_report.rb +++ b/app/models/abuse_report.rb @@ -3,14 +3,20 @@ class AbuseReport < ApplicationRecord include CacheMarkdownField include Sortable + include Gitlab::FileTypeDetection + include WithUploads + include Gitlab::Utils::StrongMemoize MAX_CHAR_LIMIT_URL = 512 + MAX_FILE_SIZE = 1.megabyte cache_markdown_field :message, pipeline: :single_line belongs_to :reporter, class_name: 'User' belongs_to :user + has_many :events, class_name: 'ResourceEvents::AbuseReportEvent', inverse_of: :abuse_report + validates :reporter, presence: true validates :user, presence: true validates :message, presence: true @@ -24,25 +30,31 @@ class AbuseReport < ApplicationRecord } validates :reported_from_url, - allow_blank: true, - length: { maximum: MAX_CHAR_LIMIT_URL }, - addressable_url: { - dns_rebind_protection: true, - blocked_message: 'is an invalid URL. You can try reporting the abuse again, ' \ - 'or contact a GitLab administrator for help.' - } + allow_blank: true, + length: { maximum: MAX_CHAR_LIMIT_URL }, + addressable_url: { + dns_rebind_protection: true, + blocked_message: 'is an invalid URL. You can try reporting the abuse again, ' \ + 'or contact a GitLab administrator for help.' + } validates :links_to_spam, - allow_blank: true, - length: { - maximum: 20, - message: N_("exceeds the limit of %{count} links") - } + allow_blank: true, + length: { + maximum: 20, + message: N_("exceeds the limit of %{count} links") + } before_validation :filter_empty_strings_from_links_to_spam validate :links_to_spam_contains_valid_urls - scope :by_user, ->(user) { where(user_id: user) } + mount_uploader :screenshot, AttachmentUploader + validates :screenshot, file_size: { maximum: MAX_FILE_SIZE } + validate :validate_screenshot_is_image + + scope :by_user_id, ->(id) { where(user_id: id) } + scope :by_reporter_id, ->(id) { where(reporter_id: id) } + scope :by_category, ->(category) { where(category: category) } scope :with_users, -> { includes(:reporter, :user) } enum category: { @@ -56,6 +68,11 @@ class AbuseReport < ApplicationRecord other: 8 } + enum status: { + open: 1, + closed: 2 + } + # For CacheMarkdownField alias_method :author, :reporter @@ -63,6 +80,12 @@ class AbuseReport < ApplicationRecord reported_from_url: "Reported from" }.freeze + CONTROLLER_TO_REPORT_TYPE = { + 'users' => :profile, + 'projects/issues' => :issue, + 'projects/merge_requests' => :merge_request + }.freeze + def self.human_attribute_name(attr, options = {}) HUMANIZED_ATTRIBUTES[attr.to_sym] || super end @@ -77,8 +100,66 @@ class AbuseReport < ApplicationRecord AbuseReportMailer.notify(id).deliver_later end + def screenshot_path + return unless screenshot + return screenshot.url unless screenshot.upload + + asset_host = ActionController::Base.asset_host || Gitlab.config.gitlab.base_url + local_path = Gitlab::Routing.url_helpers.abuse_report_upload_path( + filename: screenshot.filename, + id: screenshot.upload.model_id, + model: 'abuse_report', + mounted_as: 'screenshot') + + Gitlab::Utils.append_path(asset_host, local_path) + end + + def report_type + type = CONTROLLER_TO_REPORT_TYPE[route_hash[:controller]] + type = :comment if type.in?([:issue, :merge_request]) && note_id_from_url.present? + + type + end + + def reported_content + case report_type + when :issue + project.issues.iid_in(route_hash[:id]).pick(:description_html) + when :merge_request + project.merge_requests.iid_in(route_hash[:id]).pick(:description_html) + when :comment + project.notes.id_in(note_id_from_url).pick(:note_html) + end + end + + def other_reports_for_user + user.abuse_reports.id_not_in(id) + end + private + def project + Project.find_by_full_path(route_hash.values_at(:namespace_id, :project_id).join('/')) + end + + def route_hash + match = Rails.application.routes.recognize_path(reported_from_url) + return {} if match[:unmatched_route].present? + + match + rescue ActionController::RoutingError + {} + end + strong_memoize_attr :route_hash + + def note_id_from_url + fragment = URI(reported_from_url).fragment + Gitlab::UntrustedRegexp.new('^note_(\d+)$').match(fragment).to_a.second if fragment + rescue URI::InvalidURIError + nil + end + strong_memoize_attr :note_id_from_url + def filter_empty_strings_from_links_to_spam return if links_to_spam.blank? @@ -91,9 +172,9 @@ class AbuseReport < ApplicationRecord links_to_spam.each do |link| Gitlab::UrlBlocker.validate!( link, - schemes: %w[http https], - allow_localhost: true, - dns_rebind_protection: true + schemes: %w[http https], + allow_localhost: true, + dns_rebind_protection: true ) next unless link.length > MAX_CHAR_LIMIT_URL @@ -106,4 +187,26 @@ class AbuseReport < ApplicationRecord rescue ::Gitlab::UrlBlocker::BlockedUrlError errors.add(:links_to_spam, _('only supports valid HTTP(S) URLs')) end + + def filename + screenshot&.filename + end + + def valid_image_extensions + Gitlab::FileTypeDetection::SAFE_IMAGE_EXT + end + + def validate_screenshot_is_image + return if screenshot.blank? + return if image? + + errors.add( + :screenshot, + format( + _('must match one of the following file types: %{extension_list}'), + extension_list: valid_image_extensions.to_sentence(last_word_connector: ' or ')) + ) + end end + +AbuseReport.prepend_mod diff --git a/app/models/achievements/achievement.rb b/app/models/achievements/achievement.rb index 95606e50ad4..834c12fee5a 100644 --- a/app/models/achievements/achievement.rb +++ b/app/models/achievements/achievement.rb @@ -4,9 +4,6 @@ module Achievements class Achievement < ApplicationRecord include Avatarable include StripAttribute - include IgnorableColumns - - ignore_column :revokable, remove_with: '15.11', remove_after: '2023-04-22' belongs_to :namespace, inverse_of: :achievements, optional: false @@ -16,9 +13,9 @@ module Achievements strip_attributes! :name, :description validates :name, - presence: true, - length: { maximum: 255 }, - uniqueness: { case_sensitive: false, scope: [:namespace_id] } + presence: true, + length: { maximum: 255 }, + uniqueness: { case_sensitive: false, scope: [:namespace_id] } validates :description, length: { maximum: 1024 } end end diff --git a/app/models/achievements/user_achievement.rb b/app/models/achievements/user_achievement.rb index 885ec660cc9..08ebadaa6b0 100644 --- a/app/models/achievements/user_achievement.rb +++ b/app/models/achievements/user_achievement.rb @@ -6,12 +6,19 @@ module Achievements belongs_to :user, inverse_of: :user_achievements, optional: false belongs_to :awarded_by_user, - class_name: 'User', - inverse_of: :awarded_user_achievements, - optional: true + class_name: 'User', + inverse_of: :awarded_user_achievements, + optional: false belongs_to :revoked_by_user, - class_name: 'User', - inverse_of: :revoked_user_achievements, - optional: true + class_name: 'User', + inverse_of: :revoked_user_achievements, + optional: true + + scope :not_revoked, -> { where(revoked_by_user_id: nil) } + scope :order_by_id_asc, -> { order(id: :asc) } + + def revoked? + revoked_by_user_id.present? + end end end diff --git a/app/models/active_session.rb b/app/models/active_session.rb index 2d1dec1977d..7d025fb7738 100644 --- a/app/models/active_session.rb +++ b/app/models/active_session.rb @@ -26,8 +26,8 @@ class ActiveSession ALLOWED_NUMBER_OF_ACTIVE_SESSIONS = 100 attr_accessor :ip_address, :browser, :os, - :device_name, :device_type, - :is_impersonated, :session_id, :session_private_id + :device_name, :device_type, + :is_impersonated, :session_id, :session_private_id attr_reader :created_at, :updated_at @@ -91,13 +91,6 @@ class ActiveSession active_user_session.dump ) - # Deprecated legacy format - temporary to support mixed deployments - pipeline.setex( - key_name_v1(user.id, session_private_id), - expiry, - Marshal.dump(active_user_session) - ) - pipeline.sadd?( lookup_key_name(user.id), session_private_id @@ -107,6 +100,19 @@ class ActiveSession end end + # set marketing cookie when user has active session + def self.set_active_user_cookie(auth) + auth.cookies[:about_gitlab_active_user] = + { + value: true, + domain: Gitlab.config.gitlab.host + } + end + + def self.unset_active_user_cookie(auth) + auth.cookies.delete :about_gitlab_active_user + end + def self.list(user) Gitlab::Redis::Sessions.with do |redis| cleaned_up_lookup_entries(redis, user).map do |raw_session| diff --git a/app/models/airflow/dags.rb b/app/models/airflow/dags.rb deleted file mode 100644 index d17d4a4f3db..00000000000 --- a/app/models/airflow/dags.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Airflow - class Dags < ApplicationRecord - belongs_to :project - - validates :project, presence: true - validates :dag_name, length: { maximum: 255 }, presence: true - validates :schedule, length: { maximum: 255 } - validates :fileloc, length: { maximum: 255 } - - scope :by_project_id, ->(project_id) { where(project_id: project_id).order(id: :asc) } - end -end diff --git a/app/models/alert_management/alert.rb b/app/models/alert_management/alert.rb index a5a539eae75..74edcf12ac2 100644 --- a/app/models/alert_management/alert.rb +++ b/app/models/alert_management/alert.rb @@ -25,8 +25,9 @@ module AlertManagement has_many :assignees, through: :alert_assignees has_many :notes, as: :noteable, inverse_of: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - has_many :ordered_notes, -> { fresh }, as: :noteable, class_name: 'Note' - has_many :user_mentions, class_name: 'AlertManagement::AlertUserMention', foreign_key: :alert_management_alert_id + has_many :ordered_notes, -> { fresh }, as: :noteable, class_name: 'Note', inverse_of: :noteable + has_many :user_mentions, class_name: 'AlertManagement::AlertUserMention', foreign_key: :alert_management_alert_id, + inverse_of: :alert has_many :metric_images, class_name: '::AlertManagement::MetricImage' has_internal_id :iid, scope: :project @@ -139,7 +140,7 @@ module AlertManagement end def self.link_reference_pattern - @link_reference_pattern ||= super("alert_management", %r{(?<alert>\d+)/details(\#)?}) + @link_reference_pattern ||= compose_link_reference_pattern('alert_management', %r{(?<alert>\d+)/details(\#)?}) end def self.reference_valid?(reference) diff --git a/app/models/alert_management/alert_assignee.rb b/app/models/alert_management/alert_assignee.rb index c74b2699182..27e720c3262 100644 --- a/app/models/alert_management/alert_assignee.rb +++ b/app/models/alert_management/alert_assignee.rb @@ -3,7 +3,7 @@ module AlertManagement class AlertAssignee < ApplicationRecord belongs_to :alert, inverse_of: :alert_assignees - belongs_to :assignee, class_name: 'User', foreign_key: :user_id + belongs_to :assignee, class_name: 'User', foreign_key: :user_id, inverse_of: :alert_assignees validates :alert, presence: true validates :assignee, presence: true, uniqueness: { scope: :alert_id } diff --git a/app/models/alert_management/alert_user_mention.rb b/app/models/alert_management/alert_user_mention.rb index d36aa80ee05..1ab71127677 100644 --- a/app/models/alert_management/alert_user_mention.rb +++ b/app/models/alert_management/alert_user_mention.rb @@ -2,7 +2,10 @@ module AlertManagement class AlertUserMention < UserMention - belongs_to :alert_management_alert, class_name: '::AlertManagement::Alert' + belongs_to :alert, class_name: '::AlertManagement::Alert', + foreign_key: :alert_management_alert_id, + inverse_of: :user_mentions + belongs_to :note end end diff --git a/app/models/analytics/cycle_analytics/project_level.rb b/app/models/analytics/cycle_analytics/project_level.rb index 813263fe833..a423ea35261 100644 --- a/app/models/analytics/cycle_analytics/project_level.rb +++ b/app/models/analytics/cycle_analytics/project_level.rb @@ -11,9 +11,11 @@ module Analytics end def summary - @summary ||= ::Gitlab::CycleAnalytics::StageSummary.new(project, - options: options, - current_user: options[:current_user]).data + @summary ||= ::Gitlab::CycleAnalytics::StageSummary.new( + project, + options: options, + current_user: options[:current_user] + ).data end def permissions(user:) diff --git a/app/models/analytics/cycle_analytics/stage.rb b/app/models/analytics/cycle_analytics/stage.rb index 7e9a89975a3..c7bff7c8d7f 100644 --- a/app/models/analytics/cycle_analytics/stage.rb +++ b/app/models/analytics/cycle_analytics/stage.rb @@ -11,7 +11,7 @@ module Analytics validates :name, uniqueness: { scope: [:group_id, :group_value_stream_id] } belongs_to :value_stream, class_name: 'Analytics::CycleAnalytics::ValueStream', -foreign_key: :group_value_stream_id, inverse_of: :stages + foreign_key: :group_value_stream_id, inverse_of: :stages alias_attribute :parent, :namespace alias_attribute :parent_id, :group_id diff --git a/app/models/analytics/cycle_analytics/value_stream.rb b/app/models/analytics/cycle_analytics/value_stream.rb index 3d8a0a53f5e..59c68393d74 100644 --- a/app/models/analytics/cycle_analytics/value_stream.rb +++ b/app/models/analytics/cycle_analytics/value_stream.rb @@ -19,12 +19,7 @@ module Analytics accepts_nested_attributes_for :stages, allow_destroy: true scope :preload_associated_models, -> { - includes(:namespace, - stages: [ - :namespace, - :end_event_label, - :start_event_label - ]) + includes(:namespace, stages: [:namespace, :end_event_label, :start_event_label]) } after_save :ensure_aggregation_record_presence diff --git a/app/models/appearance.rb b/app/models/appearance.rb index b926c6abedc..4d2baf13f52 100644 --- a/app/models/appearance.rb +++ b/app/models/appearance.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Appearance < ApplicationRecord +class Appearance < MainClusterwide::ApplicationRecord include CacheableAttributes include CacheMarkdownField include WithUploads @@ -27,22 +27,25 @@ class Appearance < ApplicationRecord cache_markdown_field :footer_message, pipeline: :broadcast_message validates :pwa_name, - length: { maximum: 255, too_long: ->(object, data) { - N_("is too long (maximum is %{count} characters)") - } }, - allow_blank: true + length: { + maximum: 255, + too_long: ->(object, data) { N_("is too long (maximum is %{count} characters)") } + }, + allow_blank: true validates :pwa_short_name, - length: { maximum: 255, too_long: ->(object, data) { - N_("is too long (maximum is %{count} characters)") - } }, - allow_blank: true + length: { + maximum: 255, + too_long: ->(object, data) { N_("is too long (maximum is %{count} characters)") } + }, + allow_blank: true validates :pwa_description, - length: { maximum: 2048, too_long: ->(object, data) { - N_("is too long (maximum is %{count} characters)") - } }, - allow_blank: true + length: { + maximum: 2048, + too_long: ->(object, data) { N_("is too long (maximum is %{count} characters)") } + }, + allow_blank: true validates :logo, file_size: { maximum: 1.megabyte } validates :pwa_icon, file_size: { maximum: 1.megabyte } diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 98adbd3ab06..d2ca88aae0e 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -13,7 +13,8 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord ignore_column :user_email_lookup_limit, remove_with: '15.0', remove_after: '2022-04-18' ignore_column :send_user_confirmation_email, remove_with: '15.8', remove_after: '2022-12-18' ignore_column :web_ide_clientside_preview_enabled, remove_with: '15.11', remove_after: '2023-04-22' - ignore_column :clickhouse_connection_string, remove_with: '15.11', remove_after: '2023-04-22' + ignore_column :clickhouse_connection_string, remove_with: '16.1', remove_after: '2023-05-22' + ignore_columns %i[instance_administration_project_id instance_administrators_group_id], remove_with: '16.2', remove_after: '2023-06-22' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ @@ -22,21 +23,24 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord KROKI_URL_ERROR_MESSAGE = 'Please check your Kroki URL setting in ' \ 'Admin Area > Settings > General > Kroki' + # Validate URIs in this model according to the current value of the `deny_all_requests_except_allowed` property, + # rather than the persisted value. + ADDRESSABLE_URL_VALIDATION_OPTIONS = { deny_all_requests_except_allowed: ->(settings) { settings.deny_all_requests_except_allowed } }.freeze + + HUMANIZED_ATTRIBUTES = { + archive_builds_in_seconds: 'Archive job value' + }.freeze + enum whats_new_variant: { all_tiers: 0, current_tier: 1, disabled: 2 }, _prefix: true enum email_confirmation_setting: { off: 0, soft: 1, hard: 2 }, _prefix: true - add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption) ? :optional : :required } + add_authentication_token_field :runners_registration_token, encrypted: :required add_authentication_token_field :health_check_access_token add_authentication_token_field :static_objects_external_storage_auth_token, encrypted: :required add_authentication_token_field :error_tracking_access_token, encrypted: :required - belongs_to :self_monitoring_project, class_name: "Project", foreign_key: 'instance_administration_project_id' belongs_to :push_rule - alias_attribute :self_monitoring_project_id, :instance_administration_project_id - belongs_to :instance_group, class_name: "Group", foreign_key: 'instance_administrators_group_id' - alias_attribute :instance_group_id, :instance_administrators_group_id - alias_attribute :instance_administrators_group, :instance_group alias_attribute :housekeeping_optimize_repository_period, :housekeeping_incremental_repack_period sanitizes! :default_branch_name @@ -90,336 +94,357 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord chronic_duration_attr :project_runner_token_expiration_interval_human_readable, :project_runner_token_expiration_interval validates :grafana_url, - system_hook_url: { - blocked_message: "is blocked: %{exception_message}. #{GRAFANA_URL_ERROR_MESSAGE}" - }, - if: :grafana_url_absolute? + system_hook_url: ADDRESSABLE_URL_VALIDATION_OPTIONS.merge({ + blocked_message: "is blocked: %{exception_message}. #{GRAFANA_URL_ERROR_MESSAGE}" + }), + if: :grafana_url_absolute? validate :validate_grafana_url validates :uuid, presence: true validates :outbound_local_requests_whitelist, - length: { maximum: 1_000, message: N_('is too long (maximum is 1000 entries)') }, - allow_nil: false, - qualified_domain_array: true + length: { maximum: 1_000, message: N_('is too long (maximum is 1000 entries)') }, + allow_nil: false, + qualified_domain_array: true validates :session_expire_delay, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :minimum_password_length, - presence: true, - numericality: { only_integer: true, - greater_than_or_equal_to: DEFAULT_MINIMUM_PASSWORD_LENGTH, - less_than_or_equal_to: Devise.password_length.max } + presence: true, + numericality: { + only_integer: true, + greater_than_or_equal_to: DEFAULT_MINIMUM_PASSWORD_LENGTH, + less_than_or_equal_to: Devise.password_length.max + } validates :home_page_url, - allow_blank: true, - addressable_url: true, - if: :home_page_url_column_exists? + allow_blank: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, + if: :home_page_url_column_exists? validates :help_page_support_url, - allow_blank: true, - addressable_url: true, - if: :help_page_support_url_column_exists? + allow_blank: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, + if: :help_page_support_url_column_exists? validates :help_page_documentation_base_url, - length: { maximum: 255, message: N_("is too long (maximum is %{count} characters)") }, - allow_blank: true, - addressable_url: true + length: { maximum: 255, message: N_("is too long (maximum is %{count} characters)") }, + allow_blank: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS validates :after_sign_out_path, - allow_blank: true, - addressable_url: true + allow_blank: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS validates :abuse_notification_email, - devise_email: true, - allow_blank: true + devise_email: true, + allow_blank: true validates :two_factor_grace_period, - numericality: { greater_than_or_equal_to: 0 } + numericality: { greater_than_or_equal_to: 0 } validates :recaptcha_site_key, - presence: true, - if: :recaptcha_or_login_protection_enabled + presence: true, + if: :recaptcha_or_login_protection_enabled validates :recaptcha_private_key, - presence: true, - if: :recaptcha_or_login_protection_enabled + presence: true, + if: :recaptcha_or_login_protection_enabled validates :akismet_api_key, - presence: true, - if: :akismet_enabled + presence: true, + if: :akismet_enabled validates :spam_check_api_key, - length: { maximum: 2000, message: N_('is too long (maximum is %{count} characters)') }, - allow_blank: true + length: { maximum: 2000, message: N_('is too long (maximum is %{count} characters)') }, + allow_blank: true validates :unique_ips_limit_per_user, - numericality: { greater_than_or_equal_to: 1 }, - presence: true, - if: :unique_ips_limit_enabled + numericality: { greater_than_or_equal_to: 1 }, + presence: true, + if: :unique_ips_limit_enabled validates :unique_ips_limit_time_window, - numericality: { greater_than_or_equal_to: 0 }, - presence: true, - if: :unique_ips_limit_enabled + numericality: { greater_than_or_equal_to: 0 }, + presence: true, + if: :unique_ips_limit_enabled - validates :kroki_url, - presence: { if: :kroki_enabled } + validates :kroki_url, presence: { if: :kroki_enabled } validate :validate_kroki_url, if: :kroki_enabled validates :kroki_formats, json_schema: { filename: 'application_setting_kroki_formats' } validates :metrics_method_call_threshold, - numericality: { greater_than_or_equal_to: 0 }, - presence: true, - if: :prometheus_metrics_enabled + numericality: { greater_than_or_equal_to: 0 }, + presence: true, + if: :prometheus_metrics_enabled - validates :plantuml_url, - presence: true, - if: :plantuml_enabled + validates :plantuml_url, presence: true, if: :plantuml_enabled - validates :sourcegraph_url, - presence: true, - if: :sourcegraph_enabled + validates :sourcegraph_url, presence: true, if: :sourcegraph_enabled validates :gitpod_url, - presence: true, - addressable_url: { enforce_sanitization: true }, - if: :gitpod_enabled + presence: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS.merge({ enforce_sanitization: true }), + if: :gitpod_enabled validates :mailgun_signing_key, - presence: true, - length: { maximum: 255 }, - if: :mailgun_events_enabled + presence: true, + length: { maximum: 255 }, + if: :mailgun_events_enabled validates :snowplow_collector_hostname, - presence: true, - hostname: true, - if: :snowplow_enabled + presence: true, + hostname: true, + if: :snowplow_enabled validates :max_attachment_size, - presence: true, - numericality: { only_integer: true, greater_than: 0 } + presence: true, + numericality: { only_integer: true, greater_than: 0 } validates :max_artifacts_size, - presence: true, - numericality: { only_integer: true, greater_than: 0 } + presence: true, + numericality: { only_integer: true, greater_than: 0 } validates :max_export_size, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :max_import_size, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :max_pages_size, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0, - less_than: ::Gitlab::Pages::MAX_SIZE / 1.megabyte } + presence: true, + numericality: { + only_integer: true, greater_than_or_equal_to: 0, + less_than: ::Gitlab::Pages::MAX_SIZE / 1.megabyte + } validates :max_pages_custom_domains_per_project, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :jobs_per_stage_page_size, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :max_terraform_state_size_bytes, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :default_artifacts_expire_in, presence: true, duration: true validates :container_expiration_policies_enable_historic_entries, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :container_registry_token_expire_delay, - presence: true, - numericality: { only_integer: true, greater_than: 0 } + presence: true, + numericality: { only_integer: true, greater_than: 0 } validates :repository_storages, presence: true validate :check_repository_storages validate :check_repository_storages_weighted validates :auto_devops_domain, - allow_blank: true, - hostname: { allow_numeric_hostname: true, require_valid_tld: true }, - if: :auto_devops_enabled? + allow_blank: true, + hostname: { allow_numeric_hostname: true, require_valid_tld: true }, + if: :auto_devops_enabled? validates :enabled_git_access_protocol, - inclusion: { in: %w(ssh http), allow_blank: true } + inclusion: { in: %w(ssh http), allow_blank: true } validates :domain_denylist, - presence: { message: 'Domain denylist cannot be empty if denylist is enabled.' }, - if: :domain_denylist_enabled? + presence: { message: 'Domain denylist cannot be empty if denylist is enabled.' }, + if: :domain_denylist_enabled? validates :housekeeping_optimize_repository_period, - presence: true, - numericality: { only_integer: true, greater_than: 0 } + presence: true, + numericality: { only_integer: true, greater_than: 0 } validates :terminal_max_session_time, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :polling_interval_multiplier, - presence: true, - numericality: { greater_than_or_equal_to: 0 } + presence: true, + numericality: { greater_than_or_equal_to: 0 } validates :gitaly_timeout_default, - presence: true, - if: :gitaly_timeout_default_changed?, - numericality: { - only_integer: true, - greater_than_or_equal_to: 0, - less_than_or_equal_to: Settings.gitlab.max_request_duration_seconds - } + presence: true, + if: :gitaly_timeout_default_changed?, + numericality: { + only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: Settings.gitlab.max_request_duration_seconds + } validates :gitaly_timeout_medium, - presence: true, - if: :gitaly_timeout_medium_changed?, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + if: :gitaly_timeout_medium_changed?, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :gitaly_timeout_medium, - numericality: { less_than_or_equal_to: :gitaly_timeout_default }, - if: :gitaly_timeout_default + numericality: { less_than_or_equal_to: :gitaly_timeout_default }, + if: :gitaly_timeout_default validates :gitaly_timeout_medium, - numericality: { greater_than_or_equal_to: :gitaly_timeout_fast }, - if: :gitaly_timeout_fast + numericality: { greater_than_or_equal_to: :gitaly_timeout_fast }, + if: :gitaly_timeout_fast validates :gitaly_timeout_fast, - presence: true, - if: :gitaly_timeout_fast_changed?, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + presence: true, + if: :gitaly_timeout_fast_changed?, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :gitaly_timeout_fast, - numericality: { less_than_or_equal_to: :gitaly_timeout_default }, - if: :gitaly_timeout_default + numericality: { less_than_or_equal_to: :gitaly_timeout_default }, + if: :gitaly_timeout_default validates :diff_max_patch_bytes, - presence: true, - numericality: { only_integer: true, - greater_than_or_equal_to: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, - less_than_or_equal_to: Gitlab::Git::Diff::MAX_PATCH_BYTES_UPPER_BOUND } + presence: true, + numericality: { + only_integer: true, + greater_than_or_equal_to: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, + less_than_or_equal_to: Gitlab::Git::Diff::MAX_PATCH_BYTES_UPPER_BOUND + } validates :diff_max_files, - presence: true, - numericality: { only_integer: true, - greater_than_or_equal_to: Commit::DEFAULT_MAX_DIFF_FILES_SETTING, - less_than_or_equal_to: Commit::MAX_DIFF_FILES_SETTING_UPPER_BOUND } + presence: true, + numericality: { + only_integer: true, + greater_than_or_equal_to: Commit::DEFAULT_MAX_DIFF_FILES_SETTING, + less_than_or_equal_to: Commit::MAX_DIFF_FILES_SETTING_UPPER_BOUND + } validates :diff_max_lines, - presence: true, - numericality: { only_integer: true, - greater_than_or_equal_to: Commit::DEFAULT_MAX_DIFF_LINES_SETTING, - less_than_or_equal_to: Commit::MAX_DIFF_LINES_SETTING_UPPER_BOUND } + presence: true, + numericality: { + only_integer: true, + greater_than_or_equal_to: Commit::DEFAULT_MAX_DIFF_LINES_SETTING, + less_than_or_equal_to: Commit::MAX_DIFF_LINES_SETTING_UPPER_BOUND + } validates :user_default_internal_regex, js_regex: true, allow_nil: true validates :default_preferred_language, presence: true, inclusion: { in: Gitlab::I18n.available_locales } validates :personal_access_token_prefix, - format: { with: %r{\A[a-zA-Z0-9_+=/@:.-]+\z}, - message: N_("can contain only letters of the Base64 alphabet (RFC4648) with the addition of '@', ':' and '.'") }, - length: { maximum: 20, message: N_('is too long (maximum is %{count} characters)') }, - allow_blank: true + format: { with: %r{\A[a-zA-Z0-9_+=/@:.-]+\z}, + message: N_("can contain only letters of the Base64 alphabet (RFC4648) with the addition of '@', ':' and '.'") }, + length: { maximum: 20, message: N_('is too long (maximum is %{count} characters)') }, + allow_blank: true validates :commit_email_hostname, format: { with: /\A[^@]+\z/ } validates :archive_builds_in_seconds, - allow_nil: true, - numericality: { only_integer: true, greater_than_or_equal_to: 1.day.seconds } + allow_nil: true, + numericality: { + only_integer: true, + greater_than_or_equal_to: 1.day.seconds, + message: N_('must be at least 1 day') + } validates :local_markdown_version, - allow_nil: true, - numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than: 65536 } + allow_nil: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than: 65536 } validates :asset_proxy_url, - presence: true, - allow_blank: false, - url: true, - if: :asset_proxy_enabled? + presence: true, + allow_blank: false, + url: true, + if: :asset_proxy_enabled? validates :asset_proxy_secret_key, - presence: true, - allow_blank: false, - if: :asset_proxy_enabled? + presence: true, + allow_blank: false, + if: :asset_proxy_enabled? validates :static_objects_external_storage_url, - addressable_url: true, allow_blank: true + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, allow_blank: true validates :static_objects_external_storage_auth_token, - presence: true, - if: :static_objects_external_storage_url? + presence: true, + if: :static_objects_external_storage_url? validates :protected_paths, - length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, - allow_nil: false + length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, + allow_nil: false validates :push_event_hooks_limit, - numericality: { greater_than_or_equal_to: 0 } + numericality: { greater_than_or_equal_to: 0 } validates :push_event_activities_limit, - numericality: { greater_than_or_equal_to: 0 } + numericality: { greater_than_or_equal_to: 0 } validates :snippet_size_limit, numericality: { only_integer: true, greater_than: 0 } validates :wiki_page_max_content_bytes, numericality: { only_integer: true, greater_than_or_equal_to: 1.kilobytes } validates :max_yaml_size_bytes, numericality: { only_integer: true, greater_than: 0 }, presence: true validates :max_yaml_depth, numericality: { only_integer: true, greater_than: 0 }, presence: true + validates :ci_max_includes, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, presence: true + validates :email_restrictions, untrusted_regexp: true validates :hashed_storage_enabled, inclusion: { in: [true], message: N_("Hashed storage can't be disabled anymore for new projects") } validates :container_registry_delete_tags_service_timeout, - :container_registry_cleanup_tags_service_max_list_size, - :container_registry_expiration_policies_worker_capacity, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + :container_registry_cleanup_tags_service_max_list_size, + :container_registry_expiration_policies_worker_capacity, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :container_registry_expiration_policies_caching, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :container_registry_import_max_tags_count, - :container_registry_import_max_retries, - :container_registry_import_start_max_retries, - :container_registry_import_max_step_duration, - :container_registry_pre_import_timeout, - :container_registry_import_timeout, - allow_nil: false, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + :container_registry_import_max_retries, + :container_registry_import_start_max_retries, + :container_registry_import_max_step_duration, + :container_registry_pre_import_timeout, + :container_registry_import_timeout, + allow_nil: false, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :container_registry_pre_import_tags_rate, - allow_nil: false, - numericality: { greater_than_or_equal_to: 0 } + allow_nil: false, + numericality: { greater_than_or_equal_to: 0 } validates :container_registry_import_target_plan, presence: true validates :container_registry_import_created_before, presence: true validates :dependency_proxy_ttl_group_policy_worker_capacity, - allow_nil: false, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + allow_nil: false, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :packages_cleanup_package_file_worker_capacity, - :package_registry_cleanup_policies_worker_capacity, - allow_nil: false, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + :package_registry_cleanup_policies_worker_capacity, + allow_nil: false, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :invisible_captcha_enabled, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :invitation_flow_enforcement, :can_create_group, :user_defaults_to_private_profile, - allow_nil: false, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :deactivate_dormant_users_period, - presence: true, - numericality: { only_integer: true, greater_than_or_equal_to: 90, message: N_("'%{value}' days of inactivity must be greater than or equal to 90") }, - if: :deactivate_dormant_users? + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 90, message: N_("'%{value}' days of inactivity must be greater than or equal to 90") }, + if: :deactivate_dormant_users? validates :allow_possible_spam, - allow_nil: false, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + + validates :deny_all_requests_except_allowed, + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + + validates :silent_mode_enabled, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + + validates :remember_me_enabled, + inclusion: { in: [true, false], message: N_('must be a boolean value') } Gitlab::SSHPublicKey.supported_types.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } @@ -448,93 +473,90 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validate :terms_exist, if: :enforce_terms? validates :external_authorization_service_default_label, - presence: true, - if: :external_authorization_service_enabled + presence: true, + if: :external_authorization_service_enabled validates :external_authorization_service_url, - addressable_url: true, allow_blank: true, - if: :external_authorization_service_enabled + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, allow_blank: true, + if: :external_authorization_service_enabled validates :external_authorization_service_timeout, - numericality: { greater_than: 0, less_than_or_equal_to: 10 }, - if: :external_authorization_service_enabled + numericality: { greater_than: 0, less_than_or_equal_to: 10 }, + if: :external_authorization_service_enabled validates :spam_check_endpoint_url, - addressable_url: { schemes: %w(tls grpc) }, allow_blank: true + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS.merge({ schemes: %w(tls grpc) }), allow_blank: true validates :spam_check_endpoint_url, - presence: true, - if: :spam_check_endpoint_enabled + presence: true, + if: :spam_check_endpoint_enabled validates :external_auth_client_key, - presence: true, - if: ->(setting) { setting.external_auth_client_cert.present? } + presence: true, + if: ->(setting) { setting.external_auth_client_cert.present? } validates :lets_encrypt_notification_email, - devise_email: true, - format: { without: /@example\.(com|org|net)\z/, - message: N_("Let's Encrypt does not accept emails on example.com") }, - allow_blank: true + devise_email: true, + format: { without: /@example\.(com|org|net)\z/, message: N_("Let's Encrypt does not accept emails on example.com") }, + allow_blank: true validates :lets_encrypt_notification_email, - presence: true, - if: :lets_encrypt_terms_of_service_accepted? + presence: true, + if: :lets_encrypt_terms_of_service_accepted? validates :eks_integration_enabled, - inclusion: { in: [true, false] } + inclusion: { in: [true, false] } validates :eks_account_id, - format: { with: Gitlab::Regex.aws_account_id_regex, - message: Gitlab::Regex.aws_account_id_message }, - if: :eks_integration_enabled? + format: { with: Gitlab::Regex.aws_account_id_regex, message: Gitlab::Regex.aws_account_id_message }, + if: :eks_integration_enabled? validates :eks_access_key_id, - length: { in: 16..128 }, - if: ->(setting) { setting.eks_integration_enabled? && setting.eks_access_key_id.present? } + length: { in: 16..128 }, + if: ->(setting) { setting.eks_integration_enabled? && setting.eks_access_key_id.present? } validates :eks_secret_access_key, - presence: true, - if: ->(setting) { setting.eks_integration_enabled? && setting.eks_access_key_id.present? } + presence: true, + if: ->(setting) { setting.eks_integration_enabled? && setting.eks_access_key_id.present? } validates_with X509CertificateCredentialsValidator, - certificate: :external_auth_client_cert, - pkey: :external_auth_client_key, - pass: :external_auth_client_key_pass, - if: ->(setting) { setting.external_auth_client_cert.present? } + certificate: :external_auth_client_cert, + pkey: :external_auth_client_key, + pass: :external_auth_client_key_pass, + if: ->(setting) { setting.external_auth_client_cert.present? } validates :default_ci_config_path, - format: { without: %r{(\.{2}|\A/)}, - message: N_('cannot include leading slash or directory traversal.') }, + format: { without: %r{(\.{2}|\A/)}, message: N_('cannot include leading slash or directory traversal.') }, length: { maximum: 255 }, allow_blank: true validates :issues_create_limit, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :raw_blob_request_limit, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :pipeline_limit_per_project_user_sha, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :ci_jwt_signing_key, - rsa_key: true, allow_nil: true + rsa_key: true, allow_nil: true validates :customers_dot_jwt_signing_key, - rsa_key: true, allow_nil: true + rsa_key: true, allow_nil: true validates :rate_limiting_response_text, - length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, - allow_blank: true + length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, + allow_blank: true validates :jira_connect_application_key, - length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, - allow_blank: true + length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, + allow_blank: true validates :jira_connect_proxy_url, - length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, - allow_blank: true, - public_url: true + length: { maximum: 255, message: N_('is too long (maximum is %{count} characters)') }, + allow_blank: true, + public_url: ADDRESSABLE_URL_VALIDATION_OPTIONS with_options(presence: true, numericality: { only_integer: true, greater_than: 0 }) do validates :throttle_unauthenticated_api_requests_per_period @@ -563,54 +585,52 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :throttle_protected_paths_period_in_seconds end - validates :notes_create_limit, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } - - validates :search_rate_limit, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } - - validates :search_rate_limit_unauthenticated, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + with_options(numericality: { only_integer: true, greater_than_or_equal_to: 0 }) do + validates :notes_create_limit + validates :search_rate_limit + validates :search_rate_limit_unauthenticated + validates :projects_api_rate_limit_unauthenticated + end validates :notes_create_limit_allowlist, - length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, - allow_nil: false + length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, + allow_nil: false validates :admin_mode, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :external_pipeline_validation_service_url, - addressable_url: true, allow_blank: true + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, allow_blank: true validates :external_pipeline_validation_service_timeout, - allow_nil: true, - numericality: { only_integer: true, greater_than: 0 } + allow_nil: true, + numericality: { only_integer: true, greater_than: 0 } validates :whats_new_variant, - inclusion: { in: ApplicationSetting.whats_new_variants.keys } + inclusion: { in: ApplicationSetting.whats_new_variants.keys } validates :floc_enabled, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } enum sidekiq_job_limiter_mode: { - Gitlab::SidekiqMiddleware::SizeLimiter::Validator::TRACK_MODE => 0, - Gitlab::SidekiqMiddleware::SizeLimiter::Validator::COMPRESS_MODE => 1 # The default - } + Gitlab::SidekiqMiddleware::SizeLimiter::Validator::TRACK_MODE => 0, + Gitlab::SidekiqMiddleware::SizeLimiter::Validator::COMPRESS_MODE => 1 # The default + } validates :sidekiq_job_limiter_mode, - inclusion: { in: self.sidekiq_job_limiter_modes } + inclusion: { in: self.sidekiq_job_limiter_modes } validates :sidekiq_job_limiter_compression_threshold_bytes, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :sidekiq_job_limiter_limit_bytes, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :sentry_enabled, inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :sentry_dsn, - addressable_url: true, presence: true, length: { maximum: 255 }, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, presence: true, length: { maximum: 255 }, if: :sentry_enabled? validates :sentry_clientside_dsn, - addressable_url: true, allow_blank: true, length: { maximum: 255 }, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, allow_blank: true, length: { maximum: 255 }, if: :sentry_enabled? validates :sentry_environment, presence: true, length: { maximum: 255 }, @@ -620,32 +640,39 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :error_tracking_api_url, presence: true, - addressable_url: true, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, length: { maximum: 255 }, if: :error_tracking_enabled? validates :users_get_by_id_limit, numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :users_get_by_id_limit_allowlist, - length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, - allow_nil: false + length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, + allow_nil: false - validates :public_runner_releases_url, addressable_url: true, presence: true + validates :update_runner_versions_enabled, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + validates :public_runner_releases_url, + addressable_url: ADDRESSABLE_URL_VALIDATION_OPTIONS, + presence: true, + if: :update_runner_versions_enabled? validates :inactive_projects_min_size_mb, - numericality: { only_integer: true, greater_than_or_equal_to: 0 } + numericality: { only_integer: true, greater_than_or_equal_to: 0 } validates :inactive_projects_delete_after_months, - numericality: { only_integer: true, greater_than: 0 } + numericality: { only_integer: true, greater_than: 0 } validates :inactive_projects_send_warning_email_after_months, - numericality: { only_integer: true, greater_than: 0, less_than: :inactive_projects_delete_after_months } + numericality: { only_integer: true, greater_than: 0, less_than: :inactive_projects_delete_after_months } + + validates :database_apdex_settings, json_schema: { filename: 'application_setting_database_apdex_settings' }, allow_nil: true attr_encrypted :asset_proxy_secret_key, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_truncated, - algorithm: 'aes-256-cbc', - insecure_mode: true + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_truncated, + algorithm: 'aes-256-cbc', + insecure_mode: true private_class_method def self.encryption_options_base_32_aes_256_gcm { @@ -683,24 +710,49 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord attr_encrypted :telesign_customer_xid, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) attr_encrypted :telesign_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) attr_encrypted :product_analytics_clickhouse_connection_string, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :product_analytics_configurator_connection_string, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :openai_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :anthropic_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + # TOFA API integration settngs + attr_encrypted :tofa_client_library_args, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_client_library_class, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_client_library_create_credentials_method, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_client_library_fetch_access_token_method, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_credentials, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_host, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_request_json_keys, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_request_payload, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_response_json_keys, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_url, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) + attr_encrypted :tofa_access_token_expires_in, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) validates :disable_feed_token, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :disable_admin_oauth_scopes, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :bulk_import_enabled, - allow_nil: false, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } validates :allow_runner_registration_token, - allow_nil: false, - inclusion: { in: [true, false], message: N_('must be a boolean value') } + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } + + validates :default_syntax_highlighting_theme, + allow_nil: false, + numericality: { only_integer: true, greater_than: 0 }, + inclusion: { in: Gitlab::ColorSchemes.valid_ids, message: N_('must be a valid syntax highlighting theme ID') } + + validates :gitlab_dedicated_instance, + allow_nil: false, + inclusion: { in: [true, false], message: N_('must be a boolean value') } before_validation :ensure_uuid! before_validation :coerce_repository_storages_weighted, if: :repository_storages_weighted_changed? before_validation :normalize_default_branch_name + before_validation :remove_old_import_sources before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token @@ -744,6 +796,10 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord users_count >= INSTANCE_REVIEW_MIN_USERS end + def remove_old_import_sources + self.import_sources -= %w[phabricator gitlab] if self.import_sources + end + Recursion = Class.new(RuntimeError) def self.create_from_defaults @@ -824,6 +880,10 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord private + def self.human_attribute_name(attribute, *options) + HUMANIZED_ATTRIBUTES[attribute.to_sym] || super + end + def parsed_grafana_url @parsed_grafana_url ||= Gitlab::Utils.parse_url(grafana_url) end diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index a5f262f2e1e..845d402f550 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -60,6 +60,7 @@ module ApplicationSettingImplementation default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_projects_limit: Settings.gitlab['default_projects_limit'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], + deny_all_requests_except_allowed: false, diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, diff_max_files: Commit::DEFAULT_MAX_DIFF_FILES_SETTING, diff_max_lines: Commit::DEFAULT_MAX_DIFF_LINES_SETTING, @@ -96,7 +97,7 @@ module ApplicationSettingImplementation group_import_limit: 6, help_page_hide_commercial_content: false, help_page_text: nil, - help_page_documentation_base_url: nil, + help_page_documentation_base_url: 'https://docs.gitlab.com', hide_third_party_offers: false, housekeeping_enabled: true, housekeeping_full_repack_period: 50, @@ -249,7 +250,10 @@ module ApplicationSettingImplementation can_create_group: true, bulk_import_enabled: false, allow_runner_registration_token: true, - user_defaults_to_private_profile: false + user_defaults_to_private_profile: false, + projects_api_rate_limit_unauthenticated: 400, + gitlab_dedicated_instance: false, + ci_max_includes: 150 }.tap do |hsh| hsh.merge!(non_production_defaults) unless Rails.env.production? end diff --git a/app/models/atlassian/identity.rb b/app/models/atlassian/identity.rb index 02bbe007e1b..1ad7f657db1 100644 --- a/app/models/atlassian/identity.rb +++ b/app/models/atlassian/identity.rb @@ -10,17 +10,17 @@ module Atlassian validates :user, presence: true, uniqueness: true attr_encrypted :token, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm', - encode: false, - encode_iv: false + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + encode: false, + encode_iv: false attr_encrypted :refresh_token, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm', - encode: false, - encode_iv: false + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + encode: false, + encode_iv: false end end diff --git a/app/models/audit_event.rb b/app/models/audit_event.rb index 3312216932b..163e741d990 100644 --- a/app/models/audit_event.rb +++ b/app/models/audit_event.rb @@ -21,7 +21,7 @@ class AuditEvent < ApplicationRecord serialize :details, Hash # rubocop:disable Cop/ActiveRecordSerialize - belongs_to :user, foreign_key: :author_id + belongs_to :user, foreign_key: :author_id, inverse_of: :audit_events validates :author_id, presence: true validates :entity_id, presence: true diff --git a/app/models/authentication_event.rb b/app/models/authentication_event.rb index d5a5079acd6..a70ebb42008 100644 --- a/app/models/authentication_event.rb +++ b/app/models/authentication_event.rb @@ -30,4 +30,8 @@ class AuthenticationEvent < ApplicationRecord !where(user_id: user).exists? || where(user_id: user, ip_address: ip_address).success.exists? end + + def self.most_used_ip_address_for_user(user) + select('mode() within group (order by ip_address) as ip_address').find_by(user: user).ip_address + end end diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index dbc5c7a584e..31bee8db1b4 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -7,6 +7,9 @@ class AwardEmoji < ApplicationRecord include Participable include GhostUser include Importable + include IgnorableColumns + + ignore_column :awardable_id_convert_to_bigint, remove_with: '16.2', remove_after: '2023-07-22' belongs_to :awardable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :user diff --git a/app/models/awareness_session.rb b/app/models/awareness_session.rb deleted file mode 100644 index 0b652984630..00000000000 --- a/app/models/awareness_session.rb +++ /dev/null @@ -1,245 +0,0 @@ -# frozen_string_literal: true - -# A Redis backed session store for real-time collaboration. A session is defined -# by its documents and the users that join this session. An online user can have -# two states within the session: "active" and "away". -# -# By design, session must eventually be cleaned up. If this doesn't happen -# explicitly, all keys used within the session model must have an expiry -# timestamp set. -class AwarenessSession # rubocop:disable Gitlab/NamespacedClass - # An awareness session expires automatically after 1 hour of no activity - SESSION_LIFETIME = 1.hour - private_constant :SESSION_LIFETIME - - # Expire user awareness keys after some time of inactivity - USER_LIFETIME = 1.hour - private_constant :USER_LIFETIME - - PRESENCE_LIFETIME = 10.minutes - private_constant :PRESENCE_LIFETIME - - KEY_NAMESPACE = "gitlab:awareness" - private_constant :KEY_NAMESPACE - - class << self - def for(value = nil) - # Creates a unique value for situations where we have no unique value to - # create a session with. This could be when creating a new issue, a new - # merge request, etc. - value = SecureRandom.uuid unless value.present? - - # We use SHA-256 based session identifiers (similar to abbreviated git - # hashes). There is always a chance for Hash collisions (birthday - # problem), we therefore have to pick a good tradeoff between the amount - # of data stored and the probability of a collision. - # - # The approximate probability for a collision can be calculated: - # - # p ~= n^2 / 2m - # ~= (2^18)^2 / (2 * 16^15) - # ~= 2^36 / 2^61 - # - # n is the number of awareness sessions and m the number of possibilities - # for each item. For a hex number, this is 16^c, where c is the number of - # characters. With 260k (~2^18) sessions, the probability for a collision - # is ~2^-25. - # - # The number of 15 is selected carefully. The integer representation fits - # nicely into a signed 64 bit integer and eventually allows Redis to - # optimize its memory usage. 16 chars would exceed the space for - # this datatype. - id = Digest::SHA256.hexdigest(value.to_s)[0, 15] - - AwarenessSession.new(id) - end - end - - def initialize(id) - @id = id - end - - def join(user) - user_key = user_sessions_key(user.id) - - with_redis do |redis| - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipeline| - pipeline.sadd?(user_key, id_i) - pipeline.expire(user_key, USER_LIFETIME.to_i) - - pipeline.zadd(users_key, timestamp.to_f, user.id) - - # We also mark for expiry when a session key is created (first user joins), - # because some users might never actively leave a session and the key could - # therefore become stale, w/o us noticing. - reset_session_expiry(pipeline) - end - end - end - - nil - end - - def leave(user) - user_key = user_sessions_key(user.id) - - with_redis do |redis| - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipeline| - pipeline.srem?(user_key, id_i) - pipeline.zrem(users_key, user.id) - end - end - - # cleanup orphan sessions and users - # - # this needs to be a second pipeline due to the delete operations being - # dependent on the result of the cardinality checks - user_sessions_count, session_users_count = - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipeline| - pipeline.scard(user_key) - pipeline.zcard(users_key) - end - end - - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipeline| - pipeline.del(user_key) unless user_sessions_count > 0 - - unless session_users_count > 0 - pipeline.del(users_key) - @id = nil - end - end - end - end - - nil - end - - def present?(user, threshold: PRESENCE_LIFETIME) - with_redis do |redis| - user_timestamp = redis.zscore(users_key, user.id) - break false unless user_timestamp.present? - - timestamp - user_timestamp < threshold - end - end - - def away?(user, threshold: PRESENCE_LIFETIME) - !present?(user, threshold: threshold) - end - - # Updates the last_activity timestamp for a user in this session - def touch!(user) - with_redis do |redis| - redis.pipelined do |pipeline| - pipeline.zadd(users_key, timestamp.to_f, user.id) - - # extend the session lifetime due to user activity - reset_session_expiry(pipeline) - end - end - - nil - end - - def size - with_redis do |redis| - redis.zcard(users_key) - end - end - - def to_param - id&.to_s - end - - def to_s - "awareness_session=#{id}" - end - - def online_users_with_last_activity(threshold: PRESENCE_LIFETIME) - users_with_last_activity.filter do |_user, last_activity| - user_online?(last_activity, threshold: threshold) - end - end - - def users - User.where(id: user_ids) - end - - def users_with_last_activity - # where in (x, y, [...z]) is a set and does not maintain any order, we need - # to make sure to establish a stable order for both, the pairs returned from - # redis and the ActiveRecord query. Using IDs in ascending order. - user_ids, last_activities = user_ids_with_last_activity - .sort_by(&:first) - .transpose - - return [] if user_ids.blank? - - users = User.where(id: user_ids).order(id: :asc) - users.zip(last_activities) - end - - private - - attr_reader :id - - def user_online?(last_activity, threshold:) - last_activity.to_i + threshold.to_i > Time.zone.now.to_i - end - - # converts session id from hex to integer representation - def id_i - Integer(id, 16) if id.present? - end - - def users_key - "#{KEY_NAMESPACE}:session:#{id}:users" - end - - def user_sessions_key(user_id) - "#{KEY_NAMESPACE}:user:#{user_id}:sessions" - end - - def with_redis - Gitlab::Redis::SharedState.with do |redis| - yield redis if block_given? - end - end - - def timestamp - Time.now.to_i - end - - def user_ids - with_redis do |redis| - redis.zrange(users_key, 0, -1) - end - end - - # Returns an array of tuples, where the first element in the tuple represents - # the user ID and the second part the last_activity timestamp. - def user_ids_with_last_activity - pairs = with_redis do |redis| - redis.zrange(users_key, 0, -1, with_scores: true) - end - - # map data type of score (float) to Time - pairs.map do |user_id, score| - [user_id, Time.zone.at(score.to_i)] - end - end - - # We want sessions to cleanup automatically after a certain period of - # inactivity. This sets the expiry timestamp for this session to - # [SESSION_LIFETIME]. - def reset_session_expiry(redis) - redis.expire(users_key, SESSION_LIFETIME) - - nil - end -end diff --git a/app/models/badge.rb b/app/models/badge.rb index 0676de10d02..23e6f305c32 100644 --- a/app/models/badge.rb +++ b/app/models/badge.rb @@ -42,7 +42,7 @@ class Badge < ApplicationRecord private def build_rendered_url(url, project = nil) - return url unless valid? && project + return url unless project Gitlab::StringPlaceholderReplacer.replace_string_placeholders(url, PLACEHOLDERS_REGEX) do |arg| replace_placeholder_action(PLACEHOLDERS[arg], project) diff --git a/app/models/badges/project_badge.rb b/app/models/badges/project_badge.rb index 59638df6fad..8c51ebafb5e 100644 --- a/app/models/badges/project_badge.rb +++ b/app/models/badges/project_badge.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ProjectBadge < Badge + include EachBatch + belongs_to :project validates :project, presence: true diff --git a/app/models/blob_viewer/composer_json.rb b/app/models/blob_viewer/composer_json.rb index 9d1376de0cb..aac7271242e 100644 --- a/app/models/blob_viewer/composer_json.rb +++ b/app/models/blob_viewer/composer_json.rb @@ -15,7 +15,7 @@ module BlobViewer end def package_name - @package_name ||= package_name_from_json('name') + @package_name ||= fetch_from_json('name') end def package_url diff --git a/app/models/blob_viewer/dependency_manager.rb b/app/models/blob_viewer/dependency_manager.rb index a3801025cd7..71bd90e7459 100644 --- a/app/models/blob_viewer/dependency_manager.rb +++ b/app/models/blob_viewer/dependency_manager.rb @@ -38,8 +38,10 @@ module BlobViewer end end - def package_name_from_json(key) - json_data[key] + def fetch_from_json(...) + json_data.dig(...) + rescue TypeError + nil end def package_name_from_method_call(name) diff --git a/app/models/blob_viewer/metrics_dashboard_yml.rb b/app/models/blob_viewer/metrics_dashboard_yml.rb index 4b7a178566c..b63f3022198 100644 --- a/app/models/blob_viewer/metrics_dashboard_yml.rb +++ b/app/models/blob_viewer/metrics_dashboard_yml.rb @@ -11,6 +11,10 @@ module BlobViewer self.file_types = %i(metrics_dashboard) self.binary = false + def self.can_render?(blob, verify_binary: true) + super && !Feature.enabled?(:remove_monitor_metrics) + end + def valid? errors.blank? end diff --git a/app/models/blob_viewer/package_json.rb b/app/models/blob_viewer/package_json.rb index 1d10cc82a85..5350b6b0626 100644 --- a/app/models/blob_viewer/package_json.rb +++ b/app/models/blob_viewer/package_json.rb @@ -11,7 +11,7 @@ module BlobViewer end def yarn? - json_data['engines'].present? && json_data['engines']['yarn'].present? + fetch_from_json('engines', 'yarn').present? end def manager_url @@ -19,7 +19,7 @@ module BlobViewer end def package_name - @package_name ||= package_name_from_json('name') + @package_name ||= fetch_from_json('name') end def package_type @@ -33,11 +33,11 @@ module BlobViewer private def private? - !!json_data['private'] + !!fetch_from_json('private') end def homepage - url = json_data['homepage'] + url = fetch_from_json('homepage') url if Gitlab::UrlSanitizer.valid?(url) end diff --git a/app/models/blob_viewer/podspec_json.rb b/app/models/blob_viewer/podspec_json.rb index d3f6ae269da..d606f72376d 100644 --- a/app/models/blob_viewer/podspec_json.rb +++ b/app/models/blob_viewer/podspec_json.rb @@ -5,7 +5,7 @@ module BlobViewer self.file_types = %i(podspec_json) def package_name - @package_name ||= package_name_from_json('name') + @package_name ||= fetch_from_json('name') end end end diff --git a/app/models/board.rb b/app/models/board.rb index 2181b2f0545..da9cd1548e4 100644 --- a/app/models/board.rb +++ b/app/models/board.rb @@ -1,13 +1,15 @@ # frozen_string_literal: true class Board < ApplicationRecord + include EachBatch + RECENT_BOARDS_SIZE = 4 belongs_to :group belongs_to :project - has_many :lists, -> { ordered }, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent - has_many :destroyable_lists, -> { destroyable.ordered }, class_name: "List" + has_many :lists, -> { ordered }, dependent: :delete_all, inverse_of: :board # rubocop:disable Cop/ActiveRecordDependent + has_many :destroyable_lists, -> { destroyable.ordered }, class_name: "List", inverse_of: :board validates :name, presence: true validates :project, presence: true, if: :project_needed? diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb index c5a234ffa69..733018160cd 100644 --- a/app/models/broadcast_message.rb +++ b/app/models/broadcast_message.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true -class BroadcastMessage < ApplicationRecord +class BroadcastMessage < MainClusterwide::ApplicationRecord include CacheMarkdownField include Sortable + include IgnorableColumns ALLOWED_TARGET_ACCESS_LEVELS = [ Gitlab::Access::GUEST, @@ -12,6 +13,8 @@ class BroadcastMessage < ApplicationRecord Gitlab::Access::OWNER ].freeze + ignore_column :namespace_id, remove_with: '16.0', remove_after: '2022-06-22' + cache_markdown_field :message, pipeline: :broadcast_message, whitelisted: true validates :message, presence: true @@ -85,10 +88,8 @@ class BroadcastMessage < ApplicationRecord private - def fetch_messages(cache_key, current_path, user_access_level) - messages = cache.fetch(cache_key, as: BroadcastMessage, expires_in: cache_expires_in) do - yield - end + def fetch_messages(cache_key, current_path, user_access_level, &block) + messages = cache.fetch(cache_key, as: BroadcastMessage, expires_in: cache_expires_in, &block) now_or_future = messages.select(&:now_or_future?) @@ -131,7 +132,6 @@ class BroadcastMessage < ApplicationRecord end def matches_current_user_access_level?(user_access_level) - return false if target_access_levels.present? && Feature.disabled?(:role_targeted_broadcast_messages) return true unless target_access_levels.present? target_access_levels.include? user_access_level @@ -145,9 +145,7 @@ class BroadcastMessage < ApplicationRecord # This fixes a mismatch between requests in the GUI and CLI # # This has to be reassigned due to frozen strings being provided. - unless current_path.start_with?("/") - current_path = "/#{current_path}" - end + current_path = "/#{current_path}" unless current_path.start_with?("/") escaped = Regexp.escape(target_path).gsub('\\*', '.*') regexp = Regexp.new "^#{escaped}$", Regexp::IGNORECASE diff --git a/app/models/bulk_import.rb b/app/models/bulk_import.rb index 2565ad5f2b8..c2d7529f468 100644 --- a/app/models/bulk_import.rb +++ b/app/models/bulk_import.rb @@ -42,6 +42,12 @@ class BulkImport < ApplicationRecord event :fail_op do transition any => :failed end + + # rubocop:disable Style/SymbolProc + after_transition any => [:finished, :failed, :timeout] do |bulk_import| + bulk_import.update_has_failures + end + # rubocop:enable Style/SymbolProc end def source_version_info @@ -55,4 +61,11 @@ class BulkImport < ApplicationRecord def self.all_human_statuses state_machine.states.map(&:human_name) end + + def update_has_failures + return if has_failures + return unless entities.any?(&:has_failures) + + update!(has_failures: true) + end end diff --git a/app/models/bulk_imports/batch_tracker.rb b/app/models/bulk_imports/batch_tracker.rb new file mode 100644 index 00000000000..df1fab89ee6 --- /dev/null +++ b/app/models/bulk_imports/batch_tracker.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module BulkImports + class BatchTracker < ApplicationRecord + self.table_name = 'bulk_import_batch_trackers' + + belongs_to :tracker, class_name: 'BulkImports::Tracker' + + validates :batch_number, presence: true, uniqueness: { scope: :tracker_id } + + state_machine :status, initial: :created do + state :created, value: 0 + state :started, value: 1 + state :finished, value: 2 + state :timeout, value: 3 + state :failed, value: -1 + state :skipped, value: -2 + + event :start do + transition created: :started + end + + event :retry do + transition started: :created + end + + event :finish do + transition started: :finished + transition failed: :failed + transition skipped: :skipped + end + + event :skip do + transition any => :skipped + end + + event :fail_op do + transition any => :failed + end + + event :cleanup_stale do + transition [:created, :started] => :timeout + end + end + end +end diff --git a/app/models/bulk_imports/configuration.rb b/app/models/bulk_imports/configuration.rb index 3b263ed0340..6d9f598583e 100644 --- a/app/models/bulk_imports/configuration.rb +++ b/app/models/bulk_imports/configuration.rb @@ -9,7 +9,7 @@ class BulkImports::Configuration < ApplicationRecord validates :url, :access_token, length: { maximum: 255 }, presence: true validates :url, public_url: { schemes: %w[http https], enforce_sanitization: true, ascii_only: true }, - allow_nil: true + allow_nil: true attr_encrypted :url, key: Settings.attr_encrypted_db_key_base_32, diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 6fc24c77f1d..94e4a8165eb 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -26,10 +26,11 @@ class BulkImports::Entity < ApplicationRecord belongs_to :parent, class_name: 'BulkImports::Entity', optional: true belongs_to :project, optional: true - belongs_to :group, foreign_key: :namespace_id, optional: true + belongs_to :group, foreign_key: :namespace_id, optional: true, inverse_of: :bulk_import_entities has_many :trackers, class_name: 'BulkImports::Tracker', + inverse_of: :entity, foreign_key: :bulk_import_entity_id has_many :failures, @@ -40,27 +41,14 @@ class BulkImports::Entity < ApplicationRecord validates :project, absence: true, if: :group validates :group, absence: true, if: :project validates :source_type, presence: true - validates :source_full_path, - presence: true, - format: { with: Gitlab::Regex.bulk_import_source_full_path_regex, - message: Gitlab::Regex.bulk_import_destination_namespace_path_regex_message } - - validates :destination_name, - presence: true, - format: { with: Gitlab::Regex.group_path_regex, - message: Gitlab::Regex.group_path_regex_message } - - validates :destination_namespace, - exclusion: [nil], - format: { with: Gitlab::Regex.bulk_import_destination_namespace_path_regex, - message: Gitlab::Regex.bulk_import_destination_namespace_path_regex_message }, - if: :group - - validates :destination_namespace, - presence: true, - format: { with: Gitlab::Regex.bulk_import_destination_namespace_path_regex, - message: Gitlab::Regex.bulk_import_destination_namespace_path_regex_message }, - if: :project + validates :source_full_path, presence: true, format: { + with: Gitlab::Regex.bulk_import_source_full_path_regex, + message: Gitlab::Regex.bulk_import_source_full_path_regex_message + } + + validates :destination_name, presence: true, if: -> { group || project } + validates :destination_namespace, exclusion: [nil], if: :group + validates :destination_namespace, presence: true, if: :project? validate :validate_parent_is_a_group, if: :parent validate :validate_imported_entity_type @@ -76,9 +64,8 @@ class BulkImports::Entity < ApplicationRecord alias_attribute :destination_slug, :destination_name - delegate :default_project_visibility, - :default_group_visibility, - to: :'Gitlab::CurrentSettings.current_application_settings' + delegate :default_project_visibility, :default_group_visibility, + to: :'Gitlab::CurrentSettings.current_application_settings' state_machine :status, initial: :created do state :created, value: 0 @@ -104,6 +91,12 @@ class BulkImports::Entity < ApplicationRecord transition created: :timeout transition started: :timeout end + + # rubocop:disable Style/SymbolProc + after_transition any => [:finished, :failed, :timeout] do |entity| + entity.update_has_failures + end + # rubocop:enable Style/SymbolProc end def self.all_human_statuses @@ -185,6 +178,13 @@ class BulkImports::Entity < ApplicationRecord default_project_visibility end + def update_has_failures + return if has_failures + return unless failures.any? + + update!(has_failures: true) + end + private def validate_parent_is_a_group @@ -194,13 +194,6 @@ class BulkImports::Entity < ApplicationRecord end def validate_imported_entity_type - if project_entity? && !BulkImports::Features.project_migration_enabled?(destination_namespace) - errors.add( - :base, - s_('BulkImport|invalid entity source type') - ) - end - if group.present? && project_entity? errors.add( :group, diff --git a/app/models/bulk_imports/export.rb b/app/models/bulk_imports/export.rb index 8d4d31ee92d..93cf047c690 100644 --- a/app/models/bulk_imports/export.rb +++ b/app/models/bulk_imports/export.rb @@ -14,6 +14,7 @@ module BulkImports belongs_to :group, optional: true has_one :upload, class_name: 'BulkImports::ExportUpload' + has_many :batches, class_name: 'BulkImports::ExportBatch' validates :project, presence: true, unless: :group validates :group, presence: true, unless: :project @@ -32,6 +33,7 @@ module BulkImports event :finish do transition started: :finished + transition finished: :finished transition failed: :failed end diff --git a/app/models/bulk_imports/export_batch.rb b/app/models/bulk_imports/export_batch.rb new file mode 100644 index 00000000000..9d34dae12d0 --- /dev/null +++ b/app/models/bulk_imports/export_batch.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module BulkImports + class ExportBatch < ApplicationRecord + self.table_name = 'bulk_import_export_batches' + + BATCH_SIZE = 1000 + + belongs_to :export, class_name: 'BulkImports::Export' + has_one :upload, class_name: 'BulkImports::ExportUpload', foreign_key: :batch_id, inverse_of: :batch + + validates :batch_number, presence: true, uniqueness: { scope: :export_id } + + state_machine :status, initial: :started do + state :started, value: 0 + state :finished, value: 1 + state :failed, value: -1 + + event :start do + transition any => :started + end + + event :finish do + transition started: :finished + transition failed: :failed + end + + event :fail_op do + transition any => :failed + end + end + end +end diff --git a/app/models/bulk_imports/export_upload.rb b/app/models/bulk_imports/export_upload.rb index 4304032b28c..00f8e8f1304 100644 --- a/app/models/bulk_imports/export_upload.rb +++ b/app/models/bulk_imports/export_upload.rb @@ -7,6 +7,7 @@ module BulkImports self.table_name = 'bulk_import_export_uploads' belongs_to :export, class_name: 'BulkImports::Export' + belongs_to :batch, class_name: 'BulkImports::ExportBatch', optional: true mount_uploader :export_file, ExportUploader diff --git a/app/models/bulk_imports/file_transfer.rb b/app/models/bulk_imports/file_transfer.rb index 5be954b98da..c6af4e0c833 100644 --- a/app/models/bulk_imports/file_transfer.rb +++ b/app/models/bulk_imports/file_transfer.rb @@ -9,9 +9,9 @@ module BulkImports def config_for(portable) case portable when ::Project - FileTransfer::ProjectConfig.new(portable) + ::BulkImports::FileTransfer::ProjectConfig.new(portable) when ::Group - FileTransfer::GroupConfig.new(portable) + ::BulkImports::FileTransfer::GroupConfig.new(portable) else raise(UnsupportedObjectType, "Unsupported object type: #{portable.class}") end diff --git a/app/models/bulk_imports/file_transfer/base_config.rb b/app/models/bulk_imports/file_transfer/base_config.rb index 036d511bc59..32fc794627c 100644 --- a/app/models/bulk_imports/file_transfer/base_config.rb +++ b/app/models/bulk_imports/file_transfer/base_config.rb @@ -32,6 +32,15 @@ module BulkImports tree_relations + file_relations + self_relation - skipped_relations end + def batchable_relations + portable_relations.select { |relation| portable_class.reflect_on_association(relation)&.collection? } + end + strong_memoize_attr :batchable_relations + + def batchable_relation?(relation) + batchable_relations.include?(relation) + end + def self_relation?(relation) relation == SELF_RELATION end @@ -51,7 +60,21 @@ module BulkImports end def portable_relations_tree - @portable_relations_tree ||= attributes_finder.find_relations_tree(portable_class_sym).deep_stringify_keys + @portable_relations_tree ||= attributes_finder + .find_relations_tree(portable_class_sym, include_import_only_tree: true).deep_stringify_keys + end + + # Returns an export service class for the given relation. + # @return TreeExportService if a relation is serializable and is listed in import_export.yml + # @return FileExportService if a relation is a file (uploads, lfs objects, git repository, etc.) + def export_service_for(relation) + if tree_relation?(relation) + ::BulkImports::TreeExportService + elsif file_relation?(relation) + ::BulkImports::FileExportService + else + raise ::BulkImports::Error, 'Unsupported export relation' + end end private diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index b04ef1cb7ae..55502721a76 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -7,9 +7,12 @@ class BulkImports::Tracker < ApplicationRecord belongs_to :entity, class_name: 'BulkImports::Entity', + inverse_of: :trackers, foreign_key: :bulk_import_entity_id, optional: false + has_many :batches, class_name: 'BulkImports::BatchTracker', inverse_of: :tracker + validates :relation, presence: true, uniqueness: { scope: :bulk_import_entity_id } diff --git a/app/models/chat_name.rb b/app/models/chat_name.rb index 9bd618c1008..cda19273f52 100644 --- a/app/models/chat_name.rb +++ b/app/models/chat_name.rb @@ -3,7 +3,9 @@ class ChatName < ApplicationRecord LAST_USED_AT_INTERVAL = 1.hour - belongs_to :integration + include IgnorableColumns + ignore_column :integration_id, remove_with: '16.0', remove_after: '2023-04-22' + belongs_to :user validates :user, presence: true diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 697f06fbffd..7cdd0d56a98 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -55,8 +55,6 @@ module Ci end def retryable? - return false unless Feature.enabled?(:ci_recreate_downstream_pipeline, project) - return false if failed? && (pipeline_loop_detected? || reached_max_descendant_pipelines_depth?) super @@ -81,7 +79,9 @@ module Ci case pipeline.status when 'success' success! - when 'failed', 'canceled', 'skipped' + when 'canceled' + cancel! + when 'failed', 'skipped' drop! else false diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1e70dd171ed..61585de4ff7 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -18,14 +18,15 @@ module Ci belongs_to :runner belongs_to :trigger_request belongs_to :erased_by, class_name: 'User' - belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id + belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, inverse_of: :builds RUNNER_FEATURES = { upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? }, refspecs: -> (build) { build.merge_request_ref? }, artifacts_exclude: -> (build) { build.supports_artifacts_exclude? }, multi_build_steps: -> (build) { build.multi_build_steps? }, - return_exit_code: -> (build) { build.exit_codes_defined? } + return_exit_code: -> (build) { build.exit_codes_defined? }, + fallback_cache_keys: -> (build) { build.fallback_cache_keys_defined? } }.freeze DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD' @@ -35,8 +36,8 @@ module Ci has_one :deployment, as: :deployable, class_name: 'Deployment', inverse_of: :deployable has_one :pending_state, class_name: 'Ci::BuildPendingState', foreign_key: :build_id, inverse_of: :build - has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id - has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id + has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id, inverse_of: :build + has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id, inverse_of: :build has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id, inverse_of: :build has_many :report_results, class_name: 'Ci::BuildReportResult', foreign_key: :build_id, inverse_of: :build has_one :namespace, through: :project @@ -47,7 +48,7 @@ module Ci # Details: https://gitlab.com/gitlab-org/gitlab/-/issues/24644#note_689472685 has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy, inverse_of: :job # rubocop:disable Cop/ActiveRecordDependent has_many :job_variables, class_name: 'Ci::JobVariable', foreign_key: :job_id, inverse_of: :job - has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id, inverse_of: :build has_many :pages_deployments, foreign_key: :ci_build_id, inverse_of: :ci_build @@ -55,7 +56,9 @@ module Ci has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', foreign_key: :job_id, inverse_of: :job end - has_one :runner_machine, through: :metadata, class_name: 'Ci::RunnerMachine' + has_one :runner_manager_build, class_name: 'Ci::RunnerManagerBuild', foreign_key: :build_id, inverse_of: :build, + autosave: true + has_one :runner_manager, foreign_key: :runner_machine_id, through: :runner_manager_build, class_name: 'Ci::RunnerManager' has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, foreign_key: :build_id, inverse_of: :build has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', foreign_key: :build_id, inverse_of: :build @@ -71,6 +74,7 @@ module Ci delegate :gitlab_deploy_token, to: :project delegate :harbor_integration, to: :project delegate :apple_app_store_integration, to: :project + delegate :google_play_integration, to: :project delegate :trigger_short_token, to: :trigger_request, allow_nil: true delegate :ensure_persistent_ref, to: :pipeline delegate :enable_debug_trace!, to: :metadata @@ -132,7 +136,7 @@ module Ci scope :eager_load_job_artifacts, -> { includes(:job_artifacts) } scope :eager_load_tags, -> { includes(:tags) } - scope :eager_load_for_archiving_trace, -> { includes(:project, :pending_state) } + scope :eager_load_for_archiving_trace, -> { preload(:project, :pending_state) } scope :eager_load_everything, -> do includes( @@ -180,7 +184,9 @@ module Ci acts_as_taggable - add_authentication_token_field :token, encrypted: :required + add_authentication_token_field :token, + encrypted: :required, + format_with_prefix: :partition_id_prefix_in_16_bit_encode after_save :stick_build_if_status_changed @@ -592,14 +598,21 @@ module Ci .append(key: 'CI_JOB_URL', value: Gitlab::Routing.url_helpers.project_job_url(project, self)) .append(key: 'CI_JOB_TOKEN', value: token.to_s, public: false, masked: true) .append(key: 'CI_JOB_STARTED_AT', value: started_at&.iso8601) - .append(key: 'CI_BUILD_ID', value: id.to_s) - .append(key: 'CI_BUILD_TOKEN', value: token.to_s, public: false, masked: true) + + if Feature.disabled?(:ci_remove_legacy_predefined_variables, project) + variables + .append(key: 'CI_BUILD_ID', value: id.to_s) + .append(key: 'CI_BUILD_TOKEN', value: token.to_s, public: false, masked: true) + end + + variables .append(key: 'CI_REGISTRY_USER', value: ::Gitlab::Auth::CI_JOB_USER) .append(key: 'CI_REGISTRY_PASSWORD', value: token.to_s, public: false, masked: true) .append(key: 'CI_REPOSITORY_URL', value: repo_url.to_s, public: false) .concat(deploy_token_variables) .concat(harbor_variables) .concat(apple_app_store_variables) + .concat(google_play_variables) end end @@ -650,6 +663,13 @@ module Ci Gitlab::Ci::Variables::Collection.new(apple_app_store_integration.ci_variables) end + def google_play_variables + return [] unless google_play_integration.try(:activated?) + return [] unless pipeline.protected_ref? + + Gitlab::Ci::Variables::Collection.new(google_play_integration.ci_variables) + end + def features { trace_sections: true, @@ -757,9 +777,7 @@ module Ci end def remove_token! - if Feature.enabled?(:remove_job_token_on_completion, project) - update!(token_encrypted: nil) - end + update!(token_encrypted: nil) end # acts_as_taggable uses this method create/remove tags with contexts @@ -802,7 +820,7 @@ module Ci return unless project return if user&.blocked? - ActiveRecord::Associations::Preloader.new.preload([self], { runner: :tags }) + ActiveRecord::Associations::Preloader.new(records: [self], associations: { runner: :tags }).call project.execute_hooks(build_data.dup, :job_hooks) if project.has_active_hooks?(:job_hooks) project.execute_integrations(build_data.dup, :job_hooks) if project.has_active_integrations?(:job_hooks) @@ -902,9 +920,15 @@ module Ci def cache cache = Array.wrap(options[:cache]) + cache.each do |single_cache| + single_cache[:fallback_keys] = [] unless single_cache.key?(:fallback_keys) + end + if project.jobs_cache_index cache = cache.map do |single_cache| - single_cache.merge(key: "#{single_cache[:key]}-#{project.jobs_cache_index}") + cache = single_cache.merge(key: "#{single_cache[:key]}-#{project.jobs_cache_index}") + fallback = cache.slice(:fallback_keys).transform_values { |keys| keys.map { |key| "#{key}-#{project.jobs_cache_index}" } } + cache.merge(fallback.compact) end end @@ -913,10 +937,16 @@ module Ci cache.map do |entry| type_suffix = !entry[:unprotect] && pipeline.protected_ref? ? 'protected' : 'non_protected' - entry.merge(key: "#{entry[:key]}-#{type_suffix}") + cache = entry.merge(key: "#{entry[:key]}-#{type_suffix}") + fallback = cache.slice(:fallback_keys).transform_values { |keys| keys.map { |key| "#{key}-#{type_suffix}" } } + cache.merge(fallback.compact) end end + def fallback_cache_keys_defined? + Array.wrap(options[:cache]).any? { |cache| cache[:fallback_keys].present? } + end + def credentials Gitlab::Ci::Build::Credentials::Factory.new(self).create! end @@ -1091,10 +1121,6 @@ module Ci ::Ci::PendingBuild.upsert_from_build!(self) end - def create_runtime_metadata! - ::Ci::RunningBuild.upsert_shared_runner_build!(self) - end - ## # We can have only one queuing entry or running build tracking entry, # because there is a unique index on `build_id` in each table, but we need @@ -1161,11 +1187,6 @@ module Ci end end - override :format_token - def format_token(token) - "#{partition_id.to_s(16)}_#{token}" - end - protected def run_status_commit_hooks! @@ -1231,10 +1252,10 @@ module Ci end def job_jwt_variables - if project.ci_cd_settings.opt_in_jwt? + if id_tokens? id_tokens_variables else - predefined_jwt_variables.concat(id_tokens_variables) + predefined_jwt_variables end end @@ -1251,8 +1272,6 @@ module Ci end def id_tokens_variables - return [] unless id_tokens? - Gitlab::Ci::Variables::Collection.new.tap do |variables| id_tokens.each do |var_name, token_data| token = Gitlab::Ci::JwtV2.for_build(self, aud: token_data['aud']) @@ -1308,6 +1327,10 @@ module Ci ).to_context] ) end + + def partition_id_prefix_in_16_bit_encode + "#{partition_id.to_s(16)}_" + end end end diff --git a/app/models/ci/build_metadata.rb b/app/models/ci/build_metadata.rb index b294afd405d..382f861a802 100644 --- a/app/models/ci/build_metadata.rb +++ b/app/models/ci/build_metadata.rb @@ -10,15 +10,16 @@ module Ci include Presentable include ChronicDurationAttribute include Gitlab::Utils::StrongMemoize + include SafelyChangeColumnDefault self.table_name = 'p_ci_builds_metadata' self.primary_key = 'id' + columns_changing_default :partition_id partitionable scope: :build belongs_to :build, class_name: 'CommitStatus' belongs_to :project - belongs_to :runner_machine, class_name: 'Ci::RunnerMachine' before_create :set_build_project diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index 03d1bd14bfb..940221619b3 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -6,8 +6,6 @@ module Ci include BulkInsertSafe include IgnorableColumns - ignore_column :id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-04-22' - belongs_to :build, class_name: "Ci::Processable", foreign_key: :build_id, inverse_of: :needs partitionable scope: :build diff --git a/app/models/ci/build_pending_state.rb b/app/models/ci/build_pending_state.rb index 3684dac06c7..966884ae158 100644 --- a/app/models/ci/build_pending_state.rb +++ b/app/models/ci/build_pending_state.rb @@ -3,7 +3,7 @@ class Ci::BuildPendingState < Ci::ApplicationRecord include Ci::Partitionable - belongs_to :build, class_name: 'Ci::Build', foreign_key: :build_id + belongs_to :build, class_name: 'Ci::Build', foreign_key: :build_id, inverse_of: :pending_state partitionable scope: :build diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 541a8b5bffa..03b59b19ef1 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -9,7 +9,7 @@ module Ci include ::Gitlab::ExclusiveLeaseHelpers include ::Gitlab::OptimisticLocking - belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id + belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :trace_chunks partitionable scope: :build diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb index 00cf1531483..4c76089617f 100644 --- a/app/models/ci/build_trace_metadata.rb +++ b/app/models/ci/build_trace_metadata.rb @@ -42,9 +42,7 @@ module Ci end def track_archival!(trace_artifact_id, checksum) - update!(trace_artifact_id: trace_artifact_id, - checksum: checksum, - archived_at: Time.current) + update!(trace_artifact_id: trace_artifact_id, checksum: checksum, archived_at: Time.current) end def archival_attempts_message diff --git a/app/models/ci/catalog/listing.rb b/app/models/ci/catalog/listing.rb new file mode 100644 index 00000000000..b9e777f27a0 --- /dev/null +++ b/app/models/ci/catalog/listing.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Ci + module Catalog + class Listing + # This class is the SSoT to displaying the list of resources in the + # CI/CD Catalog given a namespace as a scope. + # This model is not directly backed by a table and joins catalog resources + # with projects to return relevant data. + def initialize(namespace, current_user) + raise ArgumentError, 'Namespace is not a root namespace' unless namespace.root? + + @namespace = namespace + @current_user = current_user + end + + def resources + Ci::Catalog::Resource + .joins(:project).includes(:project) + .merge(projects_in_namespace_visible_to_user) + end + + private + + attr_reader :namespace, :current_user + + def projects_in_namespace_visible_to_user + Project + .in_namespace(namespace.self_and_descendant_ids) + .public_or_visible_to_user(current_user, ::Gitlab::Access::DEVELOPER) + end + end + end +end diff --git a/app/models/ci/catalog/resource.rb b/app/models/ci/catalog/resource.rb new file mode 100644 index 00000000000..bb4584aacae --- /dev/null +++ b/app/models/ci/catalog/resource.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Ci + module Catalog + # This class represents a CI/CD Catalog resource. + # A Catalog resource is normally associated to a project. + # This model connects to the `main` database because of its + # dependency on the Project model and its need to join with that table + # in order to generate the CI/CD catalog. + class Resource < ::ApplicationRecord + self.table_name = 'catalog_resources' + + belongs_to :project + + scope :for_projects, ->(project_ids) { where(project_id: project_ids) } + + delegate :avatar_path, :description, :name, to: :project + + def versions + project.releases.order_released_desc + end + + def latest_version + versions.first + end + end + end +end diff --git a/app/models/ci/commit_with_pipeline.rb b/app/models/ci/commit_with_pipeline.rb index dde4b534aaa..2aa249df321 100644 --- a/app/models/ci/commit_with_pipeline.rb +++ b/app/models/ci/commit_with_pipeline.rb @@ -19,7 +19,7 @@ class Ci::CommitWithPipeline < SimpleDelegator end def lazy_latest_pipeline - BatchLoader.for(sha).batch do |shas, loader| + BatchLoader.for(sha).batch(key: project.id) do |shas, loader| preload_pipelines = project.ci_pipelines.latest_pipeline_per_commit(shas.compact) shas.each do |sha| diff --git a/app/models/ci/daily_build_group_report_result.rb b/app/models/ci/daily_build_group_report_result.rb index 598d1456a48..5ec54ee2983 100644 --- a/app/models/ci/daily_build_group_report_result.rb +++ b/app/models/ci/daily_build_group_report_result.rb @@ -4,9 +4,10 @@ module Ci class DailyBuildGroupReportResult < Ci::ApplicationRecord PARAM_TYPES = %w[coverage].freeze - belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id + belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id, + inverse_of: :daily_build_group_report_results belongs_to :project - belongs_to :group + belongs_to :group, class_name: '::Group' validates :data, json_schema: { filename: "daily_build_group_report_result_data" } diff --git a/app/models/ci/group_variable.rb b/app/models/ci/group_variable.rb index b03c46a164f..f04f0d27e51 100644 --- a/app/models/ci/group_variable.rb +++ b/app/models/ci/group_variable.rb @@ -29,5 +29,13 @@ module Ci def audit_details key end + + def group_name + group.name + end + + def group_ci_cd_settings_path + Gitlab::Routing.url_helpers.group_settings_ci_cd_path(group) + end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 89a3d269a43..766155c6a99 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -132,7 +132,7 @@ module Ci PLAN_LIMIT_PREFIX = 'ci_max_artifact_size_' belongs_to :project - belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id, inverse_of: :job_artifacts mount_file_store_uploader JobArtifactUploader, skip_store_file: true @@ -155,7 +155,7 @@ module Ci scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } scope :for_job_ids, ->(job_ids) { where(job_id: job_ids) } - scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } + scope :for_job_name, ->(name) { joins(:job).merge(Ci::Build.by_name(name)) } scope :created_at_before, ->(time) { where(arel_table[:created_at].lteq(time)) } scope :id_before, ->(id) { where(arel_table[:id].lteq(id)) } scope :id_after, ->(id) { where(arel_table[:id].gt(id)) } @@ -177,6 +177,8 @@ module Ci where(file_type: self.erasable_file_types) end + scope :non_trace, -> { where.not(file_type: [:trace]) } + scope :downloadable, -> { where(file_type: DOWNLOADABLE_TYPES) } scope :unlocked, -> { joins(job: :pipeline).merge(::Ci::Pipeline.unlocked) } scope :order_expired_asc, -> { order(expire_at: :asc) } diff --git a/app/models/ci/job_token/scope.rb b/app/models/ci/job_token/scope.rb index 20775077bd8..f389c642fd8 100644 --- a/app/models/ci/job_token/scope.rb +++ b/app/models/ci/job_token/scope.rb @@ -58,8 +58,7 @@ module Ci end def inbound_accessible?(accessed_project) - # if the flag or setting is disabled any project is considered to be in scope. - return true unless Feature.enabled?(:ci_inbound_job_token_scope, accessed_project) + # if the setting is disabled any project is considered to be in scope. return true unless accessed_project.ci_inbound_job_token_scope_enabled? inbound_linked_as_accessible?(accessed_project) diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb index 998f0647ad5..573999995bc 100644 --- a/app/models/ci/job_variable.rb +++ b/app/models/ci/job_variable.rb @@ -7,7 +7,7 @@ module Ci include Ci::RawVariable include BulkInsertSafe - belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id, inverse_of: :job_variables partitionable scope: :job diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb index 5ea51fbe0a7..ff7e681217a 100644 --- a/app/models/ci/namespace_mirror.rb +++ b/app/models/ci/namespace_mirror.rb @@ -41,8 +41,7 @@ module Ci namespace = event.namespace traversal_ids = namespace.self_and_ancestor_ids(hierarchy_order: :desc) - upsert({ namespace_id: event.namespace_id, traversal_ids: traversal_ids }, - unique_by: :namespace_id) + upsert({ namespace_id: event.namespace_id, traversal_ids: traversal_ids }, unique_by: :namespace_id) end end end diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index 2b1eb67d4f2..14050a1e78e 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -14,7 +14,6 @@ module Ci validates :namespace, presence: true scope :ref_protected, -> { where(protected: true) } - scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) } scope :with_instance_runners, -> { where(instance_runners_enabled: true) } scope :for_tags, ->(tag_ids) do if tag_ids.present? diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index bd426e02b9c..babea831d85 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -11,7 +11,6 @@ module Ci include Gitlab::OptimisticLocking include Gitlab::Utils::StrongMemoize include AtomicInternalId - include EnumWithNil include Ci::HasRef include ShaAttribute include FromUnion @@ -19,6 +18,9 @@ module Ci include EachBatch include FastDestroyAll::Helpers + include IgnorableColumns + ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' + MAX_OPEN_MERGE_REQUESTS_REFS = 4 PROJECT_ROUTE_AND_NAMESPACE_ROUTE = { @@ -46,39 +48,53 @@ module Ci belongs_to :project, inverse_of: :all_pipelines belongs_to :user - belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' + belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline', inverse_of: :auto_canceled_pipelines belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule' belongs_to :merge_request, class_name: 'MergeRequest' belongs_to :external_pull_request belongs_to :ci_ref, class_name: 'Ci::Ref', foreign_key: :ci_ref_id, inverse_of: :pipelines has_internal_id :iid, scope: :project, presence: false, - track_if: -> { !importing? }, - ensure_if: -> { !importing? }, - init: ->(pipeline, scope) do - if pipeline - pipeline.project&.all_pipelines&.maximum(:iid) || pipeline.project&.all_pipelines&.count - elsif scope - ::Ci::Pipeline.where(**scope).maximum(:iid) - end - end + track_if: -> { !importing? }, + ensure_if: -> { !importing? }, + init: ->(pipeline, scope) do + if pipeline + pipeline.project&.all_pipelines&.maximum(:iid) || pipeline.project&.all_pipelines&.count + elsif scope + ::Ci::Pipeline.where(**scope).maximum(:iid) + end + end has_many :stages, -> { order(position: :asc) }, inverse_of: :pipeline + + # + # In https://gitlab.com/groups/gitlab-org/-/epics/9991, we aim to convert all CommitStatus related models to + # Ci:Job models. With that epic, we aim to replace `statuses` with `jobs`. + # + # DEPRECATED: has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline + has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline has_many :latest_statuses_ordered_by_stage, -> { latest.order(:stage_idx, :stage) }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :latest_statuses, -> { latest }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline - has_many :statuses_order_id_desc, -> { order_id_desc }, class_name: 'CommitStatus', foreign_key: :commit_id - has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline + has_many :statuses_order_id_desc, -> { order_id_desc }, class_name: 'CommitStatus', foreign_key: :commit_id, + inverse_of: :pipeline has_many :bridges, class_name: 'Ci::Bridge', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline has_many :generic_commit_statuses, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'GenericCommitStatus' + # + # NEW: + has_many :all_jobs, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline + has_many :current_jobs, -> { latest }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline + has_many :all_processable_jobs, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline + has_many :current_processable_jobs, -> { latest }, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline + has_many :job_artifacts, through: :builds has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks - has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent + has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' has_many :latest_builds, -> { latest.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' has_many :downloadable_artifacts, -> do - not_expired.or(where_exists(::Ci::Pipeline.artifacts_locked.where('ci_pipelines.id = ci_builds.commit_id'))).downloadable.with_job + not_expired.or(where_exists(Ci::Pipeline.artifacts_locked.where("#{Ci::Pipeline.quoted_table_name}.id = #{Ci::Build.quoted_table_name}.commit_id"))).downloadable.with_job end, through: :latest_builds, source: :job_artifacts has_many :latest_successful_builds, -> { latest.success.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' @@ -86,17 +102,24 @@ module Ci # Merge requests for which the current pipeline is running against # the merge request's latest commit. - has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest' + has_many :merge_requests_as_head_pipeline, foreign_key: :head_pipeline_id, class_name: 'MergeRequest', + inverse_of: :head_pipeline + has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline - has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline + has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', + inverse_of: :pipeline has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline - has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus' + has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus', + inverse_of: :pipeline has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline - has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id' - has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' - has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_pipeline_id + has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: :auto_canceled_by_id, + inverse_of: :auto_canceled_by + has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: :auto_canceled_by_id, + inverse_of: :auto_canceled_by + has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_pipeline_id, + inverse_of: :source_pipeline has_one :source_pipeline, class_name: 'Ci::Sources::Pipeline', inverse_of: :pipeline @@ -114,7 +137,9 @@ module Ci has_one :pipeline_metadata, class_name: 'Ci::PipelineMetadata', inverse_of: :pipeline - has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', foreign_key: :last_pipeline_id + has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult', + foreign_key: :last_pipeline_id, inverse_of: :last_pipeline + has_many :latest_builds_report_results, through: :latest_builds, source: :report_results has_many :pipeline_artifacts, class_name: 'Ci::PipelineArtifact', inverse_of: :pipeline, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -143,9 +168,9 @@ module Ci # We use `Enums::Ci::Pipeline.sources` here so that EE can more easily extend # this `Hash` with new values. - enum_with_nil source: Enums::Ci::Pipeline.sources + enum source: Enums::Ci::Pipeline.sources - enum_with_nil config_source: Enums::Ci::Pipeline.config_sources + enum config_source: Enums::Ci::Pipeline.config_sources # We use `Enums::Ci::Pipeline.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. @@ -336,6 +361,22 @@ module Ci AutoDevops::DisableWorker.perform_async(pipeline.id) if pipeline.auto_devops_source? end end + + after_transition any => [:running, *::Ci::Pipeline.completed_statuses] do |pipeline| + project = pipeline&.project + + next unless project + next unless Feature.enabled?(:pipeline_trigger_merge_status, project) + + pipeline.run_after_commit do + next if pipeline.child? + next unless project.only_allow_merge_if_pipeline_succeeds?(inherit_group_setting: true) + + pipeline.all_merge_requests.opened.each do |merge_request| + GraphqlTriggers.merge_request_merge_status_updated(merge_request) + end + end + end end scope :internal, -> { where(source: internal_sources) } @@ -361,18 +402,25 @@ module Ci scope :created_before_id, -> (id) { where(arel_table[:id].lt(id)) } scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } scope :with_pipeline_source, -> (source) { where(source: source) } + scope :preload_pipeline_metadata, -> { preload(:pipeline_metadata) } scope :outside_pipeline_family, ->(pipeline) do where.not(id: pipeline.same_family_pipeline_ids) end scope :with_reports, -> (reports_scope) do - where('EXISTS (?)', ::Ci::Build.latest.with_artifacts(reports_scope).where('ci_pipelines.id=ci_builds.commit_id').select(1)) + where('EXISTS (?)', + ::Ci::Build + .latest + .with_artifacts(reports_scope) + .where("#{quoted_table_name}.id = #{Ci::Build.quoted_table_name}.commit_id") + .select(1) + ) end scope :with_only_interruptible_builds, -> do where('NOT EXISTS (?)', - Ci::Build.where('ci_builds.commit_id = ci_pipelines.id') + Ci::Build.where("#{Ci::Build.quoted_table_name}.commit_id = #{quoted_table_name}.id") .with_status(STARTED_STATUSES) .not_interruptible ) @@ -382,11 +430,15 @@ module Ci # In general, please use `Ci::PipelinesForMergeRequestFinder` instead, # for checking permission of the actor. scope :triggered_by_merge_request, -> (merge_request) do - where(source: :merge_request_event, - merge_request: merge_request, - project: [merge_request.source_project, merge_request.target_project]) + where( + source: :merge_request_event, + merge_request: merge_request, + project: [merge_request.source_project, merge_request.target_project] + ) end + scope :order_id_desc, -> { order(id: :desc) } + # Returns the pipelines in descending order (= newest first), optionally # limited to a number of references. # @@ -657,7 +709,7 @@ module Ci # rubocop: enable CodeReuse/ServiceClass def lazy_ref_commit - BatchLoader.for(ref).batch do |refs, loader| + BatchLoader.for(ref).batch(key: project.id) do |refs, loader| next unless project.repository_exists? project.repository.list_commits_by_ref_name(refs).then do |commits| @@ -818,8 +870,7 @@ module Ci when 'manual' then block when 'scheduled' then delay else - raise Ci::HasStatus::UnknownStatusError, - "Unknown status `#{new_status}`" + raise Ci::HasStatus::UnknownStatusError, "Unknown status `#{new_status}`" end end end @@ -1282,7 +1333,7 @@ module Ci types_to_collect = report_types.empty? ? ::Ci::JobArtifact::SECURITY_REPORT_FILE_TYPES : report_types ::Gitlab::Ci::Reports::Security::Reports.new(self).tap do |security_reports| - latest_report_builds(reports_scope).each do |build| + latest_report_builds_in_self_and_project_descendants(reports_scope).includes(pipeline: { project: :route }).each do |build| # rubocop:disable Rails/FindEach build.collect_security_reports!(security_reports, report_types: types_to_collect) end end @@ -1294,7 +1345,7 @@ module Ci def cluster_agent_authorizations strong_memoize(:cluster_agent_authorizations) do - ::Clusters::AgentAuthorizationsFinder.new(project).execute + ::Clusters::Agents::Authorizations::CiAccess::Finder.new(project).execute end end diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 20ff07e88ba..49d27053745 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -8,14 +8,15 @@ module Ci include CronSchedulable include Limitable include EachBatch + include BatchNullifyDependentAssociations self.limit_name = 'ci_pipeline_schedules' self.limit_scope = :project belongs_to :project belongs_to :owner, class_name: 'User' - has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline' - has_many :pipelines + has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline', inverse_of: :pipeline_schedule + has_many :pipelines, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineScheduleVariable' validates :cron, unless: :importing?, cron: true, presence: { unless: :importing? } @@ -81,6 +82,14 @@ module Ci def worker_cron_expression Settings.cron_jobs['pipeline_schedule_worker']['cron'] end + + # Using destroy instead of before_destroy as we want nullify_dependent_associations_in_batches + # to run first and not in a transaction block. This prevents timeouts for schedules with numerous pipelines + def destroy + nullify_dependent_associations_in_batches + + super + end end end diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index 8e83b41cd0b..f2457af0074 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -6,6 +6,9 @@ module Ci include Ci::HasVariable include Ci::RawVariable + include IgnorableColumns + ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' + belongs_to :pipeline partitionable scope: :pipeline diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 37c82c125aa..4c421f066f9 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true module Ci + # This class is a collection of common features between Ci::Build and Ci::Bridge. + # In https://gitlab.com/groups/gitlab-org/-/epics/9991, we aim to clarify class naming conventions. class Processable < ::CommitStatus include Gitlab::Utils::StrongMemoize include FromUnion diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb index 15a161d5b7c..23cd5d92730 100644 --- a/app/models/ci/project_mirror.rb +++ b/app/models/ci/project_mirror.rb @@ -13,8 +13,7 @@ module Ci class << self def sync!(event) - upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id }, - unique_by: :project_id) + upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id }, unique_by: :project_id) end end end diff --git a/app/models/ci/ref.rb b/app/models/ci/ref.rb index af5fdabff6e..199e1cd07e7 100644 --- a/app/models/ci/ref.rb +++ b/app/models/ci/ref.rb @@ -43,8 +43,7 @@ module Ci class << self def ensure_for(pipeline) - safe_find_or_create_by(project_id: pipeline.project_id, - ref_path: pipeline.source_ref_path) + safe_find_or_create_by(project_id: pipeline.project_id, ref_path: pipeline.source_ref_path) end def failing_state?(status_name) diff --git a/app/models/ci/resource_group.rb b/app/models/ci/resource_group.rb index b788e4f58c1..48f321a236d 100644 --- a/app/models/ci/resource_group.rb +++ b/app/models/ci/resource_group.rb @@ -29,13 +29,19 @@ module Ci partition_id: processable.partition_id } - resources.free.limit(1).update_all(attrs) > 0 + success = resources.free.limit(1).update_all(attrs) > 0 + log_event(success: success, processable: processable, action: "assign resource to processable") + + success end def release_resource_from(processable) attrs = { build_id: nil, partition_id: nil } - resources.retained_by(processable).update_all(attrs) > 0 + success = resources.retained_by(processable).update_all(attrs) > 0 + log_event(success: success, processable: processable, action: "release resource from processable") + + success end def upcoming_processables @@ -52,6 +58,10 @@ module Ci end end + def current_processable + Ci::Processable.find_by('(id, partition_id) IN (?)', resources.select('build_id, partition_id')) + end + private # In order to avoid deadlock, we do NOT specify the job execution order in the same pipeline. @@ -72,5 +82,14 @@ module Ci # belong to the same resource group are executed once at time. self.resources.build if self.resources.empty? end + + def log_event(success:, processable:, action:) + Gitlab::Ci::ResourceGroups::Logger.build.info({ + resource_group_id: self.id, + processable_id: processable.id, + message: "attempted to #{action}", + success: success + }) + end end end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 09ac0fa69e7..7727e94875b 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -17,7 +17,10 @@ module Ci extend ::Gitlab::Utils::Override - add_authentication_token_field :token, encrypted: :optional, expires_at: :compute_token_expiration + add_authentication_token_field :token, + encrypted: :optional, + expires_at: :compute_token_expiration, + format_with_prefix: :prefix_for_new_and_legacy_runner enum access_level: { not_protected: 0, @@ -54,6 +57,9 @@ module Ci # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner will be considered stale STALE_TIMEOUT = 3.months + # Only allow authentication token to be visible for a short while + REGISTRATION_AVAILABILITY_TIME = 1.hour + AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze AVAILABLE_TYPES = runner_types.keys.freeze AVAILABLE_STATUSES = %w[active paused online offline never_contacted stale].freeze # TODO: Remove in %16.0: active, paused. Relevant issue: https://gitlab.com/gitlab-org/gitlab/-/issues/344648 @@ -64,7 +70,7 @@ module Ci TAG_LIST_MAX_LENGTH = 50 - has_many :runner_machines, inverse_of: :runner + has_many :runner_managers, inverse_of: :runner has_many :builds has_many :runner_projects, inverse_of: :runner, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :projects, through: :runner_projects, disable_joins: true @@ -81,8 +87,13 @@ module Ci scope :active, -> (value = true) { where(active: value) } scope :paused, -> { active(false) } scope :online, -> { where('contacted_at > ?', online_contact_time_deadline) } - scope :recent, -> { where('ci_runners.created_at >= :date OR ci_runners.contacted_at >= :date', date: stale_deadline) } - scope :stale, -> { where('ci_runners.created_at < :date AND (ci_runners.contacted_at IS NULL OR ci_runners.contacted_at < :date)', date: stale_deadline) } + scope :recent, -> do + where('ci_runners.created_at >= :datetime OR ci_runners.contacted_at >= :datetime', datetime: stale_deadline) + end + scope :stale, -> do + where('ci_runners.created_at <= :datetime AND ' \ + '(ci_runners.contacted_at IS NULL OR ci_runners.contacted_at <= :datetime)', datetime: stale_deadline) + end scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) } scope :never_contacted, -> { where(contacted_at: nil) } scope :ordered, -> { order(id: :desc) } @@ -123,7 +134,7 @@ module Ci belonging_to_group(group_self_and_ancestors_ids) } - scope :belonging_to_parent_group_of_project, -> (project_id) { + scope :belonging_to_parent_groups_of_project, -> (project_id) { raise ArgumentError, "only 1 project_id allowed for performance reasons" unless project_id.is_a?(Integer) project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) @@ -137,7 +148,7 @@ module Ci from_union( [ belonging_to_project(project_id), - project.group_runners_enabled? ? belonging_to_parent_group_of_project(project_id) : nil, + project.group_runners_enabled? ? belonging_to_parent_groups_of_project(project_id) : nil, project.shared_runners ].compact, remove_duplicates: false @@ -185,6 +196,7 @@ module Ci scope :order_token_expires_at_asc, -> { order(token_expires_at: :asc) } scope :order_token_expires_at_desc, -> { order(token_expires_at: :desc) } scope :with_tags, -> { preload(:tags) } + scope :with_creator, -> { preload(:creator) } validate :tag_constraints validates :access_level, presence: true @@ -203,16 +215,14 @@ module Ci cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at, :executor_type chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout, - error_message: 'Maximum job timeout has a value which could not be accepted' + error_message: 'Maximum job timeout has a value which could not be accepted' validates :maximum_timeout, allow_nil: true, - numericality: { greater_than_or_equal_to: 600, - message: 'needs to be at least 10 minutes' } + numericality: { greater_than_or_equal_to: 600, message: 'needs to be at least 10 minutes' } validates :public_projects_minutes_cost_factor, :private_projects_minutes_cost_factor, allow_nil: false, - numericality: { greater_than_or_equal_to: 0.0, - message: 'needs to be non-negative' } + numericality: { greater_than_or_equal_to: 0.0, message: 'needs to be non-negative' } validates :config, json_schema: { filename: 'ci_runner_config' } @@ -332,15 +342,10 @@ module Ci def stale? return false unless created_at - [created_at, contacted_at].compact.max < self.class.stale_deadline + [created_at, contacted_at].compact.max <= self.class.stale_deadline end - def status(legacy_mode = nil) - # TODO Deprecate legacy_mode in %16.0 and make it a no-op - # (see https://gitlab.com/gitlab-org/gitlab/-/issues/360545) - # TODO Remove legacy_mode in %17.0 - return deprecated_rest_status if legacy_mode == '14.5' - + def status return :stale if stale? return :never_contacted unless contacted_at @@ -434,7 +439,7 @@ module Ci ensure_runner_queue_value == value if value.present? end - def heartbeat(values) + def heartbeat(values, update_contacted_at: true) ## # We can safely ignore writes performed by a runner heartbeat. We do # not want to upgrade database connection proxy to use the primary @@ -442,20 +447,18 @@ module Ci # ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} - values[:contacted_at] = Time.current + values[:contacted_at] = Time.current if update_contacted_at if values.include?(:executor) values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) end - cache_attributes(values) + new_version = values[:version] + schedule_runner_version_update(new_version) if new_version && values[:version] != version - # We save data without validation, it will always change due to `contacted_at` - if persist_cached_data? - version_updated = values.include?(:version) && values[:version] != version + merge_cache_attributes(values) - update_columns(values) - schedule_runner_version_update if version_updated - end + # We save data without validation, it will always change due to `contacted_at` + update_columns(values) if persist_cached_data? end end @@ -488,15 +491,18 @@ module Ci end end - override :format_token - def format_token(token) - return token if registration_token_registration_type? + def ensure_manager(system_xid, &blk) + RunnerManager.safe_find_or_create_by!(runner_id: id, system_xid: system_xid.to_s, &blk) # rubocop: disable Performance/ActiveRecordSubtransactionMethods + end - "#{CREATED_RUNNER_TOKEN_PREFIX}#{token}" + def registration_available? + authenticated_user_registration_type? && + created_at > REGISTRATION_AVAILABILITY_TIME.ago && + !runner_managers.any? end - def ensure_machine(system_xid, &blk) - RunnerMachine.safe_find_or_create_by!(runner_id: id, system_xid: system_xid.to_s, &blk) # rubocop: disable Performance/ActiveRecordSubtransactionMethods + def gitlab_hosted? + Gitlab.com? && instance_type? end private @@ -586,7 +592,7 @@ module Ci end def exactly_one_group - unless runner_namespaces.one? + unless runner_namespaces.size == 1 errors.add(:runner, 'needs to be assigned to exactly one group') end end @@ -594,10 +600,16 @@ module Ci # TODO Remove in 16.0 when runners are known to send a system_id # For now, heartbeats with version updates might result in two Sidekiq jobs being queued if a runner has a system_id # This is not a problem since the jobs are deduplicated on the version - def schedule_runner_version_update - return unless version + def schedule_runner_version_update(new_version) + return unless new_version && Gitlab::Ci::RunnerReleases.instance.enabled? + + Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(new_version) + end + + def prefix_for_new_and_legacy_runner + return if registration_token_registration_type? - Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(version) + CREATED_RUNNER_TOKEN_PREFIX end end end diff --git a/app/models/ci/runner_machine.rb b/app/models/ci/runner_manager.rb index e52659a011f..e36024d9f5b 100644 --- a/app/models/ci/runner_machine.rb +++ b/app/models/ci/runner_manager.rb @@ -1,23 +1,23 @@ # frozen_string_literal: true module Ci - class RunnerMachine < Ci::ApplicationRecord + class RunnerManager < Ci::ApplicationRecord include FromUnion include RedisCacheable include Ci::HasRunnerExecutor - include IgnorableColumns - ignore_column :machine_xid, remove_with: '15.11', remove_after: '2022-03-22' + # For legacy reasons, the table name is ci_runner_machines in the database + self.table_name = 'ci_runner_machines' # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner Machine DB entry can be updated - UPDATE_CONTACT_COLUMN_EVERY = 40.minutes..55.minutes + UPDATE_CONTACT_COLUMN_EVERY = (40.minutes)..(55.minutes) belongs_to :runner - has_many :build_metadata, class_name: 'Ci::BuildMetadata' - has_many :builds, through: :build_metadata, class_name: 'Ci::Build' - belongs_to :runner_version, inverse_of: :runner_machines, primary_key: :version, foreign_key: :version, - class_name: 'Ci::RunnerVersion' + has_many :runner_manager_builds, inverse_of: :runner_manager, class_name: 'Ci::RunnerManagerBuild' + has_many :builds, through: :runner_manager_builds, class_name: 'Ci::Build' + belongs_to :runner_version, inverse_of: :runner_managers, primary_key: :version, foreign_key: :version, + class_name: 'Ci::RunnerVersion' validates :runner, presence: true validates :system_xid, presence: true, length: { maximum: 64 } @@ -30,7 +30,7 @@ module Ci cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at, :executor_type - # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner machine + # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner manager # will be considered stale STALE_TIMEOUT = 7.days @@ -44,7 +44,15 @@ module Ci remove_duplicates: false).where(created_some_time_ago) end - def heartbeat(values) + def self.online_contact_time_deadline + Ci::Runner.online_contact_time_deadline + end + + def self.stale_deadline + STALE_TIMEOUT.ago + end + + def heartbeat(values, update_contacted_at: true) ## # We can safely ignore writes performed by a runner heartbeat. We do # not want to upgrade database connection proxy to use the primary @@ -52,24 +60,40 @@ module Ci # ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} - values[:contacted_at] = Time.current + values[:contacted_at] = Time.current if update_contacted_at if values.include?(:executor) values[:executor_type] = Ci::Runner::EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) end - version_changed = values.include?(:version) && values[:version] != version + new_version = values[:version] + schedule_runner_version_update(new_version) if new_version && values[:version] != version - cache_attributes(values) - - schedule_runner_version_update if version_changed + merge_cache_attributes(values) # We save data without validation, it will always change due to `contacted_at` update_columns(values) if persist_cached_data? end end + def status + return :stale if stale? + return :never_contacted unless contacted_at + + online? ? :online : :offline + end + private + def online? + contacted_at && contacted_at > self.class.online_contact_time_deadline + end + + def stale? + return false unless created_at + + [created_at, contacted_at].compact.max <= self.class.stale_deadline + end + def persist_cached_data? # Use a random threshold to prevent beating DB updates. contacted_at_max_age = Random.rand(UPDATE_CONTACT_COLUMN_EVERY) @@ -79,10 +103,10 @@ module Ci (Time.current - real_contacted_at) >= contacted_at_max_age end - def schedule_runner_version_update - return unless version + def schedule_runner_version_update(new_version) + return unless new_version && Gitlab::Ci::RunnerReleases.instance.enabled? - Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(version) + Ci::Runners::ProcessRunnerVersionUpdateWorker.perform_async(new_version) end end end diff --git a/app/models/ci/runner_manager_build.rb b/app/models/ci/runner_manager_build.rb new file mode 100644 index 00000000000..322c5ae3a68 --- /dev/null +++ b/app/models/ci/runner_manager_build.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Ci + class RunnerManagerBuild < Ci::ApplicationRecord + include Ci::Partitionable + + self.table_name = :p_ci_runner_machine_builds + self.primary_key = :build_id + + partitionable scope: :build, partitioned: true + + alias_attribute :runner_manager_id, :runner_machine_id + + belongs_to :build, inverse_of: :runner_manager_build, class_name: 'Ci::Build' + belongs_to :runner_manager, foreign_key: :runner_machine_id, inverse_of: :runner_manager_builds, + class_name: 'Ci::RunnerManager' + + validates :build, presence: true + validates :runner_manager, presence: true + + scope :for_build, ->(build_id) { where(build_id: build_id) } + + def self.pluck_build_id_and_runner_manager_id + select(:build_id, :runner_manager_id) + .pluck(:build_id, :runner_manager_id) + .to_h + end + end +end diff --git a/app/models/ci/runner_version.rb b/app/models/ci/runner_version.rb index ec42f46b165..03b50f13989 100644 --- a/app/models/ci/runner_version.rb +++ b/app/models/ci/runner_version.rb @@ -3,9 +3,8 @@ module Ci class RunnerVersion < Ci::ApplicationRecord include EachBatch - include EnumWithNil - enum_with_nil status: { + enum status: { not_processed: nil, invalid_version: -1, unavailable: 1, @@ -20,7 +19,7 @@ module Ci recommended: 'Upgrade is available and recommended for the runner.' }.freeze - has_many :runner_machines, inverse_of: :runner_version, foreign_key: :version, class_name: 'Ci::RunnerMachine' + has_many :runner_managers, inverse_of: :runner_version, foreign_key: :version, class_name: 'Ci::RunnerManager' # This scope returns all versions that might need recalculating. For instance, once a version is considered # :recommended, it normally doesn't change status even if the instance is upgraded diff --git a/app/models/ci/running_build.rb b/app/models/ci/running_build.rb index 43214b0c336..e6f80658f5d 100644 --- a/app/models/ci/running_build.rb +++ b/app/models/ci/running_build.rb @@ -24,10 +24,12 @@ module Ci raise ArgumentError, 'build has not been picked by a shared runner' end - entry = self.new(build: build, - project: build.project, - runner: build.runner, - runner_type: build.runner.runner_type) + entry = self.new( + build: build, + project: build.project, + runner: build.runner, + runner_type: build.runner.runner_type + ) entry.validate! diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index 855e68d1db1..719d19f4169 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -10,6 +10,7 @@ module Ci belongs_to :project, class_name: "::Project" belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :source_pipeline + belongs_to :build, class_name: "Ci::Build", foreign_key: :source_job_id, inverse_of: :sourced_pipelines belongs_to :source_project, class_name: "::Project", foreign_key: :source_project_id belongs_to :source_job, class_name: "CommitStatus", foreign_key: :source_job_id diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 46a9e3f6494..d61760bd0fc 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -27,6 +27,7 @@ module Ci has_many :processables, class_name: 'Ci::Processable', foreign_key: :stage_id, inverse_of: :ci_stage has_many :builds, foreign_key: :stage_id, inverse_of: :ci_stage has_many :bridges, foreign_key: :stage_id, inverse_of: :ci_stage + has_many :generic_commit_statuses, foreign_key: :stage_id, inverse_of: :ci_stage scope :ordered, -> { order(position: :asc) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } @@ -111,12 +112,12 @@ module Ci when 'scheduled' then delay when 'skipped', nil then skip else - raise Ci::HasStatus::UnknownStatusError, - "Unknown status `#{new_status}`" + raise Ci::HasStatus::UnknownStatusError, "Unknown status `#{new_status}`" end end end + # This will be removed with ci_remove_ensure_stage_service def update_legacy_status set_status(latest_stage_status.to_s) end @@ -150,6 +151,7 @@ module Ci blocked? || skipped? end + # This will be removed with ci_remove_ensure_stage_service def latest_stage_status statuses.latest.composite_status || 'skipped' end diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index 1b2a7dc3fe4..58da1b4bd7e 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -8,7 +8,7 @@ module Ci TRIGGER_TOKEN_PREFIX = 'glptt-' - ignore_column :ref, remove_with: '15.4', remove_after: '2022-08-22' + ignore_column :ref, remove_with: '16.1', remove_after: '2023-05-22' self.limit_name = 'pipeline_triggers' self.limit_scope = :project @@ -26,8 +26,7 @@ module Ci mode: :per_attribute_iv, algorithm: 'aes-256-gcm', key: Settings.attr_encrypted_db_key_base_32, - encode: false, - encode_vi: false + encode: false before_validation :set_default_values diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb index 3478bb69707..6980ec1c2d3 100644 --- a/app/models/clusters/agent.rb +++ b/app/models/clusters/agent.rb @@ -2,6 +2,8 @@ module Clusters class Agent < ApplicationRecord + include FromUnion + self.table_name = 'cluster_agents' INACTIVE_AFTER = 1.hour.freeze @@ -11,12 +13,19 @@ module Clusters belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project has_many :agent_tokens, -> { order_last_used_at_desc }, class_name: 'Clusters::AgentToken', inverse_of: :agent + has_many :active_agent_tokens, -> { active.order_last_used_at_desc }, class_name: 'Clusters::AgentToken', inverse_of: :agent + + has_many :ci_access_group_authorizations, class_name: 'Clusters::Agents::Authorizations::CiAccess::GroupAuthorization' + has_many :ci_access_authorized_groups, class_name: '::Group', through: :ci_access_group_authorizations, source: :group - has_many :group_authorizations, class_name: 'Clusters::Agents::GroupAuthorization' - has_many :authorized_groups, class_name: '::Group', through: :group_authorizations, source: :group + has_many :ci_access_project_authorizations, class_name: 'Clusters::Agents::Authorizations::CiAccess::ProjectAuthorization' + has_many :ci_access_authorized_projects, class_name: '::Project', through: :ci_access_project_authorizations, source: :project - has_many :project_authorizations, class_name: 'Clusters::Agents::ProjectAuthorization' - has_many :authorized_projects, class_name: '::Project', through: :project_authorizations, source: :project + has_many :user_access_group_authorizations, class_name: 'Clusters::Agents::Authorizations::UserAccess::GroupAuthorization' + has_many :user_access_authorized_groups, class_name: '::Group', through: :user_access_group_authorizations, source: :group + + has_many :user_access_project_authorizations, class_name: 'Clusters::Agents::Authorizations::UserAccess::ProjectAuthorization' + has_many :user_access_authorized_projects, class_name: '::Project', through: :user_access_project_authorizations, source: :project has_many :activity_events, -> { in_timeline_order }, class_name: 'Clusters::Agents::ActivityEvent', inverse_of: :agent @@ -51,6 +60,80 @@ module Clusters def to_ability_name :cluster end + + def ci_access_authorized_for?(user) + return false unless user + return false unless ::Feature.enabled?(:expose_authorized_cluster_agents, project) + + ::Project.from_union( + all_ci_access_authorized_projects_for(user).limit(1), + all_ci_access_authorized_namespaces_for(user).limit(1) + ).exists? + end + + def user_access_authorized_for?(user) + return false unless user + return false unless ::Feature.enabled?(:expose_authorized_cluster_agents, project) + + Clusters::Agents::Authorizations::UserAccess::Finder + .new(user, agent: self, preload: false, limit: 1).execute.any? + end + + # As of today, all config values of associated authorization rows have the same value. + # See `UserAccess::RefreshService` for more information. + def user_access_config + self.class.from_union( + user_access_project_authorizations.select('config').limit(1), + user_access_group_authorizations.select('config').limit(1) + ).compact.first&.config + end + + private + + def all_ci_access_authorized_projects_for(user) + ::Project.joins(:ci_access_project_authorizations) + .joins(:project_authorizations) + .where(agent_project_authorizations: { agent_id: id }) + .where(project_authorizations: { user_id: user.id, access_level: Gitlab::Access::DEVELOPER.. }) + end + + def all_ci_access_authorized_namespaces_for(user) + ::Project.with(root_namespace_cte.to_arel) + .with(all_ci_access_authorized_namespaces_cte.to_arel) + .joins('INNER JOIN all_authorized_namespaces ON all_authorized_namespaces.id = projects.namespace_id') + .joins(:project_authorizations) + .where(project_authorizations: { user_id: user.id, access_level: Gitlab::Access::DEVELOPER.. }) + end + + def root_namespace_cte + Gitlab::SQL::CTE.new(:root_namespace, root_namespace.to_sql) + end + + def all_ci_access_authorized_namespaces_cte + Gitlab::SQL::CTE.new(:all_authorized_namespaces, all_ci_access_authorized_namespaces.to_sql) + end + + def all_ci_access_authorized_namespaces + Namespace.select("traversal_ids[array_length(traversal_ids, 1)] AS id") + .joins("INNER JOIN root_namespace ON " \ + "namespaces.traversal_ids @> ARRAY[root_namespace.root_id]") + .joins("INNER JOIN agent_group_authorizations ON " \ + "namespaces.traversal_ids @> ARRAY[agent_group_authorizations.group_id::integer]") + .where(agent_group_authorizations: { agent_id: id }) + end + + def root_namespace + Namespace.select("traversal_ids[1] AS root_id") + .where("traversal_ids @> ARRAY(?)", project_namespace) + .limit(1) + end + + def project_namespace + ::Project.select('namespace_id') + .joins(:cluster_agents) + .where(cluster_agents: { id: id }) + .limit(1) + end end end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index e2dcff13a69..b2b13f6cef7 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -20,6 +20,7 @@ module Clusters scope :order_last_used_at_desc, -> { order(arel_table[:last_used_at].desc.nulls_last) } scope :with_status, -> (status) { where(status: status) } + scope :active, -> { where(status: :active) } enum status: { active: 0, diff --git a/app/models/clusters/agents/authorizations/ci_access/group_authorization.rb b/app/models/clusters/agents/authorizations/ci_access/group_authorization.rb new file mode 100644 index 00000000000..4261fd6570f --- /dev/null +++ b/app/models/clusters/agents/authorizations/ci_access/group_authorization.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Clusters + module Agents + module Authorizations + module CiAccess + class GroupAuthorization < ApplicationRecord + include ConfigScopes + + self.table_name = 'agent_group_authorizations' + + belongs_to :agent, class_name: 'Clusters::Agent', optional: false + belongs_to :group, class_name: '::Group', optional: false + + validates :config, json_schema: { filename: 'clusters_agents_authorizations_ci_access_config' } + + def config_project + agent.project + end + end + end + end + end +end diff --git a/app/models/clusters/agents/authorizations/ci_access/implicit_authorization.rb b/app/models/clusters/agents/authorizations/ci_access/implicit_authorization.rb new file mode 100644 index 00000000000..b996ae3f92b --- /dev/null +++ b/app/models/clusters/agents/authorizations/ci_access/implicit_authorization.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Clusters + module Agents + module Authorizations + module CiAccess + class ImplicitAuthorization + attr_reader :agent + + delegate :id, to: :agent, prefix: true + + def initialize(agent:) + @agent = agent + end + + def config_project + agent.project + end + + def config + {} + end + end + end + end + end +end diff --git a/app/models/clusters/agents/authorizations/ci_access/project_authorization.rb b/app/models/clusters/agents/authorizations/ci_access/project_authorization.rb new file mode 100644 index 00000000000..7742d109cdb --- /dev/null +++ b/app/models/clusters/agents/authorizations/ci_access/project_authorization.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Clusters + module Agents + module Authorizations + module CiAccess + class ProjectAuthorization < ApplicationRecord + include ConfigScopes + + self.table_name = 'agent_project_authorizations' + + belongs_to :agent, class_name: 'Clusters::Agent', optional: false + belongs_to :project, class_name: '::Project', optional: false + + validates :config, json_schema: { filename: 'clusters_agents_authorizations_ci_access_config' } + + def config_project + agent.project + end + end + end + end + end +end diff --git a/app/models/clusters/agents/authorizations/user_access/group_authorization.rb b/app/models/clusters/agents/authorizations/user_access/group_authorization.rb new file mode 100644 index 00000000000..7027870855a --- /dev/null +++ b/app/models/clusters/agents/authorizations/user_access/group_authorization.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Clusters + module Agents + module Authorizations + module UserAccess + class GroupAuthorization < ApplicationRecord + include Scopes + + self.table_name = 'agent_user_access_group_authorizations' + + belongs_to :agent, class_name: 'Clusters::Agent', optional: false + belongs_to :group, class_name: '::Group', optional: false + + scope :for_user, ->(user) { + with(groups_with_direct_membership_cte(user).to_arel) + .with(all_groups_with_membership_cte.to_arel) + .joins('INNER JOIN all_groups_with_membership ON ' \ + 'all_groups_with_membership.id = agent_user_access_group_authorizations.group_id') + .select('DISTINCT ON (id) agent_user_access_group_authorizations.*, ' \ + 'all_groups_with_membership.access_level AS access_level') + .order('id, access_level DESC') + } + + scope :for_project, ->(project) { + where('all_groups_with_membership.traversal_ids @> ARRAY[?]', project.namespace_id) + } + + validates :config, json_schema: { filename: 'clusters_agents_authorizations_user_access_config' } + + def config_project + agent.project + end + + class << self + def upsert_configs(configs) + upsert_all(configs, unique_by: [:agent_id, :group_id]) + end + + def delete_unlisted(group_ids) + where.not(group_id: group_ids).delete_all + end + + def all_groups_with_membership_cte + Gitlab::SQL::CTE.new(:all_groups_with_membership, all_groups_with_membership.to_sql) + end + + def all_groups_with_membership + ::Group.joins('INNER JOIN groups_with_direct_membership ON ' \ + 'namespaces.traversal_ids @> ARRAY[groups_with_direct_membership.id]') + .select('namespaces.id AS id, ' \ + 'namespaces.traversal_ids AS traversal_ids, ' \ + 'groups_with_direct_membership.access_level AS access_level') + end + + def groups_with_direct_membership_cte(user) + Gitlab::SQL::CTE.new(:groups_with_direct_membership, groups_with_direct_membership_for(user).to_sql) + end + + def groups_with_direct_membership_for(user) + ::Group.joins("INNER JOIN members ON " \ + "members.source_id = namespaces.id AND members.source_type = 'Namespace'") + .where(members: { user_id: user.id, access_level: Gitlab::Access::DEVELOPER.. }) + .select('namespaces.id AS id, members.access_level AS access_level') + end + end + end + end + end + end +end diff --git a/app/models/clusters/agents/authorizations/user_access/project_authorization.rb b/app/models/clusters/agents/authorizations/user_access/project_authorization.rb new file mode 100644 index 00000000000..476666e3ad8 --- /dev/null +++ b/app/models/clusters/agents/authorizations/user_access/project_authorization.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Clusters + module Agents + module Authorizations + module UserAccess + class ProjectAuthorization < ApplicationRecord + include Scopes + + self.table_name = 'agent_user_access_project_authorizations' + + belongs_to :agent, class_name: 'Clusters::Agent', optional: false + belongs_to :project, class_name: '::Project', optional: false + + scope :for_user, ->(user) { + joins('INNER JOIN project_authorizations ON ' \ + 'project_authorizations.project_id = agent_user_access_project_authorizations.project_id') + .where(project_authorizations: { user_id: user.id, access_level: Gitlab::Access::DEVELOPER.. }) + .select('agent_user_access_project_authorizations.*, project_authorizations.access_level AS access_level') + } + + scope :for_project, ->(project) { where(project: project) } + + validates :config, json_schema: { filename: 'clusters_agents_authorizations_user_access_config' } + + def config_project + agent.project + end + + class << self + def upsert_configs(configs) + upsert_all(configs, unique_by: [:agent_id, :project_id]) + end + + def delete_unlisted(project_ids) + where.not(project_id: project_ids).delete_all + end + end + end + end + end + end +end diff --git a/app/models/clusters/agents/group_authorization.rb b/app/models/clusters/agents/group_authorization.rb deleted file mode 100644 index 58ba874ab53..00000000000 --- a/app/models/clusters/agents/group_authorization.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Agents - class GroupAuthorization < ApplicationRecord - include ::Clusters::Agents::AuthorizationConfigScopes - - self.table_name = 'agent_group_authorizations' - - belongs_to :agent, class_name: 'Clusters::Agent', optional: false - belongs_to :group, class_name: '::Group', optional: false - - validates :config, json_schema: { filename: 'cluster_agent_authorization_configuration' } - - def config_project - agent.project - end - end - end -end diff --git a/app/models/clusters/agents/implicit_authorization.rb b/app/models/clusters/agents/implicit_authorization.rb deleted file mode 100644 index a365ccdc568..00000000000 --- a/app/models/clusters/agents/implicit_authorization.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Agents - class ImplicitAuthorization - attr_reader :agent - - delegate :id, to: :agent, prefix: true - - def initialize(agent:) - @agent = agent - end - - def config_project - agent.project - end - - def config - {} - end - end - end -end diff --git a/app/models/clusters/agents/project_authorization.rb b/app/models/clusters/agents/project_authorization.rb deleted file mode 100644 index b9b44741936..00000000000 --- a/app/models/clusters/agents/project_authorization.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Agents - class ProjectAuthorization < ApplicationRecord - include ::Clusters::Agents::AuthorizationConfigScopes - - self.table_name = 'agent_project_authorizations' - - belongs_to :agent, class_name: 'Clusters::Agent', optional: false - belongs_to :project, class_name: '::Project', optional: false - - validates :config, json_schema: { filename: 'cluster_agent_authorization_configuration' } - - def config_project - agent.project - end - end - end -end diff --git a/app/models/clusters/applications/crossplane.rb b/app/models/clusters/applications/crossplane.rb deleted file mode 100644 index a7b4fb57149..00000000000 --- a/app/models/clusters/applications/crossplane.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - # DEPRECATED for removal in %14.0 - # See https://gitlab.com/groups/gitlab-org/-/epics/4280 - class Crossplane < ApplicationRecord - VERSION = '0.4.1' - - self.table_name = 'clusters_applications_crossplane' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - - attribute :version, default: VERSION - attribute :stack, default: "" - - validates :stack, presence: true - - def chart - 'crossplane/crossplane' - end - - def repository - 'https://charts.crossplane.io/alpha' - end - - def install_command - helm_command_module::InstallCommand.new( - name: 'crossplane', - repository: repository, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files - ) - end - - def values - crossplane_values.to_yaml - end - - private - - def crossplane_values - { - "clusterStacks" => { - self.stack => { - "deploy" => true - } - } - } - end - end - end -end diff --git a/app/models/clusters/applications/helm.rb b/app/models/clusters/applications/helm.rb deleted file mode 100644 index 9fac852ed5b..00000000000 --- a/app/models/clusters/applications/helm.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -require 'openssl' - -module Clusters - module Applications - # DEPRECATED: This model represents the Helm 2 Tiller server. - # It is being kept around to enable the cleanup of the unused Tiller server. - class Helm < ApplicationRecord - self.table_name = 'clusters_applications_helm' - - attr_encrypted :ca_key, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_truncated, - algorithm: 'aes-256-cbc' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Gitlab::Utils::StrongMemoize - - attribute :version, default: Gitlab::Kubernetes::Helm::V2::BaseCommand::HELM_VERSION - - before_create :create_keys_and_certs - - def issue_client_cert - ca_cert_obj.issue - end - - def set_initial_status - # The legacy Tiller server is not installable, which is the initial status of every app - end - - # DEPRECATED: This command is only for development and testing purposes, to simulate - # a Helm 2 cluster with an existing Tiller server. - def install_command - Gitlab::Kubernetes::Helm::V2::InitCommand.new( - name: name, - files: files, - rbac: cluster.platform_kubernetes_rbac? - ) - end - - def uninstall_command - Gitlab::Kubernetes::Helm::V2::ResetCommand.new( - name: name, - files: files, - rbac: cluster.platform_kubernetes_rbac? - ) - end - - def has_ssl? - ca_key.present? && ca_cert.present? - end - - private - - def files - { - 'ca.pem': ca_cert, - 'cert.pem': tiller_cert.cert_string, - 'key.pem': tiller_cert.key_string - } - end - - def create_keys_and_certs - ca_cert = Gitlab::Kubernetes::Helm::V2::Certificate.generate_root - self.ca_key = ca_cert.key_string - self.ca_cert = ca_cert.cert_string - end - - def tiller_cert - @tiller_cert ||= ca_cert_obj.issue(expires_in: Gitlab::Kubernetes::Helm::V2::Certificate::INFINITE_EXPIRY) - end - - def ca_cert_obj - return unless has_ssl? - - Gitlab::Kubernetes::Helm::V2::Certificate - .from_strings(ca_key, ca_cert) - end - end - end -end diff --git a/app/models/clusters/applications/ingress.rb b/app/models/clusters/applications/ingress.rb deleted file mode 100644 index 034b178d67d..00000000000 --- a/app/models/clusters/applications/ingress.rb +++ /dev/null @@ -1,91 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - # DEPRECATED for removal in %14.0 - # See https://gitlab.com/groups/gitlab-org/-/epics/4280 - class Ingress < ApplicationRecord - VERSION = '1.40.2' - INGRESS_CONTAINER_NAME = 'nginx-ingress-controller' - - self.table_name = 'clusters_applications_ingress' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - include AfterCommitQueue - include UsageStatistics - - attribute :version, default: VERSION - - enum ingress_type: { - nginx: 1 - }, _default: :nginx - - FETCH_IP_ADDRESS_DELAY = 30.seconds - - state_machine :status do - after_transition any => [:installed] do |application| - application.run_after_commit do - ClusterWaitForIngressIpAddressWorker.perform_in( - FETCH_IP_ADDRESS_DELAY, application.name, application.id) - end - end - end - - def chart - "#{name}/nginx-ingress" - end - - def repository - 'https://gitlab-org.gitlab.io/cluster-integration/helm-stable-archive' - end - - def values - content_values.to_yaml - end - - def allowed_to_uninstall? - external_ip_or_hostname? && !application_jupyter_installed? - end - - def install_command - helm_command_module::InstallCommand.new( - name: name, - repository: repository, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files - ) - end - - def external_ip_or_hostname? - external_ip.present? || external_hostname.present? - end - - def schedule_status_update - return unless installed? - return if external_ip - return if external_hostname - - ClusterWaitForIngressIpAddressWorker.perform_async(name, id) - end - - def ingress_service - cluster.kubeclient.get_service("ingress-#{INGRESS_CONTAINER_NAME}", Gitlab::Kubernetes::Helm::NAMESPACE) - end - - private - - def content_values - YAML.load_file(chart_values_file) - end - - def application_jupyter_installed? - cluster.application_jupyter&.installed? - end - end - end -end diff --git a/app/models/clusters/applications/jupyter.rb b/app/models/clusters/applications/jupyter.rb deleted file mode 100644 index 9c0e90d59ed..00000000000 --- a/app/models/clusters/applications/jupyter.rb +++ /dev/null @@ -1,128 +0,0 @@ -# frozen_string_literal: true - -require 'securerandom' - -module Clusters - module Applications - # DEPRECATED for removal in %14.0 - # See https://gitlab.com/groups/gitlab-org/-/epics/4280 - class Jupyter < ApplicationRecord - VERSION = '0.9.0' - - self.table_name = 'clusters_applications_jupyter' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - - belongs_to :oauth_application, class_name: 'Doorkeeper::Application' - - attribute :version, default: VERSION - - def set_initial_status - return unless not_installable? - return unless cluster&.application_ingress_available? - - ingress = cluster.application_ingress - self.status = status_states[:installable] if ingress.external_ip_or_hostname? - end - - def chart - "#{name}/jupyterhub" - end - - def repository - 'https://jupyterhub.github.io/helm-chart/' - end - - def values - content_values.to_yaml - end - - def install_command - helm_command_module::InstallCommand.new( - name: name, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files, - repository: repository - ) - end - - def callback_url - "http://#{hostname}/hub/oauth_callback" - end - - def oauth_scopes - 'api read_repository write_repository' - end - - private - - def specification - { - "ingress" => { - "hosts" => [hostname], - "tls" => [{ - "hosts" => [hostname], - "secretName" => "jupyter-cert" - }] - }, - "hub" => { - "extraEnv" => { - "GITLAB_HOST" => gitlab_url - }, - "cookieSecret" => cookie_secret - }, - "proxy" => { - "secretToken" => secret_token - }, - "auth" => { - "state" => { - "cryptoKey" => crypto_key - }, - "gitlab" => { - "clientId" => oauth_application.uid, - "clientSecret" => oauth_application.secret, - "callbackUrl" => callback_url, - "gitlabProjectIdWhitelist" => cluster.projects.ids, - "gitlabGroupWhitelist" => cluster.groups.map(&:to_param) - } - }, - "singleuser" => { - "extraEnv" => { - "GITLAB_CLUSTER_ID" => cluster.id.to_s, - "GITLAB_HOST" => gitlab_host - } - } - } - end - - def crypto_key - @crypto_key ||= SecureRandom.hex(32) - end - - def gitlab_url - Gitlab.config.gitlab.url - end - - def gitlab_host - Gitlab.config.gitlab.host - end - - def content_values - YAML.load_file(chart_values_file).deep_merge!(specification) - end - - def secret_token - @secret_token ||= SecureRandom.hex(32) - end - - def cookie_secret - @cookie_secret ||= SecureRandom.hex(32) - end - end - end -end diff --git a/app/models/clusters/applications/knative.rb b/app/models/clusters/applications/knative.rb deleted file mode 100644 index 64366594583..00000000000 --- a/app/models/clusters/applications/knative.rb +++ /dev/null @@ -1,156 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - # DEPRECATED for removal in %14.0 - # See https://gitlab.com/groups/gitlab-org/-/epics/4280 - class Knative < ApplicationRecord - VERSION = '0.10.0' - REPOSITORY = 'https://charts.gitlab.io' - METRICS_CONFIG = 'https://gitlab.com/gitlab-org/charts/knative/-/raw/v0.9.0/vendor/istio-metrics.yml' - FETCH_IP_ADDRESS_DELAY = 30.seconds - API_GROUPS_PATH = 'config/knative/api_groups.yml' - - self.table_name = 'clusters_applications_knative' - - has_one :serverless_domain_cluster, class_name: '::Serverless::DomainCluster', foreign_key: 'clusters_applications_knative_id', inverse_of: :knative - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - include AfterCommitQueue - - alias_method :original_set_initial_status, :set_initial_status - def set_initial_status - return unless cluster&.platform_kubernetes_rbac? - - original_set_initial_status - end - - state_machine :status do - after_transition any => [:installed] do |application| - application.run_after_commit do - ClusterWaitForIngressIpAddressWorker.perform_in( - FETCH_IP_ADDRESS_DELAY, application.name, application.id) - end - end - - after_transition any => [:installed, :updated] do |application| - application.run_after_commit do - ClusterConfigureIstioWorker.perform_async(application.cluster_id) - end - end - end - - attribute :version, default: VERSION - - validates :hostname, presence: true, hostname: true - - scope :for_cluster, -> (cluster) { where(cluster: cluster) } - - has_one :pages_domain, through: :serverless_domain_cluster - - def chart - 'knative/knative' - end - - def values - { "domain" => hostname }.to_yaml - end - - def available_domains - PagesDomain.instance_serverless - end - - def find_available_domain(pages_domain_id) - available_domains.find_by(id: pages_domain_id) - end - - def allowed_to_uninstall? - !pre_installed? - end - - def install_command - helm_command_module::InstallCommand.new( - name: name, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files, - repository: REPOSITORY, - postinstall: install_knative_metrics - ) - end - - def schedule_status_update - return unless installed? - return if external_ip - return if external_hostname - - ClusterWaitForIngressIpAddressWorker.perform_async(name, id) - end - - def ingress_service - cluster.kubeclient.get_service('istio-ingressgateway', Clusters::Kubernetes::ISTIO_SYSTEM_NAMESPACE) - end - - def uninstall_command - helm_command_module::DeleteCommand.new( - name: name, - rbac: cluster.platform_kubernetes_rbac?, - files: files, - predelete: delete_knative_services_and_metrics, - postdelete: delete_knative_istio_leftovers - ) - end - - private - - def delete_knative_services_and_metrics - delete_knative_services + delete_knative_istio_metrics - end - - def delete_knative_services - cluster.kubernetes_namespaces.map do |kubernetes_namespace| - Gitlab::Kubernetes::KubectlCmd.delete("ksvc", "--all", "-n", kubernetes_namespace.namespace) - end - end - - def delete_knative_istio_leftovers - delete_knative_namespaces + delete_knative_and_istio_crds - end - - def delete_knative_namespaces - [ - Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "ns", "knative-serving"), - Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "ns", "knative-build") - ] - end - - def delete_knative_and_istio_crds - api_groups.map do |group| - Gitlab::Kubernetes::KubectlCmd.delete_crds_from_group(group) - end - end - - # returns an array of CRDs to be postdelete since helm does not - # manage the CRDs it creates. - def api_groups - @api_groups ||= YAML.safe_load(File.read(Rails.root.join(API_GROUPS_PATH))) - end - - def install_knative_metrics - return [] unless cluster.application_prometheus&.available? - - [Gitlab::Kubernetes::KubectlCmd.apply_file(METRICS_CONFIG)] - end - - def delete_knative_istio_metrics - return [] unless cluster.application_prometheus&.available? - - [Gitlab::Kubernetes::KubectlCmd.delete("--ignore-not-found", "-f", METRICS_CONFIG)] - end - end - end -end diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb deleted file mode 100644 index a076c871824..00000000000 --- a/app/models/clusters/applications/prometheus.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - class Prometheus < ApplicationRecord - include ::Clusters::Concerns::PrometheusClient - - VERSION = '10.4.1' - - self.table_name = 'clusters_applications_prometheus' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - include AfterCommitQueue - - attribute :version, default: VERSION - - scope :preload_cluster_platform, -> { preload(cluster: [:platform_kubernetes]) } - - attr_encrypted :alert_manager_token, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm' - - after_initialize :set_alert_manager_token, if: :new_record? - - after_destroy do - cluster.find_or_build_integration_prometheus.destroy - end - - state_machine :status do - after_transition any => [:installed, :externally_installed] do |application| - application.cluster.find_or_build_integration_prometheus.update(enabled: true, alert_manager_token: application.alert_manager_token) - end - - after_transition any => :updating do |application| - application.update(last_update_started_at: Time.current) - end - end - - def managed_prometheus? - !externally_installed? && !uninstalled? - end - - def updated_since?(timestamp) - last_update_started_at && - last_update_started_at > timestamp && - !update_errored? - end - - def chart - "#{name}/prometheus" - end - - def repository - 'https://gitlab-org.gitlab.io/cluster-integration/helm-stable-archive' - end - - def install_command - helm_command_module::InstallCommand.new( - name: name, - repository: repository, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files, - postinstall: install_knative_metrics - ) - end - - # Deprecated, to be removed in %14.0 as part of https://gitlab.com/groups/gitlab-org/-/epics/4280 - def patch_command(values) - helm_command_module::PatchCommand.new( - name: name, - repository: repository, - version: version, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files_with_replaced_values(values) - ) - end - - def uninstall_command - helm_command_module::DeleteCommand.new( - name: name, - rbac: cluster.platform_kubernetes_rbac?, - files: files, - predelete: delete_knative_istio_metrics - ) - end - - # Returns a copy of files where the values of 'values.yaml' - # are replaced by the argument. - # - # See #values for the data format required - def files_with_replaced_values(replaced_values) - files.merge('values.yaml': replaced_values) - end - - private - - def set_alert_manager_token - self.alert_manager_token = SecureRandom.hex - end - - def install_knative_metrics - return [] unless cluster.application_knative_available? - - [Gitlab::Kubernetes::KubectlCmd.apply_file(Clusters::Applications::Knative::METRICS_CONFIG)] - end - - def delete_knative_istio_metrics - return [] unless cluster.application_knative_available? - - [ - Gitlab::Kubernetes::KubectlCmd.delete( - "-f", Clusters::Applications::Knative::METRICS_CONFIG, - "--ignore-not-found" - ) - ] - end - end - end -end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb deleted file mode 100644 index b8ed33828bc..00000000000 --- a/app/models/clusters/applications/runner.rb +++ /dev/null @@ -1,69 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Applications - class Runner < ApplicationRecord - VERSION = '0.42.1' - - self.table_name = 'clusters_applications_runners' - - include ::Clusters::Concerns::ApplicationCore - include ::Clusters::Concerns::ApplicationStatus - include ::Clusters::Concerns::ApplicationVersion - include ::Clusters::Concerns::ApplicationData - - belongs_to :runner, class_name: 'Ci::Runner', foreign_key: :runner_id - delegate :project, :group, to: :cluster - - attribute :version, default: VERSION - - def chart - "#{name}/gitlab-runner" - end - - def repository - 'https://charts.gitlab.io' - end - - def values - content_values.to_yaml - end - - def install_command - helm_command_module::InstallCommand.new( - name: name, - version: VERSION, - rbac: cluster.platform_kubernetes_rbac?, - chart: chart, - files: files, - repository: repository - ) - end - - def prepare_uninstall - # No op, see https://gitlab.com/gitlab-org/gitlab/-/issues/350180. - end - - def post_uninstall - runner.destroy! - end - - private - - def gitlab_url - Gitlab::Routing.url_helpers.root_url(only_path: false) - end - - def specification - { - "gitlabUrl" => gitlab_url, - "runners" => { "privileged" => privileged } - } - end - - def content_values - YAML.load_file(chart_values_file).deep_merge!(specification) - end - end - end -end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index a35ea6ddb46..a2903bba6d2 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -11,18 +11,8 @@ module Clusters self.table_name = 'clusters' - APPLICATIONS = { - Clusters::Applications::Helm.application_name => Clusters::Applications::Helm, - Clusters::Applications::Ingress.application_name => Clusters::Applications::Ingress, - Clusters::Applications::Crossplane.application_name => Clusters::Applications::Crossplane, - Clusters::Applications::Prometheus.application_name => Clusters::Applications::Prometheus, - Clusters::Applications::Runner.application_name => Clusters::Applications::Runner, - Clusters::Applications::Jupyter.application_name => Clusters::Applications::Jupyter, - Clusters::Applications::Knative.application_name => Clusters::Applications::Knative - }.freeze DEFAULT_ENVIRONMENT = '*' KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN' - APPLICATIONS_ASSOCIATIONS = APPLICATIONS.values.map(&:association_name).freeze self.reactive_cache_work_type = :external_dependency @@ -54,14 +44,6 @@ module Clusters has_one application.association_name, class_name: application.to_s, inverse_of: :cluster # rubocop:disable Rails/ReflectionClassName end - has_one_cluster_application :helm - has_one_cluster_application :ingress - has_one_cluster_application :crossplane - has_one_cluster_application :prometheus - has_one_cluster_application :runner - has_one_cluster_application :jupyter - has_one_cluster_application :knative - has_many :kubernetes_namespaces has_many :metrics_dashboard_annotations, class_name: 'Metrics::Dashboard::Annotation', inverse_of: :cluster @@ -88,9 +70,6 @@ module Clusters delegate :status, to: :provider, allow_nil: true delegate :status_reason, to: :provider, allow_nil: true - delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true - delegate :external_hostname, to: :application_ingress, prefix: true, allow_nil: true - alias_attribute :base_domain, :domain alias_attribute :provided_by_user?, :user? @@ -123,7 +102,6 @@ module Clusters scope :distinct_with_deployed_environments, -> { joins(:environments).merge(::Deployment.success).distinct } scope :managed, -> { where(managed: true) } - scope :with_persisted_applications, -> { eager_load(*APPLICATIONS_ASSOCIATIONS) } scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) } scope :with_management_project, -> { where.not(management_project: nil) } @@ -232,24 +210,6 @@ module Clusters connection_data.merge(Gitlab::Kubernetes::Node.new(self).all) end - def persisted_applications - APPLICATIONS_ASSOCIATIONS.filter_map { |association_name| public_send(association_name) } # rubocop:disable GitlabSecurity/PublicSend - end - - def applications - APPLICATIONS.each_value.map do |application_class| - find_or_build_application(application_class) - end - end - - def find_or_build_application(application_class) - raise ArgumentError, "#{application_class} is not in APPLICATIONS" unless APPLICATIONS.value?(application_class) - - association_name = application_class.association_name - - public_send(association_name) || public_send("build_#{association_name}") # rubocop:disable GitlabSecurity/PublicSend - end - def find_or_build_integration_prometheus integration_prometheus || build_integration_prometheus end @@ -270,18 +230,6 @@ module Clusters !!platform_kubernetes&.rbac? end - def application_helm_available? - !!application_helm&.available? - end - - def application_ingress_available? - !!application_ingress&.available? - end - - def application_knative_available? - !!application_knative&.available? - end - def integration_prometheus_available? !!integration_prometheus&.available? end @@ -365,12 +313,6 @@ module Clusters end end - def serverless_domain - strong_memoize(:serverless_domain) do - self.application_knative&.serverless_domain_cluster - end - end - def prometheus_adapter integration_prometheus end diff --git a/app/models/clusters/kubernetes_namespace.rb b/app/models/clusters/kubernetes_namespace.rb index 42332bdc193..dfb5c4cc5eb 100644 --- a/app/models/clusters/kubernetes_namespace.rb +++ b/app/models/clusters/kubernetes_namespace.rb @@ -22,9 +22,9 @@ module Clusters delegate :api_url, to: :platform_kubernetes, allow_nil: true attr_encrypted :service_account_token, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_truncated, - algorithm: 'aes-256-cbc' + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_truncated, + algorithm: 'aes-256-cbc' scope :has_service_account_token, -> { where.not(encrypted_service_account_token: nil) } scope :with_environment_name, -> (name) { joins(:environment).where(environments: { name: name }) } diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 165285b34b2..123ad0ebfaf 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -4,7 +4,6 @@ module Clusters module Platforms class Kubernetes < ApplicationRecord include Gitlab::Kubernetes - include EnumWithNil include AfterCommitQueue include ReactiveCaching include NullifyIfBlank @@ -63,7 +62,7 @@ module Clusters alias_attribute :ca_pem, :ca_cert - enum_with_nil authorization_type: { + enum authorization_type: { unknown_authorization: nil, rbac: 1, abac: 2 diff --git a/app/models/commit.rb b/app/models/commit.rb index 4517b3ef216..6d17d7f495d 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -206,7 +206,8 @@ class Commit def self.link_reference_pattern @link_reference_pattern ||= - super("commit", /(?<commit>#{COMMIT_SHA_PATTERN})?(\.(?<extension>#{LINK_EXTENSION_PATTERN}))?/o) + compose_link_reference_pattern('commit', + /(?<commit>#{COMMIT_SHA_PATTERN})?(\.(?<extension>#{LINK_EXTENSION_PATTERN}))?/o) end def to_reference(from = nil, full: false) @@ -387,8 +388,6 @@ class Commit Gitlab::X509::Commit.new(self).signature when :SSH Gitlab::Ssh::Commit.new(self).signature - else - nil end end end @@ -573,8 +572,43 @@ class Commit } end + def tipping_branches(limit: 0) + tipping_refs(Gitlab::Git::BRANCH_REF_PREFIX, limit: limit) + end + + def tipping_tags(limit: 0) + tipping_refs(Gitlab::Git::TAG_REF_PREFIX, limit: limit) + end + + def branches_containing(limit: 0, exclude_tipped: false) + # WARNING: This argument can be confusing, if there is a limit. + # for example set the limit to 5 and in the 5 out a total of 25 refs there is 2 tipped refs, + # then the method will only 3 refs, even though there is more. + excluded = exclude_tipped ? tipping_branches : [] + + refs = repository.branch_names_contains(id, limit: limit) || [] + refs - excluded + end + + def tags_containing(limit: 0, exclude_tipped: false) + # WARNING: This argument can be confusing, if there is a limit. + # for example set the limit to 5 and in the 5 out a total of 25 refs there is 2 tipped refs, + # then the method will only 3 refs, even though there is more. + excluded = exclude_tipped ? tipping_tags : [] + + refs = repository.tag_names_contains(id, limit: limit) || [] + refs - excluded + end + private + def tipping_refs(ref_prefix, limit: 0) + strong_memoize_with(:tipping_tags, ref_prefix, limit) do + refs = repository.refs_by_oid(oid: id, ref_patterns: [ref_prefix], limit: limit) + refs.map { |n| n.delete_prefix(ref_prefix) } + end + end + def expire_note_etag_cache_for_related_mrs MergeRequest.includes(target_project: :namespace).by_commit_sha(id).find_each(&:expire_note_etag_cache) end diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb index 47ecdfa8574..edc60a757d2 100644 --- a/app/models/commit_collection.rb +++ b/app/models/commit_collection.rb @@ -30,6 +30,10 @@ class CommitCollection User.by_any_email(emails) end + def committer_user_ids + committers.pluck(:id) + end + def without_merge_commits strong_memoize(:without_merge_commits) do # `#enrich!` the collection to ensure all commits contain @@ -118,4 +122,21 @@ class CommitCollection def next_page @pagination.next_page end + + def load_tags + oids = commits.map(&:id) + references = repository.list_refs([Gitlab::Git::TAG_REF_PREFIX], pointing_at_oids: oids, peel_tags: true) + oid_to_references = references.group_by { |reference| reference.peeled_target.presence || reference.target } + + return self if oid_to_references.empty? + + commits.each do |commit| + grouped_references = oid_to_references[commit.id] + next unless grouped_references + + commit.referenced_by = grouped_references.map(&:name) + end + + self + end end diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index 87029cb2033..90cdd267cbd 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -50,7 +50,7 @@ class CommitRange end def self.link_reference_pattern - @link_reference_pattern ||= super("compare", /(?<commit_range>#{PATTERN})/o) + @link_reference_pattern ||= compose_link_reference_pattern('compare', /(?<commit_range>#{PATTERN})/o) end # Initialize a CommitRange diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 333a176b8f3..6dfea7ef9a7 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -6,17 +6,20 @@ class CommitStatus < Ci::ApplicationRecord include Importable include AfterCommitQueue include Presentable - include EnumWithNil include BulkInsertableAssociations include TaggableQueries + include SafelyChangeColumnDefault self.table_name = 'ci_builds' + self.sequence_name = 'ci_builds_id_seq' + self.primary_key = :id partitionable scope: :pipeline + columns_changing_default :partition_id belongs_to :user belongs_to :project - belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id - belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline' + belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, inverse_of: :statuses + belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline', inverse_of: :auto_canceled_jobs belongs_to :ci_stage, class_name: 'Ci::Stage', foreign_key: :stage_id has_many :needs, class_name: 'Ci::BuildNeed', foreign_key: :build_id, inverse_of: :build @@ -26,13 +29,14 @@ class CommitStatus < Ci::ApplicationRecord enum scheduling_type: { stage: 0, dag: 1 }, _prefix: true # We use `Enums::Ci::CommitStatus.failure_reasons` here so that EE can more easily # extend this `Hash` with new values. - enum_with_nil failure_reason: Enums::Ci::CommitStatus.failure_reasons + enum failure_reason: Enums::Ci::CommitStatus.failure_reasons delegate :commit, to: :pipeline delegate :sha, :short_sha, :before_sha, to: :pipeline validates :pipeline, presence: true, unless: :importing? validates :name, presence: true, unless: :importing? + validates :stage, :ref, :target_url, :description, length: { maximum: 255 } alias_attribute :author, :user alias_attribute :pipeline_id, :commit_id @@ -43,14 +47,6 @@ class CommitStatus < Ci::ApplicationRecord scope :order_id_desc, -> { order(id: :desc) } - scope :exclude_ignored, -> do - # We want to ignore failed but allowed to fail jobs. - # - # TODO, we also skip ignored optional manual actions. - where("allow_failure = ? OR status IN (?)", - false, all_state_names - [:failed, :canceled, :manual]) - end - scope :latest, -> { where(retried: [false, nil]) } scope :retried, -> { where(retried: true) } scope :ordered, -> { order(:name) } @@ -66,12 +62,13 @@ class CommitStatus < Ci::ApplicationRecord scope :by_name, -> (name) { where(name: name) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } scope :with_pipeline, -> { joins(:pipeline) } - scope :updated_at_before, ->(date) { where('ci_builds.updated_at < ?', date) } - scope :created_at_before, ->(date) { where('ci_builds.created_at < ?', date) } + scope :updated_at_before, ->(date) { where("#{quoted_table_name}.updated_at < ?", date) } + scope :created_at_before, ->(date) { where("#{quoted_table_name}.created_at < ?", date) } scope :scheduled_at_before, ->(date) { - where('ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?', date) + where("#{quoted_table_name}.scheduled_at IS NOT NULL AND #{quoted_table_name}.scheduled_at < ?", date) } scope :with_when_executed, ->(when_executed) { where(when: when_executed) } + scope :with_type, ->(type) { where(type: type) } # The scope applies `pluck` to split the queries. Use with care. scope :for_project_paths, -> (paths) do @@ -239,10 +236,6 @@ class CommitStatus < Ci::ApplicationRecord name.to_s.sub(regex, '').strip end - def failed_but_allowed? - allow_failure? && (failed? || canceled?) - end - # Time spent running. def duration calculate_duration(started_at, finished_at) diff --git a/app/models/compare.rb b/app/models/compare.rb index f03390334f4..58279cb58aa 100644 --- a/app/models/compare.rb +++ b/app/models/compare.rb @@ -30,7 +30,7 @@ class Compare # See `namespace_project_compare_url` def to_param { - from: @straight ? start_commit_sha : base_commit_sha, + from: @straight ? start_commit_sha : (base_commit_sha || start_commit_sha), to: head_commit_sha } end diff --git a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb index 1bdb89349aa..c01399184ad 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage_event_model.rb @@ -74,7 +74,7 @@ module Analytics query = <<~SQL INSERT INTO #{quoted_table_name} ( - stage_event_hash_id, + stage_event_hash_id, #{connection.quote_column_name(issuable_id_column)}, group_id, project_id, diff --git a/app/models/concerns/analytics/cycle_analytics/stageable.rb b/app/models/concerns/analytics/cycle_analytics/stageable.rb index caac4f31e1a..d1dd46883e3 100644 --- a/app/models/concerns/analytics/cycle_analytics/stageable.rb +++ b/app/models/concerns/analytics/cycle_analytics/stageable.rb @@ -7,8 +7,8 @@ module Analytics include Gitlab::Utils::StrongMemoize included do - belongs_to :start_event_label, class_name: 'GroupLabel', optional: true - belongs_to :end_event_label, class_name: 'GroupLabel', optional: true + belongs_to :start_event_label, class_name: 'Label', optional: true + belongs_to :end_event_label, class_name: 'Label', optional: true belongs_to :stage_event_hash, class_name: 'Analytics::CycleAnalytics::StageEventHash', optional: true validates :name, presence: true @@ -119,10 +119,11 @@ module Analytics end def label_available_for_namespace?(label_id) - subject = is_a?(::Analytics::CycleAnalytics::Stage) ? namespace : project.group + subject = namespace.is_a?(Namespaces::ProjectNamespace) ? namespace.project.group : namespace return unless subject - LabelsFinder.new(nil, { group_id: subject.id, include_ancestor_groups: true, only_group_labels: true }) + LabelsFinder.new(nil, + { group_id: subject.id, include_ancestor_groups: true, only_group_labels: namespace.is_a?(Group) }) .execute(skip_authorization: true) .id_in(label_id) .exists? diff --git a/app/models/concerns/atomic_internal_id.rb b/app/models/concerns/atomic_internal_id.rb index 14be924f9da..ec4ee7985fe 100644 --- a/app/models/concerns/atomic_internal_id.rb +++ b/app/models/concerns/atomic_internal_id.rb @@ -61,6 +61,8 @@ module AtomicInternalId AtomicInternalId.project_init(self) when :group AtomicInternalId.group_init(self) + when :namespace + AtomicInternalId.namespace_init(self) else # We require init here to retain the ability to recalculate in the absence of a # InternalId record (we may delete records in `internal_ids` for example). @@ -241,6 +243,16 @@ module AtomicInternalId end end + def self.namespace_init(klass, column_name = :iid) + ->(instance, scope) do + if instance + klass.where(namespace_id: instance.namespace_id).maximum(column_name) + elsif scope.present? + klass.where(**scope).maximum(column_name) + end + end + end + def internal_id_read_scope(scope) association(scope).reader end diff --git a/app/models/concerns/awareness.rb b/app/models/concerns/awareness.rb deleted file mode 100644 index da87d87e838..00000000000 --- a/app/models/concerns/awareness.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Awareness - extend ActiveSupport::Concern - - KEY_NAMESPACE = "gitlab:awareness" - private_constant :KEY_NAMESPACE - - def join(session) - session.join(self) - - nil - end - - def leave(session) - session.leave(self) - - nil - end - - def session_ids - with_redis do |redis| - redis - .smembers(user_sessions_key) - # converts session ids from (internal) integer to hex presentation - .map { |key| key.to_i.to_s(16) } - end - end - - private - - def user_sessions_key - "#{KEY_NAMESPACE}:user:#{id}:sessions" - end - - def with_redis - Gitlab::Redis::SharedState.with do |redis| - yield redis if block_given? - end - end -end diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb index c3aa3019abb..11e88ee3372 100644 --- a/app/models/concerns/bulk_member_access_load.rb +++ b/app/models/concerns/bulk_member_access_load.rb @@ -5,16 +5,20 @@ module BulkMemberAccessLoad included do def merge_value_to_request_store(resource_klass, resource_id, value) - Gitlab::SafeRequestLoader.execute(resource_key: max_member_access_for_resource_key(resource_klass), - resource_ids: [resource_id], - default_value: Gitlab::Access::NO_ACCESS) do + Gitlab::SafeRequestLoader.execute( + resource_key: max_member_access_for_resource_key(resource_klass), + resource_ids: [resource_id], + default_value: Gitlab::Access::NO_ACCESS + ) do { resource_id => value } end end def purge_resource_id_from_request_store(resource_klass, resource_id) - Gitlab::SafeRequestPurger.execute(resource_key: max_member_access_for_resource_key(resource_klass), - resource_ids: [resource_id]) + Gitlab::SafeRequestPurger.execute( + resource_key: max_member_access_for_resource_key(resource_klass), + resource_ids: [resource_id] + ) end def max_member_access_for_resource_key(klass) diff --git a/app/models/concerns/cached_commit.rb b/app/models/concerns/cached_commit.rb index 0fb72552dd5..8a53fec0612 100644 --- a/app/models/concerns/cached_commit.rb +++ b/app/models/concerns/cached_commit.rb @@ -14,4 +14,9 @@ module CachedCommit def parent_ids [] end + + # These are not saved + def referenced_by + [] + end end diff --git a/app/models/concerns/cascading_namespace_setting_attribute.rb b/app/models/concerns/cascading_namespace_setting_attribute.rb index 731729a1ed5..d0ee4f33ce6 100644 --- a/app/models/concerns/cascading_namespace_setting_attribute.rb +++ b/app/models/concerns/cascading_namespace_setting_attribute.rb @@ -57,11 +57,13 @@ module CascadingNamespaceSettingAttribute # private methods define_validator_methods(attribute) + define_attr_before_save(attribute) define_after_update(attribute) validate :"#{attribute}_changeable?" validate :"lock_#{attribute}_changeable?" + before_save :"before_save_#{attribute}", if: -> { will_save_change_to_attribute?(attribute) } after_update :"clear_descendant_#{attribute}_locks", if: -> { saved_change_to_attribute?("lock_#{attribute}", to: true) } end end @@ -92,13 +94,26 @@ module CascadingNamespaceSettingAttribute def define_attr_writer(attribute) define_method("#{attribute}=") do |value| - return value if value == cascaded_ancestor_value(attribute) + return value if read_attribute(attribute).nil? && to_bool(value) == cascaded_ancestor_value(attribute) clear_memoization(attribute) super(value) end end + def define_attr_before_save(attribute) + # rubocop:disable GitlabSecurity/PublicSend + define_method("before_save_#{attribute}") do + new_value = public_send(attribute) + if public_send("#{attribute}_was").nil? && new_value == cascaded_ancestor_value(attribute) + write_attribute(attribute, nil) + end + end + # rubocop:enable GitlabSecurity/PublicSend + + private :"before_save_#{attribute}" + end + def define_lock_attr_writer(attribute) define_method("lock_#{attribute}=") do |value| attr_value = public_send(attribute) # rubocop:disable GitlabSecurity/PublicSend @@ -239,4 +254,8 @@ module CascadingNamespaceSettingAttribute namespace.descendants.pluck(:id) end end + + def to_bool(value) + ActiveModel::Type::Boolean.new.cast(value) + end end diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb index 9a04776f1c6..2971ecb04b8 100644 --- a/app/models/concerns/ci/has_status.rb +++ b/app/models/concerns/ci/has_status.rb @@ -13,7 +13,7 @@ module Ci STOPPED_STATUSES = COMPLETED_STATUSES + BLOCKED_STATUS ORDERED_STATUSES = %w[failed preparing pending running waiting_for_resource manual scheduled canceled success skipped created].freeze PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze - EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze + IGNORED_STATUSES = %w[manual].to_set.freeze ALIVE_STATUSES = (ACTIVE_STATUSES + ['created']).freeze CANCELABLE_STATUSES = (ALIVE_STATUSES + ['scheduled']).freeze STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, @@ -23,6 +23,7 @@ module Ci UnknownStatusError = Class.new(StandardError) class_methods do + # This will be removed with ci_remove_ensure_stage_service def composite_status Gitlab::Ci::Status::Composite .new(all, with_allow_failure: columns_hash.key?('allow_failure')) diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index d91f33452a0..1c6b82d6ea7 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -9,10 +9,11 @@ module Ci extend ActiveSupport::Concern included do - has_one :metadata, class_name: 'Ci::BuildMetadata', - foreign_key: :build_id, - inverse_of: :build, - autosave: true + has_one :metadata, + class_name: 'Ci::BuildMetadata', + foreign_key: :build_id, + inverse_of: :build, + autosave: true accepts_nested_attributes_for :metadata diff --git a/app/models/concerns/ci/partitionable.rb b/app/models/concerns/ci/partitionable.rb index d6ba0f4488f..d8417773dbd 100644 --- a/app/models/concerns/ci/partitionable.rb +++ b/app/models/concerns/ci/partitionable.rb @@ -2,7 +2,7 @@ module Ci ## - # This module implements a way to set the `partion_id` value on a dependent + # This module implements a way to set the `partition_id` value on a dependent # resource from a parent record. # Usage: # @@ -36,6 +36,7 @@ module Ci Ci::Pipeline Ci::PendingBuild Ci::RunningBuild + Ci::RunnerManagerBuild Ci::PipelineVariable Ci::Sources::Pipeline Ci::Stage @@ -70,8 +71,8 @@ module Ci class_methods do def partitionable(scope:, through: nil, partitioned: false) handle_partitionable_through(through) - handle_partitionable_dml(partitioned) handle_partitionable_scope(scope) + handle_partitionable_ddl(partitioned) end private @@ -85,13 +86,6 @@ module Ci include Partitionable::Switch end - def handle_partitionable_dml(partitioned) - define_singleton_method(:partitioned?) { partitioned } - return unless partitioned - - include Partitionable::PartitionedFilter - end - def handle_partitionable_scope(scope) define_method(:partition_scope_value) do strong_memoize(:partition_scope_value) do @@ -102,6 +96,17 @@ module Ci end end end + + def handle_partitionable_ddl(partitioned) + return unless partitioned + + include ::PartitionedTable + + partitioned_by :partition_id, + strategy: :ci_sliding_list, + next_partition_if: proc { false }, + detach_partition_if: proc { false } + end end end end diff --git a/app/models/concerns/ci/partitionable/partitioned_filter.rb b/app/models/concerns/ci/partitionable/partitioned_filter.rb deleted file mode 100644 index 4adae3be26a..00000000000 --- a/app/models/concerns/ci/partitionable/partitioned_filter.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Ci - module Partitionable - # Used to patch the save, update, delete, destroy methods to use the - # partition_id attributes for their SQL queries. - module PartitionedFilter - extend ActiveSupport::Concern - - if Rails::VERSION::MAJOR >= 7 - # These methods are updated in Rails 7 to use `_primary_key_constraints_hash` - # by default, so this patch will no longer be required. - # - # rubocop:disable Gitlab/NoCodeCoverageComment - # :nocov: - raise "`#{__FILE__}` should be double checked" if Rails.env.test? - - warn "Update `#{__FILE__}`. Patches Rails internals for partitioning" - # :nocov: - # rubocop:enable Gitlab/NoCodeCoverageComment - else - def _update_row(attribute_names, attempted_action = "update") - self.class._update_record( - attributes_with_values(attribute_names), - _primary_key_constraints_hash - ) - end - - def _delete_row - self.class._delete_record(_primary_key_constraints_hash) - end - end - - # Introduced in Rails 7, but updated to include `partition_id` filter. - # https://github.com/rails/rails/blob/a4dbb153fd390ac31bb9808809e7ac4d3a2c5116/activerecord/lib/active_record/persistence.rb#L1031-L1033 - def _primary_key_constraints_hash - { @primary_key => id_in_database, partition_id: partition_id } # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - end - end -end diff --git a/app/models/concerns/clusters/agents/authorization_config_scopes.rb b/app/models/concerns/clusters/agents/authorization_config_scopes.rb deleted file mode 100644 index 0a0406c3389..00000000000 --- a/app/models/concerns/clusters/agents/authorization_config_scopes.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Clusters - module Agents - module AuthorizationConfigScopes - extend ActiveSupport::Concern - - included do - scope :with_available_ci_access_fields, ->(project) { - where("config->'access_as' IS NULL") - .or(where("config->'access_as' = '{}'")) - .or(where("config->'access_as' ?| array[:fields]", fields: available_ci_access_fields(project))) - } - end - - class_methods do - def available_ci_access_fields(_project) - %w(agent) - end - end - end - end -end - -Clusters::Agents::AuthorizationConfigScopes.prepend_mod diff --git a/app/models/concerns/clusters/agents/authorizations/ci_access/config_scopes.rb b/app/models/concerns/clusters/agents/authorizations/ci_access/config_scopes.rb new file mode 100644 index 00000000000..eef68bfd349 --- /dev/null +++ b/app/models/concerns/clusters/agents/authorizations/ci_access/config_scopes.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Clusters + module Agents + module Authorizations + module CiAccess + module ConfigScopes + extend ActiveSupport::Concern + + included do + scope :with_available_ci_access_fields, ->(project) { + where("config->'access_as' IS NULL") + .or(where("config->'access_as' = '{}'")) + .or(where("config->'access_as' ?| array[:fields]", fields: available_ci_access_fields(project))) + } + end + + class_methods do + def available_ci_access_fields(_project) + %w(agent) + end + end + end + end + end + end +end + +Clusters::Agents::Authorizations::CiAccess::ConfigScopes.prepend_mod diff --git a/app/models/concerns/clusters/agents/authorizations/user_access/scopes.rb b/app/models/concerns/clusters/agents/authorizations/user_access/scopes.rb new file mode 100644 index 00000000000..515b4ed3c87 --- /dev/null +++ b/app/models/concerns/clusters/agents/authorizations/user_access/scopes.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Clusters + module Agents + module Authorizations + module UserAccess + module Scopes + extend ActiveSupport::Concern + + included do + scope :for_agent, ->(agent) { where(agent: agent) } + scope :preloaded, -> { joins(agent: :project).preload(agent: :project) } + end + end + end + end + end +end diff --git a/app/models/concerns/counter_attribute.rb b/app/models/concerns/counter_attribute.rb index 58ea57962c5..56608c49a6b 100644 --- a/app/models/concerns/counter_attribute.rb +++ b/app/models/concerns/counter_attribute.rb @@ -5,7 +5,7 @@ # after a period of time (10 minutes). # When an attribute is incremented by a value, the increment is added # to a Redis key. Then, FlushCounterIncrementsWorker will execute -# `flush_increments_to_database!` which removes increments from Redis for a +# `commit_increment!` which removes increments from Redis for a # given model attribute and updates the values in the database. # # @example: @@ -29,8 +29,24 @@ # counter_attribute :conditional_one, if: -> { |object| object.use_counter_attribute? } # end # +# The `counter_attribute` by default will return last persisted value. +# It's possible to always return accurate (real) value instead by using `returns_current: true`. +# While doing this the `counter_attribute` will overwrite attribute accessor to fetch +# the buffered information added to the last persisted value. This will incur cost a Redis call per attribute fetched. +# +# @example: +# +# class ProjectStatistics +# include CounterAttribute +# +# counter_attribute :commit_count, returns_current: true +# end +# +# in that case +# model.commit_count => persisted value + buffered amount to be added +# # To increment the counter we can use the method: -# increment_counter(:commit_count, 3) +# increment_amount(:commit_count, 3) # # This method would determine whether it would increment the counter using Redis, # or fallback to legacy increment on ActiveRecord counters. @@ -50,11 +66,22 @@ module CounterAttribute include Gitlab::Utils::StrongMemoize class_methods do - def counter_attribute(attribute, if: nil) + def counter_attribute(attribute, if: nil, returns_current: false) counter_attributes << { attribute: attribute, - if_proc: binding.local_variable_get(:if) # can't read `if` directly + if_proc: binding.local_variable_get(:if), # can't read `if` directly + returns_current: returns_current } + + if returns_current + define_method(attribute) do + current_counter(attribute) + end + end + + define_method("increment_#{attribute}") do |amount| + increment_amount(attribute, amount) + end end def counter_attributes @@ -87,6 +114,15 @@ module CounterAttribute end end + def increment_amount(attribute, amount) + counter = Gitlab::Counters::Increment.new(amount: amount) + increment_counter(attribute, counter) + end + + def current_counter(attribute) + read_attribute(attribute) + counter(attribute).get + end + def increment_counter(attribute, increment) return if increment.amount == 0 @@ -165,14 +201,13 @@ module CounterAttribute # # It does not guarantee that there will not be any concurrent updates. def detect_race_on_record(log_fields: {}) - return yield unless Feature.enabled?(:counter_attribute_db_lease_for_update, project) - # Ensure attributes is always an array before we log log_fields[:attributes] = Array(log_fields[:attributes]) Gitlab::AppLogger.info( message: 'Acquiring lease for project statistics update', - project_statistics_id: id, + model: self.class.name, + model_id: id, project_id: project.id, **log_fields, **Gitlab::ApplicationContext.current @@ -184,7 +219,8 @@ module CounterAttribute rescue Gitlab::ExclusiveLeaseHelpers::FailedToObtainLockError Gitlab::AppLogger.warn( message: 'Concurrent project statistics update detected', - project_statistics_id: id, + model: self.class.name, + model_id: id, project_id: project.id, **log_fields, **Gitlab::ApplicationContext.current diff --git a/app/models/concerns/database_event_tracking.rb b/app/models/concerns/database_event_tracking.rb index 9f75b3ed4d8..26e184c202f 100644 --- a/app/models/concerns/database_event_tracking.rb +++ b/app/models/concerns/database_event_tracking.rb @@ -3,6 +3,8 @@ module DatabaseEventTracking extend ActiveSupport::Concern + FEATURE_FLAG_BATCH2_CLASSES = %w[Vulnerability MergeRequest::Metrics].freeze + included do after_create_commit :publish_database_create_event after_destroy_commit :publish_database_destroy_event @@ -22,7 +24,8 @@ module DatabaseEventTracking end def publish_database_event(name) - return unless Feature.enabled?(:product_intelligence_database_event_tracking) + return unless database_events_for_class_enabled? + return unless database_events_feature_flag_enabled? # Gitlab::Tracking#event is triggering Snowplow event # Snowplow events are sent with usage of @@ -30,11 +33,12 @@ module DatabaseEventTracking # that reports data asynchronously and does not impact performance nor carries a risk of # rollback in case of error - Gitlab::Tracking.event( + Gitlab::Tracking.database_event( self.class.to_s, "database_event_#{name}", label: self.class.table_name, - namespace: try(:group) || try(:namespace), + project: try(:project), + namespace: (try(:group) || try(:namespace)) || try(:project)&.namespace, property: name, **filtered_record_attributes ) @@ -50,4 +54,14 @@ module DatabaseEventTracking .with_indifferent_access .slice(*self.class::SNOWPLOW_ATTRIBUTES) end + + def database_events_for_class_enabled? + is_batch2 = FEATURE_FLAG_BATCH2_CLASSES.include?(self.class.to_s) + + !is_batch2 || Feature.enabled?(:product_intelligence_database_event_tracking_batch2) + end + + def database_events_feature_flag_enabled? + Feature.enabled?(:product_intelligence_database_event_tracking) + end end diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb index 40891073738..d3ebda2702d 100644 --- a/app/models/concerns/discussion_on_diff.rb +++ b/app/models/concerns/discussion_on_diff.rb @@ -7,20 +7,20 @@ module DiscussionOnDiff NUMBER_OF_TRUNCATED_DIFF_LINES = 16 included do - delegate :line_code, - :original_line_code, - :note_diff_file, - :diff_line, - :active?, - :created_at_diff?, - to: :first_note - - delegate :file_path, - :blob, - :highlighted_diff_lines, - :diff_lines, - to: :diff_file, - allow_nil: true + delegate :line_code, + :original_line_code, + :note_diff_file, + :diff_line, + :active?, + :created_at_diff?, + to: :first_note + + delegate :file_path, + :blob, + :highlighted_diff_lines, + :diff_lines, + to: :diff_file, + allow_nil: true end def diff_discussion? diff --git a/app/models/concerns/each_batch.rb b/app/models/concerns/each_batch.rb index dbc0887dc97..79fb81e7820 100644 --- a/app/models/concerns/each_batch.rb +++ b/app/models/concerns/each_batch.rb @@ -161,5 +161,81 @@ module EachBatch break unless stop end end + + # Iterates over the relation and counts the rows. The counting + # logic is combined with the iteration query which saves one query + # compared to a standard each_batch approach. + # + # Basic usage: + # count, _last_value = Project.each_batch_count + # + # The counting can be stopped by passing a block and making the last statement true. + # Example: + # + # query_count = 0 + # count, last_value = Project.each_batch_count do + # query_count += 1 + # query_count == 5 # stop counting after 5 loops + # end + # + # Resume where the previous counting has stopped: + # + # count, last_value = Project.each_batch_count(last_count: count, last_value: last_value) + # + # Another example, counting issues in project: + # + # project = Project.find(1) + # count, _ = project.issues.each_batch_count(column: :iid) + def each_batch_count(of: 1000, column: :id, last_count: 0, last_value: nil) + arel_table = self.arel_table + window = Arel::Nodes::Window.new.order(arel_table[column]) + last_value_column = Arel::Nodes::NamedFunction + .new('LAST_VALUE', [arel_table[column]]) + .over(window) + .as(column.to_s) + + loop do + count_column = Arel::Nodes::Addition + .new(Arel::Nodes::NamedFunction.new('ROW_NUMBER', []).over(window), last_count) + .as('count') + + projections = [count_column, last_value_column] + scope = limit(1).offset(of - 1) + scope = scope.where(arel_table[column].gt(last_value)) if last_value + new_count, last_value = scope.pick(*projections) + + # When reaching the last batch the offset query might return no data, to address this + # problem, we invoke a specialized query that takes the last row out of the resultset. + # We could do this for each batch, however it would add unnecessary overhead to all + # queries. + if new_count.nil? + inner_query = scope + .select(*projections) + .limit(nil) + .offset(nil) + .arel + .as(quoted_table_name) + + new_count, last_value = + unscoped + .from(inner_query) + .order(count: :desc) + .limit(1) + .pick(:count, column) + + last_count = new_count if new_count + last_value = nil + break + end + + last_count = new_count + + if block_given? + should_break = yield(last_count, last_value) + break if should_break + end + end + [last_count, last_value] + end end end diff --git a/app/models/concerns/enum_with_nil.rb b/app/models/concerns/enum_with_nil.rb deleted file mode 100644 index c66942025d7..00000000000 --- a/app/models/concerns/enum_with_nil.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module EnumWithNil - extend ActiveSupport::Concern - - included do - def self.enum_with_nil(definitions) - # use original `enum` to auto-define all methods - enum(definitions) - - # override auto-defined methods only for the - # key which uses nil value - definitions.each do |name, values| - # E.g. for enum_with_nil failure_reason: { unknown_failure: nil } - # this overrides auto-generated method `failure_reason` - define_method(name) do - orig = super() - - return orig unless orig.nil? - - self.class.public_send(name.to_s.pluralize).key(nil) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end -end diff --git a/app/models/concerns/enums/abuse/source.rb b/app/models/concerns/enums/abuse/source.rb new file mode 100644 index 00000000000..80703126aae --- /dev/null +++ b/app/models/concerns/enums/abuse/source.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Enums + module Abuse + module Source + def self.sources + { + spamcheck: 0, + virus_total: 1, + arkose_custom_score: 2, + arkose_global_score: 3, + telesign: 4, + pvs: 5 + } + end + end + end +end diff --git a/app/models/concerns/enums/ci/pipeline.rb b/app/models/concerns/enums/ci/pipeline.rb index 8ed6c54441b..778471eac8b 100644 --- a/app/models/concerns/enums/ci/pipeline.rb +++ b/app/models/concerns/enums/ci/pipeline.rb @@ -11,7 +11,6 @@ module Enums config_error: 1, external_validation_failure: 2, user_not_verified: 3, - activity_limit_exceeded: 20, size_limit_exceeded: 21, job_activity_limit_exceeded: 22, deployments_limit_exceeded: 23, diff --git a/app/models/concerns/enums/internal_id.rb b/app/models/concerns/enums/internal_id.rb index a8227363a22..8e161c1513f 100644 --- a/app/models/concerns/enums/internal_id.rb +++ b/app/models/concerns/enums/internal_id.rb @@ -17,7 +17,8 @@ module Enums sprints: 9, # iterations design_management_designs: 10, incident_management_oncall_schedules: 11, - ml_experiments: 12 + ml_experiments: 12, + ml_candidates: 13 } end end diff --git a/app/models/concerns/enums/package_metadata.rb b/app/models/concerns/enums/package_metadata.rb index e15fe758e69..3f107987ef6 100644 --- a/app/models/concerns/enums/package_metadata.rb +++ b/app/models/concerns/enums/package_metadata.rb @@ -10,11 +10,46 @@ module Enums maven: 5, npm: 6, nuget: 7, - pypi: 8 + pypi: 8, + apk: 9, + rpm: 10, + deb: 11, + cbl_mariner: 12 + }.with_indifferent_access.freeze + + ADVISORY_SOURCES = { + glad: 1, # gitlab advisory db + trivy: 2 + }.with_indifferent_access.freeze + + DATA_TYPES = { + advisories: 1, + licenses: 2 + }.with_indifferent_access.freeze + + VERSION_FORMATS = { + v1: 1, + v2: 2 }.with_indifferent_access.freeze def self.purl_types PURL_TYPES end + + def self.purl_types_numerical + purl_types.invert + end + + def self.advisory_sources + ADVISORY_SOURCES + end + + def self.data_types + DATA_TYPES + end + + def self.version_formats + VERSION_FORMATS + end end end diff --git a/app/models/concerns/enums/sbom.rb b/app/models/concerns/enums/sbom.rb index 8848c0c5555..3ba911dbcc5 100644 --- a/app/models/concerns/enums/sbom.rb +++ b/app/models/concerns/enums/sbom.rb @@ -14,7 +14,11 @@ module Enums maven: 5, npm: 6, nuget: 7, - pypi: 8 + pypi: 8, + apk: 9, + rpm: 10, + deb: 11, + cbl_mariner: 12 }.with_indifferent_access.freeze def self.component_types diff --git a/app/models/concerns/expirable.rb b/app/models/concerns/expirable.rb index 5975ea23723..cc55315d6d7 100644 --- a/app/models/concerns/expirable.rb +++ b/app/models/concerns/expirable.rb @@ -8,7 +8,7 @@ module Expirable included do scope :not, ->(scope) { where(scope.arel.constraints.reduce(:and).not) } - scope :expired, -> { where('expires_at IS NOT NULL AND expires_at <= ?', Time.current) } + scope :expired, -> { where.not(expires_at: nil).where(arel_table[:expires_at].lteq(Time.current)) } scope :not_expired, -> { self.not(expired) } end diff --git a/app/models/concerns/group_descendant.rb b/app/models/concerns/group_descendant.rb index 224ac8930b5..de316446e14 100644 --- a/app/models/concerns/group_descendant.rb +++ b/app/models/concerns/group_descendant.rb @@ -60,10 +60,7 @@ module GroupDescendant end if parent && parent != hierarchy_top - expand_hierarchy_for_child(parent, - { parent => hierarchy }, - hierarchy_top, - preloaded) + expand_hierarchy_for_child(parent, { parent => hierarchy }, hierarchy_top, preloaded) else hierarchy end diff --git a/app/models/concerns/has_unique_internal_users.rb b/app/models/concerns/has_unique_internal_users.rb index 4d60cfa03b0..25b56f6d70f 100644 --- a/app/models/concerns/has_unique_internal_users.rb +++ b/app/models/concerns/has_unique_internal_users.rb @@ -28,7 +28,7 @@ module HasUniqueInternalUsers existing_user = uncached { scope.first } return existing_user if existing_user.present? - uniquify = Uniquify.new + uniquify = Gitlab::Utils::Uniquify.new username = uniquify.string(username) { |s| User.find_by_username(s) } diff --git a/app/models/concerns/has_user_type.rb b/app/models/concerns/has_user_type.rb index b02c95c9662..468ea26c51a 100644 --- a/app/models/concerns/has_user_type.rb +++ b/app/models/concerns/has_user_type.rb @@ -4,7 +4,8 @@ module HasUserType extend ActiveSupport::Concern USER_TYPES = { - human: nil, + human_deprecated: nil, + human: 0, support_bot: 1, alert_bot: 2, visual_review_bot: 3, @@ -14,8 +15,11 @@ module HasUserType migration_bot: 7, security_bot: 8, automation_bot: 9, + security_policy_bot: 10, # Currently not in use. See https://gitlab.com/gitlab-org/gitlab/-/issues/384174 admin_bot: 11, - suggested_reviewers_bot: 12 + suggested_reviewers_bot: 12, + service_account: 13, + llm_bot: 14 }.with_indifferent_access.freeze BOT_USER_TYPES = %w[ @@ -26,15 +30,24 @@ module HasUserType migration_bot security_bot automation_bot + security_policy_bot admin_bot suggested_reviewers_bot + service_account + llm_bot ].freeze - NON_INTERNAL_USER_TYPES = %w[human project_bot service_user].freeze + # `service_account` allows instance/namespaces to configure a user for external integrations/automations + # `service_user` is an internal, `gitlab-com`-specific user type for integrations like suggested reviewers + NON_INTERNAL_USER_TYPES = %w[human human_deprecated project_bot service_user service_account].freeze INTERNAL_USER_TYPES = (USER_TYPES.keys - NON_INTERNAL_USER_TYPES).freeze included do - scope :humans, -> { where(user_type: :human) } + enum user_type: USER_TYPES + + scope :humans, -> { where(user_type: :human).or(where(user_type: :human_deprecated)) } + # Override default scope to include temporary human type. See https://gitlab.com/gitlab-org/gitlab/-/issues/386474 + scope :human, -> { humans } scope :bots, -> { where(user_type: BOT_USER_TYPES) } scope :without_bots, -> { humans.or(where(user_type: USER_TYPES.keys - BOT_USER_TYPES)) } scope :non_internal, -> { humans.or(where(user_type: NON_INTERNAL_USER_TYPES)) } @@ -42,10 +55,8 @@ module HasUserType scope :without_project_bot, -> { humans.or(where(user_type: USER_TYPES.keys - ['project_bot'])) } scope :human_or_service_user, -> { humans.or(where(user_type: :service_user)) } - enum user_type: USER_TYPES - def human? - super || user_type.nil? + super || human_deprecated? || user_type.nil? end end @@ -53,10 +64,8 @@ module HasUserType BOT_USER_TYPES.include?(user_type) end - # The explicit check for project_bot will be removed with Bot Categorization - # Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/213945 def internal? - ghost? || (bot? && !project_bot?) + INTERNAL_USER_TYPES.include?(user_type) end def redacted_name(viewing_user) diff --git a/app/models/concerns/integrations/has_issue_tracker_fields.rb b/app/models/concerns/integrations/has_issue_tracker_fields.rb index 57f8e21c5a6..223191fb963 100644 --- a/app/models/concerns/integrations/has_issue_tracker_fields.rb +++ b/app/models/concerns/integrations/has_issue_tracker_fields.rb @@ -8,29 +8,29 @@ module Integrations self.field_storage = :data_fields field :project_url, - required: true, - title: -> { _('Project URL') }, - help: -> do - s_('IssueTracker|The URL to the project in the external issue tracker.') - end + required: true, + title: -> { _('Project URL') }, + help: -> do + s_('IssueTracker|The URL to the project in the external issue tracker.') + end field :issues_url, - required: true, - title: -> { s_('IssueTracker|Issue URL') }, - help: -> do - ERB::Util.html_escape( - s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.') - ) % { - colon_id: '<code>:id</code>'.html_safe - } - end + required: true, + title: -> { s_('IssueTracker|Issue URL') }, + help: -> do + ERB::Util.html_escape( + s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.') + ) % { + colon_id: '<code>:id</code>'.html_safe + } + end field :new_issue_url, - required: true, - title: -> { s_('IssueTracker|New issue URL') }, - help: -> do - s_('IssueTracker|The URL to create an issue in the external issue tracker.') - end + required: true, + title: -> { s_('IssueTracker|New issue URL') }, + help: -> do + s_('IssueTracker|The URL to create an issue in the external issue tracker.') + end end end end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 50696c7b5e1..b1ec6b8ba32 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -84,11 +84,11 @@ module Issuable has_one :metrics, inverse_of: model_name.singular.to_sym, autosave: true delegate :name, - :email, - :public_email, - to: :author, - allow_nil: true, - prefix: true + :email, + :public_email, + to: :author, + allow_nil: true, + prefix: true validates :author, presence: true validates :title, presence: true, length: { maximum: TITLE_LENGTH_MAX } @@ -174,6 +174,10 @@ module Issuable end end + def issuable_type + self.class.name.underscore + end + # We want to use optimistic lock for cases when only title or description are involved # http://api.rubyonrails.org/classes/ActiveRecord/Locking/Optimistic.html def locking_enabled? @@ -197,15 +201,15 @@ module Issuable end def supports_severity? - incident? + incident_type_issue? end def supports_escalation? - incident? + incident_type_issue? end - def incident? - is_a?(Issue) && super + def incident_type_issue? + is_a?(Issue) && work_item_type&.incident? end def supports_issue_type? @@ -345,8 +349,7 @@ module Issuable order_milestone_due_asc .order_labels_priority(excluded_labels: excluded_labels, extra_select_columns: [milestones_due_date]) - .reorder(milestones_due_date_with_direction.nulls_last, - highest_priority_arel_with_direction.nulls_last) + .reorder(milestones_due_date_with_direction.nulls_last, highest_priority_arel_with_direction.nulls_last) end def order_labels_priority(direction = 'ASC', excluded_labels: [], extra_select_columns: [], with_cte: false) @@ -620,8 +623,10 @@ module Issuable end def updated_tasks - Taskable.get_updated_tasks(old_content: previous_changes['description'].first, - new_content: description) + Taskable.get_updated_tasks( + old_content: previous_changes['description'].first, + new_content: description + ) end ## @@ -640,10 +645,6 @@ module Issuable false end - def ensure_metrics - self.metrics || create_metrics - end - ## # Overridden in MergeRequest # @@ -658,6 +659,10 @@ module Issuable { name: name, subject: self } end + + def supports_health_status? + false + end end Issuable.prepend_mod_with('Issuable') diff --git a/app/models/concerns/limitable.rb b/app/models/concerns/limitable.rb index 0cccb7b51a8..7ed7f65ca57 100644 --- a/app/models/concerns/limitable.rb +++ b/app/models/concerns/limitable.rb @@ -59,7 +59,10 @@ module Limitable def check_plan_limit_not_exceeded(limits, relation) return unless limits&.exceeded?(limit_name, relation) - errors.add(:base, _("Maximum number of %{name} (%{count}) exceeded") % - { name: limit_name.humanize(capitalize: false), count: limits.public_send(limit_name) }) # rubocop:disable GitlabSecurity/PublicSend + errors.add( + :base, + _("Maximum number of %{name} (%{count}) exceeded") % + { name: limit_name.humanize(capitalize: false), count: limits.public_send(limit_name) } # rubocop:disable GitlabSecurity/PublicSend + ) end end diff --git a/app/models/concerns/mentionable/reference_regexes.rb b/app/models/concerns/mentionable/reference_regexes.rb index b05beb6c764..0b6075fbeb8 100644 --- a/app/models/concerns/mentionable/reference_regexes.rb +++ b/app/models/concerns/mentionable/reference_regexes.rb @@ -5,9 +5,7 @@ module Mentionable extend Gitlab::Utils::StrongMemoize def self.reference_pattern(link_patterns, issue_pattern) - Regexp.union(link_patterns, - issue_pattern, - *other_patterns) + Regexp.union(link_patterns, issue_pattern, *other_patterns) end def self.other_patterns @@ -22,14 +20,14 @@ module Mentionable def self.default_pattern strong_memoize(:default_pattern) do issue_pattern = Issue.reference_pattern - link_patterns = Regexp.union([Issue, Commit, MergeRequest, Epic, Vulnerability].map(&:link_reference_pattern).compact) + link_patterns = Regexp.union([Issue, WorkItem, Commit, MergeRequest, Epic, Vulnerability].map(&:link_reference_pattern).compact) reference_pattern(link_patterns, issue_pattern) end end def self.external_pattern strong_memoize(:external_pattern) do - issue_pattern = Integrations::BaseIssueTracker.reference_pattern + issue_pattern = Integrations::BaseIssueTracker.base_reference_pattern link_patterns = URI::DEFAULT_PARSER.make_regexp(%w(http https)) reference_pattern(link_patterns, issue_pattern) end diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index 7addcf9e2ec..65e7f734233 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -169,6 +169,7 @@ module Noteable def expire_note_etag_cache return unless discussions_rendered_on_frontend? return unless etag_caching_enabled? + return unless project.present? Gitlab::EtagCaching::Store.new.touch(note_etag_key) end @@ -197,7 +198,7 @@ module Noteable def creatable_note_email_address(author) return unless supports_creating_notes_by_email? - project_email = project.new_issuable_address(author, self.class.name.underscore) + project_email = project&.new_issuable_address(author, base_class_name.underscore) return unless project_email project_email.sub('@', "-#{iid}@") diff --git a/app/models/concerns/packages/debian/component_file.rb b/app/models/concerns/packages/debian/component_file.rb index 77409549e85..cc7279d05f8 100644 --- a/app/models/concerns/packages/debian/component_file.rb +++ b/app/models/concerns/packages/debian/component_file.rb @@ -8,6 +8,9 @@ module Packages included do include Sortable include FileStoreMounter + include IgnorableColumns + + ignore_column :file_md5, remove_with: '16.2', remove_after: '2023-06-22' def self.container_foreign_key "#{container_type}_id".to_sym @@ -30,7 +33,6 @@ module Packages validates :file, length: { minimum: 0, allow_nil: false } validates :size, presence: true validates :file_store, presence: true - validates :file_md5, presence: true validates :file_sha256, presence: true scope :with_container, ->(container) do @@ -88,6 +90,10 @@ module Packages end end + def empty? + size == 0 + end + private def extension diff --git a/app/models/concerns/partitioned_table.rb b/app/models/concerns/partitioned_table.rb index f95f9dd8ad7..c322a736e79 100644 --- a/app/models/concerns/partitioned_table.rb +++ b/app/models/concerns/partitioned_table.rb @@ -8,7 +8,8 @@ module PartitionedTable PARTITIONING_STRATEGIES = { monthly: Gitlab::Database::Partitioning::MonthlyStrategy, - sliding_list: Gitlab::Database::Partitioning::SlidingListStrategy + sliding_list: Gitlab::Database::Partitioning::SlidingListStrategy, + ci_sliding_list: Gitlab::Database::Partitioning::CiSlidingListStrategy }.freeze def partitioned_by(partitioning_key, strategy:, **kwargs) diff --git a/app/models/concerns/protected_branch_access.rb b/app/models/concerns/protected_branch_access.rb index 58761fce952..8156090fd9c 100644 --- a/app/models/concerns/protected_branch_access.rb +++ b/app/models/concerns/protected_branch_access.rb @@ -9,10 +9,4 @@ module ProtectedBranchAccess delegate :project, to: :protected_branch end - - def check_access(user) - return false if access_level == Gitlab::Access::NO_ACCESS - - super - end end diff --git a/app/models/concerns/protected_ref_access.rb b/app/models/concerns/protected_ref_access.rb index facf0808e7a..c1c670db543 100644 --- a/app/models/concerns/protected_ref_access.rb +++ b/app/models/concerns/protected_ref_access.rb @@ -2,38 +2,45 @@ module ProtectedRefAccess extend ActiveSupport::Concern - HUMAN_ACCESS_LEVELS = { - Gitlab::Access::MAINTAINER => "Maintainers", - Gitlab::Access::DEVELOPER => "Developers + Maintainers", - Gitlab::Access::NO_ACCESS => "No one" - }.freeze class_methods do + def human_access_levels + { + Gitlab::Access::DEVELOPER => 'Developers + Maintainers', + Gitlab::Access::MAINTAINER => 'Maintainers', + Gitlab::Access::ADMIN => 'Instance admins', + Gitlab::Access::NO_ACCESS => 'No one' + }.slice(*allowed_access_levels) + end + def allowed_access_levels - [ - Gitlab::Access::MAINTAINER, + levels = [ Gitlab::Access::DEVELOPER, + Gitlab::Access::MAINTAINER, + Gitlab::Access::ADMIN, Gitlab::Access::NO_ACCESS ] + + return levels unless Gitlab.com? + + levels.excluding(Gitlab::Access::ADMIN) + end + + def humanize(access_level) + human_access_levels[access_level] end end included do scope :maintainer, -> { where(access_level: Gitlab::Access::MAINTAINER) } scope :developer, -> { where(access_level: Gitlab::Access::DEVELOPER) } - scope :by_user, -> (user) { where(user_id: user) } - scope :by_group, -> (group) { where(group_id: group) } scope :for_role, -> { where(user_id: nil, group_id: nil) } - scope :for_user, -> { where.not(user_id: nil) } - scope :for_group, -> { where.not(group_id: nil) } - validates :access_level, presence: true, if: :role?, inclusion: { - in: self.allowed_access_levels - } + validates :access_level, presence: true, if: :role?, inclusion: { in: allowed_access_levels } end def humanize - HUMAN_ACCESS_LEVELS[self.access_level] + self.class.humanize(access_level) end def type @@ -44,12 +51,28 @@ module ProtectedRefAccess type == :role end - def check_access(user) - return false unless user - return true if user.admin? + def check_access(current_user) + return false if current_user.nil? || no_access? + return current_user.admin? if admin_access? + + yield if block_given? + + user_can_access?(current_user) + end + + private + + def admin_access? + role? && access_level == ::Gitlab::Access::ADMIN + end + + def no_access? + role? && access_level == Gitlab::Access::NO_ACCESS + end - user.can?(:push_code, project) && - project.team.max_member_access(user.id) >= access_level + def user_can_access?(current_user) + current_user.can?(:push_code, project) && + project.team.max_member_access(current_user.id) >= access_level end end diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index f1d29ad5a90..460cb529715 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -33,6 +33,14 @@ module RedisCacheable clear_memoization(:cached_attributes) end + def merge_cache_attributes(values) + existing_attributes = Hash(cached_attributes) + merged_attributes = existing_attributes.merge(values.symbolize_keys) + return if merged_attributes == existing_attributes + + cache_attributes(merged_attributes) + end + private def cache_attribute_key diff --git a/app/models/concerns/referable.rb b/app/models/concerns/referable.rb index 9a17131c91c..5303d110078 100644 --- a/app/models/concerns/referable.rb +++ b/app/models/concerns/referable.rb @@ -76,7 +76,11 @@ module Referable true end - def link_reference_pattern(route, pattern) + def link_reference_pattern + raise NotImplementedError, "#{self} does not implement #{__method__}" + end + + def compose_link_reference_pattern(route, pattern) %r{ (?<url> #{Regexp.escape(Gitlab.config.gitlab.url)} diff --git a/app/models/concerns/require_email_verification.rb b/app/models/concerns/require_email_verification.rb index 5ff4f520d24..d7182778b36 100644 --- a/app/models/concerns/require_email_verification.rb +++ b/app/models/concerns/require_email_verification.rb @@ -47,6 +47,7 @@ module RequireEmailVerification def override_devise_lockable? Feature.enabled?(:require_email_verification, self) && !two_factor_enabled? && + identities.none? && Feature.disabled?(:skip_require_email_verification, self, type: :ops) end strong_memoize_attr :override_devise_lockable? diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index 141c480ea1f..45818942326 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -24,14 +24,14 @@ module ResolvableDiscussion ) delegate :potentially_resolvable?, - :noteable_id, - :noteable_type, - to: :first_note - - delegate :resolved_at, - :resolved_by, - to: :last_resolved_note, - allow_nil: true + :noteable_id, + :noteable_type, + to: :first_note + + delegate :resolved_at, + :resolved_by, + to: :last_resolved_note, + allow_nil: true end def resolved_by_push? diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 262839a3fa6..d70aad4e9ae 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -99,39 +99,11 @@ module Routable end def full_name - # We have to test for persistence as the cache key uses #updated_at - return (route&.name || build_full_name) unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops) - - # Return the name as-is if the parent is missing - return name if route.nil? && parent.nil? && name.present? - - # If the route is already preloaded, return directly, preventing an extra load - return route.name if route_loaded? && route.present? - - # Similarly, we can allow the build if the parent is loaded - return build_full_name if parent_loaded? - - Gitlab::Cache.fetch_once([cache_key, :full_name]) do - route&.name || build_full_name - end + full_attribute(:name) end def full_path - # We have to test for persistence as the cache key uses #updated_at - return (route&.path || build_full_path) unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops) - - # Return the path as-is if the parent is missing - return path if route.nil? && parent.nil? && path.present? - - # If the route is already preloaded, return directly, preventing an extra load - return route.path if route_loaded? && route.present? - - # Similarly, we can allow the build if the parent is loaded - return build_full_path if parent_loaded? - - Gitlab::Cache.fetch_once([cache_key, :full_path]) do - route&.path || build_full_path - end + full_attribute(:path) end # Overriden in the Project model @@ -163,6 +135,31 @@ module Routable private + # rubocop: disable GitlabSecurity/PublicSend + def full_attribute(attribute) + attribute_from_route_or_self = ->(attribute) do + route&.public_send(attribute) || send("build_full_#{attribute}") + end + + unless persisted? && Feature.enabled?(:cached_route_lookups, self, type: :ops) + return attribute_from_route_or_self.call(attribute) + end + + # Return the attribute as-is if the parent is missing + return public_send(attribute) if route.nil? && parent.nil? && public_send(attribute).present? + + # If the route is already preloaded, return directly, preventing an extra load + return route.public_send(attribute) if route_loaded? && route.present? && route.public_send(attribute) + + # Similarly, we can allow the build if the parent is loaded + return send("build_full_#{attribute}") if parent_loaded? + + Gitlab::Cache.fetch_once([cache_key, :"full_#{attribute}"]) do + attribute_from_route_or_self.call(attribute) + end + end + # rubocop: enable GitlabSecurity/PublicSend + def set_path_errors route_path_errors = self.errors.delete(:"route.path") route_path_errors&.each do |msg| diff --git a/app/models/concerns/subscribable.rb b/app/models/concerns/subscribable.rb index 5a10ea7a248..fe47393c554 100644 --- a/app/models/concerns/subscribable.rb +++ b/app/models/concerns/subscribable.rb @@ -27,8 +27,6 @@ module Subscribable def lazy_subscription(user, project = nil) return unless user - # handle project and group labels as well as issuable subscriptions - subscribable_type = self.class.ancestors.include?(Label) ? 'Label' : self.class.name BatchLoader.for(id: id, subscribable_type: subscribable_type, project_id: project&.id).batch do |items, loader| values = items.each_with_object({ ids: Set.new, subscribable_types: Set.new, project_ids: Set.new }) do |item, result| result[:ids] << item[:id] @@ -121,4 +119,15 @@ module Subscribable subscriptions .where(t[:project_id].eq(nil).or(t[:project_id].eq(project.try(:id)))) end + + def subscribable_type + # handle project and group labels as well as issuable subscriptions + if self.class.ancestors.include?(Label) + 'Label' + elsif self.class.ancestors.include?(Issue) + 'Issue' + else + self.class.name + end + end end diff --git a/app/models/concerns/taskable.rb b/app/models/concerns/taskable.rb index dee1c820f23..bf645e99b5e 100644 --- a/app/models/concerns/taskable.rb +++ b/app/models/concerns/taskable.rb @@ -15,19 +15,19 @@ module Taskable INCOMPLETE_PATTERN = /\[[[:space:]]\]/.freeze ITEM_PATTERN = %r{ ^ - (?:(?:>\s{0,4})*) # optional blockquote characters - ((?:\s*(?:[-+*]|(?:\d+\.)))+) # list prefix (one or more) required - task item has to be always in a list - \s+ # whitespace prefix has to be always presented for a list item - ( # checkbox + (?:(?:>\s{0,4})*) # optional blockquote characters + ((?:\s*(?:[-+*]|(?:\d+[.)])))+) # list prefix (one or more) required - task item has to be always in a list + \s+ # whitespace prefix has to be always presented for a list item + ( # checkbox #{COMPLETE_PATTERN}|#{INCOMPLETE_PATTERN} ) - (\s.+) # followed by whitespace and some text. + (\s.+) # followed by whitespace and some text. }x.freeze ITEM_PATTERN_UNTRUSTED = '^' \ '(?:(?:>\s{0,4})*)' \ - '(?P<prefix>(?:\s*(?:[-+*]|(?:\d+\.)))+)' \ + '(?P<prefix>(?:\s*(?:[-+*]|(?:\d+[.)])))+)' \ '\s+' \ '(?P<checkbox>' \ "#{COMPLETE_PATTERN.source}|#{INCOMPLETE_PATTERN.source}" \ diff --git a/app/models/concerns/token_authenticatable_strategies/base.rb b/app/models/concerns/token_authenticatable_strategies/base.rb index 2b677f37c89..d0085b60d98 100644 --- a/app/models/concerns/token_authenticatable_strategies/base.rb +++ b/app/models/concerns/token_authenticatable_strategies/base.rb @@ -31,9 +31,13 @@ module TokenAuthenticatableStrategies result end - # Default implementation returns the token as-is + # If a `format_with_prefix` option is provided, it applies and returns the formatted token. + # Otherwise, default implementation returns the token as-is def format_token(instance, token) - instance.send("format_#{@token_field}", token) # rubocop:disable GitlabSecurity/PublicSend + prefix = prefix_for(instance) + prefixed_token = prefix ? "#{prefix}#{token}" : token + + instance.send("format_#{@token_field}", prefixed_token) # rubocop:disable GitlabSecurity/PublicSend end def ensure_token(instance) @@ -88,6 +92,17 @@ module TokenAuthenticatableStrategies protected + def prefix_for(instance) + case prefix_option = options[:format_with_prefix] + when nil + nil + when Symbol + instance.send(prefix_option) # rubocop:disable GitlabSecurity/PublicSend + else + raise NotImplementedError + end + end + def write_new_token(instance) new_token = generate_available_token formatted_token = format_token(instance, new_token) diff --git a/app/models/concerns/token_authenticatable_strategies/encrypted.rb b/app/models/concerns/token_authenticatable_strategies/encrypted.rb index 1db88c27181..4b3b80437db 100644 --- a/app/models/concerns/token_authenticatable_strategies/encrypted.rb +++ b/app/models/concerns/token_authenticatable_strategies/encrypted.rb @@ -106,11 +106,7 @@ module TokenAuthenticatableStrategies end def matches_prefix?(instance, token) - prefix = options[:prefix] - prefix = prefix.call(instance) if prefix.is_a?(Proc) - prefix = '' unless prefix.is_a?(String) - - token.start_with?(prefix) + !options[:require_prefix_for_validation] || token.start_with?(prefix_for(instance)) end def token_set?(instance) diff --git a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb index 447521ad8c1..5e77dfde397 100644 --- a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb +++ b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb @@ -20,8 +20,6 @@ module TokenAuthenticatableStrategies end def self.encrypt_token(plaintext_token) - return Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token) unless Feature.enabled?(:dynamic_nonce, type: :ops) - iv = ::Digest::SHA256.hexdigest(plaintext_token).bytes.take(NONCE_SIZE).pack('c*') token = Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token, nonce: iv) "#{DYNAMIC_NONCE_IDENTIFIER}#{token}#{iv}" diff --git a/app/models/concerns/uniquify.rb b/app/models/concerns/uniquify.rb deleted file mode 100644 index 382e826ec58..00000000000 --- a/app/models/concerns/uniquify.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -# Uniquify -# -# Return a version of the given 'base' string that is unique -# by appending a counter to it. Uniqueness is determined by -# repeated calls to the passed block. -# -# You can pass an initial value for the counter, if not given -# counting starts from 1. -# -# If `base` is a function/proc, we expect that calling it with a -# candidate counter returns a string to test/return. -class Uniquify - def initialize(counter = nil) - @counter = counter - end - - def string(base) - @base = base - - increment_counter! while yield(base_string) - base_string - end - - private - - def base_string - if @base.respond_to?(:call) - @base.call(@counter) - else - "#{@base}#{@counter}" - end - end - - def increment_counter! - @counter ||= 0 - @counter += 1 - end -end diff --git a/app/models/concerns/vulnerability_finding_helpers.rb b/app/models/concerns/vulnerability_finding_helpers.rb index 1e8a290c050..a5b69997900 100644 --- a/app/models/concerns/vulnerability_finding_helpers.rb +++ b/app/models/concerns/vulnerability_finding_helpers.rb @@ -47,8 +47,9 @@ module VulnerabilityFindingHelpers report_finding = report_finding_for(security_finding) return Vulnerabilities::Finding.new unless report_finding - finding_data = report_finding.to_hash.except(:compare_key, :identifiers, :location, :scanner, :links, :signatures, - :flags, :evidence) + finding_data = report_finding.to_hash.except( + :compare_key, :identifiers, :location, :scanner, :links, :signatures, :flags, :evidence + ) identifiers = report_finding.identifiers.uniq(&:fingerprint).map do |identifier| Vulnerabilities::Identifier.new(identifier.to_hash.merge({ project: project })) end diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index 2cc17a6f185..2ad2e47ec4e 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -4,7 +4,32 @@ module WebHooks module AutoDisabling extend ActiveSupport::Concern + ENABLED_HOOK_TYPES = %w[ProjectHook].freeze + MAX_FAILURES = 100 + FAILURE_THRESHOLD = 3 + EXCEEDED_FAILURE_THRESHOLD = FAILURE_THRESHOLD + 1 + INITIAL_BACKOFF = 1.minute.freeze + MAX_BACKOFF = 1.day.freeze + BACKOFF_GROWTH_FACTOR = 2.0 + + class_methods do + def auto_disabling_enabled? + enabled_hook_types.include?(name) && + Gitlab::SafeRequestStore.fetch(:auto_disabling_web_hooks) do + Feature.enabled?(:auto_disabling_web_hooks, type: :ops) + end + end + + private + + def enabled_hook_types + ENABLED_HOOK_TYPES + end + end + included do + delegate :auto_disabling_enabled?, to: :class, private: true + # A hook is disabled if: # # - we are no longer in the grace-perod (recent_failures > ?) @@ -12,8 +37,13 @@ module WebHooks # - disabled_until is nil (i.e. this was set by WebHook#fail!) # - or disabled_until is in the future (i.e. this was set by WebHook#backoff!) scope :disabled, -> do - where('recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)', - WebHook::FAILURE_THRESHOLD, Time.current) + return none unless auto_disabling_enabled? + + where( + 'recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)', + FAILURE_THRESHOLD, + Time.current + ) end # A hook is executable if: @@ -23,40 +53,85 @@ module WebHooks # - disabled_until is nil (i.e. this was set by WebHook#fail!) # - disabled_until is in the future (i.e. this was set by WebHook#backoff!) scope :executable, -> do - where('recent_failures <= ? OR (recent_failures > ? AND (disabled_until IS NOT NULL) AND (disabled_until < ?))', - WebHook::FAILURE_THRESHOLD, WebHook::FAILURE_THRESHOLD, Time.current) + return all unless auto_disabling_enabled? + + where( + 'recent_failures <= ? OR (recent_failures > ? AND (disabled_until IS NOT NULL) AND (disabled_until < ?))', + FAILURE_THRESHOLD, + FAILURE_THRESHOLD, + Time.current + ) end end def executable? + return true unless auto_disabling_enabled? + !temporarily_disabled? && !permanently_disabled? end def temporarily_disabled? - return false if recent_failures <= WebHook::FAILURE_THRESHOLD + return false unless auto_disabling_enabled? - disabled_until.present? && disabled_until >= Time.current + disabled_until.present? && disabled_until >= Time.current && recent_failures > FAILURE_THRESHOLD end def permanently_disabled? - return false if disabled_until.present? + return false unless auto_disabling_enabled? - recent_failures > WebHook::FAILURE_THRESHOLD + recent_failures > FAILURE_THRESHOLD && disabled_until.blank? end def disable! - return if permanently_disabled? + return if !auto_disabling_enabled? || permanently_disabled? - super + update_attribute(:recent_failures, EXCEEDED_FAILURE_THRESHOLD) end + def enable! + return unless auto_disabling_enabled? + return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0 + + assign_attributes(recent_failures: 0, disabled_until: nil, backoff_count: 0) + save(validate: false) + end + + # Don't actually back-off until FAILURE_THRESHOLD failures have been seen + # we mark the grace-period using the recent_failures counter def backoff! - return if permanently_disabled? || (backoff_count >= WebHook::MAX_FAILURES && temporarily_disabled?) + return unless auto_disabling_enabled? + return if permanently_disabled? || (backoff_count >= MAX_FAILURES && temporarily_disabled?) + + attrs = { recent_failures: next_failure_count } - super + if recent_failures >= FAILURE_THRESHOLD + attrs[:backoff_count] = next_backoff_count + attrs[:disabled_until] = next_backoff.from_now + end + + assign_attributes(attrs) + save(validate: false) if changed? + end + + def failed! + return unless auto_disabling_enabled? + return unless recent_failures < MAX_FAILURES + + assign_attributes(disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count) + save(validate: false) + end + + def next_backoff + return MAX_BACKOFF if backoff_count >= 8 # optimization to prevent expensive exponentiation and possible overflows + + (INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**backoff_count)) + .clamp(INITIAL_BACKOFF, MAX_BACKOFF) + .seconds end def alert_status + return :executable unless auto_disabling_enabled? + if temporarily_disabled? :temporarily_disabled elsif permanently_disabled? @@ -65,5 +140,18 @@ module WebHooks :executable end end + + private + + def next_failure_count + recent_failures.succ.clamp(1, MAX_FAILURES) + end + + def next_backoff_count + backoff_count.succ.clamp(1, MAX_FAILURES) + end end end + +WebHooks::AutoDisabling.prepend_mod +WebHooks::AutoDisabling::ClassMethods.prepend_mod diff --git a/app/models/concerns/web_hooks/has_web_hooks.rb b/app/models/concerns/web_hooks/has_web_hooks.rb index 161ce106b9b..2183cc3c44b 100644 --- a/app/models/concerns/web_hooks/has_web_hooks.rb +++ b/app/models/concerns/web_hooks/has_web_hooks.rb @@ -2,8 +2,6 @@ module WebHooks module HasWebHooks - extend ActiveSupport::Concern - WEB_HOOK_CACHE_EXPIRY = 1.hour def any_hook_failed? @@ -15,7 +13,7 @@ module WebHooks end def last_failure_redis_key - "web_hooks:last_failure:project-#{id}" + "web_hooks:last_failure:#{self.class.name.underscore}-#{id}" end def get_web_hook_failure @@ -42,5 +40,13 @@ module WebHooks state end end + + def last_webhook_failure + last_failure = Gitlab::Redis::SharedState.with do |redis| + redis.get(last_failure_redis_key) + end + + DateTime.parse(last_failure) if last_failure + end end end diff --git a/app/models/concerns/web_hooks/unstoppable.rb b/app/models/concerns/web_hooks/unstoppable.rb deleted file mode 100644 index 26284fe3c36..00000000000 --- a/app/models/concerns/web_hooks/unstoppable.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module WebHooks - module Unstoppable - extend ActiveSupport::Concern - - included do - scope :executable, -> { all } - - scope :disabled, -> { none } - end - - def executable? - true - end - - def temporarily_disabled? - false - end - - def permanently_disabled? - false - end - - def alert_status - :executable - end - end -end diff --git a/app/models/concerns/with_uploads.rb b/app/models/concerns/with_uploads.rb index d90f32d8b1c..caaf2b33ef0 100644 --- a/app/models/concerns/with_uploads.rb +++ b/app/models/concerns/with_uploads.rb @@ -25,6 +25,13 @@ module WithUploads FILE_UPLOADERS = %w(PersonalFileUploader NamespaceFileUploader FileUploader).freeze included do + around_destroy :ignore_uploads_table_in_transaction + + def ignore_uploads_table_in_transaction(&blk) + Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.temporary_ignore_tables_in_transaction( + %w[uploads], url: "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", &blk) + end + has_many :uploads, as: :model has_many :file_uploads, -> { where(uploader: FILE_UPLOADERS) }, class_name: 'Upload', as: :model, diff --git a/app/models/container_registry/data_repair_detail.rb b/app/models/container_registry/data_repair_detail.rb new file mode 100644 index 00000000000..a2616490905 --- /dev/null +++ b/app/models/container_registry/data_repair_detail.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module ContainerRegistry + class DataRepairDetail < ApplicationRecord + include EachBatch + + self.table_name = 'container_registry_data_repair_details' + self.primary_key = :project_id + + belongs_to :project, optional: false + + enum status: { ongoing: 0, completed: 1, failed: 2 } + + scope :ongoing_since, ->(threshold) { where(status: :ongoing).where('updated_at < ?', threshold) } + end +end diff --git a/app/models/container_registry/event.rb b/app/models/container_registry/event.rb index c4d06be8841..dd2675e17d8 100644 --- a/app/models/container_registry/event.rb +++ b/app/models/container_registry/event.rb @@ -8,7 +8,7 @@ module ContainerRegistry PUSH_ACTION = 'push' DELETE_ACTION = 'delete' EVENT_TRACKING_CATEGORY = 'container_registry:notification' - EVENT_PREFIX = "i_container_registry" + EVENT_PREFIX = 'i_container_registry' ALLOWED_ACTOR_TYPES = %w( personal_access_token @@ -48,8 +48,12 @@ module ContainerRegistry ::Gitlab::Tracking.event(EVENT_TRACKING_CATEGORY, tracking_action) - event = usage_data_event_for(tracking_action) - ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: originator.id) if event + if manifest_delete_event? + ::Gitlab::UsageDataCounters::ContainerRegistryEventCounter.count("#{EVENT_PREFIX}_delete_manifest") + else + event = usage_data_event_for(tracking_action) + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: originator.id) if event + end end private @@ -122,9 +126,13 @@ module ContainerRegistry end end + def manifest_delete_event? + action_delete? && target_digest? + end + def update_project_statistics return unless supported? - return unless target_tag? || (action_delete? && target_digest?) + return unless target_tag? || manifest_delete_event? return unless project Rails.cache.delete(project.root_ancestor.container_repositories_size_cache_key) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 98ce981ad8e..0f0abeae795 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -22,6 +22,12 @@ class ContainerRepository < ApplicationRecord MAX_TAGS_PAGES = 2000 + # The Registry client uses JWT token to authenticate to Registry. We cache the client using expiration + # time of JWT token. However it's possible that the token is valid but by the time the request is made to + # Regsitry, it's already expired. To prevent this case, we are subtracting a few seconds, defined by this constant + # from the cache expiration time. + AUTH_TOKEN_USAGE_RESERVED_TIME_IN_SECS = 5 + TooManyImportsError = Class.new(StandardError) belongs_to :project @@ -32,8 +38,8 @@ class ContainerRepository < ApplicationRecord validates :migration_aborted_in_state, inclusion: { in: ABORTABLE_MIGRATION_STATES }, allow_nil: true validates :migration_retries_count, presence: true, - numericality: { greater_than_or_equal_to: 0 }, - allow_nil: false + numericality: { greater_than_or_equal_to: 0 }, + allow_nil: false enum status: { delete_scheduled: 0, delete_failed: 1, delete_ongoing: 2 } enum expiration_policy_cleanup_status: { cleanup_unscheduled: 0, cleanup_scheduled: 1, cleanup_unfinished: 2, cleanup_ongoing: 3 } @@ -69,7 +75,7 @@ class ContainerRepository < ApplicationRecord scope :with_migration_import_started_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_import_started_at, '01-01-1970') < ?", timestamp) } scope :with_migration_pre_import_started_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_pre_import_started_at, '01-01-1970') < ?", timestamp) } scope :with_migration_pre_import_done_at_nil_or_before, ->(timestamp) { where("COALESCE(migration_pre_import_done_at, '01-01-1970') < ?", timestamp) } - scope :with_stale_ongoing_cleanup, ->(threshold) { cleanup_ongoing.where('expiration_policy_started_at < ?', threshold) } + scope :with_stale_ongoing_cleanup, ->(threshold) { cleanup_ongoing.expiration_policy_started_at_nil_or_before(threshold) } scope :with_stale_delete_at, ->(threshold) { where('delete_started_at < ?', threshold) } scope :import_in_process, -> { where(migration_state: %w[pre_importing pre_import_done importing]) } @@ -118,9 +124,7 @@ class ContainerRepository < ApplicationRecord state :import_done state :import_skipped do - validates :migration_skipped_reason, - :migration_skipped_at, - presence: true + validates :migration_skipped_reason, :migration_skipped_at, presence: true end state :import_aborted do @@ -289,6 +293,10 @@ class ContainerRepository < ApplicationRecord all end + def self.registry_client_expiration_time + (Gitlab::CurrentSettings.container_registry_token_expire_delay * 60) - AUTH_TOKEN_USAGE_RESERVED_TIME_IN_SECS + end + class << self alias_method :pending_destruction, :delete_scheduled # needed by Packages::Destructible end @@ -395,7 +403,7 @@ class ContainerRepository < ApplicationRecord end def migrated? - (self.created_at && MIGRATION_PHASE_1_ENDED_AT < self.created_at) || import_done? + Gitlab.com? end def last_import_step_done_at @@ -410,7 +418,7 @@ class ContainerRepository < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def registry - @registry ||= begin + strong_memoize_with_expiration(:registry, self.class.registry_client_expiration_time) do token = Auth::ContainerRegistryAuthenticationService.full_access_token(path) url = Gitlab.config.registry.api_url @@ -509,7 +517,11 @@ class ContainerRepository < ApplicationRecord end def start_expiration_policy! - update!(expiration_policy_started_at: Time.zone.now, last_cleanup_deleted_tags_count: nil) + update!( + expiration_policy_started_at: Time.zone.now, + last_cleanup_deleted_tags_count: nil, + expiration_policy_cleanup_status: :cleanup_ongoing + ) end def size @@ -589,8 +601,7 @@ class ContainerRepository < ApplicationRecord end def self.build_from_path(path) - self.new(project: path.repository_project, - name: path.repository_name) + self.new(project: path.repository_project, name: path.repository_name) end def self.find_or_create_from_path(path) @@ -608,13 +619,11 @@ class ContainerRepository < ApplicationRecord end def self.find_by_path!(path) - self.find_by!(project: path.repository_project, - name: path.repository_name) + self.find_by!(project: path.repository_project, name: path.repository_name) end def self.find_by_path(path) - self.find_by(project: path.repository_project, - name: path.repository_name) + self.find_by(project: path.repository_project, name: path.repository_name) end private diff --git a/app/models/cycle_analytics/project_level_stage_adapter.rb b/app/models/cycle_analytics/project_level_stage_adapter.rb index 9b9c0822f63..ae21a4a6bfe 100644 --- a/app/models/cycle_analytics/project_level_stage_adapter.rb +++ b/app/models/cycle_analytics/project_level_stage_adapter.rb @@ -16,12 +16,12 @@ module CycleAnalytics presenter = Analytics::CycleAnalytics::StagePresenter.new(stage) serializer.new.represent(ProjectLevelStage.new( - title: presenter.title, - description: presenter.description, - legend: presenter.legend, - name: stage.name, - project_median: median - )) + title: presenter.title, + description: presenter.description, + legend: presenter.legend, + name: stage.name, + project_median: median + )) end # rubocop: enable CodeReuse/Presenter diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index 5ad746e4cd1..11fe0503f50 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -12,6 +12,11 @@ class DependencyProxy::Manifest < ApplicationRecord MAX_FILE_SIZE = 10.megabytes.freeze DIGEST_HEADER = 'Docker-Content-Digest' + ACCEPTED_TYPES = [ + ContainerRegistry::BaseClient::DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, + ContainerRegistry::BaseClient::OCI_MANIFEST_V1_TYPE, + ContainerRegistry::BaseClient::OCI_DISTRIBUTION_INDEX_TYPE + ].freeze validates :group, presence: true validates :file, presence: true diff --git a/app/models/dependency_proxy/registry.rb b/app/models/dependency_proxy/registry.rb index 6492acf325a..3073dd59c7b 100644 --- a/app/models/dependency_proxy/registry.rb +++ b/app/models/dependency_proxy/registry.rb @@ -33,3 +33,5 @@ class DependencyProxy::Registry end end end + +::DependencyProxy::Registry.prepend_mod diff --git a/app/models/deployment.rb b/app/models/deployment.rb index f8873d388a3..f3ee21ea4e0 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -372,9 +372,11 @@ class Deployment < ApplicationRecord # i.e.: # MergeRequest.select(1, 2).to_sql #=> SELECT 1, 2 FROM "merge_requests" # MergeRequest.select(1, 1).to_sql #=> SELECT 1 FROM "merge_requests" - select = relation.select('merge_requests.id', - "#{id} as deployment_id", - "#{environment_id} as environment_id").to_sql + select = relation.select( + 'merge_requests.id', + "#{id} as deployment_id", + "#{environment_id} as environment_id" + ).to_sql # We don't use `ApplicationRecord.legacy_bulk_insert` here so that we don't need to # first pluck lots of IDs into memory. diff --git a/app/models/design_management/design.rb b/app/models/design_management/design.rb index 317399e780a..505935bb230 100644 --- a/app/models/design_management/design.rb +++ b/app/models/design_management/design.rb @@ -13,6 +13,9 @@ module DesignManagement include RelativePositioning include Todoable include Participable + include CacheMarkdownField + + cache_markdown_field :description belongs_to :project, inverse_of: :designs belongs_to :issue @@ -28,12 +31,13 @@ module DesignManagement has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_internal_id :iid, scope: :project, presence: true, - hook_names: %i[create update], # Deal with old records - track_if: -> { !importing? } + hook_names: %i[create update], # Deal with old records + track_if: -> { !importing? } validates :project, :filename, presence: true validates :issue, presence: true, unless: :importing? validates :filename, uniqueness: { scope: :issue_id }, length: { maximum: 255 } + validates :description, length: { maximum: Gitlab::Database::MAX_TEXT_SIZE_LIMIT } validate :validate_file_is_image alias_attribute :title, :filename @@ -43,7 +47,7 @@ module DesignManagement # Pre-fetching scope to include the data necessary to construct a # reference using `to_reference`. - scope :for_reference, -> { includes(issue: [{ project: [:route, :namespace] }]) } + scope :for_reference, -> { includes(issue: [{ namespace: :project }, { project: [:route, :namespace] }]) } # A design can be uniquely identified by issue_id and filename # Takes one or more sets of composite IDs of the form: @@ -174,7 +178,7 @@ module DesignManagement (?<url_filename> #{valid_char}+ \. #{ext}) }x - super(path_segment, filename_pattern) + compose_link_reference_pattern(path_segment, filename_pattern) end end @@ -182,10 +186,6 @@ module DesignManagement File.join(DesignManagement.designs_directory, "issue-#{issue.iid}", design.filename) end - def description - '' - end - def new_design? strong_memoize(:new_design) { actions.none? } end diff --git a/app/models/design_management/git_repository.rb b/app/models/design_management/git_repository.rb new file mode 100644 index 00000000000..38c457c7991 --- /dev/null +++ b/app/models/design_management/git_repository.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module DesignManagement + class GitRepository < ::Repository + extend ::Gitlab::Utils::Override + + # We define static git attributes for the design repository as this + # repository is entirely GitLab-managed rather than user-facing. + # + # Enable all uploaded files to be stored in LFS. + MANAGED_GIT_ATTRIBUTES = <<~GA.freeze + /#{DesignManagement.designs_directory}/* filter=lfs diff=lfs merge=lfs -text + GA + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def info_attributes + @info_attributes ||= Gitlab::Git::AttributesParser.new(MANAGED_GIT_ATTRIBUTES) + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def attributes(path) + info_attributes.attributes(path) + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def gitattribute(path, name) + attributes(path)[name] + end + + # Override of a method called on Repository instances but sent via + # method_missing to Gitlab::Git::Repository where it is defined + def attributes_at(_ref = nil) + info_attributes + end + + override :copy_gitattributes + def copy_gitattributes(_ref = nil) + true + end + end +end diff --git a/app/models/design_management/repository.rb b/app/models/design_management/repository.rb index 2b1e6070e6b..33c5dc15fa4 100644 --- a/app/models/design_management/repository.rb +++ b/app/models/design_management/repository.rb @@ -1,51 +1,36 @@ # frozen_string_literal: true module DesignManagement - class Repository < ::Repository - extend ::Gitlab::Utils::Override - - # We define static git attributes for the design repository as this - # repository is entirely GitLab-managed rather than user-facing. - # - # Enable all uploaded files to be stored in LFS. - MANAGED_GIT_ATTRIBUTES = <<~GA - /#{DesignManagement.designs_directory}/* filter=lfs diff=lfs merge=lfs -text - GA - - def initialize(project) - full_path = project.full_path + Gitlab::GlRepository::DESIGN.path_suffix - disk_path = project.disk_path + Gitlab::GlRepository::DESIGN.path_suffix - - super(full_path, project, shard: project.repository_storage, disk_path: disk_path, repo_type: Gitlab::GlRepository::DESIGN) - end - - # Override of a method called on Repository instances but sent via - # method_missing to Gitlab::Git::Repository where it is defined - def info_attributes - @info_attributes ||= Gitlab::Git::AttributesParser.new(MANAGED_GIT_ATTRIBUTES) - end - - # Override of a method called on Repository instances but sent via - # method_missing to Gitlab::Git::Repository where it is defined - def attributes(path) - info_attributes.attributes(path) + class Repository < ApplicationRecord + include ::Gitlab::Utils::StrongMemoize + include HasRepository + + belongs_to :project, inverse_of: :design_management_repository + validates :project, presence: true, uniqueness: true + + delegate :lfs_enabled?, :storage, :repository_storage, to: :project + + def repository + ::DesignManagement::GitRepository.new( + full_path, + self, + shard: repository_storage, + disk_path: disk_path, + repo_type: repo_type + ) end + strong_memoize_attr :repository - # Override of a method called on Repository instances but sent via - # method_missing to Gitlab::Git::Repository where it is defined - def gitattribute(path, name) - attributes(path)[name] + def full_path + project.full_path + repo_type.path_suffix end - # Override of a method called on Repository instances but sent via - # method_missing to Gitlab::Git::Repository where it is defined - def attributes_at(_ref = nil) - info_attributes + def disk_path + project.disk_path + repo_type.path_suffix end - override :copy_gitattributes - def copy_gitattributes(_ref = nil) - true + def repo_type + Gitlab::GlRepository::DESIGN end end end diff --git a/app/models/design_management/version.rb b/app/models/design_management/version.rb index 5819404efb9..dd6812f0eac 100644 --- a/app/models/design_management/version.rb +++ b/app/models/design_management/version.rb @@ -36,10 +36,10 @@ module DesignManagement belongs_to :author, class_name: 'User' has_many :actions has_many :designs, - through: :actions, - class_name: "DesignManagement::Design", - source: :design, - inverse_of: :versions + through: :actions, + class_name: "DesignManagement::Design", + source: :design, + inverse_of: :versions validates :designs, presence: true, unless: :importing? validates :sha, presence: true diff --git a/app/models/diff_discussion.rb b/app/models/diff_discussion.rb index 041ec98ffc9..e2ee951522d 100644 --- a/app/models/diff_discussion.rb +++ b/app/models/diff_discussion.rb @@ -10,13 +10,13 @@ class DiffDiscussion < Discussion DiffNote end - delegate :position, - :original_position, - :change_position, - :diff_note_positions, - :on_text?, - :on_image?, - to: :first_note + delegate :position, + :original_position, + :change_position, + :diff_note_positions, + :on_text?, + :on_image?, + to: :first_note def legacy_diff_discussion? false diff --git a/app/models/diff_viewer/base.rb b/app/models/diff_viewer/base.rb index 75aa51348c8..05552e83700 100644 --- a/app/models/diff_viewer/base.rb +++ b/app/models/diff_viewer/base.rb @@ -101,8 +101,9 @@ module DiffViewer def render_error_options options = [] - blob_url = Gitlab::Routing.url_helpers.project_blob_path(diff_file.repository.project, - File.join(diff_file.content_sha, diff_file.file_path)) + blob_url = Gitlab::Routing.url_helpers.project_blob_path( + diff_file.repository.project, File.join(diff_file.content_sha, diff_file.file_path) + ) options << ActionController::Base.helpers.link_to(_('view the blob'), blob_url) options diff --git a/app/models/draft_note.rb b/app/models/draft_note.rb index 9f7977fce68..ffc04f9bf90 100644 --- a/app/models/draft_note.rb +++ b/app/models/draft_note.rb @@ -108,7 +108,7 @@ class DraftNote < ApplicationRecord end def self.preload_author(draft_notes) - ActiveRecord::Associations::Preloader.new.preload(draft_notes, { author: :status }) + ActiveRecord::Associations::Preloader.new(records: draft_notes, associations: { author: :status }).call end def diff_file diff --git a/app/models/environment_status.rb b/app/models/environment_status.rb index d06d0a99948..7687bc2be60 100644 --- a/app/models/environment_status.rb +++ b/app/models/environment_status.rb @@ -9,6 +9,7 @@ class EnvironmentStatus delegate :name, to: :environment delegate :status, to: :deployment, allow_nil: true delegate :deployed_at, to: :deployment, allow_nil: true + delegate :deployable, to: :deployment, allow_nil: true def self.for_merge_request(mr, user) build_environments_status(mr, user, mr.actual_head_pipeline) @@ -100,11 +101,14 @@ class EnvironmentStatus def self.build_environments_status(mr, user, pipeline) return [] unless pipeline - pipeline.environments_in_self_and_project_descendants.includes(:project).available.map do |environment| + environments = pipeline.environments_in_self_and_project_descendants.includes(:project) + environments = environments.available if Feature.disabled?(:review_apps_redeploy_mr_widget, mr.project) + environments.map do |environment| next unless Ability.allowed?(user, :read_environment, environment) EnvironmentStatus.new(pipeline.project, environment, mr, pipeline.sha) end.compact end + private_class_method :build_environments_status end diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb index 1c7a8d93e6e..c52f8a58c00 100644 --- a/app/models/error_tracking/project_error_tracking_setting.rb +++ b/app/models/error_tracking/project_error_tracking_setting.rb @@ -145,7 +145,7 @@ module ErrorTracking ensure_issue_belongs_to_project!(issue_to_be_updated.project_id) handle_exceptions do - { updated: sentry_client.update_issue(opts) } + { updated: sentry_client.update_issue(**opts) } end end diff --git a/app/models/event.rb b/app/models/event.rb index 333841b1f90..9345776c32b 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -9,6 +9,9 @@ class Event < ApplicationRecord include Gitlab::Utils::StrongMemoize include UsageStatistics include ShaAttribute + include IgnorableColumns + + ignore_column :target_id_convert_to_bigint, remove_with: '16.2', remove_after: '2023-07-22' ACTIONS = HashWithIndifferentAccess.new( created: 1, @@ -66,7 +69,7 @@ class Event < ApplicationRecord # If the association for "target" defines an "author" association we want to # eager-load this so Banzai & friends don't end up performing N+1 queries to # get the authors of notes, issues, etc. (likewise for "noteable"). - incs = %i(author noteable).select do |a| + incs = %i(author noteable work_item_type).select do |a| reflections['events'].active_record.reflect_on_association(a) end diff --git a/app/models/external_pull_request.rb b/app/models/external_pull_request.rb index 4654f7e2341..94c242782c1 100644 --- a/app/models/external_pull_request.rb +++ b/app/models/external_pull_request.rb @@ -14,6 +14,7 @@ class ExternalPullRequest < Ci::ApplicationRecord include Gitlab::Utils::StrongMemoize include ShaAttribute + include EachBatch belongs_to :project diff --git a/app/models/group.rb b/app/models/group.rb index 7e09280dfff..ab8e0101684 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -7,7 +7,6 @@ class Group < Namespace include AfterCommitQueue include AccessRequestable include Avatarable - include Referable include SelectForProjectAuthorization include LoadedInGroupList include GroupDescendant @@ -21,7 +20,6 @@ class Group < Namespace include ChronicDurationAttribute include RunnerTokenExpirationInterval include Todoable - include IssueParent extend ::Gitlab::Utils::Override @@ -111,6 +109,7 @@ class Group < Namespace has_one :import_state, class_name: 'GroupImportState', inverse_of: :group has_many :bulk_import_exports, class_name: 'BulkImports::Export', inverse_of: :group + has_many :bulk_import_entities, class_name: 'BulkImports::Entity', foreign_key: :namespace_id, inverse_of: :group has_many :group_deploy_keys_groups, inverse_of: :group has_many :group_deploy_keys, through: :group_deploy_keys_groups @@ -162,7 +161,8 @@ class Group < Namespace add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:groups_tokens_optional_encryption) ? :optional : :required }, - prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + format_with_prefix: :runners_token_prefix, + require_prefix_for_validation: true after_create :post_create_hook after_create -> { create_or_load_association(:group_feature) } @@ -198,14 +198,27 @@ class Group < Namespace .where(project_authorizations: { user_id: user_ids }) end + scope :with_project_creation_levels, -> (project_creation_levels) do + where(project_creation_level: project_creation_levels) + end + scope :project_creation_allowed, -> do - permitted_levels = [ + project_creation_allowed_on_levels = [ ::Gitlab::Access::DEVELOPER_MAINTAINER_PROJECT_ACCESS, ::Gitlab::Access::MAINTAINER_PROJECT_ACCESS, nil ] - where(project_creation_level: permitted_levels) + # When the value of application_settings.default_project_creation is set to `NO_ONE_PROJECT_ACCESS`, + # it means that a `nil` value for `groups.project_creation_level` is telling us: + # do not allow project creation in such groups. + # ie, `nil` is a placeholder value for inheriting the value from the ApplicationSetting. + # So we remove `nil` from the list when the application_setting's value is `NO_ONE_PROJECT_ACCESS` + if ::Gitlab::CurrentSettings.default_project_creation == ::Gitlab::Access::NO_ONE_PROJECT_ACCESS + project_creation_allowed_on_levels.delete(nil) + end + + with_project_creation_levels(project_creation_allowed_on_levels) end scope :shared_into_ancestors, -> (group) do @@ -240,14 +253,6 @@ class Group < Namespace end end - def reference_prefix - User.reference_prefix - end - - def reference_pattern - User.reference_pattern - end - # WARNING: This method should never be used on its own # please do make sure the number of rows you are filtering is small # enough for this query @@ -364,10 +369,6 @@ class Group < Namespace notification_settings.find { |n| n.notification_email.present? }&.notification_email end - def to_reference(_from = nil, target_project: nil, full: nil) - "#{self.class.reference_prefix}#{full_path}" - end - def web_url(only_path: nil) Gitlab::UrlBuilder.build(self, only_path: only_path) end @@ -561,7 +562,7 @@ class Group < Namespace # rubocop: enable CodeReuse/ServiceClass def users_ids_of_direct_members - direct_members.pluck(:user_id) + direct_members.pluck_user_ids end def user_ids_for_project_authorizations @@ -762,11 +763,6 @@ class Group < Namespace ensure_runners_token! end - override :format_runners_token - def format_runners_token(token) - "#{RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX}#{token}" - end - def project_creation_level super || ::Gitlab::CurrentSettings.default_project_creation end @@ -814,8 +810,10 @@ class Group < Namespace end def preload_shared_group_links - preloader = ActiveRecord::Associations::Preloader.new - preloader.preload(self, shared_with_group_links: [shared_with_group: :route]) + ActiveRecord::Associations::Preloader.new( + records: [self], + associations: { shared_with_group_links: [shared_with_group: :route] } + ).call end def update_shared_runners_setting!(state) @@ -907,6 +905,10 @@ class Group < Namespace ].compact.min end + def content_editor_on_issues_feature_flag_enabled? + feature_flag_enabled_for_self_or_ancestor?(:content_editor_on_issues) + end + def work_items_feature_flag_enabled? feature_flag_enabled_for_self_or_ancestor?(:work_items) end @@ -1095,6 +1097,10 @@ class Group < Namespace def enable_shared_runners! update!(shared_runners_enabled: true) end + + def runners_token_prefix + RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + end end Group.prepend_mod_with('Group') diff --git a/app/models/group_group_link.rb b/app/models/group_group_link.rb index 15949570f9c..fdb8fb9ed75 100644 --- a/app/models/group_group_link.rb +++ b/app/models/group_group_link.rb @@ -19,6 +19,14 @@ class GroupGroupLink < ApplicationRecord where(group_access: [Gitlab::Access::OWNER, Gitlab::Access::MAINTAINER]) end + scope :with_developer_maintainer_owner_access, -> do + where(group_access: [Gitlab::Access::DEVELOPER, Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER]) + end + + scope :with_developer_access, -> do + where(group_access: [Gitlab::Access::DEVELOPER]) + end + scope :with_owner_access, -> do where(group_access: [Gitlab::Access::OWNER]) end diff --git a/app/models/group_label.rb b/app/models/group_label.rb index 0d2eb524929..46e56166951 100644 --- a/app/models/group_label.rb +++ b/app/models/group_label.rb @@ -11,4 +11,8 @@ class GroupLabel < Label def subject_foreign_key 'group_id' end + + def preloaded_parent_container + association(:group).loaded? ? group : parent_container + end end diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index 8e9a74a68d0..695041f0247 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -2,7 +2,6 @@ class ProjectHook < WebHook include TriggerableHooks - include WebHooks::AutoDisabling include Presentable include Limitable extend ::Gitlab::Utils::Override diff --git a/app/models/hooks/service_hook.rb b/app/models/hooks/service_hook.rb index 6af70c249a0..453b986ca4d 100644 --- a/app/models/hooks/service_hook.rb +++ b/app/models/hooks/service_hook.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class ServiceHook < WebHook - include WebHooks::Unstoppable include Presentable extend ::Gitlab::Utils::Override diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb index eaffe83cab3..3c7f0ef9ffc 100644 --- a/app/models/hooks/system_hook.rb +++ b/app/models/hooks/system_hook.rb @@ -2,7 +2,6 @@ class SystemHook < WebHook include TriggerableHooks - include WebHooks::Unstoppable triggerable_hooks [ :repository_update_hooks, diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 7202a530feb..5ccbc926a71 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -2,15 +2,10 @@ class WebHook < ApplicationRecord include Sortable + include WebHooks::AutoDisabling InterpolationError = Class.new(StandardError) - MAX_FAILURES = 100 - FAILURE_THRESHOLD = 3 # three strikes - EXCEEDED_FAILURE_THRESHOLD = FAILURE_THRESHOLD + 1 - INITIAL_BACKOFF = 1.minute - MAX_BACKOFF = 1.day - BACKOFF_GROWTH_FACTOR = 2.0 SECRET_MASK = '************' attr_encrypted :token, @@ -78,46 +73,6 @@ class WebHook < ApplicationRecord 'user/project/integrations/webhooks' end - def next_backoff - return MAX_BACKOFF if backoff_count >= 8 # optimization to prevent expensive exponentiation and possible overflows - - (INITIAL_BACKOFF * (BACKOFF_GROWTH_FACTOR**backoff_count)) - .clamp(INITIAL_BACKOFF, MAX_BACKOFF) - .seconds - end - - def disable! - update_attribute(:recent_failures, EXCEEDED_FAILURE_THRESHOLD) - end - - def enable! - return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0 - - assign_attributes(recent_failures: 0, disabled_until: nil, backoff_count: 0) - save(validate: false) - end - - # Don't actually back-off until FAILURE_THRESHOLD failures have been seen - # we mark the grace-period using the recent_failures counter - def backoff! - attrs = { recent_failures: next_failure_count } - - if recent_failures >= FAILURE_THRESHOLD - attrs[:backoff_count] = next_backoff_count - attrs[:disabled_until] = next_backoff.from_now - end - - assign_attributes(attrs) - save(validate: false) if changed? - end - - def failed! - return unless recent_failures < MAX_FAILURES - - assign_attributes(disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count) - save(validate: false) - end - # @return [Boolean] Whether or not the WebHook is currently throttled. def rate_limited? rate_limiter.rate_limited? @@ -178,7 +133,7 @@ class WebHook < ApplicationRecord def reset_url_variables interpolated_url_was = interpolated_url(decrypt_url_was, url_variables_were) - return if url_variables_were.empty? || interpolated_url_was == interpolated_url + return if url_variables_were.blank? || interpolated_url_was == interpolated_url self.url_variables = {} if url_changed? && url_variables_were.to_a.intersection(url_variables.to_a).any? end @@ -191,14 +146,6 @@ class WebHook < ApplicationRecord self.class.decrypt_url_variables(encrypted_url_variables_was, iv: encrypted_url_variables_iv_was) end - def next_failure_count - recent_failures.succ.clamp(1, MAX_FAILURES) - end - - def next_backoff_count - backoff_count.succ.clamp(1, MAX_FAILURES) - end - def initialize_url_variables self.url_variables = {} if encrypted_url_variables.nil? end diff --git a/app/models/import_failure.rb b/app/models/import_failure.rb index 109c0c82487..e5b27009115 100644 --- a/app/models/import_failure.rb +++ b/app/models/import_failure.rb @@ -6,6 +6,9 @@ class ImportFailure < ApplicationRecord validates :project, presence: true, unless: :group validates :group, presence: true, unless: :project + validates :external_identifiers, json_schema: { filename: "import_failure_external_identifiers" } + + scope :with_external_identifiers, -> { where.not(external_identifiers: {}) } # Returns any `import_failures` for relations that were unrecoverable errors or failed after # several retries. An import can be successful even if some relations failed to import correctly. @@ -13,4 +16,8 @@ class ImportFailure < ApplicationRecord scope :hard_failures_by_correlation_id, ->(correlation_id) { where(correlation_id_value: correlation_id, retry_count: 0).order(created_at: :desc) } + + scope :failures_by_correlation_id, ->(correlation_id) { + where(correlation_id_value: correlation_id).order(created_at: :desc) + } end diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index 8a8c1a29375..64c9680ce90 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -138,7 +138,6 @@ class InstanceConfiguration plan.actual_limits.slice( :ci_pipeline_size, :ci_active_jobs, - :ci_active_pipelines, :ci_project_subscriptions, :ci_pipeline_schedules, :ci_needs_size_limit, diff --git a/app/models/integration.rb b/app/models/integration.rb index d3006f00ba1..860739fe5aa 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -21,13 +21,14 @@ class Integration < ApplicationRecord asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord drone_ci emails_on_push ewm external_wiki hangouts_chat harbor irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email - pivotaltracker prometheus pumble pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao + pivotaltracker prometheus pumble pushover redmine slack slack_slash_commands squash_tm teamcity + unify_circuit webex_teams youtrack zentao ].freeze # TODO Shimo is temporary disabled on group and instance-levels. # See: https://gitlab.com/gitlab-org/gitlab/-/issues/345677 PROJECT_SPECIFIC_INTEGRATION_NAMES = %w[ - apple_app_store jenkins shimo + apple_app_store google_play jenkins shimo ].freeze # Fake integrations to help with local development. diff --git a/app/models/integrations/apple_app_store.rb b/app/models/integrations/apple_app_store.rb index 84185542939..5e502cce927 100644 --- a/app/models/integrations/apple_app_store.rb +++ b/app/models/integrations/apple_app_store.rb @@ -6,11 +6,15 @@ module Integrations class AppleAppStore < Integration ISSUER_ID_REGEX = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/.freeze KEY_ID_REGEX = /\A(?=.*[A-Z])(?=.*[0-9])[A-Z0-9]+\z/.freeze + IS_KEY_CONTENT_BASE64 = "true" + + SECTION_TYPE_APPLE_APP_STORE = 'apple_app_store' with_options if: :activated? do validates :app_store_issuer_id, presence: true, format: { with: ISSUER_ID_REGEX } validates :app_store_key_id, presence: true, format: { with: KEY_ID_REGEX } validates :app_store_private_key, presence: true, certificate_key: true + validates :app_store_private_key_file_name, presence: true end field :app_store_issuer_id, @@ -21,15 +25,12 @@ module Integrations field :app_store_key_id, section: SECTION_TYPE_CONNECTION, required: true, - title: -> { s_('AppleAppStore|The Apple App Store Connect Key ID.') }, - is_secret: false + title: -> { s_('AppleAppStore|The Apple App Store Connect Key ID.') } - field :app_store_private_key, - section: SECTION_TYPE_CONNECTION, - required: true, - type: 'textarea', - title: -> { s_('AppleAppStore|The Apple App Store Connect Private Key.') }, - is_secret: false + field :app_store_private_key_file_name, + section: SECTION_TYPE_CONNECTION + + field :app_store_private_key, api_only: true def title 'Apple App Store Connect' @@ -43,7 +44,8 @@ module Integrations variable_list = [ '<code>APP_STORE_CONNECT_API_KEY_ISSUER_ID</code>', '<code>APP_STORE_CONNECT_API_KEY_KEY_ID</code>', - '<code>APP_STORE_CONNECT_API_KEY_KEY</code>' + '<code>APP_STORE_CONNECT_API_KEY_KEY</code>', + '<code>APP_STORE_CONNECT_API_KEY_IS_KEY_CONTENT_BASE64</code>' ] # rubocop:disable Layout/LineLength @@ -51,7 +53,7 @@ module Integrations s_("Use the Apple App Store Connect integration to easily connect to the Apple App Store with Fastlane in CI/CD pipelines."), s_("After the Apple App Store Connect integration is activated, the following protected variables will be created for CI/CD use."), variable_list.join('<br>'), - s_(format("To get started, see the <a href='%{url}' target='_blank'>integration documentation</a> for instructions on how to generate App Store Connect credentials, and how to use this integration.", url: "https://docs.gitlab.com/ee/integration/apple_app_store.html")).html_safe + s_(format("To get started, see the <a href='%{url}' target='_blank'>integration documentation</a> for instructions on how to generate App Store Connect credentials, and how to use this integration.", url: Rails.application.routes.url_helpers.help_page_url('user/project/integrations/apple_app_store'))).html_safe ] # rubocop:enable Layout/LineLength @@ -69,7 +71,7 @@ module Integrations def sections [ { - type: SECTION_TYPE_CONNECTION, + type: SECTION_TYPE_APPLE_APP_STORE, title: s_('Integrations|Integration details'), description: help } @@ -92,20 +94,20 @@ module Integrations { key: 'APP_STORE_CONNECT_API_KEY_ISSUER_ID', value: app_store_issuer_id, masked: true, public: false }, { key: 'APP_STORE_CONNECT_API_KEY_KEY', value: Base64.encode64(app_store_private_key), masked: true, public: false }, - { key: 'APP_STORE_CONNECT_API_KEY_KEY_ID', value: app_store_key_id, masked: true, public: false } + { key: 'APP_STORE_CONNECT_API_KEY_KEY_ID', value: app_store_key_id, masked: true, public: false }, + { key: 'APP_STORE_CONNECT_API_KEY_IS_KEY_CONTENT_BASE64', value: IS_KEY_CONTENT_BASE64, masked: false, + public: false } ] end private def client - config = { + AppStoreConnect::Client.new( issuer_id: app_store_issuer_id, key_id: app_store_key_id, private_key: app_store_private_key - } - - AppStoreConnect::Client.new(config) + ) end end end diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index fc5e6a88c2d..4638ca0c5f1 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -17,7 +17,8 @@ module Integrations non_empty_password_title: -> { s_('BambooService|Enter new build key') }, non_empty_password_help: -> { s_('BambooService|Leave blank to use your current build key.') }, placeholder: -> { _('KEY') }, - required: true + required: true, + is_secret: true field :username, help: -> { s_('BambooService|The user with API access to the Bamboo server.') } diff --git a/app/models/integrations/base_issue_tracker.rb b/app/models/integrations/base_issue_tracker.rb index e0994305e9d..7a54d354007 100644 --- a/app/models/integrations/base_issue_tracker.rb +++ b/app/models/integrations/base_issue_tracker.rb @@ -14,7 +14,7 @@ module Integrations # This pattern does not support cross-project references # The other code assumes that this pattern is a superset of all # overridden patterns. See ReferenceRegexes.external_pattern - def self.reference_pattern(only_long: false) + def self.base_reference_pattern(only_long: false) if only_long /(\b[A-Z][A-Z0-9_]*-)#{Gitlab::Regex.issue}/ else @@ -22,6 +22,10 @@ module Integrations end end + def reference_pattern(only_long: false) + self.class.base_reference_pattern(only_long: only_long) + end + def handle_properties # this has been moved from initialize_properties and should be improved # as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 diff --git a/app/models/integrations/base_slack_notification.rb b/app/models/integrations/base_slack_notification.rb index 7a2a91aa0d2..c83a559e0da 100644 --- a/app/models/integrations/base_slack_notification.rb +++ b/app/models/integrations/base_slack_notification.rb @@ -44,8 +44,6 @@ module Integrations Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user_id) - return unless Feature.enabled?(:route_hll_to_snowplow_phase2) - optional_arguments = { project: project, namespace: group || project&.namespace diff --git a/app/models/integrations/base_slash_commands.rb b/app/models/integrations/base_slash_commands.rb index 619579a543a..7662da933ba 100644 --- a/app/models/integrations/base_slash_commands.rb +++ b/app/models/integrations/base_slash_commands.rb @@ -6,10 +6,6 @@ module Integrations class BaseSlashCommands < Integration attribute :category, default: 'chat' - prop_accessor :token - - has_many :chat_names, foreign_key: :integration_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - def valid_token?(token) self.respond_to?(:token) && self.token.present? && @@ -24,18 +20,6 @@ module Integrations false end - def fields - [ - { - type: 'password', - name: 'token', - non_empty_password_title: s_('ProjectService|Enter new token'), - non_empty_password_help: s_('ProjectService|Leave blank to use your current token.'), - placeholder: 'XXxxXXxxXXxxXXxxXXxxXXxx' - } - ] - end - def trigger(params) return unless valid_token?(params[:token]) diff --git a/app/models/integrations/campfire.rb b/app/models/integrations/campfire.rb index 3f7fa1c51b2..9b837faf79b 100644 --- a/app/models/integrations/campfire.rb +++ b/app/models/integrations/campfire.rb @@ -68,7 +68,7 @@ module Integrations def execute(data) return unless supported_events.include?(data[:object_kind]) - message = build_message(data) + message = create_message(data) speak(self.room, message, auth) end @@ -116,7 +116,7 @@ module Integrations res.code == 200 ? res["rooms"] : [] end - def build_message(push) + def create_message(push) ref = Gitlab::Git.ref_name(push[:ref]) before = push[:before] after = push[:after] diff --git a/app/models/integrations/ewm.rb b/app/models/integrations/ewm.rb index 1b86ef73c85..003c896704a 100644 --- a/app/models/integrations/ewm.rb +++ b/app/models/integrations/ewm.rb @@ -6,7 +6,7 @@ module Integrations validates :project_url, :issues_url, :new_issue_url, presence: true, public_url: true, if: :activated? - def self.reference_pattern(only_long: true) + def reference_pattern(only_long: true) @reference_pattern ||= %r{(?<issue>\b(bug|task|work item|workitem|rtcwi|defect)\b\s+\d+)}i end diff --git a/app/models/integrations/field.rb b/app/models/integrations/field.rb index 329c046075f..9f2274216f6 100644 --- a/app/models/integrations/field.rb +++ b/app/models/integrations/field.rb @@ -2,8 +2,6 @@ module Integrations class Field - SECRET_NAME = %r/token|key|password|passphrase|secret/.freeze - BOOLEAN_ATTRIBUTES = %i[required api_only is_secret exposes_secrets].freeze ATTRIBUTES = %i[ @@ -17,11 +15,11 @@ module Integrations attr_reader :name, :integration_class - def initialize(name:, integration_class:, type: 'text', is_secret: true, api_only: false, **attributes) + def initialize(name:, integration_class:, type: 'text', is_secret: false, api_only: false, **attributes) @name = name.to_s.freeze @integration_class = integration_class - attributes[:type] = SECRET_NAME.match?(@name) && is_secret ? 'password' : type + attributes[:type] = is_secret ? 'password' : type attributes[:api_only] = api_only attributes[:is_secret] = is_secret @attributes = attributes.freeze diff --git a/app/models/integrations/gitlab_slack_application.rb b/app/models/integrations/gitlab_slack_application.rb new file mode 100644 index 00000000000..b0f54f39e8c --- /dev/null +++ b/app/models/integrations/gitlab_slack_application.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +module Integrations + class GitlabSlackApplication < BaseSlackNotification + attribute :alert_events, default: false + attribute :commit_events, default: false + attribute :confidential_issues_events, default: false + attribute :confidential_note_events, default: false + attribute :deployment_events, default: false + attribute :issues_events, default: false + attribute :job_events, default: false + attribute :merge_requests_events, default: false + attribute :note_events, default: false + attribute :pipeline_events, default: false + attribute :push_events, default: false + attribute :tag_push_events, default: false + attribute :vulnerability_events, default: false + attribute :wiki_page_events, default: false + + has_one :slack_integration, foreign_key: :integration_id, inverse_of: :integration + delegate :bot_access_token, :bot_user_id, to: :slack_integration, allow_nil: true + + def update_active_status + update(active: !!slack_integration) + end + + def title + s_('Integrations|GitLab for Slack app') + end + + def description + s_('Integrations|Enable slash commands and notifications for a Slack workspace.') + end + + def self.to_param + 'gitlab_slack_application' + end + + override :show_active_box? + def show_active_box? + false + end + + override :test + def test(_data) + failures = test_notification_channels + + { success: failures.blank?, result: failures } + end + + # The form fields of this integration are editable only after the Slack App installation + # flow has been completed, which causes the integration to become activated/enabled. + override :editable? + def editable? + activated? + end + + override :fields + def fields + return [] unless editable? + + super + end + + override :sections + def sections + return [] unless editable? + + [ + { + type: SECTION_TYPE_TRIGGER, + title: s_('Integrations|Trigger'), + description: s_('Integrations|An event will be triggered when one of the following items happen.') + }, + { + type: SECTION_TYPE_CONFIGURATION, + title: s_('Integrations|Notification settings'), + description: s_('Integrations|Configure the scope of notifications.') + } + ] + end + + override :configurable_events + def configurable_events + return [] unless editable? + + super + end + + override :requires_webhook? + def requires_webhook? + false + end + + def upgrade_needed? + slack_integration.present? && slack_integration.upgrade_needed? + end + + private + + override :notify + def notify(message, opts) + channels = Array(opts[:channel]) + return false if channels.empty? + + payload = { + attachments: message.attachments, + text: message.pretext, + unfurl_links: false, + unfurl_media: false + } + + successes = channels.map do |channel| + notify_slack_channel!(channel, payload) + end + + successes.any? + end + + def notify_slack_channel!(channel, payload) + response = api_client.post( + 'chat.postMessage', + payload.merge(channel: channel) + ) + + log_error('Slack API error when notifying', api_response: response.parsed_response) unless response['ok'] + + response['ok'] + rescue *Gitlab::HTTP::HTTP_ERRORS => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, + { + integration_id: id, + slack_integration_id: slack_integration.id + } + ) + + false + end + + def api_client + @slack_api ||= ::Slack::API.new(slack_integration) + end + + def test_notification_channels + return if unique_channels.empty? + return s_('Integrations|GitLab for Slack app must be reinstalled to enable notifications') unless bot_access_token + + test_payload = { + text: 'Test', + user: bot_user_id + } + + not_found_channels = unique_channels.first(10).select do |channel| + test_payload[:channel] = channel + + response = ::Slack::API.new(slack_integration).post('chat.postEphemeral', test_payload) + response['error'] == 'channel_not_found' + end + + return if not_found_channels.empty? + + format( + s_( + 'Integrations|Unable to post to %{channel_list}, ' \ + 'please add the GitLab Slack app to any private Slack channels' + ), + channel_list: not_found_channels.to_sentence + ) + end + + override :metrics_key_prefix + def metrics_key_prefix + 'i_integrations_gitlab_for_slack_app' + end + end +end diff --git a/app/models/integrations/google_play.rb b/app/models/integrations/google_play.rb new file mode 100644 index 00000000000..9fa6dc19f11 --- /dev/null +++ b/app/models/integrations/google_play.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Integrations + class GooglePlay < Integration + PACKAGE_NAME_REGEX = /\A[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z][A-Za-z0-9_]*){1,20}\z/ + + SECTION_TYPE_GOOGLE_PLAY = 'google_play' + + with_options if: :activated? do + validates :service_account_key, presence: true, json_schema: { + filename: "google_service_account_key", parse_json: true + } + validates :service_account_key_file_name, presence: true + validates :package_name, presence: true, format: { with: PACKAGE_NAME_REGEX } + end + + field :package_name, + section: SECTION_TYPE_CONNECTION, + placeholder: 'com.example.myapp', + required: true + + field :service_account_key_file_name, + section: SECTION_TYPE_CONNECTION, + required: true + + field :service_account_key, api_only: true + + def title + s_('GooglePlay|Google Play') + end + + def description + s_('GooglePlay|Use GitLab to build and release an app in Google Play.') + end + + def help + variable_list = [ + '<code>SUPPLY_PACKAGE_NAME</code>', + '<code>SUPPLY_JSON_KEY_DATA</code>' + ] + + # rubocop:disable Layout/LineLength + texts = [ + s_("Use the Google Play integration to connect to Google Play with fastlane in CI/CD pipelines."), + s_("After you enable the integration, the following protected variable is created for CI/CD use:"), + variable_list.join('<br>'), + s_(format("To generate a Google Play service account key and use this integration, see the <a href='%{url}' target='_blank'>integration documentation</a>.", url: Rails.application.routes.url_helpers.help_page_url('user/project/integrations/google_play'))).html_safe + ] + # rubocop:enable Layout/LineLength + + texts.join('<br><br>'.html_safe) + end + + def self.to_param + 'google_play' + end + + def self.supported_events + [] + end + + def sections + [ + { + type: SECTION_TYPE_GOOGLE_PLAY, + title: s_('Integrations|Integration details'), + description: help + } + ] + end + + def test(*_args) + client.list_reviews(package_name) + { success: true } + rescue Google::Apis::ClientError => error + { success: false, message: error } + end + + def ci_variables + return [] unless activated? + + [ + { key: 'SUPPLY_JSON_KEY_DATA', value: service_account_key, masked: true, public: false }, + { key: 'SUPPLY_PACKAGE_NAME', value: package_name, masked: false, public: false } + ] + end + + private + + def client + service = Google::Apis::AndroidpublisherV3::AndroidPublisherService.new # rubocop: disable CodeReuse/ServiceClass + + service.authorization = Google::Auth::ServiceAccountCredentials.make_creds( + json_key_io: StringIO.new(service_account_key), + scope: [Google::Apis::AndroidpublisherV3::AUTH_ANDROIDPUBLISHER] + ) + + service + end + end +end diff --git a/app/models/integrations/harbor.rb b/app/models/integrations/harbor.rb index 01a04743d5d..079811e0df0 100644 --- a/app/models/integrations/harbor.rb +++ b/app/models/integrations/harbor.rb @@ -17,7 +17,8 @@ module Integrations field :project_name, title: -> { s_('HarborIntegration|Harbor project name') }, - help: -> { s_('HarborIntegration|The name of the project in Harbor.') } + help: -> { s_('HarborIntegration|The name of the project in Harbor.') }, + required: true field :username, title: -> { s_('HarborIntegration|Harbor username') }, @@ -62,7 +63,7 @@ module Integrations end def test(*_args) - client.ping + client.check_project_availability end def ci_variables diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index d96a848c72e..2520d3bfc9c 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -17,12 +17,19 @@ module Integrations SECTION_TYPE_JIRA_TRIGGER = 'jira_trigger' SECTION_TYPE_JIRA_ISSUES = 'jira_issues' + AUTH_TYPE_BASIC = 0 + AUTH_TYPE_PAT = 1 + SNOWPLOW_EVENT_CATEGORY = self.name validates :url, public_url: true, presence: true, if: :activated? validates :api_url, public_url: true, allow_blank: true - validates :username, presence: true, if: :activated? + validates :username, presence: true, if: ->(object) { object.activated? && !object.personal_access_token_authorization? } validates :password, presence: true, if: :activated? + validates :jira_auth_type, presence: true, inclusion: { in: [AUTH_TYPE_BASIC, AUTH_TYPE_PAT] }, if: :activated? + validates :jira_issue_prefix, untrusted_regexp: true, length: { maximum: 255 }, if: :activated? + validates :jira_issue_regex, untrusted_regexp: true, length: { maximum: 255 }, if: :activated? + validate :validate_jira_cloud_auth_type_is_basic, if: :activated? validates :jira_issue_transition_id, format: { @@ -58,19 +65,44 @@ module Integrations help: -> { s_('JiraService|If different from the Web URL') }, exposes_secrets: true + field :jira_auth_type, + type: 'select', + required: true, + section: SECTION_TYPE_CONNECTION, + title: -> { s_('JiraService|Authentication type') }, + choices: -> { + [ + [s_('JiraService|Basic'), AUTH_TYPE_BASIC], + [s_('JiraService|Jira personal access token (Jira Data Center and Jira Server only)'), AUTH_TYPE_PAT] + ] + } + field :username, section: SECTION_TYPE_CONNECTION, - required: true, - title: -> { s_('JiraService|Username or email') }, - help: -> { s_('JiraService|Username for the server version or an email for the cloud version') } + required: false, + title: -> { s_('JiraService|Email or username') }, + help: -> { s_('JiraService|Only required for Basic authentication. Email for Jira Cloud or username for Jira Data Center and Jira Server') } field :password, section: SECTION_TYPE_CONNECTION, required: true, title: -> { s_('JiraService|Password or API token') }, - non_empty_password_title: -> { s_('JiraService|Enter new password or API token') }, - non_empty_password_help: -> { s_('JiraService|Leave blank to use your current password or API token.') }, - help: -> { s_('JiraService|Password for the server version or an API token for the cloud version') } + non_empty_password_title: -> { s_('JiraService|New API token, password, or Jira personal access token') }, + non_empty_password_help: -> { s_('JiraService|Leave blank to use your current configuration') }, + help: -> { s_('JiraService|API token for Jira Cloud or password for Jira Data Center and Jira Server') }, + is_secret: true + + field :jira_issue_regex, + section: SECTION_TYPE_CONFIGURATION, + required: false, + title: -> { s_('JiraService|Jira issue regex') }, + help: -> { s_('JiraService|Use regular expression to match Jira issue keys.') } + + field :jira_issue_prefix, + section: SECTION_TYPE_CONFIGURATION, + required: false, + title: -> { s_('JiraService|Jira issue prefix') }, + help: -> { s_('JiraService|Use a prefix to match Jira issue keys.') } field :jira_issue_transition_id, api_only: true @@ -90,8 +122,8 @@ module Integrations end # {PROJECT-KEY}-{NUMBER} Examples: JIRA-1, PROJECT-1 - def self.reference_pattern(only_long: true) - @reference_pattern ||= /(?<issue>\b#{Gitlab::Regex.jira_issue_key_regex})/ + def reference_pattern(only_long: true) + @reference_pattern ||= jira_issue_match_regex end def self.valid_jira_cloud_url?(url) @@ -119,16 +151,23 @@ module Integrations def options url = URI.parse(client_url) - { - username: username&.strip, - password: password, - site: URI.join(url, '/').to_s.delete_suffix('/'), # Intended to find the root + options = { + site: URI.join(url, '/').to_s.chomp('/'), # Find the root URL context_path: (url.path.presence || '/').delete_suffix('/'), auth_type: :basic, - use_cookies: true, - additional_cookies: ['OBBasicAuth=fromDialog'], use_ssl: url.scheme == 'https' } + + if personal_access_token_authorization? + options[:default_headers] = { 'Authorization' => "Bearer #{password}" } + else + options[:username] = username&.strip + options[:password] = password + options[:use_cookies] = true + options[:additional_cookies] = ['OBBasicAuth=fromDialog'] + end + + options end def client @@ -166,6 +205,11 @@ module Integrations type: SECTION_TYPE_JIRA_TRIGGER, title: _('Trigger'), description: s_('JiraService|When a Jira issue is mentioned in a commit or merge request, a remote link and comment (if enabled) will be created.') + }, + { + type: SECTION_TYPE_CONFIGURATION, + title: _('Jira issue matching'), + description: s_('Configure custom rules for Jira issue key matching') } ] @@ -323,8 +367,18 @@ module Integrations jira_issue_transition_automatic || jira_issue_transition_id.present? end + def personal_access_token_authorization? + jira_auth_type == AUTH_TYPE_PAT + end + private + def jira_issue_match_regex + match_regex = (jira_issue_regex.presence || Gitlab::Regex.jira_issue_key_regex) + + /\b#{jira_issue_prefix}(?<issue>#{match_regex})/ + end + def parse_project_from_issue_key(issue_key) issue_key.gsub(Gitlab::Regex.jira_issue_key_project_key_extraction_regex, '') end @@ -391,8 +445,6 @@ module Integrations Gitlab::UsageDataCounters::HLLRedisCounter.track_event(key, values: user.id) - return unless Feature.enabled?(:route_hll_to_snowplow_phase2) - optional_arguments = { project: project, namespace: group || project&.namespace @@ -606,7 +658,6 @@ module Integrations # If API-based detection methods fail here then # we can only assume it's either Cloud or Server # based on the URL being *.atlassian.net - if self.class.valid_jira_cloud_url?(client_url) data_fields.deployment_cloud! else @@ -626,6 +677,17 @@ module Integrations description end + + def validate_jira_cloud_auth_type_is_basic + return unless self.class.valid_jira_cloud_url?(client_url) && jira_auth_type != AUTH_TYPE_BASIC + + errors.add(:base, + format( + s_('JiraService|For Jira Cloud, the authentication type must be %{basic}'), + basic: s_('JiraService|Basic') + ) + ) + end end end diff --git a/app/models/integrations/mattermost_slash_commands.rb b/app/models/integrations/mattermost_slash_commands.rb index 30a8ba973c1..e075400d9b5 100644 --- a/app/models/integrations/mattermost_slash_commands.rb +++ b/app/models/integrations/mattermost_slash_commands.rb @@ -4,18 +4,22 @@ module Integrations class MattermostSlashCommands < BaseSlashCommands include Ci::TriggersHelper - prop_accessor :token + field :token, + type: 'password', + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, + placeholder: '' def testable? false end def title - 'Mattermost slash commands' + s_('Integrations|Mattermost slash commands') end def description - "Perform common tasks with slash commands." + s_('Integrations|Perform common tasks with slash commands.') end def self.to_param @@ -37,10 +41,6 @@ module Integrations [[], e.message] end - def chat_responder - ::Gitlab::Chat::Responder::Mattermost - end - private def command(params) diff --git a/app/models/integrations/prometheus.rb b/app/models/integrations/prometheus.rb index 2f0995e9ab0..2dc0fd7d011 100644 --- a/app/models/integrations/prometheus.rb +++ b/app/models/integrations/prometheus.rb @@ -30,12 +30,9 @@ module Integrations help: -> { s_('PrometheusService|The contents of the credentials.json file of your service account.') }, required: false - # We need to allow the self-monitoring project to connect to the internal - # Prometheus instance. # Since the internal Prometheus instance is usually a localhost URL, we need # to allow localhost URLs when the following conditions are true: - # 1. project is the self-monitoring project. - # 2. api_url is the internal Prometheus URL. + # 1. api_url is the internal Prometheus URL. with_options presence: true do validates :api_url, public_url: true, if: ->(object) { object.manual_configuration? && !object.allow_local_api_url? } validates :api_url, url: true, if: ->(object) { object.manual_configuration? && object.allow_local_api_url? } @@ -99,8 +96,7 @@ module Integrations end def allow_local_api_url? - allow_local_requests_from_web_hooks_and_services? || - (self_monitoring_project? && internal_prometheus_url?) + allow_local_requests_from_web_hooks_and_services? || internal_prometheus_url? end def configured? @@ -127,10 +123,6 @@ module Integrations delegate :allow_local_requests_from_web_hooks_and_services?, to: :current_settings, private: true - def self_monitoring_project? - project && project.id == current_settings.self_monitoring_project_id - end - def internal_prometheus_url? api_url.present? && api_url == ::Gitlab::Prometheus::Internal.uri end diff --git a/app/models/integrations/slack_slash_commands.rb b/app/models/integrations/slack_slash_commands.rb index 72e3c4a8cbc..343c8d68166 100644 --- a/app/models/integrations/slack_slash_commands.rb +++ b/app/models/integrations/slack_slash_commands.rb @@ -4,6 +4,12 @@ module Integrations class SlackSlashCommands < BaseSlashCommands include Ci::TriggersHelper + field :token, + type: 'password', + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, + placeholder: '' + def title 'Slack slash commands' end @@ -23,10 +29,6 @@ module Integrations end end - def chat_responder - ::Gitlab::Chat::Responder::Slack - end - private def format(text) diff --git a/app/models/integrations/slack_workspace/api_scope.rb b/app/models/integrations/slack_workspace/api_scope.rb new file mode 100644 index 00000000000..3c4d25bff10 --- /dev/null +++ b/app/models/integrations/slack_workspace/api_scope.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Integrations + module SlackWorkspace + class ApiScope < ApplicationRecord + self.table_name = 'slack_api_scopes' + + def self.find_or_initialize_by_names(names) + found = where(name: names).to_a + missing_names = names - found.pluck(:name) + + if missing_names.any? + insert_all(missing_names.map { |name| { name: name } }) + missing = where(name: missing_names) + found += missing + end + + found + end + end + end +end diff --git a/app/models/integrations/slack_workspace/integration_api_scope.rb b/app/models/integrations/slack_workspace/integration_api_scope.rb new file mode 100644 index 00000000000..d33c8e0d816 --- /dev/null +++ b/app/models/integrations/slack_workspace/integration_api_scope.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Integrations + module SlackWorkspace + class IntegrationApiScope < ApplicationRecord + self.table_name = 'slack_integrations_scopes' + + belongs_to :slack_api_scope, class_name: 'Integrations::SlackWorkspace::ApiScope' + belongs_to :slack_integration + + # Efficient scope propagation + def self.update_scopes(integration_ids, scopes) + return if integration_ids.empty? + + scope_ids = scopes.pluck(:id) + + attrs = scope_ids.flat_map do |scope_id| + integration_ids.map { |si_id| { slack_integration_id: si_id, slack_api_scope_id: scope_id } } + end + + # We don't know which ones to preserve - so just delete them all in a single query + transaction do + where(slack_integration_id: integration_ids).delete_all + insert_all(attrs) unless attrs.empty? + end + end + end + end +end diff --git a/app/models/integrations/squash_tm.rb b/app/models/integrations/squash_tm.rb new file mode 100644 index 00000000000..e0a63b5ae6a --- /dev/null +++ b/app/models/integrations/squash_tm.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Integrations + class SquashTm < Integration + include HasWebHook + + field :url, + placeholder: 'https://your-instance.squashcloud.io/squash/plugin/xsquash4gitlab/webhook/issue', + title: -> { s_('SquashTmIntegration|Squash TM webhook URL') }, + exposes_secrets: true, + required: true + + field :token, + type: 'password', + title: -> { s_('SquashTmIntegration|Secret token (optional)') }, + non_empty_password_title: -> { s_('ProjectService|Enter new token') }, + non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, + required: false + + with_options if: :activated? do + validates :url, presence: true, public_url: true + validates :token, length: { maximum: 255 }, allow_blank: true + end + + def title + 'Squash TM' + end + + def description + s_("SquashTmIntegration|Update Squash TM requirements when GitLab issues are modified.") + end + + def help + docs_link = ActionController::Base.helpers.link_to( + _('Learn more.'), + Rails.application.routes.url_helpers.help_page_url('user/project/integrations/squash_tm'), + target: '_blank', + rel: 'noopener noreferrer' + ) + + Kernel.format( + s_('SquashTmIntegration|Update Squash TM requirements when GitLab issues are modified. %{docs_link}'), + { docs_link: docs_link.html_safe } + ).html_safe + end + + def self.supported_events + %w[issue confidential_issue] + end + + def self.to_param + 'squash_tm' + end + + def self.default_test_event + 'issue' + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + + execute_web_hook!(data, "#{data[:object_kind]} Hook") + end + + def test(data) + result = execute_web_hook!(data, "Test Configuration Hook") + + { success: result.payload[:http_status] == 200, result: result.message } + rescue StandardError => error + { success: false, result: error.message } + end + + override :hook_url + def hook_url + format("#{url}%s", ('?token={token}' unless token.blank?)) + end + + def url_variables + { 'token' => token }.compact + end + end +end diff --git a/app/models/integrations/youtrack.rb b/app/models/integrations/youtrack.rb index fa719f925ed..15246a37aa7 100644 --- a/app/models/integrations/youtrack.rb +++ b/app/models/integrations/youtrack.rb @@ -7,12 +7,11 @@ module Integrations validates :project_url, :issues_url, presence: true, public_url: true, if: :activated? # {PROJECT-KEY}-{NUMBER} Examples: YT-1, PRJ-1, gl-030 - def self.reference_pattern(only_long: false) - if only_long - /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)/ - else - /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)|(#{Issue.reference_prefix}#{Gitlab::Regex.issue})/ - end + def reference_pattern(only_long: false) + return @reference_pattern if defined?(@reference_pattern) + + regex_suffix = "|(#{Issue.reference_prefix}#{Gitlab::Regex.issue})" + @reference_pattern = /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)#{regex_suffix if only_long}/ end def title diff --git a/app/models/issue.rb b/app/models/issue.rb index bea86168c8d..b7125617034 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -39,6 +39,9 @@ class Issue < ApplicationRecord DueThisMonth = DueDateStruct.new('Due This Month', 'month').freeze DueNextMonthAndPreviousTwoWeeks = DueDateStruct.new('Due Next Month And Previous Two Weeks', 'next_month_and_previous_two_weeks').freeze + IssueTypeOutOfSyncError = Class.new(StandardError) + ForbiddenColumnUsed = Class.new(StandardError) + SORTING_PREFERENCE_FIELD = :issues_sort MAX_BRANCH_TEMPLATE = 255 @@ -52,18 +55,37 @@ class Issue < ApplicationRecord # Types of issues that should be displayed on issue board lists TYPES_FOR_BOARD_LIST = %w(issue incident).freeze + # This default came from the enum `issue_type` column. Defined as default in the DB + DEFAULT_ISSUE_TYPE = :issue + belongs_to :project belongs_to :namespace, inverse_of: :issues belongs_to :duplicated_to, class_name: 'Issue' belongs_to :closed_by, class_name: 'User' - belongs_to :iteration, foreign_key: 'sprint_id' belongs_to :work_item_type, class_name: 'WorkItems::Type', inverse_of: :work_items - belongs_to :moved_to, class_name: 'Issue' - has_one :moved_from, class_name: 'Issue', foreign_key: :moved_to_id - - has_internal_id :iid, scope: :project, track_if: -> { !importing? } + belongs_to :moved_to, class_name: 'Issue', inverse_of: :moved_from + has_one :moved_from, class_name: 'Issue', foreign_key: :moved_to_id, inverse_of: :moved_to + + has_internal_id :iid, scope: :namespace, track_if: -> { !importing? }, init: ->(issue, scope) do + # we need this init for the case where the IID allocation in internal_ids#last_value + # is higher than the actual issues.max(iid) value for a given project. For instance + # in case of an import where a batch of IIDs may be prealocated + # + # TODO: remove this once the UpdateIssuesInternalIdScope migration completes + if issue + [ + InternalId.where(project: issue.project, usage: :issues).pick(:last_value).to_i, + issue.namespace&.issues&.maximum(:iid).to_i + ].max + else + [ + InternalId.where(**scope, usage: :issues).pick(:last_value).to_i, + where(**scope).maximum(:iid).to_i + ].max + end + end has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -97,6 +119,7 @@ class Issue < ApplicationRecord has_many :issue_customer_relations_contacts, class_name: 'CustomerRelations::IssueContact', inverse_of: :issue has_many :customer_relations_contacts, through: :issue_customer_relations_contacts, source: :contact, class_name: 'CustomerRelations::Contact', inverse_of: :issues has_many :incident_management_timeline_events, class_name: 'IncidentManagement::TimelineEvent', foreign_key: :issue_id, inverse_of: :incident + has_many :assignment_events, class_name: 'ResourceEvents::IssueAssignmentEvent', inverse_of: :issue alias_attribute :escalation_status, :incident_management_issuable_escalation_status @@ -104,17 +127,41 @@ class Issue < ApplicationRecord accepts_nested_attributes_for :sentry_issue accepts_nested_attributes_for :incident_management_issuable_escalation_status, update_only: true - validates :project, presence: true - validates :issue_type, presence: true + validates :project, presence: true, if: -> { !namespace || namespace.is_a?(Namespaces::ProjectNamespace) } validates :namespace, presence: true validates :work_item_type, presence: true + validates :confidential, inclusion: { in: [true, false], message: 'must be a boolean' } validate :allowed_work_item_type_change, on: :update, if: :work_item_type_id_changed? validate :due_date_after_start_date validate :parent_link_confidentiality + # using a custom validation since we are overwriting the `issue_type` method to use the work_item_types table + validate :issue_type_attribute_present enum issue_type: WorkItems::Type.base_types + # TODO: Remove with https://gitlab.com/gitlab-org/gitlab/-/issues/402699 + WorkItems::Type.base_types.each do |base_type, _value| + define_method "#{base_type}?".to_sym do + error_message = <<~ERROR + `#{base_type}?` uses the `issue_type` column underneath. As we want to remove the column, + its usage is forbidden. You should use the `work_item_types` table instead. + + # Before + + issue.requirement? => true + + # After + + issue.work_item_type.requirement? => true + + More details in https://gitlab.com/groups/gitlab-org/-/epics/10529 + ERROR + + raise ForbiddenColumnUsed, error_message + end + end + alias_method :issuing_parent, :project alias_attribute :issuing_parent_id, :project_id @@ -136,7 +183,7 @@ class Issue < ApplicationRecord scope :order_due_date_asc, -> { reorder(arel_table[:due_date].asc.nulls_last) } scope :order_due_date_desc, -> { reorder(arel_table[:due_date].desc.nulls_last) } - scope :order_closest_future_date, -> { reorder(Arel.sql('CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC')) } + scope :order_closest_future_date, -> { reorder(Arel.sql("CASE WHEN issues.due_date >= CURRENT_DATE THEN 0 ELSE 1 END ASC, ABS(CURRENT_DATE - issues.due_date) ASC")) } scope :order_created_at_desc, -> { reorder(created_at: :desc) } scope :order_severity_asc, -> do build_keyset_order_on_joined_column( @@ -162,15 +209,15 @@ class Issue < ApplicationRecord scope :order_closed_at_desc, -> { reorder(arel_table[:closed_at].desc.nulls_last) } scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } - scope :with_web_entity_associations, -> { preload(:author, project: [:project_feature, :route, namespace: :route]) } + scope :with_web_entity_associations, -> { preload(:author, :namespace, project: [:project_feature, :route, namespace: :route]) } scope :preload_awardable, -> { preload(:award_emoji) } scope :with_alert_management_alerts, -> { joins(:alert_management_alert) } scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) } scope :with_self_managed_prometheus_alert_events, -> { joins(:issues_self_managed_prometheus_alert_events) } scope :with_api_entity_associations, -> { - preload(:timelogs, :closed_by, :assignees, :author, :labels, :issuable_severity, - milestone: { project: [:route, { namespace: :route }] }, - project: [:project_feature, :route, { namespace: :route }], + preload(:work_item_type, :timelogs, :closed_by, :assignees, :author, :labels, :issuable_severity, + namespace: [{ parent: :route }, :route], milestone: { project: [:route, { namespace: :route }] }, + project: [:project_namespace, :project_feature, :route, { group: :route }, { namespace: :route }], duplicated_to: { project: [:project_feature] }) } scope :with_issue_type, ->(types) { where(issue_type: types) } @@ -213,8 +260,9 @@ class Issue < ApplicationRecord scope :with_projects_matching_search_data, -> { where('issue_search_data.project_id = issues.project_id') } before_validation :ensure_namespace_id, :ensure_work_item_type + before_save :check_issue_type_in_sync! - after_save :ensure_metrics, unless: :importing? + after_save :ensure_metrics!, unless: :importing? after_commit :expire_etag_cache, unless: :importing? after_create_commit :record_create_action, unless: :importing? @@ -345,7 +393,7 @@ class Issue < ApplicationRecord end def self.link_reference_pattern - @link_reference_pattern ||= super(%r{issues(?:\/incident)?}, Gitlab::Regex.issue) + @link_reference_pattern ||= compose_link_reference_pattern(%r{issues(?:\/incident)?}, Gitlab::Regex.issue) end def self.reference_valid?(reference) @@ -450,7 +498,7 @@ class Issue < ApplicationRecord def to_reference(from = nil, full: false) reference = "#{self.class.reference_prefix}#{iid}" - "#{project.to_reference_base(from, full: full)}#{reference}" + "#{namespace.to_reference_base(from, full: full)}#{reference}" end def suggested_branch_name @@ -463,7 +511,7 @@ class Issue < ApplicationRecord "#{to_branch_name}-#{suffix}" end - Uniquify.new(start_counting_from).string(branch_name_generator) do |suggested_branch_name| + Gitlab::Utils::Uniquify.new(start_counting_from).string(branch_name_generator) do |suggested_branch_name| project.repository.branch_exists?(suggested_branch_name) end end @@ -576,6 +624,10 @@ class Issue < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def update_project_counter_caches + # TODO: Fix counter cache for issues in group + # TODO: see https://gitlab.com/gitlab-org/gitlab/-/work_items/393125 + return unless project + Projects::OpenIssuesCountService.new(project).refresh_cache end # rubocop: enable CodeReuse/ServiceClass @@ -614,7 +666,7 @@ class Issue < ApplicationRecord end def supports_assignee? - issue_type_supports?(:assignee) + work_item_type_with_default.supports_assignee? end def supports_time_tracking? @@ -655,13 +707,13 @@ class Issue < ApplicationRecord elsif project.personal? && project.team.owner?(user) true elsif confidential? && !assignee_or_author?(user) - project.team.member?(user, Gitlab::Access::REPORTER) + project.member?(user, Gitlab::Access::REPORTER) elsif hidden? false elsif project.public? || (project.internal? && !user.external?) project.feature_available?(:issues, user) else - project.team.member?(user) + project.member?(user) end end @@ -670,6 +722,10 @@ class Issue < ApplicationRecord end def expire_etag_cache + # TODO: Fix this for the case when issues is created at group level + # TODO: https://gitlab.com/gitlab-org/gitlab/-/work_items/395814 + return unless project + key = Gitlab::Routing.url_helpers.realtime_changes_project_issue_path(project, self) Gitlab::EtagCaching::Store.new.touch(key) end @@ -684,8 +740,60 @@ class Issue < ApplicationRecord ::Gitlab::GlobalId.as_global_id(id, model_name: WorkItem.name) end + def resource_parent + project || namespace + end + + # Persisted records will always have a work_item_type. This method is useful + # in places where we use a non persisted issue to perform feature checks + def work_item_type_with_default + work_item_type || WorkItems::Type.default_by_type(DEFAULT_ISSUE_TYPE) + end + + def issue_type + if ::Feature.enabled?(:issue_type_uses_work_item_types_table) + work_item_type_with_default.base_type + else + super + end + end + private + def check_issue_type_in_sync! + # We might have existing records out of sync, so we need to skip this check unless the value is changed + # so those records can still be updated until we fix them and remove the issue_type column + # https://gitlab.com/gitlab-org/gitlab/-/work_items/403158 + return unless (changes.keys & %w[issue_type work_item_type_id]).any? + + # Do not replace the use of attributes with `issue_type` here + if attributes['issue_type'] != work_item_type.base_type + error = IssueTypeOutOfSyncError.new( + <<~ERROR + Issue `issue_type` out of sync with `work_item_type_id` column. + `issue_type` must be equal to `work_item.base_type`. + You can assign the correct work_item_type like this for example: + + Issue.new(issue_type: :incident, work_item_type: WorkItems::Type.default_by_type(:incident)) + + More details in https://gitlab.com/gitlab-org/gitlab/-/issues/338005 + ERROR + ) + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + error, + issue_type: attributes['issue_type'], + work_item_type_id: work_item_type_id + ) + end + end + + def issue_type_attribute_present + return if attributes['issue_type'].present? + + errors.add(:issue_type, 'Must be present') + end + def due_date_after_start_date return unless start_date.present? && due_date.present? @@ -711,6 +819,10 @@ class Issue < ApplicationRecord override :persist_pg_full_text_search_vector def persist_pg_full_text_search_vector(search_vector) + # TODO: Fix search vector for issues at group level + # TODO: https://gitlab.com/gitlab-org/gitlab/-/work_items/393126 + return unless project + Issues::SearchData.upsert({ project_id: project_id, issue_id: id, search_vector: search_vector }, unique_by: %i(project_id issue_id)) end @@ -722,18 +834,19 @@ class Issue < ApplicationRecord confidential_changed?(from: true, to: false) end - override :ensure_metrics - def ensure_metrics + def ensure_metrics! Issue::Metrics.record!(self) end def record_create_action - Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_created_action(author: author, project: project) + Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_created_action( + author: author, namespace: namespace.reset + ) end # Returns `true` if this Issue is visible to everybody. def publicly_visible? - project.public? && project.feature_available?(:issues, nil) && + resource_parent.public? && resource_parent.feature_available?(:issues, nil) && !confidential? && !hidden? && !::Gitlab::ExternalAuthorization.enabled? end @@ -749,7 +862,9 @@ class Issue < ApplicationRecord def ensure_work_item_type return if work_item_type_id.present? || work_item_type_id_change&.last.present? - self.work_item_type = WorkItems::Type.default_by_type(issue_type) + # TODO: We should switch to DEFAULT_ISSUE_TYPE here when the issue_type column is dropped + # https://gitlab.com/gitlab-org/gitlab/-/work_items/402700 + self.work_item_type = WorkItems::Type.default_by_type(attributes['issue_type']) end def allowed_work_item_type_change diff --git a/app/models/iteration.rb b/app/models/iteration.rb deleted file mode 100644 index ebec24731ed..00000000000 --- a/app/models/iteration.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -# Placeholder class for model that is implemented in EE -class Iteration < ApplicationRecord - include IgnorableColumns - - self.table_name = 'sprints' - - def self.reference_prefix - '*iteration:' - end - - def self.reference_pattern - nil - end -end - -Iteration.prepend_mod_with('Iteration') diff --git a/app/models/key.rb b/app/models/key.rb index 596186276bb..2ea71bfcd6d 100644 --- a/app/models/key.rb +++ b/app/models/key.rb @@ -92,7 +92,7 @@ class Key < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def update_last_used_at - Keys::LastUsedService.new(self).execute + Keys::LastUsedService.new(self).execute_async end # rubocop: enable CodeReuse/ServiceClass diff --git a/app/models/label.rb b/app/models/label.rb index aa53c0e0f3f..32b399ac461 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -9,6 +9,7 @@ class Label < ApplicationRecord include Sortable include FromUnion include Presentable + include EachBatch cache_markdown_field :description, pipeline: :single_line @@ -66,6 +67,10 @@ class Label < ApplicationRecord .with_preloaded_container end + def self.pluck_titles + pluck(:title) + end + def self.prioritized(project) joins(:priorities) .where(label_priorities: { project_id: project }) diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index f28e8f81b40..7f64606e97b 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -9,23 +9,23 @@ class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel self.ignored_columns = %i[partition] partitioned_by :partition, strategy: :sliding_list, - next_partition_if: -> (active_partition) do - oldest_record_in_partition = LooseForeignKeys::DeletedRecord - .select(:id, :created_at) - .for_partition(active_partition.value) - .order(:id) - .limit(1) - .take - - oldest_record_in_partition.present? && - oldest_record_in_partition.created_at < PARTITION_DURATION.ago - end, - detach_partition_if: -> (partition) do - !LooseForeignKeys::DeletedRecord - .for_partition(partition.value) - .status_pending - .exists? - end + next_partition_if: -> (active_partition) do + oldest_record_in_partition = LooseForeignKeys::DeletedRecord + .select(:id, :created_at) + .for_partition(active_partition.value) + .order(:id) + .limit(1) + .take + + oldest_record_in_partition.present? && + oldest_record_in_partition.created_at < PARTITION_DURATION.ago + end, + detach_partition_if: -> (partition) do + !LooseForeignKeys::DeletedRecord + .for_partition(partition.value) + .status_pending + .exists? + end scope :for_table, -> (table) { where(fully_qualified_table_name: table) } scope :for_partition, -> (partition) { where(partition: partition) } diff --git a/app/models/member.rb b/app/models/member.rb index e97c9e929ac..529666a069c 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class Member < ApplicationRecord + extend ::Gitlab::Utils::Override include EachBatch include AfterCommitQueue include Sortable @@ -320,6 +321,12 @@ class Member < ApplicationRecord end end + def filter_by_user_type(value) + return unless ::User.user_types.key?(value) + + left_join_users.merge(::User.where(user_type: value)) + end + def sort_by_attribute(method) case method.to_s when 'access_level_asc' then reorder(access_level: :asc) @@ -353,6 +360,10 @@ class Member < ApplicationRecord def valid_email?(email) Devise.email_regexp.match?(email) end + + def pluck_user_ids + pluck(:user_id) + end end def real_source_type @@ -566,7 +577,7 @@ class Member < ApplicationRecord end def after_decline_invite - # override in subclass + notification_service.decline_invite(self) end def after_accept_request diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index f23d7208b6e..aabc902fe03 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class GroupMember < Member - extend ::Gitlab::Utils::Override include FromUnion include CreatedAtFilterable @@ -38,10 +37,6 @@ class GroupMember < Member Gitlab::Access.options_with_owner end - def self.pluck_user_ids - pluck(:user_id) - end - def group source end @@ -112,12 +107,6 @@ class GroupMember < Member super end - def after_decline_invite - notification_service.decline_group_invite(self) - - super - end - def send_welcome_email? true end diff --git a/app/models/members/member_role.rb b/app/models/members/member_role.rb deleted file mode 100644 index 42ce228c318..00000000000 --- a/app/models/members/member_role.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -class MemberRole < ApplicationRecord # rubocop:disable Gitlab/NamespacedClass - include IgnorableColumns - ignore_column :download_code, remove_with: '15.9', remove_after: '2023-01-22' - - has_many :members - belongs_to :namespace - - validates :namespace, presence: true - validates :base_access_level, presence: true - validate :belongs_to_top_level_namespace - validate :validate_namespace_locked, on: :update - validate :attributes_locked_after_member_associated, on: :update - - validates_associated :members - - before_destroy :prevent_delete_after_member_associated - - private - - def belongs_to_top_level_namespace - return if !namespace || namespace.root? - - errors.add(:namespace, s_("MemberRole|must be top-level namespace")) - end - - def validate_namespace_locked - return unless namespace_id_changed? - - errors.add(:namespace, s_("MemberRole|can't be changed")) - end - - def attributes_locked_after_member_associated - return unless members.present? - - errors.add(:base, s_("MemberRole|cannot be changed because it is already assigned to a user. "\ - "Please create a new Member Role instead")) - end - - def prevent_delete_after_member_associated - return unless members.present? - - errors.add(:base, s_("MemberRole|cannot be deleted because it is already assigned to a user. "\ - "Please disassociate the member role from all users before deletion.")) - - throw :abort # rubocop:disable Cop/BanCatchThrow - end -end diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 733b7c4bc87..e0fecf702de 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class ProjectMember < Member - extend ::Gitlab::Utils::Override SOURCE_TYPE = 'Project' SOURCE_TYPE_FORMAT = /\AProject\z/.freeze @@ -21,40 +20,6 @@ class ProjectMember < Member end class << self - # Add members to projects with passed access option - # - # access can be an integer representing a access code - # or symbol like :maintainer representing role - # - # Ex. - # add_members_to_projects( - # project_ids, - # user_ids, - # ProjectMember::MAINTAINER - # ) - # - # add_members_to_projects( - # project_ids, - # user_ids, - # :maintainer - # ) - # - def add_members_to_projects(project_ids, users, access_level, current_user: nil, expires_at: nil) - self.transaction do - project_ids.each do |project_id| - project = Project.find(project_id) - - Members::Projects::CreatorService.add_members( # rubocop:disable CodeReuse/ServiceClass - project, - users, - access_level, - current_user: current_user, - expires_at: expires_at - ) - end - end - end - def truncate_teams(project_ids) ProjectMember.transaction do members = ProjectMember.where(source_id: project_ids) @@ -180,12 +145,6 @@ class ProjectMember < Member super end - def after_decline_invite - notification_service.decline_project_invite(self) - - super - end - # rubocop: disable CodeReuse/ServiceClass def event_service EventCreateService.new diff --git a/app/models/members_preloader.rb b/app/models/members_preloader.rb index ba7e4b39989..1fef155e6ea 100644 --- a/app/models/members_preloader.rb +++ b/app/models/members_preloader.rb @@ -8,12 +8,14 @@ class MembersPreloader end def preload_all - ActiveRecord::Associations::Preloader.new.preload(members, :user) - ActiveRecord::Associations::Preloader.new.preload(members, :source) - ActiveRecord::Associations::Preloader.new.preload(members, :created_by) - ActiveRecord::Associations::Preloader.new.preload(members, user: :status) - ActiveRecord::Associations::Preloader.new.preload(members, user: :u2f_registrations) - ActiveRecord::Associations::Preloader.new.preload(members, user: :webauthn_registrations) if Feature.enabled?(:webauthn) + ActiveRecord::Associations::Preloader.new( + records: members, + associations: [ + :source, + :created_by, + { user: [:status, :webauthn_registrations] } + ] + ).call end end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index f3488f6ea60..7b1d4b97d3b 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -36,24 +36,18 @@ class MergeRequest < ApplicationRecord SORTING_PREFERENCE_FIELD = :merge_requests_sort - ALLOWED_TO_USE_MERGE_BASE_PIPELINE_FOR_COMPARISON = { - 'Ci::CompareMetricsReportsService' => ->(project) { true }, - 'Ci::CompareCodequalityReportsService' => ->(project) { true } - }.freeze - belongs_to :target_project, class_name: "Project" belongs_to :source_project, class_name: "Project" belongs_to :merge_user, class_name: "User" - belongs_to :iteration, foreign_key: 'sprint_id' has_internal_id :iid, scope: :target_project, track_if: -> { !importing? }, - init: ->(mr, scope) do - if mr - mr.target_project&.merge_requests&.maximum(:iid) - elsif scope[:project] - where(target_project: scope[:project]).maximum(:iid) - end - end + init: ->(mr, scope) do + if mr + mr.target_project&.merge_requests&.maximum(:iid) + elsif scope[:project] + where(target_project: scope[:project]).maximum(:iid) + end + end has_many :merge_request_diffs, -> { regular }, inverse_of: :merge_request @@ -92,7 +86,7 @@ class MergeRequest < ApplicationRecord fallback || super || MergeRequestDiff.new(merge_request_id: id) end - belongs_to :head_pipeline, foreign_key: "head_pipeline_id", class_name: "Ci::Pipeline" + belongs_to :head_pipeline, class_name: "Ci::Pipeline", inverse_of: :merge_requests_as_head_pipeline has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -123,6 +117,7 @@ class MergeRequest < ApplicationRecord has_many :reviews, inverse_of: :merge_request has_many :reviewed_by_users, -> { distinct }, through: :reviews, source: :author has_many :created_environments, class_name: 'Environment', foreign_key: :merge_request_id, inverse_of: :merge_request + has_many :assignment_events, class_name: 'ResourceEvents::MergeRequestAssignmentEvent', inverse_of: :merge_request KNOWN_MERGE_PARAMS = [ :auto_merge_strategy, @@ -141,7 +136,7 @@ class MergeRequest < ApplicationRecord after_update :clear_memoized_shas after_update :reload_diff_if_branch_changed after_save :keep_around_commit, unless: :importing? - after_commit :ensure_metrics, on: [:create, :update], unless: :importing? + after_commit :ensure_metrics!, on: [:create, :update], unless: :importing? after_commit :expire_etag_cache, unless: :importing? # When this attribute is true some MR validation is ignored @@ -156,10 +151,15 @@ class MergeRequest < ApplicationRecord # when creating new merge request attr_accessor :can_be_created, :compare_commits, :diff_options, :compare + # Flag to skip triggering mergeRequestMergeStatusUpdated GraphQL subscription. + attr_accessor :skip_merge_status_trigger + participant :reviewers - # Keep states definition to be evaluated before the state_machine block to avoid spec failures. - # If this gets evaluated after, the `merged` and `locked` states which are overrided can be nil. + # Keep states definition to be evaluated before the state_machine block to + # avoid spec failures. If this gets evaluated after, the `merged` and `locked` + # states (which are overriden) can be nil. + # def self.available_state_names super + [:merged, :locked] end @@ -195,6 +195,7 @@ class MergeRequest < ApplicationRecord before_transition any => :merged do |merge_request| merge_request.merge_error = nil + merge_request.metrics.first_contribution = true if merge_request.first_contribution? end after_transition any => :opened do |merge_request| @@ -251,7 +252,9 @@ class MergeRequest < ApplicationRecord Gitlab::Timeless.timeless(merge_request, &block) end - after_transition any => [:unchecked, :cannot_be_merged_recheck, :checking, :cannot_be_merged_rechecking, :can_be_merged, :cannot_be_merged] do |merge_request, transition| + after_transition any => [:unchecked, :cannot_be_merged_recheck, :can_be_merged, :cannot_be_merged] do |merge_request, transition| + next if merge_request.skip_merge_status_trigger + merge_request.run_after_commit do GraphqlTriggers.merge_request_merge_status_updated(merge_request) end @@ -347,11 +350,12 @@ class MergeRequest < ApplicationRecord end scope :references_project, -> { references(:target_project) } scope :with_api_entity_associations, -> { - preload_routables - .preload(:assignees, :author, :unresolved_notes, :labels, :milestone, - :timelogs, :latest_merge_request_diff, :reviewers, - target_project: :project_feature, - metrics: [:latest_closed_by, :merged_by]) + preload_routables.preload( + :assignees, :author, :unresolved_notes, :labels, :milestone, + :timelogs, :latest_merge_request_diff, :reviewers, + target_project: :project_feature, + metrics: [:latest_closed_by, :merged_by] + ) } scope :with_csv_entity_associations, -> { preload(:assignees, :approved_by_users, :author, :milestone, metrics: [:merged_by]) } @@ -394,8 +398,10 @@ class MergeRequest < ApplicationRecord scope :preload_target_project, -> { preload(:target_project) } scope :preload_target_project_with_namespace, -> { preload(target_project: [:namespace]) } scope :preload_routables, -> do - preload(target_project: [:route, { namespace: :route }], - source_project: [:route, { namespace: :route }]) + preload( + target_project: [:route, { namespace: :route }], + source_project: [:route, { namespace: :route }] + ) end scope :preload_author, -> { preload(:author) } scope :preload_approved_by_users, -> { preload(:approved_by_users) } @@ -451,7 +457,12 @@ class MergeRequest < ApplicationRecord def self.total_time_to_merge join_metrics - .merge(MergeRequest::Metrics.with_valid_time_to_merge) + .where( + # Replicating the scope MergeRequest::Metrics.with_valid_time_to_merge + MergeRequest::Metrics.arel_table[:merged_at].gt( + MergeRequest::Metrics.arel_table[:created_at] + ) + ) .pick(MergeRequest::Metrics.time_to_merge_expression) end @@ -558,7 +569,7 @@ class MergeRequest < ApplicationRecord end def self.link_reference_pattern - @link_reference_pattern ||= super("merge_requests", Gitlab::Regex.merge_request) + @link_reference_pattern ||= compose_link_reference_pattern('merge_requests', Gitlab::Regex.merge_request) end def self.reference_valid?(reference) @@ -1011,8 +1022,7 @@ class MergeRequest < ApplicationRecord return true if target_project == source_project return true unless source_project_missing? - errors.add :validate_fork, - 'Source project is not a fork of the target project' + errors.add :validate_fork, 'Source project is not a fork of the target project' end def validate_reviewer_size_length @@ -1179,8 +1189,10 @@ class MergeRequest < ApplicationRecord alias_method :wip_title, :draft_title def mergeable?(skip_ci_check: false, skip_discussions_check: false) - return false unless mergeable_state?(skip_ci_check: skip_ci_check, - skip_discussions_check: skip_discussions_check) + return false unless mergeable_state?( + skip_ci_check: skip_ci_check, + skip_discussions_check: skip_discussions_check + ) check_mergeability @@ -1201,10 +1213,12 @@ class MergeRequest < ApplicationRecord end def mergeable_state?(skip_ci_check: false, skip_discussions_check: false) - additional_checks = execute_merge_checks(params: { - skip_ci_check: skip_ci_check, - skip_discussions_check: skip_discussions_check - }) + additional_checks = execute_merge_checks( + params: { + skip_ci_check: skip_ci_check, + skip_discussions_check: skip_discussions_check + } + ) additional_checks.success? end @@ -1693,7 +1707,7 @@ class MergeRequest < ApplicationRecord def compare_reports(service_class, current_user = nil, report_type = nil, additional_params = {}) with_reactive_cache(service_class.name, current_user&.id, report_type) do |data| unless service_class.new(project, current_user, id: id, report_type: report_type, additional_params: additional_params) - .latest?(comparison_base_pipeline(service_class.name), actual_head_pipeline, data) + .latest?(comparison_base_pipeline(service_class), actual_head_pipeline, data) raise InvalidateReactiveCache end @@ -1729,7 +1743,7 @@ class MergeRequest < ApplicationRecord raise NameError, service_class unless service_class < Ci::CompareReportsBaseService current_user = User.find_by(id: current_user_id) - service_class.new(project, current_user, id: id, report_type: report_type).execute(comparison_base_pipeline(identifier), actual_head_pipeline) + service_class.new(project, current_user, id: id, report_type: report_type).execute(comparison_base_pipeline(service_class), actual_head_pipeline) end MAX_RECENT_DIFF_HEAD_SHAS = 100 @@ -1870,8 +1884,9 @@ class MergeRequest < ApplicationRecord end end - def use_merge_base_pipeline_for_comparison?(service_class) - ALLOWED_TO_USE_MERGE_BASE_PIPELINE_FOR_COMPARISON[service_class]&.call(project) + # Overridden in EE + def use_merge_base_pipeline_for_comparison?(_) + false end def comparison_base_pipeline(service_class) @@ -1901,7 +1916,7 @@ class MergeRequest < ApplicationRecord # rubocop: enable CodeReuse/ServiceClass def first_contribution? - return false if project.team.max_member_access(author_id) > Gitlab::Access::GUEST + return metrics&.first_contribution if merged? & metrics.present? !project.merge_requests.merged.exists?(author_id: author_id) end @@ -1944,8 +1959,7 @@ class MergeRequest < ApplicationRecord super.merge(label_url_method: :project_merge_requests_url) end - override :ensure_metrics - def ensure_metrics + def ensure_metrics! MergeRequest::Metrics.record!(self) end diff --git a/app/models/merge_request/diff_llm_summary.rb b/app/models/merge_request/diff_llm_summary.rb new file mode 100644 index 00000000000..5e7d80712e2 --- /dev/null +++ b/app/models/merge_request/diff_llm_summary.rb @@ -0,0 +1,13 @@ +# rubocop:disable Style/ClassAndModuleChildren +# frozen_string_literal: true + +class MergeRequest::DiffLlmSummary < ApplicationRecord + belongs_to :merge_request_diff + belongs_to :user, optional: true + + validates :provider, presence: true + validates :content, presence: true, length: { maximum: 2056 } + + enum provider: { openai: 0 } +end +# rubocop:enable Style/ClassAndModuleChildren diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb index 87d8704561f..70216144035 100644 --- a/app/models/merge_request/metrics.rb +++ b/app/models/merge_request/metrics.rb @@ -2,6 +2,7 @@ class MergeRequest::Metrics < ApplicationRecord include IgnorableColumns + include DatabaseEventTracking belongs_to :merge_request, inverse_of: :metrics belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :pipeline_id @@ -24,16 +25,19 @@ class MergeRequest::Metrics < ApplicationRecord end def record!(mr) + inserted_columns = %i[merge_request_id target_project_id updated_at created_at] sql = <<~SQL - INSERT INTO #{self.table_name} (merge_request_id, target_project_id, updated_at, created_at) + INSERT INTO #{self.table_name} (#{inserted_columns.join(', ')}) VALUES (#{mr.id}, #{mr.target_project_id}, NOW(), NOW()) ON CONFLICT (merge_request_id) DO UPDATE SET target_project_id = EXCLUDED.target_project_id, updated_at = NOW() + RETURNING id, #{inserted_columns.join(', ')} SQL - connection.execute(sql) + result = connection.execute(sql).first + new(result).publish_database_create_event end end @@ -47,6 +51,31 @@ class MergeRequest::Metrics < ApplicationRecord with_valid_time_to_merge .pick(time_to_merge_expression) end + + SNOWPLOW_ATTRIBUTES = %i[ + id + merge_request_id + latest_build_started_at + latest_build_finished_at + first_deployed_to_production_at + merged_at + created_at + updated_at + pipeline_id + merged_by_id + latest_closed_by_id + latest_closed_at + first_comment_at + first_commit_at + last_commit_at + diff_size + modified_paths_size + commits_count + first_approved_at + first_reassigned_at + added_lines + removed_lines + ].freeze end MergeRequest::Metrics.prepend_mod_with('MergeRequest::Metrics') diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 1395b8ff162..0e699d7a81d 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -622,10 +622,12 @@ class MergeRequestDiff < ApplicationRecord end def diffs_in_batch_collection(batch_page, batch_size, diff_options:) - Gitlab::Diff::FileCollection::MergeRequestDiffBatch.new(self, - batch_page, - batch_size, - diff_options: diff_options) + Gitlab::Diff::FileCollection::MergeRequestDiffBatch.new( + self, + batch_page, + batch_size, + diff_options: diff_options + ) end def encode_in_base64?(diff_text) diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index 7e2efa2049b..fc08dd4d9c8 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -80,7 +80,7 @@ class MergeRequestDiffCommit < ApplicationRecord def self.prepare_commits_for_bulk_insert(commits) user_tuples = Set.new hashes = commits.map do |commit| - hash = commit.to_hash.except(:parent_ids) + hash = commit.to_hash.except(:parent_ids, :referenced_by) TRIM_USER_KEYS.each do |key| hash[key] = MergeRequest::DiffCommitUser.prepare(hash[key]) diff --git a/app/models/merge_requests_closing_issues.rb b/app/models/merge_requests_closing_issues.rb index 5c53cfd8c27..54cb6b7888b 100644 --- a/app/models/merge_requests_closing_issues.rb +++ b/app/models/merge_requests_closing_issues.rb @@ -17,10 +17,11 @@ class MergeRequestsClosingIssues < ApplicationRecord scope :accessible_by, ->(user) do joins(:merge_request) .joins('INNER JOIN project_features ON merge_requests.target_project_id = project_features.project_id') - .where('project_features.merge_requests_access_level >= :access OR EXISTS(:authorizations)', - access: ProjectFeature::ENABLED, - authorizations: user.authorizations_for_projects(min_access_level: Gitlab::Access::REPORTER, related_project_column: "merge_requests.target_project_id") - ) + .where( + 'project_features.merge_requests_access_level >= :access OR EXISTS(:authorizations)', + access: ProjectFeature::ENABLED, + authorizations: user.authorizations_for_projects(min_access_level: Gitlab::Access::REPORTER, related_project_column: "merge_requests.target_project_id") + ) end class << self diff --git a/app/models/milestone.rb b/app/models/milestone.rb index b0676c25f8e..d300b938fc0 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -8,6 +8,8 @@ class Milestone < ApplicationRecord include FromUnion include Importable include IidRoutes + include UpdatedAtFilterable + include EachBatch prepend_mod_with('Milestone') # rubocop: disable Cop/InjectEnterpriseEditionModule @@ -26,6 +28,7 @@ class Milestone < ApplicationRecord has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + scope :by_iid, ->(iid) { where(iid: iid) } scope :active, -> { with_state(:active) } scope :started, -> { active.where('milestones.start_date <= CURRENT_DATE') } scope :not_started, -> { active.where('milestones.start_date > CURRENT_DATE') } @@ -112,7 +115,7 @@ class Milestone < ApplicationRecord end def self.link_reference_pattern - @link_reference_pattern ||= super("milestones", /(?<milestone>\d+)/) + @link_reference_pattern ||= compose_link_reference_pattern('milestones', /(?<milestone>\d+)/) end def self.upcoming_ids(projects, groups) diff --git a/app/models/milestone_note.rb b/app/models/milestone_note.rb index 19171e682b7..14808158fd0 100644 --- a/app/models/milestone_note.rb +++ b/app/models/milestone_note.rb @@ -17,6 +17,7 @@ class MilestoneNote < SyntheticNote def note_text(html: false) format = milestone&.group_milestone? ? :name : :iid - event.remove? ? 'removed milestone' : "changed milestone to #{milestone.to_reference(project, format: format)}" + reference = milestone&.to_reference(project, format: format) + event.remove? ? "removed milestone #{reference}" : "changed milestone to #{reference}" end end diff --git a/app/models/ml/candidate.rb b/app/models/ml/candidate.rb index f973b00c568..6f4728a1d98 100644 --- a/app/models/ml/candidate.rb +++ b/app/models/ml/candidate.rb @@ -3,25 +3,35 @@ module Ml class Candidate < ApplicationRecord include Sortable + include AtomicInternalId + include IgnorableColumns - PACKAGE_PREFIX = 'ml_candidate_' + ignore_column :iid, remove_with: '16.0', remove_after: '2023-05-01' enum status: { running: 0, scheduled: 1, finished: 2, failed: 3, killed: 4 } - validates :iid, :experiment, presence: true + validates :eid, :experiment, presence: true validates :status, inclusion: { in: statuses.keys } belongs_to :experiment, class_name: 'Ml::Experiment' belongs_to :user + belongs_to :package, class_name: 'Packages::Package' + belongs_to :project + belongs_to :ci_build, class_name: 'Ci::Build', optional: true has_many :metrics, class_name: 'Ml::CandidateMetric' has_many :params, class_name: 'Ml::CandidateParam' has_many :metadata, class_name: 'Ml::CandidateMetadata' has_many :latest_metrics, -> { latest }, class_name: 'Ml::CandidateMetric', inverse_of: :candidate - attribute :iid, default: -> { SecureRandom.uuid } + attribute :eid, default: -> { SecureRandom.uuid } - scope :including_relationships, -> { includes(:latest_metrics, :params, :user) } + has_internal_id :internal_id, + scope: :project, + init: AtomicInternalId.project_init(self, :internal_id) + + scope :including_relationships, -> { includes(:latest_metrics, :params, :user, :package, :project, :ci_build) } scope :by_name, ->(name) { where("ml_candidates.name LIKE ?", "%#{sanitize_sql_like(name)}%") } # rubocop:disable GitlabSecurity/SqlInjection + scope :order_by_metric, ->(metric, direction) do subquery = Ml::CandidateMetric.latest.where(name: metric) column_expression = Arel::Table.new('latest')[:value] @@ -46,40 +56,34 @@ module Ml ) end - delegate :project_id, :project, to: :experiment + alias_attribute :artifact, :package + alias_attribute :iid, :internal_id + + delegate :package_name, to: :experiment def artifact_root "/#{package_name}/#{package_version}/" end - def artifact - artifact_lazy&.itself + def package_version + iid end - def artifact_lazy - BatchLoader.for(id).batch do |candidate_ids, loader| - Packages::Package - .joins("INNER JOIN ml_candidates ON packages_packages.name=(concat('#{PACKAGE_PREFIX}', ml_candidates.id))") - .where(ml_candidates: { id: candidate_ids }) - .find_each do |package| - loader.call(package.name.delete_prefix(PACKAGE_PREFIX).to_i, package) - end - end + def from_ci? + ci_build_id.present? end - def package_name - "#{PACKAGE_PREFIX}#{id}" - end + class << self + def with_project_id_and_eid(project_id, eid) + return unless project_id.present? && eid.present? - def package_version - '-' - end + find_by(project_id: project_id, eid: eid) + end - class << self def with_project_id_and_iid(project_id, iid) return unless project_id.present? && iid.present? - joins(:experiment).find_by(experiment: { project_id: project_id }, iid: iid) + find_by(project_id: project_id, internal_id: iid) end end end diff --git a/app/models/ml/candidate_metadata.rb b/app/models/ml/candidate_metadata.rb index 06b893c211f..1191051b1a3 100644 --- a/app/models/ml/candidate_metadata.rb +++ b/app/models/ml/candidate_metadata.rb @@ -4,9 +4,9 @@ module Ml class CandidateMetadata < ApplicationRecord validates :candidate, presence: true validates :name, - length: { maximum: 250 }, - presence: true, - uniqueness: { scope: :candidate, message: ->(candidate, _) { "'#{candidate.name}' already taken" } } + length: { maximum: 250 }, + presence: true, + uniqueness: { scope: :candidate, message: ->(candidate, _) { "'#{candidate.name}' already taken" } } validates :value, length: { maximum: 5000 }, presence: true belongs_to :candidate, class_name: 'Ml::Candidate' diff --git a/app/models/ml/experiment.rb b/app/models/ml/experiment.rb index 7bb80a170c5..d1277efac7b 100644 --- a/app/models/ml/experiment.rb +++ b/app/models/ml/experiment.rb @@ -4,6 +4,8 @@ module Ml class Experiment < ApplicationRecord include AtomicInternalId + PACKAGE_PREFIX = 'ml_experiment_' + validates :name, :project, presence: true validates :name, uniqueness: { scope: :project, message: "should be unique in the project" } @@ -20,6 +22,10 @@ module Ml has_internal_id :iid, scope: :project + def package_name + "#{PACKAGE_PREFIX}#{iid}" + end + class << self def by_project_id_and_iid(project_id, iid) find_by(project_id: project_id, iid: iid) @@ -32,6 +38,20 @@ module Ml def by_project_id(project_id) where(project_id: project_id).order(id: :desc) end + + def package_for_experiment?(package_name) + return false unless package_name&.starts_with?(PACKAGE_PREFIX) + + iid = package_name.delete_prefix(PACKAGE_PREFIX) + + numeric?(iid) + end + + private + + def numeric?(value) + value.match?(/\A\d+\z/) + end end end end diff --git a/app/models/ml/experiment_metadata.rb b/app/models/ml/experiment_metadata.rb index 93496807e1a..37cb2714268 100644 --- a/app/models/ml/experiment_metadata.rb +++ b/app/models/ml/experiment_metadata.rb @@ -4,9 +4,9 @@ module Ml class ExperimentMetadata < ApplicationRecord validates :experiment, presence: true validates :name, - length: { maximum: 250 }, - presence: true, - uniqueness: { scope: :experiment, message: ->(exp, _) { "'#{exp.name}' already taken" } } + length: { maximum: 250 }, + presence: true, + uniqueness: { scope: :experiment, message: ->(exp, _) { "'#{exp.name}' already taken" } } validates :value, length: { maximum: 5000 }, presence: true belongs_to :experiment, class_name: 'Ml::Experiment' diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 9d9b09e3562..7c6fa24cd4d 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -16,6 +16,7 @@ class Namespace < ApplicationRecord include EachBatch include BlocksUnsafeSerialization include Ci::NamespaceSettings + include Referable # Tells ActiveRecord not to store the full class name, in order to save some space # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69794 @@ -35,12 +36,6 @@ class Namespace < ApplicationRecord SHARED_RUNNERS_SETTINGS = [SR_DISABLED_AND_UNOVERRIDABLE, SR_DISABLED_WITH_OVERRIDE, SR_DISABLED_AND_OVERRIDABLE, SR_ENABLED].freeze URL_MAX_LENGTH = 255 - # This date is just a placeholder until namespace storage enforcement timeline is confirmed at which point - # this should be replaced, see https://about.gitlab.com/pricing/faq-efficient-free-tier/#user-limits-on-gitlab-saas-free-tier - MIN_STORAGE_ENFORCEMENT_DATE = 3.months.from_now.to_date - # https://gitlab.com/gitlab-org/gitlab/-/issues/367531 - MIN_STORAGE_ENFORCEMENT_USAGE = 5.gigabytes - cache_markdown_field :description, pipeline: :description has_many :projects, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -51,7 +46,8 @@ class Namespace < ApplicationRecord has_one :namespace_statistics has_one :namespace_route, foreign_key: :namespace_id, autosave: false, inverse_of: :namespace, class_name: 'Route' has_many :namespace_members, foreign_key: :member_namespace_id, inverse_of: :member_namespace, class_name: 'Member' - has_many :member_roles + + has_one :namespace_ldap_settings, inverse_of: :namespace, class_name: 'Namespaces::LdapSetting', autosave: true has_many :runner_namespaces, inverse_of: :namespace, class_name: 'Ci::RunnerNamespace' has_many :runners, through: :runner_namespaces, source: :runner, class_name: 'Ci::Runner' @@ -97,6 +93,7 @@ class Namespace < ApplicationRecord validates :path, presence: true, length: { maximum: URL_MAX_LENGTH } + validate :container_registry_namespace_path_validation validates :path, namespace_path: true, if: ->(n) { !n.project_namespace? } # Project path validator is used for project namespaces for now to assure @@ -127,19 +124,18 @@ class Namespace < ApplicationRecord delegate :name, to: :owner, allow_nil: true, prefix: true delegate :avatar_url, to: :owner, allow_nil: true delegate :prevent_sharing_groups_outside_hierarchy, :prevent_sharing_groups_outside_hierarchy=, - to: :namespace_settings, allow_nil: true + to: :namespace_settings, allow_nil: true delegate :show_diff_preview_in_email, :show_diff_preview_in_email?, :show_diff_preview_in_email=, - to: :namespace_settings + to: :namespace_settings delegate :runner_registration_enabled, :runner_registration_enabled?, :runner_registration_enabled=, - to: :namespace_settings + to: :namespace_settings delegate :allow_runner_registration_token, - :allow_runner_registration_token?, - :allow_runner_registration_token=, - to: :namespace_settings + :allow_runner_registration_token=, + to: :namespace_settings delegate :maven_package_requests_forwarding, - :pypi_package_requests_forwarding, - :npm_package_requests_forwarding, - to: :package_settings + :pypi_package_requests_forwarding, + :npm_package_requests_forwarding, + to: :package_settings before_save :update_new_emails_created_column, if: -> { emails_disabled_changed? } before_create :sync_share_with_group_lock_with_parent @@ -244,27 +240,42 @@ class Namespace < ApplicationRecord def clean_path(path, limited_to: Namespace.all) slug = Gitlab::Slug::Path.new(path).generate path = Namespaces::RandomizedSuffixPath.new(slug) - Uniquify.new.string(path) { |s| limited_to.find_by_path_or_name(s) } + Gitlab::Utils::Uniquify.new.string(path) { |s| limited_to.find_by_path_or_name(s) } end def clean_name(value) value.scan(Gitlab::Regex.group_name_regex_chars).join(' ') end - def find_by_pages_host(host) - gitlab_host = "." + Settings.pages.host.downcase - host = host.downcase - return unless host.ends_with?(gitlab_host) + def top_most + by_parent(nil) + end - name = host.delete_suffix(gitlab_host) - Namespace.top_most.by_path(name) + def reference_prefix + User.reference_prefix end - def top_most - by_parent(nil) + def reference_pattern + User.reference_pattern end end + def to_reference_base(from = nil, full: false) + return full_path if full || cross_namespace_reference?(from) + return path if cross_project_reference?(from) + end + + def to_reference(*) + "#{self.class.reference_prefix}#{full_path}" + end + + def container_registry_namespace_path_validation + return if Feature.disabled?(:restrict_special_characters_in_namespace_path, self) + return if !path_changed? || path.match?(Gitlab::Regex.oci_repository_path_regex) + + errors.add(:path, Gitlab::Regex.oci_repository_path_regex_message) + end + def package_settings package_setting_relation || build_package_setting_relation end @@ -286,11 +297,15 @@ class Namespace < ApplicationRecord end def any_project_has_container_registry_tags? - all_projects.includes(:container_repositories).any?(&:has_container_registry_tags?) + first_project_with_container_registry_tags.present? end def first_project_with_container_registry_tags - all_projects.find(&:has_container_registry_tags?) + if ContainerRegistry::GitlabApiClient.supports_gitlab_api? + ContainerRegistry::GitlabApiClient.one_project_with_container_registry_tag(full_path) + else + all_projects.includes(:container_repositories).find(&:has_container_registry_tags?) + end end def send_update_instructions @@ -381,12 +396,8 @@ class Namespace < ApplicationRecord # Includes projects from this namespace and projects from all subgroups # that belongs to this namespace def all_projects - if Feature.enabled?(:recursive_approach_for_all_projects) - namespace = user_namespace? ? self : self_and_descendant_ids - Project.where(namespace: namespace) - else - Project.inside_path(full_path) - end + namespace = user_namespace? ? self : self_and_descendant_ids + Project.where(namespace: namespace) end def has_parent? @@ -473,18 +484,6 @@ class Namespace < ApplicationRecord ContainerRepository.for_project_id(all_projects) end - def pages_virtual_domain - cache = if Feature.enabled?(:cache_pages_domain_api, root_ancestor) - ::Gitlab::Pages::CacheControl.for_namespace(root_ancestor.id) - end - - Pages::VirtualDomain.new( - projects: all_projects_with_pages.includes(:route, :project_feature, pages_metadatum: :pages_deployment), - trim_prefix: full_path, - cache: cache - ) - end - def any_project_with_pages_deployed? all_projects.with_pages_deployed.any? end @@ -577,12 +576,6 @@ class Namespace < ApplicationRecord Feature.enabled?(:block_issue_repositioning, self, type: :ops) end - def storage_enforcement_date - return Date.current if Feature.enabled?(:namespace_storage_limit_bypass_date_check, self) - - MIN_STORAGE_ENFORCEMENT_DATE - end - def certificate_based_clusters_enabled? cluster_enabled_granted? || certificate_based_clusters_enabled_ff? end @@ -599,8 +592,48 @@ class Namespace < ApplicationRecord namespace_settings&.all_ancestors_have_runner_registration_enabled? end + def allow_runner_registration_token? + !!namespace_settings&.allow_runner_registration_token? + end + + def all_projects_with_pages + all_projects.with_pages_deployed.includes( + :route, + :project_setting, + :project_feature, + pages_metadatum: :pages_deployment + ) + end + private + def cross_namespace_reference?(from) + return false if from == self + + comparable_namespace_id = project_namespace? ? parent_id : id + + case from + when Project + from.namespace_id != comparable_namespace_id + when Namespaces::ProjectNamespace + from.parent_id != comparable_namespace_id + when Namespace + parent != from + when User + true + end + end + + # Check if a reference is being done cross-project + def cross_project_reference?(from) + case from + when Project + from.project_namespace_id != id + else + from && self != from + end + end + def update_new_emails_created_column return if namespace_settings.nil? return if namespace_settings.emails_enabled == !emails_disabled @@ -630,10 +663,6 @@ class Namespace < ApplicationRecord end end - def all_projects_with_pages - all_projects.with_pages_deployed - end - def parent_changed? parent_id_changed? end diff --git a/app/models/namespace/aggregation_schedule.rb b/app/models/namespace/aggregation_schedule.rb index cd7d4fc409a..e08c08f9ced 100644 --- a/app/models/namespace/aggregation_schedule.rb +++ b/app/models/namespace/aggregation_schedule.rb @@ -12,11 +12,11 @@ class Namespace::AggregationSchedule < ApplicationRecord after_create :schedule_root_storage_statistics - def self.default_lease_timeout - if Feature.enabled?(:remove_namespace_aggregator_delay) - 30.minutes.to_i + def default_lease_timeout + if Feature.enabled?(:reduce_aggregation_schedule_lease, namespace.root_ancestor) + 2.minutes.to_i else - 1.hour.to_i + 30.minutes.to_i end end @@ -27,7 +27,7 @@ class Namespace::AggregationSchedule < ApplicationRecord .perform_async(namespace_id) Namespaces::RootStatisticsWorker - .perform_in(self.class.default_lease_timeout, namespace_id) + .perform_in(default_lease_timeout, namespace_id) end end end @@ -36,7 +36,7 @@ class Namespace::AggregationSchedule < ApplicationRecord # Used by ExclusiveLeaseGuard def lease_timeout - self.class.default_lease_timeout + default_lease_timeout end # Used by ExclusiveLeaseGuard diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb index 77974a0f36b..0443e1d9231 100644 --- a/app/models/namespace/root_storage_statistics.rb +++ b/app/models/namespace/root_storage_statistics.rb @@ -45,8 +45,9 @@ class Namespace::RootStorageStatistics < ApplicationRecord attributes_from_project_statistics.merge!( attributes_from_personal_snippets, attributes_from_namespace_statistics, - attributes_for_container_registry_size - ) { |key, v1, v2| v1 + v2 } + attributes_for_container_registry_size, + attributes_for_forks_statistics + ) { |_, v1, v2| v1 + v2 } end def attributes_for_container_registry_size @@ -58,6 +59,32 @@ class Namespace::RootStorageStatistics < ApplicationRecord }.with_indifferent_access end + def attributes_for_forks_statistics + return {} unless ::Feature.enabled?(:root_storage_statistics_calculate_forks, namespace) + + visibility_levels_to_storage_size_columns = { + Gitlab::VisibilityLevel::PRIVATE => :private_forks_storage_size, + Gitlab::VisibilityLevel::INTERNAL => :internal_forks_storage_size, + Gitlab::VisibilityLevel::PUBLIC => :public_forks_storage_size + } + + defaults = { + private_forks_storage_size: 0, + internal_forks_storage_size: 0, + public_forks_storage_size: 0 + } + + defaults.merge(for_forks_statistics.transform_keys { |k| visibility_levels_to_storage_size_columns[k] }) + end + + def for_forks_statistics + all_projects + .joins([:statistics, :fork_network]) + .where('fork_networks.root_project_id != projects.id') + .group('projects.visibility_level') + .sum('project_statistics.storage_size') + end + def attributes_from_project_statistics from_project_statistics .take diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb index aeb4d7a5694..e7f6db38047 100644 --- a/app/models/namespace_setting.rb +++ b/app/models/namespace_setting.rb @@ -13,6 +13,7 @@ class NamespaceSetting < ApplicationRecord enum enabled_git_access_protocol: { all: 0, ssh: 1, http: 2 }, _suffix: true validates :enabled_git_access_protocol, inclusion: { in: enabled_git_access_protocols.keys } + validates :code_suggestions, allow_nil: false, inclusion: { in: [true, false] } validate :allow_mfa_for_group validate :allow_resource_access_token_creation_for_group @@ -63,6 +64,8 @@ class NamespaceSetting < ApplicationRecord end def all_ancestors_have_runner_registration_enabled? + return false unless Gitlab::CurrentSettings.valid_runner_registrars.include?('group') + return true unless namespace.has_parent? !self.class.where(namespace_id: namespace.ancestors, runner_registration_enabled: false).exists? diff --git a/app/models/namespaces/ldap_setting.rb b/app/models/namespaces/ldap_setting.rb new file mode 100644 index 00000000000..73125d347cc --- /dev/null +++ b/app/models/namespaces/ldap_setting.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Namespaces + class LdapSetting < ApplicationRecord + belongs_to :namespace, inverse_of: :namespace_ldap_settings + validates :namespace, presence: true + + self.primary_key = :namespace_id + self.table_name = 'namespace_ldap_settings' + end +end diff --git a/app/models/namespaces/project_namespace.rb b/app/models/namespaces/project_namespace.rb index 2a2ea11ddc5..cf2612b7f33 100644 --- a/app/models/namespaces/project_namespace.rb +++ b/app/models/namespaces/project_namespace.rb @@ -11,6 +11,8 @@ module Namespaces alias_attribute :namespace_id, :parent_id has_one :project, foreign_key: :project_namespace_id, inverse_of: :project_namespace + delegate :execute_hooks, :execute_integrations, to: :project, allow_nil: true + def self.sti_name 'Project' end diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 0e9760832af..9006f104c64 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -117,17 +117,13 @@ module Namespaces traversal_ids.present? end - def use_traversal_ids_for_root_ancestor? - return false unless Feature.enabled?(:use_traversal_ids_for_root_ancestor) - - traversal_ids.present? - end - def root_ancestor - return super unless use_traversal_ids_for_root_ancestor? - strong_memoize(:root_ancestor) do - if parent_id.nil? + if association(:parent).loaded? && parent.present? + # This case is possible when parent has not been persisted or we're inside a transaction. + parent.root_ancestor + elsif parent_id.nil? + # There is no parent, so we are the root ancestor. self else Namespace.find_by(id: traversal_ids.first) @@ -215,6 +211,16 @@ module Namespaces hierarchy_order == :desc ? traversal_ids : traversal_ids.reverse end + def parent=(obj) + super(obj) + set_traversal_ids + end + + def parent_id=(id) + super(id) + set_traversal_ids + end + private attr_accessor :transient_traversal_ids @@ -232,11 +238,11 @@ module Namespaces end def set_traversal_ids + return if id.blank? + # This is a temporary guard and will be removed. return if is_a?(Namespaces::ProjectNamespace) - return unless Feature.enabled?(:set_traversal_ids_on_save, root_ancestor) - self.transient_traversal_ids = if parent_id parent.traversal_ids + [id] else @@ -244,7 +250,7 @@ module Namespaces end # Clear root_ancestor memo if changed. - if read_attribute(traversal_ids)&.first != transient_traversal_ids.first + if read_attribute(:traversal_ids)&.first != transient_traversal_ids.first clear_memoization(:root_ancestor) end diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index 843de9bce33..792964a6c7f 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -27,9 +27,11 @@ module Namespaces def self_and_ancestors(include_self: true, upto: nil, hierarchy_order: nil) return super unless use_traversal_ids_for_ancestor_scopes? - self_and_ancestors_from_inner_join(include_self: include_self, - upto: upto, hierarchy_order: - hierarchy_order) + self_and_ancestors_from_inner_join( + include_self: include_self, + upto: upto, hierarchy_order: + hierarchy_order + ) end def self_and_ancestor_ids(include_self: true) diff --git a/app/models/note.rb b/app/models/note.rb index a64f7311725..ac2b54629ae 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -60,6 +60,9 @@ class Note < ApplicationRecord # Attribute used to store the attributes that have been changed by quick actions. attr_writer :commands_changes + # Attribute used to store the quick action command names. + attr_accessor :command_names + # Attribute used to determine whether keep_around_commits will be skipped for diff notes. attr_accessor :skip_keep_around_commits @@ -84,6 +87,7 @@ class Note < ApplicationRecord inverse_of: :note, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_many :events, as: :target, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent has_one :system_note_metadata + has_one :note_metadata, inverse_of: :note, class_name: 'Notes::NoteMetadata' has_one :note_diff_file, inverse_of: :diff_note, foreign_key: :diff_note_id has_many :diff_note_positions @@ -92,6 +96,8 @@ class Note < ApplicationRecord delegate :name, :email, to: :author, prefix: true delegate :title, to: :noteable, allow_nil: true + accepts_nested_attributes_for :note_metadata + validates :note, presence: true validates :note, length: { maximum: Gitlab::Database::MAX_TEXT_SIZE_LIMIT } validates :project, presence: true, if: :for_project_noteable? @@ -165,11 +171,20 @@ class Note < ApplicationRecord scope :with_associations, -> do # FYI noteable cannot be loaded for LegacyDiffNote for commits - includes(:author, :noteable, :updated_by, - project: [:project_members, :namespace, { group: [:group_members] }]) + includes( + :author, :noteable, :updated_by, + project: [:project_members, :namespace, { group: [:group_members] }] + ) end scope :with_metadata, -> { includes(:system_note_metadata) } - scope :with_web_entity_associations, -> { preload(:project, :author, :noteable) } + + scope :without_hidden, -> { + if Feature.enabled?(:hidden_notes) + where_not_exists(Users::BannedUser.where('notes.author_id = banned_users.user_id')) + else + all + end + } scope :for_note_or_capitalized_note, ->(text) { where(note: [text, text.capitalize]) } scope :like_note_or_capitalized_note, ->(text) { where('(note LIKE ? OR note LIKE ?)', text, text.capitalize) } @@ -288,6 +303,10 @@ class Note < ApplicationRecord def cherry_picked_merge_requests(shas) where(noteable_type: 'MergeRequest', commit_id: shas).select(:noteable_id) end + + def with_web_entity_associations + preload(:project, :author, :noteable) + end end # rubocop: disable CodeReuse/ServiceClass @@ -330,6 +349,10 @@ class Note < ApplicationRecord noteable_type == "Issue" end + def for_work_item? + noteable.is_a?(WorkItem) + end + def for_merge_request? noteable_type == "MergeRequest" end @@ -382,8 +405,6 @@ class Note < ApplicationRecord project.merge_requests.by_commit_sha(commit_id) elsif for_merge_request? MergeRequest.id_in(noteable_id) - else - nil end end diff --git a/app/models/note_diff_file.rb b/app/models/note_diff_file.rb index 4238de0a2f8..e4936de7b40 100644 --- a/app/models/note_diff_file.rb +++ b/app/models/note_diff_file.rb @@ -19,9 +19,11 @@ class NoteDiffFile < ApplicationRecord def raw_diff_file raw_diff = Gitlab::Git::Diff.new(to_hash) - Gitlab::Diff::File.new(raw_diff, - repository: project.repository, - diff_refs: original_position.diff_refs, - unique_identifier: id) + Gitlab::Diff::File.new( + raw_diff, + repository: project.repository, + diff_refs: original_position.diff_refs, + unique_identifier: id + ) end end diff --git a/app/models/notes/note_metadata.rb b/app/models/notes/note_metadata.rb new file mode 100644 index 00000000000..54c3688170f --- /dev/null +++ b/app/models/notes/note_metadata.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Notes + class NoteMetadata < ApplicationRecord + self.table_name = :note_metadata + + EMAIL_PARTICIPANT_LENGTH = 255 + + belongs_to :note, inverse_of: :note_metadata + + alias_attribute :external_author, :email_participant + + before_save :ensure_email_participant_length + + private + + def ensure_email_participant_length + return unless email_participant.present? + + self.email_participant = email_participant.truncate(EMAIL_PARTICIPANT_LENGTH) + end + end +end diff --git a/app/models/oauth_access_token.rb b/app/models/oauth_access_token.rb index 8e79a750793..601381f1c65 100644 --- a/app/models/oauth_access_token.rb +++ b/app/models/oauth_access_token.rb @@ -4,6 +4,8 @@ class OauthAccessToken < Doorkeeper::AccessToken belongs_to :resource_owner, class_name: 'User' belongs_to :application, class_name: 'Doorkeeper::Application' + validates :expires_in, presence: true + alias_attribute :user, :resource_owner scope :latest_per_application, -> { select('distinct on(application_id) *').order(application_id: :desc, created_at: :desc) } diff --git a/app/models/onboarding/completion.rb b/app/models/onboarding/completion.rb index 269283df826..afbd671f82e 100644 --- a/app/models/onboarding/completion.rb +++ b/app/models/onboarding/completion.rb @@ -5,66 +5,66 @@ module Onboarding include Gitlab::Utils::StrongMemoize include Gitlab::Experiment::Dsl - ACTION_ISSUE_IDS = { - trial_started: 2, - required_mr_approvals_enabled: 11, - code_owners_enabled: 10 - }.freeze - ACTION_PATHS = [ :pipeline_created, + :trial_started, + :required_mr_approvals_enabled, + :code_owners_enabled, :issue_created, :git_write, :merge_request_created, - :user_added + :user_added, + :license_scanning_run, + :secure_dependency_scanning_run, + :secure_dast_run ].freeze - def initialize(namespace, current_user = nil) - @namespace = namespace + def initialize(project, current_user = nil) + @project = project + @namespace = project.namespace @current_user = current_user end def percentage return 0 unless onboarding_progress - attributes = onboarding_progress.attributes.symbolize_keys - total_actions = action_columns.count - completed_actions = action_columns.count { |column| attributes[column].present? } + completed_actions = action_columns.count { |column| completed?(column) } (completed_actions.to_f / total_actions * 100).round end + def completed?(column) + if column == :code_added + repository.commit_count > 1 || repository.branch_count > 1 + else + attributes[column].present? + end + end + private - def onboarding_progress - strong_memoize(:onboarding_progress) do - ::Onboarding::Progress.find_by(namespace: namespace) - end + def repository + project.repository end + strong_memoize_attr :repository - def action_columns - strong_memoize(:action_columns) do - tracked_actions.map { |action_key| ::Onboarding::Progress.column_name(action_key) } - end + def attributes + onboarding_progress.attributes.symbolize_keys end + strong_memoize_attr :attributes - def tracked_actions - ACTION_ISSUE_IDS.keys + ACTION_PATHS + deploy_section_tracked_actions + def onboarding_progress + ::Onboarding::Progress.find_by(namespace: namespace) end + strong_memoize_attr :onboarding_progress - def deploy_section_tracked_actions - experiment( - :security_actions_continuous_onboarding, - namespace: namespace, - user: current_user, - sticky_to: current_user - ) do |e| - e.control { [:security_scan_enabled] } - e.candidate { [:license_scanning_run, :secure_dependency_scanning_run, :secure_dast_run] } - end.run + def action_columns + [:code_added] + + ACTION_PATHS.map { |action_key| ::Onboarding::Progress.column_name(action_key) } end + strong_memoize_attr :action_columns - attr_reader :namespace, :current_user + attr_reader :project, :namespace, :current_user end end diff --git a/app/models/operations/feature_flag.rb b/app/models/operations/feature_flag.rb index 0df8c87f73f..6876af09c2c 100644 --- a/app/models/operations/feature_flag.rb +++ b/app/models/operations/feature_flag.rb @@ -72,7 +72,7 @@ module Operations end def link_reference_pattern - @link_reference_pattern ||= super("feature_flags", %r{(?<feature_flag>\d+)/edit}) + @link_reference_pattern ||= compose_link_reference_pattern('feature_flags', %r{(?<feature_flag>\d+)/edit}) end def reference_postfix diff --git a/app/models/organization.rb b/app/models/organization.rb new file mode 100644 index 00000000000..cfbbbf1183e --- /dev/null +++ b/app/models/organization.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class Organization < ApplicationRecord + DEFAULT_ORGANIZATION_ID = 1 + + scope :without_default, -> { where.not(id: DEFAULT_ORGANIZATION_ID) } + + before_destroy :check_if_default_organization + + validates :name, + presence: true, + length: { maximum: 255 }, + uniqueness: { case_sensitive: false } + + def default? + id == DEFAULT_ORGANIZATION_ID + end + + private + + def check_if_default_organization + return unless default? + + raise ActiveRecord::RecordNotDestroyed, _('Cannot delete the default organization') + end +end diff --git a/app/models/packages/debian.rb b/app/models/packages/debian.rb index 9c615c20250..2b8d0a4f51e 100644 --- a/app/models/packages/debian.rb +++ b/app/models/packages/debian.rb @@ -10,6 +10,10 @@ module Packages LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze + EMPTY_FILE_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'.freeze + + INCOMING_PACKAGE_NAME = 'incoming' + def self.table_name_prefix 'packages_debian_' end diff --git a/app/models/packages/debian/file_metadatum.rb b/app/models/packages/debian/file_metadatum.rb index eb1b03a8e9d..325ae0c468e 100644 --- a/app/models/packages/debian/file_metadatum.rb +++ b/app/models/packages/debian/file_metadatum.rb @@ -1,59 +1,69 @@ # frozen_string_literal: true -class Packages::Debian::FileMetadatum < ApplicationRecord - self.primary_key = :package_file_id +module Packages + module Debian + class FileMetadatum < ApplicationRecord + include UpdatedAtFilterable - belongs_to :package_file, inverse_of: :debian_file_metadatum + self.primary_key = :package_file_id - validates :package_file, presence: true - validate :valid_debian_package_type + belongs_to :package_file, inverse_of: :debian_file_metadatum - enum file_type: { - unknown: 1, source: 2, dsc: 3, deb: 4, udeb: 5, buildinfo: 6, changes: 7 - } + validates :package_file, presence: true + validate :valid_debian_package_type - validates :file_type, presence: true - validates :file_type, inclusion: { in: %w[unknown] }, - if: -> { package_file&.package&.debian_incoming? || package_file&.package&.processing? } - validates :file_type, - inclusion: { in: %w[source dsc deb udeb buildinfo changes] }, - if: -> { package_file&.package&.debian_package? && !package_file&.package&.processing? } + enum file_type: { + unknown: 1, source: 2, dsc: 3, deb: 4, udeb: 5, buildinfo: 6, changes: 7, ddeb: 8 + } - validates :component, - presence: true, - format: { with: Gitlab::Regex.debian_component_regex }, - if: :requires_component? - validates :component, absence: true, unless: :requires_component? + validates :file_type, presence: true + validates :file_type, inclusion: { in: %w[unknown] }, + if: -> { package_file&.package&.debian_incoming? || package_file&.package&.processing? } + validates :file_type, + inclusion: { in: %w[source dsc deb udeb buildinfo changes ddeb] }, + if: -> { package_file&.package&.debian_package? && !package_file&.package&.processing? } - validates :architecture, - presence: true, - format: { with: Gitlab::Regex.debian_architecture_regex }, - if: :requires_architecture? - validates :architecture, absence: true, unless: :requires_architecture? + validates :component, + presence: true, + format: { with: Gitlab::Regex.debian_component_regex }, + if: :requires_component? + validates :component, absence: true, unless: :requires_component? - validates :fields, - presence: true, - json_schema: { filename: "debian_fields" }, - if: :requires_fields? - validates :fields, absence: true, unless: :requires_fields? + validates :architecture, + presence: true, + format: { with: Gitlab::Regex.debian_architecture_regex }, + if: :requires_architecture? + validates :architecture, absence: true, unless: :requires_architecture? - private + validates :fields, + presence: true, + json_schema: { filename: "debian_fields" }, + if: :requires_fields? + validates :fields, absence: true, unless: :requires_fields? - def valid_debian_package_type - return if package_file&.package&.debian? + scope :with_file_type, ->(file_type) do + where(file_type: file_type) + end - errors.add(:package_file, _('Package type must be Debian')) - end + private - def requires_architecture? - deb? || udeb? - end + def valid_debian_package_type + return if package_file&.package&.debian? - def requires_component? - source? || dsc? || requires_architecture? || buildinfo? - end + errors.add(:package_file, _('Package type must be Debian')) + end + + def requires_architecture? + deb? || udeb? || ddeb? + end + + def requires_component? + source? || dsc? || requires_architecture? || buildinfo? + end - def requires_fields? - dsc? || requires_architecture? || buildinfo? || changes? + def requires_fields? + dsc? || requires_architecture? || buildinfo? || changes? + end + end end end diff --git a/app/models/packages/dependency.rb b/app/models/packages/dependency.rb index ad3944b5f21..c39b46dcc20 100644 --- a/app/models/packages/dependency.rb +++ b/app/models/packages/dependency.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true class Packages::Dependency < ApplicationRecord + include EachBatch + has_many :dependency_links, class_name: 'Packages::DependencyLink' validates :name, :version_pattern, presence: true @@ -41,6 +43,11 @@ class Packages::Dependency < ApplicationRecord pluck(:id, :name) end + def self.orphaned + subquery = Packages::DependencyLink.where(Packages::DependencyLink.arel_table[:dependency_id].eq(Packages::Dependency.arel_table[:id])) + where_not_exists(subquery) + end + def orphaned? self.dependency_links.empty? end diff --git a/app/models/packages/event.rb b/app/models/packages/event.rb index bb2c33594e5..d93c22adcda 100644 --- a/app/models/packages/event.rb +++ b/app/models/packages/event.rb @@ -1,61 +1,60 @@ # frozen_string_literal: true -class Packages::Event < ApplicationRecord - belongs_to :package, optional: true - - UNIQUE_EVENTS_ALLOWED = %i[push_package delete_package pull_package pull_symbol_package push_symbol_package].freeze - EVENT_SCOPES = ::Packages::Package.package_types.merge(container: 1000, tag: 1001, dependency_proxy: 1002).freeze - - EVENT_PREFIX = "i_package" - - enum event_scope: EVENT_SCOPES - - enum event_type: { - push_package: 0, - delete_package: 1, - pull_package: 2, - search_package: 3, - list_package: 4, - list_repositories: 5, - delete_repository: 6, - delete_tag: 7, - delete_tag_bulk: 8, - list_tags: 9, - cli_metadata: 10, - pull_symbol_package: 11, - push_symbol_package: 12, - pull_manifest: 13, - pull_manifest_from_cache: 14, - pull_blob: 15, - pull_blob_from_cache: 16 - } - - enum originator_type: { user: 0, deploy_token: 1, guest: 2 } - - # Remove some of the events, for now, so we don't hammer Redis too hard. - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/280770 - def self.event_allowed?(event_type) - return true if UNIQUE_EVENTS_ALLOWED.include?(event_type.to_sym) - - false - end - - # counter names for unique user tracking (for MAU) - def self.unique_counters_for(event_scope, event_type, originator_type) - return [] unless event_allowed?(event_type) - return [] if originator_type.to_s == 'guest' - - ["#{EVENT_PREFIX}_#{event_scope}_#{originator_type}"] - end - - # total counter names for tracking number of events - def self.counters_for(event_scope, event_type, originator_type) - return [] unless event_allowed?(event_type) - - [ - "#{EVENT_PREFIX}_#{event_type}", - "#{EVENT_PREFIX}_#{event_type}_by_#{originator_type}", - "#{EVENT_PREFIX}_#{event_scope}_#{event_type}" - ] +module Packages + class Event + UNIQUE_EVENTS_ALLOWED = %i[push_package delete_package pull_package pull_symbol_package push_symbol_package].freeze + EVENT_SCOPES = ::Packages::Package.package_types.merge(container: 1000, tag: 1001, dependency_proxy: 1002).freeze + + EVENT_PREFIX = "i_package" + + EVENT_TYPES = %i[ + push_package + delete_package + pull_package + search_package + list_package + list_repositories + delete_repository + delete_tag + delete_tag_bulk + list_tags + create_tag + cli_metadata + pull_symbol_package + push_symbol_package + pull_manifest + pull_manifest_from_cache + pull_blob + pull_blob_from_cache + ].freeze + + ORIGINATOR_TYPES = %i[user deploy_token guest].freeze + + # Remove some of the events, for now, so we don't hammer Redis too hard. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/280770 + def self.event_allowed?(event_type) + return true if UNIQUE_EVENTS_ALLOWED.include?(event_type.to_sym) + + false + end + + # counter names for unique user tracking (for MAU) + def self.unique_counters_for(event_scope, event_type, originator_type) + return [] unless event_allowed?(event_type) + return [] if originator_type.to_s == 'guest' + + ["#{EVENT_PREFIX}_#{event_scope}_#{originator_type}"] + end + + # total counter names for tracking number of events + def self.counters_for(event_scope, event_type, originator_type) + return [] unless event_allowed?(event_type) + + [ + "#{EVENT_PREFIX}_#{event_type}", + "#{EVENT_PREFIX}_#{event_type}_by_#{originator_type}", + "#{EVENT_PREFIX}_#{event_scope}_#{event_type}" + ] + end end end diff --git a/app/models/packages/npm/metadata_cache.rb b/app/models/packages/npm/metadata_cache.rb new file mode 100644 index 00000000000..7a7c66d7a45 --- /dev/null +++ b/app/models/packages/npm/metadata_cache.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Packages + module Npm + class MetadataCache < ApplicationRecord + include FileStoreMounter + + belongs_to :project, inverse_of: :npm_metadata_caches + + validates :file, :object_storage_key, :package_name, :project, :size, presence: true + validates :package_name, uniqueness: { scope: :project_id } + validates :package_name, format: { with: Gitlab::Regex.package_name_regex } + validates :package_name, format: { with: Gitlab::Regex.npm_package_name_regex } + + mount_file_store_uploader MetadataCacheUploader + + before_validation :set_object_storage_key + attr_readonly :object_storage_key + + def self.find_or_build(package_name:, project_id:) + find_or_initialize_by( + package_name: package_name, + project_id: project_id + ) + end + + private + + def set_object_storage_key + return unless package_name && project_id + + self.object_storage_key = Gitlab::HashedPath.new( + 'packages', 'metadata_caches', 'npm', OpenSSL::Digest::SHA256.hexdigest(package_name), + root_hash: project_id + ).to_s + end + end + end +end diff --git a/app/models/packages/npm/metadatum.rb b/app/models/packages/npm/metadatum.rb index 7388c4bdbd2..ccbf056ec7b 100644 --- a/app/models/packages/npm/metadatum.rb +++ b/app/models/packages/npm/metadatum.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class Packages::Npm::Metadatum < ApplicationRecord + MAX_PACKAGE_JSON_SIZE = 20_000 + MIN_PACKAGE_JSON_FIELD_SIZE_FOR_ERROR_TRACKING = 5_000 + NUM_FIELDS_FOR_ERROR_TRACKING = 5 + belongs_to :package, -> { where(package_type: :npm) }, inverse_of: :npm_metadatum validates :package, presence: true @@ -9,6 +13,8 @@ class Packages::Npm::Metadatum < ApplicationRecord validate :ensure_npm_package_type validate :ensure_package_json_size + scope :package_id_in, ->(package_ids) { where(package_id: package_ids) } + private def ensure_npm_package_type @@ -18,7 +24,7 @@ class Packages::Npm::Metadatum < ApplicationRecord end def ensure_package_json_size - return if package_json.to_s.size < 20000 + return if package_json.to_s.size < MAX_PACKAGE_JSON_SIZE errors.add(:package_json, _('structure is too large')) end diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 970538b45e7..c58ad92d7a6 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -31,6 +31,8 @@ class Packages::Package < ApplicationRecord belongs_to :project belongs_to :creator, class_name: 'User' + after_create_commit :publish_creation_event, if: :generic? + # package_files must be destroyed by ruby code in order to properly remove carrierwave uploads and update project statistics has_many :package_files, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # TODO: put the installable default scope on the :package_files association once the dependent: :destroy is removed @@ -70,9 +72,8 @@ class Packages::Package < ApplicationRecord scope: %i[project_id version package_type], conditions: -> { not_pending_destruction } }, - unless: -> { pending_destruction? || conan? || debian_package? } + unless: -> { pending_destruction? || conan? } - validate :unique_debian_package_name, if: :debian_package? validate :valid_conan_package_recipe, if: :conan? validate :valid_composer_global_name, if: :composer? validate :npm_package_already_taken, if: :npm? @@ -84,7 +85,7 @@ class Packages::Package < ApplicationRecord validates :name, format: { with: Gitlab::Regex.nuget_package_name_regex }, if: :nuget? validates :name, format: { with: Gitlab::Regex.terraform_module_package_name_regex }, if: :terraform_module? validates :name, format: { with: Gitlab::Regex.debian_package_name_regex }, if: :debian_package? - validates :name, inclusion: { in: %w[incoming] }, if: :debian_incoming? + validates :name, inclusion: { in: [Packages::Debian::INCOMING_PACKAGE_NAME] }, if: :debian_incoming? validates :version, format: { with: Gitlab::Regex.nuget_version_regex }, if: :nuget? validates :version, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan? validates :version, format: { with: Gitlab::Regex.maven_version_regex }, if: -> { version? && maven? } @@ -155,6 +156,7 @@ class Packages::Package < ApplicationRecord scope :preload_npm_metadatum, -> { preload(:npm_metadatum) } scope :preload_nuget_metadatum, -> { preload(:nuget_metadatum) } scope :preload_pypi_metadatum, -> { preload(:pypi_metadatum) } + scope :preload_conan_metadatum, -> { preload(:conan_metadatum) } scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) } @@ -179,6 +181,7 @@ class Packages::Package < ApplicationRecord scope :order_project_name, -> { joins(:project).reorder('projects.name ASC') } scope :order_project_name_desc, -> { joins(:project).reorder('projects.name DESC') } scope :order_by_package_file, -> { joins(:package_files).order('packages_package_files.created_at ASC') } + scope :with_npm_scope, ->(scope) { npm.where("name ILIKE :package_name", package_name: "@#{sanitize_sql_like(scope)}/%") } scope :order_project_path, -> do keyset_order = keyset_pagination_order(join_class: Project, column_name: :path, direction: :asc) @@ -222,6 +225,12 @@ class Packages::Package < ApplicationRecord find_by!(name: name, version: version) end + def self.existing_debian_packages_with(name:, version:) + debian.with_name(name) + .with_version(version) + .not_pending_destruction + end + def self.pluck_names pluck(:name) end @@ -288,9 +297,14 @@ class Packages::Package < ApplicationRecord end # Technical debt: to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/281937 + # TODO: rename the method https://gitlab.com/gitlab-org/gitlab/-/issues/410352 def original_build_info strong_memoize(:original_build_info) do - build_infos.first + if Feature.enabled?(:packages_display_last_pipeline, project) + build_infos.last + else + build_infos.first + end end end @@ -353,6 +367,18 @@ class Packages::Package < ApplicationRecord end end + def publish_creation_event + ::Gitlab::EventStore.publish( + ::Packages::PackageCreatedEvent.new(data: { + project_id: project_id, + id: id, + name: name, + version: version, + package_type: package_type + }) + ) + end + private def composer_tag_version? @@ -404,19 +430,6 @@ class Packages::Package < ApplicationRecord project.root_namespace.path == ::Packages::Npm.scope_of(name) end - def unique_debian_package_name - return unless debian_publication&.distribution - - package_exists = debian_publication.distribution.packages - .with_name(name) - .with_version(version) - .not_pending_destruction - .id_not_in(id) - .exists? - - errors.add(:base, _('Debian package already exists in Distribution')) if package_exists - end - def forbidden_debian_changes return unless persisted? @@ -426,3 +439,5 @@ class Packages::Package < ApplicationRecord end end end + +Packages::Package.prepend_mod diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index e1486c11298..c164d150bce 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -85,6 +85,13 @@ class Packages::PackageFile < ApplicationRecord .where(packages_debian_file_metadata: { architecture: architecture_name }) end + scope :with_debian_unknown_since, ->(updated_before) do + file_metadata = Packages::Debian::FileMetadatum.with_file_type(:unknown) + .updated_before(updated_before) + .where('packages_package_files.id = packages_debian_file_metadata.package_file_id') + where('EXISTS (?)', file_metadata.select(1)) + end + scope :with_conan_package_reference, ->(conan_package_reference) do joins(:conan_file_metadatum) .where(packages_conan_file_metadata: { conan_package_reference: conan_package_reference }) diff --git a/app/models/packages/rpm/repository_file.rb b/app/models/packages/rpm/repository_file.rb index 614ec9b3e56..bbd435691d2 100644 --- a/app/models/packages/rpm/repository_file.rb +++ b/app/models/packages/rpm/repository_file.rb @@ -13,7 +13,7 @@ module Packages enum status: { default: 0, pending_destruction: 1, processing: 2, error: 3 } - belongs_to :project, inverse_of: :repository_files + belongs_to :project, inverse_of: :rpm_repository_files validates :project, presence: true validates :file, presence: true diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index a1ba48f3ab0..864ea04c019 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -49,19 +49,32 @@ module Pages if project.pages_namespace_url == project.pages_url '/' else - project.full_path.delete_prefix(trim_prefix) + '/' + "#{project.full_path.delete_prefix(trim_prefix)}/" end end strong_memoize_attr :prefix + def unique_host + return unless project.project_setting.pages_unique_domain_enabled? + + project.pages_unique_host + end + strong_memoize_attr :unique_host + + def root_directory + return unless deployment + + deployment.root_directory + end + strong_memoize_attr :root_directory + private attr_reader :project, :trim_prefix, :domain def deployment - strong_memoize(:deployment) do - project.pages_metadatum.pages_deployment - end + project.pages_metadatum.pages_deployment end + strong_memoize_attr :deployment end end diff --git a/app/models/pages_deployment.rb b/app/models/pages_deployment.rb index da6ef035c54..fa29cbf8352 100644 --- a/app/models/pages_deployment.rb +++ b/app/models/pages_deployment.rb @@ -4,6 +4,7 @@ class PagesDeployment < ApplicationRecord include EachBatch include FileStoreMounter + include Gitlab::Utils::StrongMemoize MIGRATED_FILE_NAME = "_migrated.zip" @@ -28,15 +29,29 @@ class PagesDeployment < ApplicationRecord mount_file_store_uploader ::Pages::DeploymentUploader + skip_callback :save, :after, :store_file!, if: :store_after_commit? + after_commit :store_file_after_commit!, on: [:create, :update], if: :store_after_commit? + def migrated? file.filename == MIGRATED_FILE_NAME end + def store_after_commit? + Feature.enabled?(:pages_deploy_upload_file_outside_transaction, project) + end + strong_memoize_attr :store_after_commit? + private def set_size self.size = file.size end + + def store_file_after_commit! + return unless previous_changes.key?(:file) + + store_file_now! + end end PagesDeployment.prepend_mod diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb index 909658214fd..10ac10295fc 100644 --- a/app/models/pages_domain.rb +++ b/app/models/pages_domain.rb @@ -15,7 +15,6 @@ class PagesDomain < ApplicationRecord belongs_to :project has_many :acme_orders, class_name: "PagesDomainAcmeOrder" - has_many :serverless_domain_clusters, class_name: 'Serverless::DomainCluster', inverse_of: :pages_domain after_initialize :set_verification_code before_validation :clear_auto_ssl_failure, unless: :auto_ssl_enabled @@ -173,6 +172,10 @@ class PagesDomain < ApplicationRecord "#{VERIFICATION_KEY}=#{verification_code}" end + def verification_record + "#{verification_domain} TXT #{keyed_verification_code}" + end + def certificate=(certificate) super(certificate) @@ -209,20 +212,6 @@ class PagesDomain < ApplicationRecord self.certificate_source = 'gitlab_provided' if attribute_changed?(:key) end - def pages_virtual_domain - return unless pages_deployed? - - cache = if Feature.enabled?(:cache_pages_domain_api, project.root_namespace) - ::Gitlab::Pages::CacheControl.for_domain(id) - end - - Pages::VirtualDomain.new( - projects: [project], - domain: self, - cache: cache - ) - end - def clear_auto_ssl_failure self.auto_ssl_failed = false end @@ -237,14 +226,14 @@ class PagesDomain < ApplicationRecord end end - private - def pages_deployed? return false unless project project.pages_metadatum&.deployed? end + private + def set_verification_code return if self.verification_code.present? diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb index f99c4c6c39d..75afff6a2fa 100644 --- a/app/models/personal_access_token.rb +++ b/app/models/personal_access_token.rb @@ -9,10 +9,13 @@ class PersonalAccessToken < ApplicationRecord include Gitlab::SQL::Pattern extend ::Gitlab::Utils::Override - add_authentication_token_field :token, digest: true + add_authentication_token_field :token, + digest: true, + format_with_prefix: :prefix_from_application_current_settings # PATs are 20 characters + optional configurable settings prefix (0..20) TOKEN_LENGTH_RANGE = (20..40).freeze + MAX_PERSONAL_ACCESS_TOKEN_LIFETIME_IN_DAYS = 365 serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize @@ -39,13 +42,14 @@ class PersonalAccessToken < ApplicationRecord scope :for_users, -> (users) { where(user: users) } scope :preload_users, -> { preload(:user) } scope :order_expires_at_asc_id_desc, -> { reorder(expires_at: :asc, id: :desc) } - scope :project_access_token, -> { includes(:user).where(user: { user_type: :project_bot }) } - scope :owner_is_human, -> { includes(:user).where(user: { user_type: :human }) } + scope :project_access_token, -> { includes(:user).references(:user).merge(User.project_bot) } + scope :owner_is_human, -> { includes(:user).references(:user).merge(User.human) } scope :last_used_before, -> (date) { where("last_used_at <= ?", date) } scope :last_used_after, -> (date) { where("last_used_at >= ?", date) } validates :scopes, presence: true validate :validate_scopes + validate :expires_at_before_instance_max_expiry_date, on: :create def revoke! update!(revoked: true) @@ -55,6 +59,19 @@ class PersonalAccessToken < ApplicationRecord !revoked? && !expired? end + # fall back to default value until background migration has updated all + # existing PATs and we can add a validation + # https://gitlab.com/gitlab-org/gitlab/-/issues/369123 + def expires_at=(value) + datetime = if Feature.enabled?(:default_pat_expiration) + value.presence || MAX_PERSONAL_ACCESS_TOKEN_LIFETIME_IN_DAYS.days.from_now + else + value + end + + super(datetime) + end + override :simple_sorts def self.simple_sorts super.merge( @@ -72,11 +89,6 @@ class PersonalAccessToken < ApplicationRecord fuzzy_search(query, [:name]) end - override :format_token - def format_token(token) - "#{self.class.token_prefix}#{token}" - end - def project_access_token? user&.project_bot? end @@ -107,6 +119,19 @@ class PersonalAccessToken < ApplicationRecord def add_admin_mode_scope self.scopes += [Gitlab::Auth::ADMIN_MODE_SCOPE.to_s] end + + def prefix_from_application_current_settings + self.class.token_prefix + end + + def expires_at_before_instance_max_expiry_date + return unless Feature.enabled?(:default_pat_expiration) + return unless expires_at + + if expires_at > MAX_PERSONAL_ACCESS_TOKEN_LIFETIME_IN_DAYS.days.from_now + errors.add(:expires_at, _('must expire in 365 days')) + end + end end PersonalAccessToken.prepend_mod_with('PersonalAccessToken') diff --git a/app/models/preloaders/commit_status_preloader.rb b/app/models/preloaders/commit_status_preloader.rb index 535dd24ba6b..79c2549e371 100644 --- a/app/models/preloaders/commit_status_preloader.rb +++ b/app/models/preloaders/commit_status_preloader.rb @@ -9,10 +9,11 @@ module Preloaders end def execute(relations) - preloader = ActiveRecord::Associations::Preloader.new - CLASSES.each do |klass| - preloader.preload(objects(klass), associations(klass, relations)) + ActiveRecord::Associations::Preloader.new( + records: objects(klass), + associations: associations(klass, relations) + ).call end end diff --git a/app/models/preloaders/labels_preloader.rb b/app/models/preloaders/labels_preloader.rb index b6e73c1cd02..7ee0ec0ca43 100644 --- a/app/models/preloaders/labels_preloader.rb +++ b/app/models/preloaders/labels_preloader.rb @@ -19,17 +19,32 @@ module Preloaders end def preload_all - preloader = ActiveRecord::Associations::Preloader.new + ActiveRecord::Associations::Preloader.new( + records: project_labels, + associations: { project: [:project_feature, namespace: :route] } + ).call - preloader.preload(labels, parent_container: :route) - preloader.preload(labels.select { |l| l.is_a? ProjectLabel }, { project: [:project_feature, namespace: :route] }) - preloader.preload(labels.select { |l| l.is_a? GroupLabel }, { group: :route }) + ActiveRecord::Associations::Preloader.new( + records: group_labels, + associations: { group: :route } + ).call + Preloaders::UserMaxAccessLevelInProjectsPreloader.new(project_labels.map(&:project), user).execute labels.each do |label| label.lazy_subscription(user) label.lazy_subscription(user, project) if project.present? end end + + private + + def group_labels + @group_labels ||= labels.select { |l| l.is_a? GroupLabel } + end + + def project_labels + @project_labels ||= labels.select { |l| l.is_a? ProjectLabel } + end end end diff --git a/app/models/preloaders/project_policy_preloader.rb b/app/models/preloaders/project_policy_preloader.rb index fe9db3464c7..e16eabf40a1 100644 --- a/app/models/preloaders/project_policy_preloader.rb +++ b/app/models/preloaders/project_policy_preloader.rb @@ -10,7 +10,10 @@ module Preloaders def execute return if projects.is_a?(ActiveRecord::NullRelation) - ActiveRecord::Associations::Preloader.new.preload(projects, { group: :route, namespace: :owner }) + ActiveRecord::Associations::Preloader.new( + records: projects, + associations: { group: :route, namespace: :owner } + ).call ::Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects, current_user).execute end diff --git a/app/models/preloaders/project_root_ancestor_preloader.rb b/app/models/preloaders/project_root_ancestor_preloader.rb index 6192f79ce2c..ccb9d2eab98 100644 --- a/app/models/preloaders/project_root_ancestor_preloader.rb +++ b/app/models/preloaders/project_root_ancestor_preloader.rb @@ -19,7 +19,7 @@ module Preloaders root_ancestors_by_id = root_query.group_by(&:source_id) - ActiveRecord::Associations::Preloader.new.preload(@projects, :namespace) + ActiveRecord::Associations::Preloader.new(records: @projects, associations: :namespace).call @projects.each do |project| root_ancestor = root_ancestors_by_id[project.id]&.first project.namespace.root_ancestor = root_ancestor if root_ancestor.present? diff --git a/app/models/preloaders/runner_manager_policy_preloader.rb b/app/models/preloaders/runner_manager_policy_preloader.rb new file mode 100644 index 00000000000..788a3d25a87 --- /dev/null +++ b/app/models/preloaders/runner_manager_policy_preloader.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Preloaders + class RunnerManagerPolicyPreloader + def initialize(runner_managers, current_user) + @runner_managers = runner_managers + @current_user = current_user + end + + def execute + return if runner_managers.is_a?(ActiveRecord::NullRelation) + + ActiveRecord::Associations::Preloader.new( + records: runner_managers, + associations: [:runner] + ).call + end + + private + + attr_reader :runner_managers, :current_user + end +end diff --git a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb index 0c747ad9c84..16d46facb96 100644 --- a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb +++ b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb @@ -46,14 +46,10 @@ module Preloaders end def all_memberships - if Feature.enabled?(:include_memberships_from_group_shares_in_preloader) - [ - direct_memberships.select(*GroupMember.cached_column_list), - memberships_from_group_shares - ] - else - [direct_memberships] - end + [ + direct_memberships.select(*GroupMember.cached_column_list), + memberships_from_group_shares + ] end def direct_memberships diff --git a/app/models/preloaders/users_max_access_level_by_project_preloader.rb b/app/models/preloaders/users_max_access_level_by_project_preloader.rb new file mode 100644 index 00000000000..37842665e7d --- /dev/null +++ b/app/models/preloaders/users_max_access_level_by_project_preloader.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Preloaders + # This class preloads the max access level (role) for the users within the given projects and + # stores the values in requests store via the ProjectTeam class. + class UsersMaxAccessLevelByProjectPreloader + include Gitlab::Utils::StrongMemoize + + def initialize(project_users:) + @project_users = project_users.transform_values { |users| Array.wrap(users) } + end + + def execute + return unless @project_users.present? + + all_users = @project_users.values.flatten.uniq + preload_users_namespace_bans(all_users) + + @project_users.each do |project, users| + users.each do |user| + access_level = access_levels.fetch([project.id, user.id], Gitlab::Access::NO_ACCESS) + project.team.write_member_access_for_user_id(user.id, access_level) + end + end + end + + private + + def access_levels + query = ProjectAuthorization.none + + @project_users.each do |project, users| + query = query.or( + ProjectAuthorization + .where(project_id: project.id, user_id: users.map(&:id)) + ) + end + + query + .group(:project_id, :user_id) + .maximum(:access_level) + end + strong_memoize_attr :access_levels + + def preload_users_namespace_bans(_users) + # overridden in EE + end + end +end + +Preloaders::UsersMaxAccessLevelByProjectPreloader.prepend_mod diff --git a/app/models/preloaders/users_max_access_level_in_projects_preloader.rb b/app/models/preloaders/users_max_access_level_in_projects_preloader.rb deleted file mode 100644 index f32184f168d..00000000000 --- a/app/models/preloaders/users_max_access_level_in_projects_preloader.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Preloaders - # This class preloads the max access level (role) for the users within the given projects and - # stores the values in requests store via the ProjectTeam class. - class UsersMaxAccessLevelInProjectsPreloader - def initialize(projects:, users:) - @projects = projects - @users = users - end - - def execute - return unless @projects.present? && @users.present? - - preload_users_namespace_bans(@users) - - access_levels.each do |(project_id, user_id), access_level| - project = projects_by_id[project_id] - - project.team.write_member_access_for_user_id(user_id, access_level) - end - end - - private - - def access_levels - ProjectAuthorization - .where(project_id: project_ids, user_id: user_ids) - .group(:project_id, :user_id) - .maximum(:access_level) - end - - # Use reselect to override the existing select to prevent - # the error `subquery has too many columns` - # NotificationsController passes in an Array so we need to check the type - def project_ids - @projects.is_a?(ActiveRecord::Relation) ? @projects.reselect(:id) : @projects - end - - def user_ids - @users.is_a?(ActiveRecord::Relation) ? @users.reselect(:id) : @users - end - - def projects_by_id - @projects_by_id ||= @projects.index_by(&:id) - end - - def preload_users_namespace_bans(_users) - # overridden in EE - end - end -end - -Preloaders::UsersMaxAccessLevelInProjectsPreloader.prepend_mod diff --git a/app/models/project.rb b/app/models/project.rb index 43ec26be786..224193fba08 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -19,6 +19,7 @@ class Project < ApplicationRecord include Presentable include HasRepository include HasWiki + include WebHooks::HasWebHooks include CanMoveRepositoryStorage include Routable include GroupDescendant @@ -41,7 +42,7 @@ class Project < ApplicationRecord include BlocksUnsafeSerialization include Subquery include IssueParent - include WebHooks::HasWebHooks + include UpdatedAtFilterable extend Gitlab::Cache::RequestCache extend Gitlab::Utils::Override @@ -89,6 +90,14 @@ class Project < ApplicationRecord DEFAULT_SQUASH_COMMIT_TEMPLATE = '%{title}' + PROJECT_FEATURES_DEFAULTS = { + issues: gitlab_config_features.issues, + merge_requests: gitlab_config_features.merge_requests, + builds: gitlab_config_features.builds, + wiki: gitlab_config_features.wiki, + snippets: gitlab_config_features.snippets + }.freeze + cache_markdown_field :description, pipeline: :description attribute :packages_enabled, default: true @@ -101,18 +110,14 @@ class Project < ApplicationRecord attribute :autoclose_referenced_issues, default: true attribute :ci_config_path, default: -> { Gitlab::CurrentSettings.default_ci_config_path } - default_value_for :issues_enabled, gitlab_config_features.issues - default_value_for :merge_requests_enabled, gitlab_config_features.merge_requests - default_value_for :builds_enabled, gitlab_config_features.builds - default_value_for :wiki_enabled, gitlab_config_features.wiki - default_value_for :snippets_enabled, gitlab_config_features.snippets - add_authentication_token_field :runners_token, encrypted: -> { Feature.enabled?(:projects_tokens_optional_encryption) ? :optional : :required }, - prefix: RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + format_with_prefix: :runners_token_prefix, + require_prefix_for_validation: true # Storage specific hooks after_initialize :use_hashed_storage + after_initialize :set_project_feature_defaults, if: :new_record? before_validation :mark_remote_mirrors_for_removal, if: -> { RemoteMirror.table_exists? } before_validation :ensure_project_namespace_in_sync @@ -128,7 +133,6 @@ class Project < ApplicationRecord after_create -> { create_or_load_association(:pages_metadatum) } after_create :set_timestamps_for_create after_create :check_repository_absence! - after_update :update_forks_visibility_level before_destroy :remove_private_deploy_keys after_destroy :remove_exports after_save :update_project_statistics, if: :saved_change_to_namespace_id? @@ -165,6 +169,7 @@ class Project < ApplicationRecord alias_method :parent, :namespace alias_attribute :parent_id, :namespace_id + has_one :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :project has_one :last_event, -> { order 'events.created_at DESC' }, class_name: 'Event' has_many :boards @@ -183,11 +188,13 @@ class Project < ApplicationRecord has_one :confluence_integration, class_name: 'Integrations::Confluence' has_one :custom_issue_tracker_integration, class_name: 'Integrations::CustomIssueTracker' has_one :datadog_integration, class_name: 'Integrations::Datadog' + has_one :container_registry_data_repair_detail, class_name: 'ContainerRegistry::DataRepairDetail' has_one :discord_integration, class_name: 'Integrations::Discord' has_one :drone_ci_integration, class_name: 'Integrations::DroneCi' has_one :emails_on_push_integration, class_name: 'Integrations::EmailsOnPush' has_one :ewm_integration, class_name: 'Integrations::Ewm' has_one :external_wiki_integration, class_name: 'Integrations::ExternalWiki' + has_one :google_play_integration, class_name: 'Integrations::GooglePlay' has_one :hangouts_chat_integration, class_name: 'Integrations::HangoutsChat' has_one :harbor_integration, class_name: 'Integrations::Harbor' has_one :irker_integration, class_name: 'Integrations::Irker' @@ -208,6 +215,7 @@ class Project < ApplicationRecord has_one :shimo_integration, class_name: 'Integrations::Shimo' has_one :slack_integration, class_name: 'Integrations::Slack' has_one :slack_slash_commands_integration, class_name: 'Integrations::SlackSlashCommands' + has_one :squash_tm_integration, class_name: 'Integrations::SquashTm' has_one :teamcity_integration, class_name: 'Integrations::Teamcity' has_one :unify_circuit_integration, class_name: 'Integrations::UnifyCircuit' has_one :webex_teams_integration, class_name: 'Integrations::WebexTeams' @@ -215,6 +223,7 @@ class Project < ApplicationRecord has_one :zentao_integration, class_name: 'Integrations::Zentao' has_one :wiki_repository, class_name: 'Projects::WikiRepository', inverse_of: :project + has_one :design_management_repository, class_name: 'DesignManagement::Repository', inverse_of: :project has_one :root_of_fork_network, foreign_key: 'root_project_id', inverse_of: :root_project, @@ -238,14 +247,24 @@ class Project < ApplicationRecord has_many :fork_network_projects, through: :fork_network, source: :projects # Packages - has_many :packages, class_name: 'Packages::Package' - has_many :package_files, through: :packages, class_name: 'Packages::PackageFile' + has_many :packages, + class_name: 'Packages::Package' + has_many :package_files, + through: :packages, class_name: 'Packages::PackageFile' # repository_files must be destroyed by ruby code in order to properly remove carrierwave uploads - has_many :repository_files, inverse_of: :project, class_name: 'Packages::Rpm::RepositoryFile', - dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :rpm_repository_files, + inverse_of: :project, + class_name: 'Packages::Rpm::RepositoryFile', + dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads - has_many :debian_distributions, class_name: 'Packages::Debian::ProjectDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_one :packages_cleanup_policy, class_name: 'Packages::Cleanup::Policy', inverse_of: :project + has_many :debian_distributions, + class_name: 'Packages::Debian::ProjectDistribution', + dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :npm_metadata_caches, + class_name: 'Packages::Npm::MetadataCache' + has_one :packages_cleanup_policy, + class_name: 'Packages::Cleanup::Policy', + inverse_of: :project has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -259,12 +278,15 @@ class Project < ApplicationRecord has_one :project_setting, inverse_of: :project, autosave: true has_one :alerting_setting, inverse_of: :project, class_name: 'Alerting::ProjectAlertingSetting' has_one :service_desk_setting, class_name: 'ServiceDeskSetting' + has_one :service_desk_custom_email_verification, class_name: 'ServiceDesk::CustomEmailVerification' + has_one :service_desk_custom_email_credential, class_name: 'ServiceDesk::CustomEmailCredential' # Merge requests for target project should be removed with it has_many :merge_requests, foreign_key: 'target_project_id', inverse_of: :target_project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :merge_request_metrics, foreign_key: 'target_project', class_name: 'MergeRequest::Metrics', inverse_of: :target_project has_many :source_of_merge_requests, foreign_key: 'source_project_id', class_name: 'MergeRequest' has_many :issues, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :work_items # the issues relation will handle any destroys has_many :incident_management_issuable_escalation_statuses, through: :issues, inverse_of: :project, class_name: 'IncidentManagement::IssuableEscalationStatus' has_many :incident_management_timeline_event_tags, inverse_of: :project, class_name: 'IncidentManagement::TimelineEventTag' has_many :labels, class_name: 'ProjectLabel', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -342,6 +364,7 @@ class Project < ApplicationRecord has_many :kubernetes_namespaces, class_name: 'Clusters::KubernetesNamespace' has_many :management_clusters, class_name: 'Clusters::Cluster', foreign_key: :management_project_id, inverse_of: :management_project has_many :cluster_agents, class_name: 'Clusters::Agent' + has_many :ci_access_project_authorizations, class_name: 'Clusters::Agents::Authorizations::CiAccess::ProjectAuthorization' has_many :prometheus_metrics has_many :prometheus_alerts, inverse_of: :project @@ -371,7 +394,6 @@ class Project < ApplicationRecord inverse_of: :project has_many :stages, class_name: 'Ci::Stage', inverse_of: :project has_many :ci_refs, class_name: 'Ci::Ref', inverse_of: :project - has_many :pipeline_metadata, class_name: 'Ci::PipelineMetadata', inverse_of: :project has_many :pending_builds, class_name: 'Ci::PendingBuild' has_many :builds, class_name: 'Ci::Build', inverse_of: :project @@ -476,6 +498,7 @@ class Project < ApplicationRecord to: :project_setting, allow_nil: true delegate :show_diff_preview_in_email, :show_diff_preview_in_email=, :show_diff_preview_in_email?, + :runner_registration_enabled, :runner_registration_enabled?, :runner_registration_enabled=, to: :project_setting delegate :squash_always?, :squash_never?, :squash_enabled_by_default?, :squash_readonly?, to: :project_setting @@ -484,7 +507,7 @@ class Project < ApplicationRecord delegate :previous_default_branch, :previous_default_branch=, to: :project_setting delegate :name, to: :owner, allow_nil: true, prefix: true delegate :members, to: :team, prefix: true - delegate :add_member, :add_members, to: :team + delegate :add_member, :add_members, :member?, to: :team delegate :add_guest, :add_reporter, :add_developer, :add_maintainer, :add_owner, :add_role, to: :team delegate :group_runners_enabled, :group_runners_enabled=, to: :ci_cd_settings, allow_nil: true delegate :root_ancestor, to: :namespace, allow_nil: true @@ -496,7 +519,6 @@ class Project < ApplicationRecord delegate :job_token_scope_enabled, :job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci_outbound, allow_nil: true delegate :inbound_job_token_scope_enabled, :inbound_job_token_scope_enabled=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :keep_latest_artifact, :keep_latest_artifact=, to: :ci_cd_settings, allow_nil: true - delegate :opt_in_jwt, :opt_in_jwt=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :allow_fork_pipelines_to_run_in_parent_project, :allow_fork_pipelines_to_run_in_parent_project=, to: :ci_cd_settings, prefix: :ci, allow_nil: true delegate :restrict_user_defined_variables, :restrict_user_defined_variables=, to: :ci_cd_settings, allow_nil: true delegate :separated_caches, :separated_caches=, to: :ci_cd_settings, prefix: :ci, allow_nil: true @@ -719,6 +741,11 @@ class Project < ApplicationRecord topic ? with_topic(topic) : none end + scope :pending_data_repair_analysis, -> do + left_outer_joins(:container_registry_data_repair_detail) + .where(container_registry_data_repair_details: { project_id: nil }) + end + enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } chronic_duration_attr :build_timeout_human_readable, :build_timeout, @@ -874,7 +901,7 @@ class Project < ApplicationRecord def reference_pattern %r{ (?<!#{Gitlab::PathRegex::PATH_START_CHAR}) - ((?<namespace>#{Gitlab::PathRegex::FULL_NAMESPACE_FORMAT_REGEX})\/)? + ((?<namespace>#{Gitlab::PathRegex::FULL_NAMESPACE_FORMAT_REGEX})/)? (?<project>#{Gitlab::PathRegex::PROJECT_PATH_FORMAT_REGEX}) }xo end @@ -950,27 +977,44 @@ class Project < ApplicationRecord .where(pending_delete: false) .where(archived: false) end + + def project_features_defaults + PROJECT_FEATURES_DEFAULTS + end + + def by_pages_enabled_unique_domain(domain) + without_deleted + .joins(:project_setting) + .find_by(project_setting: { + pages_unique_domain_enabled: true, + pages_unique_domain: domain + }) + end end def initialize(attributes = nil) - # We can't use default_value_for because the database has a default - # value of 0 for visibility_level. If someone attempts to create a - # private project, default_value_for will assume that the - # visibility_level hasn't changed and will use the application - # setting default, which could be internal or public. For projects - # inside a private group, those levels are invalid. - # - # To fix the problem, we assign the actual default in the application if - # no explicit visibility has been initialized. + # We assign the actual snippet default if no explicit visibility has been initialized. attributes ||= {} unless visibility_attribute_present?(attributes) attributes[:visibility_level] = Gitlab::CurrentSettings.default_project_visibility end + @init_attributes = attributes + super end + # Remove along with ProjectFeaturesCompatibility module + def set_project_feature_defaults + self.class.project_features_defaults.each do |attr, value| + # If the deprecated _enabled or the accepted _access_level attribute is specified, we don't need to set the default + next unless @init_attributes[:"#{attr}_enabled"].nil? && @init_attributes[:"#{attr}_access_level"].nil? + + public_send("#{attr}_enabled=", value) # rubocop:disable GitlabSecurity/PublicSend + end + end + def parent_loaded? association(:namespace).loaded? end @@ -1077,8 +1121,10 @@ class Project < ApplicationRecord end def preload_protected_branches - preloader = ActiveRecord::Associations::Preloader.new - preloader.preload(self, protected_branches: [:push_access_levels, :merge_access_levels]) + ActiveRecord::Associations::Preloader.new( + records: [self], + associations: { protected_branches: [:push_access_levels, :merge_access_levels] } + ).call end # returns all ancestor-groups upto but excluding the given namespace @@ -1089,11 +1135,7 @@ class Project < ApplicationRecord end def ancestors(hierarchy_order: nil) - if Feature.enabled?(:linear_project_ancestors, self) - group&.self_and_ancestors(hierarchy_order: hierarchy_order) || Group.none - else - ancestors_upto(hierarchy_order: hierarchy_order) - end + group&.self_and_ancestors(hierarchy_order: hierarchy_order) || Group.none end def ancestors_upto_ids(...) @@ -1140,10 +1182,6 @@ class Project < ApplicationRecord auto_devops_config[:scope] != :project && !auto_devops_config[:status] end - def has_packages?(package_type) - packages.where(package_type: package_type).exists? - end - def packages_cleanup_policy super || build_packages_cleanup_policy end @@ -1154,10 +1192,6 @@ class Project < ApplicationRecord { scope: :project, status: auto_devops&.enabled || Feature.enabled?(:force_autodevops_on_by_default, self) } end - def unlink_forks_upon_visibility_decrease_enabled? - Feature.enabled?(:unlink_fork_network_upon_visibility_decrease, self) - end - # LFS and hashed repository storage are required for using Design Management. def design_management_enabled? lfs_enabled? && hashed_storage?(:repository) @@ -1171,21 +1205,16 @@ class Project < ApplicationRecord @repository ||= Gitlab::GlRepository::PROJECT.repository_for(self) end + def design_management_repository + super || create_design_management_repository + end + def design_repository strong_memoize(:design_repository) do Gitlab::GlRepository::DESIGN.repository_for(self) end end - # Because we use default_value_for we need to be sure - # packages_enabled= method does exist even if we rollback migration. - # Otherwise many tests from spec/migrations will fail. - def packages_enabled=(value) - if has_attribute?(:packages_enabled) - write_attribute(:packages_enabled, value) - end - end - def cleanup @repository = nil end @@ -1239,12 +1268,16 @@ class Project < ApplicationRecord latest_successful_build_for_ref(job_name, ref) || raise(ActiveRecord::RecordNotFound, "Couldn't find job #{job_name}") end - def latest_pipeline(ref = default_branch, sha = nil) + def latest_pipelines(ref = default_branch, sha = nil) ref = ref.presence || default_branch sha ||= commit(ref)&.sha - return unless sha + return ci_pipelines.none unless sha + + ci_pipelines.newest_first(ref: ref, sha: sha) + end - ci_pipelines.newest_first(ref: ref, sha: sha).take + def latest_pipeline(ref = default_branch, sha = nil) + latest_pipelines(ref, sha).take end def merge_base_commit(first_commit_id, second_commit_id) @@ -1272,6 +1305,18 @@ class Project < ApplicationRecord import_state&.human_status_name || 'none' end + def beautified_import_status_name + if import_finished? + return 'completed' unless import_checksums.present? + + fetched = import_checksums['fetched'] + imported = import_checksums['imported'] + fetched.keys.any? { |key| fetched[key] != imported[key] } ? 'partially completed' : 'completed' + else + import_status + end + end + def add_import_job job_id = if forked? @@ -1314,6 +1359,11 @@ class Project < ApplicationRecord super(value&.delete("\0")) end + # Used by Import/Export to export commit notes + def commit_notes + notes.where(noteable_type: "Commit") + end + def import_url=(value) if Gitlab::UrlSanitizer.valid?(value) import_url = Gitlab::UrlSanitizer.new(value) @@ -1355,7 +1405,7 @@ class Project < ApplicationRecord end def import? - external_import? || forked? || gitlab_project_import? || jira_import? || bare_repository_import? || gitlab_project_migration? + external_import? || forked? || gitlab_project_import? || jira_import? || gitlab_project_migration? end def external_import? @@ -1366,10 +1416,6 @@ class Project < ApplicationRecord Gitlab::UrlSanitizer.new(import_url).masked_url end - def bare_repository_import? - import_type == 'bare_repository' - end - def jira_import? import_type == 'jira' && latest_jira_import.present? end @@ -1545,7 +1591,7 @@ class Project < ApplicationRecord end def new_issuable_address(author, address_type) - return unless Gitlab::IncomingEmail.supports_issue_creation? && author + return unless Gitlab::Email::IncomingEmail.supports_issue_creation? && author # check since this can come from a request parameter return unless %w(issue merge_request).include?(address_type) @@ -1556,7 +1602,7 @@ class Project < ApplicationRecord # example: incoming+h5bp-html5-boilerplate-8-1234567890abcdef123456789-issue@localhost.com # example: incoming+h5bp-html5-boilerplate-8-1234567890abcdef123456789-merge-request@localhost.com - Gitlab::IncomingEmail.reply_address("#{full_path_slug}-#{project_id}-#{author.incoming_email_token}-#{suffix}") + Gitlab::Email::IncomingEmail.reply_address("#{full_path_slug}-#{project_id}-#{author.incoming_email_token}-#{suffix}") end def build_commit_note(commit) @@ -1590,7 +1636,7 @@ class Project < ApplicationRecord end def external_issue_reference_pattern - external_issue_tracker.class.reference_pattern(only_long: issues_enabled?) + external_issue_tracker.reference_pattern(only_long: issues_enabled?) end def default_issues_tracker? @@ -1630,9 +1676,7 @@ class Project < ApplicationRecord end def disabled_integrations - disabled_integrations = [] - disabled_integrations << 'apple_app_store' unless Feature.enabled?(:apple_app_store_integration, self) - disabled_integrations + %w[shimo zentao] end def find_or_initialize_integration(name) @@ -1835,11 +1879,11 @@ class Project < ApplicationRecord repository.update!(shard_name: repository_storage, disk_path: disk_path) end - def create_repository(force: false) + def create_repository(force: false, default_branch: nil) # Forked import is handled asynchronously return if forked? && !force - repository.create_repository + repository.create_repository(default_branch) repository.after_create true @@ -1935,19 +1979,6 @@ class Project < ApplicationRecord create_repository(force: true) unless repository_exists? end - # update visibility_level of forks - def update_forks_visibility_level - return if unlink_forks_upon_visibility_decrease_enabled? - return unless visibility_level < visibility_level_before_last_save - - forks.each do |forked_project| - if forked_project.visibility_level > visibility_level - forked_project.visibility_level = visibility_level - forked_project.save! - end - end - end - def allowed_to_share_with_group? !namespace.share_with_group_lock end @@ -2039,7 +2070,7 @@ class Project < ApplicationRecord end def group_runners - @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_parent_group_of_project(self.id) : Ci::Runner.none + @group_runners ||= group_runners_enabled? ? Ci::Runner.belonging_to_parent_groups_of_project(self.id) : Ci::Runner.none end def all_runners @@ -2080,7 +2111,11 @@ class Project < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def open_merge_requests_count(_current_user = nil) - Projects::OpenMergeRequestsCountService.new(self).count + BatchLoader.for(self).batch do |projects, loader| + ::Projects::BatchOpenMergeRequestsCountService.new(projects) + .refresh_cache_and_retrieve_data + .each { |project, count| loader.call(project, count) } + end end # rubocop: enable CodeReuse/ServiceClass @@ -2107,23 +2142,13 @@ class Project < ApplicationRecord ensure_runners_token! end - override :format_runners_token - def format_runners_token(token) - "#{RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX}#{token}" - end - def pages_deployed? pages_metadatum&.deployed? end - def pages_namespace_url - # The host in URL always needs to be downcased - Gitlab.config.pages.url.sub(%r{^https?://}) do |prefix| - "#{prefix}#{pages_subdomain}." - end.downcase - end + def pages_url(with_unique_domain: false) + return pages_unique_url if with_unique_domain && pages_unique_domain_enabled? - def pages_url url = pages_namespace_url url_path = full_path.partition('/').last namespace_url = "#{Settings.pages.protocol}://#{url_path}".downcase @@ -2141,6 +2166,18 @@ class Project < ApplicationRecord "#{url}/#{url_path}" end + def pages_unique_url + pages_url_for(project_setting.pages_unique_domain) + end + + def pages_unique_host + URI(pages_unique_url).host + end + + def pages_namespace_url + pages_url_for(pages_subdomain) + end + def pages_subdomain full_path.partition('/').first end @@ -2214,7 +2251,7 @@ class Project < ApplicationRecord wiki.repository.expire_content_cache DetectRepositoryLanguagesWorker.perform_async(id) - ProjectCacheWorker.perform_async(self.id, [], [:repository_size]) + ProjectCacheWorker.perform_async(self.id, [], [:repository_size, :wiki_size]) AuthorizedProjectUpdate::ProjectRecalculateWorker.perform_async(id) enqueue_record_project_target_platforms @@ -2376,6 +2413,8 @@ class Project < ApplicationRecord .append(key: 'CI_SERVER_HOST', value: Gitlab.config.gitlab.host) .append(key: 'CI_SERVER_PORT', value: Gitlab.config.gitlab.port.to_s) .append(key: 'CI_SERVER_PROTOCOL', value: Gitlab.config.gitlab.protocol) + .append(key: 'CI_SERVER_SHELL_SSH_HOST', value: Gitlab.config.gitlab_shell.ssh_host.to_s) + .append(key: 'CI_SERVER_SHELL_SSH_PORT', value: Gitlab.config.gitlab_shell.ssh_port.to_s) .append(key: 'CI_SERVER_NAME', value: 'GitLab') .append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) .append(key: 'CI_SERVER_VERSION_MAJOR', value: Gitlab.version_info.major.to_s) @@ -2396,6 +2435,7 @@ class Project < ApplicationRecord def api_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_API_V4_URL', value: API::Helpers::Version.new('v4').root_url) + variables.append(key: 'CI_API_GRAPHQL_URL', value: Gitlab::Routing.url_helpers.api_graphql_url) end end @@ -2809,15 +2849,15 @@ class Project < ApplicationRecord end def all_protected_branches - if Feature.enabled?(:group_protected_branches) + if allow_protected_branches_for_group? @all_protected_branches ||= ProtectedBranch.from_union([protected_branches, group_protected_branches]) else protected_branches end end - def self_monitoring? - Gitlab::CurrentSettings.self_monitoring_project_id == id + def allow_protected_branches_for_group? + Feature.enabled?(:group_protected_branches, group) || Feature.enabled?(:allow_protected_branches_for_group, group) end def deploy_token_create_url(opts = {}) @@ -2868,11 +2908,11 @@ class Project < ApplicationRecord end def service_desk_custom_address - return unless Gitlab::ServiceDeskEmail.enabled? + return unless Gitlab::Email::ServiceDeskEmail.enabled? key = service_desk_setting&.project_key || default_service_desk_suffix - Gitlab::ServiceDeskEmail.address_for_key("#{full_path_slug}-#{key}") + Gitlab::Email::ServiceDeskEmail.address_for_key("#{full_path_slug}-#{key}") end def default_service_desk_suffix @@ -2918,6 +2958,12 @@ class Project < ApplicationRecord ).exists? end + def has_namespaced_npm_packages? + packages.with_npm_scope(root_namespace.path) + .not_pending_destruction + .exists? + end + def default_branch_or_main return default_branch if default_branch @@ -2971,7 +3017,7 @@ class Project < ApplicationRecord end def ci_inbound_job_token_scope_enabled? - return false unless ci_cd_settings + return true unless ci_cd_settings ci_cd_settings.inbound_job_token_scope_enabled? end @@ -3060,6 +3106,10 @@ class Project < ApplicationRecord pending_delete? || hidden? end + def content_editor_on_issues_feature_flag_enabled? + group&.content_editor_on_issues_feature_flag_enabled? || Feature.enabled?(:content_editor_on_issues, self) + end + def work_items_feature_flag_enabled? group&.work_items_feature_flag_enabled? || Feature.enabled?(:work_items, self) end @@ -3119,8 +3169,31 @@ class Project < ApplicationRecord false end + def crm_enabled? + return false unless group + + group.crm_enabled? + end + + def frozen_outbound_job_token_scopes? + Feature.enabled?(:frozen_outbound_job_token_scopes, self) && Feature.disabled?(:frozen_outbound_job_token_scopes_override, self) + end + strong_memoize_attr :frozen_outbound_job_token_scopes? + private + def pages_unique_domain_enabled? + Feature.enabled?(:pages_unique_domain, self) && + project_setting.pages_unique_domain_enabled? + end + + def pages_url_for(domain) + # The host in URL always needs to be downcased + Gitlab.config.pages.url.sub(%r{^https?://}) do |prefix| + "#{prefix}#{domain}." + end.downcase + end + # overridden in EE def project_group_links_with_preload project_group_links @@ -3224,6 +3297,8 @@ class Project < ApplicationRecord case from when Project namespace_id != from.namespace_id + when Namespaces::ProjectNamespace + namespace_id != from.parent_id when Namespace namespace != from when User @@ -3233,9 +3308,14 @@ class Project < ApplicationRecord # Check if a reference is being done cross-project def cross_project_reference?(from) - return true if from.is_a?(Namespace) - - from && self != from + case from + when Namespaces::ProjectNamespace + project_namespace_id != from.id + when Namespace + true + else + from && self != from + end end def update_project_statistics @@ -3401,6 +3481,10 @@ class Project < ApplicationRecord project_setting.emails_enabled = !emails_disabled end end + + def runners_token_prefix + RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + end end Project.prepend_mod_with('Project') diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index 8741a341ad3..aa65f27870d 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -2,6 +2,7 @@ class ProjectCiCdSetting < ApplicationRecord include ChronicDurationAttribute + include IgnorableColumns belongs_to :project, inverse_of: :ci_cd_settings @@ -20,12 +21,10 @@ class ProjectCiCdSetting < ApplicationRecord attribute :forward_deployment_enabled, default: true attribute :separated_caches, default: true - default_value_for :inbound_job_token_scope_enabled do |settings| - Feature.enabled?(:ci_inbound_job_token_scope, settings.project) - end - chronic_duration_attr :runner_token_expiration_interval_human_readable, :runner_token_expiration_interval + ignore_column :opt_in_jwt, remove_with: '16.2', remove_after: '2023-07-01' + def keep_latest_artifacts_available? # The project level feature can only be enabled when the feature is enabled instance wide Gitlab::CurrentSettings.current_application_settings.keep_latest_artifact? && keep_latest_artifact? diff --git a/app/models/project_custom_attribute.rb b/app/models/project_custom_attribute.rb index b0da586988a..8d3d45715ca 100644 --- a/app/models/project_custom_attribute.rb +++ b/app/models/project_custom_attribute.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ProjectCustomAttribute < ApplicationRecord + include EachBatch + belongs_to :project validates :project, :key, :value, presence: true diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 23b0665cb74..772a82fa173 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -163,6 +163,12 @@ class ProjectFeature < ApplicationRecord end end + def public_packages? + return false unless Gitlab.config.packages.enabled + + package_registry_access_level == PUBLIC || project.public? + end + private def set_pages_access_level @@ -200,7 +206,7 @@ class ProjectFeature < ApplicationRecord override :resource_member? def resource_member?(user, feature) - project.team.member?(user, ProjectFeature.required_minimum_access_level(feature)) + project.member?(user, ProjectFeature.required_minimum_access_level(feature)) end end diff --git a/app/models/project_label.rb b/app/models/project_label.rb index dc647901b46..05d7b7429ff 100644 --- a/app/models/project_label.rb +++ b/app/models/project_label.rb @@ -23,6 +23,10 @@ class ProjectLabel < Label super(project, target_project: target_project, format: format, full: full) end + def preloaded_parent_container + association(:project).loaded? ? project : parent_container + end + private def title_must_not_exist_at_group_level diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index db86bb5e1fb..1256ef0f2fc 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -10,6 +10,27 @@ class ProjectSetting < ApplicationRecord scope :for_projects, ->(projects) { where(project_id: projects) } + attr_encrypted :cube_api_key, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + encode: false, + encode_iv: false + + attr_encrypted :jitsu_administrator_password, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + encode: false, + encode_iv: false + + attr_encrypted :product_analytics_clickhouse_connection_string, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + encode: false, + encode_iv: false + enum squash_option: { never: 0, always: 1, @@ -25,6 +46,10 @@ class ProjectSetting < ApplicationRecord validates :target_platforms, inclusion: { in: ALLOWED_TARGET_PLATFORMS } validates :suggested_reviewers_enabled, inclusion: { in: [true, false] } + validates :pages_unique_domain, + uniqueness: { if: -> { pages_unique_domain.present? } }, + presence: { if: :require_unique_domain? } + validate :validates_mr_default_target_self attribute :legacy_open_source_license_available, default: -> do @@ -61,6 +86,10 @@ class ProjectSetting < ApplicationRecord end strong_memoize_attr :show_diff_preview_in_email? + def runner_registration_enabled + Gitlab::CurrentSettings.valid_runner_registrars.include?('project') && read_attribute(:runner_registration_enabled) + end + private def validates_mr_default_target_self @@ -68,6 +97,11 @@ class ProjectSetting < ApplicationRecord errors.add :mr_default_target_self, _('This setting is allowed for forked projects only') end end + + def require_unique_domain? + pages_unique_domain_enabled || + pages_unique_domain_in_database.present? + end end ProjectSetting.prepend_mod diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 5641fbfb867..dd200aec807 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -121,7 +121,7 @@ class ProjectTeam target_project = project source_members = source_project.project_members.to_a - target_user_ids = target_project.project_members.pluck(:user_id) + target_user_ids = target_project.project_members.pluck_user_ids source_members.reject! do |member| # Skip if user already present in team diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index ffffa803011..e64892dfa03 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -12,6 +12,13 @@ class ProjectWiki < Wiki container.disk_path + '.wiki' end + override :create_wiki_repository + def create_wiki_repository + super + + track_wiki_repository + end + override :after_wiki_activity def after_wiki_activity # Update activity columns, this is done synchronously to avoid @@ -28,6 +35,16 @@ class ProjectWiki < Wiki # the activity columns for Git pushes as well. after_wiki_activity end + + private + + def track_wiki_repository + return unless ::Gitlab::Database.read_write? + return if container.wiki_repository + + # This is the ActiveRecord auto-generated method for a Project's has_one :wiki_repository + container.create_wiki_repository! + end end # TODO: Remove this once we implement ES support for group wikis. diff --git a/app/models/projects/data_transfer.rb b/app/models/projects/data_transfer.rb index a93aea55781..c7f5132fbc7 100644 --- a/app/models/projects/data_transfer.rb +++ b/app/models/projects/data_transfer.rb @@ -4,12 +4,28 @@ # This class ensures that we keep 1 record per project per month. module Projects class DataTransfer < ApplicationRecord + include AfterCommitQueue + include CounterAttribute + self.table_name = 'project_data_transfers' belongs_to :project belongs_to :namespace scope :current_month, -> { where(date: beginning_of_month) } + scope :with_project_between_dates, ->(project, from, to) { + where(project: project, date: from..to) + } + scope :with_namespace_between_dates, ->(namespace, from, to) { + where(namespace: namespace, date: from..to) + .group(:date, :namespace_id) + .order(date: :desc) + } + + counter_attribute :repository_egress, returns_current: true + counter_attribute :artifacts_egress, returns_current: true + counter_attribute :packages_egress, returns_current: true + counter_attribute :registry_egress, returns_current: true def self.beginning_of_month(time = Time.current) time.utc.beginning_of_month diff --git a/app/models/projects/forks/divergence_counts.rb b/app/models/projects/forks/details.rb index 7d630b00083..9e09ef09022 100644 --- a/app/models/projects/forks/divergence_counts.rb +++ b/app/models/projects/forks/details.rb @@ -3,8 +3,11 @@ module Projects module Forks # Class for calculating the divergence of a fork with the source project - class DivergenceCounts + class Details + include Gitlab::Utils::StrongMemoize + LATEST_COMMITS_COUNT = 10 + LEASE_TIMEOUT = 15.minutes.to_i EXPIRATION_TIME = 8.hours def initialize(project, ref) @@ -20,32 +23,55 @@ module Projects { ahead: ahead, behind: behind } end + def exclusive_lease + key = ['project_details', project.id, ref].join(':') + uuid = Gitlab::ExclusiveLease.get_uuid(key) + + Gitlab::ExclusiveLease.new(key, uuid: uuid, timeout: LEASE_TIMEOUT) + end + strong_memoize_attr :exclusive_lease + + def syncing? + exclusive_lease.exists? + end + + def has_conflicts? + !(attrs && attrs[:has_conflicts]).nil? + end + + def update!(params) + Rails.cache.write(cache_key, params, expires_in: EXPIRATION_TIME) + + @attrs = nil + end + private attr_reader :project, :fork_repo, :source_repo, :ref def cache_key - @cache_key ||= ['project_forks', project.id, ref, 'divergence_counts'] + @cache_key ||= ['project_fork_details', project.id, ref].join(':') end def divergence_counts - fork_sha = fork_repo.commit(ref).sha - source_sha = source_repo.commit.sha + sha = fork_repo.commit(ref)&.sha + source_sha = source_repo.commit&.sha - cached_source_sha, cached_fork_sha, counts = Rails.cache.read(cache_key) - return counts if cached_source_sha == source_sha && cached_fork_sha == fork_sha + return if sha.blank? || source_sha.blank? - counts = calculate_divergence_counts(fork_sha, source_sha) + return attrs[:counts] if attrs.present? && attrs[:source_sha] == source_sha && attrs[:sha] == sha - Rails.cache.write(cache_key, [source_sha, fork_sha, counts], expires_in: EXPIRATION_TIME) + counts = calculate_divergence_counts(sha, source_sha) + + update!({ sha: sha, source_sha: source_sha, counts: counts }) counts end - def calculate_divergence_counts(fork_sha, source_sha) + def calculate_divergence_counts(sha, source_sha) # If the upstream latest commit exists in the fork repo, then # it's possible to calculate divergence counts within the fork repository. - return fork_repo.diverging_commit_count(fork_sha, source_sha) if fork_repo.commit(source_sha) + return fork_repo.diverging_commit_count(sha, source_sha) if fork_repo.commit(source_sha) # Otherwise, we need to find a commit that exists both in the fork and upstream # in order to use this commit as a base for calculating divergence counts. @@ -67,6 +93,10 @@ module Projects [ahead, behind] end + + def attrs + @attrs ||= Rails.cache.read(cache_key) + end end end end diff --git a/app/models/projects/import_export/relation_export.rb b/app/models/projects/import_export/relation_export.rb index 9bdf10d7c0e..2771c5131b2 100644 --- a/app/models/projects/import_export/relation_export.rb +++ b/app/models/projects/import_export/relation_export.rb @@ -51,12 +51,16 @@ module Projects transition queued: :started end + event :retry do + transition started: :queued + end + event :finish do transition started: :finished end event :fail_op do - transition [:queued, :started] => :failed + transition [:queued, :started, :failed] => :failed end end @@ -65,6 +69,14 @@ module Projects project_tree_relation_names + EXTRA_RELATION_LIST end + + def mark_as_failed(export_error) + sanitized_error = Gitlab::UrlSanitizer.sanitize(export_error) + + fail_op + + update_column(:export_error, sanitized_error) + end end end end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index b3331b99a6b..09a0cfc91dc 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -4,10 +4,13 @@ class ProtectedBranch < ApplicationRecord include ProtectedRef include Gitlab::SQL::Pattern include FromUnion + include EachBatch belongs_to :group, foreign_key: :namespace_id, touch: true, inverse_of: :protected_branches validate :validate_either_project_or_top_group + validates :name, presence: true + validates :name, uniqueness: { scope: [:project_id, :namespace_id] }, if: :name_changed? scope :requiring_code_owner_approval, -> { where(code_owner_approval_required: true) } scope :allowing_force_push, -> { where(allow_force_push: true) } @@ -26,7 +29,7 @@ class ProtectedBranch < ApplicationRecord # Maintainers, owners and admins are allowed to create the default branch if project.empty_repo? && project.default_branch_protected? - return true if user.admin? || project.team.max_member_access(user.id) > Gitlab::Access::DEVELOPER + return true if user.admin? || user.can?(:admin_project, project) end super @@ -37,38 +40,13 @@ class ProtectedBranch < ApplicationRecord return true if project.empty_repo? && project.default_branch_protected? return false if ref_name.blank? - dry_run = Feature.disabled?(:rely_on_protected_branches_cache, project) - - new_cache_result = new_cache(project, ref_name, dry_run: dry_run) - - return new_cache_result unless new_cache_result.nil? - - deprecated_cache(project, ref_name) - end - - def self.new_cache(project, ref_name, dry_run: true) - ProtectedBranches::CacheService.new(project).fetch(ref_name, dry_run: dry_run) do # rubocop: disable CodeReuse/ServiceClass - self.matching(ref_name, protected_refs: protected_refs(project)).present? - end - end - - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/370608 - # ---------------------------------------------------------------- - CACHE_EXPIRE_IN = 1.hour - - def self.deprecated_cache(project, ref_name) - Rails.cache.fetch(protected_ref_cache_key(project, ref_name), expires_in: CACHE_EXPIRE_IN) do + ProtectedBranches::CacheService.new(project).fetch(ref_name) do # rubocop: disable CodeReuse/ServiceClass self.matching(ref_name, protected_refs: protected_refs(project)).present? end end - def self.protected_ref_cache_key(project, ref_name) - "protected_ref-#{project.cache_key}-#{Digest::SHA1.hexdigest(ref_name)}" - end - # End of deprecation -------------------------------------------- - def self.allow_force_push?(project, ref_name) - if Feature.enabled?(:group_protected_branches) + if allow_protected_branches_for_group?(project.group) protected_branches = project.all_protected_branches.matching(ref_name) project_protected_branches, group_protected_branches = protected_branches.partition(&:project_id) @@ -83,6 +61,10 @@ class ProtectedBranch < ApplicationRecord end end + def self.allow_protected_branches_for_group?(group) + Feature.enabled?(:group_protected_branches, group) || Feature.enabled?(:allow_protected_branches_for_group, group) + end + def self.any_protected?(project, ref_names) protected_refs(project).any? do |protected_ref| ref_names.any? do |ref_name| @@ -92,11 +74,7 @@ class ProtectedBranch < ApplicationRecord end def self.protected_refs(project) - if Feature.enabled?(:group_protected_branches) - project.all_protected_branches - else - project.protected_branches - end + project.all_protected_branches end # overridden in EE diff --git a/app/models/protected_branch/push_access_level.rb b/app/models/protected_branch/push_access_level.rb index 66fe57be25f..c86ca5723fa 100644 --- a/app/models/protected_branch/push_access_level.rb +++ b/app/models/protected_branch/push_access_level.rb @@ -21,6 +21,12 @@ class ProtectedBranch::PushAccessLevel < ApplicationRecord end end + def humanize + return "Deploy key" if deploy_key.present? + + super + end + def check_access(user) if user && deploy_key.present? return user.can?(:read_project, project) && enabled_deploy_key_for_user?(deploy_key, user) diff --git a/app/models/protected_ref/access_level.rb b/app/models/protected_ref/access_level.rb new file mode 100644 index 00000000000..ffd3b480b70 --- /dev/null +++ b/app/models/protected_ref/access_level.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module ProtectedRef + class AccessLevel + extend ProtectedRefAccess::ClassMethods + end +end diff --git a/app/models/protected_tag.rb b/app/models/protected_tag.rb index e89cb3aabc7..5d215a364b7 100644 --- a/app/models/protected_tag.rb +++ b/app/models/protected_tag.rb @@ -2,6 +2,7 @@ class ProtectedTag < ApplicationRecord include ProtectedRef + include EachBatch validates :name, uniqueness: { scope: :project_id } validates :project, presence: true diff --git a/app/models/protected_tag/create_access_level.rb b/app/models/protected_tag/create_access_level.rb index abb233d3800..5837f3a5afb 100644 --- a/app/models/protected_tag/create_access_level.rb +++ b/app/models/protected_tag/create_access_level.rb @@ -12,35 +12,39 @@ class ProtectedTag::CreateAccessLevel < ApplicationRecord validate :validate_deploy_key_membership def type - if deploy_key.present? - :deploy_key - else - super - end - end + return :deploy_key if deploy_key.present? - def check_access(user) - return false if access_level == Gitlab::Access::NO_ACCESS + super + end - if user && deploy_key.present? - return user.can?(:read_project, project) && enabled_deploy_key_for_user?(deploy_key, user) - end + def humanize + return "Deploy key" if deploy_key.present? super end + def check_access(current_user) + super do + break enabled_deploy_key_for_user?(current_user) if deploy_key? + end + end + private + def deploy_key? + type == :deploy_key + end + def validate_deploy_key_membership return unless deploy_key - return if project.deploy_keys_projects.where(deploy_key: deploy_key).exists? errors.add(:deploy_key, 'is not enabled for this project') end - def enabled_deploy_key_for_user?(deploy_key, user) - deploy_key.user_id == user.id && + def enabled_deploy_key_for_user?(current_user) + current_user.can?(:read_project, project) && + deploy_key.user_id == current_user.id && DeployKey.with_write_access_for_project(protected_tag.project, deploy_key: deploy_key).any? end end diff --git a/app/models/releases/link.rb b/app/models/releases/link.rb index e02486fbc5b..67d765a15c0 100644 --- a/app/models/releases/link.rb +++ b/app/models/releases/link.rb @@ -37,15 +37,9 @@ module Releases url.start_with?(release.project.web_url) end - # `external?` is deprecated in 15.9 and will be removed in 16.0. - def external? - !internal? - end - def hook_attrs { id: id, - external: external?, # `external` is deprecated in 15.9 and will be removed in 16.0. link_type: link_type, name: name, url: url diff --git a/app/models/repository.rb b/app/models/repository.rb index f951418c0bf..e942157993b 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -48,7 +48,7 @@ class Repository # For example, for entry `:commit_count` there's a method called `commit_count` which # stores its data in the `commit_count` cache key. CACHED_METHODS = %i(size commit_count readme_path contribution_guide - changelog license_blob license_licensee license_gitaly gitignore + changelog license_blob license_gitaly gitignore gitlab_ci_yml branch_names tag_names branch_count tag_count avatar exists? root_ref merged_branch_names has_visible_content? issue_template_names_hash merge_request_template_names_hash @@ -60,7 +60,7 @@ class Repository METHOD_CACHES_FOR_FILE_TYPES = { readme: %i(readme_path), changelog: :changelog, - license: %i(license_blob license_licensee license_gitaly), + license: %i(license_blob license_gitaly), contributing: :contribution_guide, gitignore: :gitignore, gitlab_ci: :gitlab_ci_yml, @@ -161,7 +161,8 @@ class Repository first_parent: !!opts[:first_parent], order: opts[:order], literal_pathspec: opts.fetch(:literal_pathspec, true), - trailers: opts[:trailers] + trailers: opts[:trailers], + include_referenced_by: opts[:include_referenced_by] } commits = Gitlab::Git::Commit.where(options) @@ -198,7 +199,7 @@ class Repository def list_commits_by(query, ref, author: nil, before: nil, after: nil, limit: 1000) return [] unless exists? return [] unless has_visible_content? - return [] unless query.present? && ref.present? + return [] unless ref.present? commits = raw_repository.list_commits_by( query, ref, author: author, before: before, after: after, limit: limit).map do |c| @@ -655,24 +656,13 @@ class Repository end def license - if Feature.enabled?(:license_from_gitaly) - license_gitaly - else - license_licensee - end + license_gitaly end - def license_licensee - return unless exists? - - raw_repository.license(false) - end - cache_method :license_licensee - def license_gitaly return unless exists? - raw_repository.license(true) + raw_repository.license end cache_method :license_gitaly @@ -720,8 +710,6 @@ class Repository if last_commit blob_at(last_commit.sha, path) - else - nil end end @@ -844,6 +832,26 @@ class Repository commit_files(user, **options) end + def move_dir_files(user, path, previous_path, **options) + regex = Regexp.new("^#{Regexp.escape(previous_path + '/')}", 'i') + files = ls_files(options[:branch_name]) + + options[:actions] = files.each_with_object([]) do |item, list| + next unless item =~ regex + + list.push( + action: :move, + file_path: "#{path}/#{item[regex.match(item)[0].size..]}", + previous_path: item, + infer_content: true + ) + end + + return if options[:actions].blank? + + commit_files(user, **options) + end + def delete_file(user, path, **options) options[:actions] = [{ action: :delete, file_path: path }] @@ -948,19 +956,19 @@ class Repository end def merged_to_root_ref?(branch_or_name) + return unless head_commit + branch = Gitlab::Git::Branch.find(self, branch_or_name) if branch same_head = branch.target == root_ref_sha merged = ancestor?(branch.target, root_ref_sha) !same_head && merged - else - nil end end def root_ref_sha - @root_ref_sha ||= commit(root_ref).sha + @root_ref_sha ||= head_commit.sha end # If this method is not provided a set of branch names to check merge status, diff --git a/app/models/resource_events/abuse_report_event.rb b/app/models/resource_events/abuse_report_event.rb new file mode 100644 index 00000000000..2cddfc393e3 --- /dev/null +++ b/app/models/resource_events/abuse_report_event.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module ResourceEvents + class AbuseReportEvent < ApplicationRecord + belongs_to :abuse_report, optional: false + belongs_to :user + + validates :action, presence: true + + enum action: { + ban_user: 1, + block_user: 2, + delete_user: 3, + close_report: 4, + ban_user_and_close_report: 5, + block_user_and_close_report: 6, + delete_user_and_close_report: 7 + } + + enum reason: { + spam: 1, + offensive: 2, + phishing: 3, + crypto: 4, + credentials: 5, + copyright: 6, + malware: 7, + other: 8, + unconfirmed: 9 + } + end +end diff --git a/app/models/resource_events/issue_assignment_event.rb b/app/models/resource_events/issue_assignment_event.rb new file mode 100644 index 00000000000..393e2aa8942 --- /dev/null +++ b/app/models/resource_events/issue_assignment_event.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module ResourceEvents + class IssueAssignmentEvent < ApplicationRecord + self.table_name = :issue_assignment_events + + belongs_to :user, optional: true + belongs_to :issue + + validates :issue, presence: true + + enum action: { add: 1, remove: 2 } + + def self.issuable_id_column + :issue_id + end + end +end diff --git a/app/models/resource_events/merge_request_assignment_event.rb b/app/models/resource_events/merge_request_assignment_event.rb new file mode 100644 index 00000000000..778b9101858 --- /dev/null +++ b/app/models/resource_events/merge_request_assignment_event.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module ResourceEvents + class MergeRequestAssignmentEvent < ApplicationRecord + self.table_name = :merge_request_assignment_events + + belongs_to :user, optional: true + belongs_to :merge_request + + validates :merge_request, presence: true + + enum action: { add: 1, remove: 2 } + + def self.issuable_id_column + :merge_request_id + end + end +end diff --git a/app/models/resource_label_event.rb b/app/models/resource_label_event.rb index efffc1bd6dc..13610d37a74 100644 --- a/app/models/resource_label_event.rb +++ b/app/models/resource_label_event.rb @@ -29,9 +29,8 @@ class ResourceLabelEvent < ResourceEvent labels = events.map(&:label).compact project_labels, group_labels = labels.partition { |label| label.is_a? ProjectLabel } - preloader = ActiveRecord::Associations::Preloader.new - preloader.preload(project_labels, { project: :project_feature }) - preloader.preload(group_labels, :group) + ActiveRecord::Associations::Preloader.new(records: project_labels, associations: { project: :project_feature }).call + ActiveRecord::Associations::Preloader.new(records: group_labels, associations: :group).call end def issuable diff --git a/app/models/resource_milestone_event.rb b/app/models/resource_milestone_event.rb index def7e91af3f..d305a4ace51 100644 --- a/app/models/resource_milestone_event.rb +++ b/app/models/resource_milestone_event.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class ResourceMilestoneEvent < ResourceTimeboxEvent - include IgnorableColumns - belongs_to :milestone scope :include_relations, -> { includes(:user, milestone: [:project, :group]) } @@ -10,8 +8,6 @@ class ResourceMilestoneEvent < ResourceTimeboxEvent # state is used for issue and merge request states. enum state: Issue.available_states.merge(MergeRequest.available_states) - ignore_columns %i[reference reference_html cached_markdown_version], remove_with: '13.1', remove_after: '2020-06-22' - def milestone_title milestone&.title end @@ -24,3 +20,5 @@ class ResourceMilestoneEvent < ResourceTimeboxEvent MilestoneNote end end + +ResourceMilestoneEvent.prepend_mod diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index 1a0a65df6a3..580e4cd277c 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -16,8 +16,6 @@ class SentNotification < ApplicationRecord validates :in_reply_to_discussion_id, format: { with: /\A\h{40}\z/, allow_nil: true } validate :note_valid - ignore_column :id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' - after_save :keep_around_commit, if: :for_commit? class << self @@ -105,9 +103,18 @@ class SentNotification < ApplicationRecord self.reply_key end - def create_reply(message, dryrun: false) + def create_reply(message, external_author = nil, dryrun: false) klass = dryrun ? Notes::BuildService : Notes::CreateService - klass.new(self.project, self.recipient, reply_params.merge(note: message)).execute + params = reply_params.merge( + note: message + ) + + params[:external_author] = external_author if external_author.present? + + klass.new(self.project, + self.recipient, + params + ).execute end private diff --git a/app/models/serverless/domain.rb b/app/models/serverless/domain.rb deleted file mode 100644 index 164f93afa9a..00000000000 --- a/app/models/serverless/domain.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class Domain - include ActiveModel::Model - - REGEXP = %r{^(?<scheme>https?://)?(?<function_name>[^.]+)-(?<cluster_left>\h{2})a1(?<cluster_middle>\h{10})f2(?<cluster_right>\h{2})(?<environment_id>\h+)-(?<environment_slug>[^.]+)\.(?<pages_domain_name>.+)}.freeze - UUID_LENGTH = 14 - - attr_accessor :function_name, :serverless_domain_cluster, :environment - - validates :function_name, presence: true, allow_blank: false - validates :serverless_domain_cluster, presence: true - validates :environment, presence: true - - def self.generate_uuid - SecureRandom.hex(UUID_LENGTH / 2) - end - - def uri - URI("https://#{function_name}-#{serverless_domain_cluster_uuid}#{"%x" % environment.id}-#{environment.slug}.#{serverless_domain_cluster.domain}") - end - - def knative_uri - URI("http://#{function_name}.#{namespace}.#{serverless_domain_cluster.knative.hostname}") - end - - private - - def namespace - serverless_domain_cluster.cluster.kubernetes_namespace_for(environment) - end - - def serverless_domain_cluster_uuid - [ - serverless_domain_cluster.uuid[0..1], - 'a1', - serverless_domain_cluster.uuid[2..-3], - 'f2', - serverless_domain_cluster.uuid[-2..] - ].join - end - end -end diff --git a/app/models/serverless/domain_cluster.rb b/app/models/serverless/domain_cluster.rb deleted file mode 100644 index 561bfc65b2b..00000000000 --- a/app/models/serverless/domain_cluster.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class DomainCluster < ApplicationRecord - self.table_name = 'serverless_domain_cluster' - - HEX_REGEXP = %r{\A\h+\z}.freeze - - belongs_to :pages_domain - belongs_to :knative, class_name: 'Clusters::Applications::Knative', foreign_key: 'clusters_applications_knative_id' - belongs_to :creator, class_name: 'User', optional: true - - attr_encrypted :key, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm' - - validates :pages_domain, :knative, presence: true - validates :uuid, presence: true, uniqueness: true, length: { is: ::Serverless::Domain::UUID_LENGTH }, - format: { with: HEX_REGEXP, message: 'only allows hex characters' } - - after_initialize :set_uuid, if: :new_record? - - delegate :domain, to: :pages_domain - delegate :cluster, to: :knative - - def self.for_uuid(uuid) - joins(:pages_domain, :knative) - .includes(:pages_domain, :knative) - .find_by(uuid: uuid) - end - - private - - def set_uuid - self.uuid = ::Serverless::Domain.generate_uuid - end - end -end diff --git a/app/models/serverless/function.rb b/app/models/serverless/function.rb deleted file mode 100644 index 5d4f8e0c9e2..00000000000 --- a/app/models/serverless/function.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class Function - attr_accessor :name, :namespace - - def initialize(project, name, namespace) - @project = project - @name = name - @namespace = namespace - end - - def id - @project.id.to_s + "/" + @name + "/" + @namespace - end - - def self.find_by_id(id) - array = id.split("/") - project = Project.find_by_id(array[0]) - name = array[1] - namespace = array[2] - - self.new(project, name, namespace) - end - end -end diff --git a/app/models/serverless/lookup_path.rb b/app/models/serverless/lookup_path.rb deleted file mode 100644 index c09b3718651..00000000000 --- a/app/models/serverless/lookup_path.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class LookupPath - attr_reader :serverless_domain - - delegate :serverless_domain_cluster, to: :serverless_domain - delegate :knative, to: :serverless_domain_cluster - delegate :certificate, to: :serverless_domain_cluster - delegate :key, to: :serverless_domain_cluster - - def initialize(serverless_domain) - @serverless_domain = serverless_domain - end - - def source - { - type: 'serverless', - service: serverless_domain.knative_uri.host, - cluster: { - hostname: knative.hostname, - address: knative.external_ip, - port: 443, - cert: certificate, - key: key - } - } - end - end -end diff --git a/app/models/serverless/virtual_domain.rb b/app/models/serverless/virtual_domain.rb deleted file mode 100644 index d6a23a4c0ce..00000000000 --- a/app/models/serverless/virtual_domain.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Serverless - class VirtualDomain - attr_reader :serverless_domain - - delegate :serverless_domain_cluster, to: :serverless_domain - delegate :pages_domain, to: :serverless_domain_cluster - delegate :certificate, to: :pages_domain - delegate :key, to: :pages_domain - - def initialize(serverless_domain) - @serverless_domain = serverless_domain - end - - def lookup_paths - [ - ::Serverless::LookupPath.new(serverless_domain) - ] - end - end -end diff --git a/app/models/airflow.rb b/app/models/service_desk.rb index 2e5642a2639..cb9c924c01f 100644 --- a/app/models/airflow.rb +++ b/app/models/service_desk.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true -module Airflow + +module ServiceDesk def self.table_name_prefix - 'airflow_' + 'service_desk_' end end diff --git a/app/models/service_desk/custom_email_credential.rb b/app/models/service_desk/custom_email_credential.rb new file mode 100644 index 00000000000..8ccdd6f2261 --- /dev/null +++ b/app/models/service_desk/custom_email_credential.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module ServiceDesk + class CustomEmailCredential < ApplicationRecord + attr_encrypted :smtp_username, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_iv: false + attr_encrypted :smtp_password, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_iv: false + + belongs_to :project + + validates :project, presence: true + + validates :smtp_address, + presence: true, + length: { maximum: 255 }, + hostname: { allow_numeric_hostname: true } + validate :validate_smtp_address + + validates :smtp_port, + presence: true, + numericality: { only_integer: true, greater_than: 0 } + + validates :smtp_username, + presence: true, + length: { maximum: 255 } + validates :smtp_password, + presence: true, + length: { minimum: 8, maximum: 128 } + + delegate :service_desk_setting, to: :project + + def delivery_options + { + user_name: smtp_username, + password: smtp_password, + address: smtp_address, + domain: Mail::Address.new(service_desk_setting.custom_email).domain, + port: smtp_port || 587 + } + end + + private + + def validate_smtp_address + # Addressable::URI always needs a scheme otherwise it interprets the host as the path + Gitlab::UrlBlocker.validate!("smtp://#{smtp_address}", + schemes: %w[smtp], + ascii_only: true, + enforce_sanitization: true, + allow_localhost: false, + allow_local_network: false + ) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + errors.add(:smtp_address, e) + end + end +end diff --git a/app/models/service_desk/custom_email_verification.rb b/app/models/service_desk/custom_email_verification.rb new file mode 100644 index 00000000000..482a10447ed --- /dev/null +++ b/app/models/service_desk/custom_email_verification.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module ServiceDesk + class CustomEmailVerification < ApplicationRecord + TIMEFRAME = 30.minutes + STATES = { started: 0, finished: 1, failed: 2 }.freeze + + enum error: { + incorrect_token: 0, + incorrect_from: 1, + mail_not_received_within_timeframe: 2, + invalid_credentials: 3, + smtp_host_issue: 4 + } + + attr_encrypted :token, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_32, + encode: false, + encode_iv: false + + belongs_to :project + belongs_to :triggerer, class_name: 'User', optional: true + + validates :project, presence: true + validates :state, presence: true + + delegate :service_desk_setting, to: :project + + state_machine :state do + state :started do + validates :token, presence: true, length: { is: 12 } + validates :triggerer, presence: true + validates :triggered_at, presence: true + validates :error, absence: true + end + + state :finished do + validates :token, absence: true + validates :error, absence: true + end + + state :failed do + validates :token, absence: true + validates :error, presence: true + end + + event :mark_as_started do + transition all => :started + end + + event :mark_as_finished do + transition started: :finished + end + + event :mark_as_failed do + transition all => :failed + end + + before_transition any => :started do |verification, transition| + triggerer = transition.args.first + + verification.triggerer = triggerer + verification.token = verification.class.generate_token + verification.triggered_at = Time.current + verification.error = nil + end + + before_transition started: :finished do |verification| + verification.token = nil + end + + before_transition started: :failed do |verification, transition| + error = transition.args.first + + verification.error = error + verification.token = nil + end + + # Supress warning: + # both enum and its state_machine have defined a different default for "state". + # State machine uses `nil` and the enum should use the same. + def owner_class_attribute_default + nil + end + end + + # Needs to be below `state_machine` definition to suppress + # its method override warnings + enum state: STATES + + class << self + def generate_token + SecureRandom.alphanumeric(12) + end + end + + def accepted_until + return unless started? + return unless triggered_at.present? + + TIMEFRAME.since(triggered_at) + end + + def in_timeframe? + return false unless started? + + !!accepted_until&.future? + end + end +end diff --git a/app/models/service_desk_setting.rb b/app/models/service_desk_setting.rb index 5152746abb4..4216ad7e70f 100644 --- a/app/models/service_desk_setting.rb +++ b/app/models/service_desk_setting.rb @@ -2,59 +2,59 @@ class ServiceDeskSetting < ApplicationRecord include Gitlab::Utils::StrongMemoize + include IgnorableColumns + + CUSTOM_EMAIL_VERIFICATION_SUBADDRESS = '+verify' + + ignore_columns %i[ + custom_email_smtp_address + custom_email_smtp_port + custom_email_smtp_username + encrypted_custom_email_smtp_password + encrypted_custom_email_smtp_password_iv + ], remove_with: '16.1', remove_after: '2023-05-22' attribute :custom_email_enabled, default: false - attr_encrypted :custom_email_smtp_password, - mode: :per_attribute_iv, - algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_32, - encode: false, - encode_iv: false belongs_to :project + validates :project_id, presence: true validate :valid_issue_template validate :valid_project_key validates :outgoing_name, length: { maximum: 255 }, allow_blank: true validates :project_key, - length: { maximum: 255 }, - allow_blank: true, - format: { with: /\A[a-z0-9_]+\z/, message: -> (setting, data) { _("can contain only lowercase letters, digits, and '_'.") } } + length: { maximum: 255 }, + allow_blank: true, + format: { with: /\A[a-z0-9_]+\z/, message: -> (setting, data) { _("can contain only lowercase letters, digits, and '_'.") } } validates :custom_email, - length: { maximum: 255 }, - uniqueness: true, - allow_nil: true, - format: /\A[\w\-._]+@[\w\-.]+\.{1}[a-zA-Z]{2,}\z/ - validates :custom_email_smtp_address, length: { maximum: 255 } - validates :custom_email_smtp_username, length: { maximum: 255 } - + length: { maximum: 255 }, + uniqueness: true, + allow_nil: true, + format: /\A[\w\-._]+@[\w\-.]+\.{1}[a-zA-Z]{2,}\z/ + + validates :custom_email_credential, + presence: true, + if: :needs_custom_email_credentials? validates :custom_email, - presence: true, - devise_email: true, - if: :custom_email_enabled? - validates :custom_email_smtp_address, - presence: true, - hostname: { allow_numeric_hostname: true, require_valid_tld: true }, - if: :custom_email_enabled? - validates :custom_email_smtp_username, - presence: true, - if: :custom_email_enabled? - validates :custom_email_smtp_port, - presence: true, - numericality: { only_integer: true, greater_than: 0 }, - if: :custom_email_enabled? + presence: true, + devise_email: true, + if: :needs_custom_email_credentials? scope :with_project_key, ->(key) { where(project_key: key) } - def custom_email_delivery_options - { - user_name: custom_email_smtp_username, - password: custom_email_smtp_password, - address: custom_email_smtp_address, - domain: Mail::Address.new(custom_email).domain, - port: custom_email_smtp_port || 587 - } + def custom_email_credential + project&.service_desk_custom_email_credential + end + + def custom_email_verification + project&.service_desk_custom_email_verification + end + + def custom_email_address_for_verification + return unless custom_email.present? + + custom_email.sub("@", "#{CUSTOM_EMAIL_VERIFICATION_SUBADDRESS}@") end def issue_template_content @@ -102,6 +102,10 @@ class ServiceDeskSetting < ApplicationRecord setting.project.full_path_slug == project_slug end end + + def needs_custom_email_credentials? + custom_email_enabled? || custom_email_verification.present? + end end ServiceDeskSetting.prepend_mod diff --git a/app/models/slack_integration.rb b/app/models/slack_integration.rb new file mode 100644 index 00000000000..22e911aeacd --- /dev/null +++ b/app/models/slack_integration.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +class SlackIntegration < ApplicationRecord + include EachBatch + + ALL_FEATURES = %i[commands notifications].freeze + + SCOPE_COMMANDS = 'commands' + SCOPE_CHAT_WRITE = 'chat:write' + SCOPE_CHAT_WRITE_PUBLIC = 'chat:write.public' + + # These scopes are requested when installing the app, additional scopes + # will need reauthorization. + # https://api.slack.com/authentication/oauth-v2#asking + SCOPES = [SCOPE_COMMANDS, SCOPE_CHAT_WRITE, SCOPE_CHAT_WRITE_PUBLIC].freeze + + belongs_to :integration + + attr_encrypted :bot_access_token, + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + encode: false, + encode_iv: false + + has_many :slack_integrations_scopes, + class_name: '::Integrations::SlackWorkspace::IntegrationApiScope' + + has_many :slack_api_scopes, + class_name: '::Integrations::SlackWorkspace::ApiScope', + through: :slack_integrations_scopes + + scope :with_bot, -> { where.not(bot_user_id: nil) } + scope :by_team, ->(team_id) { where(team_id: team_id) } + + validates :team_id, presence: true + validates :team_name, presence: true + validates :alias, presence: true, + uniqueness: { scope: :team_id, message: 'This alias has already been taken' }, + length: 2..4096 + validates :user_id, presence: true + validates :integration, presence: true + + after_commit :update_active_status_of_integration, on: [:create, :destroy] + + def update_active_status_of_integration + integration.update_active_status + end + + def feature_available?(feature_name) + case feature_name + when :commands + # The slash commands feature requires 'commands' scope. + # All records will support this scope, as this was the original feature. + true + when :notifications + scoped_to?(SCOPE_CHAT_WRITE, SCOPE_CHAT_WRITE_PUBLIC) + else + false + end + end + + def upgrade_needed? + !all_features_supported? + end + + def all_features_supported? + ALL_FEATURES.all? { |feature| feature_available?(feature) } # rubocop: disable Gitlab/FeatureAvailableUsage + end + + def authorized_scope_names=(names) + names = Array.wrap(names).flat_map { |name| name.split(',') }.map(&:strip) + + scopes = ::Integrations::SlackWorkspace::ApiScope.find_or_initialize_by_names(names) + self.slack_api_scopes = scopes + end + + def authorized_scope_names + slack_api_scopes.pluck(:name) + end + + private + + def scoped_to?(*names) + return false if names.empty? + + names.to_set <= all_scopes + end + + def all_scopes + @all_scopes = authorized_scope_names.to_set + end +end diff --git a/app/models/snippet.rb b/app/models/snippet.rb index 9ec685c5580..3c40f4beedc 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -19,6 +19,7 @@ class Snippet < ApplicationRecord include AfterCommitQueue extend ::Gitlab::Utils::Override include CreatedAtFilterable + include EachBatch MAX_FILE_COUNT = 10 @@ -156,7 +157,7 @@ class Snippet < ApplicationRecord def for_project_with_user(project, user = nil) return none unless project.snippets_visible?(user) - if user && project.team.member?(user) + if project.member?(user) project.snippets else project.snippets.public_to_user(user) @@ -183,7 +184,7 @@ class Snippet < ApplicationRecord end def link_reference_pattern - @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/) + @link_reference_pattern ||= compose_link_reference_pattern('snippets', /(?<snippet>\d+)/) end def find_by_id_and_project(id:, project:) @@ -203,14 +204,7 @@ class Snippet < ApplicationRecord end def initialize(attributes = {}) - # We can't use default_value_for because the database has a default - # value of 0 for visibility_level. If someone attempts to create a - # private snippet, default_value_for will assume that the - # visibility_level hasn't changed and will use the application - # setting default, which could be internal or public. - # - # To fix the problem, we assign the actual snippet default if no - # explicit visibility has been initialized. + # We assign the actual snippet default if no explicit visibility has been initialized. attributes ||= {} unless visibility_attribute_present?(attributes) diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index bb8527d8c01..0e0534d45ae 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -26,8 +26,7 @@ class SystemNoteMetadata < ApplicationRecord title time_tracking branch milestone discussion task moved cloned opened closed merged duplicate locked unlocked outdated reviewer tag due_date start_date_or_due_date pinned_embed cherry_pick health_status approved unapproved - status alert_issue_added relate unrelate new_alert_added severity - attention_requested attention_request_removed contact timeline_event + status alert_issue_added relate unrelate new_alert_added severity contact timeline_event issue_type relate_to_child unrelate_from_child relate_to_parent unrelate_from_parent ].freeze diff --git a/app/models/terraform/state.rb b/app/models/terraform/state.rb index 8a207c891e2..93c128c989c 100644 --- a/app/models/terraform/state.rb +++ b/app/models/terraform/state.rb @@ -8,6 +8,8 @@ module Terraform HEX_REGEXP = %r{\A\h+\z}.freeze UUID_LENGTH = 32 + self.locking_column = :activerecord_lock_version + belongs_to :project belongs_to :locked_by_user, class_name: 'User' diff --git a/app/models/terraform/state_version.rb b/app/models/terraform/state_version.rb index d6a16ad5b99..6727c81f17b 100644 --- a/app/models/terraform/state_version.rb +++ b/app/models/terraform/state_version.rb @@ -5,7 +5,7 @@ module Terraform include EachBatch include FileStoreMounter - belongs_to :terraform_state, class_name: 'Terraform::State', optional: false + belongs_to :terraform_state, class_name: 'Terraform::State', optional: false, touch: true belongs_to :created_by_user, class_name: 'User', optional: true belongs_to :build, class_name: 'Ci::Build', optional: true, foreign_key: :ci_build_id diff --git a/app/models/todo.rb b/app/models/todo.rb index 62252912c32..e1b5076e3d8 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -13,6 +13,7 @@ class Todo < ApplicationRecord # and giving it back again. WAIT_FOR_DELETE = 1.hour + # Actions ASSIGNED = 1 MENTIONED = 2 BUILD_FAILED = 3 @@ -76,10 +77,11 @@ class Todo < ApplicationRecord scope :for_target, -> (id) { where(target_id: id) } scope :for_commit, -> (id) { where(commit_id: id) } scope :with_entity_associations, -> do - preload(:target, :author, :note, group: :route, project: [:route, { namespace: [:route, :owner] }, :project_setting]) + preload(:target, :author, :note, group: :route, project: [:route, :group, { namespace: [:route, :owner] }, :project_setting]) end scope :joins_issue_and_assignees, -> { left_joins(issue: :assignees) } scope :for_internal_notes, -> { joins(:note).where(note: { confidential: true }) } + scope :with_preloaded_user, -> { preload(:user) } enum resolved_by_action: { system_done: 0, api_all_done: 1, api_done: 2, mark_all_done: 3, mark_done: 4 }, _prefix: :resolved_by diff --git a/app/models/u2f_registration.rb b/app/models/u2f_registration.rb deleted file mode 100644 index ba6c1ee6af1..00000000000 --- a/app/models/u2f_registration.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -# Registration information for U2F (universal 2nd factor) devices, like Yubikeys - -class U2fRegistration < ApplicationRecord - belongs_to :user - - after_create :create_webauthn_registration - after_update :update_webauthn_registration, if: :saved_change_to_counter? - - def self.register(user, app_id, params, challenges) - u2f = U2F::U2F.new(app_id) - registration = self.new - - begin - response = U2F::RegisterResponse.load_from_json(params[:device_response]) - registration_data = u2f.register!(challenges, response) - registration.update(certificate: registration_data.certificate, - key_handle: registration_data.key_handle, - public_key: registration_data.public_key, - counter: registration_data.counter, - user: user, - name: params[:name]) - rescue JSON::ParserError, NoMethodError, ArgumentError - registration.errors.add(:base, _('Your U2F device did not send a valid JSON response.')) - rescue U2F::Error => e - registration.errors.add(:base, e.message) - end - - registration - end - - def self.authenticate(user, app_id, json_response, challenges) - response = U2F::SignResponse.load_from_json(json_response) - registration = user.u2f_registrations.find_by_key_handle(response.key_handle) - u2f = U2F::U2F.new(app_id) - - if registration - u2f.authenticate!(challenges, response, Base64.decode64(registration.public_key), registration.counter) - registration.update(counter: response.counter) - true - end - rescue JSON::ParserError, NoMethodError, ArgumentError, U2F::Error - false - end - - private - - def create_webauthn_registration - converter = Gitlab::Auth::U2fWebauthnConverter.new(self) - WebauthnRegistration.create!(converter.convert) - rescue StandardError => e - Gitlab::ErrorTracking.track_exception(e, u2f_registration_id: self.id) - end - - def update_webauthn_registration - # When we update the sign count of this registration - # we need to update the sign count of the corresponding webauthn registration - # as well if it exists already - WebauthnRegistration.find_by_credential_xid(webauthn_credential_xid) - &.update_attribute(:counter, counter) - end - - def webauthn_credential_xid - Base64.strict_encode64(Base64.urlsafe_decode64(key_handle)) - end -end diff --git a/app/models/user.rb b/app/models/user.rb index f3e8f14adf5..dc70ff2e232 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -9,7 +9,6 @@ class User < ApplicationRecord include Gitlab::SQL::Pattern include AfterCommitQueue include Avatarable - include Awareness include Referable include Sortable include CaseSensitivity @@ -28,6 +27,7 @@ class User < ApplicationRecord include UpdateHighestRole include HasUserType include Gitlab::Auth::Otp::Fortinet + include Gitlab::Auth::Otp::DuoAuth include RestrictedSignup include StripAttribute include EachBatch @@ -71,6 +71,7 @@ class User < ApplicationRecord attribute :notified_of_own_activity, default: false attribute :preferred_language, default: -> { Gitlab::CurrentSettings.default_preferred_language } attribute :theme_id, default: -> { gitlab_config.default_theme } + attribute :color_scheme_id, default: -> { Gitlab::CurrentSettings.default_syntax_highlighting_theme } attr_encrypted :otp_secret, key: Gitlab::Application.secrets.otp_key_base, @@ -79,14 +80,14 @@ class User < ApplicationRecord algorithm: 'aes-256-cbc' devise :two_factor_authenticatable, - otp_secret_encryption_key: Gitlab::Application.secrets.otp_key_base + otp_secret_encryption_key: Gitlab::Application.secrets.otp_key_base devise :two_factor_backupable, otp_number_of_backup_codes: 10 devise :two_factor_backupable_pbkdf2 serialize :otp_backup_codes, JSON # rubocop:disable Cop/ActiveRecordSerialize devise :lockable, :recoverable, :rememberable, :trackable, - :validatable, :omniauthable, :confirmable, :registerable + :validatable, :omniauthable, :confirmable, :registerable # Must be included after `devise` include EncryptedUserPassword @@ -101,8 +102,6 @@ class User < ApplicationRecord MINIMUM_DAYS_CREATED = 7 - ignore_columns %i[linkedin twitter skype website_url location organization], remove_with: '15.10', remove_after: '2023-02-22' - # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour # rubocop: disable CodeReuse/ServiceClass @@ -133,11 +132,11 @@ class User < ApplicationRecord # Namespace for personal projects has_one :namespace, - -> { where(type: Namespaces::UserNamespace.sti_name) }, - dependent: :destroy, # rubocop:disable Cop/ActiveRecordDependent - foreign_key: :owner_id, - inverse_of: :owner, - autosave: true # rubocop:disable Cop/ActiveRecordDependent + -> { where(type: Namespaces::UserNamespace.sti_name) }, + dependent: :destroy, # rubocop:disable Cop/ActiveRecordDependent + foreign_key: :owner_id, + inverse_of: :owner, + autosave: true # rubocop:disable Cop/ActiveRecordDependent # Profile has_many :keys, -> { regular_keys }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -150,7 +149,6 @@ class User < ApplicationRecord has_many :emails has_many :personal_access_tokens, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :identities, dependent: :destroy, autosave: true # rubocop:disable Cop/ActiveRecordDependent - has_many :u2f_registrations, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :webauthn_registrations has_many :chat_names, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :saved_replies, class_name: '::Users::SavedReply' @@ -173,18 +171,18 @@ class User < ApplicationRecord has_many :maintainers_groups, -> { where(members: { access_level: Gitlab::Access::MAINTAINER }) }, through: :group_members, source: :group has_many :developer_groups, -> { where(members: { access_level: ::Gitlab::Access::DEVELOPER }) }, through: :group_members, source: :group has_many :owned_or_maintainers_groups, - -> { where(members: { access_level: [Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, - through: :group_members, - source: :group + -> { where(members: { access_level: [Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, + through: :group_members, + source: :group alias_attribute :masters_groups, :maintainers_groups has_many :developer_maintainer_owned_groups, - -> { where(members: { access_level: [Gitlab::Access::DEVELOPER, Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, - through: :group_members, - source: :group + -> { where(members: { access_level: [Gitlab::Access::DEVELOPER, Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, + through: :group_members, + source: :group has_many :reporter_developer_maintainer_owned_groups, - -> { where(members: { access_level: [Gitlab::Access::REPORTER, Gitlab::Access::DEVELOPER, Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, - through: :group_members, - source: :group + -> { where(members: { access_level: [Gitlab::Access::REPORTER, Gitlab::Access::DEVELOPER, Gitlab::Access::MAINTAINER, Gitlab::Access::OWNER] }) }, + through: :group_members, + source: :group has_many :minimal_access_group_members, -> { where(access_level: [Gitlab::Access::MINIMAL_ACCESS]) }, class_name: 'GroupMember' has_many :minimal_access_groups, through: :minimal_access_group_members, source: :group @@ -220,6 +218,7 @@ class User < ApplicationRecord has_many :abuse_reports, dependent: :destroy, foreign_key: :user_id # rubocop:disable Cop/ActiveRecordDependent has_many :reported_abuse_reports, dependent: :destroy, foreign_key: :reporter_id, class_name: "AbuseReport" # rubocop:disable Cop/ActiveRecordDependent has_many :spam_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent + has_many :abuse_trust_scores, class_name: 'Abuse::TrustScore', foreign_key: :user_id has_many :builds, class_name: 'Ci::Build' has_many :pipelines, class_name: 'Ci::Pipeline' has_many :todos, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -227,7 +226,9 @@ class User < ApplicationRecord has_many :notification_settings has_many :award_emoji, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :triggers, class_name: 'Ci::Trigger', foreign_key: :owner_id + has_many :audit_events, foreign_key: :author_id, inverse_of: :user + has_many :alert_assignees, class_name: '::AlertManagement::AlertAssignee', inverse_of: :assignee has_many :issue_assignees, inverse_of: :assignee has_many :merge_request_assignees, inverse_of: :assignee, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :merge_request_reviewers, inverse_of: :reviewer, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -264,6 +265,8 @@ class User < ApplicationRecord has_many :resource_label_events, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :resource_state_events, dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent + has_many :issue_assignment_events, class_name: 'ResourceEvents::IssueAssignmentEvent', dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent + has_many :merge_request_assignment_events, class_name: 'ResourceEvents::MergeRequestAssignmentEvent', dependent: :nullify # rubocop:disable Cop/ActiveRecordDependent has_many :authored_events, class_name: 'Event', dependent: :destroy, foreign_key: :author_id # rubocop:disable Cop/ActiveRecordDependent has_many :namespace_commit_emails, class_name: 'Users::NamespaceCommitEmail' has_many :user_achievements, class_name: 'Achievements::UserAchievement', inverse_of: :user @@ -289,7 +292,7 @@ class User < ApplicationRecord validate :check_password_weakness, if: :encrypted_password_changed? validates :namespace, presence: true - validate :namespace_move_dir_allowed, if: :username_changed? + validate :namespace_move_dir_allowed, if: :username_changed?, unless: :new_record? validate :unique_email, if: :email_changed? validate :notification_email_verified, if: :notification_email_changed? @@ -345,26 +348,28 @@ class User < ApplicationRecord # User's role enum role: { software_developer: 0, development_team_lead: 1, devops_engineer: 2, systems_administrator: 3, security_analyst: 4, data_analyst: 5, product_manager: 6, product_designer: 7, other: 8 }, _suffix: true - delegate :notes_filter_for, - :set_notes_filter, - :first_day_of_week, :first_day_of_week=, - :timezone, :timezone=, - :time_display_relative, :time_display_relative=, - :time_format_in_24h, :time_format_in_24h=, - :show_whitespace_in_diffs, :show_whitespace_in_diffs=, - :view_diffs_file_by_file, :view_diffs_file_by_file=, - :tab_width, :tab_width=, - :sourcegraph_enabled, :sourcegraph_enabled=, - :gitpod_enabled, :gitpod_enabled=, - :setup_for_company, :setup_for_company=, - :render_whitespace_in_code, :render_whitespace_in_code=, - :markdown_surround_selection, :markdown_surround_selection=, - :markdown_automatic_lists, :markdown_automatic_lists=, - :diffs_deletion_color, :diffs_deletion_color=, - :diffs_addition_color, :diffs_addition_color=, - :use_legacy_web_ide, :use_legacy_web_ide=, - :use_new_navigation, :use_new_navigation=, - to: :user_preference + delegate :notes_filter_for, + :set_notes_filter, + :first_day_of_week, :first_day_of_week=, + :timezone, :timezone=, + :time_display_relative, :time_display_relative=, + :show_whitespace_in_diffs, :show_whitespace_in_diffs=, + :view_diffs_file_by_file, :view_diffs_file_by_file=, + :pass_user_identities_to_ci_jwt, :pass_user_identities_to_ci_jwt=, + :tab_width, :tab_width=, + :sourcegraph_enabled, :sourcegraph_enabled=, + :gitpod_enabled, :gitpod_enabled=, + :setup_for_company, :setup_for_company=, + :render_whitespace_in_code, :render_whitespace_in_code=, + :markdown_surround_selection, :markdown_surround_selection=, + :markdown_automatic_lists, :markdown_automatic_lists=, + :diffs_deletion_color, :diffs_deletion_color=, + :diffs_addition_color, :diffs_addition_color=, + :use_new_navigation, :use_new_navigation=, + :pinned_nav_items, :pinned_nav_items=, + :achievements_enabled, :achievements_enabled=, + :enabled_following, :enabled_following=, + to: :user_preference delegate :path, to: :namespace, allow_nil: true, prefix: true delegate :job_title, :job_title=, to: :user_detail, allow_nil: true @@ -373,7 +378,6 @@ class User < ApplicationRecord delegate :pronouns, :pronouns=, to: :user_detail, allow_nil: true delegate :pronunciation, :pronunciation=, to: :user_detail, allow_nil: true delegate :registration_objective, :registration_objective=, to: :user_detail, allow_nil: true - delegate :requires_credit_card_verification, :requires_credit_card_verification=, to: :user_detail, allow_nil: true delegate :linkedin, :linkedin=, to: :user_detail, allow_nil: true delegate :twitter, :twitter=, to: :user_detail, allow_nil: true delegate :skype, :skype=, to: :user_detail, allow_nil: true @@ -513,28 +517,27 @@ class User < ApplicationRecord scope :with_dashboard, -> (dashboard) { where(dashboard: dashboard) } scope :with_public_profile, -> { where(private_profile: false) } scope :with_expiring_and_not_notified_personal_access_tokens, ->(at) do - where('EXISTS (?)', - ::PersonalAccessToken - .where('personal_access_tokens.user_id = users.id') - .without_impersonation - .expiring_and_not_notified(at).select(1)) + where('EXISTS (?)', ::PersonalAccessToken + .where('personal_access_tokens.user_id = users.id') + .without_impersonation + .expiring_and_not_notified(at).select(1) + ) end scope :with_personal_access_tokens_expired_today, -> do - where('EXISTS (?)', - ::PersonalAccessToken - .select(1) - .where('personal_access_tokens.user_id = users.id') - .without_impersonation - .expired_today_and_not_notified) + where('EXISTS (?)', ::PersonalAccessToken + .select(1) + .where('personal_access_tokens.user_id = users.id') + .without_impersonation + .expired_today_and_not_notified + ) end scope :with_ssh_key_expiring_soon, -> do includes(:expiring_soon_and_unnotified_keys) - .where('EXISTS (?)', - ::Key - .select(1) - .where('keys.user_id = users.id') - .expiring_soon_and_not_notified) + .where('EXISTS (?)', ::Key + .select(1) + .where('keys.user_id = users.id') + .expiring_soon_and_not_notified) end scope :order_recent_sign_in, -> { reorder(arel_table[:current_sign_in_at].desc.nulls_last) } scope :order_oldest_sign_in, -> { reorder(arel_table[:current_sign_in_at].asc.nulls_last) } @@ -614,13 +617,12 @@ class User < ApplicationRecord def self.with_two_factor where(otp_required_for_login: true) - .or(where_exists(U2fRegistration.where(U2fRegistration.arel_table[:user_id].eq(arel_table[:id])))) .or(where_exists(WebauthnRegistration.where(WebauthnRegistration.arel_table[:user_id].eq(arel_table[:id])))) end def self.without_two_factor where - .missing(:u2f_registrations, :webauthn_registrations) + .missing(:webauthn_registrations) .where(otp_required_for_login: false) end @@ -922,6 +924,17 @@ class User < ApplicationRecord end end + def llm_bot + email_pattern = "llm-bot%s@#{Settings.gitlab.host}" + + unique_internal(where(user_type: :llm_bot), 'GitLab-Llm-Bot', email_pattern) do |u| + u.bio = 'The Gitlab LLM bot used for fetching LLM-generated content' + u.name = 'GitLab LLM Bot' + u.avatar = bot_avatar(image: 'support-bot.png') # todo: add an avatar for llm-bot + u.confirmed_at = Time.zone.now + end + end + def admin_bot email_pattern = "admin-bot%s@#{Settings.gitlab.host}" @@ -1025,17 +1038,32 @@ class User < ApplicationRecord password_allowed end + # Override Devise Rememberable#remember_me! + # + # In Devise this method sets `remember_created_at` and writes the session token + # to the session cookie. When remember me is disabled this method ensures these + # values aren't set. def remember_me! - super if ::Gitlab::Database.read_write? + super if ::Gitlab::Database.read_write? && ::Gitlab::CurrentSettings.remember_me_enabled? end def forget_me! super if ::Gitlab::Database.read_write? end + # Override Devise Rememberable#remember_me? + # + # In Devise this method compares the remember me token received from the user session + # and compares to the stored value. When remember me is disabled this method ensures + # the upstream comparison does not happen. + def remember_me?(token, generated_at) + return false unless ::Gitlab::CurrentSettings.remember_me_enabled? + + super + end + def disable_two_factor! transaction do - self.u2f_registrations.destroy_all # rubocop:disable Cop/DestroyAll self.disable_webauthn! self.disable_two_factor_otp! self.reset_backup_codes! @@ -1062,32 +1090,17 @@ class User < ApplicationRecord end def two_factor_enabled? - two_factor_otp_enabled? || two_factor_webauthn_u2f_enabled? + two_factor_otp_enabled? || two_factor_webauthn_enabled? end def two_factor_otp_enabled? otp_required_for_login? || forti_authenticator_enabled?(self) || - forti_token_cloud_enabled?(self) - end - - def two_factor_u2f_enabled? - return false if Feature.enabled?(:webauthn) - - if u2f_registrations.loaded? - u2f_registrations.any? - else - u2f_registrations.exists? - end - end - - def two_factor_webauthn_u2f_enabled? - two_factor_u2f_enabled? || two_factor_webauthn_enabled? + forti_token_cloud_enabled?(self) || + duo_auth_enabled?(self) end def two_factor_webauthn_enabled? - return false unless Feature.enabled?(:webauthn) - (webauthn_registrations.loaded? && webauthn_registrations.any?) || (!webauthn_registrations.loaded? && webauthn_registrations.exists?) end @@ -1646,9 +1659,19 @@ class User < ApplicationRecord end # rubocop: enable CodeReuse/ServiceClass + DELETION_DELAY_IN_DAYS = 7.days + def delete_async(deleted_by:, params: {}) - block if params[:hard_delete] - DeleteUserWorker.perform_async(deleted_by.id, id, params.to_h) + is_deleting_own_record = deleted_by.id == id + + if is_deleting_own_record && ::Feature.enabled?(:delay_delete_own_user) + block + DeleteUserWorker.perform_in(DELETION_DELAY_IN_DAYS, deleted_by.id, id, params.to_h) + else + block if params[:hard_delete] + + DeleteUserWorker.perform_async(deleted_by.id, id, params.to_h) + end end # rubocop: disable CodeReuse/ServiceClass @@ -1693,7 +1716,7 @@ class User < ApplicationRecord end def follow(user) - return false if self.id == user.id + return false unless following_users_allowed?(user) begin followee = Users::UserFollowUser.create(follower_id: self.id, followee_id: user.id) @@ -1712,24 +1735,33 @@ class User < ApplicationRecord end end + def following_users_allowed?(user) + return false if self.id == user.id + + following_users_enabled? && user.following_users_enabled? + end + + def following_users_enabled? + return true unless ::Feature.enabled?(:disable_follow_users, self) + + enabled_following + end + def forkable_namespaces strong_memoize(:forkable_namespaces) do personal_namespace = Namespace.where(id: namespace_id) + groups_allowing_project_creation = Groups::AcceptingProjectCreationsFinder.new(self).execute Namespace.from_union( [ - manageable_groups(include_groups_with_developer_maintainer_access: true), + groups_allowing_project_creation, personal_namespace ]) end end def manageable_groups(include_groups_with_developer_maintainer_access: false) - owned_and_maintainer_group_hierarchy = if Feature.enabled?(:linear_user_manageable_groups, self) - owned_or_maintainers_groups.self_and_descendants - else - Gitlab::ObjectHierarchy.new(owned_or_maintainers_groups).base_and_descendants - end + owned_and_maintainer_group_hierarchy = owned_or_maintainers_groups.self_and_descendants if include_groups_with_developer_maintainer_access union_sql = ::Gitlab::SQL::Union.new( @@ -1988,7 +2020,7 @@ class User < ApplicationRecord end def enabled_incoming_email_token - incoming_email_token if Gitlab::IncomingEmail.supports_issue_creation? + incoming_email_token if Gitlab::Email::IncomingEmail.supports_issue_creation? end def sync_attribute?(attribute) @@ -2017,9 +2049,11 @@ class User < ApplicationRecord # # Returns a Hash mapping project ID -> maximum access level. def max_member_access_for_project_ids(project_ids) - Gitlab::SafeRequestLoader.execute(resource_key: max_member_access_for_resource_key(Project), - resource_ids: project_ids, - default_value: Gitlab::Access::NO_ACCESS) do |project_ids| + Gitlab::SafeRequestLoader.execute( + resource_key: max_member_access_for_resource_key(Project), + resource_ids: project_ids, + default_value: Gitlab::Access::NO_ACCESS + ) do |project_ids| project_authorizations.where(project: project_ids) .group(:project_id) .maximum(:access_level) @@ -2034,9 +2068,11 @@ class User < ApplicationRecord # # Returns a Hash mapping project ID -> maximum access level. def max_member_access_for_group_ids(group_ids) - Gitlab::SafeRequestLoader.execute(resource_key: max_member_access_for_resource_key(Group), - resource_ids: group_ids, - default_value: Gitlab::Access::NO_ACCESS) do |group_ids| + Gitlab::SafeRequestLoader.execute( + resource_key: max_member_access_for_resource_key(Group), + resource_ids: group_ids, + default_value: Gitlab::Access::NO_ACCESS + ) do |group_ids| group_members.where(source: group_ids).group(:source_id).maximum(:access_level) end end @@ -2136,7 +2172,15 @@ class User < ApplicationRecord end def confirmation_required_on_sign_in? - !confirmed? && !confirmation_period_valid? + return false if confirmed? + + if ::Gitlab::CurrentSettings.email_confirmation_setting_off? + false + elsif ::Gitlab::CurrentSettings.email_confirmation_setting_soft? + !in_confirmation_period? + elsif ::Gitlab::CurrentSettings.email_confirmation_setting_hard? + true + end end def impersonated? @@ -2206,21 +2250,39 @@ class User < ApplicationRecord namespace_commit_emails.find_by(namespace: project.root_namespace) end + def spam_score + abuse_trust_scores.spamcheck.average(:score) || 0.0 + end + + def trust_scores_for_source(source) + abuse_trust_scores.where(source: source) + end + + def abuse_metadata + { + account_age: account_age_in_days, + two_factor_enabled: two_factor_enabled? ? 1 : 0 + } + end + protected # override, from Devise::Validatable def password_required? - return false if internal? || project_bot? + return false if internal? || project_bot? || security_policy_bot? super end # override from Devise::Confirmable def confirmation_period_valid? - return false if Feature.disabled?(:soft_email_confirmation) + return super if ::Gitlab::CurrentSettings.email_confirmation_setting_soft? - super + # Following devise logic for method, we want to return `true` + # See: https://github.com/heartcombo/devise/blob/main/lib/devise/models/confirmable.rb#L191-L218 + true end + alias_method :in_confirmation_period?, :confirmation_period_valid? # This is copied from Devise::Models::TwoFactorAuthenticatable#consume_otp! # diff --git a/app/models/user_custom_attribute.rb b/app/models/user_custom_attribute.rb index 4ebb8ba9f00..9a186cb9038 100644 --- a/app/models/user_custom_attribute.rb +++ b/app/models/user_custom_attribute.rb @@ -13,6 +13,8 @@ class UserCustomAttribute < ApplicationRecord BLOCKED_BY = 'blocked_by' UNBLOCKED_BY = 'unblocked_by' + ARKOSE_RISK_BAND = 'arkose_risk_band' + AUTO_BANNED_BY_ABUSE_REPORT_ID = 'auto_banned_by_abuse_report_id' class << self def upsert_custom_attributes(custom_attributes) diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index 9d3df3d6400..293a20fcc5a 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -1,8 +1,11 @@ # frozen_string_literal: true class UserDetail < ApplicationRecord + include IgnorableColumns extend ::Gitlab::Utils::Override + ignore_column :requires_credit_card_verification, remove_with: '16.1', remove_after: '2023-06-22' + REGISTRATION_OBJECTIVE_PAIRS = { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5, joining_team: 6 }.freeze belongs_to :user diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index bc2c6b526b8..90449411f8a 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -20,17 +20,24 @@ class UserPreference < ApplicationRecord less_than_or_equal_to: Gitlab::TabWidth::MAX } validates :diffs_deletion_color, :diffs_addition_color, - format: { with: ColorsHelper::HEX_COLOR_PATTERN }, - allow_blank: true - validates :use_legacy_web_ide, allow_nil: false, inclusion: { in: [true, false] } + format: { with: ColorsHelper::HEX_COLOR_PATTERN }, + allow_blank: true + + validates :pass_user_identities_to_ci_jwt, allow_nil: false, inclusion: { in: [true, false] } + + validates :pinned_nav_items, json_schema: { filename: 'pinned_nav_items' } ignore_columns :experience_level, remove_with: '14.10', remove_after: '2021-03-22' + ignore_columns :time_format_in_24h, remove_with: '16.2', remove_after: '2023-07-22' + # 2023-06-22 is after 16.1 release and during 16.2 release https://docs.gitlab.com/ee/development/database/avoiding_downtime_in_migrations.html#ignoring-the-column-release-m + ignore_columns :use_legacy_web_ide, remove_with: '16.2', remove_after: '2023-06-22' attribute :tab_width, default: -> { Gitlab::TabWidth::DEFAULT } attribute :time_display_relative, default: true - attribute :time_format_in_24h, default: false attribute :render_whitespace_in_code, default: false + enum visibility_pipeline_id_type: { id: 0, iid: 1 } + class << self def notes_filters { @@ -88,22 +95,6 @@ class UserPreference < ApplicationRecord end end - def time_format_in_24h - value = read_attribute(:time_format_in_24h) - return value unless value.nil? - - self.class.column_defaults['time_format_in_24h'] - end - - def time_format_in_24h=(value) - if value.nil? - default = self.class.column_defaults['time_format_in_24h'] - super(default) - else - super(value) - end - end - def render_whitespace_in_code value = read_attribute(:render_whitespace_in_code) return value unless value.nil? diff --git a/app/models/user_status.rb b/app/models/user_status.rb index 0c66f465356..da24ef47a2a 100644 --- a/app/models/user_status.rb +++ b/app/models/user_status.rb @@ -17,7 +17,7 @@ class UserStatus < ApplicationRecord '30_days' => 30.days }.freeze - belongs_to :user + belongs_to :user, inverse_of: :status enum availability: { not_set: 0, busy: 1 } diff --git a/app/models/user_synced_attributes_metadata.rb b/app/models/user_synced_attributes_metadata.rb index 4cd0e3fb828..6b23bce6406 100644 --- a/app/models/user_synced_attributes_metadata.rb +++ b/app/models/user_synced_attributes_metadata.rb @@ -14,7 +14,7 @@ class UserSyncedAttributesMetadata < ApplicationRecord def read_only_attributes return [] unless sync_profile_from_provider? - self.class.syncable_attributes.select { |key| synced?(key) } + SYNCABLE_ATTRIBUTES.select { |key| synced?(key) } end def synced?(attribute) @@ -26,12 +26,11 @@ class UserSyncedAttributesMetadata < ApplicationRecord end class << self - def syncable_attributes - if Gitlab.config.ldap.enabled && !Gitlab.config.ldap.sync_name - SYNCABLE_ATTRIBUTES - %i[name] - else - SYNCABLE_ATTRIBUTES - end + def syncable_attributes(provider = nil) + return SYNCABLE_ATTRIBUTES unless provider && ldap_provider?(provider) + return SYNCABLE_ATTRIBUTES if ldap_sync_name?(provider) + + SYNCABLE_ATTRIBUTES - %i[name] end end @@ -40,4 +39,17 @@ class UserSyncedAttributesMetadata < ApplicationRecord def sync_profile_from_provider? Gitlab::Auth::OAuth::Provider.sync_profile_from_provider?(provider) end + + class << self + def ldap_provider?(provider) + Gitlab::Auth::OAuth::Provider.ldap_provider?(provider) + end + + def ldap_sync_name?(provider) + return false unless provider + + config = Gitlab::Auth::Ldap::Config.new(provider) + config.enabled? && config.sync_name + end + end end diff --git a/app/models/users/banned_user.rb b/app/models/users/banned_user.rb index 615668e2b55..8a62744c7d6 100644 --- a/app/models/users/banned_user.rb +++ b/app/models/users/banned_user.rb @@ -5,8 +5,12 @@ module Users self.primary_key = :user_id belongs_to :user + has_one :credit_card_validation, class_name: '::Users::CreditCardValidation', primary_key: 'user_id', + foreign_key: 'user_id', inverse_of: :banned_user validates :user, presence: true validates :user_id, uniqueness: { message: N_("banned user already exists") } end end + +Users::BannedUser.prepend_mod_with('Users::BannedUser') diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 3f9353214ee..896cccfa0e5 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -43,10 +43,9 @@ module Users verification_reminder: 40, # EE-only ci_deprecation_warning_for_types_keyword: 41, security_training_feature_promotion: 42, # EE-only - storage_enforcement_banner_first_enforcement_threshold: 43, # EE-only - storage_enforcement_banner_second_enforcement_threshold: 44, # EE-only - storage_enforcement_banner_third_enforcement_threshold: 45, # EE-only - storage_enforcement_banner_fourth_enforcement_threshold: 46, # EE-only + namespace_storage_pre_enforcement_banner: 43, # EE-only + # 44, 45, 46 were unused and removed with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118330, + # they can be replaced. # 47 and 48 were removed with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95446 # 49 was removed with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91533 # because the banner was no longer relevant. @@ -60,12 +59,13 @@ module Users namespace_storage_limit_banner_warning_threshold: 56, # EE-only namespace_storage_limit_banner_alert_threshold: 57, # EE-only namespace_storage_limit_banner_error_threshold: 58, # EE-only - project_quality_summary_feedback: 59, # EE-only + project_quality_summary_feedback: 59, # EE-only merge_request_settings_moved_callout: 60, new_top_level_group_alert: 61, artifacts_management_page_feedback_banner: 62, - vscode_web_ide: 63, - vscode_web_ide_callout: 64 + # 63 and 64 were removed with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120233 + branch_rules_info_callout: 65, + create_runner_workflow_banner: 66 } validates :feature_name, diff --git a/app/models/users/credit_card_validation.rb b/app/models/users/credit_card_validation.rb index 272f31aa9ce..1b0fd8682db 100644 --- a/app/models/users/credit_card_validation.rb +++ b/app/models/users/credit_card_validation.rb @@ -7,6 +7,8 @@ module Users self.table_name = 'user_credit_card_validations' belongs_to :user + belongs_to :banned_user, class_name: '::Users::BannedUser', foreign_key: :user_id, + inverse_of: :credit_card_validation validates :holder_name, length: { maximum: 50 } validates :network, length: { maximum: 32 } @@ -14,18 +16,32 @@ module Users greater_than_or_equal_to: 0, less_than_or_equal_to: 9999 } + scope :by_banned_user, -> { joins(:banned_user) } + scope :similar_by_holder_name, ->(holder_name) do + if holder_name.present? + where('lower(holder_name) = lower(:value)', value: holder_name) + else + none + end + end + scope :similar_to, ->(credit_card_validation) do + where( + expiration_date: credit_card_validation.expiration_date, + last_digits: credit_card_validation.last_digits, + network: credit_card_validation.network + ) + end + def similar_records - self.class.where( - expiration_date: expiration_date, - last_digits: last_digits, - network: network - ).order(credit_card_validated_at: :desc).includes(:user) + self.class.similar_to(self).order(credit_card_validated_at: :desc).includes(:user) end def similar_holder_names_count - return 0 unless holder_name + self.class.similar_by_holder_name(holder_name).count + end - self.class.where('lower(holder_name) = lower(:value)', value: holder_name).count + def used_by_banned_user? + self.class.by_banned_user.similar_to(self).similar_by_holder_name(holder_name).exists? end end end diff --git a/app/models/users/group_callout.rb b/app/models/users/group_callout.rb index 2552407fa4c..1cc9f1f50ad 100644 --- a/app/models/users/group_callout.rb +++ b/app/models/users/group_callout.rb @@ -11,10 +11,9 @@ module Users enum feature_name: { invite_members_banner: 1, approaching_seat_count_threshold: 2, # EE-only - storage_enforcement_banner_first_enforcement_threshold: 3, # EE-only - storage_enforcement_banner_second_enforcement_threshold: 4, # EE-only - storage_enforcement_banner_third_enforcement_threshold: 5, # EE-only - storage_enforcement_banner_fourth_enforcement_threshold: 6, # EE-only + namespace_storage_pre_enforcement_banner: 3, # EE-only + # 4,5,6 were unused and removed with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118330, + # they can be replaced. preview_user_over_limit_free_plan_alert: 7, # EE-only user_reached_limit_free_plan_alert: 8, # EE-only free_group_limited_alert: 9, # EE-only @@ -24,14 +23,16 @@ module Users namespace_storage_limit_banner_error_threshold: 13, # EE-only usage_quota_trial_alert: 14, # EE-only preview_usage_quota_free_plan_alert: 15, # EE-only - enforcement_at_limit_alert: 16 # EE-only + enforcement_at_limit_alert: 16, # EE-only + web_hook_disabled: 17, # EE-only + unlimited_members_during_trial_alert: 18 # EE-only } validates :group, presence: true validates :feature_name, - presence: true, - uniqueness: { scope: [:user_id, :group_id] }, - inclusion: { in: GroupCallout.feature_names.keys } + presence: true, + uniqueness: { scope: [:user_id, :group_id] }, + inclusion: { in: GroupCallout.feature_names.keys } def source_feature_name "#{feature_name}_#{group_id}" diff --git a/app/models/users/phone_number_validation.rb b/app/models/users/phone_number_validation.rb index b9e4e908ddd..52f16a7861f 100644 --- a/app/models/users/phone_number_validation.rb +++ b/app/models/users/phone_number_validation.rb @@ -8,28 +8,25 @@ module Users belongs_to :user, foreign_key: :user_id belongs_to :banned_user, class_name: '::Users::BannedUser', foreign_key: :user_id - validates :country, - presence: true, - length: { maximum: 3 } + validates :country, presence: true, length: { maximum: 3 } validates :international_dial_code, - presence: true, - numericality: { - only_integer: true, - greater_than_or_equal_to: 1, - less_than_or_equal_to: 999 - } + presence: true, + numericality: { + only_integer: true, + greater_than_or_equal_to: 1, + less_than_or_equal_to: 999 + } validates :phone_number, - presence: true, - format: { - with: /\A\d+\Z/, - message: -> (object, data) { _('can contain only digits') } - }, - length: { maximum: 12 } - - validates :telesign_reference_xid, - length: { maximum: 255 } + presence: true, + format: { + with: /\A\d+\Z/, + message: -> (object, data) { _('can contain only digits') } + }, + length: { maximum: 12 } + + validates :telesign_reference_xid, length: { maximum: 255 } scope :for_user, -> (user_id) { where(user_id: user_id) } diff --git a/app/models/users/project_callout.rb b/app/models/users/project_callout.rb index c73b3a4ee71..3964f202be6 100644 --- a/app/models/users/project_callout.rb +++ b/app/models/users/project_callout.rb @@ -12,16 +12,16 @@ module Users awaiting_members_banner: 1, # EE-only web_hook_disabled: 2, ultimate_feature_removal_banner: 3, - storage_enforcement_banner_first_enforcement_threshold: 4, # EE-only - storage_enforcement_banner_second_enforcement_threshold: 5, # EE-only - storage_enforcement_banner_third_enforcement_threshold: 6, # EE-only - storage_enforcement_banner_fourth_enforcement_threshold: 7 # EE-only + namespace_storage_pre_enforcement_banner: 4, # EE-only + # 5,6,7 were unused and removed with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118330, + # they can be replaced. + license_check_deprecation_alert: 8 # EE-only } validates :project, presence: true validates :feature_name, - presence: true, - uniqueness: { scope: [:user_id, :project_id] }, - inclusion: { in: ProjectCallout.feature_names.keys } + presence: true, + uniqueness: { scope: [:user_id, :project_id] }, + inclusion: { in: ProjectCallout.feature_names.keys } end end diff --git a/app/models/users/user_follow_user.rb b/app/models/users/user_follow_user.rb index 5a82a81364a..c9d4bee496c 100644 --- a/app/models/users/user_follow_user.rb +++ b/app/models/users/user_follow_user.rb @@ -14,9 +14,13 @@ module Users followee_count = self.class.where(follower_id: follower_id).limit(MAX_FOLLOWEE_LIMIT).count return if followee_count < MAX_FOLLOWEE_LIMIT - errors.add(:base, format( - _("You can't follow more than %{limit} users. To follow more users, unfollow some others."), - limit: MAX_FOLLOWEE_LIMIT)) + errors.add( + :base, + format( + _("You can't follow more than %{limit} users. To follow more users, unfollow some others."), + limit: MAX_FOLLOWEE_LIMIT + ) + ) end end end diff --git a/app/models/vulnerability.rb b/app/models/vulnerability.rb index 8bb598ee316..700e4e0e0ec 100644 --- a/app/models/vulnerability.rb +++ b/app/models/vulnerability.rb @@ -7,6 +7,14 @@ class Vulnerability < ApplicationRecord alias_attribute :vulnerability_id, :id + scope :with_projects, -> { includes(:project) } + + # Policy class inferring logic is causing performance + # issues therefore we need to explicitly set it. + def self.declarative_policy_class + :VulnerabilityPolicy + end + def self.link_reference_pattern nil end diff --git a/app/models/web_ide_terminal.rb b/app/models/web_ide_terminal.rb index ef70df2405f..fe69ca80c32 100644 --- a/app/models/web_ide_terminal.rb +++ b/app/models/web_ide_terminal.rb @@ -39,12 +39,14 @@ class WebIdeTerminal private def web_ide_terminal_route_generator(action, options = {}) - options.reverse_merge!(action: action, - controller: 'projects/web_ide_terminals', - namespace_id: project.namespace.to_param, - project_id: project.to_param, - id: build.id, - only_path: true) + options.reverse_merge!( + action: action, + controller: 'projects/web_ide_terminals', + namespace_id: project.namespace.to_param, + project_id: project.to_param, + id: build.id, + only_path: true + ) url_for(options) end diff --git a/app/models/webauthn_registration.rb b/app/models/webauthn_registration.rb index 71b50192e29..c8b2513e702 100644 --- a/app/models/webauthn_registration.rb +++ b/app/models/webauthn_registration.rb @@ -3,10 +3,14 @@ # Registration information for WebAuthn credentials class WebauthnRegistration < ApplicationRecord + include IgnorableColumns + + ignore_column :u2f_registration_id, remove_with: '16.2', remove_after: '2023-06-22' + belongs_to :user validates :credential_xid, :public_key, :counter, presence: true validates :name, length: { minimum: 0, allow_nil: false } validates :counter, - numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 2**32 - 1 } + numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 2**32 - 1 } end diff --git a/app/models/wiki.rb b/app/models/wiki.rb index 57488749b76..39d22ea0e07 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -6,11 +6,10 @@ class Wiki include Repositories::CanHousekeepRepository include Gitlab::Utils::StrongMemoize include GlobalID::Identification + include Gitlab::Git::WrapsGitalyErrors extend ActiveModel::Naming - DuplicatePageError = Class.new(StandardError) - MARKUPS = { # rubocop:disable Style/MultilineIfModifier markdown: { name: 'Markdown', @@ -187,6 +186,8 @@ class Wiki def has_home_page? !!find_page(HOMEPAGE) + rescue StandardError + false end def empty? @@ -287,9 +288,7 @@ class Wiki def create_page(title, content, format = :markdown, message = nil) with_valid_format(format) do |default_extension| - if file_exists_by_regex?(title) - raise_duplicate_page_error! - end + next duplicated_page_error if file_exists_by_regex?(title) capture_git_error(:created) do create_wiki_repository unless repository_exists? @@ -300,13 +299,9 @@ class Wiki true rescue Gitlab::Git::Index::IndexError - raise_duplicate_page_error! + duplicated_page_error end end - rescue DuplicatePageError => e - @error_message = _("Duplicate page: %{error_message}" % { error_message: e.message }) - - false end def update_page(page, content:, title: nil, format: :markdown, message: nil) @@ -326,10 +321,17 @@ class Wiki content, previous_path: page.path, **multi_commit_options(:updated, message, title)) + repository.move_dir_files( + user, + sluggified_title(title), + page.url_path, + **multi_commit_options(:moved, message, title)) after_wiki_activity true + rescue Gitlab::Git::Index::IndexError + duplicated_page_error end end end @@ -398,13 +400,11 @@ class Wiki # Callbacks for synchronous processing after wiki changes. # These will be executed after any change made through GitLab itself (web UI and API), # but not for Git pushes. - def after_wiki_activity - end + def after_wiki_activity; end # Callbacks for background processing after wiki changes. # These will be executed after any change to the wiki repository. - def after_post_receive - end + def after_post_receive; end override :git_garbage_collect_worker_klass def git_garbage_collect_worker_klass @@ -416,12 +416,14 @@ class Wiki end def capture_git_error(action, &block) - yield block + wrapped_gitaly_errors(&block) rescue Gitlab::Git::Index::IndexError, - Gitlab::Git::CommitError, - Gitlab::Git::PreReceiveError, - Gitlab::Git::CommandError, - ArgumentError => e + Gitlab::Git::CommitError, + Gitlab::Git::PreReceiveError, + Gitlab::Git::CommandError, + ArgumentError => e + + @error_message = e.message Gitlab::ErrorTracking.log_exception(e, action: action, wiki_id: id) @@ -471,8 +473,9 @@ class Wiki repository.ls_files(default_branch).any? { |s| s =~ regex } end - def raise_duplicate_page_error! - raise ::Wiki::DuplicatePageError, _('A page with that title already exists') + def duplicated_page_error + @error_message = _("Duplicate page: A page with that title already exists") + false end def sluggified_full_path(title, extension) @@ -491,7 +494,9 @@ class Wiki escaped_path = RE2::Regexp.escape(sluggified_title(title)) path_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)^#{escaped_path}\\.(#{file_extension_regexp})$") - matched_files = repository.search_files_by_regexp(path_regexp, version, limit: 1) + matched_files = capture_git_error(:find) do + repository.search_files_by_regexp(path_regexp, version, limit: 1) + end return if matched_files.blank? Gitlab::EncodingHelper.encode_utf8_no_detect(matched_files.first) diff --git a/app/models/wiki_directory.rb b/app/models/wiki_directory.rb index 76fe664f23d..e57d186a3e3 100644 --- a/app/models/wiki_directory.rb +++ b/app/models/wiki_directory.rb @@ -7,34 +7,48 @@ class WikiDirectory validates :slug, presence: true alias_method :to_param, :slug - # Groups a list of wiki pages into a nested collection of WikiPage and WikiDirectory objects, - # preserving the order of the passed pages. - # - # Returns an array with all entries for the toplevel directory. - # - # @param [Array<WikiPage>] pages - # @return [Array<WikiPage, WikiDirectory>] - # - def self.group_pages(pages) - # Build a hash to map paths to created WikiDirectory objects, - # and recursively create them for each level of the path. - # For the toplevel directory we use '' as path, as that's what WikiPage#directory returns. - directories = Hash.new do |_, path| - directories[path] = new(path).tap do |directory| - if path.present? - parent = File.dirname(path) - parent = '' if parent == '.' - directories[parent].entries << directory - directories[parent].entries.delete_if { |item| item.is_a?(WikiPage) && item.slug == directory.slug } + class << self + # Groups a list of wiki pages into a nested collection of WikiPage and WikiDirectory objects, + # preserving the order of the passed pages. + # + # Returns an array with all entries for the toplevel directory. + # + # @param [Array<WikiPage>] pages + # @return [Array<WikiPage, WikiDirectory>] + # + def group_pages(pages) + # Build a hash to map paths to created WikiDirectory objects, + # and recursively create them for each level of the path. + # For the toplevel directory we use '' as path, as that's what WikiPage#directory returns. + directories = Hash.new do |_, path| + directories[path] = new(path).tap do |directory| + if path.present? + parent = File.dirname(path) + parent = '' if parent == '.' + directories[parent].entries << directory + directories[parent].entries.delete_if do |item| + item.is_a?(WikiPage) && item.slug.casecmp?(directory.slug) + end + end end end - end - pages.each do |page| - directories[page.directory].entries << page + pages.each do |page| + next unless directory_for_page?(directories[page.directory], page) + + directories[page.directory].entries << page + end + + directories[''].entries end - directories[''].entries + private + + def directory_for_page?(directory, page) + directory.entries.none? do |item| + item.is_a?(WikiDirectory) && item.slug.casecmp?(page.slug) + end + end end def initialize(slug, entries = []) diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index b04aa196883..e1468872f52 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -145,10 +145,12 @@ class WikiPage default_per_page = Kaminari.config.default_per_page offset = [options[:page].to_i - 1, 0].max * options.fetch(:per_page, default_per_page) - wiki.repository.commits(wiki.default_branch, - path: page.path, - limit: options.fetch(:limit, default_per_page), - offset: offset) + wiki.repository.commits( + wiki.default_branch, + path: page.path, + limit: options.fetch(:limit, default_per_page), + offset: offset + ) end def count_versions diff --git a/app/models/work_item.rb b/app/models/work_item.rb index 5ae3fb6cf78..24d1078516e 100644 --- a/app/models/work_item.rb +++ b/app/models/work_item.rb @@ -4,7 +4,7 @@ class WorkItem < Issue include Gitlab::Utils::StrongMemoize COMMON_QUICK_ACTIONS_COMMANDS = [ - :title, :reopen, :close, :cc, :tableflip, :shrug + :title, :reopen, :close, :cc, :tableflip, :shrug, :type ].freeze self.table_name = 'issues' @@ -16,15 +16,13 @@ class WorkItem < Issue has_many :child_links, class_name: '::WorkItems::ParentLink', foreign_key: :work_item_parent_id has_many :work_item_children, through: :child_links, class_name: 'WorkItem', - foreign_key: :work_item_id, source: :work_item + foreign_key: :work_item_id, source: :work_item has_many :work_item_children_by_relative_position, -> { work_item_children_keyset_order }, - through: :child_links, class_name: 'WorkItem', - foreign_key: :work_item_id, source: :work_item + through: :child_links, class_name: 'WorkItem', + foreign_key: :work_item_id, source: :work_item scope :inc_relations_for_permission_check, -> { includes(:author, project: :project_feature) } - delegate :supports_assignee?, to: :work_item_type - class << self def assignee_association_name 'issue' @@ -34,6 +32,14 @@ class WorkItem < Issue 'issues.id' end + # def reference_pattern + # # no-op: We currently only support link_reference_pattern parsing + # end + + def link_reference_pattern + @link_reference_pattern ||= compose_link_reference_pattern('work_items', Gitlab::Regex.work_item) + end + def work_item_children_keyset_order keyset_order = Gitlab::Pagination::Keyset::Order.build([ Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( @@ -51,7 +57,7 @@ class WorkItem < Issue ) ]) - includes(:child_links).order(keyset_order) + includes(:parent_link).order(keyset_order) end end @@ -67,6 +73,16 @@ class WorkItem < Issue end end + # Returns widget object if available + # type parameter can be a symbol, for example, `:description`. + def get_widget(type) + widgets.find do |widget| + widget.instance_of?(WorkItems::Widgets.const_get(type.to_s.camelize, false)) + end + rescue NameError + nil + end + def ancestors hierarchy.ancestors(hierarchy_order: :asc) end @@ -85,6 +101,26 @@ class WorkItem < Issue COMMON_QUICK_ACTIONS_COMMANDS + commands_for_widgets end + # Widgets have a set of quick action params that they must process. + # Map them to widget_params so they can be picked up by widget services. + def transform_quick_action_params(command_params) + common_params = command_params.dup + widget_params = {} + + work_item_type.widgets + .filter { |widget| widget.respond_to?(:quick_action_params) } + .each do |widget| + widget.quick_action_params + .filter { |param_name| common_params.key?(param_name) } + .each do |param_name| + widget_params[widget.api_symbol] ||= {} + widget_params[widget.api_symbol][param_name] = common_params.delete(param_name) + end + end + + { common: common_params, widgets: widget_params } + end + private override :parent_link_confidentiality @@ -110,6 +146,75 @@ class WorkItem < Issue ::Gitlab::WorkItems::WorkItemHierarchy.new(base, options: options) end + + override :allowed_work_item_type_change + def allowed_work_item_type_change + return unless work_item_type_id_changed? + + child_links = WorkItems::ParentLink.for_parents(id) + parent_link = ::WorkItems::ParentLink.find_by(work_item: self) + + validate_parent_restrictions(parent_link) + validate_child_restrictions(child_links) + validate_depth(parent_link, child_links) + end + + def validate_parent_restrictions(parent_link) + return unless parent_link + + parent_link.work_item.work_item_type_id = work_item_type_id + + unless parent_link.valid? + errors.add( + :work_item_type_id, + format( + _('cannot be changed to %{new_type} with %{parent_type} as parent type.'), + new_type: work_item_type.name, parent_type: parent_link.work_item_parent.work_item_type.name + ) + ) + end + end + + def validate_child_restrictions(child_links) + return if child_links.empty? + + child_type_ids = child_links.joins(:work_item).select(self.class.arel_table[:work_item_type_id]).distinct + restrictions = ::WorkItems::HierarchyRestriction.where( + parent_type_id: work_item_type_id, + child_type_id: child_type_ids + ) + + # We expect a restriction for every child type + if restrictions.size < child_type_ids.size + errors.add( + :work_item_type_id, + format(_('cannot be changed to %{new_type} with these child item types.'), new_type: work_item_type.name) + ) + end + end + + def validate_depth(parent_link, child_links) + restriction = ::WorkItems::HierarchyRestriction.find_by_parent_type_id_and_child_type_id( + work_item_type_id, + work_item_type_id + ) + return unless restriction&.maximum_depth + + children_with_new_type = self.class.where(id: child_links.select(:work_item_id)) + .where(work_item_type_id: work_item_type_id) + max_child_depth = ::Gitlab::WorkItems::WorkItemHierarchy.new(children_with_new_type).max_descendants_depth.to_i + + ancestor_depth = + if parent_link&.work_item_parent && parent_link.work_item_parent.work_item_type_id == work_item_type_id + parent_link.work_item_parent.same_type_base_and_ancestors.count + else + 0 + end + + if max_child_depth + ancestor_depth > restriction.maximum_depth - 1 + errors.add(:work_item_type_id, _('reached maximum depth')) + end + end end WorkItem.prepend_mod diff --git a/app/models/work_items/parent_link.rb b/app/models/work_items/parent_link.rb index 21e31980fda..5dff9e8e8d5 100644 --- a/app/models/work_items/parent_link.rb +++ b/app/models/work_items/parent_link.rb @@ -41,6 +41,10 @@ module WorkItems def relative_positioning_parent_column :work_item_parent_id end + + def for_work_item(work_item) + find_or_initialize_by(work_item: work_item) + end end private diff --git a/app/models/work_items/resource_link_event.rb b/app/models/work_items/resource_link_event.rb new file mode 100644 index 00000000000..6725acf8c68 --- /dev/null +++ b/app/models/work_items/resource_link_event.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module WorkItems + class ResourceLinkEvent < ResourceEvent + belongs_to :child_work_item, class_name: 'WorkItem' + + validates :child_work_item, presence: true + + enum action: { + add: 1, + remove: 2 + } + end +end + +WorkItems::ResourceLinkEvent.prepend_mod diff --git a/app/models/work_items/widget_definition.rb b/app/models/work_items/widget_definition.rb index 5d4414e95d8..763b1a79069 100644 --- a/app/models/work_items/widget_definition.rb +++ b/app/models/work_items/widget_definition.rb @@ -28,7 +28,10 @@ module WorkItems progress: 10, # EE-only status: 11, # EE-only requirement_legacy: 12, # EE-only - test_reports: 13 # EE-only + test_reports: 13, # EE-only + notifications: 14, + current_user_todos: 15, + award_emoji: 16 } def self.available_widgets diff --git a/app/models/work_items/widgets/award_emoji.rb b/app/models/work_items/widgets/award_emoji.rb new file mode 100644 index 00000000000..3c862d7c267 --- /dev/null +++ b/app/models/work_items/widgets/award_emoji.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class AwardEmoji < Base + delegate :award_emoji, :downvotes, :upvotes, to: :work_item + end + end +end diff --git a/app/models/work_items/widgets/base.rb b/app/models/work_items/widgets/base.rb index 3a5b03bd514..b54b84f1e1b 100644 --- a/app/models/work_items/widgets/base.rb +++ b/app/models/work_items/widgets/base.rb @@ -15,6 +15,12 @@ module WorkItems [] end + def self.callback_class + Issuable::Callbacks.const_get(name.demodulize, false) + rescue NameError + nil + end + def type self.class.type end diff --git a/app/models/work_items/widgets/current_user_todos.rb b/app/models/work_items/widgets/current_user_todos.rb new file mode 100644 index 00000000000..61c4fcb453b --- /dev/null +++ b/app/models/work_items/widgets/current_user_todos.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class CurrentUserTodos < Base + end + end +end diff --git a/app/models/work_items/widgets/notifications.rb b/app/models/work_items/widgets/notifications.rb new file mode 100644 index 00000000000..9a13e5ebbea --- /dev/null +++ b/app/models/work_items/widgets/notifications.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class Notifications < Base + delegate :subscribed?, to: :work_item + end + end +end |