diff options
Diffstat (limited to 'app/models')
118 files changed, 1763 insertions, 977 deletions
diff --git a/app/models/analytics/cycle_analytics/issue_stage_event.rb b/app/models/analytics/cycle_analytics/issue_stage_event.rb new file mode 100644 index 00000000000..1da8973ff21 --- /dev/null +++ b/app/models/analytics/cycle_analytics/issue_stage_event.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Analytics + module CycleAnalytics + class IssueStageEvent < ApplicationRecord + extend SuppressCompositePrimaryKeyWarning + + validates(*%i[stage_event_hash_id issue_id group_id project_id start_event_timestamp], presence: true) + end + end +end diff --git a/app/models/analytics/cycle_analytics/merge_request_stage_event.rb b/app/models/analytics/cycle_analytics/merge_request_stage_event.rb new file mode 100644 index 00000000000..d2f899ae933 --- /dev/null +++ b/app/models/analytics/cycle_analytics/merge_request_stage_event.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Analytics + module CycleAnalytics + class MergeRequestStageEvent < ApplicationRecord + extend SuppressCompositePrimaryKeyWarning + + validates(*%i[stage_event_hash_id merge_request_id group_id project_id start_event_timestamp], presence: true) + end + end +end diff --git a/app/models/application_record.rb b/app/models/application_record.rb index d9375b55e89..d2757d8c17d 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true class ApplicationRecord < ActiveRecord::Base + self.gitlab_schema = :gitlab_main self.abstract_class = true alias_method :reset, :reload @@ -30,7 +31,7 @@ class ApplicationRecord < ActiveRecord::Base end def self.safe_ensure_unique(retries: 0) - transaction(requires_new: true) do + transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions yield end rescue ActiveRecord::RecordNotUnique @@ -54,7 +55,7 @@ class ApplicationRecord < ActiveRecord::Base # currently one third of the default 15-second timeout def self.with_fast_read_statement_timeout(timeout_ms = 5000) ::Gitlab::Database::LoadBalancing::Session.current.fallback_to_replicas_for_ambiguous_queries do - transaction(requires_new: true) do + transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions connection.exec_query("SET LOCAL statement_timeout = #{timeout_ms}") yield @@ -63,14 +64,6 @@ class ApplicationRecord < ActiveRecord::Base end def self.safe_find_or_create_by(*args, &block) - return optimized_safe_find_or_create_by(*args, &block) if Feature.enabled?(:optimize_safe_find_or_create_by, default_enabled: :yaml) - - safe_ensure_unique(retries: 1) do - find_or_create_by(*args, &block) - end - end - - def self.optimized_safe_find_or_create_by(*args, &block) record = find_by(*args) return record if record.present? @@ -79,7 +72,7 @@ class ApplicationRecord < ActiveRecord::Base # # When calling this method on an association, just calling `self.create` would call `ActiveRecord::Persistence.create` # and that skips some code that adds the newly created record to the association. - transaction(requires_new: true) { all.create(*args, &block) } + transaction(requires_new: true) { all.create(*args, &block) } # rubocop:disable Performance/ActiveRecordSubtransactions rescue ActiveRecord::RecordNotUnique find_by(*args) end @@ -103,23 +96,18 @@ class ApplicationRecord < ActiveRecord::Base enum(enum_mod.key => values) end - def self.transaction(**options, &block) - if options[:requires_new] && track_subtransactions? - ::Gitlab::Database::Metrics.subtransactions_increment(self.name) - end - - super(**options, &block) - end - - def self.track_subtransactions? - ::Feature.enabled?(:active_record_subtransactions_counter, type: :ops, default_enabled: :yaml) && - connection.transaction_open? - end - def self.cached_column_list self.column_names.map { |column_name| self.arel_table[column_name] } end + def self.default_select_columns + if ignored_columns.any? + cached_column_list + else + arel_table[Arel.star] + end + end + def readable_by?(user) Ability.allowed?(user, "read_#{to_ability_name}".to_sym, self) end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index c4b6bcb9395..5f16b990d01 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -6,6 +6,7 @@ class ApplicationSetting < ApplicationRecord include TokenAuthenticatable include ChronicDurationAttribute include IgnorableColumns + include Sanitizable ignore_columns %i[elasticsearch_shards elasticsearch_replicas], remove_with: '14.4', remove_after: '2021-09-22' ignore_column :seat_link_enabled, remove_with: '14.4', remove_after: '2021-09-22' @@ -32,6 +33,8 @@ class ApplicationSetting < ApplicationRecord alias_attribute :instance_group_id, :instance_administrators_group_id alias_attribute :instance_administrators_group, :instance_group + sanitizes! :default_branch_name + def self.kroki_formats_attributes { blockdiag: { @@ -204,6 +207,10 @@ class ApplicationSetting < ApplicationRecord numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than: ::Gitlab::Pages::MAX_SIZE / 1.megabyte } + validates :jobs_per_stage_page_size, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :default_artifacts_expire_in, presence: true, duration: true validates :container_expiration_policies_enable_historic_entries, @@ -343,6 +350,8 @@ class ApplicationSetting < ApplicationRecord validates :snippet_size_limit, numericality: { only_integer: true, greater_than: 0 } validates :wiki_page_max_content_bytes, numericality: { only_integer: true, greater_than_or_equal_to: 1.kilobytes } + validates :max_yaml_size_bytes, numericality: { only_integer: true, greater_than: 0 }, presence: true + validates :max_yaml_depth, numericality: { only_integer: true, greater_than: 0 }, presence: true validates :email_restrictions, untrusted_regexp: true @@ -463,53 +472,28 @@ class ApplicationSetting < ApplicationRecord length: { maximum: 255, message: _('is too long (maximum is %{count} characters)') }, allow_blank: true - validates :throttle_unauthenticated_requests_per_period, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_unauthenticated_period_in_seconds, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_unauthenticated_packages_api_requests_per_period, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_unauthenticated_packages_api_period_in_seconds, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_authenticated_api_requests_per_period, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_authenticated_api_period_in_seconds, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_authenticated_web_requests_per_period, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_authenticated_web_period_in_seconds, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_authenticated_packages_api_requests_per_period, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_authenticated_packages_api_period_in_seconds, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_protected_paths_requests_per_period, - presence: true, - numericality: { only_integer: true, greater_than: 0 } - - validates :throttle_protected_paths_period_in_seconds, - presence: true, - numericality: { only_integer: true, greater_than: 0 } + with_options(presence: true, numericality: { only_integer: true, greater_than: 0 }) do + validates :throttle_unauthenticated_api_requests_per_period + validates :throttle_unauthenticated_api_period_in_seconds + validates :throttle_unauthenticated_requests_per_period + validates :throttle_unauthenticated_period_in_seconds + validates :throttle_unauthenticated_packages_api_requests_per_period + validates :throttle_unauthenticated_packages_api_period_in_seconds + validates :throttle_unauthenticated_files_api_requests_per_period + validates :throttle_unauthenticated_files_api_period_in_seconds + validates :throttle_authenticated_api_requests_per_period + validates :throttle_authenticated_api_period_in_seconds + validates :throttle_authenticated_git_lfs_requests_per_period + validates :throttle_authenticated_git_lfs_period_in_seconds + validates :throttle_authenticated_web_requests_per_period + validates :throttle_authenticated_web_period_in_seconds + validates :throttle_authenticated_packages_api_requests_per_period + validates :throttle_authenticated_packages_api_period_in_seconds + validates :throttle_authenticated_files_api_requests_per_period + validates :throttle_authenticated_files_api_period_in_seconds + validates :throttle_protected_paths_requests_per_period + validates :throttle_protected_paths_period_in_seconds + end validates :notes_create_limit, numericality: { only_integer: true, greater_than_or_equal_to: 0 } @@ -534,6 +518,18 @@ class ApplicationSetting < ApplicationRecord validates :floc_enabled, inclusion: { in: [true, false], message: _('must be a boolean value') } + enum sidekiq_job_limiter_mode: { + Gitlab::SidekiqMiddleware::SizeLimiter::Validator::TRACK_MODE => 0, + Gitlab::SidekiqMiddleware::SizeLimiter::Validator::COMPRESS_MODE => 1 # The default + } + + validates :sidekiq_job_limiter_mode, + inclusion: { in: self.sidekiq_job_limiter_modes } + validates :sidekiq_job_limiter_compression_threshold_bytes, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :sidekiq_job_limiter_limit_bytes, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + attr_encrypted :asset_proxy_secret_key, mode: :per_attribute_iv, key: Settings.attr_encrypted_db_key_base_truncated, @@ -573,7 +569,7 @@ class ApplicationSetting < ApplicationRecord before_validation :ensure_uuid! before_validation :coerce_repository_storages_weighted, if: :repository_storages_weighted_changed? - before_validation :sanitize_default_branch_name + before_validation :normalize_default_branch_name before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token @@ -603,12 +599,8 @@ class ApplicationSetting < ApplicationRecord !!(sourcegraph_url =~ %r{\Ahttps://(www\.)?sourcegraph\.com}) end - def sanitize_default_branch_name - self.default_branch_name = if default_branch_name.blank? - nil - else - Sanitize.fragment(self.default_branch_name) - end + def normalize_default_branch_name + self.default_branch_name = default_branch_name.presence end def instance_review_permitted? @@ -622,7 +614,7 @@ class ApplicationSetting < ApplicationRecord def self.create_from_defaults check_schema! - transaction(requires_new: true) do + transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions super end rescue ActiveRecord::RecordNotUnique diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 060c831a11b..612fda158d3 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -109,6 +109,8 @@ module ApplicationSettingImplementation max_artifacts_size: Settings.artifacts['max_size'], max_attachment_size: Settings.gitlab['max_attachment_size'], max_import_size: 0, + max_yaml_size_bytes: 1.megabyte, + max_yaml_depth: 100, minimum_password_length: DEFAULT_MINIMUM_PASSWORD_LENGTH, mirror_available: true, notes_create_limit: 300, @@ -161,24 +163,36 @@ module ApplicationSettingImplementation throttle_authenticated_api_enabled: false, throttle_authenticated_api_period_in_seconds: 3600, throttle_authenticated_api_requests_per_period: 7200, + throttle_authenticated_git_lfs_enabled: false, + throttle_authenticated_git_lfs_period_in_seconds: 60, + throttle_authenticated_git_lfs_requests_per_period: 1000, throttle_authenticated_web_enabled: false, throttle_authenticated_web_period_in_seconds: 3600, throttle_authenticated_web_requests_per_period: 7200, throttle_authenticated_packages_api_enabled: false, throttle_authenticated_packages_api_period_in_seconds: 15, throttle_authenticated_packages_api_requests_per_period: 1000, + throttle_authenticated_files_api_enabled: false, + throttle_authenticated_files_api_period_in_seconds: 15, + throttle_authenticated_files_api_requests_per_period: 500, throttle_incident_management_notification_enabled: false, throttle_incident_management_notification_per_period: 3600, throttle_incident_management_notification_period_in_seconds: 3600, throttle_protected_paths_enabled: false, throttle_protected_paths_in_seconds: 10, throttle_protected_paths_per_period: 60, + throttle_unauthenticated_api_enabled: false, + throttle_unauthenticated_api_period_in_seconds: 3600, + throttle_unauthenticated_api_requests_per_period: 3600, throttle_unauthenticated_enabled: false, throttle_unauthenticated_period_in_seconds: 3600, throttle_unauthenticated_requests_per_period: 3600, throttle_unauthenticated_packages_api_enabled: false, throttle_unauthenticated_packages_api_period_in_seconds: 15, throttle_unauthenticated_packages_api_requests_per_period: 800, + throttle_unauthenticated_files_api_enabled: false, + throttle_unauthenticated_files_api_period_in_seconds: 15, + throttle_unauthenticated_files_api_requests_per_period: 125, time_tracking_limit_to_hours: false, two_factor_grace_period: 48, unique_ips_limit_enabled: false, @@ -197,7 +211,8 @@ module ApplicationSettingImplementation kroki_url: nil, kroki_formats: { blockdiag: false, bpmn: false, excalidraw: false }, rate_limiting_response_text: nil, - whats_new_variant: 0 + whats_new_variant: 0, + user_deactivation_emails_enabled: true } end diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index c8f6b9aaedb..d251b0adbd3 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -66,3 +66,5 @@ class AwardEmoji < ApplicationRecord awardable.try(:update_upvotes_count) if upvote? end end + +AwardEmoji.prepend_mod_with('AwardEmoji') diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 24f86b44841..ab5d248ff8c 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -78,6 +78,30 @@ class BulkImports::Entity < ApplicationRecord ERB::Util.url_encode(source_full_path) end + def pipelines + @pipelines ||= case source_type + when 'group_entity' + BulkImports::Groups::Stage.pipelines + when 'project_entity' + BulkImports::Projects::Stage.pipelines + end + end + + def pipeline_exists?(name) + pipelines.any? { |_, pipeline| pipeline.to_s == name.to_s } + end + + def create_pipeline_trackers! + self.class.transaction do + pipelines.each do |stage, pipeline| + trackers.create!( + stage: stage, + pipeline_name: pipeline + ) + end + end + end + private def validate_parent_is_a_group diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index 1b108d5c042..c185470b1c2 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -34,8 +34,8 @@ class BulkImports::Tracker < ApplicationRecord end def pipeline_class - unless BulkImports::Stage.pipeline_exists?(pipeline_name) - raise NameError, "'#{pipeline_name}' is not a valid BulkImport Pipeline" + unless entity.pipeline_exists?(pipeline_name) + raise BulkImports::Error, "'#{pipeline_name}' is not a valid BulkImport Pipeline" end pipeline_name.constantize diff --git a/app/models/ci/application_record.rb b/app/models/ci/application_record.rb index 9d4a8f0648e..913e7a62c66 100644 --- a/app/models/ci/application_record.rb +++ b/app/models/ci/application_record.rb @@ -2,6 +2,7 @@ module Ci class ApplicationRecord < ::ApplicationRecord + self.gitlab_schema = :gitlab_ci self.abstract_class = true def self.table_name_prefix diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 577bca282ef..97fb8233d34 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -28,10 +28,10 @@ module Ci state_machine :status do after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge| - next unless bridge.downstream_project + next unless bridge.triggers_downstream_pipeline? bridge.run_after_commit do - bridge.schedule_downstream_pipeline! + ::Ci::CreateCrossProjectPipelineWorker.perform_async(bridge.id) end end @@ -64,12 +64,6 @@ module Ci ) end - def schedule_downstream_pipeline! - raise InvalidBridgeTypeError unless downstream_project - - ::Ci::CreateCrossProjectPipelineWorker.perform_async(self.id) - end - def inherit_status_from_downstream!(pipeline) case pipeline.status when 'success' @@ -112,10 +106,18 @@ module Ci pipeline if triggers_child_pipeline? end + def triggers_downstream_pipeline? + triggers_child_pipeline? || triggers_cross_project_pipeline? + end + def triggers_child_pipeline? yaml_for_downstream.present? end + def triggers_cross_project_pipeline? + downstream_project_path.present? + end + def tags [:bridge] end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 1ca291a659b..e2e24247679 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -90,6 +90,10 @@ module Ci end end + def persisted_environment=(environment) + strong_memoize(:persisted_environment) { environment } + end + serialize :options # rubocop:disable Cop/ActiveRecordSerialize serialize :yaml_variables, Gitlab::Serializer::Ci::Variables # rubocop:disable Cop/ActiveRecordSerialize @@ -166,8 +170,6 @@ module Ci scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) } scope :finished_before, -> (date) { finished.where('finished_at < ?', date) } - scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") } - scope :with_secure_reports_from_config_options, -> (job_types) do joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types) end @@ -306,7 +308,9 @@ module Ci end after_transition pending: :running do |build| - build.deployment&.run + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment&.run + end build.run_after_commit do build.pipeline.persistent_ref.create @@ -328,7 +332,9 @@ module Ci end after_transition any => [:success] do |build| - build.deployment&.succeed + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment&.succeed + end build.run_after_commit do BuildSuccessWorker.perform_async(id) @@ -341,7 +347,9 @@ module Ci next unless build.deployment begin - build.deployment.drop! + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + build.deployment.drop! + end rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, build_id: build.id) end @@ -362,10 +370,12 @@ module Ci end after_transition any => [:skipped, :canceled] do |build, transition| - if transition.to_name == :skipped - build.deployment&.skip - else - build.deployment&.cancel + Gitlab::Database.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338867') do + if transition.to_name == :skipped + build.deployment&.skip + else + build.deployment&.cancel + end end end end @@ -712,6 +722,10 @@ module Ci update_column(:trace, nil) end + def ensure_trace_metadata! + Ci::BuildTraceMetadata.find_or_upsert_for!(id) + end + def artifacts_expose_as options.dig(:artifacts, :expose_as) end @@ -748,7 +762,9 @@ module Ci def any_runners_available? cache_for_available_runners do - project.active_runners.exists? + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do + project.active_runners.exists? + end end end @@ -1013,9 +1029,10 @@ module Ci # Consider this object to have a structural integrity problems def doom! - update_columns( - status: :failed, - failure_reason: :data_integrity_failure) + transaction do + update_columns(status: :failed, failure_reason: :data_integrity_failure) + all_queuing_entries.delete_all + end end def degradation_threshold diff --git a/app/models/ci/build_trace_chunks/fog.rb b/app/models/ci/build_trace_chunks/fog.rb index 3bfac2b33c0..1cae2279434 100644 --- a/app/models/ci/build_trace_chunks/fog.rb +++ b/app/models/ci/build_trace_chunks/fog.rb @@ -80,12 +80,10 @@ module Ci private def append_strings(old_data, new_data) - if Feature.enabled?(:ci_job_trace_force_encode, default_enabled: :yaml) - # When object storage is in use, old_data may be retrieved in UTF-8. - old_data = old_data.force_encoding(Encoding::ASCII_8BIT) - # new_data should already be in ASCII-8BIT, but just in case it isn't, do this. - new_data = new_data.force_encoding(Encoding::ASCII_8BIT) - end + # When object storage is in use, old_data may be retrieved in UTF-8. + old_data = old_data.force_encoding(Encoding::ASCII_8BIT) + # new_data should already be in ASCII-8BIT, but just in case it isn't, do this. + new_data = new_data.force_encoding(Encoding::ASCII_8BIT) old_data + new_data end diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb index 05bdb3d8b7b..901b84ceec6 100644 --- a/app/models/ci/build_trace_metadata.rb +++ b/app/models/ci/build_trace_metadata.rb @@ -2,6 +2,7 @@ module Ci class BuildTraceMetadata < Ci::ApplicationRecord + MAX_ATTEMPTS = 5 self.table_name = 'ci_build_trace_metadata' self.primary_key = :build_id @@ -9,5 +10,49 @@ module Ci belongs_to :trace_artifact, class_name: 'Ci::JobArtifact' validates :build, presence: true + validates :archival_attempts, presence: true + + def self.find_or_upsert_for!(build_id) + record = find_by(build_id: build_id) + return record if record + + upsert({ build_id: build_id }, unique_by: :build_id) + find_by!(build_id: build_id) + end + + # The job is retried around 5 times during the 7 days retention period for + # trace chunks as defined in `Ci::BuildTraceChunks::RedisBase::CHUNK_REDIS_TTL` + def can_attempt_archival_now? + return false unless archival_attempts_available? + return true unless last_archival_attempt_at + + last_archival_attempt_at + backoff < Time.current + end + + def archival_attempts_available? + archival_attempts <= MAX_ATTEMPTS + end + + def increment_archival_attempts! + increment!(:archival_attempts, touch: :last_archival_attempt_at) + end + + def track_archival!(trace_artifact_id) + update!(trace_artifact_id: trace_artifact_id, archived_at: Time.current) + end + + def archival_attempts_message + if archival_attempts_available? + 'The job can not be archived right now.' + else + 'The job is out of archival attempts.' + end + end + + private + + def backoff + ::Gitlab::Ci::Trace::Backoff.new(archival_attempts).value_with_jitter + end end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 1f0da4345f2..ad3e867f9d5 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -10,6 +10,9 @@ module Ci include Artifactable include FileStoreMounter include EachBatch + include IgnorableColumns + + ignore_columns %i[id_convert_to_bigint job_id_convert_to_bigint], remove_with: '14.5', remove_after: '2021-11-22' TEST_REPORT_FILE_TYPES = %w[junit].freeze COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze @@ -182,7 +185,6 @@ module Ci scope :order_expired_desc, -> { order(expire_at: :desc) } scope :with_destroy_preloads, -> { includes(project: [:route, :statistics]) } - scope :scoped_project, -> { where('ci_job_artifacts.project_id = projects.id') } scope :for_project, ->(project) { where(project_id: project) } scope :created_in_time_range, ->(from: nil, to: nil) { where(created_at: from..to) } @@ -232,6 +234,17 @@ module Ci hashed_path: 2 } + # `locked` will be populated from the source of truth on Ci::Pipeline + # in order to clean up expired job artifacts in a performant way. + # The values should be the same as `Ci::Pipeline.lockeds` with the + # additional value of `unknown` to indicate rows that have not + # yet been populated from the parent Ci::Pipeline + enum locked: { + unlocked: 0, + artifacts_locked: 1, + unknown: 2 + }, _prefix: :artifact + def validate_file_format! unless TYPE_AND_FORMAT_PAIRS[self.file_type&.to_sym] == self.file_format&.to_sym errors.add(:base, _('Invalid file format with specified file type')) diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index 7cf3a387516..ccad6290fac 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -2,6 +2,8 @@ module Ci class PendingBuild < Ci::ApplicationRecord + include EachBatch + belongs_to :project belongs_to :build, class_name: 'Ci::Build' belongs_to :namespace, inverse_of: :pending_builds, class_name: 'Namespace' @@ -11,52 +13,62 @@ module Ci scope :ref_protected, -> { where(protected: true) } scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) } scope :with_instance_runners, -> { where(instance_runners_enabled: true) } + scope :for_tags, ->(tag_ids) do + if tag_ids.present? + where('ci_pending_builds.tag_ids <@ ARRAY[?]::int[]', Array.wrap(tag_ids)) + else + where("ci_pending_builds.tag_ids = '{}'") + end + end - def self.upsert_from_build!(build) - entry = self.new(args_from_build(build)) + class << self + def upsert_from_build!(build) + entry = self.new(args_from_build(build)) - entry.validate! + entry.validate! - self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) - end + self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) + end - def self.args_from_build(build) - args = { - build: build, - project: build.project, - protected: build.protected?, - namespace: build.project.namespace - } + private + + def args_from_build(build) + project = build.project + + args = { + build: build, + project: project, + protected: build.protected?, + namespace: project.namespace + } + + if Feature.enabled?(:ci_pending_builds_maintain_tags_data, type: :development, default_enabled: :yaml) + args.store(:tag_ids, build.tags_ids) + end + + if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) + args.store(:instance_runners_enabled, shared_runners_enabled?(project)) + end + + if Feature.enabled?(:ci_pending_builds_maintain_namespace_traversal_ids, type: :development, default_enabled: :yaml) + args.store(:namespace_traversal_ids, project.namespace.traversal_ids) if group_runners_enabled?(project) + end - if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) - args.merge(instance_runners_enabled: shareable?(build)) - else args end - end - private_class_method :args_from_build - - def self.shareable?(build) - shared_runner_enabled?(build) && - builds_access_level?(build) && - project_not_removed?(build) - end - private_class_method :shareable? - def self.shared_runner_enabled?(build) - build.project.shared_runners.exists? - end - private_class_method :shared_runner_enabled? + def shared_runners_enabled?(project) + builds_enabled?(project) && project.shared_runners_enabled? + end - def self.project_not_removed?(build) - !build.project.pending_delete? - end - private_class_method :project_not_removed? + def group_runners_enabled?(project) + builds_enabled?(project) && project.group_runners_enabled? + end - def self.builds_access_level?(build) - build.project.project_feature.builds_access_level.nil? || build.project.project_feature.builds_access_level > 0 + def builds_enabled?(project) + project.builds_enabled? && !project.pending_delete? + end end - private_class_method :builds_access_level? end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 70e67953e31..1a0cec3c935 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -66,6 +66,7 @@ module Ci has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline has_many :bridges, class_name: 'Ci::Bridge', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline + has_many :generic_commit_statuses, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'GenericCommitStatus' has_many :job_artifacts, through: :builds has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id # rubocop:disable Cop/ActiveRecordDependent has_many :variables, class_name: 'Ci::PipelineVariable' @@ -307,6 +308,7 @@ module Ci scope :ci_and_parent_sources, -> { where(source: Enums::Ci::Pipeline.ci_and_parent_sources.values) } scope :for_user, -> (user) { where(user: user) } scope :for_sha, -> (sha) { where(sha: sha) } + scope :where_not_sha, -> (sha) { where.not(sha: sha) } scope :for_source_sha, -> (source_sha) { where(source_sha: source_sha) } scope :for_sha_or_source_sha, -> (sha) { for_sha(sha).or(for_source_sha(sha)) } scope :for_ref, -> (ref) { where(ref: ref) } @@ -317,7 +319,6 @@ module Ci scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) } scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) } scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } - scope :eager_load_project, -> { eager_load(project: [:route, { namespace: :route }]) } scope :with_pipeline_source, -> (source) { where(source: source)} scope :outside_pipeline_family, ->(pipeline) do @@ -588,13 +589,11 @@ module Ci end def cancel_running(retries: 1) - commit_status_relations = [:project, :pipeline] - ci_build_relations = [:deployment, :taggings] + preloaded_relations = [:project, :pipeline, :deployment, :taggings] retry_lock(cancelable_statuses, retries, name: 'ci_pipeline_cancel_running') do |cancelables| cancelables.find_in_batches do |batch| - ActiveRecord::Associations::Preloader.new.preload(batch, commit_status_relations) - ActiveRecord::Associations::Preloader.new.preload(batch.select { |job| job.is_a?(Ci::Build) }, ci_build_relations) + Preloaders::CommitStatusPreloader.new(batch).execute(preloaded_relations) batch.each do |job| yield(job) if block_given? @@ -1108,7 +1107,7 @@ module Ci merge_request.modified_paths elsif branch_updated? push_details.modified_paths - elsif external_pull_request? && ::Feature.enabled?(:ci_modified_paths_of_external_prs, project, default_enabled: :yaml) + elsif external_pull_request? external_pull_request.modified_paths end end @@ -1220,24 +1219,12 @@ module Ci self.ci_ref = Ci::Ref.ensure_for(self) end - # We need `base_and_ancestors` in a specific order to "break" when needed. - # If we use `find_each`, then the order is broken. - # rubocop:disable Rails/FindEach def reset_source_bridge!(current_user) - if ::Feature.enabled?(:ci_reset_bridge_with_subsequent_jobs, project, default_enabled: :yaml) - return unless bridge_waiting? + return unless bridge_waiting? - source_bridge.pending! - Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass - else - self_and_upstreams.includes(:source_bridge).each do |pipeline| - break unless pipeline.bridge_waiting? - - pipeline.source_bridge.pending! - end - end + source_bridge.pending! + Ci::AfterRequeueJobService.new(project, current_user).execute(source_bridge) # rubocop:disable CodeReuse/ServiceClass end - # rubocop:enable Rails/FindEach # EE-only def merge_train_pipeline? diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index a0e8886414b..3dca77af051 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -8,7 +8,7 @@ module Ci alias_attribute :secret_value, :value - validates :key, uniqueness: { scope: :pipeline_id } + validates :key, presence: true def hook_attrs { key: key, value: value } diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 432c3a408a9..4aa232ad26b 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -208,16 +208,18 @@ module Ci Arel.sql("(#{arel_tag_names_array.to_sql})") ] - group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| - Gitlab::Ci::Matching::RunnerMatcher.new({ - runner_ids: values[0], - runner_type: values[1], - public_projects_minutes_cost_factor: values[2], - private_projects_minutes_cost_factor: values[3], - run_untagged: values[4], - access_level: values[5], - tag_list: values[6] - }) + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339621') do + group(*unique_params).pluck('array_agg(ci_runners.id)', *unique_params).map do |values| + Gitlab::Ci::Matching::RunnerMatcher.new({ + runner_ids: values[0], + runner_type: values[1], + public_projects_minutes_cost_factor: values[2], + private_projects_minutes_cost_factor: values[3], + run_untagged: values[4], + access_level: values[5], + tag_list: values[6] + }) + end end end @@ -385,6 +387,12 @@ module Ci read_attribute(:contacted_at) end + def namespace_ids + strong_memoize(:namespace_ids) do + runner_namespaces.pluck(:namespace_id).compact + end + end + private def cleanup_runner_queue @@ -420,14 +428,18 @@ module Ci end def no_projects - if projects.any? - errors.add(:runner, 'cannot have projects assigned') + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do + if projects.any? + errors.add(:runner, 'cannot have projects assigned') + end end end def no_groups - if groups.any? - errors.add(:runner, 'cannot have groups assigned') + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/338659') do + if groups.any? + errors.add(:runner, 'cannot have groups assigned') + end end end diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index f78caf710a6..95842d944f9 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -4,6 +4,9 @@ module Ci module Sources class Pipeline < Ci::ApplicationRecord include Ci::NamespacedModelName + include IgnorableColumns + + ignore_columns 'source_job_id_convert_to_bigint', remove_with: '14.5', remove_after: '2021-11-22' self.table_name = "ci_sources_pipelines" diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb index 9fb8cd024c5..cf6d95fc6df 100644 --- a/app/models/clusters/agent.rb +++ b/app/models/clusters/agent.rb @@ -10,6 +10,12 @@ module Clusters has_many :agent_tokens, class_name: 'Clusters::AgentToken' has_many :last_used_agent_tokens, -> { order_last_used_at_desc }, class_name: 'Clusters::AgentToken', inverse_of: :agent + has_many :group_authorizations, class_name: 'Clusters::Agents::GroupAuthorization' + has_many :authorized_groups, class_name: '::Group', through: :group_authorizations, source: :group + + has_many :project_authorizations, class_name: 'Clusters::Agents::ProjectAuthorization' + has_many :authorized_projects, class_name: '::Project', through: :project_authorizations, source: :project + scope :ordered_by_name, -> { order(:name) } scope :with_name, -> (name) { where(name: name) } diff --git a/app/models/clusters/agents/group_authorization.rb b/app/models/clusters/agents/group_authorization.rb new file mode 100644 index 00000000000..74c0cec3b7e --- /dev/null +++ b/app/models/clusters/agents/group_authorization.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class GroupAuthorization < ApplicationRecord + self.table_name = 'agent_group_authorizations' + + belongs_to :agent, class_name: 'Clusters::Agent', optional: false + belongs_to :group, class_name: '::Group', optional: false + + validates :config, json_schema: { filename: 'cluster_agent_authorization_configuration' } + + delegate :project, to: :agent + end + end +end diff --git a/app/models/clusters/agents/implicit_authorization.rb b/app/models/clusters/agents/implicit_authorization.rb new file mode 100644 index 00000000000..967cc686045 --- /dev/null +++ b/app/models/clusters/agents/implicit_authorization.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class ImplicitAuthorization + attr_reader :agent + + delegate :id, to: :agent, prefix: true + delegate :project, to: :agent + + def initialize(agent:) + @agent = agent + end + + def config + nil + end + end + end +end diff --git a/app/models/clusters/agents/project_authorization.rb b/app/models/clusters/agents/project_authorization.rb new file mode 100644 index 00000000000..1c71a0a432a --- /dev/null +++ b/app/models/clusters/agents/project_authorization.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class ProjectAuthorization < ApplicationRecord + self.table_name = 'agent_project_authorizations' + + belongs_to :agent, class_name: 'Clusters::Agent', optional: false + belongs_to :project, class_name: '::Project', optional: false + + validates :config, json_schema: { filename: 'cluster_agent_authorization_configuration' } + end + end +end diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 2fff0a69a26..feac7bbc363 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -148,6 +148,7 @@ module Clusters scope :with_management_project, -> { where.not(management_project: nil) } scope :for_project_namespace, -> (namespace_id) { joins(:projects).where(projects: { namespace_id: namespace_id }) } + scope :with_name, -> (name) { where(name: name) } # with_application_prometheus scope is deprecated, and scheduled for removal # in %14.0. See https://gitlab.com/groups/gitlab-org/-/epics/4280 diff --git a/app/models/clusters/clusters_hierarchy.rb b/app/models/clusters/clusters_hierarchy.rb index 162a1a3290d..9435d258d67 100644 --- a/app/models/clusters/clusters_hierarchy.rb +++ b/app/models/clusters/clusters_hierarchy.rb @@ -83,7 +83,7 @@ module Clusters project_id: clusterable.id } - model.sanitize_sql_array([Arel.sql(order), values]) + Arel.sql(model.sanitize_sql_array([Arel.sql(order), values])) end def group_clusters_base_query diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 7f5f87e3e36..7ec614b048c 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -137,6 +137,14 @@ module Clusters kubeclient.patch_ingress(ingress.name, data, namespace) end + def kubeconfig(namespace) + to_kubeconfig( + url: api_url, + namespace: namespace, + token: token, + ca_pem: ca_pem) + end + private def default_namespace(project, environment_name:) @@ -154,14 +162,6 @@ module Clusters ).execute end - def kubeconfig(namespace) - to_kubeconfig( - url: api_url, - namespace: namespace, - token: token, - ca_pem: ca_pem) - end - def read_pods(namespace) kubeclient.get_pods(namespace: namespace).as_json rescue Kubeclient::ResourceNotFoundError diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index b34d64de101..8cba3d04502 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -56,15 +56,19 @@ class CommitStatus < Ci::ApplicationRecord scope :for_ref, -> (ref) { where(ref: ref) } scope :by_name, -> (name) { where(name: name) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } - scope :eager_load_pipeline, -> { eager_load(:pipeline, project: { namespace: :route }) } scope :with_pipeline, -> { joins(:pipeline) } - scope :updated_at_before, ->(date) { where('updated_at < ?', date) } + scope :updated_at_before, ->(date) { where('ci_builds.updated_at < ?', date) } + scope :created_at_before, ->(date) { where('ci_builds.created_at < ?', date) } scope :updated_before, ->(lookback:, timeout:) { where('(ci_builds.created_at BETWEEN ? AND ?) AND (ci_builds.updated_at BETWEEN ? AND ?)', lookback, timeout, lookback, timeout) } + # The scope applies `pluck` to split the queries. Use with care. scope :for_project_paths, -> (paths) do - where(project: Project.where_full_path_in(Array(paths))) + # Pluck is used to split this query. Splitting the query is required for database decomposition for `ci_*` tables. + # https://docs.gitlab.com/ee/development/database/transaction_guidelines.html#database-decomposition-and-sharding + project_ids = Project.where_full_path_in(Array(paths)).pluck(:id) + where(project: project_ids) end scope :with_preloads, -> do diff --git a/app/models/compare.rb b/app/models/compare.rb index 2eaaf98c260..f1b0bf19c11 100644 --- a/app/models/compare.rb +++ b/app/models/compare.rb @@ -25,6 +25,16 @@ class Compare @straight = straight end + # Return a Hash of parameters for passing to a URL helper + # + # See `namespace_project_compare_url` + def to_param + { + from: @straight ? start_commit_sha : base_commit_sha, + to: head_commit_sha + } + end + def cache_key [@project, :compare, diff_refs.hash] end diff --git a/app/models/concerns/approvable_base.rb b/app/models/concerns/approvable_base.rb index ef7ba7b1089..8240f9bd6ea 100644 --- a/app/models/concerns/approvable_base.rb +++ b/app/models/concerns/approvable_base.rb @@ -54,4 +54,8 @@ module ApprovableBase def can_be_approved_by?(user) user && !approved_by?(user) && user.can?(:approve_merge_request, self) end + + def can_be_unapproved_by?(user) + user && approved_by?(user) && user.can?(:approve_merge_request, self) + end end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 44d9beff27e..9414d16beef 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -160,39 +160,6 @@ module CacheMarkdownField # We can only store mentions if the mentionable is a database object return unless self.is_a?(ApplicationRecord) - return store_mentions_without_subtransaction! if Feature.enabled?(:store_mentions_without_subtransaction, default_enabled: :yaml) - - refs = all_references(self.author) - - references = {} - references[:mentioned_users_ids] = refs.mentioned_user_ids.presence - references[:mentioned_groups_ids] = refs.mentioned_group_ids.presence - references[:mentioned_projects_ids] = refs.mentioned_project_ids.presence - - # One retry is enough as next time `model_user_mention` should return the existing mention record, - # that threw the `ActiveRecord::RecordNotUnique` exception in first place. - self.class.safe_ensure_unique(retries: 1) do - user_mention = model_user_mention - - # this may happen due to notes polymorphism, so noteable_id may point to a record - # that no longer exists as we cannot have FK on noteable_id - break if user_mention.blank? - - user_mention.mentioned_users_ids = references[:mentioned_users_ids] - user_mention.mentioned_groups_ids = references[:mentioned_groups_ids] - user_mention.mentioned_projects_ids = references[:mentioned_projects_ids] - - if user_mention.has_mentions? - user_mention.save! - else - user_mention.destroy! - end - end - - true - end - - def store_mentions_without_subtransaction! identifier = user_mention_identifier # this may happen due to notes polymorphism, so noteable_id may point to a record diff --git a/app/models/concerns/calloutable.rb b/app/models/concerns/calloutable.rb new file mode 100644 index 00000000000..8b9cfae6a32 --- /dev/null +++ b/app/models/concerns/calloutable.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Calloutable + extend ActiveSupport::Concern + + included do + belongs_to :user + + validates :user, presence: true + end + + def dismissed_after?(dismissed_after) + dismissed_at > dismissed_after + end +end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index bdba2d3e251..27a704c1de0 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -33,13 +33,13 @@ module Ci # def simple_variables strong_memoize(:simple_variables) do - scoped_variables(environment: nil).to_runner_variables + scoped_variables(environment: nil) end end def simple_variables_without_dependencies strong_memoize(:variables_without_dependencies) do - scoped_variables(environment: nil, dependencies: false).to_runner_variables + scoped_variables(environment: nil, dependencies: false) end end diff --git a/app/models/concerns/cron_schedulable.rb b/app/models/concerns/cron_schedulable.rb index 48605ecc3d7..d5b86db2640 100644 --- a/app/models/concerns/cron_schedulable.rb +++ b/app/models/concerns/cron_schedulable.rb @@ -14,12 +14,10 @@ module CronSchedulable # The `next_run_at` column is set to the actual execution date of worker that # triggers the schedule. This way, a schedule like `*/1 * * * *` won't be triggered # in a short interval when the worker runs irregularly by Sidekiq Memory Killer. - def calculate_next_run_at - now = Time.zone.now + def calculate_next_run_at(start_time = Time.zone.now) + ideal_next_run = ideal_next_run_from(start_time) - ideal_next_run = ideal_next_run_from(now) - - if ideal_next_run == cron_worker_next_run_from(now) + if ideal_next_run == cron_worker_next_run_from(start_time) ideal_next_run else cron_worker_next_run_from(ideal_next_run) diff --git a/app/models/concerns/enums/ci/commit_status.rb b/app/models/concerns/enums/ci/commit_status.rb index 16dec5fb081..7f46e44697e 100644 --- a/app/models/concerns/enums/ci/commit_status.rb +++ b/app/models/concerns/enums/ci/commit_status.rb @@ -26,6 +26,7 @@ module Enums pipeline_loop_detected: 17, no_matching_runner: 18, # not used anymore, but cannot be deleted because of old data trace_size_exceeded: 19, + builds_disabled: 20, insufficient_bridge_permissions: 1_001, downstream_bridge_project_not_found: 1_002, invalid_bridge_trigger: 1_003, diff --git a/app/models/concerns/featurable.rb b/app/models/concerns/featurable.rb index ed9bce87da1..70d67fc7559 100644 --- a/app/models/concerns/featurable.rb +++ b/app/models/concerns/featurable.rb @@ -83,6 +83,10 @@ module Featurable end end + included do + validate :allowed_access_levels + end + def access_level(feature) public_send(self.class.access_level_attribute(feature)) # rubocop:disable GitlabSecurity/PublicSend end @@ -94,4 +98,21 @@ module Featurable def string_access_level(feature) self.class.str_from_access_level(access_level(feature)) end + + private + + def allowed_access_levels + validator = lambda do |field| + level = public_send(field) || ENABLED # rubocop:disable GitlabSecurity/PublicSend + not_allowed = level > ENABLED + self.errors.add(field, "cannot have public visibility level") if not_allowed + end + + (self.class.available_features - feature_validation_exclusion).each {|f| validator.call("#{f}_access_level")} + end + + # Features that we should exclude from the validation + def feature_validation_exclusion + [] + end end diff --git a/app/models/concerns/has_repository.rb b/app/models/concerns/has_repository.rb index 1b4c590694a..9218ba47d20 100644 --- a/app/models/concerns/has_repository.rb +++ b/app/models/concerns/has_repository.rb @@ -122,4 +122,8 @@ module HasRepository def after_repository_change_head reload_default_branch end + + def after_change_head_branch_does_not_exist(branch) + # No-op (by default) + end end diff --git a/app/models/concerns/integrations/has_data_fields.rb b/app/models/concerns/integrations/has_data_fields.rb index e9aaaac8226..1709b56080e 100644 --- a/app/models/concerns/integrations/has_data_fields.rb +++ b/app/models/concerns/integrations/has_data_fields.rb @@ -46,6 +46,7 @@ module Integrations has_one :issue_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id, class_name: 'Integrations::IssueTrackerData' has_one :jira_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id, class_name: 'Integrations::JiraTrackerData' has_one :open_project_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :service_id, class_name: 'Integrations::OpenProjectTrackerData' + has_one :zentao_tracker_data, autosave: true, inverse_of: :integration, foreign_key: :integration_id, class_name: 'Integrations::ZentaoTrackerData' def data_fields raise NotImplementedError diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 8d0f8b01d64..5c307158a9a 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -26,6 +26,7 @@ module Issuable include UpdatedAtFilterable include ClosedAtFilterable include VersionedDescription + include SortableTitle TITLE_LENGTH_MAX = 255 TITLE_HTML_LENGTH_MAX = 800 @@ -116,20 +117,6 @@ module Issuable end # rubocop:enable GitlabSecurity/SqlInjection - scope :without_particular_labels, ->(label_names) do - labels_table = Label.arel_table - label_links_table = LabelLink.arel_table - issuables_table = klass.arel_table - inner_query = label_links_table.project('true') - .join(labels_table, Arel::Nodes::InnerJoin).on(labels_table[:id].eq(label_links_table[:label_id])) - .where(label_links_table[:target_type].eq(name) - .and(label_links_table[:target_id].eq(issuables_table[:id])) - .and(labels_table[:title].in(label_names))) - .exists.not - - where(inner_query) - end - scope :without_label, -> { joins("LEFT OUTER JOIN label_links ON label_links.target_type = '#{name}' AND label_links.target_id = #{table_name}.id").where(label_links: { id: nil }) } scope :with_label_ids, ->(label_ids) { joins(:label_links).where(label_links: { label_id: label_ids }) } scope :join_project, -> { joins(:project) } @@ -293,6 +280,8 @@ module Issuable when 'popularity', 'popularity_desc', 'upvotes_desc' then order_upvotes_desc when 'priority', 'priority_asc' then order_due_date_and_labels_priority(excluded_labels: excluded_labels) when 'priority_desc' then order_due_date_and_labels_priority('DESC', excluded_labels: excluded_labels) + when 'title_asc' then order_title_asc.with_order_id_desc + when 'title_desc' then order_title_desc.with_order_id_desc else order_by(method) end diff --git a/app/models/concerns/loose_foreign_key.rb b/app/models/concerns/loose_foreign_key.rb new file mode 100644 index 00000000000..4e822a04869 --- /dev/null +++ b/app/models/concerns/loose_foreign_key.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module LooseForeignKey + extend ActiveSupport::Concern + + # This concern adds loose foreign key support to ActiveRecord models. + # Loose foreign keys allow delayed processing of associated database records + # with similar guarantees than a database foreign key. + # + # TODO: finalize this later once the async job is in place + # + # Prerequisites: + # + # To start using the concern, you'll need to install a database trigger to the parent + # table in a standard DB migration (not post-migration). + # + # > add_loose_foreign_key_support(:projects, :gitlab_main) + # + # Usage: + # + # > class Ci::Build < ApplicationRecord + # > + # > loose_foreign_key :security_scans, :build_id, on_delete: :async_delete, gitlab_schema: :gitlab_main + # > + # > # associations can be still defined, the dependent options is no longer necessary: + # > has_many :security_scans, class_name: 'Security::Scan' + # > + # > end + # + # Options for on_delete: + # + # - :async_delete - deletes the children rows via an asynchronous process. + # - :async_nullify - sets the foreign key column to null via an asynchronous process. + # + # Options for gitlab_schema: + # + # - :gitlab_ci + # - :gitlab_main + # + # The value can be determined by calling `Model.gitlab_schema` where the Model represents + # the model for the child table. + # + # How it works: + # + # When adding loose foreign key support to the table, a DELETE trigger is installed + # which tracks the record deletions (stores primary key value of the deleted row) in + # a database table. + # + # These deletion records are processed asynchronously and records are cleaned up + # according to the loose foreign key definitions described in the model. + # + # The cleanup happens in batches, which reduces the likelyhood of statement timeouts. + # + # When all associations related to the deleted record are cleaned up, the record itself + # is deleted. + included do + class_attribute :loose_foreign_key_definitions, default: [] + end + + class_methods do + def loose_foreign_key(to_table, column, options) + symbolized_options = options.symbolize_keys + + unless base_class? + raise <<~MSG + loose_foreign_key can be only used on base classes, inherited classes are not supported. + Please define the loose_foreign_key on the #{base_class.name} class. + MSG + end + + on_delete_options = %i[async_delete async_nullify] + gitlab_schema_options = [ApplicationRecord.gitlab_schema, Ci::ApplicationRecord.gitlab_schema] + + unless on_delete_options.include?(symbolized_options[:on_delete]&.to_sym) + raise "Invalid on_delete option given: #{symbolized_options[:on_delete]}. Valid options: #{on_delete_options.join(', ')}" + end + + unless gitlab_schema_options.include?(symbolized_options[:gitlab_schema]&.to_sym) + raise "Invalid gitlab_schema option given: #{symbolized_options[:gitlab_schema]}. Valid options: #{gitlab_schema_options.join(', ')}" + end + + definition = ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new( + table_name.to_s, + to_table.to_s, + { + column: column.to_s, + on_delete: symbolized_options[:on_delete].to_sym, + gitlab_schema: symbolized_options[:gitlab_schema].to_sym + } + ) + + self.loose_foreign_key_definitions += [definition] + end + end +end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index 4df9e32d8ec..a0ea5ac8012 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -217,17 +217,6 @@ module Mentionable def user_mention_association association(:user_mentions).reflection end - - # User mention that is parsed from model description rather then its related notes. - # Models that have a description attribute like Issue, MergeRequest, Epic, Snippet may have such a user mention. - # Other mentionable models like Commit, DesignManagement::Design, will never have such record as those do not have - # a description attribute. - # - # Using this method followed by a call to *save* may result in *ActiveRecord::RecordNotUnique* exception - # in a multi-threaded environment. Make sure to use it within a *safe_ensure_unique* block. - def model_user_mention - user_mentions.where(note_id: nil).first_or_initialize - end end Mentionable.prepend_mod_with('Mentionable') diff --git a/app/models/concerns/optimized_issuable_label_filter.rb b/app/models/concerns/optimized_issuable_label_filter.rb deleted file mode 100644 index 19d2ac620f3..00000000000 --- a/app/models/concerns/optimized_issuable_label_filter.rb +++ /dev/null @@ -1,121 +0,0 @@ -# frozen_string_literal: true - -module OptimizedIssuableLabelFilter - extend ActiveSupport::Concern - - prepended do - extend Gitlab::Cache::RequestCache - - # Avoid repeating label queries times when the finder is instantiated multiple times during the request. - request_cache(:find_label_ids) { [root_namespace.id, params.label_names] } - end - - def by_label(items) - return items unless params.labels? - - return super if Feature.disabled?(:optimized_issuable_label_filter, default_enabled: :yaml) - - target_model = items.model - - if params.filter_by_no_label? - items.where('NOT EXISTS (?)', optimized_any_label_query(target_model)) - elsif params.filter_by_any_label? - items.where('EXISTS (?)', optimized_any_label_query(target_model)) - else - issuables_with_selected_labels(items, target_model) - end - end - - # Taken from IssuableFinder - def count_by_state - return super if Feature.disabled?(:optimized_issuable_label_filter, default_enabled: :yaml) - - count_params = params.merge(state: nil, sort: nil, force_cte: true) - finder = self.class.new(current_user, count_params) - - state_counts = finder - .execute - .reorder(nil) - .group(:state_id) - .count - - counts = Hash.new(0) - - state_counts.each do |key, value| - counts[count_key(key)] += value - end - - counts[:all] = counts.values.sum - counts.with_indifferent_access - end - - private - - def issuables_with_selected_labels(items, target_model) - if root_namespace - all_label_ids = find_label_ids - # Found less labels in the DB than we were searching for. Return nothing. - return items.none if all_label_ids.size != params.label_names.size - - all_label_ids.each do |label_ids| - items = items.where('EXISTS (?)', optimized_label_query_by_label_ids(target_model, label_ids)) - end - else - params.label_names.each do |label_name| - items = items.where('EXISTS (?)', optimized_label_query_by_label_name(target_model, label_name)) - end - end - - items - end - - def find_label_ids - group_labels = Label - .where(project_id: nil) - .where(title: params.label_names) - .where(group_id: root_namespace.self_and_descendants.select(:id)) - - project_labels = Label - .where(group_id: nil) - .where(title: params.label_names) - .where(project_id: Project.select(:id).where(namespace_id: root_namespace.self_and_descendants.select(:id))) - - Label - .from_union([group_labels, project_labels], remove_duplicates: false) - .reorder(nil) - .pluck(:title, :id) - .group_by(&:first) - .values - .map { |labels| labels.map(&:last) } - end - - def root_namespace - strong_memoize(:root_namespace) do - (params.project || params.group)&.root_ancestor - end - end - - def optimized_any_label_query(target_model) - LabelLink - .where(target_type: target_model.name) - .where(LabelLink.arel_table['target_id'].eq(target_model.arel_table['id'])) - .limit(1) - end - - def optimized_label_query_by_label_ids(target_model, label_ids) - LabelLink - .where(target_type: target_model.name) - .where(LabelLink.arel_table['target_id'].eq(target_model.arel_table['id'])) - .where(label_id: label_ids) - .limit(1) - end - - def optimized_label_query_by_label_name(target_model, label_name) - LabelLink - .joins(:label) - .where(target_type: target_model.name) - .where(LabelLink.arel_table['target_id'].eq(target_model.arel_table['id'])) - .where(labels: { name: label_name }) - .limit(1) - end -end diff --git a/app/models/concerns/partitioned_table.rb b/app/models/concerns/partitioned_table.rb index eab5d4c35bb..23d2d00b346 100644 --- a/app/models/concerns/partitioned_table.rb +++ b/app/models/concerns/partitioned_table.rb @@ -14,8 +14,6 @@ module PartitionedTable strategy_class = PARTITIONING_STRATEGIES[strategy.to_sym] || raise(ArgumentError, "Unknown partitioning strategy: #{strategy}") @partitioning_strategy = strategy_class.new(self, partitioning_key, **kwargs) - - Gitlab::Database::Partitioning::PartitionManager.register(self) end end end diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index 75dfed6d58f..c32e499c329 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -135,21 +135,21 @@ module RelativePositioning before, after = [before, after].sort_by(&:relative_position) if before && after RelativePositioning.mover.move(self, before, after) - rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + rescue NoSpaceLeft => e could_not_move(e) raise e end def move_after(before = self) RelativePositioning.mover.move(self, before, nil) - rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + rescue NoSpaceLeft => e could_not_move(e) raise e end def move_before(after = self) RelativePositioning.mover.move(self, nil, after) - rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + rescue NoSpaceLeft => e could_not_move(e) raise e end @@ -159,9 +159,6 @@ module RelativePositioning rescue NoSpaceLeft => e could_not_move(e) self.relative_position = MAX_POSITION - rescue ActiveRecord::QueryCanceled => e - could_not_move(e) - raise e end def move_to_start @@ -169,9 +166,6 @@ module RelativePositioning rescue NoSpaceLeft => e could_not_move(e) self.relative_position = MIN_POSITION - rescue ActiveRecord::QueryCanceled => e - could_not_move(e) - raise e end # This method is used during rebalancing - override it to customise the update diff --git a/app/models/concerns/sanitizable.rb b/app/models/concerns/sanitizable.rb new file mode 100644 index 00000000000..05756beb404 --- /dev/null +++ b/app/models/concerns/sanitizable.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +# == Sanitizable concern +# +# This concern adds HTML sanitization and validation to models. The intention is +# to help prevent XSS attacks in the event of a by-pass in the frontend +# sanitizer due to a configuration issue or a vulnerability in the sanitizer. +# This approach is commonly referred to as defense-in-depth. +# +# Example: +# +# module Dast +# class Profile < ApplicationRecord +# include Sanitizable +# +# sanitizes! :name, :description + +module Sanitizable + extend ActiveSupport::Concern + + class_methods do + def sanitize(input) + return unless input + + # We return the input unchanged to avoid escaping pre-escaped HTML fragments. + # Please see gitlab-org/gitlab#293634 for an example. + return input unless input == CGI.unescapeHTML(input.to_s) + + CGI.unescapeHTML(Sanitize.fragment(input)) + end + + def sanitizes!(*attrs) + instance_eval do + before_validation do + attrs.each do |attr| + input = public_send(attr) # rubocop: disable GitlabSecurity/PublicSend + + public_send("#{attr}=", self.class.sanitize(input)) # rubocop: disable GitlabSecurity/PublicSend + end + end + + validates_each(*attrs) do |record, attr, input| + # We reject pre-escaped HTML fragments as invalid to avoid saving them + # to the database. + unless input.to_s == CGI.unescapeHTML(input.to_s) + record.errors.add(attr, 'cannot contain escaped HTML entities') + end + end + end + end + end +end diff --git a/app/models/concerns/sortable_title.rb b/app/models/concerns/sortable_title.rb new file mode 100644 index 00000000000..7c5cad17f4c --- /dev/null +++ b/app/models/concerns/sortable_title.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module SortableTitle + extend ActiveSupport::Concern + + included do + scope :order_title_asc, -> { reorder(Arel::Nodes::Ascending.new(arel_table[:title].lower)) } + scope :order_title_desc, -> { reorder(Arel::Nodes::Descending.new(arel_table[:title].lower)) } + end + + class_methods do + def simple_sorts + super.merge( + { + 'title_asc' => -> { order_title_asc }, + 'title_desc' => -> { order_title_desc } + } + ) + end + end +end diff --git a/app/models/concerns/taggable_queries.rb b/app/models/concerns/taggable_queries.rb index cba2e93a86d..06799f0a9f4 100644 --- a/app/models/concerns/taggable_queries.rb +++ b/app/models/concerns/taggable_queries.rb @@ -3,6 +3,10 @@ module TaggableQueries extend ActiveSupport::Concern + MAX_TAGS_IDS = 50 + + TooManyTagsError = Class.new(StandardError) + class_methods do # context is a name `acts_as_taggable context` def arel_tag_names_array(context = :tags) @@ -34,4 +38,10 @@ module TaggableQueries where("EXISTS (?)", matcher) end end + + def tags_ids + tags.limit(MAX_TAGS_IDS).order('id ASC').pluck(:id).tap do |ids| + raise TooManyTagsError if ids.size >= MAX_TAGS_IDS + end + end end diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb new file mode 100644 index 00000000000..aaa7e2ae175 --- /dev/null +++ b/app/models/customer_relations/contact.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class CustomerRelations::Contact < ApplicationRecord + include StripAttribute + + self.table_name = "customer_relations_contacts" + + belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'group_id' + belongs_to :organization, optional: true + + strip_attributes! :phone, :first_name, :last_name + + enum state: { + inactive: 0, + active: 1 + } + + validates :group, presence: true + validates :phone, length: { maximum: 32 } + validates :first_name, presence: true, length: { maximum: 255 } + validates :last_name, presence: true, length: { maximum: 255 } + validates :email, length: { maximum: 255 } + validates :description, length: { maximum: 1024 } + validate :validate_email_format + + private + + def validate_email_format + return unless email + + self.errors.add(:email, I18n.t(:invalid, scope: 'valid_email.validations.email')) unless ValidateEmail.valid?(self.email) + end +end diff --git a/app/models/customer_relations/organization.rb b/app/models/customer_relations/organization.rb index caf1cd68cc5..a18d3ab8148 100644 --- a/app/models/customer_relations/organization.rb +++ b/app/models/customer_relations/organization.rb @@ -1,11 +1,13 @@ # frozen_string_literal: true class CustomerRelations::Organization < ApplicationRecord + include StripAttribute + self.table_name = "customer_relations_organizations" belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'group_id' - before_validation :strip_whitespace! + strip_attributes! :name enum state: { inactive: 0, @@ -22,10 +24,4 @@ class CustomerRelations::Organization < ApplicationRecord where(group: group_id) .where('LOWER(name) = LOWER(?)', name) end - - private - - def strip_whitespace! - name&.strip! - end end diff --git a/app/models/dependency_proxy/blob.rb b/app/models/dependency_proxy/blob.rb index 3a81112340a..5de6b1cf28f 100644 --- a/app/models/dependency_proxy/blob.rb +++ b/app/models/dependency_proxy/blob.rb @@ -8,6 +8,9 @@ class DependencyProxy::Blob < ApplicationRecord validates :group, presence: true validates :file, presence: true validates :file_name, presence: true + validates :status, presence: true + + enum status: { default: 0, expired: 1 } mount_file_store_uploader DependencyProxy::FileUploader diff --git a/app/models/dependency_proxy/image_ttl_group_policy.rb b/app/models/dependency_proxy/image_ttl_group_policy.rb new file mode 100644 index 00000000000..5a1b8cb8f1f --- /dev/null +++ b/app/models/dependency_proxy/image_ttl_group_policy.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class DependencyProxy::ImageTtlGroupPolicy < ApplicationRecord + self.primary_key = :group_id + + belongs_to :group + + validates :group, presence: true + validates :enabled, inclusion: { in: [true, false] } + validates :ttl, numericality: { greater_than: 0 }, allow_nil: true +end diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index d613d5708f0..15e5137b50a 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -9,6 +9,9 @@ class DependencyProxy::Manifest < ApplicationRecord validates :file, presence: true validates :file_name, presence: true validates :digest, presence: true + validates :status, presence: true + + enum status: { default: 0, expired: 1 } mount_file_store_uploader DependencyProxy::FileUploader diff --git a/app/models/deploy_keys_project.rb b/app/models/deploy_keys_project.rb index 40c66d5bc4c..363ef0b1c9a 100644 --- a/app/models/deploy_keys_project.rb +++ b/app/models/deploy_keys_project.rb @@ -3,7 +3,6 @@ class DeployKeysProject < ApplicationRecord belongs_to :project, inverse_of: :deploy_keys_projects belongs_to :deploy_key, inverse_of: :deploy_keys_projects - scope :without_project_deleted, -> { joins(:project).where(projects: { pending_delete: false }) } scope :in_project, ->(project) { where(project: project) } scope :with_write_access, -> { where(can_push: true) } diff --git a/app/models/design_management/action.rb b/app/models/design_management/action.rb index ecd7973a523..b9df2873a73 100644 --- a/app/models/design_management/action.rb +++ b/app/models/design_management/action.rb @@ -17,6 +17,8 @@ module DesignManagement # we assume sequential ordering. scope :ordered, -> { order(version_id: :asc) } + scope :by_design, -> (design) { where(design: design) } + scope :by_event, -> (event) { where(event: event) } # For each design, only select the most recent action scope :most_recent, -> do diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb index c8a0773cc5b..6ebac6384bc 100644 --- a/app/models/diff_note.rb +++ b/app/models/diff_note.rb @@ -22,7 +22,7 @@ class DiffNote < Note validate :verify_supported, unless: :importing? before_validation :set_line_code, if: :on_text?, unless: :importing? - after_save :keep_around_commits, unless: :importing? + after_save :keep_around_commits, unless: -> { importing? || skip_keep_around_commits } NoteDiffFileCreationError = Class.new(StandardError) @@ -115,6 +115,20 @@ class DiffNote < Note position&.multiline? end + def shas + [ + self.original_position.base_sha, + self.original_position.start_sha, + self.original_position.head_sha + ].tap do |a| + if self.position != self.original_position + a << self.position.base_sha + a << self.position.start_sha + a << self.position.head_sha + end + end + end + private def enqueue_diff_file_creation_job @@ -173,18 +187,6 @@ class DiffNote < Note end def keep_around_commits - shas = [ - self.original_position.base_sha, - self.original_position.start_sha, - self.original_position.head_sha - ] - - if self.position != self.original_position - shas << self.position.base_sha - shas << self.position.start_sha - shas << self.position.head_sha - end - repository.keep_around(*shas) end diff --git a/app/models/environment.rb b/app/models/environment.rb index 963249c018a..48522a23068 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -27,11 +27,10 @@ class Environment < ApplicationRecord has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :environment has_one :last_deployment, -> { success.distinct_on_environment }, class_name: 'Deployment', inverse_of: :environment - has_one :last_deployable, through: :last_deployment, source: 'deployable', source_type: 'CommitStatus' - has_one :last_pipeline, through: :last_deployable, source: 'pipeline' has_one :last_visible_deployment, -> { visible.distinct_on_environment }, inverse_of: :environment, class_name: 'Deployment' - has_one :last_visible_deployable, through: :last_visible_deployment, source: 'deployable', source_type: 'CommitStatus' - has_one :last_visible_pipeline, through: :last_visible_deployable, source: 'pipeline' + has_one :last_visible_deployable, through: :last_visible_deployment, source: 'deployable', source_type: 'CommitStatus', disable_joins: -> { ::Feature.enabled?(:environment_last_visible_pipeline_disable_joins, default_enabled: :yaml) } + has_one :last_visible_pipeline, through: :last_visible_deployable, source: 'pipeline', disable_joins: -> { ::Feature.enabled?(:environment_last_visible_pipeline_disable_joins, default_enabled: :yaml) } + has_one :upcoming_deployment, -> { running.distinct_on_environment }, class_name: 'Deployment', inverse_of: :environment has_one :latest_opened_most_severe_alert, -> { order_severity_with_open_prometheus_alert }, class_name: 'AlertManagement::Alert', inverse_of: :environment @@ -77,6 +76,7 @@ class Environment < ApplicationRecord scope :in_review_folder, -> { where(environment_type: "review") } scope :for_name, -> (name) { where(name: name) } scope :preload_cluster, -> { preload(last_deployment: :cluster) } + scope :preload_project, -> { preload(:project) } scope :auto_stoppable, -> (limit) { available.where('auto_stop_at < ?', Time.zone.now).limit(limit) } scope :auto_deletable, -> (limit) { stopped.where('auto_delete_at < ?', Time.zone.now).limit(limit) } @@ -132,6 +132,10 @@ class Environment < ApplicationRecord state :available state :stopped + before_transition any => :stopped do |environment| + environment.auto_stop_at = nil + end + after_transition do |environment| environment.expire_etag_cache end @@ -168,33 +172,6 @@ class Environment < ApplicationRecord end class << self - ## - # This method returns stop actions (jobs) for multiple environments within one - # query. It's useful to avoid N+1 problem. - # - # NOTE: The count of environments should be small~medium (e.g. < 5000) - def stop_actions - cte = cte_for_deployments_with_stop_action - ci_builds = Ci::Build.arel_table - - inner_join_stop_actions = ci_builds.join(cte.table).on( - ci_builds[:project_id].eq(cte.table[:project_id]) - .and(ci_builds[:ref].eq(cte.table[:ref])) - .and(ci_builds[:name].eq(cte.table[:on_stop])) - ).join_sources - - pipeline_ids = ci_builds.join(cte.table).on( - ci_builds[:id].eq(cte.table[:deployable_id]) - ).project(:commit_id) - - Ci::Build.joins(inner_join_stop_actions) - .with(cte.to_arel) - .where(ci_builds[:commit_id].in(pipeline_ids)) - .where(status: Ci::HasStatus::BLOCKED_STATUS) - .preload_project_and_pipeline_project - .preload(:user, :metadata, :deployment) - end - def count_by_state environments_count_by_state = group(:state).count @@ -202,15 +179,35 @@ class Environment < ApplicationRecord count_hash[state] = environments_count_by_state[state.to_s] || 0 end end + end + + def last_deployable + last_deployment&.deployable + end - private + # NOTE: Below assocation overrides is a workaround for issue https://gitlab.com/gitlab-org/gitlab/-/issues/339908 + # It helps to avoid cross joins with the CI database. + # Caveat: It also overrides and losses the default AR caching mechanism. + # Read - https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68870#note_677227727 - def cte_for_deployments_with_stop_action - Gitlab::SQL::CTE.new(:deployments_with_stop_action, - Deployment.where(environment_id: select(:id)) - .distinct_on_environment - .stoppable) - end + # NOTE: Association Preloads does not use the overriden definitions below. + # Association Preloads when preloading uses the original definitions from the relationships above. + # https://github.com/rails/rails/blob/75ac626c4e21129d8296d4206a1960563cc3d4aa/activerecord/lib/active_record/associations/preloader.rb#L158 + # But after preloading, when they are called it is using the overriden methods below. + # So we are checking for `association_cached?(:association_name)` in the overridden methods and calling `super` which inturn fetches the preloaded values. + + # Overriding association + def last_visible_deployable + return super if association_cached?(:last_visible_deployable) || ::Feature.disabled?(:environment_last_visible_pipeline_disable_joins, default_enabled: :yaml) + + last_visible_deployment&.deployable + end + + # Overriding association + def last_visible_pipeline + return super if association_cached?(:last_visible_pipeline) || ::Feature.disabled?(:environment_last_visible_pipeline_disable_joins, default_enabled: :yaml) + + last_visible_deployable&.pipeline end def clear_prometheus_reactive_cache!(query_name) diff --git a/app/models/environment_status.rb b/app/models/environment_status.rb index 07c0983f239..3be7af2e4bf 100644 --- a/app/models/environment_status.rb +++ b/app/models/environment_status.rb @@ -100,11 +100,13 @@ class EnvironmentStatus def self.build_environments_status(mr, user, pipeline) return [] unless pipeline - pipeline.environments_in_self_and_descendants.includes(:project).available.map do |environment| - next unless Ability.allowed?(user, :read_environment, environment) + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/340781') do + pipeline.environments_in_self_and_descendants.includes(:project).available.map do |environment| + next unless Ability.allowed?(user, :read_environment, environment) - EnvironmentStatus.new(pipeline.project, environment, mr, pipeline.sha) - end.compact + EnvironmentStatus.new(pipeline.project, environment, mr, pipeline.sha) + end.compact + end end private_class_method :build_environments_status end diff --git a/app/models/error_tracking/client_key.rb b/app/models/error_tracking/client_key.rb index 9d12c0ed6f1..8e59f6f9ecb 100644 --- a/app/models/error_tracking/client_key.rb +++ b/app/models/error_tracking/client_key.rb @@ -14,9 +14,13 @@ class ErrorTracking::ClientKey < ApplicationRecord find_by(public_key: key) end + def sentry_dsn + @sentry_dsn ||= ErrorTracking::Collector::Dsn.build_url(public_key, project_id) + end + private def generate_key - self.public_key = "glet_#{SecureRandom.hex}" + self.public_key ||= "glet_#{SecureRandom.hex}" end end diff --git a/app/models/error_tracking/error.rb b/app/models/error_tracking/error.rb index 32932c4d045..39ecc487806 100644 --- a/app/models/error_tracking/error.rb +++ b/app/models/error_tracking/error.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ErrorTracking::Error < ApplicationRecord + include Sortable + belongs_to :project has_many :events, class_name: 'ErrorTracking::ErrorEvent' @@ -22,11 +24,28 @@ class ErrorTracking::Error < ApplicationRecord def self.report_error(name:, description:, actor:, platform:, timestamp:) safe_find_or_create_by( name: name, - description: description, actor: actor, platform: platform - ) do |error| - error.update!(last_seen_at: timestamp) + ).tap do |error| + error.update!( + # Description can contain object id, so it can't be + # used as a group criteria for similar errors. + description: description, + last_seen_at: timestamp + ) + end + end + + def self.sort_by_attribute(method) + case method.to_s + when 'last_seen' + order(last_seen_at: :desc) + when 'first_seen' + order(first_seen_at: :desc) + when 'frequency' + order(events_count: :desc) + else + order_id_desc end end diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb index c5a77427588..dd5ce9f7387 100644 --- a/app/models/error_tracking/project_error_tracking_setting.rb +++ b/app/models/error_tracking/project_error_tracking_setting.rb @@ -51,7 +51,7 @@ module ErrorTracking end def integrated_client? - integrated && ::Feature.enabled?(:integrated_error_tracking, project) + integrated end def api_url=(value) diff --git a/app/models/event.rb b/app/models/event.rb index f6174589a84..d6588699d27 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -9,6 +9,9 @@ class Event < ApplicationRecord include Gitlab::Utils::StrongMemoize include UsageStatistics include ShaAttribute + include IgnorableColumns + + ignore_columns :id_convert_to_bigint, remove_with: '14.5', remove_after: '2021-10-22' default_scope { reorder(nil) } # rubocop:disable Cop/DefaultScope diff --git a/app/models/group.rb b/app/models/group.rb index f6b45a755e4..437c750afa6 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -18,6 +18,10 @@ class Group < Namespace include EachBatch include BulkMemberAccessLoad + def self.sti_name + 'Group' + end + has_many :all_group_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'GroupMember' # rubocop:disable Cop/ActiveRecordDependent has_many :group_members, -> { where(requested_at: nil).where.not(members: { access_level: Gitlab::Access::MINIMAL_ACCESS }) }, dependent: :destroy, as: :source # rubocop:disable Cop/ActiveRecordDependent alias_method :members, :group_members @@ -74,13 +78,16 @@ class Group < Namespace has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :dependency_proxy_setting, class_name: 'DependencyProxy::GroupSetting' + has_one :dependency_proxy_image_ttl_policy, class_name: 'DependencyProxy::ImageTtlGroupPolicy' has_many :dependency_proxy_blobs, class_name: 'DependencyProxy::Blob' has_many :dependency_proxy_manifests, class_name: 'DependencyProxy::Manifest' # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads has_many :debian_distributions, class_name: 'Packages::Debian::GroupDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - delegate :prevent_sharing_groups_outside_hierarchy, :new_user_signups_cap, to: :namespace_settings + has_many :group_callouts, class_name: 'Users::GroupCallout', foreign_key: :group_id + + delegate :prevent_sharing_groups_outside_hierarchy, :new_user_signups_cap, :setup_for_company, :jobs_to_be_done, to: :namespace_settings accepts_nested_attributes_for :variables, allow_destroy: true @@ -260,6 +267,15 @@ class Group < Namespace Gitlab::UrlBuilder.build(self, only_path: only_path) end + def dependency_proxy_image_prefix + # The namespace path can include uppercase letters, which + # Docker doesn't allow. The proxy expects it to be downcased. + url = "#{web_url.downcase}#{DependencyProxy::URL_SUFFIX}" + + # Docker images do not include the protocol + url.partition('//').last + end + def human_name full_name end @@ -296,7 +312,7 @@ class Group < Namespace end def add_users(users, access_level, current_user: nil, expires_at: nil) - Members::Groups::CreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass + Members::Groups::BulkCreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass self, users, access_level, @@ -642,6 +658,10 @@ class Group < Namespace members.owners.connected_to_user.order_recent_sign_in.limit(Member::ACCESS_REQUEST_APPROVERS_TO_BE_NOTIFIED_LIMIT) end + def membership_locked? + false # to support project and group calling this as 'source' + end + def supports_events? false end @@ -734,6 +754,22 @@ class Group < Namespace Timelog.in_group(self) end + def cached_issues_state_count_enabled? + Feature.enabled?(:cached_issues_state_count, self, default_enabled: :yaml) + end + + def organizations + ::CustomerRelations::Organization.where(group_id: self.id) + end + + def contacts + ::CustomerRelations::Contact.where(group_id: self.id) + end + + def dependency_proxy_image_ttl_policy + super || build_dependency_proxy_image_ttl_policy + end + private def max_member_access(user_ids) @@ -822,9 +858,15 @@ class Group < Namespace end def self.groups_including_descendants_by(group_ids) - Gitlab::ObjectHierarchy - .new(Group.where(id: group_ids)) + groups = Group.where(id: group_ids) + + if Feature.enabled?(:linear_group_including_descendants_by, default_enabled: :yaml) + groups.self_and_descendants + else + Gitlab::ObjectHierarchy + .new(groups) .base_and_descendants + end end def disable_shared_runners! diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index 9a78fe3971c..cb5c1ac48cd 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -80,6 +80,8 @@ class WebHook < ApplicationRecord end def backoff! + return if backoff_count >= MAX_FAILURES && disabled_until && disabled_until > Time.current + assign_attributes(disabled_until: next_backoff.from_now, backoff_count: backoff_count.succ.clamp(0, MAX_FAILURES)) save(validate: false) end diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index 09a60e9dd10..9565dae08b5 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -13,7 +13,7 @@ class InstanceConfiguration { ssh_algorithms_hashes: ssh_algorithms_hashes, host: host, gitlab_pages: gitlab_pages, - gitlab_ci: gitlab_ci, + size_limits: size_limits, package_file_size_limits: package_file_size_limits, rate_limits: rate_limits }.deep_symbolize_keys end @@ -38,11 +38,16 @@ class InstanceConfiguration rescue Resolv::ResolvError end - def gitlab_ci - Settings.gitlab_ci - .to_h - .merge(artifacts_max_size: { value: Gitlab::CurrentSettings.max_artifacts_size.megabytes, - default: 100.megabytes }) + def size_limits + { + max_attachment_size: application_settings[:max_attachment_size].megabytes, + receive_max_input_size: application_settings[:receive_max_input_size]&.megabytes, + max_import_size: application_settings[:max_import_size] > 0 ? application_settings[:max_import_size].megabytes : nil, + diff_max_patch_bytes: application_settings[:diff_max_patch_bytes].bytes, + max_artifacts_size: application_settings[:max_artifacts_size].megabytes, + max_pages_size: application_settings[:max_pages_size] > 0 ? application_settings[:max_pages_size].megabytes : nil, + snippet_size_limit: application_settings[:snippet_size_limit]&.bytes + } end def package_file_size_limits diff --git a/app/models/integration.rb b/app/models/integration.rb index a9c865569d0..158764bb783 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -274,7 +274,7 @@ class Integration < ApplicationRecord end def self.closest_group_integration(type, scope) - group_ids = scope.ancestors(hierarchy_order: :asc).select(:id) + group_ids = scope.ancestors(hierarchy_order: :asc).reselect(:id) array = group_ids.to_sql.present? ? "array(#{group_ids.to_sql})" : 'ARRAY[]' where(type: type, group_id: group_ids, inherit_from_id: nil) @@ -357,6 +357,10 @@ class Integration < ApplicationRecord [] end + def password_fields + fields.select { |f| f[:type] == 'password' }.pluck(:name) + end + # Expose a list of fields in the JSON endpoint. # # This list is used in `Integration#as_json(only: json_fields)`. diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index 5eae8bce92a..c6335782b5e 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -253,3 +253,5 @@ module Integrations end end end + +Integrations::BaseChatNotification.prepend_mod_with('Integrations::BaseChatNotification') diff --git a/app/models/integrations/datadog.rb b/app/models/integrations/datadog.rb index 6422f6bddab..72e0ca22ac2 100644 --- a/app/models/integrations/datadog.rb +++ b/app/models/integrations/datadog.rb @@ -7,7 +7,7 @@ module Integrations extend Gitlab::Utils::Override DEFAULT_DOMAIN = 'datadoghq.com' - URL_TEMPLATE = 'https://webhooks-http-intake.logs.%{datadog_domain}/api/v2/webhook' + URL_TEMPLATE = 'https://webhook-intake.%{datadog_domain}/api/v2/webhook' URL_API_KEYS_DOCS = "https://docs.#{DEFAULT_DOMAIN}/account_management/api-app-keys/" SUPPORTED_EVENTS = %w[ diff --git a/app/models/integrations/prometheus.rb b/app/models/integrations/prometheus.rb index 54cb823d606..5746343c31c 100644 --- a/app/models/integrations/prometheus.rb +++ b/app/models/integrations/prometheus.rb @@ -76,7 +76,7 @@ module Integrations name: 'google_iap_audience_client_id', title: 'Google IAP Audience Client ID', placeholder: s_('PrometheusService|IAP_CLIENT_ID.apps.googleusercontent.com'), - help: s_('PrometheusService|PrometheusService|The ID of the IAP-secured resource.'), + help: s_('PrometheusService|The ID of the IAP-secured resource.'), autocomplete: 'off', required: false }, diff --git a/app/models/integrations/slack_slash_commands.rb b/app/models/integrations/slack_slash_commands.rb index ff1f806df45..72e3c4a8cbc 100644 --- a/app/models/integrations/slack_slash_commands.rb +++ b/app/models/integrations/slack_slash_commands.rb @@ -9,7 +9,7 @@ module Integrations end def description - "Perform common operations in Slack" + "Perform common operations in Slack." end def self.to_param diff --git a/app/models/integrations/zentao.rb b/app/models/integrations/zentao.rb new file mode 100644 index 00000000000..68c02f54c61 --- /dev/null +++ b/app/models/integrations/zentao.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Integrations + class Zentao < Integration + data_field :url, :api_url, :api_token, :zentao_product_xid + + validates :url, public_url: true, presence: true, if: :activated? + validates :api_url, public_url: true, allow_blank: true + validates :api_token, presence: true, if: :activated? + validates :zentao_product_xid, presence: true, if: :activated? + + def data_fields + zentao_tracker_data || self.build_zentao_tracker_data + end + + def title + self.class.name.demodulize + end + + def description + s_("ZentaoIntegration|Use Zentao as this project's issue tracker.") + end + + def self.to_param + name.demodulize.downcase + end + + def test(*_args) + client.ping + end + + def self.supported_events + %w() + end + + def self.supported_event_actions + %w() + end + + def fields + [ + { + type: 'text', + name: 'url', + title: s_('ZentaoIntegration|Zentao Web URL'), + placeholder: 'https://www.zentao.net', + help: s_('ZentaoIntegration|Base URL of the Zentao instance.'), + required: true + }, + { + type: 'text', + name: 'api_url', + title: s_('ZentaoIntegration|Zentao API URL (optional)'), + help: s_('ZentaoIntegration|If different from Web URL.') + }, + { + type: 'password', + name: 'api_token', + title: s_('ZentaoIntegration|Zentao API token'), + non_empty_password_title: s_('ZentaoIntegration|Enter API token'), + required: true + }, + { + type: 'text', + name: 'zentao_product_xid', + title: s_('ZentaoIntegration|Zentao Product ID'), + required: true + } + ] + end + + private + + def client + @client ||= ::Gitlab::Zentao::Client.new(self) + end + end +end diff --git a/app/models/integrations/zentao_tracker_data.rb b/app/models/integrations/zentao_tracker_data.rb new file mode 100644 index 00000000000..468e4e5d7d7 --- /dev/null +++ b/app/models/integrations/zentao_tracker_data.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Integrations + class ZentaoTrackerData < ApplicationRecord + belongs_to :integration, inverse_of: :zentao_tracker_data, foreign_key: :integration_id + delegate :activated?, to: :integration + validates :integration, presence: true + + scope :encryption_options, -> do + { + key: Settings.attr_encrypted_db_key_base_32, + encode: true, + mode: :per_attribute_iv, + algorithm: 'aes-256-gcm' + } + end + + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + attr_encrypted :zentao_product_xid, encryption_options + attr_encrypted :api_token, encryption_options + end +end diff --git a/app/models/internal_id.rb b/app/models/internal_id.rb index a54de3c82d1..10d24ab50b2 100644 --- a/app/models/internal_id.rb +++ b/app/models/internal_id.rb @@ -4,7 +4,7 @@ # generated for a given scope and usage. # # The monotone sequence may be broken if an ID is explicitly provided -# to `.track_greatest_and_save!` or `#track_greatest`. +# to `#track_greatest`. # # For example, issues use their project to scope internal ids: # In that sense, scope is "project" and usage is "issues". @@ -29,32 +29,6 @@ class InternalId < ApplicationRecord where(**scope, usage: usage) end - # Increments #last_value and saves the record - # - # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL). - # As such, the increment is atomic and safe to be called concurrently. - def increment_and_save! - update_and_save { self.last_value = (last_value || 0) + 1 } - end - - # Increments #last_value with new_value if it is greater than the current, - # and saves the record - # - # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL). - # As such, the increment is atomic and safe to be called concurrently. - def track_greatest_and_save!(new_value) - update_and_save { self.last_value = [last_value || 0, new_value].max } - end - - private - - def update_and_save(&block) - lock! - yield - save! - last_value - end - class << self def track_greatest(subject, scope, usage, new_value, init) build_generator(subject, scope, usage, init).track_greatest(new_value) @@ -99,132 +73,7 @@ class InternalId < ApplicationRecord private def build_generator(subject, scope, usage, init = nil) - if Feature.enabled?(:generate_iids_without_explicit_locking) - ImplicitlyLockingInternalIdGenerator.new(subject, scope, usage, init) - else - InternalIdGenerator.new(subject, scope, usage, init) - end - end - end - - class InternalIdGenerator - # Generate next internal id for a given scope and usage. - # - # For currently supported usages, see #usage enum. - # - # The method implements a locking scheme that has the following properties: - # 1) Generated sequence of internal ids is unique per (scope and usage) - # 2) The method is thread-safe and may be used in concurrent threads/processes. - # 3) The generated sequence is gapless. - # 4) In the absence of a record in the internal_ids table, one will be created - # and last_value will be calculated on the fly. - # - # subject: The instance or class we're generating an internal id for. - # scope: Attributes that define the scope for id generation. - # Valid keys are `project/project_id` and `namespace/namespace_id`. - # usage: Symbol to define the usage of the internal id, see InternalId.usages - # init: Proc that accepts the subject and the scope and returns Integer|NilClass - attr_reader :subject, :scope, :scope_attrs, :usage, :init - - def initialize(subject, scope, usage, init = nil) - @subject = subject - @scope = scope - @usage = usage - @init = init - - raise ArgumentError, 'Scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty? - - unless InternalId.usages.has_key?(usage.to_s) - raise ArgumentError, "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages" - end - end - - # Generates next internal id and returns it - # init: Block that gets called to initialize InternalId record if not present - # Make sure to not throw exceptions in the absence of records (if this is expected). - def generate - InternalId.internal_id_transactions_increment(operation: :generate, usage: usage) - - subject.transaction do - # Create a record in internal_ids if one does not yet exist - # and increment its last value - # - # Note this will acquire a ROW SHARE lock on the InternalId record - record.increment_and_save! - end - end - - # Reset tries to rewind to `value-1`. This will only succeed, - # if `value` stored in database is equal to `last_value`. - # value: The expected last_value to decrement - def reset(value) - return false unless value - - InternalId.internal_id_transactions_increment(operation: :reset, usage: usage) - - updated = - InternalId - .where(**scope, usage: usage_value) - .where(last_value: value) - .update_all('last_value = last_value - 1') - - updated > 0 - end - - # Create a record in internal_ids if one does not yet exist - # and set its new_value if it is higher than the current last_value - # - # Note this will acquire a ROW SHARE lock on the InternalId record - - def track_greatest(new_value) - InternalId.internal_id_transactions_increment(operation: :track_greatest, usage: usage) - - subject.transaction do - record.track_greatest_and_save!(new_value) - end - end - - def record - @record ||= (lookup || create_record) - end - - def with_lock(&block) - InternalId.internal_id_transactions_increment(operation: :with_lock, usage: usage) - - record.with_lock(&block) - end - - private - - # Retrieve InternalId record for (project, usage) combination, if it exists - def lookup - InternalId.find_by(**scope, usage: usage_value) - end - - def usage_value - @usage_value ||= InternalId.usages[usage.to_s] - end - - # Create InternalId record for (scope, usage) combination, if it doesn't exist - # - # We blindly insert without synchronization. If another process - # was faster in doing this, we'll realize once we hit the unique key constraint - # violation. We can safely roll-back the nested transaction and perform - # a lookup instead to retrieve the record. - def create_record - raise ArgumentError, 'Cannot initialize without init!' unless init - - instance = subject.is_a?(::Class) ? nil : subject - - subject.transaction(requires_new: true) do - InternalId.create!( - **scope, - usage: usage_value, - last_value: init.call(instance, scope) || 0 - ) - end - rescue ActiveRecord::RecordNotUnique - lookup + ImplicitlyLockingInternalIdGenerator.new(subject, scope, usage, init) end end @@ -247,6 +96,8 @@ class InternalId < ApplicationRecord # init: Proc that accepts the subject and the scope and returns Integer|NilClass attr_reader :subject, :scope, :scope_attrs, :usage, :init + RecordAlreadyExists = Class.new(StandardError) + def initialize(subject, scope, usage, init = nil) @subject = subject @scope = scope @@ -270,10 +121,8 @@ class InternalId < ApplicationRecord return next_iid if next_iid - create_record!(subject, scope, usage, init) do |iid| - iid.last_value += 1 - end - rescue ActiveRecord::RecordNotUnique + create_record!(subject, scope, usage, initial_value(subject, scope) + 1) + rescue RecordAlreadyExists retry end @@ -302,10 +151,8 @@ class InternalId < ApplicationRecord next_iid = update_record!(subject, scope, usage, function) return next_iid if next_iid - create_record!(subject, scope, usage, init) do |object| - object.last_value = [object.last_value, new_value].max - end - rescue ActiveRecord::RecordNotUnique + create_record!(subject, scope, usage, [initial_value(subject, scope), new_value].max) + rescue RecordAlreadyExists retry end @@ -317,27 +164,45 @@ class InternalId < ApplicationRecord stmt.set(arel_table[:last_value] => new_value) stmt.wheres = InternalId.filter_by(scope, usage).arel.constraints - ActiveRecord::Base.connection.insert(stmt, 'Update InternalId', 'last_value') # rubocop: disable Database/MultipleDatabases + InternalId.connection.insert(stmt, 'Update InternalId', 'last_value') end - def create_record!(subject, scope, usage, init) - raise ArgumentError, 'Cannot initialize without init!' unless init + def create_record!(subject, scope, usage, value) + scope[:project].save! if scope[:project] && !scope[:project].persisted? + scope[:namespace].save! if scope[:namespace] && !scope[:namespace].persisted? - instance = subject.is_a?(::Class) ? nil : subject + attributes = { + project_id: scope[:project]&.id || scope[:project_id], + namespace_id: scope[:namespace]&.id || scope[:namespace_id], + usage: usage_value, + last_value: value + } - subject.transaction(requires_new: true) do - last_value = init.call(instance, scope) || 0 + result = InternalId.insert(attributes) - internal_id = InternalId.create!(**scope, usage: usage, last_value: last_value) do |subject| - yield subject if block_given? - end + raise RecordAlreadyExists if result.empty? - internal_id.last_value - end + value end def arel_table InternalId.arel_table end + + def initial_value(subject, scope) + raise ArgumentError, 'Cannot initialize without init!' unless init + + # `init` computes the maximum based on actual records. We use the + # primary to make sure we have up to date results + Gitlab::Database::LoadBalancing::Session.current.use_primary do + instance = subject.is_a?(::Class) ? nil : subject + + init.call(instance, scope) || 0 + end + end + + def usage_value + @usage_value ||= InternalId.usages[usage.to_s] + end end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 48e3fdd51e9..e0b0c352c22 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -128,13 +128,15 @@ class Issue < ApplicationRecord } scope :with_issue_type, ->(types) { where(issue_type: types) } - scope :public_only, -> { where(confidential: false) } + scope :public_only, -> { + without_hidden.where(confidential: false) + } + scope :confidential_only, -> { where(confidential: true) } scope :without_hidden, -> { if Feature.enabled?(:ban_user_feature_flag) - where(id: joins('LEFT JOIN banned_users ON banned_users.user_id = issues.author_id WHERE banned_users.user_id IS NULL') - .select('issues.id')) + where('NOT EXISTS (?)', Users::BannedUser.select(1).where('issues.author_id = banned_users.user_id')) else all end @@ -323,6 +325,13 @@ class Issue < ApplicationRecord ) end + def self.column_order_id_asc + Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: 'id', + order_expression: arel_table[:id].asc + ) + end + def self.to_branch_name(*args) branch_name = args.map(&:to_s).each_with_index.map do |arg, i| arg.parameterize(preserve_case: i == 0).presence @@ -584,15 +593,9 @@ class Issue < ApplicationRecord confidential_changed?(from: true, to: false) end - # Ensure that the metrics association is safely created and respecting the unique constraint on issue_id override :ensure_metrics def ensure_metrics - if !association(:metrics).loaded? || metrics.blank? - metrics_record = Issue::Metrics.safe_find_or_create_by(issue: self) - self.metrics = metrics_record - end - - metrics.record! + Issue::Metrics.record!(self) end def record_create_action diff --git a/app/models/issue/metrics.rb b/app/models/issue/metrics.rb index 86523bbd023..25afd9bf58d 100644 --- a/app/models/issue/metrics.rb +++ b/app/models/issue/metrics.rb @@ -9,25 +9,30 @@ class Issue::Metrics < ApplicationRecord .or(where(arel_table['first_mentioned_in_commit_at'].gteq(timestamp))) } - def record! - if issue.milestone_id.present? && self.first_associated_with_milestone_at.blank? - self.first_associated_with_milestone_at = Time.current + class << self + def record!(issue) + now = connection.quote(Time.current) + first_associated_with_milestone_at = issue.milestone_id.present? ? now : 'NULL' + first_added_to_board_at = issue_assigned_to_list_label?(issue) ? now : 'NULL' + + sql = <<~SQL + INSERT INTO #{self.table_name} (issue_id, first_associated_with_milestone_at, first_added_to_board_at, created_at, updated_at) + VALUES (#{issue.id}, #{first_associated_with_milestone_at}, #{first_added_to_board_at}, NOW(), NOW()) + ON CONFLICT (issue_id) + DO UPDATE SET + first_associated_with_milestone_at = LEAST(#{self.table_name}.first_associated_with_milestone_at, EXCLUDED.first_associated_with_milestone_at), + first_added_to_board_at = LEAST(#{self.table_name}.first_added_to_board_at, EXCLUDED.first_added_to_board_at), + updated_at = NOW() + RETURNING id + SQL + + connection.execute(sql) end - if issue_assigned_to_list_label? && self.first_added_to_board_at.blank? - self.first_added_to_board_at = Time.current - end - - self.save - end + private - private - - def issue_assigned_to_list_label? - # Avoid another DB lookup when issue.labels are empty by adding a guard clause here - # We can't use issue.labels.empty? because that will cause a `Label Exists?` DB lookup - return false if issue.labels.length == 0 # rubocop:disable Style/ZeroLengthPredicate - - issue.labels.includes(:lists).any? { |label| label.lists.present? } + def issue_assigned_to_list_label?(issue) + issue.labels.joins(:lists).exists? + end end end diff --git a/app/models/loose_foreign_keys.rb b/app/models/loose_foreign_keys.rb new file mode 100644 index 00000000000..0f45c0b5568 --- /dev/null +++ b/app/models/loose_foreign_keys.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module LooseForeignKeys + def self.table_name_prefix + 'loose_foreign_keys_' + end +end diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb new file mode 100644 index 00000000000..a39d88b2e49 --- /dev/null +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class LooseForeignKeys::DeletedRecord < ApplicationRecord + extend SuppressCompositePrimaryKeyWarning + include PartitionedTable + + partitioned_by :created_at, strategy: :monthly, retain_for: 3.months, retain_non_empty_partitions: true + + scope :ordered_by_primary_keys, -> { order(:created_at, :deleted_table_name, :deleted_table_primary_key_value) } + + def self.load_batch(batch_size) + ordered_by_primary_keys + .limit(batch_size) + .to_a + end + + # Because the table has composite primary keys, the delete_all or delete methods are not going to work. + # This method implements deletion that benefits from the primary key index, example: + # + # > DELETE + # > FROM "loose_foreign_keys_deleted_records" + # > WHERE (created_at, + # > deleted_table_name, + # > deleted_table_primary_key_value) IN + # > (SELECT created_at::TIMESTAMP WITH TIME ZONE, + # > deleted_table_name, + # > deleted_table_primary_key_value + # > FROM (VALUES (LIST_OF_VALUES)) AS primary_key_values (created_at, deleted_table_name, deleted_table_primary_key_value)) + def self.delete_records(records) + values = records.pluck(:created_at, :deleted_table_name, :deleted_table_primary_key_value) + + primary_keys = connection.primary_keys(table_name).join(', ') + + primary_keys_with_type_cast = [ + Arel.sql('created_at::timestamp with time zone'), + Arel.sql('deleted_table_name'), + Arel.sql('deleted_table_primary_key_value') + ] + + value_list = Arel::Nodes::ValuesList.new(values) + + # (SELECT primary keys FROM VALUES) + inner_query = Arel::SelectManager.new + inner_query.from("#{Arel::Nodes::Grouping.new([value_list]).as('primary_key_values').to_sql} (#{primary_keys})") + inner_query.projections = primary_keys_with_type_cast + + where(Arel::Nodes::Grouping.new([Arel.sql(primary_keys)]).in(inner_query)).delete_all + end +end diff --git a/app/models/member.rb b/app/models/member.rb index 397e60be3a8..beb4c05f2a6 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -147,7 +147,6 @@ class Member < ApplicationRecord scope :owners, -> { active.where(access_level: OWNER) } scope :owners_and_maintainers, -> { active.where(access_level: [OWNER, MAINTAINER]) } scope :with_user, -> (user) { where(user: user) } - scope :with_user_by_email, -> (email) { left_join_users.where(users: { email: email } ) } scope :preload_user_and_notification_settings, -> { preload(user: :notification_settings) } @@ -278,12 +277,14 @@ class Member < ApplicationRecord def accept_invite!(new_user) return false unless invite? + return false unless new_user + + self.user = new_user + return false unless self.user.save self.invite_token = nil self.invite_accepted_at = Time.current.utc - self.user = new_user - saved = self.save after_accept_invite if saved diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index b45c0b6a0cc..72cb831cc88 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -44,7 +44,7 @@ class ProjectMember < Member project_ids.each do |project_id| project = Project.find(project_id) - Members::Projects::CreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass + Members::Projects::BulkCreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass project, users, access_level, diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index a090ac87cc9..db49ec6f412 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -615,8 +615,8 @@ class MergeRequest < ApplicationRecord context_commits.count end - def commits(limit: nil) - return merge_request_diff.commits(limit: limit) if merge_request_diff.persisted? + def commits(limit: nil, load_from_gitaly: false) + return merge_request_diff.commits(limit: limit, load_from_gitaly: load_from_gitaly) if merge_request_diff.persisted? commits_arr = if compare_commits reversed_commits = compare_commits.reverse @@ -628,8 +628,8 @@ class MergeRequest < ApplicationRecord CommitCollection.new(source_project, commits_arr, source_branch) end - def recent_commits - commits(limit: MergeRequestDiff::COMMITS_SAFE_SIZE) + def recent_commits(load_from_gitaly: false) + commits(limit: MergeRequestDiff::COMMITS_SAFE_SIZE, load_from_gitaly: load_from_gitaly) end def commits_count @@ -1349,7 +1349,9 @@ class MergeRequest < ApplicationRecord def has_ci? return false if has_no_commits? - !!(head_pipeline_id || all_pipelines.any? || source_project&.ci_integration) + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336891') do + !!(head_pipeline_id || all_pipelines.any? || source_project&.ci_integration) + end end def branch_missing? @@ -1835,15 +1837,10 @@ class MergeRequest < ApplicationRecord Ability.allowed?(user, :push_code, source_project) end - def squash_in_progress? - # The source project can be deleted - return false unless source_project - - source_project.repository.squash_in_progress?(id) - end - def find_actual_head_pipeline - all_pipelines.for_sha_or_source_sha(diff_head_sha).first + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336891') do + all_pipelines.for_sha_or_source_sha(diff_head_sha).first + end end def etag_caching_enabled? @@ -1860,25 +1857,29 @@ class MergeRequest < ApplicationRecord override :ensure_metrics def ensure_metrics - # Backward compatibility: some merge request metrics records will not have target_project_id filled in. - # In that case the first `safe_find_or_create_by` will return false. - # The second finder call will be eliminated in https://gitlab.com/gitlab-org/gitlab/-/issues/233507 - metrics_record = MergeRequest::Metrics.safe_find_or_create_by(merge_request_id: id, target_project_id: target_project_id) || MergeRequest::Metrics.safe_find_or_create_by(merge_request_id: id) - - metrics_record.tap do |metrics_record| - # Make sure we refresh the loaded association object with the newly created/loaded item. - # This is needed in order to have the exact functionality than before. - # - # Example: - # - # merge_request.metrics.destroy - # merge_request.ensure_metrics - # merge_request.metrics # should return the metrics record and not nil - # merge_request.metrics.merge_request # should return the same MR record - - metrics_record.target_project_id = target_project_id - metrics_record.association(:merge_request).target = self - association(:metrics).target = metrics_record + if Feature.enabled?(:use_upsert_query_for_mr_metrics) + MergeRequest::Metrics.record!(self) + else + # Backward compatibility: some merge request metrics records will not have target_project_id filled in. + # In that case the first `safe_find_or_create_by` will return false. + # The second finder call will be eliminated in https://gitlab.com/gitlab-org/gitlab/-/issues/233507 + metrics_record = MergeRequest::Metrics.safe_find_or_create_by(merge_request_id: id, target_project_id: target_project_id) || MergeRequest::Metrics.safe_find_or_create_by(merge_request_id: id) + + metrics_record.tap do |metrics_record| + # Make sure we refresh the loaded association object with the newly created/loaded item. + # This is needed in order to have the exact functionality than before. + # + # Example: + # + # merge_request.metrics.destroy + # merge_request.ensure_metrics + # merge_request.metrics # should return the metrics record and not nil + # merge_request.metrics.merge_request # should return the same MR record + + metrics_record.target_project_id = target_project_id + metrics_record.association(:merge_request).target = self + association(:metrics).target = metrics_record + end end end @@ -1917,6 +1918,20 @@ class MergeRequest < ApplicationRecord end end + def lazy_upvotes_count + BatchLoader.for(id).batch(default_value: 0) do |ids, loader| + counts = AwardEmoji + .where(awardable_id: ids) + .upvotes + .group(:awardable_id) + .count + + counts.each do |id, count| + loader.call(id, count) + end + end + end + private def set_draft_status diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb index b9460afa8e7..b984228eb13 100644 --- a/app/models/merge_request/metrics.rb +++ b/app/models/merge_request/metrics.rb @@ -14,8 +14,23 @@ class MergeRequest::Metrics < ApplicationRecord scope :with_valid_time_to_merge, -> { where(arel_table[:merged_at].gt(arel_table[:created_at])) } scope :by_target_project, ->(project) { where(target_project_id: project) } - def self.time_to_merge_expression - Arel.sql('EXTRACT(epoch FROM SUM(AGE(merge_request_metrics.merged_at, merge_request_metrics.created_at)))') + class << self + def time_to_merge_expression + Arel.sql('EXTRACT(epoch FROM SUM(AGE(merge_request_metrics.merged_at, merge_request_metrics.created_at)))') + end + + def record!(mr) + sql = <<~SQL + INSERT INTO #{self.table_name} (merge_request_id, target_project_id, updated_at, created_at) + VALUES (#{mr.id}, #{mr.target_project_id}, NOW(), NOW()) + ON CONFLICT (merge_request_id) + DO UPDATE SET + target_project_id = EXCLUDED.target_project_id, + updated_at = NOW() + SQL + + connection.execute(sql) + end end private diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index bea75927b2c..d2b3ca753b1 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -288,9 +288,9 @@ class MergeRequestDiff < ApplicationRecord end end - def commits(limit: nil) - strong_memoize(:"commits_#{limit || 'all'}") do - load_commits(limit: limit) + def commits(limit: nil, load_from_gitaly: false) + strong_memoize(:"commits_#{limit || 'all'}_#{load_from_gitaly}") do + load_commits(limit: limit, load_from_gitaly: load_from_gitaly) end end @@ -700,9 +700,14 @@ class MergeRequestDiff < ApplicationRecord end end - def load_commits(limit: nil) - commits = merge_request_diff_commits.with_users.limit(limit) - .map { |commit| Commit.from_hash(commit.to_hash, project) } + def load_commits(limit: nil, load_from_gitaly: false) + if load_from_gitaly + commits = Gitlab::Git::Commit.batch_by_oid(repository, merge_request_diff_commits.limit(limit).map(&:sha)) + commits = Commit.decorate(commits, project) + else + commits = merge_request_diff_commits.with_users.limit(limit) + .map { |commit| Commit.from_hash(commit.to_hash, project) } + end CommitCollection .new(merge_request.source_project, commits, merge_request.source_branch) diff --git a/app/models/milestone.rb b/app/models/milestone.rb index 0e2842c3c11..868bee9961b 100644 --- a/app/models/milestone.rb +++ b/app/models/milestone.rb @@ -61,18 +61,10 @@ class Milestone < ApplicationRecord end def self.reference_pattern - if Feature.enabled?(:milestone_reference_pattern, default_enabled: :yaml) - new_reference_pattern - else - old_reference_pattern - end - end - - def self.new_reference_pattern # NOTE: The iid pattern only matches when all characters on the expression # are digits, so it will match %2 but not %2.1 because that's probably a # milestone name and we want it to be matched as such. - @new_reference_pattern ||= %r{ + @reference_pattern ||= %r{ (#{Project.reference_pattern})? #{Regexp.escape(reference_prefix)} (?: @@ -87,26 +79,6 @@ class Milestone < ApplicationRecord }x end - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/336268 - def self.old_reference_pattern - # NOTE: The iid pattern only matches when all characters on the expression - # are digits, so it will match %2 but not %2.1 because that's probably a - # milestone name and we want it to be matched as such. - @old_reference_pattern ||= %r{ - (#{Project.reference_pattern})? - #{Regexp.escape(reference_prefix)} - (?: - (?<milestone_iid> - \d+(?!\S\w)\b # Integer-based milestone iid, or - ) | - (?<milestone_name> - [^"\s]+\b | # String-based single-word milestone title, or - "[^"]+" # String-based multi-word milestone surrounded in quotes - ) - ) - }x - end - def self.link_reference_pattern @link_reference_pattern ||= super("milestones", /(?<milestone>\d+)/) end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 261639a4ec1..0c160cedb4d 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -18,6 +18,11 @@ class Namespace < ApplicationRecord ignore_column :delayed_project_removal, remove_with: '14.1', remove_after: '2021-05-22' + # Tells ActiveRecord not to store the full class name, in order to space some space + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69794 + self.store_full_sti_class = false + self.store_full_class_name = false + # Prevent users from creating unreasonably deep level of nesting. # The number 20 was taken based on maximum nesting level of # Android repo (15) + some extra backup. @@ -52,7 +57,7 @@ class Namespace < ApplicationRecord has_one :admin_note, inverse_of: :namespace accepts_nested_attributes_for :admin_note, update_only: true - validates :owner, presence: true, unless: ->(n) { n.type == "Group" } + validates :owner, presence: true, if: ->(n) { n.owner_required? } validates :name, presence: true, length: { maximum: 255 } @@ -131,6 +136,21 @@ class Namespace < ApplicationRecord attr_writer :root_ancestor, :emails_disabled_memoized class << self + def sti_class_for(type_name) + case type_name + when 'Group' + Group + when 'Project' + Namespaces::ProjectNamespace + when 'User' + # TODO: We create a normal Namespace until + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68894 is ready + Namespace + else + Namespace + end + end + def by_path(path) find_by('lower(path) = :value', value: path.downcase) end @@ -227,15 +247,27 @@ class Namespace < ApplicationRecord end def kind - type == 'Group' ? 'group' : 'user' + return 'group' if group? + return 'project' if project? + + 'user' # defaults to user + end + + def group? + type == Group.sti_name + end + + def project? + type == Namespaces::ProjectNamespace.sti_name end def user? - kind == 'user' + # That last bit ensures we're considered a user namespace as a default + type.nil? || type == Namespaces::UserNamespace.sti_name || !(group? || project?) end - def group? - type == 'Group' + def owner_required? + user? end def find_fork_of(project) @@ -498,17 +530,27 @@ class Namespace < ApplicationRecord def nesting_level_allowed if ancestors.count > Group::NUMBER_OF_ANCESTORS_ALLOWED - errors.add(:parent_id, 'has too deep level of nesting') + errors.add(:parent_id, _('has too deep level of nesting')) end end def validate_parent_type - return unless has_parent? + unless has_parent? + if project? + errors.add(:parent_id, _('must be set for a project namespace')) + end + + return + end + + if parent.project? + errors.add(:parent_id, _('project namespace cannot be the parent of another namespace')) + end if user? - errors.add(:parent_id, 'a user namespace cannot have a parent') + errors.add(:parent_id, _('cannot not be used for user namespace')) elsif group? - errors.add(:parent_id, 'a group cannot have a user namespace as its parent') if parent.user? + errors.add(:parent_id, _('user namespace cannot be the parent of another namespace')) if parent.user? end end diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb index 4a39bfebda0..170b29e9e21 100644 --- a/app/models/namespace_setting.rb +++ b/app/models/namespace_setting.rb @@ -2,6 +2,7 @@ class NamespaceSetting < ApplicationRecord include CascadingNamespaceSettingAttribute + include Sanitizable cascading_attr :delayed_project_removal @@ -16,12 +17,17 @@ class NamespaceSetting < ApplicationRecord before_validation :normalize_default_branch_name + enum jobs_to_be_done: { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5 }, _suffix: true + NAMESPACE_SETTINGS_PARAMS = [:default_branch_name, :delayed_project_removal, :lock_delayed_project_removal, :resource_access_token_creation_allowed, - :prevent_sharing_groups_outside_hierarchy, :new_user_signups_cap].freeze + :prevent_sharing_groups_outside_hierarchy, :new_user_signups_cap, + :setup_for_company, :jobs_to_be_done].freeze self.primary_key = :namespace_id + sanitizes! :default_branch_name + def prevent_sharing_groups_outside_hierarchy return super if namespace.root? @@ -31,11 +37,7 @@ class NamespaceSetting < ApplicationRecord private def normalize_default_branch_name - self.default_branch_name = if default_branch_name.blank? - nil - else - Sanitize.fragment(self.default_branch_name) - end + self.default_branch_name = default_branch_name.presence end def default_branch_name_content diff --git a/app/models/namespaces/project_namespace.rb b/app/models/namespaces/project_namespace.rb new file mode 100644 index 00000000000..d1806c1c088 --- /dev/null +++ b/app/models/namespaces/project_namespace.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Namespaces + class ProjectNamespace < Namespace + has_one :project, foreign_key: :project_namespace_id, inverse_of: :project_namespace + + validates :project, presence: true + + def self.sti_name + 'Project' + end + end +end diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 33e8c3e5172..d7130322ed1 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -74,7 +74,7 @@ module Namespaces return super unless use_traversal_ids_for_root_ancestor? strong_memoize(:root_ancestor) do - if parent.nil? + if parent_id.nil? self else Namespace.find_by(id: traversal_ids.first) @@ -176,13 +176,14 @@ module Namespaces # if you are walking up the ancestors or down the descendants. if hierarchy_order depth_sql = "ABS(#{traversal_ids.count} - array_length(traversal_ids, 1))" - skope = skope.select(skope.arel_table[Arel.star], "#{depth_sql} as depth") + skope = skope.select(skope.default_select_columns, "#{depth_sql} as depth") # The SELECT includes an extra depth attribute. We wrap the SQL in a # standard SELECT to avoid mismatched attribute errors when trying to # chain future ActiveRelation commands, and retain the ordering. skope = self.class .without_sti_condition .from(skope, self.class.table_name) + .select(skope.arel_table[Arel.star]) .order(depth: hierarchy_order) end diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index 90fae8ef35d..2da0e48c2da 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -15,6 +15,28 @@ module Namespaces select('namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id') end + def self_and_ancestors(include_self: true, hierarchy_order: nil) + return super unless use_traversal_ids_for_ancestor_scopes? + + records = unscoped + .without_sti_condition + .where(id: without_sti_condition.select('unnest(traversal_ids)')) + .order_by_depth(hierarchy_order) + .normal_select + + if include_self + records + else + records.where.not(id: all.as_ids) + end + end + + def self_and_ancestor_ids(include_self: true) + return super unless use_traversal_ids_for_ancestor_scopes? + + self_and_ancestors(include_self: include_self).as_ids + end + def self_and_descendants(include_self: true) return super unless use_traversal_ids? @@ -22,11 +44,7 @@ module Namespaces distinct = records.select('DISTINCT on(namespaces.id) namespaces.*') - # Produce a query of the form: SELECT * FROM namespaces; - # - # When we have queries that break this SELECT * format we can run in to errors. - # For example `SELECT DISTINCT on(...)` will fail when we chain a `.count` c - unscoped.without_sti_condition.from(distinct, :namespaces) + distinct.normal_select end def self_and_descendant_ids(include_self: true) @@ -42,12 +60,35 @@ module Namespaces unscope(where: :type) end + def order_by_depth(hierarchy_order) + return all unless hierarchy_order + + depth_order = hierarchy_order == :asc ? :desc : :asc + + all + .select(Arel.star, 'array_length(traversal_ids, 1) as depth') + .order(depth: depth_order, id: :asc) + end + + # Produce a query of the form: SELECT * FROM namespaces; + # + # When we have queries that break this SELECT * format we can run in to errors. + # For example `SELECT DISTINCT on(...)` will fail when we chain a `.count` c + def normal_select + unscoped.without_sti_condition.from(all, :namespaces) + end + private def use_traversal_ids? Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) end + def use_traversal_ids_for_ancestor_scopes? + Feature.enabled?(:use_traversal_ids_for_ancestor_scopes, default_enabled: :yaml) && + use_traversal_ids? + end + def self_and_descendants_with_duplicates(include_self: true) base_ids = select(:id) diff --git a/app/models/namespaces/traversal/recursive.rb b/app/models/namespaces/traversal/recursive.rb index c1ada715d6d..8d2c5d3be5a 100644 --- a/app/models/namespaces/traversal/recursive.rb +++ b/app/models/namespaces/traversal/recursive.rb @@ -7,12 +7,12 @@ module Namespaces include RecursiveScopes def root_ancestor - return self if parent.nil? - - if persisted? + if persisted? && !parent_id.nil? strong_memoize(:root_ancestor) do - recursive_self_and_ancestors.reorder(nil).find_by(parent_id: nil) + recursive_ancestors.reorder(nil).find_by(parent_id: nil) end + elsif parent.nil? + self else parent.root_ancestor end diff --git a/app/models/namespaces/traversal/recursive_scopes.rb b/app/models/namespaces/traversal/recursive_scopes.rb index be49d5d9d55..6659cefe095 100644 --- a/app/models/namespaces/traversal/recursive_scopes.rb +++ b/app/models/namespaces/traversal/recursive_scopes.rb @@ -10,6 +10,22 @@ module Namespaces select('id') end + def self_and_ancestors(include_self: true, hierarchy_order: nil) + records = Gitlab::ObjectHierarchy.new(all).base_and_ancestors(hierarchy_order: hierarchy_order) + + if include_self + records + else + records.where.not(id: all.as_ids) + end + end + alias_method :recursive_self_and_ancestors, :self_and_ancestors + + def self_and_ancestor_ids(include_self: true) + self_and_ancestors(include_self: include_self).as_ids + end + alias_method :recursive_self_and_ancestor_ids, :self_and_ancestor_ids + def descendant_ids recursive_descendants.as_ids end diff --git a/app/models/namespaces/user_namespace.rb b/app/models/namespaces/user_namespace.rb new file mode 100644 index 00000000000..517d68b118d --- /dev/null +++ b/app/models/namespaces/user_namespace.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# TODO: currently not created/mapped in the database, will be done in another issue +# https://gitlab.com/gitlab-org/gitlab/-/issues/337102 +module Namespaces + class UserNamespace < Namespace + def self.sti_name + 'User' + end + end +end diff --git a/app/models/note.rb b/app/models/note.rb index 34ffd7c91af..a8f5c305d9b 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -48,6 +48,9 @@ class Note < ApplicationRecord # Attribute used to store the attributes that have been changed by quick actions. attr_accessor :commands_changes + # Attribute used to determine whether keep_around_commits will be skipped for diff notes. + attr_accessor :skip_keep_around_commits + default_value_for :system, false attr_mentionable :note, pipeline: :note @@ -112,7 +115,6 @@ class Note < ApplicationRecord scope :updated_after, ->(time) { where('updated_at > ?', time) } scope :with_updated_at, ->(time) { where(updated_at: time) } scope :with_suggestions, -> { joins(:suggestions) } - scope :inc_author_project, -> { includes(:project, :author) } scope :inc_author, -> { includes(:author) } scope :with_api_entity_associations, -> { preload(:note_diff_file, :author) } scope :inc_relations_for_view, -> do @@ -579,7 +581,8 @@ class Note < ApplicationRecord end def post_processed_cache_key - cache_key_items = [cache_key, author.cache_key] + cache_key_items = [cache_key, author&.cache_key] + cache_key_items << project.team.human_max_access(author&.id) if author.present? cache_key_items << Digest::SHA1.hexdigest(redacted_note_html) if redacted_note_html.present? cache_key_items.join(':') @@ -603,14 +606,6 @@ class Note < ApplicationRecord private - # Using this method followed by a call to *save* may result in *ActiveRecord::RecordNotUnique* exception - # in a multi-threaded environment. Make sure to use it within a *safe_ensure_unique* block. - def model_user_mention - return if user_mentions.is_a?(ActiveRecord::NullRelation) - - user_mentions.first_or_initialize - end - def system_note_viewable_by?(user) return true unless system_note_metadata @@ -648,7 +643,7 @@ class Note < ApplicationRecord user_visible_reference_count > 0 && user_visible_reference_count == total_reference_count else refs = all_references(user) - refs.all.any? && refs.stateful_not_visible_counter == 0 + refs.all.any? && refs.all_visible? end end diff --git a/app/models/onboarding_progress.rb b/app/models/onboarding_progress.rb index 9185547d7cd..c12309d1852 100644 --- a/app/models/onboarding_progress.rb +++ b/app/models/onboarding_progress.rb @@ -45,7 +45,7 @@ class OnboardingProgress < ApplicationRecord def onboard(namespace) return unless root_namespace?(namespace) - safe_find_or_create_by(namespace: namespace) + create(namespace: namespace) end def onboarding?(namespace) diff --git a/app/models/operations/feature_flag.rb b/app/models/operations/feature_flag.rb index 450a5970ad8..46810749b18 100644 --- a/app/models/operations/feature_flag.rb +++ b/app/models/operations/feature_flag.rb @@ -17,6 +17,7 @@ module Operations has_internal_id :iid, scope: :project default_value_for :active, true + default_value_for :version, :new_version_flag # scopes exists only for the first version has_many :scopes, class_name: 'Operations::FeatureFlagScope' @@ -39,8 +40,6 @@ module Operations validate :first_default_scope, on: :create, if: :has_scopes? validate :version_associations - before_create :build_default_scope, if: -> { legacy_flag? && scopes.none? } - accepts_nested_attributes_for :scopes, allow_destroy: true accepts_nested_attributes_for :strategies, allow_destroy: true @@ -52,7 +51,6 @@ module Operations scope :new_version_only, -> { where(version: :new_version_flag)} enum version: { - legacy_flag: 1, new_version_flag: 2 } @@ -127,8 +125,6 @@ module Operations def version_associations if new_version_flag? && scopes.any? errors.add(:version_associations, 'version 2 feature flags may not have scopes') - elsif legacy_flag? && strategies.any? - errors.add(:version_associations, 'version 1 feature flags may not have strategies') end end diff --git a/app/models/operations/feature_flag_scope.rb b/app/models/operations/feature_flag_scope.rb index 78be29f2531..9068ca0f588 100644 --- a/app/models/operations/feature_flag_scope.rb +++ b/app/models/operations/feature_flag_scope.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true +# All of the legacy flags have been removed in 14.1, including all of the +# `operations_feature_flag_scopes` rows. Therefore, this model and the database +# table are unused and should be removed. + module Operations class FeatureFlagScope < ApplicationRecord prepend HasEnvironmentScope diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index 4ea127fc222..34eae6ab5dc 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true class Packages::Package < ApplicationRecord + include EachBatch include Sortable include Gitlab::SQL::Pattern include UsageStatistics @@ -104,6 +105,7 @@ class Packages::Package < ApplicationRecord scope :including_build_info, -> { includes(pipelines: :user) } scope :including_project_route, -> { includes(project: { namespace: :route }) } scope :including_tags, -> { includes(:tags) } + scope :including_dependency_links, -> { includes(dependency_links: :dependency) } scope :with_conan_channel, ->(package_channel) do joins(:conan_metadatum).where(packages_conan_metadata: { package_channel: package_channel }) @@ -291,6 +293,13 @@ class Packages::Package < ApplicationRecord ::Packages::Maven::Metadata::SyncWorker.perform_async(user.id, project.id, name) end + def create_build_infos!(build) + return unless build&.pipeline + + # TODO: use an upsert call when https://gitlab.com/gitlab-org/gitlab/-/issues/339093 is implemented + build_infos.find_or_create_by!(pipeline: build.pipeline) + end + private def composer_tag_version? diff --git a/app/models/packages/package_file.rb b/app/models/packages/package_file.rb index 8aa19397086..14701b8a800 100644 --- a/app/models/packages/package_file.rb +++ b/app/models/packages/package_file.rb @@ -77,6 +77,10 @@ class Packages::PackageFile < ApplicationRecord .where(packages_conan_file_metadata: { conan_package_reference: conan_package_reference }) end + def self.most_recent! + recent.first! + end + mount_file_store_uploader Packages::PackageFileUploader update_project_statistics project_statistics_name: :packages_size @@ -89,6 +93,24 @@ class Packages::PackageFile < ApplicationRecord skip_callback :commit, :after, :remove_previously_stored_file, if: :execute_move_in_object_storage? after_commit :move_in_object_storage, if: :execute_move_in_object_storage? + # Returns the most recent package files for *each* of the given packages. + # The order is not guaranteed. + def self.most_recent_for(packages, extra_join: nil, extra_where: nil) + cte_name = :packages_cte + cte = Gitlab::SQL::CTE.new(cte_name, packages.select(:id)) + + package_files = ::Packages::PackageFile.limit_recent(1) + .where(arel_table[:package_id].eq(Arel.sql("#{cte_name}.id"))) + + package_files = package_files.joins(extra_join) if extra_join + package_files = package_files.where(extra_where) if extra_where + + query = select('finder.*') + .from([Arel.sql(cte_name.to_s), package_files.arel.lateral.as('finder')]) + + query.with(cte.to_arel) + end + def download_path Gitlab::Routing.url_helpers.download_project_package_file_path(project, self) end diff --git a/app/models/pages_deployment.rb b/app/models/pages_deployment.rb index 294a4e85d1f..da6ef035c54 100644 --- a/app/models/pages_deployment.rb +++ b/app/models/pages_deployment.rb @@ -16,6 +16,7 @@ class PagesDeployment < ApplicationRecord scope :migrated_from_legacy_storage, -> { where(file: MIGRATED_FILE_NAME) } scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) } scope :with_files_stored_remotely, -> { where(file_store: ::ObjectStorage::Store::REMOTE) } + scope :project_id_in, ->(ids) { where(project_id: ids) } validates :file, presence: true validates :file_store, presence: true, inclusion: { in: ObjectStorage::SUPPORTED_STORES } @@ -27,10 +28,6 @@ class PagesDeployment < ApplicationRecord mount_file_store_uploader ::Pages::DeploymentUploader - def log_geo_deleted_event - # this is to be adressed in https://gitlab.com/groups/gitlab-org/-/epics/589 - end - def migrated? file.filename == MIGRATED_FILE_NAME end @@ -41,3 +38,5 @@ class PagesDeployment < ApplicationRecord self.size = file.size end end + +PagesDeployment.prepend_mod diff --git a/app/models/postgresql/detached_partition.rb b/app/models/postgresql/detached_partition.rb index 76b299ff9d4..12b48895e0c 100644 --- a/app/models/postgresql/detached_partition.rb +++ b/app/models/postgresql/detached_partition.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Postgresql - class DetachedPartition < ApplicationRecord + class DetachedPartition < ::Gitlab::Database::SharedModel scope :ready_to_drop, -> { where('drop_after < ?', Time.current) } end end diff --git a/app/models/preloaders/commit_status_preloader.rb b/app/models/preloaders/commit_status_preloader.rb new file mode 100644 index 00000000000..535dd24ba6b --- /dev/null +++ b/app/models/preloaders/commit_status_preloader.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Preloaders + class CommitStatusPreloader + CLASSES = [::Ci::Build, ::Ci::Bridge, ::GenericCommitStatus].freeze + + def initialize(statuses) + @statuses = statuses + end + + def execute(relations) + preloader = ActiveRecord::Associations::Preloader.new + + CLASSES.each do |klass| + preloader.preload(objects(klass), associations(klass, relations)) + end + end + + private + + def objects(klass) + @statuses.select { |job| job.is_a?(klass) } + end + + def associations(klass, relations) + klass.reflections.keys.map(&:to_sym) & relations.map(&:to_sym) + end + end +end diff --git a/app/models/preloaders/merge_requests_preloader.rb b/app/models/preloaders/merge_requests_preloader.rb new file mode 100644 index 00000000000..cefe8408cab --- /dev/null +++ b/app/models/preloaders/merge_requests_preloader.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Preloaders + class MergeRequestsPreloader + attr_reader :merge_requests + + def initialize(merge_requests) + @merge_requests = merge_requests + end + + def execute + preloader = ActiveRecord::Associations::Preloader.new + preloader.preload(merge_requests, { target_project: [:project_feature] }) + merge_requests.each do |merge_request| + merge_request.lazy_upvotes_count + end + end + end +end diff --git a/app/models/preloaders/user_max_access_level_in_groups_preloader.rb b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb new file mode 100644 index 00000000000..14f1d271572 --- /dev/null +++ b/app/models/preloaders/user_max_access_level_in_groups_preloader.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Preloaders + # This class preloads the max access level (role) for the user within the given groups and + # stores the values in requests store. + # Will only be able to preload max access level for groups where the user is a direct member + class UserMaxAccessLevelInGroupsPreloader + include BulkMemberAccessLoad + + def initialize(groups, user) + @groups = groups + @user = user + end + + def execute + group_memberships = GroupMember.active_without_invites_and_requests + .non_minimal_access + .where(user: @user, source_id: @groups) + .group(:source_id) + .maximum(:access_level) + + group_memberships.each do |group_id, max_access_level| + merge_value_to_request_store(User, @user.id, group_id, max_access_level) + end + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 81b04e1316c..74ffeef797e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -103,6 +103,8 @@ class Project < ApplicationRecord after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? } + after_save :save_topics + after_create -> { create_or_load_association(:project_feature) } after_create -> { create_or_load_association(:ci_cd_settings) } @@ -118,7 +120,6 @@ class Project < ApplicationRecord use_fast_destroy :build_trace_chunks - after_destroy -> { run_after_commit { legacy_remove_pages } } after_destroy :remove_exports after_validation :check_pending_delete @@ -127,12 +128,31 @@ class Project < ApplicationRecord after_initialize :use_hashed_storage after_create :check_repository_absence! + # Required during the `ActsAsTaggableOn::Tag -> Topic` migration + # TODO: remove 'acts_as_ordered_taggable_on' and ':topics_acts_as_taggable' in the further process of the migration + # https://gitlab.com/gitlab-org/gitlab/-/issues/335946 acts_as_ordered_taggable_on :topics + has_many :topics_acts_as_taggable, -> { order("#{ActsAsTaggableOn::Tagging.table_name}.id") }, + class_name: 'ActsAsTaggableOn::Tag', + through: :topic_taggings, + source: :tag + + has_many :project_topics, -> { order(:id) }, class_name: 'Projects::ProjectTopic' + has_many :topics, through: :project_topics, class_name: 'Projects::Topic' + + # Required during the `ActsAsTaggableOn::Tag -> Topic` migration + # TODO: remove 'topics' in the further process of the migration + # https://gitlab.com/gitlab-org/gitlab/-/issues/335946 + alias_method :topics_new, :topics + def topics + self.topics_acts_as_taggable + self.topics_new + end attr_accessor :old_path_with_namespace attr_accessor :template_name attr_writer :pipeline_status attr_accessor :skip_disk_validation + attr_writer :topic_list alias_attribute :title, :name @@ -141,6 +161,9 @@ class Project < ApplicationRecord belongs_to :creator, class_name: 'User' belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id' belongs_to :namespace + # Sync deletion via DB Trigger to ensure we do not have + # a project without a project_namespace (or vice-versa) + belongs_to :project_namespace, class_name: 'Namespaces::ProjectNamespace', foreign_key: 'project_namespace_id', inverse_of: :project alias_method :parent, :namespace alias_attribute :parent_id, :namespace_id @@ -188,6 +211,7 @@ class Project < ApplicationRecord has_one :unify_circuit_integration, class_name: 'Integrations::UnifyCircuit' has_one :webex_teams_integration, class_name: 'Integrations::WebexTeams' has_one :youtrack_integration, class_name: 'Integrations::Youtrack' + has_one :zentao_integration, class_name: 'Integrations::Zentao' has_one :root_of_fork_network, foreign_key: 'root_project_id', @@ -317,6 +341,7 @@ class Project < ApplicationRecord # build traces. Currently there's no efficient way of removing this data in # bulk that doesn't involve loading the rows into memory. As a result we're # still using `dependent: :destroy` here. + has_many :pending_builds, class_name: 'Ci::PendingBuild' has_many :builds, class_name: 'Ci::Build', inverse_of: :project, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :processables, class_name: 'Ci::Processable', inverse_of: :project has_many :build_trace_chunks, class_name: 'Ci::BuildTraceChunk', through: :builds, source: :trace_chunks @@ -355,6 +380,7 @@ class Project < ApplicationRecord has_many :jira_imports, -> { order 'jira_imports.created_at' }, class_name: 'JiraImportState', inverse_of: :project has_many :daily_build_group_report_results, class_name: 'Ci::DailyBuildGroupReportResult' + has_many :ci_feature_usages, class_name: 'Projects::CiFeatureUsage' has_many :repository_storage_moves, class_name: 'Projects::RepositoryStorageMove', inverse_of: :container @@ -503,6 +529,7 @@ class Project < ApplicationRecord scope :sorted_by_stars_desc, -> { reorder(self.arel_table['star_count'].desc) } scope :sorted_by_stars_asc, -> { reorder(self.arel_table['star_count'].asc) } # Sometimes queries (e.g. using CTEs) require explicit disambiguation with table name + scope :projects_order_id_asc, -> { reorder(self.arel_table['id'].asc) } scope :projects_order_id_desc, -> { reorder(self.arel_table['id'].desc) } scope :sorted_by_similarity_desc, -> (search, include_in_select: false) do @@ -623,6 +650,19 @@ class Project < ApplicationRecord joins(:service_desk_setting).where('service_desk_settings.project_key' => key) end + scope :with_topic, ->(topic_name) do + topic = Projects::Topic.find_by_name(topic_name) + acts_as_taggable_on_topic = ActsAsTaggableOn::Tag.find_by_name(topic_name) + + return none unless topic || acts_as_taggable_on_topic + + relations = [] + relations << where(id: topic.project_topics.select(:project_id)) if topic + relations << where(id: acts_as_taggable_on_topic.taggings.select(:taggable_id)) if acts_as_taggable_on_topic + + Project.from_union(relations) + end + enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 } chronic_duration_attr :build_timeout_human_readable, :build_timeout, @@ -638,7 +678,7 @@ class Project < ApplicationRecord mount_uploader :bfg_object_map, AttachmentUploader def self.with_api_entity_associations - preload(:project_feature, :route, :topics, :group, :timelogs, namespace: [:route, :owner]) + preload(:project_feature, :route, :topics, :topics_acts_as_taggable, :group, :timelogs, namespace: [:route, :owner]) end def self.with_web_entity_associations @@ -1421,7 +1461,7 @@ class Project < ApplicationRecord end def disabled_integrations - [] + [:zentao] end def find_or_initialize_integration(name) @@ -1640,6 +1680,10 @@ class Project < ApplicationRecord end end + def membership_locked? + false + end + def bots users.project_bot end @@ -1747,6 +1791,9 @@ class Project < ApplicationRecord Ci::Runner.from_union([runners, group_runners, available_shared_runners]) end + # Once issue 339937 is fixed, please search for all mentioned of + # https://gitlab.com/gitlab-org/gitlab/-/issues/339937, + # and remove the allow_cross_joins_across_databases. def active_runners strong_memoize(:active_runners) do all_available_runners.active @@ -1754,7 +1801,9 @@ class Project < ApplicationRecord end def any_online_runners?(&block) - online_runners_with_tags.any?(&block) + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/339937') do + online_runners_with_tags.any?(&block) + end end def valid_runners_token?(token) @@ -1763,7 +1812,15 @@ class Project < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def open_issues_count(current_user = nil) - Projects::OpenIssuesCountService.new(self, current_user).count + return Projects::OpenIssuesCountService.new(self, current_user).count unless current_user.nil? + + BatchLoader.for(self).batch(replace_methods: false) do |projects, loader| + issues_count_per_project = ::Projects::BatchOpenIssuesCountService.new(projects).refresh_cache_and_retrieve_data + + issues_count_per_project.each do |project, count| + loader.call(project, count) + end + end end # rubocop: enable CodeReuse/ServiceClass @@ -1849,27 +1906,6 @@ class Project < ApplicationRecord .delete_all end - # TODO: remove this method https://gitlab.com/gitlab-org/gitlab/-/issues/320775 - # rubocop: disable CodeReuse/ServiceClass - def legacy_remove_pages - return unless ::Settings.pages.local_store.enabled - - # Projects with a missing namespace cannot have their pages removed - return unless namespace - - mark_pages_as_not_deployed unless destroyed? - - # 1. We rename pages to temporary directory - # 2. We wait 5 minutes, due to NFS caching - # 3. We asynchronously remove pages with force - temp_path = "#{path}.#{SecureRandom.hex}.deleted" - - if Gitlab::PagesTransfer.new.rename_project(path, temp_path, namespace.full_path) - PagesWorker.perform_in(5.minutes, :remove, namespace.full_path, temp_path) - end - end - # rubocop: enable CodeReuse/ServiceClass - def mark_pages_as_deployed(artifacts_archive: nil) ensure_pages_metadatum.update!(deployed: true, artifacts_archive: artifacts_archive) end @@ -2093,6 +2129,10 @@ class Project < ApplicationRecord # Docker doesn't allow. The proxy expects it to be downcased. value: "#{Gitlab.host_with_port}/#{namespace.root_ancestor.path.downcase}#{DependencyProxy::URL_SUFFIX}" ) + variables.append( + key: 'CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX', + value: "#{Gitlab.host_with_port}/#{namespace.full_path.downcase}#{DependencyProxy::URL_SUFFIX}" + ) end end @@ -2239,7 +2279,7 @@ class Project < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def forks_count - BatchLoader.for(self).batch do |projects, loader| + BatchLoader.for(self).batch(replace_methods: false) do |projects, loader| fork_count_per_project = ::Projects::BatchForksCountService.new(projects).refresh_cache_and_retrieve_data fork_count_per_project.each do |project, count| @@ -2491,6 +2531,10 @@ class Project < ApplicationRecord ci_config_path.blank? || ci_config_path == Gitlab::FileDetector::PATTERNS[:gitlab_ci] end + def uses_external_project_ci_config? + !!(ci_config_path =~ %r{@.+/.+}) + end + def limited_protected_branches(limit) protected_branches.limit(limit) end @@ -2599,6 +2643,10 @@ class Project < ApplicationRecord repository.gitlab_ci_yml_for(sha, ci_config_path_or_default) end + def ci_config_external_project + Project.find_by_full_path(ci_config_path.split('@', 2).last) + end + def enabled_group_deploy_keys return GroupDeployKey.none unless group @@ -2669,8 +2717,37 @@ class Project < ApplicationRecord ci_cd_settings.group_runners_enabled? end + def topic_list + self.topics.map(&:name) + end + + override :after_change_head_branch_does_not_exist + def after_change_head_branch_does_not_exist(branch) + self.errors.add(:base, _("Could not change HEAD: branch '%{branch}' does not exist") % { branch: branch }) + end + private + def save_topics + return if @topic_list.nil? + + @topic_list = @topic_list.split(',') if @topic_list.instance_of?(String) + @topic_list = @topic_list.map(&:strip).uniq.reject(&:empty?) + + if @topic_list != self.topic_list || self.topics_acts_as_taggable.any? + self.topics_new.delete_all + self.topics = @topic_list.map { |topic| Projects::Topic.find_or_create_by(name: topic) } + + # Remove old topics (ActsAsTaggableOn::Tag) + # Required during the `ActsAsTaggableOn::Tag -> Topic` migration + # TODO: remove in the further process of the migration + # https://gitlab.com/gitlab-org/gitlab/-/issues/335946 + self.topic_taggings.clear + end + + @topic_list = nil + end + def find_integration(integrations, name) integrations.find { _1.to_param == name } end @@ -2832,12 +2909,8 @@ class Project < ApplicationRecord update_column(:has_external_issue_tracker, integrations.external_issue_trackers.any?) if Gitlab::Database.read_write? end - def active_runners_with_tags - @active_runners_with_tags ||= active_runners.with_tags - end - def online_runners_with_tags - @online_runners_with_tags ||= active_runners_with_tags.online + @online_runners_with_tags ||= active_runners.with_tags.online end end diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index aea8abecd74..676c28d5e1b 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -54,7 +54,6 @@ class ProjectFeature < ApplicationRecord validates :project, presence: true validate :repository_children_level - validate :allowed_access_levels default_value_for :builds_access_level, value: ENABLED, allows_nil: false default_value_for :issues_access_level, value: ENABLED, allows_nil: false @@ -110,17 +109,6 @@ class ProjectFeature < ApplicationRecord %i(merge_requests_access_level builds_access_level).each(&validator) end - # Validates access level for other than pages cannot be PUBLIC - def allowed_access_levels - validator = lambda do |field| - level = public_send(field) || ENABLED # rubocop:disable GitlabSecurity/PublicSend - not_allowed = level > ENABLED - self.errors.add(field, "cannot have public visibility level") if not_allowed - end - - (FEATURES - %i(pages)).each {|f| validator.call("#{f}_access_level")} - end - def get_permission(user, feature) case access_level(feature) when DISABLED @@ -142,6 +130,10 @@ class ProjectFeature < ApplicationRecord project.team.member?(user, ProjectFeature.required_minimum_access_level(feature)) end + + def feature_validation_exclusion + %i(pages) + end end ProjectFeature.prepend_mod_with('ProjectFeature') diff --git a/app/models/project_team.rb b/app/models/project_team.rb index 4ae3bc01a01..774d81156b7 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -42,7 +42,7 @@ class ProjectTeam end def add_users(users, access_level, current_user: nil, expires_at: nil) - Members::Projects::CreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass + Members::Projects::BulkCreatorService.add_users( # rubocop:disable CodeReuse/ServiceClass project, users, access_level, diff --git a/app/models/projects/project_topic.rb b/app/models/projects/project_topic.rb new file mode 100644 index 00000000000..d4b456ef482 --- /dev/null +++ b/app/models/projects/project_topic.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Projects + class ProjectTopic < ApplicationRecord + belongs_to :project + belongs_to :topic + end +end diff --git a/app/models/projects/topic.rb b/app/models/projects/topic.rb new file mode 100644 index 00000000000..a17aa550edb --- /dev/null +++ b/app/models/projects/topic.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Projects + class Topic < ApplicationRecord + validates :name, presence: true, uniqueness: true, length: { maximum: 255 } + + has_many :project_topics, class_name: 'Projects::ProjectTopic' + has_many :projects, through: :project_topics + end +end diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb index 3df8fe31826..3d32144e0f8 100644 --- a/app/models/protected_branch.rb +++ b/app/models/protected_branch.rb @@ -26,7 +26,9 @@ class ProtectedBranch < ApplicationRecord def self.protected?(project, ref_name) return true if project.empty_repo? && project.default_branch_protected? - self.matching(ref_name, protected_refs: protected_refs(project)).present? + Rails.cache.fetch("protected_ref-#{ref_name}-#{project.cache_key}") do + self.matching(ref_name, protected_refs: protected_refs(project)).present? + end end def self.allow_force_push?(project, ref_name) diff --git a/app/models/push_event_payload.rb b/app/models/push_event_payload.rb index 8358be35470..441b94e1855 100644 --- a/app/models/push_event_payload.rb +++ b/app/models/push_event_payload.rb @@ -2,6 +2,9 @@ class PushEventPayload < ApplicationRecord extend SuppressCompositePrimaryKeyWarning + include IgnorableColumns + + ignore_columns :event_id_convert_to_bigint, remove_with: '14.4', remove_after: '2021-10-22' include ShaAttribute diff --git a/app/models/release.rb b/app/models/release.rb index aad1cbeabdb..0dd71c6ebfb 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -33,7 +33,6 @@ class Release < ApplicationRecord includes(:author, :evidences, :milestones, :links, :sorted_links, project: [:project_feature, :route, { namespace: :route }]) } - scope :with_project_and_namespace, -> { includes(project: :namespace) } scope :recent, -> { sorted.limit(MAX_NUMBER_TO_DISPLAY) } scope :without_evidence, -> { left_joins(:evidences).where(::Releases::Evidence.arel_table[:id].eq(nil)) } scope :released_within_2hrs, -> { where(released_at: Time.zone.now - 1.hour..Time.zone.now + 1.hour) } diff --git a/app/models/repository.rb b/app/models/repository.rb index 0164d6fed93..f20b306c806 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -161,8 +161,8 @@ class Repository CommitCollection.new(container, commits, ref) end - def commits_between(from, to) - commits = Gitlab::Git::Commit.between(raw_repository, from, to) + def commits_between(from, to, limit: nil) + commits = Gitlab::Git::Commit.between(raw_repository, from, to, limit: limit) commits = Commit.decorate(commits, container) if commits.present? commits end @@ -191,7 +191,11 @@ class Repository end def find_tag(name) - tags.find { |tag| tag.name == name } + if @tags.blank? && Feature.enabled?(:find_tag_via_gitaly, project, default_enabled: :yaml) + raw_repository.find_tag(name) + else + tags.find { |tag| tag.name == name } + end end def ambiguous_ref?(ref) @@ -627,7 +631,14 @@ class Repository def license return unless license_key - Licensee::License.new(license_key) + licensee_object = Licensee::License.new(license_key) + + return if licensee_object.name.blank? + + licensee_object + rescue Licensee::InvalidLicense => ex + Gitlab::ErrorTracking.track_exception(ex) + nil end memoize_method :license @@ -721,18 +732,9 @@ class Repository end def tags_sorted_by(value) - case value - when 'name_asc' - VersionSorter.sort(tags) { |tag| tag.name } - when 'name_desc' - VersionSorter.rsort(tags) { |tag| tag.name } - when 'updated_desc' - tags_sorted_by_committed_date.reverse - when 'updated_asc' - tags_sorted_by_committed_date - else - tags - end + return raw_repository.tags(sort_by: value) if Feature.enabled?(:gitaly_tags_finder, project, default_enabled: :yaml) + + tags_ruby_sort(value) end # Params: @@ -1125,11 +1127,16 @@ class Repository copy_gitattributes(branch) after_change_head else - container.errors.add(:base, _("Could not change HEAD: branch '%{branch}' does not exist") % { branch: branch }) + container.after_change_head_branch_does_not_exist(branch) + false end end + def cache + @cache ||= Gitlab::RepositoryCache.new(self) + end + private # TODO Genericize finder, later split this on finders by Ref or Oid @@ -1144,10 +1151,6 @@ class Repository ::Commit.new(commit, container) if commit end - def cache - @cache ||= Gitlab::RepositoryCache.new(self) - end - def redis_set_cache @redis_set_cache ||= Gitlab::RepositorySetCache.new(self) end @@ -1160,6 +1163,23 @@ class Repository @request_store_cache ||= Gitlab::RepositoryCache.new(self, backend: Gitlab::SafeRequestStore) end + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/339741 + def tags_ruby_sort(value) + case value + when 'name_asc' + VersionSorter.sort(tags) { |tag| tag.name } + when 'name_desc' + VersionSorter.rsort(tags) { |tag| tag.name } + when 'updated_desc' + tags_sorted_by_committed_date.reverse + when 'updated_asc' + tags_sorted_by_committed_date + else + tags + end + end + + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/339741 def tags_sorted_by_committed_date # Annotated tags can point to any object (e.g. a blob), but generally # tags point to a commit. If we don't have a commit, then just default diff --git a/app/models/service_desk_setting.rb b/app/models/service_desk_setting.rb index 1c854cc9941..6dd7415d928 100644 --- a/app/models/service_desk_setting.rb +++ b/app/models/service_desk_setting.rb @@ -19,7 +19,11 @@ class ServiceDeskSetting < ApplicationRecord strong_memoize(:issue_template_content) do next unless issue_template_key.present? - Gitlab::Template::IssueTemplate.find(issue_template_key, project).content + TemplateFinder.new( + :issues, project, + name: issue_template_key, + source_template_project: source_template_project + ).execute.content rescue ::Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError end end @@ -42,6 +46,10 @@ class ServiceDeskSetting < ApplicationRecord private + def source_template_project + nil + end + def projects_with_same_slug_and_key_exists? return false unless project_key @@ -53,3 +61,5 @@ class ServiceDeskSetting < ApplicationRecord end end end + +ServiceDeskSetting.prepend_mod diff --git a/app/models/shard.rb b/app/models/shard.rb index 335a279c6aa..9f0039d8bf9 100644 --- a/app/models/shard.rb +++ b/app/models/shard.rb @@ -18,10 +18,6 @@ class Shard < ApplicationRecord end def self.by_name(name) - transaction(requires_new: true) do - find_or_create_by(name: name) - end - rescue ActiveRecord::RecordNotUnique - retry + safe_find_or_create_by(name: name) end end diff --git a/app/models/user.rb b/app/models/user.rb index cb0f15c04cb..b5f0251f639 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -39,6 +39,12 @@ class User < ApplicationRecord MAX_USERNAME_LENGTH = 255 MIN_USERNAME_LENGTH = 2 + SECONDARY_EMAIL_ATTRIBUTES = [ + :commit_email, + :notification_email, + :public_email + ].freeze + add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } add_authentication_token_field :feed_token add_authentication_token_field :static_object_token @@ -181,7 +187,7 @@ class User < ApplicationRecord has_many :todos has_many :notification_settings has_many :award_emoji, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger', foreign_key: :owner_id # rubocop:disable Cop/ActiveRecordDependent + has_many :triggers, class_name: 'Ci::Trigger', foreign_key: :owner_id has_many :issue_assignees, inverse_of: :assignee has_many :merge_request_assignees, inverse_of: :assignee @@ -194,6 +200,7 @@ class User < ApplicationRecord has_many :custom_attributes, class_name: 'UserCustomAttribute' has_many :callouts, class_name: 'UserCallout' + has_many :group_callouts, class_name: 'Users::GroupCallout' has_many :term_agreements belongs_to :accepted_term, class_name: 'ApplicationSetting::Term' @@ -222,10 +229,9 @@ class User < ApplicationRecord validates :first_name, length: { maximum: 127 } validates :last_name, length: { maximum: 127 } validates :email, confirmation: true - validates :notification_email, presence: true - validates :notification_email, devise_email: true, if: ->(user) { user.notification_email != user.email } + validates :notification_email, devise_email: true, allow_blank: true, if: ->(user) { user.notification_email != user.email } validates :public_email, uniqueness: true, devise_email: true, allow_blank: true - validates :commit_email, devise_email: true, allow_nil: true, if: ->(user) { user.commit_email != user.email } + validates :commit_email, devise_email: true, allow_blank: true, if: ->(user) { user.commit_email != user.email && user.commit_email != Gitlab::PrivateCommitEmail::TOKEN } validates :projects_limit, presence: true, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: Gitlab::Database::MAX_INT_VALUE } @@ -247,12 +253,10 @@ class User < ApplicationRecord validates :color_scheme_id, allow_nil: true, inclusion: { in: Gitlab::ColorSchemes.valid_ids, message: _("%{placeholder} is not a valid color scheme") % { placeholder: '%{value}' } } + validates :website_url, allow_blank: true, url: true, if: :website_url_changed? + before_validation :sanitize_attrs - before_validation :set_public_email, if: :public_email_changed? - before_validation :set_commit_email, if: :commit_email_changed? before_save :default_private_profile_to_false - before_save :set_public_email, if: :public_email_changed? # in case validation is skipped - before_save :set_commit_email, if: :commit_email_changed? # in case validation is skipped before_save :ensure_incoming_email_token before_save :ensure_user_rights_and_limits, if: ->(user) { user.new_record? || user.external_changed? } before_save :skip_reconfirmation!, if: ->(user) { user.email_changed? && user.read_only_attribute?(:email) } @@ -302,14 +306,13 @@ class User < ApplicationRecord :gitpod_enabled, :gitpod_enabled=, :setup_for_company, :setup_for_company=, :render_whitespace_in_code, :render_whitespace_in_code=, - :experience_level, :experience_level=, :markdown_surround_selection, :markdown_surround_selection=, to: :user_preference delegate :path, to: :namespace, allow_nil: true, prefix: true delegate :job_title, :job_title=, to: :user_detail, allow_nil: true delegate :other_role, :other_role=, to: :user_detail, allow_nil: true - delegate :bio, :bio=, :bio_html, to: :user_detail, allow_nil: true + delegate :bio, :bio=, to: :user_detail, allow_nil: true delegate :webauthn_xid, :webauthn_xid=, to: :user_detail, allow_nil: true delegate :pronouns, :pronouns=, to: :user_detail, allow_nil: true delegate :pronunciation, :pronunciation=, to: :user_detail, allow_nil: true @@ -347,6 +350,10 @@ class User < ApplicationRecord transition active: :banned end + event :unban do + transition banned: :active + end + event :deactivate do # Any additional changes to this event should be also # reflected in app/workers/users/deactivate_dormant_users_worker.rb @@ -374,7 +381,9 @@ class User < ApplicationRecord end after_transition any => :deactivated do |user| - NotificationService.new.user_deactivated(user.name, user.notification_email) + next unless Gitlab::CurrentSettings.user_deactivation_emails_enabled + + NotificationService.new.user_deactivated(user.name, user.notification_email_or_default) end # rubocop: enable CodeReuse/ServiceClass @@ -922,51 +931,18 @@ class User < ApplicationRecord end end - def notification_email_verified - return if read_attribute(:notification_email).blank? || temp_oauth_email? - - errors.add(:notification_email, _("must be an email you have verified")) unless verified_emails.include?(notification_email) - end - - def public_email_verified - return if public_email.blank? - - errors.add(:public_email, _("must be an email you have verified")) unless verified_emails.include?(public_email) - end - - def commit_email_verified - return if read_attribute(:commit_email).blank? - - errors.add(:commit_email, _("must be an email you have verified")) unless verified_emails.include?(commit_email) - end - - # Define commit_email-related attribute methods explicitly instead of relying - # on ActiveRecord to provide them. Some of the specs use the current state of - # the model code but an older database schema, so we need to guard against the - # possibility of the commit_email column not existing. - - def commit_email - return self.email unless has_attribute?(:commit_email) - - if super == Gitlab::PrivateCommitEmail::TOKEN + def commit_email_or_default + if self.commit_email == Gitlab::PrivateCommitEmail::TOKEN return private_commit_email end # The commit email is the same as the primary email if undefined - super.presence || self.email + self.commit_email.presence || self.email end - def commit_email=(email) - super if has_attribute?(:commit_email) - end - - def commit_email_changed? - has_attribute?(:commit_email) && super - end - - def notification_email + def notification_email_or_default # The notification email is the same as the primary email if undefined - super.presence || self.email + self.notification_email.presence || self.email end def private_commit_email @@ -1009,7 +985,11 @@ class User < ApplicationRecord # Returns the groups a user is a member of, either directly or through a parent group def membership_groups - Gitlab::ObjectHierarchy.new(groups).base_and_descendants + if Feature.enabled?(:linear_user_membership_groups, self, default_enabled: :yaml) + groups.self_and_descendants + else + Gitlab::ObjectHierarchy.new(groups).base_and_descendants + end end # Returns a relation of groups the user has access to, including their parent @@ -1292,29 +1272,15 @@ class User < ApplicationRecord self.name = self.name.gsub(%r{</?[^>]*>}, '') end - def set_notification_email - if notification_email.blank? || all_emails.exclude?(notification_email) - self.notification_email = email - end - end - - def set_public_email - if public_email.blank? || all_emails.exclude?(public_email) - self.public_email = '' - end - end - - def set_commit_email - if commit_email.blank? || verified_emails.exclude?(commit_email) - self.commit_email = nil + def unset_secondary_emails_matching_deleted_email!(deleted_email) + secondary_email_attribute_changed = false + SECONDARY_EMAIL_ATTRIBUTES.each do |attribute| + if read_attribute(attribute) == deleted_email + self.write_attribute(attribute, nil) + secondary_email_attribute_changed = true + end end - end - - def update_secondary_emails! - set_notification_email - set_public_email - set_commit_email - save if notification_email_changed? || public_email_changed? || commit_email_changed? + save if secondary_email_attribute_changed end def admin_unsubscribe! @@ -1569,7 +1535,11 @@ class User < ApplicationRecord end def manageable_groups(include_groups_with_developer_maintainer_access: false) - owned_and_maintainer_group_hierarchy = Gitlab::ObjectHierarchy.new(owned_or_maintainers_groups).base_and_descendants + owned_and_maintainer_group_hierarchy = if Feature.enabled?(:linear_user_manageable_groups, self, default_enabled: :yaml) + owned_or_maintainers_groups.self_and_descendants + else + Gitlab::ObjectHierarchy.new(owned_or_maintainers_groups).base_and_descendants + end if include_groups_with_developer_maintainer_access union_sql = ::Gitlab::SQL::Union.new( @@ -1628,6 +1598,8 @@ class User < ApplicationRecord true end + # TODO Please check all callers and remove allow_cross_joins_across_databases, + # when https://gitlab.com/gitlab-org/gitlab/-/issues/336436 is done. def ci_owned_runners @ci_owned_runners ||= begin project_runners = Ci::RunnerProject @@ -1644,9 +1616,15 @@ class User < ApplicationRecord end end + def owns_runner?(runner) + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do + ci_owned_runners.exists?(runner.id) + end + end + def notification_email_for(notification_group) # Return group-specific email address if present, otherwise return global notification email address - notification_group&.notification_email_for(self) || notification_email + notification_group&.notification_email_for(self) || notification_email_or_default end def notification_settings_for(source, inherit: false) @@ -1935,10 +1913,14 @@ class User < ApplicationRecord def dismissed_callout?(feature_name:, ignore_dismissal_earlier_than: nil) callout = callouts_by_feature_name[feature_name] - return false unless callout - return callout.dismissed_after?(ignore_dismissal_earlier_than) if ignore_dismissal_earlier_than + callout_dismissed?(callout, ignore_dismissal_earlier_than) + end - true + def dismissed_callout_for_group?(feature_name:, group:, ignore_dismissal_earlier_than: nil) + source_feature_name = "#{feature_name}_#{group.id}" + callout = group_callouts_by_feature_name[source_feature_name] + + callout_dismissed?(callout, ignore_dismissal_earlier_than) end # Load the current highest access by looking directly at the user's memberships @@ -1962,6 +1944,11 @@ class User < ApplicationRecord callouts.find_or_initialize_by(feature_name: ::UserCallout.feature_names[feature_name]) end + def find_or_initialize_group_callout(feature_name, group_id) + group_callouts + .find_or_initialize_by(feature_name: ::Users::GroupCallout.feature_names[feature_name], group_id: group_id) + end + def can_trigger_notifications? confirmed? && !blocked? && !ghost? end @@ -2015,10 +2002,39 @@ class User < ApplicationRecord private + def notification_email_verified + return if notification_email.blank? || temp_oauth_email? + + errors.add(:notification_email, _("must be an email you have verified")) unless verified_emails.include?(notification_email_or_default) + end + + def public_email_verified + return if public_email.blank? + + errors.add(:public_email, _("must be an email you have verified")) unless verified_emails.include?(public_email) + end + + def commit_email_verified + return if commit_email.blank? + + errors.add(:commit_email, _("must be an email you have verified")) unless verified_emails.include?(commit_email_or_default) + end + + def callout_dismissed?(callout, ignore_dismissal_earlier_than) + return false unless callout + return callout.dismissed_after?(ignore_dismissal_earlier_than) if ignore_dismissal_earlier_than + + true + end + def callouts_by_feature_name @callouts_by_feature_name ||= callouts.index_by(&:feature_name) end + def group_callouts_by_feature_name + @group_callouts_by_feature_name ||= group_callouts.index_by(&:source_feature_name) + end + def authorized_groups_without_shared_membership Group.from_union([ groups.select(Namespace.arel_table[Arel.star]), @@ -2080,7 +2096,7 @@ class User < ApplicationRecord def check_username_format return if username.blank? || Mime::EXTENSION_LOOKUP.keys.none? { |type| username.end_with?(".#{type}") } - errors.add(:username, _('ending with MIME type format is not allowed.')) + errors.add(:username, _('ending with a file extension is not allowed.')) end def groups_with_developer_maintainer_project_access @@ -2090,9 +2106,12 @@ class User < ApplicationRecord project_creation_levels << nil end - developer_groups_hierarchy = ::Gitlab::ObjectHierarchy.new(developer_groups).base_and_descendants - ::Group.where(id: developer_groups_hierarchy.select(:id), - project_creation_level: project_creation_levels) + if Feature.enabled?(:linear_user_groups_with_developer_maintainer_project_access, self, default_enabled: :yaml) + developer_groups.self_and_descendants.where(project_creation_level: project_creation_levels) + else + developer_groups_hierarchy = ::Gitlab::ObjectHierarchy.new(developer_groups).base_and_descendants + ::Group.where(id: developer_groups_hierarchy.select(:id), project_creation_level: project_creation_levels) + end end def no_recent_activity? diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb index 1172b2ee5e8..04bc29755f8 100644 --- a/app/models/user_callout.rb +++ b/app/models/user_callout.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class UserCallout < ApplicationRecord - belongs_to :user + include Calloutable enum feature_name: { gke_cluster_integration: 1, @@ -15,7 +15,7 @@ class UserCallout < ApplicationRecord suggest_popover_dismissed: 9, tabs_position_highlight: 10, threat_monitoring_info: 11, # EE-only - account_recovery_regular_check: 12, # EE-only + two_factor_auth_recovery_settings_check: 12, # EE-only web_ide_alert_dismissed: 16, # no longer in use active_user_count_threshold: 18, # EE-only buy_pipeline_minutes_notification_dot: 19, # EE-only @@ -39,13 +39,8 @@ class UserCallout < ApplicationRecord terraform_notification_dismissed: 38 } - validates :user, presence: true validates :feature_name, presence: true, uniqueness: { scope: :user_id }, inclusion: { in: UserCallout.feature_names.keys } - - def dismissed_after?(dismissed_after) - dismissed_at > dismissed_after - end end diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index b3cca1e0cc0..c41cff67864 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -2,7 +2,8 @@ class UserDetail < ApplicationRecord extend ::Gitlab::Utils::Override - include CacheMarkdownField + include IgnorableColumns + ignore_columns %i[bio_html cached_markdown_version], remove_with: '13.6', remove_after: '2021-10-22' belongs_to :user @@ -13,20 +14,6 @@ class UserDetail < ApplicationRecord before_save :prevent_nil_bio - cache_markdown_field :bio - - def bio_html - read_attribute(:bio_html) || bio - end - - # For backward compatibility. - # Older migrations (and their tests) reference the `User.migration_bot` where the `bio` attribute is set. - # Here we disable writing the markdown cache when the `bio_html` column does not exist. - override :invalidated_markdown_cache? - def invalidated_markdown_cache? - self.class.column_names.include?('bio_html') && super - end - private def prevent_nil_bio diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index 2735e169b5f..337ae7125f3 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -20,7 +20,7 @@ class UserPreference < ApplicationRecord less_than_or_equal_to: Gitlab::TabWidth::MAX } - enum experience_level: { novice: 0, experienced: 1 } + ignore_columns :experience_level, remove_with: '14.10', remove_after: '2021-03-22' default_value_for :tab_width, value: Gitlab::TabWidth::DEFAULT, allows_nil: false default_value_for :timezone, value: Time.zone.tzinfo.name, allows_nil: false diff --git a/app/models/users/group_callout.rb b/app/models/users/group_callout.rb new file mode 100644 index 00000000000..540d1a1d242 --- /dev/null +++ b/app/models/users/group_callout.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Users + class GroupCallout < ApplicationRecord + include Calloutable + + self.table_name = 'user_group_callouts' + + belongs_to :group + + enum feature_name: { + invite_members_banner: 1 + } + + validates :group, presence: true + validates :feature_name, + presence: true, + uniqueness: { scope: [:user_id, :group_id] }, + inclusion: { in: GroupCallout.feature_names.keys } + + def source_feature_name + "#{feature_name}_#{group_id}" + end + end +end diff --git a/app/models/work_item/type.rb b/app/models/work_item/type.rb index 16cb7a8be45..7038beadd62 100644 --- a/app/models/work_item/type.rb +++ b/app/models/work_item/type.rb @@ -9,14 +9,18 @@ class WorkItem::Type < ApplicationRecord include CacheMarkdownField + # Base types need to exist on the DB on app startup + # This constant is used by the DB seeder + BASE_TYPES = { + issue: { name: 'Issue', icon_name: 'issue-type-issue', enum_value: 0 }, + incident: { name: 'Incident', icon_name: 'issue-type-incident', enum_value: 1 }, + test_case: { name: 'Test Case', icon_name: 'issue-type-test-case', enum_value: 2 }, ## EE-only + requirement: { name: 'Requirement', icon_name: 'issue-type-requirements', enum_value: 3 } ## EE-only + }.freeze + cache_markdown_field :description, pipeline: :single_line - enum base_type: { - issue: 0, - incident: 1, - test_case: 2, ## EE-only - requirement: 3 ## EE-only - } + enum base_type: BASE_TYPES.transform_values { |value| value[:enum_value] } belongs_to :namespace, optional: true has_many :work_items, class_name: 'Issue', foreign_key: :work_item_type_id, inverse_of: :work_item_type @@ -30,6 +34,14 @@ class WorkItem::Type < ApplicationRecord validates :name, length: { maximum: 255 } validates :icon_name, length: { maximum: 255 } + def self.default_by_type(type) + find_by(namespace_id: nil, base_type: type) + end + + def self.default_issue_type + default_by_type(:issue) + end + private def strip_whitespace diff --git a/app/models/zoom_meeting.rb b/app/models/zoom_meeting.rb index f684f9e6fe0..dd230b46d4b 100644 --- a/app/models/zoom_meeting.rb +++ b/app/models/zoom_meeting.rb @@ -10,7 +10,7 @@ class ZoomMeeting < ApplicationRecord validates :project, presence: true, unless: :importing? validates :issue, presence: true, unless: :importing? - validates :url, presence: true, length: { maximum: 255 }, 'gitlab/utils/zoom_url': true + validates :url, presence: true, length: { maximum: 255 }, 'gitlab/zoom_url': true validates :issue, same_project_association: true, unless: :importing? enum issue_status: { |