diff options
Diffstat (limited to 'app/models')
116 files changed, 1457 insertions, 903 deletions
diff --git a/app/models/active_session.rb b/app/models/active_session.rb index a0e74c7f48e..0094d98fb73 100644 --- a/app/models/active_session.rb +++ b/app/models/active_session.rb @@ -4,7 +4,7 @@ # # The raw session information is stored by the Rails session store # (config/initializers/session_store.rb). These entries are accessible by the -# rack_key_name class method and consistute the base of the session data +# rack_key_name class method and constitute the base of the session data # entries. All other entries in the session store can be traced back to these # entries. # @@ -21,14 +21,24 @@ # class ActiveSession include ActiveModel::Model + include ::Gitlab::Redis::SessionsStoreHelper SESSION_BATCH_SIZE = 200 ALLOWED_NUMBER_OF_ACTIVE_SESSIONS = 100 - attr_accessor :created_at, :updated_at, - :ip_address, :browser, :os, - :device_name, :device_type, - :is_impersonated, :session_id, :session_private_id + attr_accessor :ip_address, :browser, :os, + :device_name, :device_type, + :is_impersonated, :session_id, :session_private_id + + attr_reader :created_at, :updated_at + + def created_at=(time) + @created_at = time.is_a?(String) ? Time.zone.parse(time) : time + end + + def updated_at=(time) + @updated_at = time.is_a?(String) ? Time.zone.parse(time) : time + end def current?(rack_session) return false if session_private_id.nil? || rack_session.id.nil? @@ -38,15 +48,29 @@ class ActiveSession session_private_id == rack_session.id.private_id end + def eql?(other) + other.is_a?(self.class) && id == other.id + end + alias_method :==, :eql? + + def id + session_private_id.presence || session_id + end + + def ids + [session_private_id, session_id].compact + end + def human_device_type device_type&.titleize end def self.set(user, request) - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| session_private_id = request.session.id.private_id client = DeviceDetector.new(request.user_agent) timestamp = Time.current + expiry = Settings.gitlab['session_expire_delay'] * 60 active_user_session = new( ip_address: request.remote_ip, @@ -63,7 +87,14 @@ class ActiveSession redis.pipelined do redis.setex( key_name(user.id, session_private_id), - Settings.gitlab['session_expire_delay'] * 60, + expiry, + active_user_session.dump + ) + + # Deprecated legacy format - temporary to support mixed deployments + redis.setex( + key_name_v1(user.id, session_private_id), + expiry, Marshal.dump(active_user_session) ) @@ -76,7 +107,7 @@ class ActiveSession end def self.list(user) - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| cleaned_up_lookup_entries(redis, user).map do |raw_session| load_raw_session(raw_session) end @@ -84,14 +115,17 @@ class ActiveSession end def self.cleanup(user) - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| clean_up_old_sessions(redis, user) cleaned_up_lookup_entries(redis, user) end end def self.destroy_sessions(redis, user, session_ids) + return if session_ids.empty? + key_names = session_ids.map { |session_id| key_name(user.id, session_id) } + key_names += session_ids.map { |session_id| key_name_v1(user.id, session_id) } redis.srem(lookup_key_name(user.id), session_ids) @@ -104,7 +138,7 @@ class ActiveSession def self.destroy_session(user, session_id) return unless session_id - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| destroy_sessions(redis, user, [session_id].compact) end end @@ -113,26 +147,31 @@ class ActiveSession sessions = not_impersonated(user) sessions.reject! { |session| session.current?(current_rack_session) } if current_rack_session - Gitlab::Redis::SharedState.with do |redis| - session_ids = (sessions.map(&:session_id) | sessions.map(&:session_private_id)).compact + redis_store_class.with do |redis| + session_ids = sessions.flat_map(&:ids) destroy_sessions(redis, user, session_ids) if session_ids.any? end end - def self.not_impersonated(user) + private_class_method def self.not_impersonated(user) list(user).reject(&:is_impersonated) end - def self.rack_key_name(session_id) - "#{Gitlab::Redis::SharedState::SESSION_NAMESPACE}:#{session_id}" + private_class_method def self.rack_key_name(session_id) + "#{Gitlab::Redis::Sessions::SESSION_NAMESPACE}:#{session_id}" end def self.key_name(user_id, session_id = '*') - "#{Gitlab::Redis::SharedState::USER_SESSIONS_NAMESPACE}:#{user_id}:#{session_id}" + "#{Gitlab::Redis::Sessions::USER_SESSIONS_NAMESPACE}::v2:#{user_id}:#{session_id}" + end + + # Deprecated + def self.key_name_v1(user_id, session_id = '*') + "#{Gitlab::Redis::Sessions::USER_SESSIONS_NAMESPACE}:#{user_id}:#{session_id}" end def self.lookup_key_name(user_id) - "#{Gitlab::Redis::SharedState::USER_SESSIONS_LOOKUP_NAMESPACE}:#{user_id}" + "#{Gitlab::Redis::Sessions::USER_SESSIONS_LOOKUP_NAMESPACE}:#{user_id}" end def self.list_sessions(user) @@ -143,7 +182,7 @@ class ActiveSession # # Returns an array of strings def self.session_ids_for_user(user_id) - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| redis.smembers(lookup_key_name(user_id)) end end @@ -156,7 +195,7 @@ class ActiveSession def self.sessions_from_ids(session_ids) return [] if session_ids.empty? - Gitlab::Redis::SharedState.with do |redis| + redis_store_class.with do |redis| session_keys = rack_session_keys(session_ids) session_keys.each_slice(SESSION_BATCH_SIZE).flat_map do |session_keys_batch| @@ -169,71 +208,102 @@ class ActiveSession end end - # Deserializes a session Hash object from Redis. - # + def dump + "v2:#{Gitlab::Json.dump(self)}" + end + + # Private: + # raw_session - Raw bytes from Redis # - # Returns an ActiveSession object - def self.load_raw_session(raw_session) - # rubocop:disable Security/MarshalLoad - Marshal.load(raw_session) - # rubocop:enable Security/MarshalLoad + # Returns an instance of this class + private_class_method def self.load_raw_session(raw_session) + return unless raw_session + + if raw_session.start_with?('v2:') + session_data = Gitlab::Json.parse(raw_session[3..]).symbolize_keys + new(**session_data) + else + # Deprecated legacy format. To be removed in 15.0 + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/30516 + # Explanation of why this Marshal.load call is OK: + # https://gitlab.com/gitlab-com/gl-security/appsec/appsec-reviews/-/issues/124#note_744576714 + # rubocop:disable Security/MarshalLoad + Marshal.load(raw_session) + # rubocop:enable Security/MarshalLoad + end end - def self.rack_session_keys(rack_session_ids) - rack_session_ids.map { |session_id| rack_key_name(session_id)} + private_class_method def self.rack_session_keys(rack_session_ids) + rack_session_ids.map { |session_id| rack_key_name(session_id) } end - def self.raw_active_session_entries(redis, session_ids, user_id) - return [] if session_ids.empty? + private_class_method def self.raw_active_session_entries(redis, session_ids, user_id) + return {} if session_ids.empty? + + found = Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + entry_keys = session_ids.map { |session_id| key_name(user_id, session_id) } + session_ids.zip(redis.mget(entry_keys)).to_h + end - entry_keys = session_ids.map { |session_id| key_name(user_id, session_id) } + found.compact! + missing = session_ids - found.keys + return found if missing.empty? - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.mget(entry_keys) + fallbacks = Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + entry_keys = missing.map { |session_id| key_name_v1(user_id, session_id) } + missing.zip(redis.mget(entry_keys)).to_h end + + fallbacks.merge(found.compact) end - def self.active_session_entries(session_ids, user_id, redis) + private_class_method def self.active_session_entries(session_ids, user_id, redis) return [] if session_ids.empty? - entry_keys = raw_active_session_entries(redis, session_ids, user_id) - - entry_keys.compact.map do |raw_session| - load_raw_session(raw_session) - end + raw_active_session_entries(redis, session_ids, user_id) + .values + .compact + .map { load_raw_session(_1) } end - def self.clean_up_old_sessions(redis, user) + private_class_method def self.clean_up_old_sessions(redis, user) session_ids = session_ids_for_user(user.id) return if session_ids.count <= ALLOWED_NUMBER_OF_ACTIVE_SESSIONS - # remove sessions if there are more than ALLOWED_NUMBER_OF_ACTIVE_SESSIONS. sessions = active_session_entries(session_ids, user.id, redis) - sessions.sort_by! {|session| session.updated_at }.reverse! - destroyable_sessions = sessions.drop(ALLOWED_NUMBER_OF_ACTIVE_SESSIONS) - destroyable_session_ids = destroyable_sessions.flat_map { |session| [session.session_id, session.session_private_id] }.compact - destroy_sessions(redis, user, destroyable_session_ids) if destroyable_session_ids.any? + sessions.sort_by!(&:updated_at).reverse! + + # remove sessions if there are more than ALLOWED_NUMBER_OF_ACTIVE_SESSIONS. + destroyable_session_ids = sessions + .drop(ALLOWED_NUMBER_OF_ACTIVE_SESSIONS) + .flat_map(&:ids) + + destroy_sessions(redis, user, destroyable_session_ids) end # Cleans up the lookup set by removing any session IDs that are no longer present. # # Returns an array of marshalled ActiveModel objects that are still active. - def self.cleaned_up_lookup_entries(redis, user) + # Records removed keys in the optional `removed` argument array. + def self.cleaned_up_lookup_entries(redis, user, removed = []) + lookup_key = lookup_key_name(user.id) session_ids = session_ids_for_user(user.id) - entries = raw_active_session_entries(redis, session_ids, user.id) + session_ids_and_entries = raw_active_session_entries(redis, session_ids, user.id) # remove expired keys. # only the single key entries are automatically expired by redis, the # lookup entries in the set need to be removed manually. - session_ids_and_entries = session_ids.zip(entries) - redis.pipelined do - session_ids_and_entries.reject { |_session_id, entry| entry }.each do |session_id, _entry| - redis.srem(lookup_key_name(user.id), session_id) + redis.pipelined do |pipeline| + session_ids_and_entries.each do |session_id, entry| + next if entry + + pipeline.srem(lookup_key, session_id) + removed << session_id end end - entries.compact + session_ids_and_entries.values.compact end end diff --git a/app/models/analytics/cycle_analytics/project_stage.rb b/app/models/analytics/cycle_analytics/project_stage.rb index e8b03fa066a..8d3a032812e 100644 --- a/app/models/analytics/cycle_analytics/project_stage.rb +++ b/app/models/analytics/cycle_analytics/project_stage.rb @@ -26,6 +26,12 @@ module Analytics :project_id end + def self.distinct_stages_within_hierarchy(group) + with_preloaded_labels + .where(project_id: group.all_projects.select(:id)) + .select("DISTINCT ON(stage_event_hash_id) #{quoted_table_name}.*") + end + private # Project should belong to a group when the stage has Label based events since only GroupLabels are allowed. diff --git a/app/models/application_record.rb b/app/models/application_record.rb index bcd8bdd6638..b64e6c59817 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -7,6 +7,10 @@ class ApplicationRecord < ActiveRecord::Base self.abstract_class = true + # We should avoid using pluck https://docs.gitlab.com/ee/development/sql.html#plucking-ids + # but, if we are going to use it, let's try and limit the number of records + MAX_PLUCK = 1_000 + alias_method :reset, :reload def self.without_order diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index af5796d682f..65472615f42 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -21,7 +21,7 @@ class ApplicationSetting < ApplicationRecord add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption) ? :optional : :required } add_authentication_token_field :health_check_access_token - add_authentication_token_field :static_objects_external_storage_auth_token + add_authentication_token_field :static_objects_external_storage_auth_token, encrypted: :optional belongs_to :self_monitoring_project, class_name: "Project", foreign_key: 'instance_administration_project_id' belongs_to :push_rule @@ -144,10 +144,6 @@ class ApplicationSetting < ApplicationRecord length: { maximum: 2000, message: _('is too long (maximum is %{count} characters)') }, allow_blank: true - validates :spam_check_api_key, - presence: true, - if: :spam_check_endpoint_enabled - validates :unique_ips_limit_per_user, numericality: { greater_than_or_equal_to: 1 }, presence: true, @@ -410,7 +406,7 @@ class ApplicationSetting < ApplicationRecord if: :external_authorization_service_enabled validates :spam_check_endpoint_url, - addressable_url: { schemes: %w(grpc) }, allow_blank: true + addressable_url: { schemes: %w(tls grpc) }, allow_blank: true validates :spam_check_endpoint_url, presence: true, diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 54ec8b2c3e4..5e20aac3b92 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -363,6 +363,14 @@ module ApplicationSettingImplementation super(levels&.map { |level| Gitlab::VisibilityLevel.level_value(level) }) end + def static_objects_external_storage_auth_token=(token) + if token.present? + set_static_objects_external_storage_auth_token(token) + else + self.static_objects_external_storage_auth_token_encrypted = nil + end + end + def performance_bar_allowed_group Group.find_by_id(performance_bar_allowed_group_id) end diff --git a/app/models/bulk_imports/entity.rb b/app/models/bulk_imports/entity.rb index 2368be6196c..38b7da76306 100644 --- a/app/models/bulk_imports/entity.rb +++ b/app/models/bulk_imports/entity.rb @@ -20,8 +20,6 @@ class BulkImports::Entity < ApplicationRecord self.table_name = 'bulk_import_entities' - EXPORT_RELATIONS_URL = '/%{resource}/%{full_path}/export_relations' - belongs_to :bulk_import, optional: false belongs_to :parent, class_name: 'BulkImports::Entity', optional: true @@ -104,18 +102,42 @@ class BulkImports::Entity < ApplicationRecord end end + def entity_type + source_type.gsub('_entity', '') + end + def pluralized_name - source_type.gsub('_entity', '').pluralize + entity_type.pluralize + end + + def base_resource_url_path + "/#{pluralized_name}/#{encoded_source_full_path}" end def export_relations_url_path - @export_relations_url_path ||= EXPORT_RELATIONS_URL % { resource: pluralized_name, full_path: encoded_source_full_path } + "#{base_resource_url_path}/export_relations" end def relation_download_url_path(relation) "#{export_relations_url_path}/download?relation=#{relation}" end + def wikis_url_path + "#{base_resource_url_path}/wikis" + end + + def project? + source_type == 'project_entity' + end + + def group? + source_type == 'group_entity' + end + + def update_service + "::#{pluralized_name.capitalize}::UpdateService".constantize + end + private def validate_parent_is_a_group diff --git a/app/models/bulk_imports/file_transfer/base_config.rb b/app/models/bulk_imports/file_transfer/base_config.rb index 4d370315ad5..036d511bc59 100644 --- a/app/models/bulk_imports/file_transfer/base_config.rb +++ b/app/models/bulk_imports/file_transfer/base_config.rb @@ -5,6 +5,9 @@ module BulkImports class BaseConfig include Gitlab::Utils::StrongMemoize + UPLOADS_RELATION = 'uploads' + SELF_RELATION = 'self' + def initialize(portable) @portable = portable end @@ -26,7 +29,11 @@ module BulkImports end def portable_relations - tree_relations + file_relations - skipped_relations + tree_relations + file_relations + self_relation - skipped_relations + end + + def self_relation?(relation) + relation == SELF_RELATION end def tree_relation?(relation) @@ -43,6 +50,10 @@ module BulkImports portable_tree[:include].find { |include| include[relation.to_sym] } end + def portable_relations_tree + @portable_relations_tree ||= attributes_finder.find_relations_tree(portable_class_sym).deep_stringify_keys + end + private attr_reader :portable @@ -65,10 +76,6 @@ module BulkImports @portable_class_sym ||= portable_class.to_s.demodulize.underscore.to_sym end - def portable_relations_tree - @portable_relations_tree ||= attributes_finder.find_relations_tree(portable_class_sym).deep_stringify_keys - end - def import_export_yaml raise NotImplementedError end @@ -78,12 +85,16 @@ module BulkImports end def file_relations - [] + [UPLOADS_RELATION] end def skipped_relations [] end + + def self_relation + [SELF_RELATION] + end end end end diff --git a/app/models/bulk_imports/file_transfer/project_config.rb b/app/models/bulk_imports/file_transfer/project_config.rb index 9a0434da08a..fdfb0dd0186 100644 --- a/app/models/bulk_imports/file_transfer/project_config.rb +++ b/app/models/bulk_imports/file_transfer/project_config.rb @@ -3,8 +3,6 @@ module BulkImports module FileTransfer class ProjectConfig < BaseConfig - UPLOADS_RELATION = 'uploads' - SKIPPED_RELATIONS = %w( project_members group_members @@ -14,10 +12,6 @@ module BulkImports ::Gitlab::ImportExport.config_file end - def file_relations - [UPLOADS_RELATION] - end - def skipped_relations SKIPPED_RELATIONS end diff --git a/app/models/bulk_imports/tracker.rb b/app/models/bulk_imports/tracker.rb index 9de3239ee0f..cfe33c013ba 100644 --- a/app/models/bulk_imports/tracker.rb +++ b/app/models/bulk_imports/tracker.rb @@ -29,7 +29,7 @@ class BulkImports::Tracker < ApplicationRecord def self.stage_running?(entity_id, stage) where(stage: stage, bulk_import_entity_id: entity_id) - .with_status(:created, :started) + .with_status(:created, :enqueued, :started) .exists? end @@ -45,15 +45,24 @@ class BulkImports::Tracker < ApplicationRecord state :created, value: 0 state :started, value: 1 state :finished, value: 2 + state :enqueued, value: 3 state :failed, value: -1 state :skipped, value: -2 event :start do - transition created: :started + transition enqueued: :started # To avoid errors when re-starting a pipeline in case of network errors transition started: :started end + event :retry do + transition started: :enqueued + end + + event :enqueue do + transition created: :enqueued + end + event :finish do transition started: :finished transition failed: :failed diff --git a/app/models/chat_name.rb b/app/models/chat_name.rb index da7312df18b..ff3f2663b73 100644 --- a/app/models/chat_name.rb +++ b/app/models/chat_name.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true class ChatName < ApplicationRecord - include LooseForeignKey - LAST_USED_AT_INTERVAL = 1.hour belongs_to :integration, foreign_key: :service_id @@ -16,8 +14,6 @@ class ChatName < ApplicationRecord validates :user_id, uniqueness: { scope: [:service_id] } validates :chat_id, uniqueness: { scope: [:service_id, :team_id] } - loose_foreign_key :ci_pipeline_chat_data, :chat_name_id, on_delete: :async_delete - # Updates the "last_used_timestamp" but only if it wasn't already updated # recently. # diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 3fdc44bccf3..428e440afba 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -10,6 +10,7 @@ module Ci include Presentable include Importable include Ci::HasRef + extend ::Gitlab::Utils::Override BuildArchivedError = Class.new(StandardError) @@ -58,7 +59,7 @@ module Ci has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build has_one :trace_metadata, class_name: 'Ci::BuildTraceMetadata', inverse_of: :build - has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', dependent: :nullify, inverse_of: :build, foreign_key: :ci_build_id # rubocop:disable Cop/ActiveRecordDependent + has_many :terraform_state_versions, class_name: 'Terraform::StateVersion', inverse_of: :build, foreign_key: :ci_build_id accepts_nested_attributes_for :runner_session, update_only: true accepts_nested_attributes_for :job_variables @@ -164,6 +165,7 @@ module Ci scope :with_artifacts_not_expired, -> { with_downloadable_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.current) } scope :with_expired_artifacts, -> { with_downloadable_artifacts.where('artifacts_expire_at < ?', Time.current) } + scope :with_pipeline_locked_artifacts, -> { joins(:pipeline).where('pipeline.locked': Ci::Pipeline.lockeds[:artifacts_locked]) } scope :last_month, -> { where('created_at > ?', Date.today - 1.month) } scope :manual_actions, -> { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) } scope :scheduled_actions, -> { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) } @@ -188,8 +190,6 @@ module Ci scope :without_coverage, -> { where(coverage: nil) } scope :with_coverage_regex, -> { where.not(coverage_regex: nil) } - scope :for_project, -> (project_id) { where(project_id: project_id) } - acts_as_taggable add_authentication_token_field :token, encrypted: :required @@ -286,6 +286,7 @@ module Ci build.run_after_commit do BuildQueueWorker.perform_async(id) + BuildHooksWorker.perform_async(id) end end @@ -451,7 +452,7 @@ module Ci end def retryable? - return false if retried? || archived? + return false if retried? || archived? || deployment_rejected? success? || failed? || canceled? end @@ -722,6 +723,14 @@ module Ci self.token && ActiveSupport::SecurityUtils.secure_compare(token, self.token) end + # acts_as_taggable uses this method create/remove tags with contexts + # defined by taggings and to get those contexts it executes a query. + # We don't use any other contexts except `tags`, so we don't need it. + override :custom_contexts + def custom_contexts + [] + end + def tag_list if tags.loaded? tags.map(&:name) @@ -1074,6 +1083,16 @@ module Ci runner&.instance_type? end + def job_variables_attributes + strong_memoize(:job_variables_attributes) do + job_variables.internal_source.map do |variable| + variable.attributes.except('id', 'job_id', 'encrypted_value', 'encrypted_value_iv').tap do |attrs| + attrs[:value] = variable.value + end + end + end + end + protected def run_status_commit_hooks! diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index ec1137920ef..e6dd62fab34 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -2,6 +2,7 @@ module Ci class JobArtifact < Ci::ApplicationRecord + include IgnorableColumns include AfterCommitQueue include ObjectStorage::BackgroundMove include UpdateProjectStatistics @@ -120,6 +121,9 @@ module Ci belongs_to :project belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id + # We will start using this column once we complete https://gitlab.com/gitlab-org/gitlab/-/issues/285597 + ignore_column :original_filename, remove_with: '14.7', remove_after: '2022-11-22' + mount_file_store_uploader JobArtifactUploader skip_callback :save, :after, :store_file!, if: :store_after_commit? @@ -133,6 +137,7 @@ module Ci scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } + scope :for_job_ids, ->(job_ids) { where(job_id: job_ids) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } scope :with_job, -> { joins(:job).includes(:job) } @@ -266,6 +271,10 @@ module Ci self.where(project: project).sum(:size) end + def self.distinct_job_ids + distinct.pluck(:job_id) + end + ## # FastDestroyAll concerns # rubocop: disable CodeReuse/ServiceClass diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb new file mode 100644 index 00000000000..8a4be3139e8 --- /dev/null +++ b/app/models/ci/namespace_mirror.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Ci + # This model represents a record in a shadow table of the main database's namespaces table. + # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN. + class NamespaceMirror < ApplicationRecord + belongs_to :namespace + + scope :contains_namespace, -> (id) do + where('traversal_ids @> ARRAY[?]::int[]', id) + end + + class << self + def sync!(event) + namespace = event.namespace + traversal_ids = namespace.self_and_ancestor_ids(hierarchy_order: :desc) + + upsert({ namespace_id: event.namespace_id, traversal_ids: traversal_ids }, + unique_by: :namespace_id) + + # It won't be necessary once we remove `sync_traversal_ids`. + # More info: https://gitlab.com/gitlab-org/gitlab/-/issues/347541 + sync_children_namespaces!(event.namespace_id, traversal_ids) + end + + private + + def sync_children_namespaces!(namespace_id, traversal_ids) + contains_namespace(namespace_id) + .where.not(namespace_id: namespace_id) + .update_all( + "traversal_ids = ARRAY[#{sanitize_sql(traversal_ids.join(','))}]::int[] || traversal_ids[array_position(traversal_ids, #{sanitize_sql(namespace_id)}) + 1:]" + ) + end + end + end +end diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index ccad6290fac..41dc74ef050 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -30,6 +30,10 @@ module Ci self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) end + def maintain_denormalized_data? + ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml) + end + private def args_from_build(build) @@ -42,15 +46,9 @@ module Ci namespace: project.namespace } - if Feature.enabled?(:ci_pending_builds_maintain_tags_data, type: :development, default_enabled: :yaml) + if maintain_denormalized_data? args.store(:tag_ids, build.tags_ids) - end - - if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) args.store(:instance_runners_enabled, shared_runners_enabled?(project)) - end - - if Feature.enabled?(:ci_pending_builds_maintain_namespace_traversal_ids, type: :development, default_enabled: :yaml) args.store(:namespace_traversal_ids, project.namespace.traversal_ids) if group_runners_enabled?(project) end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index a29aa756e38..a90bd739741 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -63,6 +63,7 @@ module Ci has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :latest_statuses_ordered_by_stage, -> { latest.order(:stage_idx, :stage) }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline has_many :latest_statuses, -> { latest }, class_name: 'CommitStatus', foreign_key: :commit_id, inverse_of: :pipeline + has_many :statuses_order_id_desc, -> { order_id_desc }, class_name: 'CommitStatus', foreign_key: :commit_id has_many :processables, class_name: 'Ci::Processable', foreign_key: :commit_id, inverse_of: :pipeline has_many :bridges, class_name: 'Ci::Bridge', foreign_key: :commit_id, inverse_of: :pipeline has_many :builds, foreign_key: :commit_id, inverse_of: :pipeline @@ -82,8 +83,6 @@ module Ci # Merge requests for which the current pipeline is running against # the merge request's latest commit. has_many :merge_requests_as_head_pipeline, foreign_key: "head_pipeline_id", class_name: 'MergeRequest' - has_many :package_build_infos, class_name: 'Packages::BuildInfo', dependent: :nullify, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent - has_many :package_file_build_infos, class_name: 'Packages::PackageFileBuildInfo', dependent: :nullify, inverse_of: :pipeline # rubocop:disable Cop/ActiveRecordDependent has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :failed_builds, -> { latest.failed }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline @@ -236,7 +235,18 @@ module Ci pipeline.run_after_commit do PipelineHooksWorker.perform_async(pipeline.id) - ExpirePipelineCacheWorker.perform_async(pipeline.id) + + if pipeline.project.jira_subscription_exists? + # Passing the seq-id ensures this is idempotent + seq_id = ::Atlassian::JiraConnect::Client.generate_update_sequence_id + ::JiraConnect::SyncBuildsWorker.perform_async(pipeline.id, seq_id) + end + + if Feature.enabled?(:expire_job_and_pipeline_cache_synchronously, pipeline.project, default_enabled: :yaml) + Ci::ExpirePipelineCacheService.new.execute(pipeline) # rubocop: disable CodeReuse/ServiceClass + else + ExpirePipelineCacheWorker.perform_async(pipeline.id) + end end end @@ -271,14 +281,6 @@ module Ci end end - after_transition any => any do |pipeline| - pipeline.run_after_commit do - # Passing the seq-id ensures this is idempotent - seq_id = ::Atlassian::JiraConnect::Client.generate_update_sequence_id - ::JiraConnect::SyncBuildsWorker.perform_async(pipeline.id, seq_id) - end - end - after_transition any => ::Ci::Pipeline.completed_statuses do |pipeline| pipeline.run_after_commit do ::Ci::TestFailureHistoryService.new(pipeline).async.perform_if_needed # rubocop: disable CodeReuse/ServiceClass @@ -643,7 +645,7 @@ module Ci def coverage coverage_array = latest_statuses.map(&:coverage).compact if coverage_array.size >= 1 - '%.2f' % (coverage_array.reduce(:+) / coverage_array.size) + coverage_array.reduce(:+) / coverage_array.size end end @@ -947,22 +949,16 @@ module Ci end def environments_in_self_and_descendants - if ::Feature.enabled?(:avoid_cross_joins_environments_in_self_and_descendants, default_enabled: :yaml) - # We limit to 100 unique environments for application safety. - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 - expanded_environment_names = - builds_in_self_and_descendants.joins(:metadata) - .where.not('ci_builds_metadata.expanded_environment_name' => nil) - .distinct('ci_builds_metadata.expanded_environment_name') - .limit(100) - .pluck(:expanded_environment_name) - - Environment.where(project: project, name: expanded_environment_names) - else - environment_ids = self_and_descendants.joins(:deployments).select(:'deployments.environment_id') + # We limit to 100 unique environments for application safety. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 + expanded_environment_names = + builds_in_self_and_descendants.joins(:metadata) + .where.not('ci_builds_metadata.expanded_environment_name' => nil) + .distinct('ci_builds_metadata.expanded_environment_name') + .limit(100) + .pluck(:expanded_environment_name) - Environment.where(id: environment_ids) - end + Environment.where(project: project, name: expanded_environment_names).with_deployment(sha) end # With multi-project and parent-child pipelines @@ -1276,18 +1272,18 @@ module Ci self.builds.latest.build_matchers(project) end - def predefined_vars_in_builder_enabled? - strong_memoize(:predefined_vars_in_builder_enabled) do - Feature.enabled?(:ci_predefined_vars_in_builder, project, default_enabled: :yaml) - end - end - def authorized_cluster_agents strong_memoize(:authorized_cluster_agents) do ::Clusters::AgentAuthorizationsFinder.new(project).execute.map(&:agent) end end + def create_deployment_in_separate_transaction? + strong_memoize(:create_deployment_in_separate_transaction) do + ::Feature.enabled?(:create_deployment_in_separate_transaction, project, default_enabled: :yaml) + end + end + private def add_message(severity, content) diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb new file mode 100644 index 00000000000..d6aaa3f50c1 --- /dev/null +++ b/app/models/ci/project_mirror.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Ci + # This model represents a shadow table of the main database's projects table. + # It allows us to navigate the project and namespace hierarchy on the ci database. + class ProjectMirror < ApplicationRecord + belongs_to :project + + class << self + def sync!(event) + upsert({ project_id: event.project_id, namespace_id: event.project.namespace_id }, + unique_by: :project_id) + end + end + end +end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 8a3025e5608..a80fd02080f 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -12,7 +12,6 @@ module Ci include Gitlab::Utils::StrongMemoize include TaggableQueries include Presentable - include LooseForeignKey add_authentication_token_field :token, encrypted: :optional @@ -27,6 +26,21 @@ module Ci project_type: 3 } + enum executor_type: { + unknown: 0, + custom: 1, + shell: 2, + docker: 3, + docker_windows: 4, + docker_ssh: 5, + ssh: 6, + parallels: 7, + virtualbox: 8, + docker_machine: 9, + docker_ssh_machine: 10, + kubernetes: 11 + }, _suffix: true + # This `ONLINE_CONTACT_TIMEOUT` needs to be larger than # `RUNNER_QUEUE_EXPIRY_TIME+UPDATE_CONTACT_COLUMN_EVERY` # @@ -40,9 +54,12 @@ module Ci # The `UPDATE_CONTACT_COLUMN_EVERY` defines how often the Runner DB entry can be updated UPDATE_CONTACT_COLUMN_EVERY = (40.minutes..55.minutes).freeze + # The `STALE_TIMEOUT` constant defines the how far past the last contact or creation date a runner will be considered stale + STALE_TIMEOUT = 3.months + AVAILABLE_TYPES_LEGACY = %w[specific shared].freeze AVAILABLE_TYPES = runner_types.keys.freeze - AVAILABLE_STATUSES = %w[active paused online offline not_connected].freeze + AVAILABLE_STATUSES = %w[active paused online offline not_connected never_contacted stale].freeze # TODO: Remove in %15.0: active, paused, not_connected. Relevant issues: https://gitlab.com/gitlab-org/gitlab/-/issues/347303, https://gitlab.com/gitlab-org/gitlab/-/issues/347305, https://gitlab.com/gitlab-org/gitlab/-/issues/344648 AVAILABLE_SCOPES = (AVAILABLE_TYPES_LEGACY + AVAILABLE_TYPES + AVAILABLE_STATUSES).freeze FORM_EDITABLE = %i[description tag_list active run_untagged locked access_level maximum_timeout_human_readable].freeze @@ -58,12 +75,14 @@ module Ci before_save :ensure_token - scope :active, -> { where(active: true) } - scope :paused, -> { where(active: false) } + scope :active, -> (value = true) { where(active: value) } + scope :paused, -> { active(false) } scope :online, -> { where('contacted_at > ?', online_contact_time_deadline) } - scope :recent, -> { where('ci_runners.created_at > :date OR ci_runners.contacted_at > :date', date: 3.months.ago) } + scope :recent, -> { where('ci_runners.created_at >= :date OR ci_runners.contacted_at >= :date', date: stale_deadline) } + scope :stale, -> { where('ci_runners.created_at < :date AND (ci_runners.contacted_at IS NULL OR ci_runners.contacted_at < :date)', date: stale_deadline) } scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) } - scope :not_connected, -> { where(contacted_at: nil) } + scope :not_connected, -> { where(contacted_at: nil) } # TODO: Remove in 15.0 + scope :never_contacted, -> { where(contacted_at: nil) } scope :ordered, -> { order(id: :desc) } scope :with_recent_runner_queue, -> { where('contacted_at > ?', recent_queue_deadline) } @@ -78,10 +97,7 @@ module Ci scope :belonging_to_group, -> (group_id, include_ancestors: false) { groups = ::Group.where(id: group_id) - - if include_ancestors - groups = Gitlab::ObjectHierarchy.new(groups).base_and_ancestors - end + groups = groups.self_and_ancestors if include_ancestors joins(:runner_namespaces) .where(ci_runner_namespaces: { namespace_id: groups }) @@ -102,10 +118,9 @@ module Ci scope :belonging_to_parent_group_of_project, -> (project_id) { project_groups = ::Group.joins(:projects).where(projects: { id: project_id }) - hierarchy_groups = Gitlab::ObjectHierarchy.new(project_groups).base_and_ancestors joins(:groups) - .where(namespaces: { id: hierarchy_groups }) + .where(namespaces: { id: project_groups.self_and_ancestors.as_ids }) .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336433') } @@ -152,7 +167,7 @@ module Ci after_destroy :cleanup_runner_queue - cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at + cached_attr_reader :version, :revision, :platform, :architecture, :ip_address, :contacted_at, :executor_type chronic_duration_attr :maximum_timeout_human_readable, :maximum_timeout, error_message: 'Maximum job timeout has a value which could not be accepted' @@ -168,8 +183,6 @@ module Ci validates :config, json_schema: { filename: 'ci_runner_config' } - loose_foreign_key :clusters_applications_runners, :runner_id, on_delete: :async_nullify - # Searches for runners matching the given query. # # This method uses ILIKE on PostgreSQL for the description field and performs a full match on tokens. @@ -185,6 +198,10 @@ module Ci ONLINE_CONTACT_TIMEOUT.ago end + def self.stale_deadline + STALE_TIMEOUT.ago + end + def self.recent_queue_deadline # we add queue expiry + online # - contacted_at can be updated at any time within this interval @@ -273,8 +290,17 @@ module Ci contacted_at && contacted_at > self.class.online_contact_time_deadline end - def status - return :not_connected unless contacted_at + def stale? + return false unless created_at + + [created_at, contacted_at].compact.max < self.class.stale_deadline + end + + def status(legacy_mode = nil) + return deprecated_rest_status if legacy_mode == '14.5' + + return :stale if stale? + return :never_contacted unless contacted_at online? ? :online : :offline end @@ -387,8 +413,9 @@ module Ci # database after heartbeat write happens. # ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes do - values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config) || {} + values = values&.slice(:version, :revision, :platform, :architecture, :ip_address, :config, :executor) || {} values[:contacted_at] = Time.current + values[:executor_type] = EXECUTOR_NAME_TO_TYPES.fetch(values.delete(:executor), :unknown) cache_attributes(values) @@ -413,6 +440,20 @@ module Ci private + EXECUTOR_NAME_TO_TYPES = { + 'custom' => :custom, + 'shell' => :shell, + 'docker' => :docker, + 'docker-windows' => :docker_windows, + 'docker-ssh' => :docker_ssh, + 'ssh' => :ssh, + 'parallels' => :parallels, + 'virtualbox' => :virtualbox, + 'docker+machine' => :docker_machine, + 'docker-ssh+machine' => :docker_ssh_machine, + 'kubernetes' => :kubernetes + }.freeze + def cleanup_runner_queue Gitlab::Redis::SharedState.with do |redis| redis.del(runner_queue_key) diff --git a/app/models/ci/runner_namespace.rb b/app/models/ci/runner_namespace.rb index 52a31863fb2..82390ccc538 100644 --- a/app/models/ci/runner_namespace.rb +++ b/app/models/ci/runner_namespace.rb @@ -7,7 +7,6 @@ module Ci self.limit_name = 'ci_registered_group_runners' self.limit_scope = :group self.limit_relation = :recent_runners - self.limit_feature_flag_for_override = :ci_runner_limits_override belongs_to :runner, inverse_of: :runner_namespaces belongs_to :namespace, inverse_of: :runner_namespaces, class_name: '::Namespace' diff --git a/app/models/ci/runner_project.rb b/app/models/ci/runner_project.rb index 148a29a0f8b..42c24c8c8d1 100644 --- a/app/models/ci/runner_project.rb +++ b/app/models/ci/runner_project.rb @@ -7,7 +7,6 @@ module Ci self.limit_name = 'ci_registered_project_runners' self.limit_scope = :project self.limit_relation = :recent_runners - self.limit_feature_flag_for_override = :ci_runner_limits_override belongs_to :runner, inverse_of: :runner_projects belongs_to :project, inverse_of: :runner_projects diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index e2b15497638..8c4e97ac840 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -22,6 +22,7 @@ module Ci scope :ordered, -> { order(position: :asc) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } scope :by_name, ->(names) { where(name: names) } + scope :by_position, ->(positions) { where(position: positions) } with_options unless: :importing? do validates :project, presence: true diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb index cf6d95fc6df..98490a13351 100644 --- a/app/models/clusters/agent.rb +++ b/app/models/clusters/agent.rb @@ -4,6 +4,8 @@ module Clusters class Agent < ApplicationRecord self.table_name = 'cluster_agents' + INACTIVE_AFTER = 1.hour.freeze + belongs_to :created_by_user, class_name: 'User', optional: true belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project @@ -16,6 +18,8 @@ module Clusters has_many :project_authorizations, class_name: 'Clusters::Agents::ProjectAuthorization' has_many :authorized_projects, class_name: '::Project', through: :project_authorizations, source: :project + has_many :activity_events, -> { in_timeline_order }, class_name: 'Clusters::Agents::ActivityEvent', inverse_of: :agent + scope :ordered_by_name, -> { order(:name) } scope :with_name, -> (name) { where(name: name) } @@ -31,5 +35,9 @@ module Clusters def has_access_to?(requested_project) requested_project == project end + + def active? + agent_tokens.where("last_used_at > ?", INACTIVE_AFTER.ago).exists? + end end end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb index 27a3cd8d13d..87dba50cd69 100644 --- a/app/models/clusters/agent_token.rb +++ b/app/models/clusters/agent_token.rb @@ -28,8 +28,12 @@ module Clusters cache_attributes(track_values) - # Use update_column so updated_at is skipped - update_columns(track_values) if can_update_track_values? + if can_update_track_values? + log_activity_event!(track_values[:last_used_at]) unless agent.active? + + # Use update_column so updated_at is skipped + update_columns(track_values) + end end private @@ -44,5 +48,14 @@ module Clusters real_last_used_at.nil? || (Time.current - real_last_used_at) >= last_used_at_max_age end + + def log_activity_event!(recorded_at) + agent.activity_events.create!( + kind: :agent_connected, + level: :info, + recorded_at: recorded_at, + agent_token: self + ) + end end end diff --git a/app/models/clusters/agents/activity_event.rb b/app/models/clusters/agents/activity_event.rb new file mode 100644 index 00000000000..5d9c885c923 --- /dev/null +++ b/app/models/clusters/agents/activity_event.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Clusters + module Agents + class ActivityEvent < ApplicationRecord + include NullifyIfBlank + + self.table_name = 'agent_activity_events' + + belongs_to :agent, class_name: 'Clusters::Agent', optional: false + belongs_to :user + belongs_to :agent_token, class_name: 'Clusters::AgentToken' + + scope :in_timeline_order, -> { order(recorded_at: :desc, id: :desc) } + + validates :recorded_at, :kind, :level, presence: true + + nullify_if_blank :detail + + enum kind: { + token_created: 0, + token_revoked: 1, + agent_connected: 2, + agent_disconnected: 3 + }, _prefix: true + + enum level: { + debug: 0, + info: 1, + warn: 2, + error: 3, + fatal: 4, + unknown: 5 + }, _prefix: true + end + end +end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 59a9251d6b7..b57a24dead0 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.34.0' + VERSION = '0.35.0' self.table_name = 'clusters_applications_runners' @@ -50,34 +50,6 @@ module Clusters private - def ensure_runner - runner || create_and_assign_runner - end - - def create_and_assign_runner - transaction do - Ci::Runner.create!(runner_create_params).tap do |runner| - update!(runner_id: runner.id) - end - end - end - - def runner_create_params - attributes = { - name: 'kubernetes-cluster', - runner_type: cluster.cluster_type, - tag_list: %w[kubernetes cluster] - } - - if cluster.group_type? - attributes[:runner_namespaces] = [::Ci::RunnerNamespace.new(namespace: group)] - elsif cluster.project_type? - attributes[:runner_projects] = [::Ci::RunnerProject.new(project: project)] - end - - attributes - end - def gitlab_url Gitlab::Routing.url_helpers.root_url(only_path: false) end @@ -85,7 +57,6 @@ module Clusters def specification { "gitlabUrl" => gitlab_url, - "runnerToken" => ensure_runner.token, "runners" => { "privileged" => privileged } } end diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb index 7ec614b048c..1bd8e8b44cb 100644 --- a/app/models/clusters/platforms/kubernetes.rb +++ b/app/models/clusters/platforms/kubernetes.rb @@ -50,12 +50,6 @@ module Clusters alias_attribute :ca_pem, :ca_cert - delegate :enabled?, to: :cluster, allow_nil: true - delegate :provided_by_user?, to: :cluster, allow_nil: true - delegate :allow_user_defined_namespace?, to: :cluster, allow_nil: true - - alias_method :active?, :enabled? - enum_with_nil authorization_type: { unknown_authorization: nil, rbac: 1, @@ -66,6 +60,19 @@ module Clusters nullify_if_blank :namespace + def enabled? + !!cluster&.enabled? + end + alias_method :active?, :enabled? + + def provided_by_user? + !!cluster&.provided_by_user? + end + + def allow_user_defined_namespace? + !!cluster&.allow_user_defined_namespace? + end + def predefined_variables(project:, environment_name:, kubernetes_namespace: nil) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'KUBE_URL', value: api_url) diff --git a/app/models/commit.rb b/app/models/commit.rb index 553681ee960..f0c5f3c2d12 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -84,43 +84,27 @@ class Commit sha[0..MIN_SHA_LENGTH] end - def diff_safe_lines(project: nil) - diff_safe_max_lines(project: project) + def diff_max_files + Gitlab::CurrentSettings.diff_max_files end - def diff_max_files(project: nil) - if Feature.enabled?(:increased_diff_limits, project) - 3000 - elsif Feature.enabled?(:configurable_diff_limits, project) - Gitlab::CurrentSettings.diff_max_files - else - 1000 - end - end - - def diff_max_lines(project: nil) - if Feature.enabled?(:increased_diff_limits, project) - 100000 - elsif Feature.enabled?(:configurable_diff_limits, project) - Gitlab::CurrentSettings.diff_max_lines - else - 50000 - end + def diff_max_lines + Gitlab::CurrentSettings.diff_max_lines end - def max_diff_options(project: nil) + def max_diff_options { - max_files: diff_max_files(project: project), - max_lines: diff_max_lines(project: project) + max_files: diff_max_files, + max_lines: diff_max_lines } end - def diff_safe_max_files(project: nil) - diff_max_files(project: project) / DIFF_SAFE_LIMIT_FACTOR + def diff_safe_max_files + diff_max_files / DIFF_SAFE_LIMIT_FACTOR end - def diff_safe_max_lines(project: nil) - diff_max_lines(project: project) / DIFF_SAFE_LIMIT_FACTOR + def diff_safe_max_lines + diff_max_lines / DIFF_SAFE_LIMIT_FACTOR end def from_hash(hash, container) diff --git a/app/models/commit_signatures/gpg_signature.rb b/app/models/commit_signatures/gpg_signature.rb new file mode 100644 index 00000000000..1ce76b53da4 --- /dev/null +++ b/app/models/commit_signatures/gpg_signature.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true +module CommitSignatures + class GpgSignature < ApplicationRecord + include CommitSignature + + sha_attribute :gpg_key_primary_keyid + + belongs_to :gpg_key + belongs_to :gpg_key_subkey + + validates :gpg_key_primary_keyid, presence: true + + def self.with_key_and_subkeys(gpg_key) + subkey_ids = gpg_key.subkeys.pluck(:id) + + where( + arel_table[:gpg_key_id].eq(gpg_key.id).or( + arel_table[:gpg_key_subkey_id].in(subkey_ids) + ) + ) + end + + def gpg_key=(model) + case model + when GpgKey + super + when GpgKeySubkey + self.gpg_key_subkey = model + when NilClass + super + self.gpg_key_subkey = nil + end + end + + def gpg_key + if gpg_key_id + super + elsif gpg_key_subkey_id + gpg_key_subkey + end + end + + def gpg_key_primary_keyid + super&.upcase + end + + def gpg_commit + return unless commit + + Gitlab::Gpg::Commit.new(commit) + end + end +end diff --git a/app/models/commit_signatures/x509_commit_signature.rb b/app/models/commit_signatures/x509_commit_signature.rb new file mode 100644 index 00000000000..2cbb331dd7e --- /dev/null +++ b/app/models/commit_signatures/x509_commit_signature.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +module CommitSignatures + class X509CommitSignature < ApplicationRecord + include CommitSignature + + belongs_to :x509_certificate, class_name: 'X509Certificate', foreign_key: 'x509_certificate_id', optional: false + + validates :x509_certificate_id, presence: true + + def x509_commit + return unless commit + + Gitlab::X509::Commit.new(commit) + end + end +end diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index d75f7984e2c..d6a2f62ca9b 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -53,15 +53,13 @@ class CommitStatus < Ci::ApplicationRecord scope :before_stage, -> (index) { where('stage_idx < ?', index) } scope :for_stage, -> (index) { where(stage_idx: index) } scope :after_stage, -> (index) { where('stage_idx > ?', index) } + scope :for_project, -> (project_id) { where(project_id: project_id) } scope :for_ref, -> (ref) { where(ref: ref) } scope :by_name, -> (name) { where(name: name) } scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) } scope :with_pipeline, -> { joins(:pipeline) } scope :updated_at_before, ->(date) { where('ci_builds.updated_at < ?', date) } scope :created_at_before, ->(date) { where('ci_builds.created_at < ?', date) } - scope :updated_before, ->(lookback:, timeout:) { - where('(ci_builds.created_at BETWEEN ? AND ?) AND (ci_builds.updated_at BETWEEN ? AND ?)', lookback, timeout, lookback, timeout) - } scope :scheduled_at_before, ->(date) { where('ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?', date) } @@ -71,7 +69,8 @@ class CommitStatus < Ci::ApplicationRecord # Pluck is used to split this query. Splitting the query is required for database decomposition for `ci_*` tables. # https://docs.gitlab.com/ee/development/database/transaction_guidelines.html#database-decomposition-and-sharding project_ids = Project.where_full_path_in(Array(paths)).pluck(:id) - where(project: project_ids) + + for_project(project_ids) end scope :with_preloads, -> do @@ -147,7 +146,7 @@ class CommitStatus < Ci::ApplicationRecord end event :drop do - transition [:created, :waiting_for_resource, :preparing, :pending, :running, :scheduled] => :failed + transition [:created, :waiting_for_resource, :preparing, :pending, :running, :manual, :scheduled] => :failed end event :success do @@ -191,7 +190,12 @@ class CommitStatus < Ci::ApplicationRecord commit_status.run_after_commit do PipelineProcessWorker.perform_async(pipeline_id) unless transition_options[:skip_pipeline_processing] - ExpireJobCacheWorker.perform_async(id) + + if Feature.enabled?(:expire_job_and_pipeline_cache_synchronously, project, default_enabled: :yaml) + expire_etag_cache! + else + ExpireJobCacheWorker.perform_async(id) + end end end @@ -217,6 +221,10 @@ class CommitStatus < Ci::ApplicationRecord false end + def self.bulk_insert_tags!(statuses, tag_list_by_build) + Gitlab::Ci::Tags::BulkInsert.new(statuses, tag_list_by_build).insert! + end + def locking_enabled? will_save_change_to_status? end @@ -300,6 +308,12 @@ class CommitStatus < Ci::ApplicationRecord .update_all(retried: true, processed: true) end + def expire_etag_cache! + job_path = Gitlab::Routing.url_helpers.project_build_path(project, id, format: :json) + + Gitlab::EtagCaching::Store.new.touch(job_path) + end + private def unrecoverable_failure? diff --git a/app/models/concerns/after_commit_queue.rb b/app/models/concerns/after_commit_queue.rb new file mode 100644 index 00000000000..7f525bec9e9 --- /dev/null +++ b/app/models/concerns/after_commit_queue.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module AfterCommitQueue + extend ActiveSupport::Concern + + included do + after_commit :_run_after_commit_queue + after_rollback :_clear_after_commit_queue + end + + def run_after_commit(&block) + _after_commit_queue << block if block + + true + end + + def run_after_commit_or_now(&block) + if self.class.inside_transaction? + if connection.current_transaction.records&.include?(self) + run_after_commit(&block) + else + # If the current transaction does not include this record, we can run + # the block now, even if it queues a Sidekiq job. + Sidekiq::Worker.skipping_transaction_check do + instance_eval(&block) + end + end + else + instance_eval(&block) + end + + true + end + + protected + + def _run_after_commit_queue + while action = _after_commit_queue.pop + self.instance_eval(&action) + end + end + + def _after_commit_queue + @after_commit_queue ||= [] + end + + def _clear_after_commit_queue + _after_commit_queue.clear + end +end diff --git a/app/models/concerns/calloutable.rb b/app/models/concerns/calloutable.rb deleted file mode 100644 index 8b9cfae6a32..00000000000 --- a/app/models/concerns/calloutable.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Calloutable - extend ActiveSupport::Concern - - included do - belongs_to :user - - validates :user, presence: true - end - - def dismissed_after?(dismissed_after) - dismissed_at > dismissed_after - end -end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index a9589cea5e9..12ddbc2cc40 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -13,7 +13,6 @@ module Ci track_duration do variables = pipeline.variables_builder.scoped_variables(self, environment: environment, dependencies: dependencies) - variables.concat(predefined_variables) unless pipeline.predefined_vars_in_builder_enabled? variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(runner.predefined_variables) if runnable? && runner @@ -71,24 +70,6 @@ module Ci end end - def predefined_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_JOB_NAME', value: name) - variables.append(key: 'CI_JOB_STAGE', value: stage) - variables.append(key: 'CI_JOB_MANUAL', value: 'true') if action? - variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if trigger_request - - variables.append(key: 'CI_NODE_INDEX', value: self.options[:instance].to_s) if self.options&.include?(:instance) - variables.append(key: 'CI_NODE_TOTAL', value: ci_node_total_value.to_s) - - # legacy variables - variables.append(key: 'CI_BUILD_NAME', value: name) - variables.append(key: 'CI_BUILD_STAGE', value: stage) - variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if trigger_request - variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if action? - end - end - def kubernetes_variables ::Gitlab::Ci::Variables::Collection.new.tap do |collection| # Should get merged with the cluster kubeconfig in deployment_variables, see @@ -123,13 +104,5 @@ module Ci def secret_project_variables(environment: expanded_environment_name) project.ci_variables_for(ref: git_ref, environment: environment) end - - private - - def ci_node_total_value - parallel = self.options&.dig(:parallel) - parallel = parallel.dig(:total) if parallel.is_a?(Hash) - parallel || 1 - end end end diff --git a/app/models/concerns/commit_signature.rb b/app/models/concerns/commit_signature.rb new file mode 100644 index 00000000000..5bdfa9a2966 --- /dev/null +++ b/app/models/concerns/commit_signature.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +module CommitSignature + extend ActiveSupport::Concern + + included do + include ShaAttribute + + sha_attribute :commit_sha + + enum verification_status: { + unverified: 0, + verified: 1, + same_user_different_email: 2, + other_user: 3, + unverified_key: 4, + unknown_key: 5, + multiple_signatures: 6 + } + + belongs_to :project, class_name: 'Project', foreign_key: 'project_id', optional: false + + validates :commit_sha, presence: true + validates :project_id, presence: true + + scope :by_commit_sha, ->(shas) { where(commit_sha: shas) } + end + + class_methods do + def safe_create!(attributes) + create_with(attributes) + .safe_find_or_create_by!(commit_sha: attributes[:commit_sha]) + end + + # Find commits that are lacking a signature in the database at present + def unsigned_commit_shas(commit_shas) + return [] if commit_shas.empty? + + signed = by_commit_sha(commit_shas).pluck(:commit_sha) + commit_shas - signed + end + end + + def commit + project.commit(commit_sha) + end + + def user + commit.committer + end +end diff --git a/app/models/concerns/diff_positionable_note.rb b/app/models/concerns/diff_positionable_note.rb index b13ca4bf06e..051158e5de5 100644 --- a/app/models/concerns/diff_positionable_note.rb +++ b/app/models/concerns/diff_positionable_note.rb @@ -3,7 +3,6 @@ module DiffPositionableNote extend ActiveSupport::Concern included do - delegate :on_text?, :on_image?, to: :position, allow_nil: true before_validation :set_original_position, on: :create before_validation :update_position, on: :create, if: :on_text?, unless: :importing? @@ -34,6 +33,14 @@ module DiffPositionableNote end end + def on_text? + !!position&.on_text? + end + + def on_image? + !!position&.on_image? + end + def supported? for_commit? || self.noteable.has_complete_diff_refs? end diff --git a/app/models/concerns/enums/ci/commit_status.rb b/app/models/concerns/enums/ci/commit_status.rb index 1b4cc14f4a2..312b88a4d6d 100644 --- a/app/models/concerns/enums/ci/commit_status.rb +++ b/app/models/concerns/enums/ci/commit_status.rb @@ -28,6 +28,7 @@ module Enums trace_size_exceeded: 19, builds_disabled: 20, environment_creation_failure: 21, + deployment_rejected: 22, insufficient_bridge_permissions: 1_001, downstream_bridge_project_not_found: 1_002, invalid_bridge_trigger: 1_003, diff --git a/app/models/concerns/import_state/sidekiq_job_tracker.rb b/app/models/concerns/import_state/sidekiq_job_tracker.rb index b7d0ed0f51b..340bf4279bc 100644 --- a/app/models/concerns/import_state/sidekiq_job_tracker.rb +++ b/app/models/concerns/import_state/sidekiq_job_tracker.rb @@ -15,7 +15,7 @@ module ImportState def refresh_jid_expiration return unless jid - Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) + Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION, value: 2) end def self.jid_by(project_id:, status:) diff --git a/app/models/concerns/incident_management/escalatable.rb b/app/models/concerns/incident_management/escalatable.rb index 78dce63f59e..81eef50603a 100644 --- a/app/models/concerns/incident_management/escalatable.rb +++ b/app/models/concerns/incident_management/escalatable.rb @@ -102,3 +102,5 @@ module IncidentManagement end end end + +::IncidentManagement::Escalatable.prepend_mod diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 4273eb331a1..dcd80201d3f 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -43,7 +43,7 @@ module Issuable included do cache_markdown_field :title, pipeline: :single_line - cache_markdown_field :description, issuable_state_filter_enabled: true + cache_markdown_field :description, issuable_reference_expansion_enabled: true redact_field :description @@ -61,6 +61,16 @@ module Issuable # We check first if we're loaded to not load unnecessarily. loaded? && to_a.all? { |note| note.association(:award_emoji).loaded? } end + + def projects_loaded? + # We check first if we're loaded to not load unnecessarily. + loaded? && to_a.all? { |note| note.association(:project).loaded? } + end + + def system_note_metadata_loaded? + # We check first if we're loaded to not load unnecessarily. + loaded? && to_a.all? { |note| note.association(:system_note_metadata).loaded? } + end end has_many :note_authors, -> { distinct }, through: :notes, source: :author @@ -183,6 +193,10 @@ module Issuable incident? end + def supports_escalation? + incident? + end + def incident? is_a?(Issue) && super end @@ -524,6 +538,8 @@ module Issuable includes = [] includes << :author unless notes.authors_loaded? includes << :award_emoji unless notes.award_emojis_loaded? + includes << :project unless notes.projects_loaded? + includes << :system_note_metadata unless notes.system_note_metadata_loaded? if includes.any? notes.includes(includes) diff --git a/app/models/concerns/loose_foreign_key.rb b/app/models/concerns/loose_foreign_key.rb deleted file mode 100644 index 102292672b3..00000000000 --- a/app/models/concerns/loose_foreign_key.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -module LooseForeignKey - extend ActiveSupport::Concern - - # This concern adds loose foreign key support to ActiveRecord models. - # Loose foreign keys allow delayed processing of associated database records - # with similar guarantees than a database foreign key. - # - # Prerequisites: - # - # To start using the concern, you'll need to install a database trigger to the parent - # table in a standard DB migration (not post-migration). - # - # > track_record_deletions(:projects) - # - # Usage: - # - # > class Ci::Build < ApplicationRecord - # > - # > loose_foreign_key :security_scans, :build_id, on_delete: :async_delete - # > - # > # associations can be still defined, the dependent options is no longer necessary: - # > has_many :security_scans, class_name: 'Security::Scan' - # > - # > end - # - # Options for on_delete: - # - # - :async_delete - deletes the children rows via an asynchronous process. - # - :async_nullify - sets the foreign key column to null via an asynchronous process. - # - # How it works: - # - # When adding loose foreign key support to the table, a DELETE trigger is installed - # which tracks the record deletions (stores primary key value of the deleted row) in - # a database table. - # - # These deletion records are processed asynchronously and records are cleaned up - # according to the loose foreign key definitions described in the model. - # - # The cleanup happens in batches, which reduces the likelyhood of statement timeouts. - # - # When all associations related to the deleted record are cleaned up, the record itself - # is deleted. - included do - class_attribute :loose_foreign_key_definitions, default: [] - end - - class_methods do - def loose_foreign_key(to_table, column, options) - symbolized_options = options.symbolize_keys - - unless base_class? - raise <<~MSG - loose_foreign_key can be only used on base classes, inherited classes are not supported. - Please define the loose_foreign_key on the #{base_class.name} class. - MSG - end - - on_delete_options = %i[async_delete async_nullify] - - unless on_delete_options.include?(symbolized_options[:on_delete]&.to_sym) - raise "Invalid on_delete option given: #{symbolized_options[:on_delete]}. Valid options: #{on_delete_options.join(', ')}" - end - - definition = ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new( - table_name.to_s, - to_table.to_s, - { - column: column.to_s, - on_delete: symbolized_options[:on_delete].to_sym - } - ) - - self.loose_foreign_key_definitions += [definition] - end - end -end diff --git a/app/models/concerns/merge_request_reviewer_state.rb b/app/models/concerns/merge_request_reviewer_state.rb index 216a3a0bd64..5859f43a70c 100644 --- a/app/models/concerns/merge_request_reviewer_state.rb +++ b/app/models/concerns/merge_request_reviewer_state.rb @@ -15,11 +15,5 @@ module MergeRequestReviewerState inclusion: { in: self.states.keys } after_initialize :set_state, unless: :persisted? - - def set_state - if Feature.enabled?(:mr_attention_requests, self.merge_request&.project, default_enabled: :yaml) - self.state = :attention_requested - end - end end end diff --git a/app/models/concerns/packages/debian/component_file.rb b/app/models/concerns/packages/debian/component_file.rb index 9cf66c756a0..77409549e85 100644 --- a/app/models/concerns/packages/debian/component_file.rb +++ b/app/models/concerns/packages/debian/component_file.rb @@ -20,13 +20,13 @@ module Packages belongs_to :component, class_name: "Packages::Debian::#{container_type.capitalize}Component", inverse_of: :files belongs_to :architecture, class_name: "Packages::Debian::#{container_type.capitalize}Architecture", inverse_of: :files, optional: true - enum file_type: { packages: 1, source: 2, di_packages: 3 } + enum file_type: { packages: 1, sources: 2, di_packages: 3 } enum compression_type: { gz: 1, bz2: 2, xz: 3 } validates :component, presence: true validates :file_type, presence: true - validates :architecture, presence: true, unless: :source? - validates :architecture, absence: true, if: :source? + validates :architecture, presence: true, unless: :sources? + validates :architecture, absence: true, if: :sources? validates :file, length: { minimum: 0, allow_nil: false } validates :size, presence: true validates :file_store, presence: true @@ -81,7 +81,7 @@ module Packages case file_type when 'packages' "#{component.name}/binary-#{architecture.name}/#{file_name}#{extension}" - when 'source' + when 'sources' "#{component.name}/source/#{file_name}#{extension}" when 'di_packages' "#{component.name}/debian-installer/binary-#{architecture.name}/#{file_name}#{extension}" diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb index 25410a859e9..1663aa6c886 100644 --- a/app/models/concerns/participable.rb +++ b/app/models/concerns/participable.rb @@ -60,6 +60,15 @@ module Participable filtered_participants_hash[user] end + # Returns only participants visible for the user + # + # Returns an Array of User instances. + def visible_participants(user) + return participants(user) unless Feature.enabled?(:verify_participants_access, project, default_enabled: :yaml) + + filter_by_ability(raw_participants(user, verify_access: true)) + end + # Checks if the user is a participant in a discussion. # # This method processes attributes of objects in breadth-first order. @@ -84,8 +93,7 @@ module Participable end end - def raw_participants(current_user = nil) - current_user ||= author + def raw_participants(current_user = nil, verify_access: false) ext = Gitlab::ReferenceExtractor.new(project, current_user) participants = Set.new process = [self] @@ -97,6 +105,8 @@ module Participable when User participants << source when Participable + next unless !verify_access || source_visible_to_user?(source, current_user) + source.class.participant_attrs.each do |attr| if attr.respond_to?(:call) source.instance_exec(current_user, ext, &attr) @@ -116,6 +126,10 @@ module Participable participants.merge(ext.users) end + def source_visible_to_user?(source, user) + Ability.allowed?(user, "read_#{source.model_name.element}".to_sym, source) + end + def filter_by_ability(participants) case self when PersonalSnippet diff --git a/app/models/concerns/partitioned_table.rb b/app/models/concerns/partitioned_table.rb index 23d2d00b346..f95f9dd8ad7 100644 --- a/app/models/concerns/partitioned_table.rb +++ b/app/models/concerns/partitioned_table.rb @@ -7,7 +7,8 @@ module PartitionedTable attr_reader :partitioning_strategy PARTITIONING_STRATEGIES = { - monthly: Gitlab::Database::Partitioning::MonthlyStrategy + monthly: Gitlab::Database::Partitioning::MonthlyStrategy, + sliding_list: Gitlab::Database::Partitioning::SlidingListStrategy }.freeze def partitioned_by(partitioning_key, strategy:, **kwargs) diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index c32e499c329..9069d3088cd 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -168,6 +168,24 @@ module RelativePositioning self.relative_position = MIN_POSITION end + def next_object_by_relative_position(ignoring: nil, order: :asc) + relation = relative_positioning_scoped_items(ignoring: ignoring).reorder(relative_position: order) + + relation = if order == :asc + relation.where(self.class.arel_table[:relative_position].gt(relative_position)) + else + relation.where(self.class.arel_table[:relative_position].lt(relative_position)) + end + + relation.first + end + + def relative_positioning_scoped_items(ignoring: nil) + relation = self.class.relative_positioning_query_base(self) + relation = exclude_self(relation, excluded: ignoring) if ignoring.present? + relation + end + # This method is used during rebalancing - override it to customise the update # logic: def update_relative_siblings(relation, range, delta) diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index 60e1dde17b9..aae338e9759 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -30,11 +30,14 @@ module ResolvableDiscussion delegate :resolved_at, :resolved_by, - :resolved_by_push?, to: :last_resolved_note, allow_nil: true end + def resolved_by_push? + !!last_resolved_note&.resolved_by_push? + end + def resolvable? strong_memoize(:resolvable) do potentially_resolvable? && notes.any?(&:resolvable?) diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index ba7c6c0cd8b..e49f4d03bda 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -3,11 +3,14 @@ module ShaAttribute extend ActiveSupport::Concern + # Needed for the database method + include DatabaseReflection + class_methods do def sha_attribute(name) return if ENV['STATIC_VERIFICATION'] - validate_binary_column_exists!(name) if Rails.env.development? + validate_binary_column_exists!(name) if Rails.env.development? || Rails.env.test? attribute(name, Gitlab::Database::ShaAttribute.new) end diff --git a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb index 3be82ed72d3..447521ad8c1 100644 --- a/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb +++ b/app/models/concerns/token_authenticatable_strategies/encryption_helper.rb @@ -11,7 +11,7 @@ module TokenAuthenticatableStrategies # The pattern of the token is "#{DYNAMIC_NONCE_IDENTIFIER}#{token}#{iv_of_12_characters}" if token.start_with?(DYNAMIC_NONCE_IDENTIFIER) && token.size > NONCE_SIZE + DYNAMIC_NONCE_IDENTIFIER.size token_to_decrypt = token[1...-NONCE_SIZE] - iv = token[-NONCE_SIZE..-1] + iv = token[-NONCE_SIZE..] Gitlab::CryptoHelper.aes256_gcm_decrypt(token_to_decrypt, nonce: iv) else diff --git a/app/models/concerns/transactions.rb b/app/models/concerns/transactions.rb index a186ebc8475..1c9bd8274f5 100644 --- a/app/models/concerns/transactions.rb +++ b/app/models/concerns/transactions.rb @@ -8,7 +8,7 @@ module Transactions # transaction. Handles special cases when running inside a test environment, # where tests may be wrapped in transactions def inside_transaction? - base = Rails.env.test? ? @open_transactions_baseline.to_i : 0 + base = Rails.env.test? ? open_transactions_baseline.to_i : 0 connection.open_transactions > base end @@ -24,5 +24,15 @@ module Transactions def reset_open_transactions_baseline @open_transactions_baseline = 0 end + + def open_transactions_baseline + return unless Rails.env.test? + + if @open_transactions_baseline.nil? + return self == ApplicationRecord ? nil : superclass.open_transactions_baseline + end + + @open_transactions_baseline + end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 8e130998f11..c914819f79d 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -145,9 +145,14 @@ class ContainerRepository < ApplicationRecord name: path.repository_name) end - def self.create_from_path!(path) - safe_find_or_create_by!(project: path.repository_project, - name: path.repository_name) + def self.find_or_create_from_path(path) + repository = safe_find_or_create_by( + project: path.repository_project, + name: path.repository_name + ) + return repository if repository.persisted? + + find_by_path!(path) end def self.build_root_repository(project) diff --git a/app/models/context_commits_diff.rb b/app/models/context_commits_diff.rb index fe1a72b79f2..3d25b60678a 100644 --- a/app/models/context_commits_diff.rb +++ b/app/models/context_commits_diff.rb @@ -3,6 +3,7 @@ class ContextCommitsDiff include ActsAsPaginatedDiff + delegate :head, :base, to: :compare attr_reader :merge_request def initialize(merge_request) diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index 5898bc3412f..d8669f1f4c2 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -25,6 +25,13 @@ class CustomerRelations::Contact < ApplicationRecord validates :description, length: { maximum: 1024 } validate :validate_email_format + def self.find_ids_by_emails(group_id, emails) + raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK + + where(group_id: group_id, email: emails) + .pluck(:id) + end + private def validate_email_format diff --git a/app/models/customer_relations/issue_contact.rb b/app/models/customer_relations/issue_contact.rb index 98faf8d6644..78f662b6a58 100644 --- a/app/models/customer_relations/issue_contact.rb +++ b/app/models/customer_relations/issue_contact.rb @@ -8,6 +8,14 @@ class CustomerRelations::IssueContact < ApplicationRecord validate :contact_belongs_to_issue_group + def self.find_contact_ids_by_emails(issue_id, emails) + raise ArgumentError, "Cannot lookup more than #{MAX_PLUCK} emails" if emails.length > MAX_PLUCK + + joins(:contact) + .where(issue_id: issue_id, customer_relations_contacts: { email: emails }) + .pluck(:contact_id) + end + private def contact_belongs_to_issue_group diff --git a/app/models/deployment.rb b/app/models/deployment.rb index ade19ce02a8..4c60ce57f49 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -46,9 +46,10 @@ class Deployment < ApplicationRecord scope :for_project, -> (project_id) { where(project_id: project_id) } scope :for_projects, -> (projects) { where(project: projects) } - scope :visible, -> { where(status: %i[running success failed canceled]) } + scope :visible, -> { where(status: %i[running success failed canceled blocked]) } scope :stoppable, -> { where.not(on_stop: nil).where.not(deployable_id: nil).success } scope :active, -> { where(status: %i[created running]) } + scope :upcoming, -> { where(status: %i[blocked running]) } scope :older_than, -> (deployment) { where('deployments.id < ?', deployment.id) } scope :with_api_entity_associations, -> { preload({ deployable: { runner: [], tags: [], user: [], job_artifacts_archive: [] } }) } @@ -64,6 +65,10 @@ class Deployment < ApplicationRecord transition created: :running end + event :block do + transition created: :blocked + end + event :succeed do transition any - [:success] => :success end @@ -119,6 +124,8 @@ class Deployment < ApplicationRecord next if transition.loopback? deployment.run_after_commit do + next unless deployment.project.jira_subscription_exists? + ::JiraConnect::SyncDeploymentsWorker.perform_async(id) end end @@ -126,6 +133,8 @@ class Deployment < ApplicationRecord after_create unless: :importing? do |deployment| run_after_commit do + next unless deployment.project.jira_subscription_exists? + ::JiraConnect::SyncDeploymentsWorker.perform_async(deployment.id) end end @@ -136,7 +145,8 @@ class Deployment < ApplicationRecord success: 2, failed: 3, canceled: 4, - skipped: 5 + skipped: 5, + blocked: 6 } def self.archivables_in(project, limit:) @@ -387,6 +397,8 @@ class Deployment < ApplicationRecord cancel! when 'skipped' skip! + when 'blocked' + block! else raise ArgumentError, "The status #{status.inspect} is invalid" end diff --git a/app/models/dev_ops_report/metric.rb b/app/models/dev_ops_report/metric.rb index 14eff725433..d30e869b155 100644 --- a/app/models/dev_ops_report/metric.rb +++ b/app/models/dev_ops_report/metric.rb @@ -6,6 +6,20 @@ module DevOpsReport self.table_name = 'conversational_development_index_metrics' + METRICS = %w[leader_issues instance_issues percentage_issues leader_notes instance_notes + percentage_notes leader_milestones instance_milestones percentage_milestones + leader_boards instance_boards percentage_boards leader_merge_requests + instance_merge_requests percentage_merge_requests leader_ci_pipelines + instance_ci_pipelines percentage_ci_pipelines leader_environments instance_environments + percentage_environments leader_deployments instance_deployments percentage_deployments + leader_projects_prometheus_active instance_projects_prometheus_active + percentage_projects_prometheus_active leader_service_desk_issues instance_service_desk_issues + percentage_service_desk_issues].freeze + + METRICS.each do |metric_name| + validates metric_name, presence: true, numericality: { greater_than_or_equal_to: 0 } + end + def instance_score(feature) self["instance_#{feature}"] end diff --git a/app/models/environment.rb b/app/models/environment.rb index 2618991c9e5..a830c04f291 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -31,7 +31,7 @@ class Environment < ApplicationRecord has_one :last_visible_deployable, through: :last_visible_deployment, source: 'deployable', source_type: 'CommitStatus', disable_joins: true has_one :last_visible_pipeline, through: :last_visible_deployable, source: 'pipeline', disable_joins: true - has_one :upcoming_deployment, -> { running.distinct_on_environment }, class_name: 'Deployment', inverse_of: :environment + has_one :upcoming_deployment, -> { upcoming.distinct_on_environment }, class_name: 'Deployment', inverse_of: :environment has_one :latest_opened_most_severe_alert, -> { order_severity_with_open_prometheus_alert }, class_name: 'AlertManagement::Alert', inverse_of: :environment before_validation :generate_slug, if: ->(env) { env.slug.blank? } diff --git a/app/models/error_tracking/error_event.rb b/app/models/error_tracking/error_event.rb index 0b638f65768..18c1467e6f6 100644 --- a/app/models/error_tracking/error_event.rb +++ b/app/models/error_tracking/error_event.rb @@ -3,6 +3,9 @@ class ErrorTracking::ErrorEvent < ApplicationRecord belongs_to :error, counter_cache: :events_count + # Scrub null bytes + attribute :payload, Gitlab::Database::Type::JsonPgSafe.new + validates :payload, json_schema: { filename: 'error_tracking_event_payload' } validates :error, presence: true diff --git a/app/models/event.rb b/app/models/event.rb index f6174589a84..409bc66c66c 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -130,10 +130,11 @@ class Event < ApplicationRecord # Update Gitlab::ContributionsCalendar#activity_dates if this changes def contributions - where("action = ? OR (target_type IN (?) AND action IN (?)) OR (target_type = ? AND action = ?)", - actions[:pushed], - %w(MergeRequest Issue), [actions[:created], actions[:closed], actions[:merged]], - "Note", actions[:commented]) + where( + 'action IN (?) OR (target_type IN (?) AND action IN (?))', + [actions[:pushed], actions[:commented]], + %w(MergeRequest Issue), [actions[:created], actions[:closed], actions[:merged]] + ) end def limit_recent(limit = 20, offset = nil) diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb index 0cb3662368c..a56e28859c9 100644 --- a/app/models/gpg_key.rb +++ b/app/models/gpg_key.rb @@ -92,13 +92,13 @@ class GpgKey < ApplicationRecord end def revoke - GpgSignature + CommitSignatures::GpgSignature .with_key_and_subkeys(self) - .where.not(verification_status: GpgSignature.verification_statuses[:unknown_key]) + .where.not(verification_status: CommitSignatures::GpgSignature.verification_statuses[:unknown_key]) .update_all( gpg_key_id: nil, gpg_key_subkey_id: nil, - verification_status: GpgSignature.verification_statuses[:unknown_key], + verification_status: CommitSignatures::GpgSignature.verification_statuses[:unknown_key], updated_at: Time.zone.now ) diff --git a/app/models/gpg_signature.rb b/app/models/gpg_signature.rb deleted file mode 100644 index 2775b520b2f..00000000000 --- a/app/models/gpg_signature.rb +++ /dev/null @@ -1,86 +0,0 @@ -# frozen_string_literal: true - -class GpgSignature < ApplicationRecord - include ShaAttribute - - sha_attribute :commit_sha - sha_attribute :gpg_key_primary_keyid - - enum verification_status: { - unverified: 0, - verified: 1, - same_user_different_email: 2, - other_user: 3, - unverified_key: 4, - unknown_key: 5, - multiple_signatures: 6 - } - - belongs_to :project - belongs_to :gpg_key - belongs_to :gpg_key_subkey - - validates :commit_sha, presence: true - validates :project_id, presence: true - validates :gpg_key_primary_keyid, presence: true - - scope :by_commit_sha, ->(shas) { where(commit_sha: shas) } - - def self.with_key_and_subkeys(gpg_key) - subkey_ids = gpg_key.subkeys.pluck(:id) - - where( - arel_table[:gpg_key_id].eq(gpg_key.id).or( - arel_table[:gpg_key_subkey_id].in(subkey_ids) - ) - ) - end - - def self.safe_create!(attributes) - create_with(attributes) - .safe_find_or_create_by!(commit_sha: attributes[:commit_sha]) - end - - # Find commits that are lacking a signature in the database at present - def self.unsigned_commit_shas(commit_shas) - return [] if commit_shas.empty? - - signed = GpgSignature.where(commit_sha: commit_shas).pluck(:commit_sha) - - commit_shas - signed - end - - def gpg_key=(model) - case model - when GpgKey - super - when GpgKeySubkey - self.gpg_key_subkey = model - when NilClass - super - self.gpg_key_subkey = nil - end - end - - def gpg_key - if gpg_key_id - super - elsif gpg_key_subkey_id - gpg_key_subkey - end - end - - def gpg_key_primary_keyid - super&.upcase - end - - def commit - project.commit(commit_sha) - end - - def gpg_commit - return unless commit - - Gitlab::Gpg::Commit.new(commit) - end -end diff --git a/app/models/group.rb b/app/models/group.rb index 2dd20300ad2..f51782785f9 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -852,15 +852,7 @@ class Group < Namespace end def self.groups_including_descendants_by(group_ids) - groups = Group.where(id: group_ids) - - if Feature.enabled?(:linear_group_including_descendants_by, default_enabled: :yaml) - groups.self_and_descendants - else - Gitlab::ObjectHierarchy - .new(groups) - .base_and_descendants - end + Group.where(id: group_ids).self_and_descendants end def disable_shared_runners! diff --git a/app/models/hooks/project_hook.rb b/app/models/hooks/project_hook.rb index d1584a62bfb..16b95d2a2b9 100644 --- a/app/models/hooks/project_hook.rb +++ b/app/models/hooks/project_hook.rb @@ -31,10 +31,6 @@ class ProjectHook < WebHook _('Webhooks') end - def web_hooks_disable_failed? - Feature.enabled?(:web_hooks_disable_failed, project) - end - override :rate_limit def rate_limit project.actual_limits.limit_for(:web_hook_calls) @@ -44,6 +40,13 @@ class ProjectHook < WebHook def application_context super.merge(project: project) end + + private + + override :web_hooks_disable_failed? + def web_hooks_disable_failed? + Feature.enabled?(:web_hooks_disable_failed, project) + end end ProjectHook.prepend_mod_with('ProjectHook') diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb index cb5c1ac48cd..e8a55abfc8f 100644 --- a/app/models/hooks/web_hook.rb +++ b/app/models/hooks/web_hook.rb @@ -34,9 +34,19 @@ class WebHook < ApplicationRecord end def executable? - return true unless web_hooks_disable_failed? + !temporarily_disabled? && !permanently_disabled? + end + + def temporarily_disabled? + return false unless web_hooks_disable_failed? + + disabled_until.present? && disabled_until >= Time.current + end + + def permanently_disabled? + return false unless web_hooks_disable_failed? - recent_failures <= FAILURE_THRESHOLD && (disabled_until.nil? || disabled_until < Time.current) + recent_failures > FAILURE_THRESHOLD end # rubocop: disable CodeReuse/ServiceClass @@ -69,6 +79,8 @@ class WebHook < ApplicationRecord end def disable! + return if permanently_disabled? + update_attribute(:recent_failures, FAILURE_THRESHOLD + 1) end @@ -80,7 +92,7 @@ class WebHook < ApplicationRecord end def backoff! - return if backoff_count >= MAX_FAILURES && disabled_until && disabled_until > Time.current + return if permanently_disabled? || (backoff_count >= MAX_FAILURES && temporarily_disabled?) assign_attributes(disabled_until: next_backoff.from_now, backoff_count: backoff_count.succ.clamp(0, MAX_FAILURES)) save(validate: false) @@ -93,7 +105,19 @@ class WebHook < ApplicationRecord save(validate: false) end - # Overridden in ProjectHook and GroupHook, other webhooks are not rate-limited. + # @return [Boolean] Whether or not the WebHook is currently throttled. + def rate_limited? + return false unless rate_limit + + Gitlab::ApplicationRateLimiter.peek( + :web_hook_calls, + scope: [self], + threshold: rate_limit + ) + end + + # Threshold for the rate-limit. + # Overridden in ProjectHook and GroupHook, other WebHooks are not rate-limited. def rate_limit nil end diff --git a/app/models/incident_management/issuable_escalation_status.rb b/app/models/incident_management/issuable_escalation_status.rb index 88aef104d88..fc881e62efd 100644 --- a/app/models/incident_management/issuable_escalation_status.rb +++ b/app/models/incident_management/issuable_escalation_status.rb @@ -7,8 +7,11 @@ module IncidentManagement self.table_name = 'incident_management_issuable_escalation_statuses' belongs_to :issue + has_one :project, through: :issue, inverse_of: :incident_management_issuable_escalation_status validates :issue, presence: true, uniqueness: true + + delegate :project, to: :issue end end diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index 0bf9e805aa8..bbddc18103a 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -62,6 +62,7 @@ class InstanceConfiguration def plan_file_size_limits(plan) { conan: plan.actual_limits[:conan_max_file_size], + helm: plan.actual_limits[:helm_max_file_size], maven: plan.actual_limits[:maven_max_file_size], npm: plan.actual_limits[:npm_max_file_size], nuget: plan.actual_limits[:nuget_max_file_size], diff --git a/app/models/integration.rb b/app/models/integration.rb index d3059fa6d4a..29d96650a81 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -14,11 +14,13 @@ class Integration < ApplicationRecord asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat irker jira mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email - pivotaltracker prometheus pushover redmine shimo slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao + pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack zentao ].freeze + # TODO Shimo is temporary disabled on group and instance-levels. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/345677 PROJECT_SPECIFIC_INTEGRATION_NAMES = %w[ - jenkins + jenkins shimo ].freeze # Fake integrations to help with local development. diff --git a/app/models/integrations/base_issue_tracker.rb b/app/models/integrations/base_issue_tracker.rb index 3fd67205e92..42a6a3a19c8 100644 --- a/app/models/integrations/base_issue_tracker.rb +++ b/app/models/integrations/base_issue_tracker.rb @@ -128,7 +128,7 @@ module Integrations false end - def create_cross_reference_note(mentioned, noteable, author) + def create_cross_reference_note(external_issue, mentioned_in, author) # implement inside child end diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 42c291abf55..d46299de1be 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -234,19 +234,19 @@ module Integrations end override :create_cross_reference_note - def create_cross_reference_note(mentioned, noteable, author) - unless can_cross_reference?(noteable) - return s_("JiraService|Events for %{noteable_model_name} are disabled.") % { noteable_model_name: noteable.model_name.plural.humanize(capitalize: false) } + def create_cross_reference_note(external_issue, mentioned_in, author) + unless can_cross_reference?(mentioned_in) + return s_("JiraService|Events for %{noteable_model_name} are disabled.") % { noteable_model_name: mentioned_in.model_name.plural.humanize(capitalize: false) } end - jira_issue = find_issue(mentioned.id) + jira_issue = find_issue(external_issue.id) return unless jira_issue.present? - noteable_id = noteable.respond_to?(:iid) ? noteable.iid : noteable.id - noteable_type = noteable_name(noteable) - entity_url = build_entity_url(noteable_type, noteable_id) - entity_meta = build_entity_meta(noteable) + mentioned_in_id = mentioned_in.respond_to?(:iid) ? mentioned_in.iid : mentioned_in.id + mentioned_in_type = mentionable_name(mentioned_in) + entity_url = build_entity_url(mentioned_in_type, mentioned_in_id) + entity_meta = build_entity_meta(mentioned_in) data = { user: { @@ -259,9 +259,9 @@ module Integrations }, entity: { id: entity_meta[:id], - name: noteable_type.humanize.downcase, + name: mentioned_in_type.humanize.downcase, url: entity_url, - title: noteable.title, + title: mentioned_in.title, description: entity_meta[:description], branch: entity_meta[:branch] } @@ -302,11 +302,11 @@ module Integrations private - def branch_name(noteable) + def branch_name(commit) if Feature.enabled?(:jira_use_first_ref_by_oid, project, default_enabled: :yaml) - noteable.first_ref_by_oid(project.repository) + commit.first_ref_by_oid(project.repository) else - noteable.ref_names(project.repository).first + commit.ref_names(project.repository).first end end @@ -316,8 +316,8 @@ module Integrations end end - def can_cross_reference?(noteable) - case noteable + def can_cross_reference?(mentioned_in) + case mentioned_in when Commit then commit_events when MergeRequest then merge_requests_events else true @@ -487,36 +487,36 @@ module Integrations "#{Settings.gitlab.base_url.chomp("/")}#{resource}" end - def build_entity_url(noteable_type, entity_id) + def build_entity_url(entity_type, entity_id) polymorphic_url( [ self.project, - noteable_type.to_sym + entity_type.to_sym ], id: entity_id, host: Settings.gitlab.base_url ) end - def build_entity_meta(noteable) - if noteable.is_a?(Commit) + def build_entity_meta(entity) + if entity.is_a?(Commit) { - id: noteable.short_id, - description: noteable.safe_message, - branch: branch_name(noteable) + id: entity.short_id, + description: entity.safe_message, + branch: branch_name(entity) } - elsif noteable.is_a?(MergeRequest) + elsif entity.is_a?(MergeRequest) { - id: noteable.to_reference, - branch: noteable.source_branch + id: entity.to_reference, + branch: entity.source_branch } else {} end end - def noteable_name(noteable) - name = noteable.model_name.singular + def mentionable_name(mentionable) + name = mentionable.model_name.singular # ProjectSnippet inherits from Snippet class so it causes # routing error building the URL. diff --git a/app/models/integrations/shimo.rb b/app/models/integrations/shimo.rb index 4f42fda2577..0e1023bb7a7 100644 --- a/app/models/integrations/shimo.rb +++ b/app/models/integrations/shimo.rb @@ -5,7 +5,11 @@ module Integrations prop_accessor :external_wiki_url validates :external_wiki_url, presence: true, public_url: true, if: :activated? + after_commit :cache_project_has_shimo + def render? + return false unless Feature.enabled?(:shimo_integration, project) + valid? && activated? end @@ -43,5 +47,14 @@ module Integrations } ] end + + private + + def cache_project_has_shimo + return unless project && !project.destroyed? + + project.project_setting.save! unless project.project_setting.persisted? + project.project_setting.update_column(:has_shimo, activated?) + end end end diff --git a/app/models/issue.rb b/app/models/issue.rb index 47dc084d69c..537e16e5cc3 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -63,6 +63,7 @@ class Issue < ApplicationRecord has_many :issue_assignees has_many :issue_email_participants + has_one :email has_many :assignees, class_name: "User", through: :issue_assignees has_many :zoom_meetings has_many :user_mentions, class_name: "IssueUserMention", dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -228,9 +229,37 @@ class Issue < ApplicationRecord end end + def next_object_by_relative_position(ignoring: nil, order: :asc) + return super unless Feature.enabled?(:optimized_issue_neighbor_queries, project, default_enabled: :yaml) + + array_mapping_scope = -> (id_expression) do + relation = Issue.where(Issue.arel_table[:project_id].eq(id_expression)) + + if order == :asc + relation.where(Issue.arel_table[:relative_position].gt(relative_position)) + else + relation.where(Issue.arel_table[:relative_position].lt(relative_position)) + end + end + + relation = Gitlab::Pagination::Keyset::InOperatorOptimization::QueryBuilder.new( + scope: Issue.order(relative_position: order, id: order), + array_scope: relative_positioning_parent_projects, + array_mapping_scope: array_mapping_scope, + finder_query: -> (_, id_expression) { Issue.where(Issue.arel_table[:id].eq(id_expression)) } + ).execute + + relation = exclude_self(relation, excluded: ignoring) if ignoring.present? + + relation.take + end + + def relative_positioning_parent_projects + project.group&.root_ancestor&.all_projects&.select(:id) || Project.id_in(project).select(:id) + end + def self.relative_positioning_query_base(issue) - projects = issue.project.group&.root_ancestor&.all_projects || issue.project - in_projects(projects) + in_projects(issue.relative_positioning_parent_projects) end def self.relative_positioning_parent_column @@ -433,8 +462,6 @@ class Issue < ApplicationRecord # Returns `true` if the current issue can be viewed by either a logged in User # or an anonymous user. def visible_to_user?(user = nil) - return false unless project && project.feature_available?(:issues, user) - return publicly_visible? unless user return false unless readable_by?(user) @@ -562,10 +589,10 @@ class Issue < ApplicationRecord project.team.member?(user, Gitlab::Access::REPORTER) elsif hidden? false + elsif project.public? || (project.internal? && !user.external?) + project.feature_available?(:issues, user) else - project.public? || - project.internal? && !user.external? || - project.team.member?(user) + project.team.member?(user) end end @@ -604,7 +631,7 @@ class Issue < ApplicationRecord def could_not_move(exception) # Symptom of running out of space - schedule rebalancing - IssueRebalancingWorker.perform_async(nil, *project.self_or_root_group_ids) + Issues::RebalancingWorker.perform_async(nil, *project.self_or_root_group_ids) end end diff --git a/app/models/issue/email.rb b/app/models/issue/email.rb new file mode 100644 index 00000000000..730fda5cdb4 --- /dev/null +++ b/app/models/issue/email.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class Issue::Email < ApplicationRecord + self.table_name = 'issue_emails' + + belongs_to :issue + + validates :email_message_id, uniqueness: true, presence: true, length: { maximum: 1000 } + validates :issue, presence: true, uniqueness: true +end diff --git a/app/models/lfs_object.rb b/app/models/lfs_object.rb index 9765ac6f2e9..caeffae7bda 100644 --- a/app/models/lfs_object.rb +++ b/app/models/lfs_object.rb @@ -13,6 +13,7 @@ class LfsObject < ApplicationRecord scope :with_files_stored_locally, -> { where(file_store: LfsObjectUploader::Store::LOCAL) } scope :with_files_stored_remotely, -> { where(file_store: LfsObjectUploader::Store::REMOTE) } scope :for_oids, -> (oids) { where(oid: oids) } + scope :for_oid_and_size, -> (oid, size) { find_by(oid: oid, size: size) } validates :oid, presence: true, uniqueness: true diff --git a/app/models/lfs_objects_project.rb b/app/models/lfs_objects_project.rb index e5632ff2842..bf6d1394569 100644 --- a/app/models/lfs_objects_project.rb +++ b/app/models/lfs_objects_project.rb @@ -21,9 +21,19 @@ class LfsObjectsProject < ApplicationRecord scope :project_id_in, ->(ids) { where(project_id: ids) } scope :lfs_object_in, -> (lfs_objects) { where(lfs_object: lfs_objects) } + def self.link_to_project!(lfs_object, project) + # We can't use an upsert here because there is no uniqueness constraint: + # https://gitlab.com/gitlab-org/gitlab/-/issues/347466 + self.safe_find_or_create_by!(lfs_object_id: lfs_object.id, project_id: project.id) # rubocop:disable Performance/ActiveRecordSubtransactionMethods + end + + def self.update_statistics_for_project_id(project_id) + ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size]) # rubocop:disable CodeReuse/Worker + end + private def update_project_statistics - ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size]) + self.class.update_statistics_for_project_id(project_id) end end diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index c3b3e76f67b..0fbdd2d8a5b 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -1,15 +1,45 @@ # frozen_string_literal: true class LooseForeignKeys::DeletedRecord < ApplicationRecord + PARTITION_DURATION = 1.day + + include PartitionedTable + self.primary_key = :id + self.ignored_columns = %i[partition] + + partitioned_by :partition, strategy: :sliding_list, + next_partition_if: -> (active_partition) do + return false if Feature.disabled?(:lfk_automatic_partition_creation, default_enabled: :yaml) + + oldest_record_in_partition = LooseForeignKeys::DeletedRecord + .select(:id, :created_at) + .for_partition(active_partition) + .order(:id) + .limit(1) + .take + + oldest_record_in_partition.present? && oldest_record_in_partition.created_at < PARTITION_DURATION.ago + end, + detach_partition_if: -> (partition) do + return false if Feature.disabled?(:lfk_automatic_partition_dropping, default_enabled: :yaml) + + !LooseForeignKeys::DeletedRecord + .for_partition(partition) + .status_pending + .exists? + end scope :for_table, -> (table) { where(fully_qualified_table_name: table) } + scope :for_partition, -> (partition) { where(partition: partition) } scope :consume_order, -> { order(:partition, :consume_after, :id) } enum status: { pending: 1, processed: 2 }, _prefix: :status def self.load_batch_for_table(table, batch_size) - for_table(table) + # selecting partition as partition_number to workaround the sliding partitioning column ignore + select(arel_table[Arel.star], arel_table[:partition].as('partition_number')) + .for_table(table) .status_pending .consume_order .limit(batch_size) @@ -20,9 +50,9 @@ class LooseForeignKeys::DeletedRecord < ApplicationRecord # Run a query for each partition to optimize the row lookup by primary key (partition, id) update_count = 0 - all_records.group_by(&:partition).each do |partition, records_within_partition| + all_records.group_by(&:partition_number).each do |partition, records_within_partition| update_count += status_pending - .where(partition: partition) + .for_partition(partition) .where(id: records_within_partition.pluck(:id)) .update_all(status: :processed) end diff --git a/app/models/member.rb b/app/models/member.rb index 11f67a77ee2..90fb281abf4 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -25,7 +25,7 @@ class Member < ApplicationRecord belongs_to :source, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations has_one :member_task - delegate :name, :username, :email, to: :user, prefix: true + delegate :name, :username, :email, :last_activity_on, to: :user, prefix: true delegate :tasks_to_be_done, to: :member_task, allow_nil: true validates :expires_at, allow_blank: true, future_date: true @@ -52,6 +52,7 @@ class Member < ApplicationRecord message: _('project bots cannot be added to other groups / projects') }, if: :project_bot? + validate :access_level_inclusion scope :with_invited_user_state, -> do joins('LEFT JOIN users as invited_user ON invited_user.email = members.invite_email') @@ -382,6 +383,12 @@ class Member < ApplicationRecord private + def access_level_inclusion + return if access_level.in?(Gitlab::Access.all_values) + + errors.add(:access_level, "is not included in the list") + end + def send_invite # override in subclass end @@ -417,11 +424,9 @@ class Member < ApplicationRecord def after_accept_invite post_create_hook - if experiment(:invite_members_for_task).enabled? - run_after_commit_or_now do - if member_task - TasksToBeDone::CreateWorker.perform_async(member_task.id, created_by_id, [user_id.to_i]) - end + run_after_commit_or_now do + if member_task + TasksToBeDone::CreateWorker.perform_async(member_task.id, created_by_id, [user_id.to_i]) end end end diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 9062a405218..1ad4cb6d368 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -6,6 +6,7 @@ class GroupMember < Member include CreatedAtFilterable SOURCE_TYPE = 'Namespace' + SOURCE_TYPE_FORMAT = /\ANamespace\z/.freeze belongs_to :group, foreign_key: 'source_id' alias_attribute :namespace_id, :source_id @@ -13,9 +14,7 @@ class GroupMember < Member # Make sure group member points only to group as it source default_value_for :source_type, SOURCE_TYPE - validates :source_type, format: { with: /\ANamespace\z/ } - validates :access_level, presence: true - validate :access_level_inclusion + validates :source_type, format: { with: SOURCE_TYPE_FORMAT } default_scope { where(source_type: SOURCE_TYPE) } # rubocop:disable Cop/DefaultScope @@ -65,12 +64,6 @@ class GroupMember < Member super end - def access_level_inclusion - return if access_level.in?(Gitlab::Access.all_values) - - errors.add(:access_level, "is not included in the list") - end - def send_invite run_after_commit_or_now { notification_service.invite_group_member(self, @raw_invite_token) } diff --git a/app/models/members/project_member.rb b/app/models/members/project_member.rb index 89b72508e84..6fc665cb87a 100644 --- a/app/models/members/project_member.rb +++ b/app/models/members/project_member.rb @@ -3,6 +3,7 @@ class ProjectMember < Member extend ::Gitlab::Utils::Override SOURCE_TYPE = 'Project' + SOURCE_TYPE_FORMAT = /\AProject\z/.freeze belongs_to :project, foreign_key: 'source_id' @@ -10,8 +11,7 @@ class ProjectMember < Member # Make sure project member points only to project as it source default_value_for :source_type, SOURCE_TYPE - validates :source_type, format: { with: /\AProject\z/ } - validates :access_level, inclusion: { in: Gitlab::Access.values } + validates :source_type, format: { with: SOURCE_TYPE_FORMAT } default_scope { where(source_type: SOURCE_TYPE) } # rubocop:disable Cop/DefaultScope scope :in_project, ->(project) { where(source_id: project.id) } @@ -92,6 +92,13 @@ class ProjectMember < Member private + override :access_level_inclusion + def access_level_inclusion + return if access_level.in?(Gitlab::Access.values) + + errors.add(:access_level, "is not included in the list") + end + override :refresh_member_authorized_projects def refresh_member_authorized_projects(blocking:) return unless user diff --git a/app/models/members_preloader.rb b/app/models/members_preloader.rb index ba7e4b39989..8b8eca54550 100644 --- a/app/models/members_preloader.rb +++ b/app/models/members_preloader.rb @@ -13,7 +13,7 @@ class MembersPreloader ActiveRecord::Associations::Preloader.new.preload(members, :created_by) ActiveRecord::Associations::Preloader.new.preload(members, user: :status) ActiveRecord::Associations::Preloader.new.preload(members, user: :u2f_registrations) - ActiveRecord::Associations::Preloader.new.preload(members, user: :webauthn_registrations) if Feature.enabled?(:webauthn) + ActiveRecord::Associations::Preloader.new.preload(members, user: :webauthn_registrations) if Feature.enabled?(:webauthn, default_enabled: :yaml) end end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 0cd8f12088c..f88aee38d67 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -506,12 +506,12 @@ class MergeRequest < ApplicationRecord def self.reference_pattern @reference_pattern ||= %r{ (#{Project.reference_pattern})? - #{Regexp.escape(reference_prefix)}(?<merge_request>\d+) + #{Regexp.escape(reference_prefix)}#{Gitlab::Regex.merge_request} }x end def self.link_reference_pattern - @link_reference_pattern ||= super("merge_requests", /(?<merge_request>\d+)/) + @link_reference_pattern ||= super("merge_requests", Gitlab::Regex.merge_request) end def self.reference_valid?(reference) @@ -768,7 +768,7 @@ class MergeRequest < ApplicationRecord def diff_size # Calling `merge_request_diff.diffs.real_size` will also perform # highlighting, which we don't need here. - merge_request_diff&.real_size || diff_stats&.real_size(project: project) || diffs.real_size + merge_request_diff&.real_size || diff_stats&.real_size || diffs.real_size end def modified_paths(past_merge_request_diff: nil, fallback_on_overflow: false) @@ -1317,7 +1317,7 @@ class MergeRequest < ApplicationRecord def default_merge_commit_message(include_description: false) if self.target_project.merge_commit_template.present? && !include_description - return ::Gitlab::MergeRequests::MergeCommitMessage.new(merge_request: self).message + return ::Gitlab::MergeRequests::CommitMessageGenerator.new(merge_request: self).merge_message end closes_issues_references = visible_closing_issues_for.map do |issue| @@ -1340,6 +1340,10 @@ class MergeRequest < ApplicationRecord end def default_squash_commit_message + if self.target_project.squash_commit_template.present? + return ::Gitlab::MergeRequests::CommitMessageGenerator.new(merge_request: self).squash_message + end + title end @@ -1798,7 +1802,7 @@ class MergeRequest < ApplicationRecord def pipeline_coverage_delta if base_pipeline&.coverage && head_pipeline&.coverage - '%.2f' % (head_pipeline.coverage.to_f - base_pipeline.coverage.to_f) + head_pipeline.coverage - base_pipeline.coverage end end @@ -1880,30 +1884,7 @@ class MergeRequest < ApplicationRecord override :ensure_metrics def ensure_metrics - if Feature.enabled?(:use_upsert_query_for_mr_metrics, default_enabled: :yaml) - MergeRequest::Metrics.record!(self) - else - # Backward compatibility: some merge request metrics records will not have target_project_id filled in. - # In that case the first `safe_find_or_create_by` will return false. - # The second finder call will be eliminated in https://gitlab.com/gitlab-org/gitlab/-/issues/233507 - metrics_record = MergeRequest::Metrics.safe_find_or_create_by(merge_request_id: id, target_project_id: target_project_id) || MergeRequest::Metrics.safe_find_or_create_by(merge_request_id: id) - - metrics_record.tap do |metrics_record| - # Make sure we refresh the loaded association object with the newly created/loaded item. - # This is needed in order to have the exact functionality than before. - # - # Example: - # - # merge_request.metrics.destroy - # merge_request.ensure_metrics - # merge_request.metrics # should return the metrics record and not nil - # merge_request.metrics.merge_request # should return the same MR record - - metrics_record.target_project_id = target_project_id - metrics_record.association(:merge_request).target = self - association(:metrics).target = metrics_record - end - end + MergeRequest::Metrics.record!(self) end def allows_reviewers? diff --git a/app/models/merge_request_assignee.rb b/app/models/merge_request_assignee.rb index fd8e5860040..77b46fa50f4 100644 --- a/app/models/merge_request_assignee.rb +++ b/app/models/merge_request_assignee.rb @@ -10,6 +10,12 @@ class MergeRequestAssignee < ApplicationRecord scope :in_projects, ->(project_ids) { joins(:merge_request).where(merge_requests: { target_project_id: project_ids }) } + def set_state + if Feature.enabled?(:mr_attention_requests, self.merge_request&.project, default_enabled: :yaml) + self.state = MergeRequestReviewer.find_by(user_id: self.user_id, merge_request_id: self.merge_request_id)&.state || :attention_requested + end + end + def cache_key [model_name.cache_key, id, state, assignee.cache_key] end diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 2516ff05bda..87afb7a489a 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -719,7 +719,7 @@ class MergeRequestDiff < ApplicationRecord if compare.commits.empty? new_attributes[:state] = :empty else - diff_collection = compare.diffs(Commit.max_diff_options(project: merge_request.project)) + diff_collection = compare.diffs(Commit.max_diff_options) new_attributes[:real_size] = diff_collection.real_size if diff_collection.any? diff --git a/app/models/merge_request_reviewer.rb b/app/models/merge_request_reviewer.rb index 4abf0fa09f0..8c75fb2e4e6 100644 --- a/app/models/merge_request_reviewer.rb +++ b/app/models/merge_request_reviewer.rb @@ -6,6 +6,12 @@ class MergeRequestReviewer < ApplicationRecord belongs_to :merge_request belongs_to :reviewer, class_name: 'User', foreign_key: :user_id, inverse_of: :merge_request_reviewers + def set_state + if Feature.enabled?(:mr_attention_requests, self.merge_request&.project, default_enabled: :yaml) + self.state = MergeRequestAssignee.find_by(user_id: self.user_id, merge_request_id: self.merge_request_id)&.state || :attention_requested + end + end + def cache_key [model_name.cache_key, id, state, reviewer.cache_key] end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 353a896b3fe..4b1cf2fa217 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -51,9 +51,7 @@ class Namespace < ApplicationRecord # This should _not_ be `inverse_of: :namespace`, because that would also set # `user.namespace` when this user creates a group with themselves as `owner`. - # TODO: can this be moved into the UserNamespace class? - # evaluate in issue https://gitlab.com/gitlab-org/gitlab/-/issues/341070 - belongs_to :owner, class_name: "User" + belongs_to :owner, class_name: 'User' belongs_to :parent, class_name: "Namespace" has_many :children, -> { where(type: Group.sti_name) }, class_name: "Namespace", foreign_key: :parent_id @@ -66,6 +64,9 @@ class Namespace < ApplicationRecord has_one :admin_note, inverse_of: :namespace accepts_nested_attributes_for :admin_note, update_only: true + has_one :ci_namespace_mirror, class_name: 'Ci::NamespaceMirror' + has_many :sync_events, class_name: 'Namespaces::SyncEvent' + validates :owner, presence: true, if: ->(n) { n.owner_required? } validates :name, presence: true, @@ -96,7 +97,7 @@ class Namespace < ApplicationRecord validates :max_artifacts_size, numericality: { only_integer: true, greater_than: 0, allow_nil: true } - validate :validate_parent_type, if: -> { Feature.enabled?(:validate_namespace_parent_type, default_enabled: :yaml) } + validate :validate_parent_type # ProjectNamespaces excluded as they are not meant to appear in the group hierarchy at the moment. validate :nesting_level_allowed, unless: -> { project_namespace? } @@ -106,6 +107,8 @@ class Namespace < ApplicationRecord delegate :name, to: :owner, allow_nil: true, prefix: true delegate :avatar_url, to: :owner, allow_nil: true + after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_parent_id? } + after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } before_create :sync_share_with_group_lock_with_parent @@ -122,12 +125,8 @@ class Namespace < ApplicationRecord saved_change_to_name? || saved_change_to_path? || saved_change_to_parent_id? } - # TODO: change to `type: Namespaces::UserNamespace.sti_name` when - # working on issue https://gitlab.com/gitlab-org/gitlab/-/issues/341070 - scope :user_namespaces, -> { where(type: [nil, Namespaces::UserNamespace.sti_name]) } - # TODO: this can be simplified with `type != 'Project'` when working on issue - # https://gitlab.com/gitlab-org/gitlab/-/issues/341070 - scope :without_project_namespaces, -> { where(Namespace.arel_table[:type].is_distinct_from(Namespaces::ProjectNamespace.sti_name)) } + scope :user_namespaces, -> { where(type: Namespaces::UserNamespace.sti_name) } + scope :without_project_namespaces, -> { where(Namespace.arel_table[:type].not_eq(Namespaces::ProjectNamespace.sti_name)) } scope :sort_by_type, -> { order(Gitlab::Database.nulls_first_order(:type)) } scope :include_route, -> { includes(:route) } scope :by_parent, -> (parent) { where(parent_id: parent) } @@ -615,6 +614,13 @@ class Namespace < ApplicationRecord def enforce_minimum_path_length? path_changed? && !project_namespace? end + + # SyncEvents are created by PG triggers (with the function `insert_namespaces_sync_event`) + def schedule_sync_event_worker + run_after_commit do + Namespaces::SyncEvent.enqueue_worker + end + end end Namespace.prepend_mod_with('Namespace') diff --git a/app/models/namespaces/project_namespace.rb b/app/models/namespaces/project_namespace.rb index 22ec550dee2..fbd87e3232d 100644 --- a/app/models/namespaces/project_namespace.rb +++ b/app/models/namespaces/project_namespace.rb @@ -7,5 +7,9 @@ module Namespaces def self.sti_name 'Project' end + + def self.polymorphic_name + 'Namespaces::ProjectNamespace' + end end end diff --git a/app/models/namespaces/sync_event.rb b/app/models/namespaces/sync_event.rb new file mode 100644 index 00000000000..8534d8afb8c --- /dev/null +++ b/app/models/namespaces/sync_event.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This model serves to keep track of changes to the namespaces table in the main database, and allowing to safely +# replicate these changes to other databases. +class Namespaces::SyncEvent < ApplicationRecord + self.table_name = 'namespaces_sync_events' + + belongs_to :namespace + + scope :preload_synced_relation, -> { preload(:namespace) } + scope :order_by_id_asc, -> { order(id: :asc) } + + def self.enqueue_worker + ::Namespaces::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker + end +end diff --git a/app/models/namespaces/traversal/linear.rb b/app/models/namespaces/traversal/linear.rb index 1736fe82ca5..5a5f2a5d063 100644 --- a/app/models/namespaces/traversal/linear.rb +++ b/app/models/namespaces/traversal/linear.rb @@ -64,6 +64,13 @@ module Namespaces traversal_ids.present? end + def use_traversal_ids_for_ancestors_upto? + return false unless use_traversal_ids? + return false unless Feature.enabled?(:use_traversal_ids_for_ancestors_upto, root_ancestor, default_enabled: :yaml) + + traversal_ids.present? + end + def use_traversal_ids_for_root_ancestor? return false unless Feature.enabled?(:use_traversal_ids_for_root_ancestor, default_enabled: :yaml) @@ -114,6 +121,35 @@ module Namespaces hierarchy_order == :desc ? traversal_ids[0..-2] : traversal_ids[0..-2].reverse end + # Returns all ancestors upto but excluding the top. + # When no top is given, all ancestors are returned. + # When top is not found, returns all ancestors. + # + # This copies the behavior of the recursive method. We will deprecate + # this behavior soon. + def ancestors_upto(top = nil, hierarchy_order: nil) + return super unless use_traversal_ids_for_ancestors_upto? + + # We can't use a default value in the method definition above because + # we need to preserve those specific parameters for super. + hierarchy_order ||= :desc + + # Get all ancestor IDs inclusively between top and our parent. + top_index = top ? traversal_ids.find_index(top.id) : 0 + ids = traversal_ids[top_index...-1] + ids_string = ids.map { |id| Integer(id) }.join(',') + + # WITH ORDINALITY lets us order the result to match traversal_ids order. + from_sql = <<~SQL + unnest(ARRAY[#{ids_string}]::bigint[]) WITH ORDINALITY AS ancestors(id, ord) + INNER JOIN namespaces ON namespaces.id = ancestors.id + SQL + + self.class + .from(Arel.sql(from_sql)) + .order('ancestors.ord': hierarchy_order) + end + def self_and_ancestors(hierarchy_order: nil) return super unless use_traversal_ids_for_ancestors? @@ -168,7 +204,7 @@ module Namespaces end if bottom - skope = skope.where(id: bottom.traversal_ids[0..-1]) + skope = skope.where(id: bottom.traversal_ids) end # The original `with_depth` attribute in ObjectHierarchy increments as you diff --git a/app/models/namespaces/traversal/linear_scopes.rb b/app/models/namespaces/traversal/linear_scopes.rb index f5c44171c42..0dfb7320461 100644 --- a/app/models/namespaces/traversal/linear_scopes.rb +++ b/app/models/namespaces/traversal/linear_scopes.rb @@ -105,27 +105,32 @@ module Namespaces :traversal_ids, 'LEAD (namespaces.traversal_ids, 1) OVER (ORDER BY namespaces.traversal_ids ASC) next_traversal_ids' ) - cte = Gitlab::SQL::CTE.new(:base_cte, base) + base_cte = Gitlab::SQL::CTE.new(:descendants_base_cte, base) namespaces = Arel::Table.new(:namespaces) - records = unscoped - .with(cte.to_arel) - .from([cte.table, namespaces]) # Bound the search space to ourselves (optional) and descendants. # # WHERE (base_cte.next_traversal_ids IS NULL OR base_cte.next_traversal_ids > namespaces.traversal_ids) # AND next_traversal_ids_sibling(base_cte.traversal_ids) > namespaces.traversal_ids - records = records - .where(cte.table[:next_traversal_ids].eq(nil).or(cte.table[:next_traversal_ids].gt(namespaces[:traversal_ids]))) - .where(next_sibling_func(cte.table[:traversal_ids]).gt(namespaces[:traversal_ids])) + records = unscoped + .from([base_cte.table, namespaces]) + .where(base_cte.table[:next_traversal_ids].eq(nil).or(base_cte.table[:next_traversal_ids].gt(namespaces[:traversal_ids]))) + .where(next_sibling_func(base_cte.table[:traversal_ids]).gt(namespaces[:traversal_ids])) # AND base_cte.traversal_ids <= namespaces.traversal_ids - if include_self - records.where(cte.table[:traversal_ids].lteq(namespaces[:traversal_ids])) - else - records.where(cte.table[:traversal_ids].lt(namespaces[:traversal_ids])) - end + records = if include_self + records.where(base_cte.table[:traversal_ids].lteq(namespaces[:traversal_ids])) + else + records.where(base_cte.table[:traversal_ids].lt(namespaces[:traversal_ids])) + end + + records_cte = Gitlab::SQL::CTE.new(:descendants_cte, records) + + unscoped + .unscope(where: [:type]) + .with(base_cte.to_arel, records_cte.to_arel) + .from(records_cte.alias_to(namespaces)) end def next_sibling_func(*args) diff --git a/app/models/namespaces/traversal/recursive.rb b/app/models/namespaces/traversal/recursive.rb index 8d2c5d3be5a..53eac27aa54 100644 --- a/app/models/namespaces/traversal/recursive.rb +++ b/app/models/namespaces/traversal/recursive.rb @@ -46,6 +46,7 @@ module Namespaces object_hierarchy(self.class.where(id: id)) .ancestors(upto: top, hierarchy_order: hierarchy_order) end + alias_method :recursive_ancestors_upto, :ancestors_upto def self_and_ancestors(hierarchy_order: nil) return self.class.where(id: id) unless parent_id diff --git a/app/models/namespaces/user_namespace.rb b/app/models/namespaces/user_namespace.rb index d4d7d352e71..14b867b2607 100644 --- a/app/models/namespaces/user_namespace.rb +++ b/app/models/namespaces/user_namespace.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -# TODO: currently not created/mapped in the database, will be done in another issue -# https://gitlab.com/gitlab-org/gitlab/-/issues/341070 module Namespaces #################################################################### # PLEASE DO NOT OVERRIDE METHODS IN THIS CLASS! diff --git a/app/models/note.rb b/app/models/note.rb index cb285028203..a143c21c0f9 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -23,7 +23,7 @@ class Note < ApplicationRecord include FromUnion include Sortable - cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true + cache_markdown_field :note, pipeline: :note, issuable_reference_expansion_enabled: true redact_field :note @@ -603,6 +603,15 @@ class Note < ApplicationRecord }) end + def show_outdated_changes? + return false unless for_merge_request? + return false unless Feature.enabled?(:display_outdated_line_diff, noteable.source_project, default_enabled: :yaml) + return false unless system? + return false unless change_position&.line_range + + change_position.line_range["end"] || change_position.line_range["start"] + end + private def system_note_viewable_by?(user) diff --git a/app/models/notification_reason.rb b/app/models/notification_reason.rb index c227626af9e..3713be6cb91 100644 --- a/app/models/notification_reason.rb +++ b/app/models/notification_reason.rb @@ -6,6 +6,7 @@ class NotificationReason OWN_ACTIVITY = 'own_activity' ASSIGNED = 'assigned' REVIEW_REQUESTED = 'review_requested' + ATTENTION_REQUESTED = 'attention_requested' MENTIONED = 'mentioned' SUBSCRIBED = 'subscribed' @@ -14,6 +15,7 @@ class NotificationReason OWN_ACTIVITY, ASSIGNED, REVIEW_REQUESTED, + ATTENTION_REQUESTED, MENTIONED, SUBSCRIBED ].freeze diff --git a/app/models/packages/build_info.rb b/app/models/packages/build_info.rb index 1b0f0ed8ffd..38245bef7a5 100644 --- a/app/models/packages/build_info.rb +++ b/app/models/packages/build_info.rb @@ -3,4 +3,10 @@ class Packages::BuildInfo < ApplicationRecord belongs_to :package, inverse_of: :build_infos belongs_to :pipeline, class_name: 'Ci::Pipeline' + + scope :pluck_pipeline_ids, -> { pluck(:pipeline_id) } + scope :without_empty_pipelines, -> { where.not(pipeline_id: nil) } + scope :order_by_pipeline_id, -> (direction) { order(pipeline_id: direction) } + scope :with_pipeline_id_less_than, -> (pipeline_id) { where("pipeline_id < ?", pipeline_id) } + scope :with_pipeline_id_greater_than, -> (pipeline_id) { where("pipeline_id > ?", pipeline_id) } end diff --git a/app/models/packages/conan/metadatum.rb b/app/models/packages/conan/metadatum.rb index 7ec2641177a..58af34879af 100644 --- a/app/models/packages/conan/metadatum.rb +++ b/app/models/packages/conan/metadatum.rb @@ -1,19 +1,19 @@ # frozen_string_literal: true class Packages::Conan::Metadatum < ApplicationRecord + NONE_VALUE = '_' + belongs_to :package, -> { where(package_type: :conan) }, inverse_of: :conan_metadatum validates :package, presence: true validates :package_username, - presence: true, - format: { with: Gitlab::Regex.conan_recipe_component_regex } - - validates :package_channel, - presence: true, - format: { with: Gitlab::Regex.conan_recipe_component_regex } + :package_channel, + presence: true, + format: { with: Gitlab::Regex.conan_recipe_user_channel_regex } validate :conan_package_type + validate :username_channel_none_values def recipe "#{package.name}/#{package.version}@#{package_username}/#{package_channel}" @@ -31,6 +31,15 @@ class Packages::Conan::Metadatum < ApplicationRecord package_username.tr('+', '/') end + def self.validate_username_and_channel(username, channel) + return if (username != NONE_VALUE && channel != NONE_VALUE) || + (username == NONE_VALUE && channel == NONE_VALUE) + + none_field = username == NONE_VALUE ? :username : :channel + + yield(none_field) + end + private def conan_package_type @@ -38,4 +47,10 @@ class Packages::Conan::Metadatum < ApplicationRecord errors.add(:base, _('Package type must be Conan')) end end + + def username_channel_none_values + self.class.validate_username_and_channel(package_username, package_channel) do |none_field| + errors.add("package_#{none_field}".to_sym, _("can't be solely blank")) + end + end end diff --git a/app/models/postgresql/replication_slot.rb b/app/models/postgresql/replication_slot.rb index 1a4d3bd5794..1c38edcca61 100644 --- a/app/models/postgresql/replication_slot.rb +++ b/app/models/postgresql/replication_slot.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Postgresql - class ReplicationSlot < ApplicationRecord + class ReplicationSlot < Gitlab::Database::SharedModel self.table_name = 'pg_replication_slots' # Returns true if there are any replication slots in use. diff --git a/app/models/preloaders/group_policy_preloader.rb b/app/models/preloaders/group_policy_preloader.rb index 95d6e0b5c1f..44030140ce3 100644 --- a/app/models/preloaders/group_policy_preloader.rb +++ b/app/models/preloaders/group_policy_preloader.rb @@ -8,15 +8,12 @@ module Preloaders end def execute - Preloaders::UserMaxAccessLevelInGroupsPreloader.new(@groups, @current_user).execute - Preloaders::GroupRootAncestorPreloader.new(@groups, root_ancestor_preloads).execute + Preloaders::UserMaxAccessLevelInGroupsPreloader.new(groups, current_user).execute end private - def root_ancestor_preloads - [] - end + attr_reader :groups, :current_user end end diff --git a/app/models/preloaders/group_root_ancestor_preloader.rb b/app/models/preloaders/group_root_ancestor_preloader.rb deleted file mode 100644 index 3ca713d9635..00000000000 --- a/app/models/preloaders/group_root_ancestor_preloader.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Preloaders - class GroupRootAncestorPreloader - def initialize(groups, root_ancestor_preloads = []) - @groups = groups - @root_ancestor_preloads = root_ancestor_preloads - end - - def execute - return unless ::Feature.enabled?(:use_traversal_ids, default_enabled: :yaml) - - # type == 'Group' condition located on subquery to prevent a filter in the query - root_query = Namespace.joins("INNER JOIN (#{join_sql}) as root_query ON root_query.root_id = namespaces.id") - .select('namespaces.*, root_query.id as source_id') - - root_query = root_query.preload(*@root_ancestor_preloads) if @root_ancestor_preloads.any? - - root_ancestors_by_id = root_query.group_by(&:source_id) - - @groups.each do |group| - group.root_ancestor = root_ancestors_by_id[group.id].first - end - end - - private - - def join_sql - Group.select('id, traversal_ids[1] as root_id').where(id: @groups.map(&:id)).to_sql - end - end -end diff --git a/app/models/project.rb b/app/models/project.rb index 45999da7839..a751e8adeb0 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -102,6 +102,8 @@ class Project < ApplicationRecord after_save :update_project_statistics, if: :saved_change_to_namespace_id? + after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_namespace_id? } + after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? } after_save :save_topics @@ -394,6 +396,9 @@ class Project < ApplicationRecord has_many :timelogs + has_one :ci_project_mirror, class_name: 'Ci::ProjectMirror' + has_many :sync_events, class_name: 'Projects::SyncEvent' + accepts_nested_attributes_for :variables, allow_destroy: true accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :project_setting, update_only: true @@ -449,10 +454,11 @@ class Project < ApplicationRecord delegate :restrict_user_defined_variables, :restrict_user_defined_variables=, to: :ci_cd_settings, allow_nil: true delegate :actual_limits, :actual_plan_name, to: :namespace, allow_nil: true delegate :allow_merge_on_skipped_pipeline, :allow_merge_on_skipped_pipeline?, - :allow_merge_on_skipped_pipeline=, :has_confluence?, + :allow_merge_on_skipped_pipeline=, :has_confluence?, :has_shimo?, to: :project_setting delegate :active?, to: :prometheus_integration, allow_nil: true, prefix: true delegate :merge_commit_template, :merge_commit_template=, to: :project_setting, allow_nil: true + delegate :squash_commit_template, :squash_commit_template=, to: :project_setting, allow_nil: true delegate :log_jira_dvcs_integration_usage, :jira_dvcs_server_last_sync_at, :jira_dvcs_cloud_last_sync_at, to: :feature_usage @@ -477,7 +483,8 @@ class Project < ApplicationRecord validates :project_feature, presence: true validates :namespace, presence: true - validates :project_namespace, presence: true, if: -> { self.namespace && self.root_namespace.project_namespace_creation_enabled? } + validates :project_namespace, presence: true, on: :create, if: -> { self.namespace && self.root_namespace.project_namespace_creation_enabled? } + validates :project_namespace, presence: true, on: :update, if: -> { self.project_namespace_id_changed?(to: nil) } validates :name, uniqueness: { scope: :namespace_id } validates :import_url, public_url: { schemes: ->(project) { project.persisted? ? VALID_MIRROR_PROTOCOLS : VALID_IMPORT_PROTOCOLS }, ports: ->(project) { project.persisted? ? VALID_MIRROR_PORTS : VALID_IMPORT_PORTS }, @@ -575,18 +582,12 @@ class Project < ApplicationRecord .where('rs.path LIKE ?', "#{sanitize_sql_like(path)}/%") end - # "enabled" here means "not disabled". It includes private features! scope :with_feature_enabled, ->(feature) { - access_level_attribute = ProjectFeature.arel_table[ProjectFeature.access_level_attribute(feature)] - enabled_feature = access_level_attribute.gt(ProjectFeature::DISABLED).or(access_level_attribute.eq(nil)) - - with_project_feature.where(enabled_feature) + with_project_feature.merge(ProjectFeature.with_feature_enabled(feature)) } - # Picks a feature where the level is exactly that given. scope :with_feature_access_level, ->(feature, level) { - access_level_attribute = ProjectFeature.access_level_attribute(feature) - with_project_feature.where(project_features: { access_level_attribute => level }) + with_project_feature.merge(ProjectFeature.with_feature_access_level(feature, level)) } # Picks projects which use the given programming language @@ -687,37 +688,8 @@ class Project < ApplicationRecord end end - # project features may be "disabled", "internal", "enabled" or "public". If "internal", - # they are only available to team members. This scope returns projects where - # the feature is either public, enabled, or internal with permission for the user. - # Note: this scope doesn't enforce that the user has access to the projects, it just checks - # that the user has access to the feature. It's important to use this scope with others - # that checks project authorizations first (e.g. `filter_by_feature_visibility`). - # - # This method uses an optimised version of `with_feature_access_level` for - # logged in users to more efficiently get private projects with the given - # feature. def self.with_feature_available_for_user(feature, user) - visible = [ProjectFeature::ENABLED, ProjectFeature::PUBLIC] - - if user&.can_read_all_resources? - with_feature_enabled(feature) - elsif user - min_access_level = ProjectFeature.required_minimum_access_level(feature) - column = ProjectFeature.quoted_access_level_column(feature) - - with_project_feature - .where("#{column} IS NULL OR #{column} IN (:public_visible) OR (#{column} = :private_visible AND EXISTS (:authorizations))", - { - public_visible: visible, - private_visible: ProjectFeature::PRIVATE, - authorizations: user.authorizations_for_projects(min_access_level: min_access_level) - }) - else - # This has to be added to include features whose value is nil in the db - visible << nil - with_feature_access_level(feature, visible) - end + with_project_feature.merge(ProjectFeature.with_feature_available_for_user(feature, user)) end def self.projects_user_can(projects, user, action) @@ -1469,7 +1441,9 @@ class Project < ApplicationRecord end def disabled_integrations - [:shimo] + disabled_integrations = [] + disabled_integrations << 'shimo' unless Feature.enabled?(:shimo_integration, self) + disabled_integrations end def find_or_initialize_integration(name) @@ -1600,6 +1574,12 @@ class Project < ApplicationRecord oids(lfs_objects, oids: oids) end + def lfs_objects_oids_from_fork_source(oids: []) + return [] unless forked? + + oids(fork_source.lfs_objects, oids: oids) + end + def personal? !group end @@ -2747,6 +2727,12 @@ class Project < ApplicationRecord end end + def remove_project_authorizations(user_ids, per_batch = 1000) + user_ids.each_slice(per_batch) do |user_ids_batch| + project_authorizations.where(user_id: user_ids_batch).delete_all + end + end + private # overridden in EE @@ -2957,6 +2943,13 @@ class Project < ApplicationRecord project_namespace.shared_runners_enabled = shared_runners_enabled project_namespace.visibility_level = visibility_level end + + # SyncEvents are created by PG triggers (with the function `insert_projects_sync_event`) + def schedule_sync_event_worker + run_after_commit do + Projects::SyncEvent.enqueue_worker + end + end end Project.prepend_mod_with('Project') diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index fed19a37a16..c76332b21cd 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -17,20 +17,6 @@ class ProjectAuthorization < ApplicationRecord .group(:project_id) end - def self.insert_authorizations(rows, per_batch = 1000) - rows.each_slice(per_batch) do |slice| - tuples = slice.map do |tuple| - tuple.map { |value| connection.quote(value) } - end - - connection.execute <<-EOF.strip_heredoc - INSERT INTO project_authorizations (user_id, project_id, access_level) - VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} - ON CONFLICT DO NOTHING - EOF - end - end - # This method overrides its ActiveRecord's version in order to work correctly # with composite primary keys and fix the tests for Rails 6.1 # @@ -39,6 +25,12 @@ class ProjectAuthorization < ApplicationRecord def self.insert_all(attributes) super(attributes, unique_by: connection.schema_cache.primary_keys(table_name)) end + + def self.insert_all_in_batches(attributes, per_batch = 1000) + attributes.each_slice(per_batch) do |attributes_batch| + insert_all(attributes_batch) + end + end end ProjectAuthorization.prepend_mod_with('ProjectAuthorization') diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb index 676c28d5e1b..0d3e50837ab 100644 --- a/app/models/project_feature.rb +++ b/app/models/project_feature.rb @@ -83,6 +83,52 @@ class ProjectFeature < ApplicationRecord end end + # "enabled" here means "not disabled". It includes private features! + scope :with_feature_enabled, ->(feature) { + feature_access_level_attribute = arel_table[access_level_attribute(feature)] + enabled_feature = feature_access_level_attribute.gt(DISABLED).or(feature_access_level_attribute.eq(nil)) + + where(enabled_feature) + } + + # Picks a feature where the level is exactly that given. + scope :with_feature_access_level, ->(feature, level) { + feature_access_level_attribute = access_level_attribute(feature) + where(project_features: { feature_access_level_attribute => level }) + } + + # project features may be "disabled", "internal", "enabled" or "public". If "internal", + # they are only available to team members. This scope returns features where + # the feature is either public, enabled, or internal with permission for the user. + # Note: this scope doesn't enforce that the user has access to the projects, it just checks + # that the user has access to the feature. It's important to use this scope with others + # that checks project authorizations first (e.g. `filter_by_feature_visibility`). + # + # This method uses an optimised version of `with_feature_access_level` for + # logged in users to more efficiently get private projects with the given + # feature. + def self.with_feature_available_for_user(feature, user) + visible = [ENABLED, PUBLIC] + + if user&.can_read_all_resources? + with_feature_enabled(feature) + elsif user + min_access_level = required_minimum_access_level(feature) + column = quoted_access_level_column(feature) + + where("#{column} IS NULL OR #{column} IN (:public_visible) OR (#{column} = :private_visible AND EXISTS (:authorizations))", + { + public_visible: visible, + private_visible: PRIVATE, + authorizations: user.authorizations_for_projects(min_access_level: min_access_level, related_project_column: 'project_features.project_id') + }) + else + # This has to be added to include features whose value is nil in the db + visible << nil + with_feature_access_level(feature, visible) + end + end + def public_pages? return true unless Gitlab.config.pages.access_control diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index 6c8d2226bc9..fc834286876 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -13,6 +13,7 @@ class ProjectSetting < ApplicationRecord self.primary_key = :project_id validates :merge_commit_template, length: { maximum: 500 } + validates :squash_commit_template, length: { maximum: 500 } def squash_enabled_by_default? %w[always default_on].include?(squash_option) diff --git a/app/models/projects/sync_event.rb b/app/models/projects/sync_event.rb new file mode 100644 index 00000000000..5221b00c55f --- /dev/null +++ b/app/models/projects/sync_event.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# This model serves to keep track of changes to the namespaces table in the main database as they relate to projects, +# allowing to safely replicate changes to other databases. +class Projects::SyncEvent < ApplicationRecord + self.table_name = 'projects_sync_events' + + belongs_to :project + + scope :preload_synced_relation, -> { preload(:project) } + scope :order_by_id_asc, -> { order(id: :asc) } + + def self.enqueue_worker + ::Projects::ProcessSyncEventsWorker.perform_async # rubocop:disable CodeReuse/Worker + end +end diff --git a/app/models/repository.rb b/app/models/repository.rb index 47482f04bca..645cc9773bd 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -519,6 +519,8 @@ class Repository raw_repository.batch_blobs(items, blob_size_limit: blob_size_limit).map do |blob| Blob.decorate(blob, container) end + rescue Gitlab::Git::Repository::NoRepository + [] end def root_ref diff --git a/app/models/serverless/domain.rb b/app/models/serverless/domain.rb index 2fef3b66b08..164f93afa9a 100644 --- a/app/models/serverless/domain.rb +++ b/app/models/serverless/domain.rb @@ -37,7 +37,7 @@ module Serverless 'a1', serverless_domain_cluster.uuid[2..-3], 'f2', - serverless_domain_cluster.uuid[-2..-1] + serverless_domain_cluster.uuid[-2..] ].join end end diff --git a/app/models/snippet.rb b/app/models/snippet.rb index dd76f2c3c84..6a8123b3c08 100644 --- a/app/models/snippet.rb +++ b/app/models/snippet.rb @@ -98,87 +98,115 @@ class Snippet < ApplicationRecord mode: :per_attribute_iv, algorithm: 'aes-256-cbc' - def self.with_optional_visibility(value = nil) - if value - where(visibility_level: value) - else - all + class << self + # Searches for snippets with a matching title, description or file name. + # + # This method uses ILIKE on PostgreSQL. + # + # query - The search query as a String. + # + # Returns an ActiveRecord::Relation. + def search(query) + fuzzy_search(query, [:title, :description, :file_name]) end - end - def self.only_personal_snippets - where(project_id: nil) - end + def parent_class + ::Project + end - def self.only_project_snippets - where.not(project_id: nil) - end + def sanitized_file_name(file_name) + file_name.gsub(/[^a-zA-Z0-9_\-\.]+/, '') + end - def self.only_include_projects_visible_to(current_user = nil) - levels = Gitlab::VisibilityLevel.levels_for_user(current_user) + def with_optional_visibility(value = nil) + if value + where(visibility_level: value) + else + all + end + end - joins(:project).where(projects: { visibility_level: levels }) - end + def only_personal_snippets + where(project_id: nil) + end - def self.only_include_projects_with_snippets_enabled(include_private: false) - column = ProjectFeature.access_level_attribute(:snippets) - levels = [ProjectFeature::ENABLED, ProjectFeature::PUBLIC] + def only_project_snippets + where.not(project_id: nil) + end - levels << ProjectFeature::PRIVATE if include_private + def only_include_projects_visible_to(current_user = nil) + levels = Gitlab::VisibilityLevel.levels_for_user(current_user) - joins(project: :project_feature) - .where(project_features: { column => levels }) - end + joins(:project).where(projects: { visibility_level: levels }) + end - def self.only_include_authorized_projects(current_user) - where( - 'EXISTS (?)', - ProjectAuthorization - .select(1) - .where('project_id = snippets.project_id') - .where(user_id: current_user.id) - ) - end + def only_include_projects_with_snippets_enabled(include_private: false) + column = ProjectFeature.access_level_attribute(:snippets) + levels = [ProjectFeature::ENABLED, ProjectFeature::PUBLIC] - def self.for_project_with_user(project, user = nil) - return none unless project.snippets_visible?(user) + levels << ProjectFeature::PRIVATE if include_private - if user && project.team.member?(user) - project.snippets - else - project.snippets.public_to_user(user) + joins(project: :project_feature) + .where(project_features: { column => levels }) end - end - def self.visible_to_or_authored_by(user) - query = where(visibility_level: Gitlab::VisibilityLevel.levels_for_user(user)) - query.or(where(author_id: user.id)) - end + def only_include_authorized_projects(current_user) + where( + 'EXISTS (?)', + ProjectAuthorization + .select(1) + .where('project_id = snippets.project_id') + .where(user_id: current_user.id) + ) + end - def self.reference_prefix - '$' - end + def for_project_with_user(project, user = nil) + return none unless project.snippets_visible?(user) + + if user && project.team.member?(user) + project.snippets + else + project.snippets.public_to_user(user) + end + end - # Pattern used to extract `$123` snippet references from text - # - # This pattern supports cross-project references. - def self.reference_pattern - @reference_pattern ||= %r{ + def visible_to_or_authored_by(user) + query = where(visibility_level: Gitlab::VisibilityLevel.levels_for_user(user)) + query.or(where(author_id: user.id)) + end + + def reference_prefix + '$' + end + + # Pattern used to extract `$123` snippet references from text + # + # This pattern supports cross-project references. + def reference_pattern + @reference_pattern ||= %r{ (#{Project.reference_pattern})? #{Regexp.escape(reference_prefix)}(?<snippet>\d+) }x - end + end - def self.link_reference_pattern - @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/) - end + def link_reference_pattern + @link_reference_pattern ||= super("snippets", /(?<snippet>\d+)/) + end - def self.find_by_id_and_project(id:, project:) - Snippet.find_by(id: id, project: project) - end + def find_by_id_and_project(id:, project:) + Snippet.find_by(id: id, project: project) + end + + def find_by_project_title_trunc_created_at(project, title, created_at) + where(project: project, title: title) + .find_by( + "date_trunc('second', created_at at time zone :tz) at time zone :tz = :created_at", + tz: created_at.zone, created_at: created_at) + end - def self.max_file_limit - MAX_FILE_COUNT + def max_file_limit + MAX_FILE_COUNT + end end def initialize(attributes = {}) @@ -230,10 +258,6 @@ class Snippet < ApplicationRecord super.to_s end - def self.sanitized_file_name(file_name) - file_name.gsub(/[^a-zA-Z0-9_\-\.]+/, '') - end - def visibility_level_field :visibility_level end @@ -371,23 +395,6 @@ class Snippet < ApplicationRecord def multiple_files? list_files.size > 1 end - - class << self - # Searches for snippets with a matching title, description or file name. - # - # This method uses ILIKE on PostgreSQL. - # - # query - The search query as a String. - # - # Returns an ActiveRecord::Relation. - def search(query) - fuzzy_search(query, [:title, :description, :file_name]) - end - - def parent_class - ::Project - end - end end Snippet.prepend_mod_with('Snippet') diff --git a/app/models/system_note_metadata.rb b/app/models/system_note_metadata.rb index 749b9dce97c..7b13109dbc4 100644 --- a/app/models/system_note_metadata.rb +++ b/app/models/system_note_metadata.rb @@ -24,6 +24,7 @@ class SystemNoteMetadata < ApplicationRecord opened closed merged duplicate locked unlocked outdated reviewer tag due_date pinned_embed cherry_pick health_status approved unapproved status alert_issue_added relate unrelate new_alert_added severity + attention_requested attention_request_removed ].freeze validates :note, presence: true, unless: :importing? diff --git a/app/models/todo.rb b/app/models/todo.rb index cfcb2201b80..dc436570f52 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -3,6 +3,7 @@ class Todo < ApplicationRecord include Sortable include FromUnion + include EachBatch # Time to wait for todos being removed when not visible for user anymore. # Prevents TODOs being removed by mistake, for example, removing access from a user diff --git a/app/models/u2f_registration.rb b/app/models/u2f_registration.rb index 65dc7a47533..7c01aa7a420 100644 --- a/app/models/u2f_registration.rb +++ b/app/models/u2f_registration.rb @@ -12,11 +12,7 @@ class U2fRegistration < ApplicationRecord converter = Gitlab::Auth::U2fWebauthnConverter.new(self) WebauthnRegistration.create!(converter.convert) rescue StandardError => ex - Gitlab::AppJsonLogger.error( - event: 'u2f_migration', - error: ex.class.name, - backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(ex.backtrace), - message: "U2F to WebAuthn conversion failed") + Gitlab::ErrorTracking.track_exception(ex, u2f_registration_id: self.id) end def update_webauthn_registration diff --git a/app/models/user.rb b/app/models/user.rb index 3ab5b7ee364..a39da30220a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -27,6 +27,7 @@ class User < ApplicationRecord include HasUserType include Gitlab::Auth::Otp::Fortinet include RestrictedSignup + include StripAttribute DEFAULT_NOTIFICATION_LEVEL = :participating @@ -112,10 +113,8 @@ class User < ApplicationRecord # # Namespace for personal projects - # TODO: change to `:namespace, -> { where(type: Namespaces::UserNamespace.sti_name}, class_name: 'Namespaces::UserNamespace'...` - # when working on issue https://gitlab.com/gitlab-org/gitlab/-/issues/341070 has_one :namespace, - -> { where(type: [nil, Namespaces::UserNamespace.sti_name]) }, + -> { where(type: Namespaces::UserNamespace.sti_name) }, dependent: :destroy, # rubocop:disable Cop/ActiveRecordDependent foreign_key: :owner_id, inverse_of: :owner, @@ -189,8 +188,8 @@ class User < ApplicationRecord has_one :abuse_report, dependent: :destroy, foreign_key: :user_id # rubocop:disable Cop/ActiveRecordDependent has_many :reported_abuse_reports, dependent: :destroy, foreign_key: :reporter_id, class_name: "AbuseReport" # rubocop:disable Cop/ActiveRecordDependent has_many :spam_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent - has_many :builds, dependent: :nullify, class_name: 'Ci::Build' # rubocop:disable Cop/ActiveRecordDependent - has_many :pipelines, dependent: :nullify, class_name: 'Ci::Pipeline' # rubocop:disable Cop/ActiveRecordDependent + has_many :builds, class_name: 'Ci::Build' + has_many :pipelines, class_name: 'Ci::Pipeline' has_many :todos has_many :notification_settings has_many :award_emoji, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -206,7 +205,7 @@ class User < ApplicationRecord has_many :bulk_imports has_many :custom_attributes, class_name: 'UserCustomAttribute' - has_many :callouts, class_name: 'UserCallout' + has_many :callouts, class_name: 'Users::Callout' has_many :group_callouts, class_name: 'Users::GroupCallout' has_many :term_agreements belongs_to :accepted_term, class_name: 'ApplicationSetting::Term' @@ -391,8 +390,10 @@ class User < ApplicationRecord # this state transition object in order to do a rollback. # For this reason the tradeoff is to disable this cop. after_transition any => :blocked do |user| - Ci::DropPipelineService.new.execute_async_for_all(user.pipelines, :user_blocked, user) - Ci::DisableUserPipelineSchedulesService.new.execute(user) + user.run_after_commit do + Ci::DropPipelineService.new.execute_async_for_all(user.pipelines, :user_blocked, user) + Ci::DisableUserPipelineSchedulesService.new.execute(user) + end end after_transition any => :deactivated do |user| @@ -466,6 +467,8 @@ class User < ApplicationRecord scope :by_provider_and_extern_uid, ->(provider, extern_uid) { joins(:identities).merge(Identity.with_extern_uid(provider, extern_uid)) } scope :get_ids_by_username, -> (username) { where(username: username).pluck(:id) } + strip_attributes! :name + def preferred_language read_attribute('preferred_language') || I18n.default_locale.to_s.presence_in(Gitlab::I18n.available_locales) || @@ -844,10 +847,6 @@ class User < ApplicationRecord # Instance methods # - def default_dashboard? - dashboard == self.class.column_defaults['dashboard'] - end - def full_path username end @@ -915,6 +914,8 @@ class User < ApplicationRecord end def two_factor_u2f_enabled? + return false if Feature.enabled?(:webauthn, default_enabled: :yaml) + if u2f_registrations.loaded? u2f_registrations.any? else @@ -927,7 +928,7 @@ class User < ApplicationRecord end def two_factor_webauthn_enabled? - return false unless Feature.enabled?(:webauthn) + return false unless Feature.enabled?(:webauthn, default_enabled: :yaml) (webauthn_registrations.loaded? && webauthn_registrations.any?) || (!webauthn_registrations.loaded? && webauthn_registrations.exists?) end @@ -989,11 +990,7 @@ class User < ApplicationRecord # Returns the groups a user is a member of, either directly or through a parent group def membership_groups - if Feature.enabled?(:linear_user_membership_groups, self, default_enabled: :yaml) - groups.self_and_descendants - else - Gitlab::ObjectHierarchy.new(groups).base_and_descendants - end + groups.self_and_descendants end # Returns a relation of groups the user has access to, including their parent @@ -1615,7 +1612,7 @@ class User < ApplicationRecord .select('ci_runners.*') group_runners = Ci::RunnerNamespace - .where(namespace_id: Gitlab::ObjectHierarchy.new(owned_groups).base_and_descendants.select(:id)) + .where(namespace_id: owned_groups.self_and_descendant_ids) .joins(:runner) .select('ci_runners.*') @@ -1796,7 +1793,7 @@ class User < ApplicationRecord # we do this on read since migrating all existing users is not a feasible # solution. def feed_token - Gitlab::CurrentSettings.disable_feed_token ? nil : ensure_feed_token! + ensure_feed_token! unless Gitlab::CurrentSettings.disable_feed_token end # Each existing user needs to have a `static_object_token`. @@ -1806,6 +1803,14 @@ class User < ApplicationRecord ensure_static_object_token! end + def enabled_static_object_token + static_object_token if Gitlab::CurrentSettings.static_objects_external_storage_enabled? + end + + def enabled_incoming_email_token + incoming_email_token if Gitlab::IncomingEmail.supports_issue_creation? + end + def sync_attribute?(attribute) return true if ldap_user? && attribute == :email @@ -1949,7 +1954,7 @@ class User < ApplicationRecord end def find_or_initialize_callout(feature_name) - callouts.find_or_initialize_by(feature_name: ::UserCallout.feature_names[feature_name]) + callouts.find_or_initialize_by(feature_name: ::Users::Callout.feature_names[feature_name]) end def find_or_initialize_group_callout(feature_name, group_id) @@ -2160,12 +2165,7 @@ class User < ApplicationRecord project_creation_levels << nil end - if Feature.enabled?(:linear_user_groups_with_developer_maintainer_project_access, self, default_enabled: :yaml) - developer_groups.self_and_descendants.where(project_creation_level: project_creation_levels) - else - developer_groups_hierarchy = ::Gitlab::ObjectHierarchy.new(developer_groups).base_and_descendants - ::Group.where(id: developer_groups_hierarchy.select(:id), project_creation_level: project_creation_levels) - end + developer_groups.self_and_descendants.where(project_creation_level: project_creation_levels) end def no_recent_activity? diff --git a/app/models/user_callout.rb b/app/models/user_callout.rb deleted file mode 100644 index b990aedd4f8..00000000000 --- a/app/models/user_callout.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -class UserCallout < ApplicationRecord - include Calloutable - - enum feature_name: { - gke_cluster_integration: 1, - gcp_signup_offer: 2, - cluster_security_warning: 3, - ultimate_trial: 4, # EE-only - geo_enable_hashed_storage: 5, # EE-only - geo_migrate_hashed_storage: 6, # EE-only - canary_deployment: 7, # EE-only - gold_trial_billings: 8, # EE-only - suggest_popover_dismissed: 9, - tabs_position_highlight: 10, - threat_monitoring_info: 11, # EE-only - two_factor_auth_recovery_settings_check: 12, # EE-only - web_ide_alert_dismissed: 16, # no longer in use - active_user_count_threshold: 18, # EE-only - buy_pipeline_minutes_notification_dot: 19, # EE-only - personal_access_token_expiry: 21, # EE-only - suggest_pipeline: 22, - customize_homepage: 23, - feature_flags_new_version: 24, - registration_enabled_callout: 25, - new_user_signups_cap_reached: 26, # EE-only - unfinished_tag_cleanup_callout: 27, - eoa_bronze_plan_banner: 28, # EE-only - pipeline_needs_banner: 29, - pipeline_needs_hover_tip: 30, - web_ide_ci_environments_guidance: 31, - security_configuration_upgrade_banner: 32, - cloud_licensing_subscription_activation_banner: 33, # EE-only - trial_status_reminder_d14: 34, # EE-only - trial_status_reminder_d3: 35, # EE-only - security_configuration_devops_alert: 36, # EE-only - profile_personal_access_token_expiry: 37, # EE-only - terraform_notification_dismissed: 38, - security_newsletter_callout: 39 - } - - validates :feature_name, - presence: true, - uniqueness: { scope: :user_id }, - inclusion: { in: UserCallout.feature_names.keys } -end diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index 6b0ed89c683..3787ad1c380 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -2,9 +2,6 @@ class UserDetail < ApplicationRecord extend ::Gitlab::Utils::Override - include IgnorableColumns - - ignore_columns %i[bio_html cached_markdown_version], remove_with: '14.5', remove_after: '2021-10-22' REGISTRATION_OBJECTIVE_PAIRS = { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5, joining_team: 6 }.freeze diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb new file mode 100644 index 00000000000..9ce0beed3b3 --- /dev/null +++ b/app/models/users/callout.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Users + class Callout < ApplicationRecord + include Users::Calloutable + + self.table_name = 'user_callouts' + + enum feature_name: { + gke_cluster_integration: 1, + gcp_signup_offer: 2, + cluster_security_warning: 3, + ultimate_trial: 4, # EE-only + geo_enable_hashed_storage: 5, # EE-only + geo_migrate_hashed_storage: 6, # EE-only + canary_deployment: 7, # EE-only + gold_trial_billings: 8, # EE-only + suggest_popover_dismissed: 9, + tabs_position_highlight: 10, + threat_monitoring_info: 11, # EE-only + two_factor_auth_recovery_settings_check: 12, # EE-only + web_ide_alert_dismissed: 16, # no longer in use + active_user_count_threshold: 18, # EE-only + buy_pipeline_minutes_notification_dot: 19, # EE-only + personal_access_token_expiry: 21, # EE-only + suggest_pipeline: 22, + feature_flags_new_version: 24, + registration_enabled_callout: 25, + new_user_signups_cap_reached: 26, # EE-only + unfinished_tag_cleanup_callout: 27, + eoa_bronze_plan_banner: 28, # EE-only + pipeline_needs_banner: 29, + pipeline_needs_hover_tip: 30, + web_ide_ci_environments_guidance: 31, + security_configuration_upgrade_banner: 32, + cloud_licensing_subscription_activation_banner: 33, # EE-only + trial_status_reminder_d14: 34, # EE-only + trial_status_reminder_d3: 35, # EE-only + security_configuration_devops_alert: 36, # EE-only + profile_personal_access_token_expiry: 37, # EE-only + terraform_notification_dismissed: 38, + security_newsletter_callout: 39, + verification_reminder: 40 # EE-only + } + + validates :feature_name, + presence: true, + uniqueness: { scope: :user_id }, + inclusion: { in: Users::Callout.feature_names.keys } + end +end diff --git a/app/models/users/calloutable.rb b/app/models/users/calloutable.rb new file mode 100644 index 00000000000..280a819e4d5 --- /dev/null +++ b/app/models/users/calloutable.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Users + module Calloutable + extend ActiveSupport::Concern + + included do + belongs_to :user + + validates :user, presence: true + end + + def dismissed_after?(dismissed_after) + dismissed_at > dismissed_after + end + end +end diff --git a/app/models/users/group_callout.rb b/app/models/users/group_callout.rb index 540d1a1d242..da9b95fd718 100644 --- a/app/models/users/group_callout.rb +++ b/app/models/users/group_callout.rb @@ -2,7 +2,7 @@ module Users class GroupCallout < ApplicationRecord - include Calloutable + include Users::Calloutable self.table_name = 'user_group_callouts' diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index 25438581f2f..3dbbbcdfe23 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -338,7 +338,7 @@ class WikiPage current_dirname = File.dirname(title) if persisted? - return title[1..-1] if current_dirname == '/' + return title[1..] if current_dirname == '/' return File.join([directory.presence, title].compact) if current_dirname == '.' end diff --git a/app/models/work_item/type.rb b/app/models/work_item/type.rb index 7038beadd62..3acb9c0011c 100644 --- a/app/models/work_item/type.rb +++ b/app/models/work_item/type.rb @@ -15,7 +15,8 @@ class WorkItem::Type < ApplicationRecord issue: { name: 'Issue', icon_name: 'issue-type-issue', enum_value: 0 }, incident: { name: 'Incident', icon_name: 'issue-type-incident', enum_value: 1 }, test_case: { name: 'Test Case', icon_name: 'issue-type-test-case', enum_value: 2 }, ## EE-only - requirement: { name: 'Requirement', icon_name: 'issue-type-requirements', enum_value: 3 } ## EE-only + requirement: { name: 'Requirement', icon_name: 'issue-type-requirements', enum_value: 3 }, ## EE-only + task: { name: 'Task', icon_name: 'issue-type-task', enum_value: 4 } }.freeze cache_markdown_field :description, pipeline: :single_line @@ -42,6 +43,10 @@ class WorkItem::Type < ApplicationRecord default_by_type(:issue) end + def self.allowed_types_for_issues + base_types.keys.excluding('task') + end + private def strip_whitespace diff --git a/app/models/x509_certificate.rb b/app/models/x509_certificate.rb index 428fd336a32..2c1d0110b7c 100644 --- a/app/models/x509_certificate.rb +++ b/app/models/x509_certificate.rb @@ -13,7 +13,7 @@ class X509Certificate < ApplicationRecord belongs_to :x509_issuer, class_name: 'X509Issuer', foreign_key: 'x509_issuer_id', optional: false - has_many :x509_commit_signatures, inverse_of: 'x509_certificate' + has_many :x509_commit_signatures, class_name: 'CommitSignatures::X509CommitSignature', inverse_of: 'x509_certificate' # rfc 5280 - 4.2.1.2 Subject Key Identifier validates :subject_key_identifier, presence: true, format: { with: /\A(\h{2}:){19}\h{2}\z/ } diff --git a/app/models/x509_commit_signature.rb b/app/models/x509_commit_signature.rb deleted file mode 100644 index 57d809f7cfb..00000000000 --- a/app/models/x509_commit_signature.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -class X509CommitSignature < ApplicationRecord - include ShaAttribute - - sha_attribute :commit_sha - - enum verification_status: { - unverified: 0, - verified: 1 - } - - belongs_to :project, class_name: 'Project', foreign_key: 'project_id', optional: false - belongs_to :x509_certificate, class_name: 'X509Certificate', foreign_key: 'x509_certificate_id', optional: false - - validates :commit_sha, presence: true - validates :project_id, presence: true - validates :x509_certificate_id, presence: true - - scope :by_commit_sha, ->(shas) { where(commit_sha: shas) } - - def self.safe_create!(attributes) - create_with(attributes) - .safe_find_or_create_by!(commit_sha: attributes[:commit_sha]) - end - - # Find commits that are lacking a signature in the database at present - def self.unsigned_commit_shas(commit_shas) - return [] if commit_shas.empty? - - signed = by_commit_sha(commit_shas).pluck(:commit_sha) - commit_shas - signed - end - - def commit - project.commit(commit_sha) - end - - def x509_commit - return unless commit - - Gitlab::X509::Commit.new(commit) - end - - def user - commit.committer - end -end |