diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-18 13:50:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-18 13:50:51 +0300 |
commit | db384e6b19af03b4c3c82a5760d83a3fd79f7982 (patch) | |
tree | 34beaef37df5f47ccbcf5729d7583aae093cffa0 /app/models | |
parent | 54fd7b1bad233e3944434da91d257fa7f63c3996 (diff) |
Add latest changes from gitlab-org/gitlab@16-3-stable-eev16.3.0-rc42
Diffstat (limited to 'app/models')
160 files changed, 1828 insertions, 941 deletions
diff --git a/app/models/abuse_report.rb b/app/models/abuse_report.rb index 1d2eee82827..75c90d370c3 100644 --- a/app/models/abuse_report.rb +++ b/app/models/abuse_report.rb @@ -18,6 +18,8 @@ class AbuseReport < ApplicationRecord belongs_to :assignee, class_name: 'User', inverse_of: :assigned_abuse_reports has_many :events, class_name: 'ResourceEvents::AbuseReportEvent', inverse_of: :abuse_report + has_many :label_links, as: :target, inverse_of: :target + has_many :labels, through: :label_links has_many :abuse_events, class_name: 'Abuse::Event', inverse_of: :abuse_report @@ -214,6 +216,24 @@ class AbuseReport < ApplicationRecord extension_list: valid_image_extensions.to_sentence(last_word_connector: ' or ')) ) end + + def self.aggregated_by_user_and_category(sort_by_count = false) + sub_query = self + .select('user_id, category, COUNT(id) as count', 'MIN(id) as min') + .group(:user_id, :category) + + reports = AbuseReport.with_users + .open + .select('aggregated.*, status, id, reporter_id, created_at, updated_at') + .from(sub_query, :aggregated) + .joins('INNER JOIN abuse_reports on aggregated.min = abuse_reports.id') + + if sort_by_count + reports.order(count: :desc, created_at: :desc) + else + reports + end + end end AbuseReport.prepend_mod diff --git a/app/models/admin/abuse_report_label.rb b/app/models/admin/abuse_report_label.rb new file mode 100644 index 00000000000..a2ccc8b5513 --- /dev/null +++ b/app/models/admin/abuse_report_label.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module Admin + class AbuseReportLabel < Label + end +end diff --git a/app/models/ai/service_access_token.rb b/app/models/ai/service_access_token.rb index 863bdfc7899..b8a2a271976 100644 --- a/app/models/ai/service_access_token.rb +++ b/app/models/ai/service_access_token.rb @@ -5,6 +5,7 @@ module Ai self.table_name = 'service_access_tokens' scope :expired, -> { where('expires_at < :now', now: Time.current) } + scope :active, -> { where('expires_at > :now', now: Time.current) } scope :for_category, ->(category) { where(category: category) } attr_encrypted :token, diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 291375f647c..7058bfd5650 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -6,6 +6,7 @@ class ApplicationRecord < ActiveRecord::Base include LegacyBulkInsert include CrossDatabaseModification include SensitiveSerializableHash + include ResetOnUnionError self.abstract_class = true @@ -95,7 +96,7 @@ class ApplicationRecord < ActiveRecord::Base end def self.underscore - Gitlab::SafeRequestStore.fetch("model:#{self}:underscore") { to_s.underscore } + @underscore ||= to_s.underscore end def self.where_exists(query) diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 827f8bc93be..f67efaf4f58 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -39,6 +39,12 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord encrypted_tofa_url_iv vertex_project ], remove_with: '16.3', remove_after: '2023-07-22' + ignore_column :database_apdex_settings, remove_with: '16.4', remove_after: '2023-08-22' + ignore_columns %i[ + dashboard_notification_limit + dashboard_enforcement_limit + dashboard_limit_new_namespace_creation_enforcement_date + ], remove_with: '16.5', remove_after: '2023-08-22' INSTANCE_REVIEW_MIN_USERS = 50 GRAFANA_URL_ERROR_MESSAGE = 'Please check your Grafana URL setting in ' \ @@ -254,6 +260,18 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :max_import_remote_file_size, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + validates :bulk_import_max_download_file_size, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + + validates :max_decompressed_archive_size, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :max_pages_size, presence: true, numericality: { @@ -407,6 +425,10 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, allow_nil: false + validates :protected_paths_for_get_request, + length: { maximum: 100, message: N_('is too long (maximum is 100 entries)') }, + allow_nil: false + validates :push_event_hooks_limit, numericality: { greater_than_or_equal_to: 0 } @@ -419,6 +441,8 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :max_yaml_size_bytes, numericality: { only_integer: true, greater_than: 0 }, presence: true validates :max_yaml_depth, numericality: { only_integer: true, greater_than: 0 }, presence: true + validates :ci_max_total_yaml_size_bytes, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, presence: true + validates :ci_max_includes, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, presence: true validates :email_restrictions, untrusted_regexp: true @@ -498,6 +522,10 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord end end + validates :default_project_visibility, :default_group_visibility, + exclusion: { in: :restricted_visibility_levels, message: "cannot be set to a restricted visibility level" }, + if: :should_prevent_visibility_restriction? + validates_each :import_sources do |record, attr, value| value&.each do |source| unless Gitlab::ImportSources.options.value?(source) @@ -712,18 +740,21 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord validates :inactive_projects_send_warning_email_after_months, numericality: { only_integer: true, greater_than: 0, less_than: :inactive_projects_delete_after_months } - validates :database_apdex_settings, json_schema: { filename: 'application_setting_database_apdex_settings' }, allow_nil: true + validates :prometheus_alert_db_indicators_settings, json_schema: { filename: 'application_setting_prometheus_alert_db_indicators_settings' }, allow_nil: true validates :namespace_aggregation_schedule_lease_duration_in_seconds, numericality: { only_integer: true, greater_than: 0 } + validates :sentry_clientside_traces_sample_rate, + presence: true, + numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1, message: N_('must be a value between 0 and 1') } + validates :instance_level_code_suggestions_enabled, allow_nil: false, inclusion: { in: [true, false], message: N_('must be a boolean value') } - validates :ai_access_token, - presence: { message: N_("is required to enable Code Suggestions") }, - if: :instance_level_code_suggestions_enabled + validates :package_registry_allow_anyone_to_pull_option, + inclusion: { in: [true, false], message: N_('must be a boolean value') } attr_encrypted :asset_proxy_secret_key, mode: :per_attribute_iv, @@ -951,7 +982,13 @@ class ApplicationSetting < MainClusterwide::ApplicationRecord def reset_deletion_warning_redis_key Gitlab::InactiveProjectsDeletionWarningTracker.reset_all end + + def should_prevent_visibility_restriction? + Feature.enabled?(:prevent_visibility_restriction) && + (default_project_visibility_changed? || + default_group_visibility_changed? || + restricted_visibility_levels_changed?) + end end -ApplicationSetting.prepend(ApplicationSettingMaskedAttrs) ApplicationSetting.prepend_mod_with('ApplicationSetting') diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 81e816a5b7c..f6bf535158a 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -45,6 +45,7 @@ module ApplicationSettingImplementation allow_possible_spam: false, asset_proxy_enabled: false, authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand + ci_max_total_yaml_size_bytes: 157286400, # max_yaml_size_bytes * ci_max_includes = 1.megabyte * 150 commit_email_hostname: default_commit_email_hostname, container_expiration_policies_enable_historic_entries: false, container_registry_features: [], @@ -61,6 +62,7 @@ module ApplicationSettingImplementation default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_projects_limit: Settings.gitlab['default_projects_limit'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], + default_syntax_highlighting_theme: 1, deny_all_requests_except_allowed: false, diff_max_patch_bytes: Gitlab::Git::Diff::DEFAULT_MAX_PATCH_BYTES, diff_max_files: Commit::DEFAULT_MAX_DIFF_FILES_SETTING, @@ -119,6 +121,8 @@ module ApplicationSettingImplementation max_attachment_size: Settings.gitlab['max_attachment_size'], max_export_size: 0, max_import_size: 0, + max_import_remote_file_size: 10240, + max_decompressed_archive_size: 25600, max_terraform_state_size_bytes: 0, max_yaml_size_bytes: 1.megabyte, max_yaml_depth: 100, @@ -254,6 +258,7 @@ module ApplicationSettingImplementation users_get_by_id_limit_allowlist: [], can_create_group: true, bulk_import_enabled: false, + bulk_import_max_download_file_size: 5120, allow_runner_registration_token: true, user_defaults_to_private_profile: false, projects_api_rate_limit_unauthenticated: 400, diff --git a/app/models/authentication_event.rb b/app/models/authentication_event.rb index a70ebb42008..e9fe49f980d 100644 --- a/app/models/authentication_event.rb +++ b/app/models/authentication_event.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class AuthenticationEvent < ApplicationRecord +class AuthenticationEvent < MainClusterwide::ApplicationRecord include UsageStatistics TWO_FACTOR = 'two-factor' diff --git a/app/models/batched_git_ref_updates/deletion.rb b/app/models/batched_git_ref_updates/deletion.rb new file mode 100644 index 00000000000..61bba8aeba9 --- /dev/null +++ b/app/models/batched_git_ref_updates/deletion.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module BatchedGitRefUpdates + class Deletion < ApplicationRecord + PARTITION_DURATION = 1.day + + include IgnorableColumns + include BulkInsertSafe + include PartitionedTable + include EachBatch + + self.table_name = 'p_batched_git_ref_updates_deletions' + self.primary_key = :id + self.sequence_name = :to_be_deleted_git_refs_id_seq + + # This column must be ignored otherwise Rails will cache the default value and `bulk_insert!` will start saving + # incorrect partition_id. + ignore_column :partition_id, remove_with: '3000.0', remove_after: '3000-01-01' + + belongs_to :project, inverse_of: :to_be_deleted_git_refs + + scope :for_partition, ->(partition) { where(partition_id: partition) } + scope :for_project, ->(project_id) { where(project_id: project_id) } + scope :select_ref_and_identity, -> { select(:ref, :id, arel_table[:partition_id].as('partition')) } + + partitioned_by :partition_id, strategy: :sliding_list, + next_partition_if: ->(active_partition) do + oldest_record_in_partition = Deletion + .select(:id, :created_at) + .for_partition(active_partition.value) + .order(:id) + .limit(1) + .take + + oldest_record_in_partition.present? && + oldest_record_in_partition.created_at < PARTITION_DURATION.ago + end, + detach_partition_if: ->(partition) do + !Deletion + .for_partition(partition.value) + .status_pending + .exists? + end + + enum status: { pending: 1, processed: 2 }, _prefix: :status + + def self.mark_records_processed(records) + update_by_partition(records) do |partitioned_scope| + partitioned_scope.update_all(status: :processed) + end + end + + # Your scope must select_ref_and_identity before calling this method as it relies on partition being explicitly + # selected + def self.update_by_partition(records) + records.group_by(&:partition).each do |partition, records_within_partition| + partitioned_scope = status_pending + .for_partition(partition) + .where(id: records_within_partition.map(&:id)) + + yield(partitioned_scope) + end + end + + private_class_method :update_by_partition + end +end diff --git a/app/models/broadcast_message.rb b/app/models/broadcast_message.rb deleted file mode 100644 index ccc5ca7395d..00000000000 --- a/app/models/broadcast_message.rb +++ /dev/null @@ -1,163 +0,0 @@ -# frozen_string_literal: true - -class BroadcastMessage < MainClusterwide::ApplicationRecord - include CacheMarkdownField - include Sortable - - ALLOWED_TARGET_ACCESS_LEVELS = [ - Gitlab::Access::GUEST, - Gitlab::Access::REPORTER, - Gitlab::Access::DEVELOPER, - Gitlab::Access::MAINTAINER, - Gitlab::Access::OWNER - ].freeze - - cache_markdown_field :message, pipeline: :broadcast_message, whitelisted: true - - validates :message, presence: true - validates :starts_at, presence: true - validates :ends_at, presence: true - validates :broadcast_type, presence: true - validates :target_access_levels, inclusion: { in: ALLOWED_TARGET_ACCESS_LEVELS } - validates :show_in_cli, allow_nil: false, inclusion: { in: [true, false], message: N_('must be a boolean value') } - - validates :color, allow_blank: true, color: true - validates :font, allow_blank: true, color: true - - attribute :color, default: '#E75E40' - attribute :font, default: '#FFFFFF' - - scope :current_and_future_messages, -> { where('ends_at > :now', now: Time.current).order_id_asc } - - CACHE_KEY = 'broadcast_message_current_json' - BANNER_CACHE_KEY = 'broadcast_message_current_banner_json' - NOTIFICATION_CACHE_KEY = 'broadcast_message_current_notification_json' - - after_commit :flush_redis_cache - - enum theme: { - indigo: 0, - 'light-indigo': 1, - blue: 2, - 'light-blue': 3, - green: 4, - 'light-green': 5, - red: 6, - 'light-red': 7, - dark: 8, - light: 9 - }, _default: 0, _prefix: true - - enum broadcast_type: { - banner: 1, - notification: 2 - } - - class << self - def current_banner_messages(current_path: nil, user_access_level: nil) - fetch_messages BANNER_CACHE_KEY, current_path, user_access_level do - current_and_future_messages.banner - end - end - - def current_show_in_cli_banner_messages - current_banner_messages.select(&:show_in_cli?) - end - - def current_notification_messages(current_path: nil, user_access_level: nil) - fetch_messages NOTIFICATION_CACHE_KEY, current_path, user_access_level do - current_and_future_messages.notification - end - end - - def current(current_path: nil, user_access_level: nil) - fetch_messages CACHE_KEY, current_path, user_access_level do - current_and_future_messages - end - end - - def cache - ::Gitlab::SafeRequestStore.fetch(:broadcast_message_json_cache) do - Gitlab::Cache::JsonCaches::JsonKeyed.new - end - end - - def cache_expires_in - 2.weeks - end - - private - - def fetch_messages(cache_key, current_path, user_access_level, &block) - messages = cache.fetch(cache_key, as: BroadcastMessage, expires_in: cache_expires_in, &block) - - now_or_future = messages.select(&:now_or_future?) - - # If there are cached entries but they don't match the ones we are - # displaying we'll refresh the cache so we don't need to keep filtering. - cache.expire(cache_key) if now_or_future != messages - - messages = now_or_future.select(&:now?) - messages = messages.select do |message| - message.matches_current_user_access_level?(user_access_level) - end - messages.select do |message| - message.matches_current_path(current_path) - end - end - end - - def active? - started? && !ended? - end - - def started? - Time.current >= starts_at - end - - def ended? - ends_at < Time.current - end - - def now? - (starts_at..ends_at).cover?(Time.current) - end - - def future? - starts_at > Time.current - end - - def now_or_future? - now? || future? - end - - def matches_current_user_access_level?(user_access_level) - return true unless target_access_levels.present? - - target_access_levels.include? user_access_level - end - - def matches_current_path(current_path) - return false if current_path.blank? && target_path.present? - return true if current_path.blank? || target_path.blank? - - # Ensure paths are consistent across callers. - # This fixes a mismatch between requests in the GUI and CLI - # - # This has to be reassigned due to frozen strings being provided. - current_path = "/#{current_path}" unless current_path.start_with?("/") - - escaped = Regexp.escape(target_path).gsub('\\*', '.*') - regexp = Regexp.new "^#{escaped}$", Regexp::IGNORECASE - - regexp.match(current_path) - end - - def flush_redis_cache - [CACHE_KEY, BANNER_CACHE_KEY, NOTIFICATION_CACHE_KEY].each do |key| - self.class.cache.expire(key) - end - end -end - -BroadcastMessage.prepend_mod_with('BroadcastMessage') diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 5052d84378f..d0ccf5c543a 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -3,7 +3,7 @@ module Ci class Bridge < Ci::Processable include Ci::Contextable - include Ci::Metadatable + include Ci::Deployable include Importable include AfterCommitQueue include Ci::HasRef @@ -71,7 +71,7 @@ module Ci def self.clone_accessors %i[pipeline project ref tag options name allow_failure stage stage_idx - yaml_variables when description needs_attributes + yaml_variables when environment description needs_attributes scheduling_type ci_stage partition_id].freeze end @@ -180,20 +180,6 @@ module Ci false end - def outdated_deployment? - false - end - - def expanded_environment_name - end - - def persisted_environment - end - - def deployment_job? - false - end - def execute_hooks raise NotImplementedError end @@ -266,6 +252,12 @@ module Ci end end + def expand_file_refs? + strong_memoize(:expand_file_refs) do + !Feature.enabled?(:ci_prevent_file_var_expansion_downstream_pipeline, project) + end + end + private def cross_project_params diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index bb1bfe8c889..7a623b0cefb 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -3,8 +3,8 @@ module Ci class Build < Ci::Processable prepend Ci::BulkInsertableTags - include Ci::Metadatable include Ci::Contextable + include Ci::Deployable include TokenAuthenticatable include AfterCommitQueue include Presentable @@ -34,7 +34,6 @@ module Ci DEPLOYMENT_NAMES = %w[deploy release rollout].freeze - has_one :deployment, as: :deployable, class_name: 'Deployment', inverse_of: :deployable has_one :pending_state, class_name: 'Ci::BuildPendingState', foreign_key: :build_id, inverse_of: :build has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id, inverse_of: :build has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id, inverse_of: :build @@ -158,16 +157,9 @@ module Ci .includes(:metadata, :job_artifacts_metadata) end - scope :with_project_and_metadata, -> do - if Feature.enabled?(:non_public_artifacts, type: :development) - joins(:metadata).includes(:metadata).preload(:project) - end - end - scope :with_artifacts_not_expired, -> { with_downloadable_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.current) } scope :with_pipeline_locked_artifacts, -> { joins(:pipeline).where('pipeline.locked': Ci::Pipeline.lockeds[:artifacts_locked]) } scope :last_month, -> { where('created_at > ?', Date.today - 1.month) } - scope :manual_actions, -> { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) } scope :scheduled_actions, -> { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) } scope :ref_protected, -> { where(protected: true) } scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where("#{quoted_table_name}.id = #{Ci::BuildTraceChunk.quoted_table_name}.build_id").select(1)) } @@ -327,7 +319,6 @@ module Ci after_transition any => [:success] do |build| build.run_after_commit do - BuildSuccessWorker.perform_async(id) PagesWorker.perform_async(:deploy, id) if build.pages_generator? end end @@ -345,18 +336,6 @@ module Ci end end end - - # Synchronize Deployment Status - # Please note that the data integirty is not assured because we can't use - # a database transaction due to DB decomposition. - after_transition do |build, transition| - next if transition.loopback? - next unless build.project - - build.run_after_commit do - build.deployment&.sync_status_with(build) - end - end end def self.build_matchers(project) @@ -400,10 +379,6 @@ module Ci .fabricate! end - def other_manual_actions - pipeline.manual_actions.reject { |action| action.name == name } - end - def other_scheduled_actions pipeline.scheduled_actions.reject { |action| action.name == name } end @@ -428,15 +403,6 @@ module Ci action? && !archived? && (manual? || scheduled? || retryable?) end - def outdated_deployment? - strong_memoize(:outdated_deployment) do - deployment_job? && - incomplete? && - project.ci_forward_deployment_enabled? && - deployment&.older_than_last_successful_deployment? - end - end - def schedulable? self.when == 'delayed' && options[:start_in].present? end @@ -478,94 +444,10 @@ module Ci Gitlab::Ci::Build::Prerequisite::Factory.new(self).unmet end - def persisted_environment - return unless has_environment_keyword? - - strong_memoize(:persisted_environment) do - # This code path has caused N+1s in the past, since environments are only indirectly - # associated to builds and pipelines; see https://gitlab.com/gitlab-org/gitlab/-/issues/326445 - # We therefore batch-load them to prevent dormant N+1s until we found a proper solution. - BatchLoader.for(expanded_environment_name).batch(key: project_id) do |names, loader, args| - Environment.where(name: names, project: args[:key]).find_each do |environment| - loader.call(environment.name, environment) - end - end - end - end - - def persisted_environment=(environment) - strong_memoize(:persisted_environment) { environment } - end - - # If build.persisted_environment is a BatchLoader, we need to remove - # the method proxy in order to clone into new item here - # https://github.com/exAspArk/batch-loader/issues/31 - def actual_persisted_environment - persisted_environment.respond_to?(:__sync) ? persisted_environment.__sync : persisted_environment - end - - def expanded_environment_name - return unless has_environment_keyword? - - strong_memoize(:expanded_environment_name) do - # We're using a persisted expanded environment name in order to avoid - # variable expansion per request. - if metadata&.expanded_environment_name.present? - metadata.expanded_environment_name - else - ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all }) - end - end - end - - def expanded_kubernetes_namespace - return unless has_environment_keyword? - - namespace = options.dig(:environment, :kubernetes, :namespace) - - if namespace.present? - strong_memoize(:expanded_kubernetes_namespace) do - ExpandVariables.expand(namespace, -> { simple_variables }) - end - end - end - - def has_environment_keyword? - environment.present? - end - - def deployment_job? - has_environment_keyword? && environment_action == 'start' - end - - def stops_environment? - has_environment_keyword? && environment_action == 'stop' - end - - def environment_action - options.fetch(:environment, {}).fetch(:action, 'start') if options - end - - def environment_tier_from_options - options.dig(:environment, :deployment_tier) if options - end - - def environment_tier - environment_tier_from_options || persisted_environment.try(:tier) - end - def triggered_by?(current_user) user == current_user end - def on_stop - options&.dig(:environment, :on_stop) - end - - def stop_action_successful? - success? - end - ## # All variables, including persisted environment variables. # @@ -649,9 +531,8 @@ module Ci def google_play_variables return [] unless google_play_integration.try(:activated?) - return [] unless pipeline.protected_ref? - Gitlab::Ci::Variables::Collection.new(google_play_integration.ci_variables) + Gitlab::Ci::Variables::Collection.new(google_play_integration.ci_variables(protected_ref: pipeline.protected_ref?)) end def features @@ -1033,19 +914,6 @@ module Ci job_artifacts.all_reports end - # Virtual deployment status depending on the environment status. - def deployment_status - return unless deployment_job? - - if success? - return successful_deployment_status - elsif failed? - return :failed - end - - :creating - end - # Consider this object to have a structural integrity problems def doom! transaction do @@ -1206,31 +1074,11 @@ module Ci strong_memoize(:build_data) { Gitlab::DataBuilder::Build.build(self) } end - def successful_deployment_status - if deployment&.last? - :last - else - :out_of_date - end - end - def job_artifacts_for_types(report_types) # Use select to leverage cached associations and avoid N+1 queries job_artifacts.select { |artifact| artifact.file_type.in?(report_types) } end - def environment_url - options&.dig(:environment, :url) || persisted_environment&.external_url - end - - def environment_status - strong_memoize(:environment_status) do - if has_environment_keyword? && merge_request - EnvironmentStatus.new(project, persisted_environment, merge_request, pipeline.sha) - end - end - end - def has_expiring_artifacts? artifacts_expire_at.present? && artifacts_expire_at > Time.current end diff --git a/app/models/ci/catalog/resource.rb b/app/models/ci/catalog/resource.rb index 38603ddfe59..799cdce4af7 100644 --- a/app/models/ci/catalog/resource.rb +++ b/app/models/ci/catalog/resource.rb @@ -11,6 +11,8 @@ module Ci self.table_name = 'catalog_resources' belongs_to :project + has_many :components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :catalog_resource + has_many :versions, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :catalog_resource scope :for_projects, ->(project_ids) { where(project_id: project_ids) } scope :order_by_created_at_desc, -> { reorder(created_at: :desc) } diff --git a/app/models/ci/catalog/resources/component.rb b/app/models/ci/catalog/resources/component.rb new file mode 100644 index 00000000000..7b95c14ba7e --- /dev/null +++ b/app/models/ci/catalog/resources/component.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Ci + module Catalog + module Resources + # This class represents a CI/CD Catalog resource component. + # The data will be used as metadata of a component. + class Component < ::ApplicationRecord + self.table_name = 'catalog_resource_components' + + belongs_to :project, inverse_of: :ci_components + belongs_to :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :components + belongs_to :version, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :components + + enum resource_type: { template: 1 } + + validates :inputs, json_schema: { filename: 'catalog_resource_component_inputs' } + validates :version, :catalog_resource, :project, :name, presence: true + end + end + end +end + +Ci::Catalog::Resources::Component.prepend_mod_with('Ci::Catalog::Resources::Component') diff --git a/app/models/ci/catalog/resources/version.rb b/app/models/ci/catalog/resources/version.rb new file mode 100644 index 00000000000..68f60e6a965 --- /dev/null +++ b/app/models/ci/catalog/resources/version.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Ci + module Catalog + module Resources + # This class represents a CI/CD Catalog resource version. + # Only versions which contain valid CI components are included in this table. + class Version < ::ApplicationRecord + self.table_name = 'catalog_resource_versions' + + belongs_to :release, inverse_of: :catalog_resource_version + belongs_to :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :versions + belongs_to :project, inverse_of: :catalog_resource_versions + has_many :components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :version + + validates :release, :catalog_resource, :project, presence: true + end + end + end +end + +Ci::Catalog::Resources::Version.prepend_mod_with('Ci::Catalog::Resources::Version') diff --git a/app/models/ci/job_annotation.rb b/app/models/ci/job_annotation.rb index a8bef02cc42..a6ce4196cc1 100644 --- a/app/models/ci/job_annotation.rb +++ b/app/models/ci/job_annotation.rb @@ -3,6 +3,7 @@ module Ci class JobAnnotation < Ci::ApplicationRecord include Ci::Partitionable + include BulkInsertSafe self.table_name = :p_ci_job_annotations self.primary_key = :id @@ -13,7 +14,6 @@ module Ci validates :data, json_schema: { filename: 'ci_job_annotation_data' } validates :name, presence: true, - length: { maximum: 255 }, - uniqueness: { scope: [:job_id, :partition_id] } + length: { maximum: 255 } end end diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 11d70e088e9..3f9d8f07b06 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -60,7 +60,8 @@ module Ci requirements_v2: 'requirements_v2.json', coverage_fuzzing: 'gl-coverage-fuzzing.json', api_fuzzing: 'gl-api-fuzzing-report.json', - cyclonedx: 'gl-sbom.cdx.json' + cyclonedx: 'gl-sbom.cdx.json', + annotations: 'gl-annotations.json' }.freeze INTERNAL_TYPES = { @@ -79,6 +80,7 @@ module Ci cluster_applications: :gzip, # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/361094 lsif: :zip, cyclonedx: :gzip, + annotations: :gzip, # Security reports and license scanning reports are raw artifacts # because they used to be fetched by the frontend, but this is not the case anymore. @@ -221,7 +223,8 @@ module Ci api_fuzzing: 26, ## EE-specific cluster_image_scanning: 27, ## EE-specific cyclonedx: 28, ## EE-specific - requirements_v2: 29 ## EE-specific + requirements_v2: 29, ## EE-specific + annotations: 30 } # `file_location` indicates where actual files are stored. @@ -341,10 +344,16 @@ module Ci end def to_deleted_object_attrs(pick_up_at = nil) + final_path_store_dir, final_path_filename = nil + if file_final_path.present? + final_path_store_dir = File.dirname(file_final_path) + final_path_filename = File.basename(file_final_path) + end + { file_store: file_store, - store_dir: file.store_dir.to_s, - file: file_identifier, + store_dir: final_path_store_dir || file.store_dir.to_s, + file: final_path_filename || file_identifier, pick_up_at: pick_up_at || expire_at || Time.current } end diff --git a/app/models/ci/job_token/project_scope_link.rb b/app/models/ci/job_token/project_scope_link.rb index 96e370bba1e..14c7ee14e71 100644 --- a/app/models/ci/job_token/project_scope_link.rb +++ b/app/models/ci/job_token/project_scope_link.rb @@ -8,7 +8,7 @@ module Ci class ProjectScopeLink < Ci::ApplicationRecord self.table_name = 'ci_job_token_project_scope_links' - PROJECT_LINK_DIRECTIONAL_LIMIT = 100 + PROJECT_LINK_DIRECTIONAL_LIMIT = 200 belongs_to :source_project, class_name: 'Project' # the project added to the scope's allowlist diff --git a/app/models/ci/persistent_ref.rb b/app/models/ci/persistent_ref.rb index f713d5952bc..57e2d943a4c 100644 --- a/app/models/ci/persistent_ref.rb +++ b/app/models/ci/persistent_ref.rb @@ -11,7 +11,7 @@ module Ci delegate :project, :sha, to: :pipeline delegate :repository, to: :project - delegate :ref_exists?, :create_ref, :delete_refs, to: :repository + delegate :ref_exists?, :create_ref, :delete_refs, :async_delete_refs, to: :repository def exist? ref_exists?(path) @@ -42,6 +42,12 @@ module Ci .track_exception(e, pipeline_id: pipeline.id) end + def async_delete + return unless should_delete? + + async_delete_refs(path) + end + def path "refs/#{Repository::REF_PIPELINES}/#{pipeline.id}" end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index bd327cfbe7b..3a5db04a687 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -23,6 +23,7 @@ module Ci include IgnorableColumns ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' + ignore_column :auto_canceled_by_id_convert_to_bigint, remove_with: '16.6', remove_after: '2023-10-22' MAX_OPEN_MERGE_REQUESTS_REFS = 4 @@ -99,7 +100,7 @@ module Ci has_many :downloadable_artifacts, -> do not_expired.or(where_exists(Ci::Pipeline.artifacts_locked.where("#{Ci::Pipeline.quoted_table_name}.id = #{Ci::Build.quoted_table_name}.commit_id"))).downloadable.with_job end, through: :latest_builds, source: :job_artifacts - has_many :latest_successful_builds, -> { latest.success.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' + has_many :latest_successful_jobs, -> { latest.success.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Processable' has_many :messages, class_name: 'Ci::PipelineMessage', inverse_of: :pipeline @@ -114,7 +115,7 @@ module Ci has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus', inverse_of: :pipeline - has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline + has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Processable', inverse_of: :pipeline has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: :auto_canceled_by_id, @@ -341,7 +342,9 @@ module Ci # This needs to be kept in sync with `Ci::PipelineRef#should_delete?` after_transition any => ::Ci::Pipeline.stopped_statuses do |pipeline| pipeline.run_after_commit do - if Feature.enabled?(:pipeline_cleanup_ref_worker_async, pipeline.project) + if Feature.enabled?(:pipeline_delete_gitaly_refs_in_batches, pipeline.project) + pipeline.persistent_ref.async_delete + elsif Feature.enabled?(:pipeline_cleanup_ref_worker_async, pipeline.project) ::Ci::PipelineCleanupRefWorker.perform_async(pipeline.id) else pipeline.persistent_ref.delete @@ -409,6 +412,7 @@ module Ci joins(:pipeline_metadata).where(name_column.eq(name)) end + scope :for_status, -> (status) { where(status: status) } scope :created_after, -> (time) { where(arel_table[:created_at].gt(time)) } scope :created_before_id, -> (id) { where(arel_table[:id].lt(id)) } scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) } @@ -960,11 +964,15 @@ module Ci Ci::Bridge.latest.where(pipeline: self_and_project_descendants) end + def jobs_in_self_and_project_descendants + Ci::Processable.latest.where(pipeline: self_and_project_descendants) + end + def environments_in_self_and_project_descendants(deployment_status: nil) # We limit to 100 unique environments for application safety. # See: https://gitlab.com/gitlab-org/gitlab/-/issues/340781#note_699114700 expanded_environment_names = - builds_in_self_and_project_descendants.joins(:metadata) + jobs_in_self_and_project_descendants.joins(:metadata) .where.not(Ci::BuildMetadata.table_name => { expanded_environment_name: nil }) .distinct("#{Ci::BuildMetadata.quoted_table_name}.expanded_environment_name") .limit(100) diff --git a/app/models/ci/pipeline_chat_data.rb b/app/models/ci/pipeline_chat_data.rb index ba20c993e36..37916c0b302 100644 --- a/app/models/ci/pipeline_chat_data.rb +++ b/app/models/ci/pipeline_chat_data.rb @@ -3,6 +3,9 @@ module Ci class PipelineChatData < Ci::ApplicationRecord include Ci::NamespacedModelName + include IgnorableColumns + + ignore_column :pipeline_id_convert_to_bigint, remove_with: '16.5', remove_after: '2023-10-22' self.table_name = 'ci_pipeline_chat_data' diff --git a/app/models/ci/pipeline_message.rb b/app/models/ci/pipeline_message.rb index 5668da915e6..c997ec5cd62 100644 --- a/app/models/ci/pipeline_message.rb +++ b/app/models/ci/pipeline_message.rb @@ -2,6 +2,10 @@ module Ci class PipelineMessage < Ci::ApplicationRecord + include IgnorableColumns + + ignore_column :pipeline_id_convert_to_bigint, remove_with: '16.5', remove_after: '2023-09-22' + MAX_CONTENT_LENGTH = 10_000 belongs_to :pipeline diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index 9747f9ef527..a422aaa7daa 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -9,7 +9,6 @@ module Ci include SafelyChangeColumnDefault columns_changing_default :partition_id - ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' ignore_column :pipeline_id_convert_to_bigint, remove_with: '16.5', remove_after: '2023-10-22' belongs_to :pipeline diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 4c421f066f9..7ad1a727a0e 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -6,6 +6,7 @@ module Ci class Processable < ::CommitStatus include Gitlab::Utils::StrongMemoize include FromUnion + include Ci::Metadatable extend ::Gitlab::Utils::Override has_one :resource, class_name: 'Ci::Resource', foreign_key: 'build_id', inverse_of: :processable @@ -16,6 +17,7 @@ module Ci accepts_nested_attributes_for :needs scope :preload_needs, -> { preload(:needs) } + scope :manual_actions, -> { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) } scope :with_needs, -> (names = nil) do needs = Ci::BuildNeed.scoped_build.select(1) @@ -138,6 +140,10 @@ module Ci raise NotImplementedError end + def other_manual_actions + pipeline.manual_actions.reject { |action| action.name == name } + end + def when read_attribute(:when) || 'on_success' end diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb index 4eb5c3c9ed2..8d93429fd24 100644 --- a/app/models/ci/runner.rb +++ b/app/models/ci/runner.rb @@ -87,19 +87,23 @@ module Ci scope :active, -> (value = true) { where(active: value) } scope :paused, -> { active(false) } - scope :online, -> { where('contacted_at > ?', online_contact_time_deadline) } + scope :online, -> { where(arel_table[:contacted_at].gt(online_contact_time_deadline)) } scope :recent, -> do - where('ci_runners.created_at >= :datetime OR ci_runners.contacted_at >= :datetime', datetime: stale_deadline) + timestamp = stale_deadline + + where(arel_table[:created_at].gteq(timestamp).or(arel_table[:contacted_at].gteq(timestamp))) end scope :stale, -> do - where('ci_runners.created_at <= :datetime AND ' \ - '(ci_runners.contacted_at IS NULL OR ci_runners.contacted_at <= :datetime)', datetime: stale_deadline) + timestamp = stale_deadline + + where(arel_table[:created_at].lteq(timestamp)) + .where(arel_table[:contacted_at].eq(nil).or(arel_table[:contacted_at].lteq(timestamp))) end scope :offline, -> { where(arel_table[:contacted_at].lteq(online_contact_time_deadline)) } scope :never_contacted, -> { where(contacted_at: nil) } scope :ordered, -> { order(id: :desc) } - scope :with_recent_runner_queue, -> { where('contacted_at > ?', recent_queue_deadline) } + scope :with_recent_runner_queue, -> { where(arel_table[:contacted_at].gt(recent_queue_deadline)) } scope :with_running_builds, -> do where('EXISTS(?)', ::Ci::Build.running.select(1) @@ -513,7 +517,7 @@ module Ci private scope :with_upgrade_status, ->(upgrade_status) do - joins(:runner_version).where(runner_version: { status: upgrade_status }) + joins(:runner_managers).merge(RunnerManager.with_upgrade_status(upgrade_status)) end EXECUTOR_NAME_TO_TYPES = { diff --git a/app/models/ci/runner_manager.rb b/app/models/ci/runner_manager.rb index 3a3f95a8c69..7d8fc097f51 100644 --- a/app/models/ci/runner_manager.rb +++ b/app/models/ci/runner_manager.rb @@ -14,7 +14,8 @@ module Ci belongs_to :runner - has_many :runner_manager_builds, inverse_of: :runner_manager, class_name: 'Ci::RunnerManagerBuild' + has_many :runner_manager_builds, inverse_of: :runner_manager, foreign_key: :runner_machine_id, + class_name: 'Ci::RunnerManagerBuild' has_many :builds, through: :runner_manager_builds, class_name: 'Ci::Build' belongs_to :runner_version, inverse_of: :runner_managers, primary_key: :version, foreign_key: :version, class_name: 'Ci::RunnerVersion' @@ -48,6 +49,23 @@ module Ci where(runner_id: runner_id) end + scope :with_running_builds, -> do + where('EXISTS(?)', + Ci::Build.select(1) + .joins(:runner_manager_build) + .running + .where("#{::Ci::Build.quoted_table_name}.runner_id = #{quoted_table_name}.runner_id") + .where("#{::Ci::RunnerManagerBuild.quoted_table_name}.runner_machine_id = #{quoted_table_name}.id") + .limit(1) + ) + end + + scope :order_id_desc, -> { order(id: :desc) } + + scope :with_upgrade_status, ->(upgrade_status) do + joins(:runner_version).where(runner_version: { status: upgrade_status }) + end + def self.online_contact_time_deadline Ci::Runner.online_contact_time_deadline end diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index 4853c57d41f..5b6946b04fd 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -6,6 +6,11 @@ module Ci include Ci::Partitionable include Ci::NamespacedModelName include SafelyChangeColumnDefault + include IgnorableColumns + + ignore_columns [ + :pipeline_id_convert_to_bigint, :source_pipeline_id_convert_to_bigint + ], remove_with: '16.6', remove_after: '2023-10-22' columns_changing_default :partition_id diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index 4f9a2e44562..3a498972153 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -8,9 +8,12 @@ module Ci include Gitlab::OptimisticLocking include Presentable include SafelyChangeColumnDefault + include IgnorableColumns columns_changing_default :partition_id + ignore_column :pipeline_id_convert_to_bigint, remove_with: '16.6', remove_after: '2023-10-22' + partitionable scope: :pipeline enum status: Ci::HasStatus::STATUSES_ENUM @@ -151,7 +154,7 @@ module Ci end def manual_playable? - blocked? + blocked? || skipped? end # This will be removed with ci_remove_ensure_stage_service diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 9cae71809fd..f9a34959675 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -45,7 +45,6 @@ module Clusters end has_many :kubernetes_namespaces - has_many :metrics_dashboard_annotations, class_name: 'Metrics::Dashboard::Annotation', inverse_of: :cluster accepts_nested_attributes_for :provider_gcp, update_only: true accepts_nested_attributes_for :provider_aws, update_only: true diff --git a/app/models/commit.rb b/app/models/commit.rb index ded4b06a028..d7aa66588d3 100644 --- a/app/models/commit.rb +++ b/app/models/commit.rb @@ -29,7 +29,8 @@ class Commit delegate :project, to: :repository, allow_nil: true MIN_SHA_LENGTH = Gitlab::Git::Commit::MIN_SHA_LENGTH - COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze + MAX_SHA_LENGTH = Gitlab::Git::Commit::MAX_SHA_LENGTH + COMMIT_SHA_PATTERN = Gitlab::Git::Commit::SHA_PATTERN.freeze EXACT_COMMIT_SHA_PATTERN = /\A#{COMMIT_SHA_PATTERN}\z/.freeze # Used by GFM to match and present link extensions on node texts and hrefs. LINK_EXTENSION_PATTERN = /(patch)/.freeze diff --git a/app/models/commit_collection.rb b/app/models/commit_collection.rb index edc60a757d2..993e1af20d5 100644 --- a/app/models/commit_collection.rb +++ b/app/models/commit_collection.rb @@ -24,8 +24,12 @@ class CommitCollection commits.each(&block) end - def committers - emails = without_merge_commits.filter_map(&:committer_email).uniq + def committers(with_merge_commits: false) + emails = if with_merge_commits + commits.filter_map(&:committer_email).uniq + else + without_merge_commits.filter_map(&:committer_email).uniq + end User.by_any_email(emails) end diff --git a/app/models/commit_range.rb b/app/models/commit_range.rb index c6e507e4b6c..d882a185464 100644 --- a/app/models/commit_range.rb +++ b/app/models/commit_range.rb @@ -31,9 +31,8 @@ class CommitRange REF_PATTERN = /[0-9a-zA-Z][0-9a-zA-Z_.-]*[0-9a-zA-Z\^]/.freeze PATTERN = /#{REF_PATTERN}\.{2,3}#{REF_PATTERN}/.freeze - # In text references, the beginning and ending refs can only be SHAs - # between 7 and 40 hex characters. - STRICT_PATTERN = /\h{7,40}\.{2,3}\h{7,40}/.freeze + # In text references, the beginning and ending refs can only be valid SHAs. + STRICT_PATTERN = /#{Gitlab::Git::Commit::RAW_SHA_PATTERN}\.{2,3}#{Gitlab::Git::Commit::RAW_SHA_PATTERN}/.freeze def self.reference_prefix '@' diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb index 3f631f583b6..c2425e9460a 100644 --- a/app/models/commit_status.rb +++ b/app/models/commit_status.rb @@ -9,10 +9,16 @@ class CommitStatus < Ci::ApplicationRecord include BulkInsertableAssociations include TaggableQueries + ROUTING_FEATURE_FLAG = :ci_partitioning_use_ci_builds_routing_table + self.table_name = 'ci_builds' self.sequence_name = 'ci_builds_id_seq' self.primary_key = :id - partitionable scope: :pipeline + + partitionable scope: :pipeline, through: { + table: :p_ci_builds, + flag: ROUTING_FEATURE_FLAG + } belongs_to :user belongs_to :project diff --git a/app/models/concerns/application_setting_masked_attrs.rb b/app/models/concerns/application_setting_masked_attrs.rb deleted file mode 100644 index 14a7185e39e..00000000000 --- a/app/models/concerns/application_setting_masked_attrs.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -# Similar to MASK_PASSWORD mechanism we do for EE, see: -# https://gitlab.com/gitlab-org/gitlab/-/blob/463bb1f855d71fadef931bd50f1692ee04f211a8/ee/app/models/ee/application_setting.rb#L15 -# but for non-EE attributes. -module ApplicationSettingMaskedAttrs - MASK = '*****' - - def ai_access_token=(value) - return if value == MASK - - super - end -end diff --git a/app/models/concerns/approvable.rb b/app/models/concerns/approvable.rb index 55e138d84fb..b2828821c70 100644 --- a/app/models/concerns/approvable.rb +++ b/app/models/concerns/approvable.rb @@ -14,6 +14,7 @@ module Approvable with_approvals .merge(Approval.with_user) .where(users: { id: user_ids }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/422085') .group(:id) .having("COUNT(users.id) = ?", user_ids.size) end @@ -21,6 +22,7 @@ module Approvable with_approvals .merge(Approval.with_user) .where(users: { username: usernames }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/422085') .group(:id) .having("COUNT(users.id) = ?", usernames.size) end @@ -34,7 +36,7 @@ module Approvable .where(app_table[:merge_request_id].eq(arel_table[:id])) .select('true') .arel.exists.not - ) + ).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/422085') end end @@ -50,8 +52,12 @@ module Approvable approvals.where(user: user).any? end + def approved? + approvals.present? + end + def eligible_for_approval_by?(user) - user && !approved_by?(user) && user.can?(:approve_merge_request, self) + user.present? && !approved_by?(user) && user.can?(:approve_merge_request, self) end def eligible_for_unapproval_by?(user) diff --git a/app/models/concerns/ci/deployable.rb b/app/models/concerns/ci/deployable.rb new file mode 100644 index 00000000000..b3b80989410 --- /dev/null +++ b/app/models/concerns/ci/deployable.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +# rubocop:disable Gitlab/StrongMemoizeAttr +module Ci + module Deployable + extend ActiveSupport::Concern + + included do + prepend_mod_with('Ci::Deployable') # rubocop: disable Cop/InjectEnterpriseEditionModule + + has_one :deployment, as: :deployable, class_name: 'Deployment', inverse_of: :deployable + + state_machine :status do + after_transition any => [:success] do |job| + job.run_after_commit do + Environments::StopJobSuccessWorker.perform_async(id) + end + end + + # Synchronize Deployment Status + # Please note that the data integirty is not assured because we can't use + # a database transaction due to DB decomposition. + after_transition do |job, transition| + next if transition.loopback? + next unless job.project + + job.run_after_commit do + job.deployment&.sync_status_with(job) + end + end + end + end + + def outdated_deployment? + strong_memoize(:outdated_deployment) do + deployment_job? && + project.ci_forward_deployment_enabled? && + (!project.ci_forward_deployment_rollback_allowed? || incomplete?) && + deployment&.older_than_last_successful_deployment? + end + end + + # Virtual deployment status depending on the environment status. + def deployment_status + return unless deployment_job? + + if success? + return successful_deployment_status + elsif failed? + return :failed + end + + :creating + end + + def successful_deployment_status + if deployment&.last? + :last + else + :out_of_date + end + end + + def persisted_environment + return unless has_environment_keyword? + + strong_memoize(:persisted_environment) do + # This code path has caused N+1s in the past, since environments are only indirectly + # associated to builds and pipelines; see https://gitlab.com/gitlab-org/gitlab/-/issues/326445 + # We therefore batch-load them to prevent dormant N+1s until we found a proper solution. + BatchLoader.for(expanded_environment_name).batch(key: project_id) do |names, loader, args| + Environment.where(name: names, project: args[:key]).find_each do |environment| + loader.call(environment.name, environment) + end + end + end + end + + def persisted_environment=(environment) + strong_memoize(:persisted_environment) { environment } + end + + # If build.persisted_environment is a BatchLoader, we need to remove + # the method proxy in order to clone into new item here + # https://github.com/exAspArk/batch-loader/issues/31 + def actual_persisted_environment + persisted_environment.respond_to?(:__sync) ? persisted_environment.__sync : persisted_environment + end + + def expanded_environment_name + return unless has_environment_keyword? + + strong_memoize(:expanded_environment_name) do + # We're using a persisted expanded environment name in order to avoid + # variable expansion per request. + if metadata&.expanded_environment_name.present? + metadata.expanded_environment_name + else + ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all }) + end + end + end + + def expanded_kubernetes_namespace + return unless has_environment_keyword? + + namespace = options.dig(:environment, :kubernetes, :namespace) + + if namespace.present? # rubocop:disable Style/GuardClause + strong_memoize(:expanded_kubernetes_namespace) do + ExpandVariables.expand(namespace, -> { simple_variables }) + end + end + end + + def has_environment_keyword? + environment.present? + end + + def deployment_job? + has_environment_keyword? && environment_action == 'start' + end + + def stops_environment? + has_environment_keyword? && environment_action == 'stop' + end + + def environment_action + options.fetch(:environment, {}).fetch(:action, 'start') if options + end + + def environment_tier_from_options + options.dig(:environment, :deployment_tier) if options + end + + def environment_tier + environment_tier_from_options || persisted_environment.try(:tier) + end + + def environment_url + options&.dig(:environment, :url) || persisted_environment&.external_url + end + + def environment_status + strong_memoize(:environment_status) do + if has_environment_keyword? && merge_request + EnvironmentStatus.new(project, persisted_environment, merge_request, pipeline.sha) + end + end + end + + def on_stop + options&.dig(:environment, :on_stop) + end + + def stop_action_successful? + success? + end + end +end +# rubocop:enable Gitlab/StrongMemoizeAttr diff --git a/app/models/concerns/ci/metadatable.rb b/app/models/concerns/ci/metadatable.rb index 1c6b82d6ea7..b785e39523d 100644 --- a/app/models/concerns/ci/metadatable.rb +++ b/app/models/concerns/ci/metadatable.rb @@ -24,6 +24,12 @@ module Ci delegate :id_tokens, to: :metadata, allow_nil: true before_validation :ensure_metadata, on: :create + + scope :with_project_and_metadata, -> do + if Feature.enabled?(:non_public_artifacts, type: :development) + joins(:metadata).includes(:metadata).preload(:project) + end + end end def has_exposed_artifacts? diff --git a/app/models/concerns/ci/partitionable.rb b/app/models/concerns/ci/partitionable.rb index a3bcc7bcbbc..ec6c85d888d 100644 --- a/app/models/concerns/ci/partitionable.rb +++ b/app/models/concerns/ci/partitionable.rb @@ -80,6 +80,7 @@ module Ci def handle_partitionable_through(options) return unless options + return if Gitlab::Utils.to_boolean(ENV['DISABLE_PARTITIONABLE_SWITCH'], default: false) define_singleton_method(:routing_table_name) { options[:table] } define_singleton_method(:routing_table_name_flag) { options[:flag] } diff --git a/app/models/concerns/ci/partitionable/switch.rb b/app/models/concerns/ci/partitionable/switch.rb index c1bbd107e9f..6195f92114f 100644 --- a/app/models/concerns/ci/partitionable/switch.rb +++ b/app/models/concerns/ci/partitionable/switch.rb @@ -2,6 +2,8 @@ module Ci module Partitionable + MUTEX = Mutex.new + module Switch extend ActiveSupport::Concern @@ -14,18 +16,39 @@ module Ci predicate_builder cached_find_by_statement].freeze included do |base| - partitioned = Class.new(base) do - self.table_name = base.routing_table_name + install_partitioned_class(base) + end + + class_methods do + # `Class.new(partitionable_model)` triggers `partitionable_model.inherited` + # and we need the mutex to break the recursion without adding extra accessors + # on the model. This will be used during code loading, not runtime. + # + def install_partitioned_class(partitionable_model) + Partitionable::MUTEX.synchronize do + partitioned = Class.new(partitionable_model) do + self.table_name = partitionable_model.routing_table_name + + def self.routing_class? + true + end + + def self.sti_name + superclass.sti_name + end + end - def self.routing_class? - true + partitionable_model.const_set(:Partitioned, partitioned) end end - base.const_set(:Partitioned, partitioned) - end + def inherited(child_class) + super + return if Partitionable::MUTEX.owned? + + install_partitioned_class(child_class) + end - class_methods do def routing_class? false end @@ -51,6 +74,13 @@ module Ci end end end + + def type_condition(table = arel_table) + sti_column = table[inheritance_column] + sti_names = ([self] + descendants).map(&:sti_name).uniq + + predicate_builder.build(sti_column, sti_names) + end end end end diff --git a/app/models/concerns/cross_database_ignored_tables.rb b/app/models/concerns/cross_database_ignored_tables.rb new file mode 100644 index 00000000000..c97e405cce4 --- /dev/null +++ b/app/models/concerns/cross_database_ignored_tables.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module CrossDatabaseIgnoredTables + extend ActiveSupport::Concern + + class_methods do + def cross_database_ignore_tables(tables, options = {}) + raise "missing issue url" if options[:url].blank? + + options[:on] = %I[save destroy] if options[:on].blank? + events = Array.wrap(options[:on]) + tables = Array.wrap(tables) + + events.each do |event| + register_ignored_cross_database_event(tables, event, options) + end + end + + private + + def register_ignored_cross_database_event(tables, event, options) + case event + when :save + around_save(prepend: true) { |_, blk| temporary_ignore_cross_database_tables(tables, options, &blk) } + when :create + around_create(prepend: true) { |_, blk| temporary_ignore_cross_database_tables(tables, options, &blk) } + when :update + around_update(prepend: true) { |_, blk| temporary_ignore_cross_database_tables(tables, options, &blk) } + when :destroy + around_destroy(prepend: true) { |_, blk| temporary_ignore_cross_database_tables(tables, options, &blk) } + else + raise "Unknown #{event}" + end + end + end + + private + + def temporary_ignore_cross_database_tables(tables, options, &blk) + return yield unless options[:if].nil? || instance_eval(&options[:if]) + + url = options[:url] + Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.temporary_ignore_tables_in_transaction( + tables, url: url, &blk + ) + end +end diff --git a/app/models/concerns/each_batch.rb b/app/models/concerns/each_batch.rb index 79fb81e7820..945d286a2fd 100644 --- a/app/models/concerns/each_batch.rb +++ b/app/models/concerns/each_batch.rb @@ -219,6 +219,7 @@ module EachBatch new_count, last_value = unscoped .from(inner_query) + .unscope(where: :type) .order(count: :desc) .limit(1) .pick(:count, column) diff --git a/app/models/concerns/enum_inheritance.rb b/app/models/concerns/enum_inheritance.rb new file mode 100644 index 00000000000..1df1f3d43fd --- /dev/null +++ b/app/models/concerns/enum_inheritance.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module EnumInheritance + # == STI through Enum + # + # WARNING: Usage of STI is heavily discouraged: https://docs.gitlab.com/ee/development/database/single_table_inheritance.html + # + # Active Record allows definition of STI through the <tt>Base.inheritance_column</tt>. However, this stores the class + # name as string into the record, which is heavy and unnecessary. EnumInheritance adapts ActiveRecord to use an enum + # instead. + # + # Details: + # - Correct class mapping is specified in the <tt>self.sti_type_map<\tt>, which maps the symbol of the type to + # a fully classified class as string. + # - If the type passed does not have an specified class, then the class will be the base class + # + # Example + # class Animal + # include EnumInheritable + # + # enum animal_type: { + # dog: 1, + # cat: 2, + # bird: 3 + # } + # + # def self.inheritance_column_to_class_map = { + # dog: 'Animals::Dog', + # cat: 'Animals::Cat' + # } + # + # def self.inheritance_column = 'animal_type' + # end + # + # class Animals::Dog < Animal; end + # class Animals::Cat < Animal; end + extend ActiveSupport::Concern + + included do + def self.sti_class_to_enum_map = inheritance_column_to_class_map.invert + end + + class_methods do + extend ::Gitlab::Utils::Override + + def inheritance_column_to_class_map = {}.freeze + + override :sti_class_for + def sti_class_for(type_name) + inheritance_column_to_class_map[type_name.to_sym]&.constantize || base_class + end + + override :sti_name + def sti_name + sti_class_to_enum_map[name].to_s + end + end +end diff --git a/app/models/concerns/from_union.rb b/app/models/concerns/from_union.rb index be6744f1b2a..e816608265b 100644 --- a/app/models/concerns/from_union.rb +++ b/app/models/concerns/from_union.rb @@ -32,6 +32,9 @@ module FromUnion # remove_duplicates - A boolean indicating if duplicate entries should be # removed. Defaults to true. # + # remove_order - A boolean indicating if the order from the relations should be + # removed. Defaults to true. + # # alias_as - The alias to use for the sub query. Defaults to the name of the # table of the current model. # rubocop: disable Gitlab/Union diff --git a/app/models/concerns/has_repository.rb b/app/models/concerns/has_repository.rb index d614d6c4584..0e7381882b5 100644 --- a/app/models/concerns/has_repository.rb +++ b/app/models/concerns/has_repository.rb @@ -119,6 +119,9 @@ module HasRepository def after_repository_change_head reload_default_branch + + Gitlab::EventStore.publish( + Repositories::DefaultBranchChangedEvent.new(data: { container_id: id, container_type: self.class.name })) end def after_change_head_branch_does_not_exist(branch) diff --git a/app/models/concerns/issuable_link.rb b/app/models/concerns/issuable_link.rb index 7f29083d6c6..e884e5acecf 100644 --- a/app/models/concerns/issuable_link.rb +++ b/app/models/concerns/issuable_link.rb @@ -21,6 +21,10 @@ module IssuableLink raise NotImplementedError end + def issuable_name + issuable_type.to_s.humanize(capitalize: false) + end + # Used to get the available types for the API # overriden in EE def available_link_types @@ -53,7 +57,7 @@ module IssuableLink return unless source && target if self.class.base_class.find_by(source: target, target: source) - errors.add(:source, "is already related to this #{self.class.issuable_type}") + errors.add(:source, "is already related to this #{self.class.issuable_name}") end end end diff --git a/app/models/concerns/linkable_item.rb b/app/models/concerns/linkable_item.rb new file mode 100644 index 00000000000..135252727ab --- /dev/null +++ b/app/models/concerns/linkable_item.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# == LinkableItem concern +# +# Contains common functionality shared between related issue links and related work item links +# +# Used by IssueLink, WorkItems::RelatedWorkItemLink +# +module LinkableItem + extend ActiveSupport::Concern + include FromUnion + include IssuableLink + + included do + validate :check_existing_parent_link + + scope :for_source, ->(item) { where(source_id: item.id) } + scope :for_target, ->(item) { where(target_id: item.id) } + scope :for_items, ->(source, target) do + where(source: source, target: target).or(where(source: target, target: source)) + end + + private + + def check_existing_parent_link + return unless source && target + + existing_relation = WorkItems::ParentLink.for_parents([source, target]).for_children([source, target]) + return if existing_relation.none? + + errors.add(:source, format(_('is a parent or child of this %{item}'), item: self.class.issuable_name)) + end + end +end + +LinkableItem.include_mod_with('LinkableItem::Callbacks') +LinkableItem.prepend_mod_with('LinkableItem') diff --git a/app/models/concerns/milestoneable.rb b/app/models/concerns/milestoneable.rb index e95a8a42aa6..b72d99d211c 100644 --- a/app/models/concerns/milestoneable.rb +++ b/app/models/concerns/milestoneable.rb @@ -52,7 +52,9 @@ module Milestoneable def milestone_available? return true if milestone_id.blank? - project_id == milestone&.project_id || project.ancestors_upto.compact.include?(milestone&.group) + (project_id.present? && project_id == milestone&.project_id) || + try(:namespace)&.self_and_ancestors&.include?(milestone&.group) || + project&.ancestors_upto&.compact&.include?(milestone&.group) end ## diff --git a/app/models/concerns/noteable.rb b/app/models/concerns/noteable.rb index 5c91f2460c4..40a91c8ac94 100644 --- a/app/models/concerns/noteable.rb +++ b/app/models/concerns/noteable.rb @@ -17,7 +17,7 @@ module Noteable # `Noteable` class names that support resolvable notes. def resolvable_types - %w(MergeRequest DesignManagement::Design) + %w(Issue MergeRequest DesignManagement::Design) end # `Noteable` class names that support creating/forwarding individual notes. @@ -49,6 +49,8 @@ module Noteable end def supports_resolvable_notes? + return false if is_a?(Issue) && Feature.disabled?(:resolvable_issue_threads, project) + self.class.resolvable_types.include?(base_class_name) end @@ -171,9 +173,9 @@ module Noteable return unless etag_caching_enabled? # TODO: We need to figure out a way to make ETag caching work for group-level work items - return if is_a?(Issue) && project.nil? + Gitlab::EtagCaching::Store.new.touch(note_etag_key) unless is_a?(Issue) && project.nil? - Gitlab::EtagCaching::Store.new.touch(note_etag_key) + Noteable::NotesChannel.broadcast_to(self, event: 'updated') if Feature.enabled?(:action_cable_notes, project || try(:group)) end def note_etag_key diff --git a/app/models/concerns/packages/nuget/version_normalizable.rb b/app/models/concerns/packages/nuget/version_normalizable.rb new file mode 100644 index 00000000000..473e5f07811 --- /dev/null +++ b/app/models/concerns/packages/nuget/version_normalizable.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Packages + module Nuget + module VersionNormalizable + extend ActiveSupport::Concern + + LEADING_ZEROES_REGEX = /^(?!0$)0+(?=\d)/ + + included do + before_validation :set_normalized_version, on: %i[create update] + + private + + def set_normalized_version + return unless package && Feature.enabled?(:nuget_normalized_version, package.project) + + self.normalized_version = normalize + end + + def normalize + version = remove_leading_zeroes + version = remove_build_metadata(version) + version = omit_zero_in_fourth_part(version) + append_suffix(version) + end + + def remove_leading_zeroes + package_version.split('.').map { |part| part.sub(LEADING_ZEROES_REGEX, '') }.join('.') + end + + def remove_build_metadata(version) + version.split('+').first.downcase + end + + def omit_zero_in_fourth_part(version) + parts = version.split('.') + parts[3] = nil if parts.fourth == '0' && parts.third.exclude?('-') + parts.compact.join('.') + end + + def append_suffix(version) + version << '.0.0' if version.count('.') == 0 + version << '.0' if version.count('.') == 1 + version + end + end + end + end +end diff --git a/app/models/concerns/reset_on_union_error.rb b/app/models/concerns/reset_on_union_error.rb new file mode 100644 index 00000000000..42e350b0bed --- /dev/null +++ b/app/models/concerns/reset_on_union_error.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module ResetOnUnionError + extend ActiveSupport::Concern + + MAX_RESET_PERIOD = 10.minutes + + included do |base| + base.rescue_from ActiveRecord::StatementInvalid, with: :reset_on_union_error + + base.class_attribute :previous_reset_columns_from_error + end + + class_methods do + def reset_on_union_error(exception) + if reset_on_statement_invalid?(exception) + class_to_be_reset = base_class + + class_to_be_reset.reset_column_information + Gitlab::ErrorTracking.log_exception(exception, { reset_model_name: class_to_be_reset.name }) + + class_to_be_reset.previous_reset_columns_from_error = Time.current + end + + raise + end + + def reset_on_statement_invalid?(exception) + return false unless exception.message.include?("each UNION query must have the same number of columns") + + return false if base_class.previous_reset_columns_from_error? && + base_class.previous_reset_columns_from_error > MAX_RESET_PERIOD.ago + + Feature.enabled?(:reset_column_information_on_statement_invalid, type: :ops) + end + end +end diff --git a/app/models/concerns/resolvable_discussion.rb b/app/models/concerns/resolvable_discussion.rb index 45818942326..e967c78154d 100644 --- a/app/models/concerns/resolvable_discussion.rb +++ b/app/models/concerns/resolvable_discussion.rb @@ -116,6 +116,8 @@ module ResolvableDiscussion # Set the notes array to the updated notes @notes = notes_relation.fresh.to_a # rubocop:disable Gitlab/ModuleWithInstanceVariables + noteable.expire_note_etag_cache + clear_memoized_values end end diff --git a/app/models/concerns/resolvable_note.rb b/app/models/concerns/resolvable_note.rb index 4e8a1bb643e..7f9a7faa3f5 100644 --- a/app/models/concerns/resolvable_note.rb +++ b/app/models/concerns/resolvable_note.rb @@ -23,12 +23,13 @@ module ResolvableNote class_methods do # This method must be kept in sync with `#resolve!` def resolve!(current_user) - unresolved.update_all(resolved_at: Time.current, resolved_by_id: current_user.id) + now = Time.current + unresolved.update_all(updated_at: now, resolved_at: now, resolved_by_id: current_user.id) end # This method must be kept in sync with `#unresolve!` def unresolve! - resolved.update_all(resolved_at: nil, resolved_by_id: nil) + resolved.update_all(updated_at: Time.current, resolved_at: nil, resolved_by_id: nil) end end @@ -57,7 +58,9 @@ module ResolvableNote return false unless resolvable? return false if resolved? - self.resolved_at = Time.current + now = Time.current + self.updated_at = now + self.resolved_at = now self.resolved_by = current_user self.resolved_by_push = resolved_by_push @@ -69,6 +72,7 @@ module ResolvableNote return false unless resolvable? return false unless resolved? + self.updated_at = Time.current self.resolved_at = nil self.resolved_by = nil diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index d70aad4e9ae..f2badfe48dd 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -25,17 +25,19 @@ module Routable # # We need to qualify the columns with the table name, to support both direct lookups on # Route/RedirectRoute, and scoped lookups through the Routable classes. - route = - route_scope.find_by(routes: { path: path }) || - route_scope.iwhere(Route.arel_table[:path] => path).take + Gitlab::Database.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") do + route = + route_scope.find_by(routes: { path: path }) || + route_scope.iwhere(Route.arel_table[:path] => path).take - if follow_redirects - route ||= redirect_route_scope.iwhere(RedirectRoute.arel_table[:path] => path).take - end + if follow_redirects + route ||= redirect_route_scope.iwhere(RedirectRoute.arel_table[:path] => path).take + end - return unless route + next unless route - route.is_a?(Routable) ? route : route.source + route.is_a?(Routable) ? route : route.source + end end included do @@ -46,7 +48,9 @@ module Routable validates :route, presence: true, unless: -> { is_a?(Namespaces::ProjectNamespace) } - scope :with_route, -> { includes(:route) } + scope :with_route, -> do + includes(:route).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') + end after_validation :set_path_errors @@ -94,7 +98,9 @@ module Routable joins(:route) end - route.where(wheres.join(' OR ')) + route + .where(wheres.join(' OR ')) + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") end end diff --git a/app/models/concerns/time_trackable.rb b/app/models/concerns/time_trackable.rb index 2b7447dc700..0f361e70a91 100644 --- a/app/models/concerns/time_trackable.rb +++ b/app/models/concerns/time_trackable.rb @@ -17,8 +17,8 @@ module TimeTrackable attribute :time_estimate, default: 0 - validates :time_estimate, numericality: { message: 'has an invalid format' }, allow_nil: false - validate :check_negative_time_spent + validate :check_time_estimate + validate :check_negative_time_spent has_many :timelogs, dependent: :destroy, autosave: true # rubocop:disable Cop/ActiveRecordDependent after_initialize :set_time_estimate_default_value @@ -106,4 +106,11 @@ module TimeTrackable def original_total_time_spent @original_total_time_spent ||= total_time_spent end + + def check_time_estimate + return unless new_record? || time_estimate_changed? + return if time_estimate.is_a?(Numeric) && time_estimate >= 0 + + errors.add(:time_estimate, _('must have a valid format and be greater than or equal to zero.')) + end end diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index 2ad2e47ec4e..72812f35f72 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -3,6 +3,7 @@ module WebHooks module AutoDisabling extend ActiveSupport::Concern + include ::Gitlab::Loggable ENABLED_HOOK_TYPES = %w[ProjectHook].freeze MAX_FAILURES = 100 @@ -36,7 +37,9 @@ module WebHooks # - and either: # - disabled_until is nil (i.e. this was set by WebHook#fail!) # - or disabled_until is in the future (i.e. this was set by WebHook#backoff!) + # - OR silent mode is enabled. scope :disabled, -> do + return all if Gitlab::SilentMode.enabled? return none unless auto_disabling_enabled? where( @@ -52,7 +55,9 @@ module WebHooks # - OR we have exceeded the grace period and neither of the following is true: # - disabled_until is nil (i.e. this was set by WebHook#fail!) # - disabled_until is in the future (i.e. this was set by WebHook#backoff!) + # - AND silent mode is not enabled. scope :executable, -> do + return none if Gitlab::SilentMode.enabled? return all unless auto_disabling_enabled? where( @@ -82,17 +87,14 @@ module WebHooks recent_failures > FAILURE_THRESHOLD && disabled_until.blank? end - def disable! - return if !auto_disabling_enabled? || permanently_disabled? - - update_attribute(:recent_failures, EXCEEDED_FAILURE_THRESHOLD) - end - def enable! return unless auto_disabling_enabled? return if recent_failures == 0 && disabled_until.nil? && backoff_count == 0 - assign_attributes(recent_failures: 0, disabled_until: nil, backoff_count: 0) + attrs = { recent_failures: 0, disabled_until: nil, backoff_count: 0 } + + assign_attributes(attrs) + logger.info(hook_id: id, action: 'enable', **attrs) save(validate: false) end @@ -110,14 +112,21 @@ module WebHooks end assign_attributes(attrs) - save(validate: false) if changed? + + return unless changed? + + logger.info(hook_id: id, action: 'backoff', **attrs) + save(validate: false) end def failed! return unless auto_disabling_enabled? return unless recent_failures < MAX_FAILURES - assign_attributes(disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count) + attrs = { disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count } + + assign_attributes(**attrs) + logger.info(hook_id: id, action: 'disable', **attrs) save(validate: false) end @@ -143,6 +152,10 @@ module WebHooks private + def logger + @logger ||= Gitlab::WebHooks::Logger.build + end + def next_failure_count recent_failures.succ.clamp(1, MAX_FAILURES) end diff --git a/app/models/customer_relations/contact.rb b/app/models/customer_relations/contact.rb index 16c741d340f..f99b8fa5549 100644 --- a/app/models/customer_relations/contact.rb +++ b/app/models/customer_relations/contact.rb @@ -35,6 +35,22 @@ class CustomerRelations::Contact < ApplicationRecord scope :order_by_organization_asc, -> { includes(:organization).order("customer_relations_organizations.name ASC NULLS LAST") } scope :order_by_organization_desc, -> { includes(:organization).order("customer_relations_organizations.name DESC NULLS LAST") } + SAFE_ATTRIBUTES = %w[ + created_at + description + first_name + group_id + id + last_name + organization_id + state + updated_at + ].freeze + + def hook_attrs + attributes.slice(*SAFE_ATTRIBUTES) + end + def self.reference_prefix '[contact:' end diff --git a/app/models/dependency_proxy/manifest.rb b/app/models/dependency_proxy/manifest.rb index 11fe0503f50..702e1679f6a 100644 --- a/app/models/dependency_proxy/manifest.rb +++ b/app/models/dependency_proxy/manifest.rb @@ -15,7 +15,8 @@ class DependencyProxy::Manifest < ApplicationRecord ACCEPTED_TYPES = [ ContainerRegistry::BaseClient::DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, ContainerRegistry::BaseClient::OCI_MANIFEST_V1_TYPE, - ContainerRegistry::BaseClient::OCI_DISTRIBUTION_INDEX_TYPE + ContainerRegistry::BaseClient::OCI_DISTRIBUTION_INDEX_TYPE, + ContainerRegistry::BaseClient::DOCKER_DISTRIBUTION_MANIFEST_LIST_V2_TYPE ].freeze validates :group, presence: true diff --git a/app/models/deployment.rb b/app/models/deployment.rb index b59b22c10c4..0bdce18bab5 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -67,7 +67,7 @@ class Deployment < ApplicationRecord state_machine :status, initial: :created do event :run do - transition created: :running + transition [:created, :blocked] => :running end event :block do @@ -79,10 +79,6 @@ class Deployment < ApplicationRecord transition skipped: :created end - event :unblock do - transition blocked: :created - end - event :succeed do transition any - [:success] => :success end @@ -184,23 +180,23 @@ class Deployment < ApplicationRecord # - deploy job B => production environment # In this case, `last_deployment_group` returns both deployments. # - # NOTE: Preload environment.last_deployment and pipeline.latest_successful_builds prior to avoid N+1. + # NOTE: Preload environment.last_deployment and pipeline.latest_successful_jobs prior to avoid N+1. def self.last_deployment_group_for_environment(env) - return self.none unless env.last_deployment_pipeline&.latest_successful_builds&.present? + return self.none unless env.last_deployment_pipeline&.latest_successful_jobs&.present? BatchLoader.for(env).batch(default_value: self.none) do |environments, loader| - latest_successful_build_ids = [] + latest_successful_job_ids = [] environments_hash = {} environments.each do |environment| environments_hash[environment.id] = environment # Refer comment note above, if not preloaded this can lead to N+1. - latest_successful_build_ids << environment.last_deployment_pipeline.latest_successful_builds.map(&:id) + latest_successful_job_ids << environment.last_deployment_pipeline.latest_successful_jobs.map(&:id) end Deployment - .where(deployable_type: 'CommitStatus', deployable_id: latest_successful_build_ids.flatten) + .where(deployable_type: 'CommitStatus', deployable_id: latest_successful_job_ids.flatten) .preload(last_deployment_group_associations) .group_by { |deployment| deployment.environment_id } .each do |env_id, deployment_group| @@ -217,14 +213,14 @@ class Deployment < ApplicationRecord # Fetching any unbounded or large intermediate dataset could lead to loading too many IDs into memory. # See: https://docs.gitlab.com/ee/development/database/multiple_databases.html#use-disable_joins-for-has_one-or-has_many-through-relations # For safety we default limit to fetch not more than 1000 records. - def self.builds(limit = 1000) + def self.jobs(limit = 1000) deployable_ids = where.not(deployable_id: nil).limit(limit).pluck(:deployable_id) - Ci::Build.where(id: deployable_ids) + Ci::Processable.where(id: deployable_ids) end - def build - deployable if deployable.is_a?(::Ci::Build) + def job + deployable if deployable.is_a?(::Ci::Processable) end class << self @@ -289,8 +285,8 @@ class Deployment < ApplicationRecord @scheduled_actions ||= deployable.try(:other_scheduled_actions) end - def playable_build - strong_memoize(:playable_build) do + def playable_job + strong_memoize(:playable_job) do deployable.try(:playable?) ? deployable : nil end end @@ -355,8 +351,8 @@ class Deployment < ApplicationRecord end def deployed_by - # We use deployable's user if available because Ci::PlayBuildService - # does not update the deployment's user, just the one for the deployable. + # We use deployable's user if available because Ci::PlayBuildService and Ci::PlayBridgeService + # do not update the deployment's user, just the one for the deployable. # TODO: use deployment's user once https://gitlab.com/gitlab-org/gitlab-foss/issues/66442 # is completed. deployable&.user || user @@ -402,14 +398,17 @@ class Deployment < ApplicationRecord false end - def sync_status_with(build) - return false unless ::Deployment.statuses.include?(build.status) - return false if build.status == self.status + def sync_status_with(job) + job_status = job.status + job_status = 'blocked' if job_status == 'manual' + + return false unless ::Deployment.statuses.include?(job_status) + return false if job_status == self.status - update_status!(build.status) + update_status!(job_status) rescue StandardError => e Gitlab::ErrorTracking.track_exception( - StatusSyncError.new(e.message), deployment_id: self.id, build_id: build.id) + StatusSyncError.new(e.message), deployment_id: self.id, job_id: job.id) false end diff --git a/app/models/discussion.rb b/app/models/discussion.rb index dc4794ed3cd..2d2519dc995 100644 --- a/app/models/discussion.rb +++ b/app/models/discussion.rb @@ -191,4 +191,8 @@ class Discussion def to_global_id(options = {}) GlobalID.new(::Gitlab::GlobalId.build(model_name: Discussion.to_s, id: id)) end + + def noteable_collection_name + noteable.class.underscore.pluralize + end end diff --git a/app/models/email.rb b/app/models/email.rb index 3896dfd5d22..5fca57520b8 100644 --- a/app/models/email.rb +++ b/app/models/email.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Email < ApplicationRecord +class Email < MainClusterwide::ApplicationRecord include Sortable include Gitlab::SQL::Pattern diff --git a/app/models/environment.rb b/app/models/environment.rb index 241b454f5ce..36445279b86 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -18,14 +18,13 @@ class Environment < ApplicationRecord belongs_to :cluster_agent, class_name: 'Clusters::Agent', optional: true, inverse_of: :environments use_fast_destroy :all_deployments - nullify_if_blank :external_url, :kubernetes_namespace + nullify_if_blank :external_url, :kubernetes_namespace, :flux_resource_path has_many :all_deployments, class_name: 'Deployment' has_many :deployments, -> { visible } has_many :successful_deployments, -> { success }, class_name: 'Deployment' has_many :active_deployments, -> { active }, class_name: 'Deployment' has_many :prometheus_alerts, inverse_of: :environment - has_many :metrics_dashboard_annotations, class_name: 'Metrics::Dashboard::Annotation', inverse_of: :environment has_many :self_managed_prometheus_alert_events, inverse_of: :environment has_many :alert_management_alerts, class_name: 'AlertManagement::Alert', inverse_of: :environment @@ -78,6 +77,10 @@ class Environment < ApplicationRecord message: Gitlab::Regex.kubernetes_namespace_regex_message } + validates :flux_resource_path, + length: { maximum: 255 }, + allow_nil: true + validates :tier, presence: true validate :safe_external_url @@ -331,9 +334,9 @@ class Environment < ApplicationRecord end def cancel_deployment_jobs! - active_deployments.builds.each do |build| - Gitlab::OptimisticLocking.retry_lock(build, name: 'environment_cancel_deployment_jobs') do |build| - build.cancel! if build&.cancelable? + active_deployments.jobs.each do |job| + Gitlab::OptimisticLocking.retry_lock(job, name: 'environment_cancel_deployment_jobs') do |job| + job.cancel! if job&.cancelable? end rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, environment_id: id, deployment_id: deployment.id) @@ -355,8 +358,12 @@ class Environment < ApplicationRecord Gitlab::OptimisticLocking.retry_lock( stop_action, name: 'environment_stop_with_actions' - ) do |build| - actions << build.play(current_user) + ) do |job| + actions << job.play(current_user) + rescue StateMachines::InvalidTransition + # Ci::PlayBuildService rescues an error of StateMachines::InvalidTransition and fall back to retry. However, + # Ci::PlayBridgeService doesn't rescue it, so we're ignoring the error if it's not playable. + # We should fix this inconsistency in https://gitlab.com/gitlab-org/gitlab/-/issues/420855. end end diff --git a/app/models/event.rb b/app/models/event.rb index 9345776c32b..4547d7b9e60 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -11,7 +11,7 @@ class Event < ApplicationRecord include ShaAttribute include IgnorableColumns - ignore_column :target_id_convert_to_bigint, remove_with: '16.2', remove_after: '2023-07-22' + ignore_column :target_id_convert_to_bigint, remove_with: '16.4', remove_after: '2023-09-22' ACTIONS = HashWithIndifferentAccess.new( created: 1, diff --git a/app/models/group.rb b/app/models/group.rb index 2b5a392e02c..9df3c143e0c 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -184,6 +184,7 @@ class Group < Namespace ids_by_full_path = Route .for_routable_type(Namespace.name) .where('LOWER(routes.path) IN (?)', paths.map(&:downcase)) + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") .select(:namespace_id) Group.from_union([by_id(ids), by_id(ids_by_full_path), where('LOWER(path) IN (?)', paths.map(&:downcase))]) @@ -397,7 +398,7 @@ class Group < Namespace end def visibility_level_allowed_by_projects?(level = self.visibility_level) - !projects.where('visibility_level > ?', level).exists? + !projects.without_deleted.where('visibility_level > ?', level).exists? end def visibility_level_allowed_by_sub_groups?(level = self.visibility_level) @@ -635,19 +636,26 @@ class Group < Namespace end # Returns all members that are part of the group, it's subgroups, and ancestor groups - def direct_and_indirect_members + def hierarchy_members GroupMember .active_without_invites_and_requests .where(source_id: self_and_hierarchy.reorder(nil).select(:id)) end - def direct_and_indirect_members_with_inactive + def hierarchy_members_with_inactive GroupMember .non_request .non_invite .where(source_id: self_and_hierarchy.reorder(nil).select(:id)) end + def descendant_project_members_with_inactive + ProjectMember + .with_source_id(all_projects) + .non_request + .non_invite + end + def users_with_parents User .where(id: members_with_parents.select(:user_id)) @@ -660,45 +668,6 @@ class Group < Namespace .reorder(nil) end - # Returns all users that are members of the group because: - # 1. They belong to the group - # 2. They belong to a project that belongs to the group - # 3. They belong to a sub-group or project in such sub-group - # 4. They belong to an ancestor group - # 5. They belong to a group that is shared with this group, if share_with_groups is true - def direct_and_indirect_users(share_with_groups: false) - members = if share_with_groups - # We only need :user_id column, but - # `members_from_self_and_ancestor_group_shares` needs more - # columns to make the CTE query work. - GroupMember.from_union([ - direct_and_indirect_members.select(:user_id, :source_type, :type), - members_from_self_and_ancestor_group_shares.reselect(:user_id, :source_type, :type) - ]) - else - direct_and_indirect_members - end - - User.from_union([ - User.where(id: members.select(:user_id)).reorder(nil), - project_users_with_descendants - ]) - end - - # Returns all users (also inactive) that are members of the group because: - # 1. They belong to the group - # 2. They belong to a project that belongs to the group - # 3. They belong to a sub-group or project in such sub-group - # 4. They belong to an ancestor group - def direct_and_indirect_users_with_inactive - User.from_union([ - User - .where(id: direct_and_indirect_members_with_inactive.select(:user_id)) - .reorder(nil), - project_users_with_descendants - ]).allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/417455") # failed in spec/tasks/gitlab/user_management_rake_spec.rb - end - def users_count members.count end @@ -925,6 +894,10 @@ class Group < Namespace feature_flag_enabled_for_self_or_ancestor?(:work_items_mvc_2) end + def linked_work_items_feature_flag_enabled? + feature_flag_enabled_for_self_or_ancestor?(:linked_work_items) + end + def usage_quotas_enabled? ::Feature.enabled?(:usage_quotas_for_all_editions, self) && root? end @@ -951,7 +924,7 @@ class Group < Namespace end def update_two_factor_requirement_for_members - direct_and_indirect_members.find_each(&:update_two_factor_requirement) + hierarchy_members.find_each(&:update_two_factor_requirement) end def readme_project diff --git a/app/models/group_group_link.rb b/app/models/group_group_link.rb index dba52aa51cd..13f74b938af 100644 --- a/app/models/group_group_link.rb +++ b/app/models/group_group_link.rb @@ -13,6 +13,7 @@ class GroupGroupLink < ApplicationRecord validates :group_access, inclusion: { in: Gitlab::Access.all_values }, presence: true scope :non_guests, -> { where('group_access > ?', Gitlab::Access::GUEST) } + scope :for_shared_groups, -> (group_ids) { where(shared_group_id: group_ids) } scope :with_owner_or_maintainer_access, -> do where(group_access: [Gitlab::Access::OWNER, Gitlab::Access::MAINTAINER]) diff --git a/app/models/identity.rb b/app/models/identity.rb index df1185f330d..a4c59694050 100644 --- a/app/models/identity.rb +++ b/app/models/identity.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Identity < ApplicationRecord +class Identity < MainClusterwide::ApplicationRecord include Sortable include CaseSensitivity diff --git a/app/models/identity/uniqueness_scopes.rb b/app/models/identity/uniqueness_scopes.rb index b41b4572e82..598b7e34738 100644 --- a/app/models/identity/uniqueness_scopes.rb +++ b/app/models/identity/uniqueness_scopes.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Identity < ApplicationRecord +class Identity < MainClusterwide::ApplicationRecord # This module and method are defined in a separate file to allow EE to # redefine the `scopes` method before it is used in the `Identity` model. module UniquenessScopes diff --git a/app/models/instance_configuration.rb b/app/models/instance_configuration.rb index 64c9680ce90..57638356362 100644 --- a/app/models/instance_configuration.rb +++ b/app/models/instance_configuration.rb @@ -53,7 +53,9 @@ class InstanceConfiguration diff_max_patch_bytes: application_settings[:diff_max_patch_bytes].bytes, max_artifacts_size: application_settings[:max_artifacts_size].megabytes, max_pages_size: application_settings[:max_pages_size] > 0 ? application_settings[:max_pages_size].megabytes : nil, - snippet_size_limit: application_settings[:snippet_size_limit]&.bytes + snippet_size_limit: application_settings[:snippet_size_limit]&.bytes, + max_import_remote_file_size: application_settings[:max_import_remote_file_size] > 0 ? application_settings[:max_import_remote_file_size].megabytes : 0, + bulk_import_max_download_file_size: application_settings[:bulk_import_max_download_file_size] > 0 ? application_settings[:bulk_import_max_download_file_size].megabytes : 0 } end diff --git a/app/models/integration.rb b/app/models/integration.rb index f823a385022..bc86b08018f 100644 --- a/app/models/integration.rb +++ b/app/models/integration.rb @@ -167,7 +167,7 @@ class Integration < ApplicationRecord raise ArgumentError, "Unknown field storage: #{storage}" end - boolean_accessor(name) if attrs[:type] == 'checkbox' && storage != :attribute + boolean_accessor(name) if attrs[:type] == :checkbox && storage != :attribute end # :nocov: @@ -472,7 +472,7 @@ class Integration < ApplicationRecord # use `#secret?` here. # See: https://gitlab.com/groups/gitlab-org/-/epics/7652 def secret_fields - fields.select { |f| f[:type] == 'password' }.pluck(:name) + fields.select { |f| f[:type] == :password }.pluck(:name) end # Expose a list of fields in the JSON endpoint. @@ -517,11 +517,11 @@ class Integration < ApplicationRecord end def api_field_names - fields.reject { _1[:type] == 'password' || _1[:name] == 'webhook' }.pluck(:name) + fields.reject { _1[:type] == :password || _1[:name] == 'webhook' || (_1.key?(:if) && _1[:if] != true) }.pluck(:name) end def form_fields - fields.reject { _1[:api_only] == true } + fields.reject { _1[:api_only] == true || (_1.key?(:if) && _1[:if] != true) } end def configurable_events diff --git a/app/models/integrations/apple_app_store.rb b/app/models/integrations/apple_app_store.rb index a4036a82cec..6f96626718f 100644 --- a/app/models/integrations/apple_app_store.rb +++ b/app/models/integrations/apple_app_store.rb @@ -32,7 +32,7 @@ module Integrations field :app_store_private_key, api_only: true field :app_store_protected_refs, - type: 'checkbox', + type: :checkbox, section: SECTION_TYPE_CONFIGURATION, title: -> { s_('AppleAppStore|Protected branches and tags only') }, checkbox_label: -> { s_('AppleAppStore|Only set variables on protected branches and tags') } diff --git a/app/models/integrations/asana.rb b/app/models/integrations/asana.rb index b8cfd718007..7436c08aa38 100644 --- a/app/models/integrations/asana.rb +++ b/app/models/integrations/asana.rb @@ -7,7 +7,7 @@ module Integrations validates :api_key, presence: true, if: :activated? field :api_key, - type: 'password', + type: :password, title: 'API key', help: -> { s_('AsanaService|User Personal Access Token. User must have access to the task. All comments are attributed to this user.') }, non_empty_password_title: -> { s_('ProjectService|Enter new API key') }, diff --git a/app/models/integrations/assembla.rb b/app/models/integrations/assembla.rb index 536d5584bf6..6831fac32e6 100644 --- a/app/models/integrations/assembla.rb +++ b/app/models/integrations/assembla.rb @@ -5,7 +5,7 @@ module Integrations validates :token, presence: true, if: :activated? field :token, - type: 'password', + type: :password, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, placeholder: '', diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index 4638ca0c5f1..0b8432136dd 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -24,7 +24,7 @@ module Integrations help: -> { s_('BambooService|The user with API access to the Bamboo server.') } field :password, - type: 'password', + type: :password, non_empty_password_title: -> { s_('ProjectService|Enter new password') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current password') } diff --git a/app/models/integrations/base_chat_notification.rb b/app/models/integrations/base_chat_notification.rb index c9de4d2b3bb..4d207574ca7 100644 --- a/app/models/integrations/base_chat_notification.rb +++ b/app/models/integrations/base_chat_notification.rb @@ -23,7 +23,6 @@ module Integrations ].freeze SECRET_MASK = '************' - CHANNEL_LIMIT_PER_EVENT = 10 attribute :category, default: 'chat' @@ -79,27 +78,27 @@ module Integrations def default_fields [ { - type: 'checkbox', + type: :checkbox, section: SECTION_TYPE_CONFIGURATION, name: 'notify_only_broken_pipelines', help: 'Do not send notifications for successful pipelines.' }.freeze, { - type: 'select', + type: :select, section: SECTION_TYPE_CONFIGURATION, name: 'branches_to_be_notified', title: s_('Integrations|Branches for which notifications are to be sent'), choices: self.class.branch_choices }.freeze, { - type: 'text', + type: :text, section: SECTION_TYPE_CONFIGURATION, name: 'labels_to_be_notified', placeholder: '~backend,~frontend', help: 'Send notifications for issue, merge request, and comment events with the listed labels only. Leave blank to receive notifications for all events.' }.freeze, { - type: 'select', + type: :select, section: SECTION_TYPE_CONFIGURATION, name: 'labels_to_be_notified_behavior', choices: [ @@ -111,8 +110,8 @@ module Integrations next unless requires_webhook? fields.unshift( - { type: 'text', name: 'webhook', help: webhook_help, required: true }.freeze, - { type: 'text', name: 'username', placeholder: 'GitLab-integration' }.freeze + { type: :text, name: 'webhook', help: webhook_help, required: true }.freeze, + { type: :text, name: 'username', placeholder: 'GitLab-integration' }.freeze ) end.freeze end @@ -186,6 +185,14 @@ module Integrations true end + def channel_limit_per_event + 10 + end + + def mask_configurable_channels? + false + end + private def should_execute?(object_kind) @@ -257,7 +264,7 @@ module Integrations def build_event_channels event_channel_names.map do |channel_field| - { type: 'text', name: channel_field, placeholder: default_channel_placeholder } + { type: :text, name: channel_field, placeholder: default_channel_placeholder } end end @@ -314,13 +321,13 @@ module Integrations def validate_channel_limit supported_events.each do |event| count = channels_for_event(event).count - next unless count > CHANNEL_LIMIT_PER_EVENT + next unless count > channel_limit_per_event errors.add( event_channel_name(event).to_sym, format( s_('SlackIntegration|cannot have more than %{limit} channels'), - limit: CHANNEL_LIMIT_PER_EVENT + limit: channel_limit_per_event ) ) end diff --git a/app/models/integrations/buildkite.rb b/app/models/integrations/buildkite.rb index 5c08eac8557..6cd36e545a5 100644 --- a/app/models/integrations/buildkite.rb +++ b/app/models/integrations/buildkite.rb @@ -16,7 +16,7 @@ module Integrations required: true field :token, - type: 'password', + type: :password, title: -> { _('Token') }, help: -> do s_('ProjectService|The token you get after you create a Buildkite pipeline with a GitLab repository.') diff --git a/app/models/integrations/campfire.rb b/app/models/integrations/campfire.rb index 9b837faf79b..007578e5830 100644 --- a/app/models/integrations/campfire.rb +++ b/app/models/integrations/campfire.rb @@ -13,7 +13,7 @@ module Integrations format: { with: SUBDOMAIN_REGEXP }, length: { in: 1..63 } field :token, - type: 'password', + type: :password, title: -> { _('Campfire token') }, help: -> { s_('CampfireService|API authentication token from Campfire.') }, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, diff --git a/app/models/integrations/chat_message/issue_message.rb b/app/models/integrations/chat_message/issue_message.rb index 1c234630370..dd516362491 100644 --- a/app/models/integrations/chat_message/issue_message.rb +++ b/app/models/integrations/chat_message/issue_message.rb @@ -27,7 +27,7 @@ module Integrations def attachments return [] unless opened_issue? - return description if markdown + return SlackMarkdownSanitizer.sanitize_slack_link(description) if markdown description_message end @@ -55,7 +55,7 @@ module Integrations [{ title: issue_title, title_link: issue_url, - text: format(description), + text: format(SlackMarkdownSanitizer.sanitize_slack_link(description)), color: '#C95823' }] end diff --git a/app/models/integrations/datadog.rb b/app/models/integrations/datadog.rb index c7306209174..1a56763fe57 100644 --- a/app/models/integrations/datadog.rb +++ b/app/models/integrations/datadog.rb @@ -32,7 +32,7 @@ module Integrations help: -> { s_('DatadogIntegration|(Advanced) The full URL for your Datadog site.') } field :api_key, - type: 'password', + type: :password, title: -> { _('API key') }, non_empty_password_title: -> { s_('ProjectService|Enter new API key') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current API key') }, @@ -48,7 +48,7 @@ module Integrations field :archive_trace_events, storage: :attribute, - type: 'checkbox', + type: :checkbox, title: -> { _('Logs') }, checkbox_label: -> { _('Enable logs collection') }, help: -> { s_('When enabled, job logs are collected by Datadog and displayed along with pipeline execution traces.') } @@ -73,7 +73,7 @@ module Integrations end field :datadog_tags, - type: 'textarea', + type: :textarea, title: -> { s_('DatadogIntegration|Tags') }, placeholder: "tag:value\nanother_tag:value", help: -> do diff --git a/app/models/integrations/discord.rb b/app/models/integrations/discord.rb index 061c491034d..7cae3ca20f9 100644 --- a/app/models/integrations/discord.rb +++ b/app/models/integrations/discord.rb @@ -10,15 +10,15 @@ module Integrations field :webhook, section: SECTION_TYPE_CONNECTION, - help: 'e.g. https://discordapp.com/api/webhooks/…', + help: 'e.g. https://discord.com/api/webhooks/…', required: true field :notify_only_broken_pipelines, - type: 'checkbox', + type: :checkbox, section: SECTION_TYPE_CONFIGURATION field :branches_to_be_notified, - type: 'select', + type: :select, section: SECTION_TYPE_CONFIGURATION, title: -> { s_('Integrations|Branches for which notifications are to be sent') }, choices: -> { branch_choices } @@ -45,7 +45,7 @@ module Integrations end def default_channel_placeholder - # No-op. + s_('DiscordService|Override the default webhook (e.g. https://discord.com/api/webhooks/…)') end def self.supported_events @@ -72,10 +72,23 @@ module Integrations ] end + def configurable_channels? + true + end + + def channel_limit_per_event + 1 + end + + def mask_configurable_channels? + true + end + private def notify(message, opts) - client = Discordrb::Webhooks::Client.new(url: webhook) + webhook_url = opts[:channel]&.first || webhook + client = Discordrb::Webhooks::Client.new(url: webhook_url) client.execute do |builder| builder.add_embed do |embed| diff --git a/app/models/integrations/drone_ci.rb b/app/models/integrations/drone_ci.rb index 781acf65c47..ac464c020dd 100644 --- a/app/models/integrations/drone_ci.rb +++ b/app/models/integrations/drone_ci.rb @@ -16,7 +16,7 @@ module Integrations required: true field :token, - type: 'password', + type: :password, help: -> { s_('ProjectService|Token for the Drone project.') }, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, diff --git a/app/models/integrations/emails_on_push.rb b/app/models/integrations/emails_on_push.rb index 25bda8c2bf0..eb893ae45d0 100644 --- a/app/models/integrations/emails_on_push.rb +++ b/app/models/integrations/emails_on_push.rb @@ -10,7 +10,7 @@ module Integrations validate :number_of_recipients_within_limit, if: :validate_recipients? field :send_from_committer_email, - type: 'checkbox', + type: :checkbox, title: -> { s_("EmailsOnPushService|Send from committer") }, help: -> do @help ||= begin @@ -21,17 +21,17 @@ module Integrations end field :disable_diffs, - type: 'checkbox', + type: :checkbox, title: -> { s_("EmailsOnPushService|Disable code diffs") }, help: -> { s_("EmailsOnPushService|Don't include possibly sensitive code diffs in notification body.") } field :branches_to_be_notified, - type: 'select', + type: :select, title: -> { s_('Integrations|Branches for which notifications are to be sent') }, choices: branch_choices field :recipients, - type: 'textarea', + type: :textarea, placeholder: -> { s_('EmailsOnPushService|tanuki@example.com gitlab@example.com') }, help: -> { s_('EmailsOnPushService|Emails separated by whitespace.') } diff --git a/app/models/integrations/field.rb b/app/models/integrations/field.rb index 9f2274216f6..9dc90629344 100644 --- a/app/models/integrations/field.rb +++ b/app/models/integrations/field.rb @@ -6,21 +6,24 @@ module Integrations ATTRIBUTES = %i[ section type placeholder choices value checkbox_label - title help + title help if non_empty_password_help non_empty_password_title ].concat(BOOLEAN_ATTRIBUTES).freeze - TYPES = %w[text textarea password checkbox select].freeze + TYPES = %i[text textarea password checkbox select].freeze attr_reader :name, :integration_class - def initialize(name:, integration_class:, type: 'text', is_secret: false, api_only: false, **attributes) + delegate :key?, to: :attributes + + def initialize(name:, integration_class:, type: :text, is_secret: false, api_only: false, **attributes) @name = name.to_s.freeze @integration_class = integration_class - attributes[:type] = is_secret ? 'password' : type + attributes[:type] = is_secret ? :password : type attributes[:api_only] = api_only + attributes[:if] = attributes.fetch(:if, true) attributes[:is_secret] = is_secret @attributes = attributes.freeze @@ -35,14 +38,14 @@ module Integrations def [](key) return name if key == :name - value = @attributes[key] + value = attributes[key] return integration_class.class_exec(&value) if value.respond_to?(:call) value end def secret? - self[:type] == 'password' + self[:type] == :password end ATTRIBUTES.each do |name| @@ -56,5 +59,9 @@ module Integrations TYPES.each do |type| define_method("#{type}?") { self[:type] == type } end + + private + + attr_reader :attributes end end diff --git a/app/models/integrations/google_play.rb b/app/models/integrations/google_play.rb index 9fa6dc19f11..5389e8dfa81 100644 --- a/app/models/integrations/google_play.rb +++ b/app/models/integrations/google_play.rb @@ -12,6 +12,7 @@ module Integrations } validates :service_account_key_file_name, presence: true validates :package_name, presence: true, format: { with: PACKAGE_NAME_REGEX } + validates :google_play_protected_refs, inclusion: [true, false] end field :package_name, @@ -25,6 +26,12 @@ module Integrations field :service_account_key, api_only: true + field :google_play_protected_refs, + type: :checkbox, + section: SECTION_TYPE_CONFIGURATION, + title: -> { s_('GooglePlayStore|Protected branches and tags only') }, + checkbox_label: -> { s_('GooglePlayStore|Only set variables on protected branches and tags') } + def title s_('GooglePlay|Google Play') end @@ -76,8 +83,9 @@ module Integrations { success: false, message: error } end - def ci_variables + def ci_variables(protected_ref:) return [] unless activated? + return [] if google_play_protected_refs && !protected_ref [ { key: 'SUPPLY_JSON_KEY_DATA', value: service_account_key, masked: true, public: false }, @@ -85,6 +93,11 @@ module Integrations ] end + def initialize_properties + super + self.google_play_protected_refs = true if google_play_protected_refs.nil? + end + private def client diff --git a/app/models/integrations/hangouts_chat.rb b/app/models/integrations/hangouts_chat.rb index 7ba9bbc38e6..037c689c75e 100644 --- a/app/models/integrations/hangouts_chat.rb +++ b/app/models/integrations/hangouts_chat.rb @@ -10,11 +10,11 @@ module Integrations required: true field :notify_only_broken_pipelines, - type: 'checkbox', + type: :checkbox, section: SECTION_TYPE_CONFIGURATION field :branches_to_be_notified, - type: 'select', + type: :select, section: SECTION_TYPE_CONFIGURATION, title: -> { s_('Integrations|Branches for which notifications are to be sent') }, choices: -> { branch_choices } diff --git a/app/models/integrations/harbor.rb b/app/models/integrations/harbor.rb index 079811e0df0..559e48afd10 100644 --- a/app/models/integrations/harbor.rb +++ b/app/models/integrations/harbor.rb @@ -25,7 +25,7 @@ module Integrations required: true field :password, - type: 'password', + type: :password, title: -> { s_('HarborIntegration|Harbor password') }, help: -> { s_('HarborIntegration|Password for your Harbor username.') }, non_empty_password_title: -> { s_('HarborIntegration|Enter new Harbor password') }, diff --git a/app/models/integrations/irker.rb b/app/models/integrations/irker.rb index 3f3e321f45e..a54946f074a 100644 --- a/app/models/integrations/irker.rb +++ b/app/models/integrations/irker.rb @@ -23,7 +23,7 @@ module Integrations placeholder: 'irc://irc.network.net:6697/' field :recipients, - type: 'textarea', + type: :textarea, title: -> { s_('IrkerService|Recipients') }, placeholder: 'irc[s]://irc.network.net[:port]/#channel', required: true, @@ -45,7 +45,7 @@ module Integrations end field :colorize_messages, - type: 'checkbox', + type: :checkbox, title: -> { _('Colorize messages') } # NOTE: This field is only used internally to store the parsed diff --git a/app/models/integrations/jenkins.rb b/app/models/integrations/jenkins.rb index d2e8393ef95..7769ea7d2dd 100644 --- a/app/models/integrations/jenkins.rb +++ b/app/models/integrations/jenkins.rb @@ -22,7 +22,7 @@ module Integrations help: -> { s_('The username for the Jenkins server.') } field :password, - type: 'password', + type: :password, help: -> { s_('The password for the Jenkins server.') }, non_empty_password_title: -> { s_('ProjectService|Enter new password.') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current password.') } diff --git a/app/models/integrations/jira.rb b/app/models/integrations/jira.rb index 4e0c2dde13b..faf0a378a17 100644 --- a/app/models/integrations/jira.rb +++ b/app/models/integrations/jira.rb @@ -74,7 +74,7 @@ module Integrations exposes_secrets: true field :jira_auth_type, - type: 'select', + type: :select, required: true, section: SECTION_TYPE_CONNECTION, title: -> { s_('JiraService|Authentication type') }, diff --git a/app/models/integrations/mattermost_slash_commands.rb b/app/models/integrations/mattermost_slash_commands.rb index e075400d9b5..73cddd163e0 100644 --- a/app/models/integrations/mattermost_slash_commands.rb +++ b/app/models/integrations/mattermost_slash_commands.rb @@ -5,7 +5,7 @@ module Integrations include Ci::TriggersHelper field :token, - type: 'password', + type: :password, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, placeholder: '' diff --git a/app/models/integrations/microsoft_teams.rb b/app/models/integrations/microsoft_teams.rb index a9ed0bd3da1..25308948d51 100644 --- a/app/models/integrations/microsoft_teams.rb +++ b/app/models/integrations/microsoft_teams.rb @@ -10,12 +10,12 @@ module Integrations required: true field :notify_only_broken_pipelines, - type: 'checkbox', + type: :checkbox, section: SECTION_TYPE_CONFIGURATION, help: 'If selected, successful pipelines do not trigger a notification event.' field :branches_to_be_notified, - type: 'select', + type: :select, section: SECTION_TYPE_CONFIGURATION, title: -> { s_('Integrations|Branches for which notifications are to be sent') }, choices: -> { branch_choices } diff --git a/app/models/integrations/packagist.rb b/app/models/integrations/packagist.rb index 3973b492b6d..c9c08ec9771 100644 --- a/app/models/integrations/packagist.rb +++ b/app/models/integrations/packagist.rb @@ -11,7 +11,7 @@ module Integrations required: true field :token, - type: 'password', + type: :password, title: -> { _('Token') }, help: -> { _('Enter your Packagist token.') }, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, diff --git a/app/models/integrations/pipelines_email.rb b/app/models/integrations/pipelines_email.rb index 55a8ce0be11..fa22bd1a73c 100644 --- a/app/models/integrations/pipelines_email.rb +++ b/app/models/integrations/pipelines_email.rb @@ -10,19 +10,19 @@ module Integrations validate :number_of_recipients_within_limit, if: :validate_recipients? field :recipients, - type: 'textarea', + type: :textarea, help: -> { _('Comma-separated list of email addresses.') }, required: true field :notify_only_broken_pipelines, - type: 'checkbox' + type: :checkbox field :notify_only_default_branch, - type: 'checkbox', + type: :checkbox, api_only: true field :branches_to_be_notified, - type: 'select', + type: :select, title: -> { s_('Integrations|Branches for which notifications are to be sent') }, choices: branch_choices diff --git a/app/models/integrations/pivotaltracker.rb b/app/models/integrations/pivotaltracker.rb index 1acdbbbf9bc..0d9a3f05a86 100644 --- a/app/models/integrations/pivotaltracker.rb +++ b/app/models/integrations/pivotaltracker.rb @@ -7,7 +7,7 @@ module Integrations validates :token, presence: true, if: :activated? field :token, - type: 'password', + type: :password, help: -> { s_('PivotalTrackerService|Pivotal Tracker API token. User must have access to the story. All comments are attributed to this user.') }, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, diff --git a/app/models/integrations/prometheus.rb b/app/models/integrations/prometheus.rb index 8969c6c13b2..736318ed707 100644 --- a/app/models/integrations/prometheus.rb +++ b/app/models/integrations/prometheus.rb @@ -6,7 +6,7 @@ module Integrations include Gitlab::Utils::StrongMemoize field :manual_configuration, - type: 'checkbox', + type: :checkbox, title: -> { s_('PrometheusService|Active') }, help: -> { s_('PrometheusService|Select this checkbox to override the auto configuration settings with your own settings.') }, required: true @@ -24,7 +24,7 @@ module Integrations required: false field :google_iap_service_account_json, - type: 'textarea', + type: :textarea, title: 'Google IAP Service Account JSON', placeholder: -> { s_('PrometheusService|{ "type": "service_account", "project_id": ... }') }, help: -> { s_('PrometheusService|The contents of the credentials.json file of your service account.') }, diff --git a/app/models/integrations/pumble.rb b/app/models/integrations/pumble.rb index e08dc6d0f51..8f0dddcc5c5 100644 --- a/app/models/integrations/pumble.rb +++ b/app/models/integrations/pumble.rb @@ -2,6 +2,24 @@ module Integrations class Pumble < BaseChatNotification + undef :notify_only_broken_pipelines + + field :webhook, + section: SECTION_TYPE_CONNECTION, + help: 'https://api.pumble.com/workspaces/x/...', + required: true + + field :notify_only_broken_pipelines, + type: :checkbox, + section: SECTION_TYPE_CONFIGURATION, + help: 'If selected, successful pipelines do not trigger a notification event.' + + field :branches_to_be_notified, + type: :select, + section: SECTION_TYPE_CONFIGURATION, + title: -> { s_('Integrations|Branches for which notifications are to be sent') }, + choices: -> { branch_choices } + def title 'Pumble' end @@ -34,17 +52,8 @@ module Integrations pipeline wiki_page] end - def default_fields - [ - { type: 'text', name: 'webhook', help: 'https://api.pumble.com/workspaces/x/...', required: true }, - { type: 'checkbox', name: 'notify_only_broken_pipelines' }, - { - type: 'select', - name: 'branches_to_be_notified', - title: s_('Integrations|Branches for which notifications are to be sent'), - choices: self.class.branch_choices - } - ] + def fields + self.class.fields + build_event_channels end private diff --git a/app/models/integrations/pushover.rb b/app/models/integrations/pushover.rb index 6bb6b6d60f6..006b731c6c2 100644 --- a/app/models/integrations/pushover.rb +++ b/app/models/integrations/pushover.rb @@ -7,7 +7,7 @@ module Integrations validates :api_key, :user_key, :priority, presence: true, if: :activated? field :api_key, - type: 'password', + type: :password, title: -> { _('API key') }, help: -> { s_('PushoverService|Enter your application key.') }, non_empty_password_title: -> { s_('ProjectService|Enter new API key') }, @@ -16,7 +16,7 @@ module Integrations required: true field :user_key, - type: 'password', + type: :password, title: -> { _('User key') }, help: -> { s_('PushoverService|Enter your user key.') }, non_empty_password_title: -> { s_('PushoverService|Enter new user key') }, @@ -30,7 +30,7 @@ module Integrations placeholder: '' field :priority, - type: 'select', + type: :select, required: true, choices: -> do [ @@ -42,7 +42,7 @@ module Integrations end field :sound, - type: 'select', + type: :select, choices: -> do [ ['Device default sound', nil], diff --git a/app/models/integrations/slack_slash_commands.rb b/app/models/integrations/slack_slash_commands.rb index 343c8d68166..b209f37ee7c 100644 --- a/app/models/integrations/slack_slash_commands.rb +++ b/app/models/integrations/slack_slash_commands.rb @@ -5,7 +5,7 @@ module Integrations include Ci::TriggersHelper field :token, - type: 'password', + type: :password, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, placeholder: '' diff --git a/app/models/integrations/squash_tm.rb b/app/models/integrations/squash_tm.rb index e0a63b5ae6a..bf3f391564f 100644 --- a/app/models/integrations/squash_tm.rb +++ b/app/models/integrations/squash_tm.rb @@ -11,7 +11,7 @@ module Integrations required: true field :token, - type: 'password', + type: :password, title: -> { s_('SquashTmIntegration|Secret token (optional)') }, non_empty_password_title: -> { s_('ProjectService|Enter new token') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, diff --git a/app/models/integrations/teamcity.rb b/app/models/integrations/teamcity.rb index af629d6ef1e..c74e0aab030 100644 --- a/app/models/integrations/teamcity.rb +++ b/app/models/integrations/teamcity.rb @@ -22,7 +22,7 @@ module Integrations help: -> { s_('ProjectService|Must have permission to trigger a manual build in TeamCity.') } field :password, - type: 'password', + type: :password, non_empty_password_title: -> { s_('ProjectService|Enter new password') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current password') } diff --git a/app/models/integrations/unify_circuit.rb b/app/models/integrations/unify_circuit.rb index 6c447c8f4e4..6de693b5278 100644 --- a/app/models/integrations/unify_circuit.rb +++ b/app/models/integrations/unify_circuit.rb @@ -10,11 +10,11 @@ module Integrations required: true field :notify_only_broken_pipelines, - type: 'checkbox', + type: :checkbox, section: SECTION_TYPE_CONFIGURATION field :branches_to_be_notified, - type: 'select', + type: :select, section: SECTION_TYPE_CONFIGURATION, title: -> { s_('Integrations|Branches for which notifications are to be sent') }, choices: -> { branch_choices } diff --git a/app/models/integrations/webex_teams.rb b/app/models/integrations/webex_teams.rb index ef1bc81ea58..21c65cc2b32 100644 --- a/app/models/integrations/webex_teams.rb +++ b/app/models/integrations/webex_teams.rb @@ -10,11 +10,11 @@ module Integrations required: true field :notify_only_broken_pipelines, - type: 'checkbox', + type: :checkbox, section: SECTION_TYPE_CONFIGURATION field :branches_to_be_notified, - type: 'select', + type: :select, section: SECTION_TYPE_CONFIGURATION, title: -> { s_('Integrations|Branches for which notifications are to be sent') }, choices: -> { branch_choices } diff --git a/app/models/integrations/zentao.rb b/app/models/integrations/zentao.rb index 459756c865b..fd2c741bd6b 100644 --- a/app/models/integrations/zentao.rb +++ b/app/models/integrations/zentao.rb @@ -19,7 +19,7 @@ module Integrations exposes_secrets: true field :api_token, - type: 'password', + type: :password, title: -> { s_('ZentaoIntegration|ZenTao API token') }, non_empty_password_title: -> { s_('ZentaoIntegration|Enter new ZenTao API token') }, non_empty_password_help: -> { s_('ProjectService|Leave blank to use your current token.') }, diff --git a/app/models/issue.rb b/app/models/issue.rb index 6e48dcab9ed..d227448961a 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -317,6 +317,10 @@ class Issue < ApplicationRecord pattern: IssuableFinder::FULL_TEXT_SEARCH_TERM_PATTERN ) end + + def related_link_class + IssueLink + end end def self.participant_includes @@ -542,18 +546,18 @@ class Issue < ApplicationRecord end def related_issues(current_user, preload: nil) - related_issues = ::Issue - .select(['issues.*', 'issue_links.id AS issue_link_id', - 'issue_links.link_type as issue_link_type_value', - 'issue_links.target_id as issue_link_source_id', - 'issue_links.created_at as issue_link_created_at', - 'issue_links.updated_at as issue_link_updated_at']) - .joins("INNER JOIN issue_links ON - (issue_links.source_id = issues.id AND issue_links.target_id = #{id}) - OR - (issue_links.target_id = issues.id AND issue_links.source_id = #{id})") - .preload(preload) - .reorder('issue_link_id') + related_issues = self.class + .select(['issues.*', 'issue_links.id AS issue_link_id', + 'issue_links.link_type as issue_link_type_value', + 'issue_links.target_id as issue_link_source_id', + 'issue_links.created_at as issue_link_created_at', + 'issue_links.updated_at as issue_link_updated_at']) + .joins("INNER JOIN issue_links ON + (issue_links.source_id = issues.id AND issue_links.target_id = #{id}) + OR + (issue_links.target_id = issues.id AND issue_links.source_id = #{id})") + .preload(preload) + .reorder('issue_link_id') related_issues = yield related_issues if block_given? @@ -642,12 +646,13 @@ class Issue < ApplicationRecord end def issue_link_type + link_class = self.class.related_link_class return unless respond_to?(:issue_link_type_value) && respond_to?(:issue_link_source_id) - type = IssueLink.link_types.key(issue_link_type_value) || IssueLink::TYPE_RELATES_TO + type = link_class.link_types.key(issue_link_type_value) || link_class::TYPE_RELATES_TO return type if issue_link_source_id == id - IssueLink.inverse_link_type(type) + link_class.inverse_link_type(type) end def relocation_target @@ -770,7 +775,7 @@ class Issue < ApplicationRecord return unless persisted? if confidential? && WorkItems::ParentLink.has_public_children?(id) - errors.add(:base, _('A confidential issue cannot have a parent that already has non-confidential children.')) + errors.add(:base, _('A confidential issue must have only confidential children. Make any child items confidential and try again.')) end if !confidential? && WorkItems::ParentLink.has_confidential_parent?(id) @@ -784,7 +789,7 @@ class Issue < ApplicationRecord # TODO: https://gitlab.com/gitlab-org/gitlab/-/work_items/393126 return unless project - Issues::SearchData.upsert({ project_id: project_id, issue_id: id, search_vector: search_vector }, unique_by: %i(project_id issue_id)) + Issues::SearchData.upsert({ namespace_id: namespace_id, project_id: project_id, issue_id: id, search_vector: search_vector }, unique_by: %i(project_id issue_id)) end def ensure_metrics! diff --git a/app/models/issue_link.rb b/app/models/issue_link.rb index af55a5dec91..1c596ad0341 100644 --- a/app/models/issue_link.rb +++ b/app/models/issue_link.rb @@ -1,18 +1,11 @@ # frozen_string_literal: true class IssueLink < ApplicationRecord - include FromUnion - include IssuableLink + include LinkableItem belongs_to :source, class_name: 'Issue' belongs_to :target, class_name: 'Issue' - scope :for_source_issue, ->(issue) { where(source_id: issue.id) } - scope :for_target_issue, ->(issue) { where(target_id: issue.id) } - scope :for_issues, ->(source, target) do - where(source: source, target: target).or(where(source: target, target: source)) - end - class << self def issuable_type :issue diff --git a/app/models/label.rb b/app/models/label.rb index 0831ba40536..d0d278b68fd 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -25,8 +25,10 @@ class Label < ApplicationRecord has_many :merge_requests, through: :label_links, source: :target, source_type: 'MergeRequest' before_validation :strip_whitespace_from_title + before_destroy :prevent_locked_label_destroy, prepend: true validates :color, color: true, presence: true + validate :ensure_lock_on_merge_allowed # Don't allow ',' for label titles validates :title, presence: true, format: { with: /\A[^,]+\z/ } @@ -42,6 +44,7 @@ class Label < ApplicationRecord scope :templates, -> { where(template: true, type: [Label.name, nil]) } scope :with_title, ->(title) { where(title: title) } scope :with_lists_and_board, -> { joins(lists: :board).merge(List.movable) } + scope :with_lock_on_merge, -> { where(lock_on_merge: true) } scope :on_project_boards, ->(project_id) { with_lists_and_board.where(boards: { project_id: project_id }) } scope :on_board, ->(board_id) { with_lists_and_board.where(boards: { id: board_id }) } scope :order_name_asc, -> { reorder(title: :asc) } @@ -319,6 +322,20 @@ class Label < ApplicationRecord def strip_whitespace_from_title self[:title] = title&.strip end + + def prevent_locked_label_destroy + return unless lock_on_merge + + errors.add(:base, format(_('%{label_name} is locked and was not removed'), label_name: name)) + throw :abort # rubocop:disable Cop/BanCatchThrow + end + + def ensure_lock_on_merge_allowed + return unless template? + return unless lock_on_merge || will_save_change_to_lock_on_merge? + + errors.add(:lock_on_merge, _('can not be set for template labels')) + end end Label.prepend_mod_with('Label') diff --git a/app/models/loose_foreign_keys/deleted_record.rb b/app/models/loose_foreign_keys/deleted_record.rb index 7f64606e97b..1d26c3c11e4 100644 --- a/app/models/loose_foreign_keys/deleted_record.rb +++ b/app/models/loose_foreign_keys/deleted_record.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel + include FromUnion + PARTITION_DURATION = 1.day include PartitionedTable @@ -34,13 +36,34 @@ class LooseForeignKeys::DeletedRecord < Gitlab::Database::SharedModel enum status: { pending: 1, processed: 2 }, _prefix: :status def self.load_batch_for_table(table, batch_size) - # selecting partition as partition_number to workaround the sliding partitioning column ignore - select(arel_table[Arel.star], arel_table[:partition].as('partition_number')) - .for_table(table) - .status_pending - .consume_order - .limit(batch_size) - .to_a + if Feature.enabled?("loose_foreign_keys_batch_load_using_union") + partition_names = Gitlab::Database::PostgresPartitionedTable.each_partition(table_name).map(&:name) + + unions = partition_names.map do |partition_name| + partition_number = partition_name[/\d+/].to_i + + select(arel_table[Arel.star], arel_table[:partition].as('partition_number')) + .from("#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partition_name} AS #{table_name}") + .for_table(table) + .where(partition: partition_number) + .status_pending + .consume_order + .limit(batch_size) + end + + select(arel_table[Arel.star]) + .from_union(unions, remove_duplicates: false, remove_order: false) + .limit(batch_size) + .to_a + else + # selecting partition as partition_number to workaround the sliding partitioning column ignore + select(arel_table[Arel.star], arel_table[:partition].as('partition_number')) + .for_table(table) + .status_pending + .consume_order + .limit(batch_size) + .to_a + end end def self.mark_records_processed(records) diff --git a/app/models/member.rb b/app/models/member.rb index f164ea244b4..cdf40eaa8f5 100644 --- a/app/models/member.rb +++ b/app/models/member.rb @@ -153,6 +153,7 @@ class Member < ApplicationRecord scope :not_accepted_invitations, -> { invite.where(invite_accepted_at: nil) } scope :not_accepted_invitations_by_user, -> (user) { not_accepted_invitations.where(created_by: user) } scope :not_expired, -> (today = Date.current) { where(arel_table[:expires_at].gt(today).or(arel_table[:expires_at].eq(nil))) } + scope :expiring_and_not_notified, ->(date) { where("expiry_notified_at is null AND expires_at >= ? AND expires_at <= ?", Date.current, date) } scope :created_today, -> do now = Date.current diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index 2773569161d..469dba42952 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -656,8 +656,8 @@ class MergeRequest < ApplicationRecord [:assignees, :reviewers] + super end - def committers - @committers ||= commits.committers + def committers(with_merge_commits: false) + @committers ||= commits.committers(with_merge_commits: with_merge_commits) end # Verifies if title has changed not taking into account Draft prefix @@ -984,6 +984,18 @@ class MergeRequest < ApplicationRecord branch_merge_base_commit.try(:sha) end + def existing_mrs_targeting_same_branch + similar_mrs = target_project + .merge_requests + .where(source_branch: source_branch, target_branch: target_branch) + .where(source_project: source_project) + .opened + + similar_mrs = similar_mrs.id_not_in(id) if persisted? + + similar_mrs + end + def validate_branches return unless target_project && source_project @@ -995,25 +1007,24 @@ class MergeRequest < ApplicationRecord [:source_branch, :target_branch].each { |attr| validate_branch_name(attr) } if opened? - similar_mrs = target_project - .merge_requests - .where(source_branch: source_branch, target_branch: target_branch) - .where(source_project_id: source_project&.id) - .opened + conflicting_mr = existing_mrs_targeting_same_branch.first - similar_mrs = similar_mrs.where.not(id: id) if persisted? - - conflict = similar_mrs.first - - if conflict.present? + if conflicting_mr errors.add( :validate_branches, - "Another open merge request already exists for this source branch: #{conflict.to_reference}" + conflicting_mr_message(conflicting_mr) ) end end end + def conflicting_mr_message(conflicting_mr) + format( + _("Another open merge request already exists for this source branch: %{conflicting_mr_reference}"), + conflicting_mr_reference: conflicting_mr.to_reference + ) + end + def validate_branch_name(attr) return unless will_save_change_to_attribute?(attr) @@ -1155,7 +1166,7 @@ class MergeRequest < ApplicationRecord MergeRequests::ReloadDiffsService.new(self, current_user).execute end - def check_mergeability(async: false) + def check_mergeability(async: false, sync_retry_lease: false) return unless recheck_merge_status? check_service = MergeRequests::MergeabilityCheckService.new(self) @@ -1163,7 +1174,7 @@ class MergeRequest < ApplicationRecord if async check_service.async_execute else - check_service.execute(retry_lease: false) + check_service.execute(retry_lease: sync_retry_lease) end end # rubocop: enable CodeReuse/ServiceClass @@ -1207,14 +1218,14 @@ class MergeRequest < ApplicationRecord } end - def mergeable?(skip_ci_check: false, skip_discussions_check: false, skip_approved_check: false) + def mergeable?(skip_ci_check: false, skip_discussions_check: false, skip_approved_check: false, check_mergeability_retry_lease: false) return false unless mergeable_state?( skip_ci_check: skip_ci_check, skip_discussions_check: skip_discussions_check, skip_approved_check: skip_approved_check ) - check_mergeability + check_mergeability(sync_retry_lease: check_mergeability_retry_lease) can_be_merged? && !should_be_rebased? end @@ -1537,20 +1548,29 @@ class MergeRequest < ApplicationRecord end def schedule_cleanup_refs(only: :all) - if Feature.enabled?(:merge_request_cleanup_ref_worker_async, target_project) + if Feature.enabled?(:merge_request_delete_gitaly_refs_in_batches, target_project) + async_cleanup_refs(only: only) + elsif Feature.enabled?(:merge_request_cleanup_ref_worker_async, target_project) MergeRequests::CleanupRefWorker.perform_async(id, only.to_s) else cleanup_refs(only: only) end end - def cleanup_refs(only: :all) + def refs_to_cleanup(only: :all) target_refs = [] target_refs << ref_path if %i[all head].include?(only) target_refs << merge_ref_path if %i[all merge].include?(only) target_refs << train_ref_path if %i[all train].include?(only) + target_refs + end + + def cleanup_refs(only: :all) + project.repository.delete_refs(*refs_to_cleanup(only: only)) + end - project.repository.delete_refs(*target_refs) + def async_cleanup_refs(only: :all) + project.repository.async_delete_refs(*refs_to_cleanup(only: only)) end def self.merge_request_ref?(ref) diff --git a/app/models/merge_request/metrics.rb b/app/models/merge_request/metrics.rb index a13cb353c7b..3c592c0008f 100644 --- a/app/models/merge_request/metrics.rb +++ b/app/models/merge_request/metrics.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true class MergeRequest::Metrics < ApplicationRecord - include IgnorableColumns include DatabaseEventTracking belongs_to :merge_request, inverse_of: :metrics @@ -17,8 +16,6 @@ class MergeRequest::Metrics < ApplicationRecord scope :with_valid_time_to_merge, -> { where(arel_table[:merged_at].gt(arel_table[:created_at])) } scope :by_target_project, ->(project) { where(target_project_id: project) } - ignore_column :id_convert_to_bigint, remove_with: '16.0', remove_after: '2023-05-22' - class << self def time_to_merge_expression Arel.sql('EXTRACT(epoch FROM SUM(AGE(merge_request_metrics.merged_at, merge_request_metrics.created_at)))') diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 33930836c48..bddc03d8b21 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -7,6 +7,7 @@ class MergeRequestDiff < ApplicationRecord include EachBatch include Gitlab::Utils::StrongMemoize include BulkInsertableAssociations + include ShaAttribute # Don't display more than 100 commits at once COMMITS_SAFE_SIZE = 100 @@ -34,6 +35,8 @@ class MergeRequestDiff < ApplicationRecord has_many :merge_request_diff_commits, -> { order(:merge_request_diff_id, :relative_order) }, inverse_of: :merge_request_diff + sha_attribute :patch_id_sha + validates :base_commit_sha, :head_commit_sha, :start_commit_sha, sha: true validates :merge_request_id, uniqueness: { scope: :diff_type }, if: :merge_head? diff --git a/app/models/metrics/dashboard/annotation.rb b/app/models/metrics/dashboard/annotation.rb index d3d3f973398..ac0fcb41089 100644 --- a/app/models/metrics/dashboard/annotation.rb +++ b/app/models/metrics/dashboard/annotation.rb @@ -7,15 +7,10 @@ module Metrics self.table_name = 'metrics_dashboard_annotations' - belongs_to :environment, inverse_of: :metrics_dashboard_annotations - belongs_to :cluster, class_name: 'Clusters::Cluster', inverse_of: :metrics_dashboard_annotations - validates :starting_at, presence: true validates :description, presence: true, length: { maximum: 255 } validates :dashboard_path, presence: true, length: { maximum: 255 } validates :panel_xid, length: { maximum: 255 } - validate :single_ownership - validate :orphaned_annotation validate :ending_at_after_starting_at scope :after, ->(after) { where('starting_at >= ?', after) } @@ -34,18 +29,6 @@ module Metrics errors.add(:ending_at, s_("MetricsDashboardAnnotation|can't be before starting_at time")) end - - def single_ownership - return if cluster.nil? ^ environment.nil? - - errors.add(:base, s_("MetricsDashboardAnnotation|Annotation can't belong to both a cluster and an environment at the same time")) - end - - def orphaned_annotation - return if cluster.present? || environment.present? - - errors.add(:base, s_("MetricsDashboardAnnotation|Annotation must belong to a cluster or an environment")) - end end end end diff --git a/app/models/ml/experiment.rb b/app/models/ml/experiment.rb index 5c5f8d3b2db..ad6c6b7b3bf 100644 --- a/app/models/ml/experiment.rb +++ b/app/models/ml/experiment.rb @@ -59,6 +59,10 @@ module Ml numeric?(iid) end + def find_or_create(project, name, user) + create_with(user: user).find_or_create_by(project: project, name: name) + end + private def numeric?(value) diff --git a/app/models/ml/model.rb b/app/models/ml/model.rb index 684b8e1983b..fb15b9fea72 100644 --- a/app/models/ml/model.rb +++ b/app/models/ml/model.rb @@ -2,6 +2,8 @@ module Ml class Model < ApplicationRecord + include Presentable + validates :project, :default_experiment, presence: true validates :name, format: Gitlab::Regex.ml_model_name_regex, @@ -14,6 +16,10 @@ module Ml has_one :default_experiment, class_name: 'Ml::Experiment' belongs_to :project has_many :versions, class_name: 'Ml::ModelVersion' + has_one :latest_version, -> { latest_by_model }, class_name: 'Ml::ModelVersion', inverse_of: :model + + scope :including_latest_version, -> { includes(:latest_version) } + scope :by_project, ->(project) { where(project_id: project.id) } def valid_default_experiment? return unless default_experiment @@ -21,5 +27,10 @@ module Ml errors.add(:default_experiment) unless default_experiment.name == name errors.add(:default_experiment) unless default_experiment.project_id == project_id end + + def self.find_or_create(project, name, experiment) + create_with(default_experiment: experiment) + .find_or_create_by(project: project, name: name) + end end end diff --git a/app/models/ml/model_version.rb b/app/models/ml/model_version.rb index 540fe6018a1..6d0e7c35865 100644 --- a/app/models/ml/model_version.rb +++ b/app/models/ml/model_version.rb @@ -5,7 +5,7 @@ module Ml validates :project, :model, presence: true validates :version, - format: Gitlab::Regex.ml_model_version_regex, + format: Gitlab::Regex.semver_regex, uniqueness: { scope: [:project, :model_id] }, presence: true, length: { maximum: 255 } @@ -18,6 +18,15 @@ module Ml delegate :name, to: :model + scope :order_by_model_id_id_desc, -> { order('model_id, id DESC') } + scope :latest_by_model, -> { order_by_model_id_id_desc.select('DISTINCT ON (model_id) *') } + + class << self + def find_or_create!(model, version, package) + create_with(package: package).find_or_create_by!(project: model.project, model: model, version: version) + end + end + private def valid_model? diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 5449f006a2e..a7d03c3688a 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -137,6 +137,7 @@ class Namespace < ApplicationRecord :pypi_package_requests_forwarding, :npm_package_requests_forwarding, to: :package_settings + delegate :default_branch_protection_defaults, to: :namespace_settings, allow_nil: true before_save :update_new_emails_created_column, if: -> { emails_disabled_changed? } before_create :sync_share_with_group_lock_with_parent @@ -234,6 +235,7 @@ class Namespace < ApplicationRecord if include_parents without_project_namespaces .where(id: Route.for_routable_type(Namespace.name) + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") .fuzzy_search(query, [Route.arel_table[:path], Route.arel_table[:name]], use_minimum_char_limit: use_minimum_char_limit) .select(:source_id)) @@ -543,8 +545,8 @@ class Namespace < ApplicationRecord def changing_allow_descendants_override_disabled_shared_runners_is_allowed return unless new_record? || changes.has_key?(:allow_descendants_override_disabled_shared_runners) - if shared_runners_enabled && !new_record? - errors.add(:allow_descendants_override_disabled_shared_runners, _('cannot be changed if shared runners are enabled')) + if shared_runners_enabled && allow_descendants_override_disabled_shared_runners + errors.add(:allow_descendants_override_disabled_shared_runners, _('can not be true if shared runners are enabled')) end if allow_descendants_override_disabled_shared_runners && has_parent? && parent.shared_runners_setting == SR_DISABLED_AND_UNOVERRIDABLE diff --git a/app/models/namespace/aggregation_schedule.rb b/app/models/namespace/aggregation_schedule.rb index 6c977505f17..08187a9273e 100644 --- a/app/models/namespace/aggregation_schedule.rb +++ b/app/models/namespace/aggregation_schedule.rb @@ -13,11 +13,7 @@ class Namespace::AggregationSchedule < ApplicationRecord after_create :schedule_root_storage_statistics def default_lease_timeout - if Feature.enabled?(:reduce_aggregation_schedule_lease, namespace.root_ancestor) - ::Gitlab::CurrentSettings.namespace_aggregation_schedule_lease_duration_in_seconds - else - 30.minutes.to_i - end + ::Gitlab::CurrentSettings.namespace_aggregation_schedule_lease_duration_in_seconds end def schedule_root_storage_statistics diff --git a/app/models/namespace/detail.rb b/app/models/namespace/detail.rb index 2660d11171e..6c825b5364f 100644 --- a/app/models/namespace/detail.rb +++ b/app/models/namespace/detail.rb @@ -4,6 +4,10 @@ class Namespace::Detail < ApplicationRecord include IgnorableColumns ignore_column :free_user_cap_over_limt_notified_at, remove_with: '15.7', remove_after: '2022-11-22' + ignore_column :dashboard_notification_at, remove_with: '16.5', remove_after: '2023-08-22' + ignore_column :dashboard_enforcement_at, remove_with: '16.5', remove_after: '2023-08-22' + ignore_column :next_over_limit_check_at, remove_with: '16.5', remove_after: '2023-08-22' + ignore_column :free_user_cap_over_limit_notified_at, remove_with: '16.5', remove_after: '2023-08-22' belongs_to :namespace, inverse_of: :namespace_details validates :namespace, presence: true diff --git a/app/models/namespace/package_setting.rb b/app/models/namespace/package_setting.rb index 22c3e41ff21..a249bb144f9 100644 --- a/app/models/namespace/package_setting.rb +++ b/app/models/namespace/package_setting.rb @@ -12,7 +12,7 @@ class Namespace::PackageSetting < ApplicationRecord PackageSettingNotImplemented = Class.new(StandardError) - PACKAGES_WITH_SETTINGS = %w[maven generic].freeze + PACKAGES_WITH_SETTINGS = %w[maven generic nuget].freeze belongs_to :namespace, inverse_of: :package_setting_relation @@ -21,6 +21,8 @@ class Namespace::PackageSetting < ApplicationRecord validates :maven_duplicate_exception_regex, untrusted_regexp: true, length: { maximum: 255 } validates :generic_duplicates_allowed, inclusion: { in: [true, false] } validates :generic_duplicate_exception_regex, untrusted_regexp: true, length: { maximum: 255 } + validates :nuget_duplicates_allowed, inclusion: { in: [true, false] } + validates :nuget_duplicate_exception_regex, untrusted_regexp: true, length: { maximum: 255 } class << self def duplicates_allowed?(package) diff --git a/app/models/namespace_setting.rb b/app/models/namespace_setting.rb index 5b114bb42aa..8d5d788c738 100644 --- a/app/models/namespace_setting.rb +++ b/app/models/namespace_setting.rb @@ -45,6 +45,7 @@ class NamespaceSetting < ApplicationRecord enabled_git_access_protocol subgroup_runner_token_expiration_interval project_runner_token_expiration_interval + default_branch_protection_defaults ].freeze # matches the size set in the database constraint diff --git a/app/models/namespaces/project_namespace.rb b/app/models/namespaces/project_namespace.rb index bf23fc21124..288c5c0d2d4 100644 --- a/app/models/namespaces/project_namespace.rb +++ b/app/models/namespaces/project_namespace.rb @@ -6,10 +6,10 @@ module Namespaces # project.namespace/project.namespace_id attribute. # # TODO: we can remove these attribute aliases when we no longer need to sync these with project model, - # see project#sync_attributes + # see ProjectNamespace#sync_attributes_from_project alias_attribute :namespace, :parent alias_attribute :namespace_id, :parent_id - has_one :project, foreign_key: :project_namespace_id, inverse_of: :project_namespace + has_one :project, inverse_of: :project_namespace delegate :execute_hooks, :execute_integrations, :group, to: :project, allow_nil: true delegate :external_references_supported?, :default_issues_tracker?, to: :project @@ -21,5 +21,40 @@ module Namespaces def self.polymorphic_name 'Namespaces::ProjectNamespace' end + + def self.create_from_project!(project) + return unless project.new_record? + return unless project.namespace + + proj_namespace = project.project_namespace || project.build_project_namespace + project.project_namespace.sync_attributes_from_project(project) + proj_namespace.save! + proj_namespace + end + + def sync_attributes_from_project(project) + attributes_to_sync = project + .changes + .slice(*%w[name path namespace_id namespace visibility_level shared_runners_enabled]) + .transform_values { |val| val[1] } + + # if visibility_level is not set explicitly for project, it defaults to 0, + # but for namespace visibility_level defaults to 20, + # so it gets out of sync right away if we do not set it explicitly when creating the project namespace + attributes_to_sync['visibility_level'] ||= project.visibility_level if project.new_record? + + # when a project is associated with a group while the group is created we need to ensure we associate the new + # group with the project namespace as well. + # E.g. + # project = create(:project) <- project is saved + # create(:group, projects: [project]) <- associate project with a group that is not yet created. + if attributes_to_sync.has_key?('namespace_id') && + attributes_to_sync['namespace_id'].blank? && + project.namespace.present? + attributes_to_sync['parent'] = project.namespace + end + + assign_attributes(attributes_to_sync) + end end end diff --git a/app/models/network/graph.rb b/app/models/network/graph.rb index 7ffcb8b9219..0f410d4810d 100644 --- a/app/models/network/graph.rb +++ b/app/models/network/graph.rb @@ -86,6 +86,7 @@ module Network skip = 0 while offset == -1 tmp_commits = find_commits(skip) + if tmp_commits.present? index = tmp_commits.index do |c| c.id == @commit.id @@ -112,15 +113,17 @@ module Network end def find_commits(skip = 0) - opts = { - max_count: self.class.max_count, - skip: skip, - order: :date - } + Gitlab::SafeRequestStore.fetch([@project, :network_graph_commits, skip]) do + opts = { + max_count: self.class.max_count, + skip: skip, + order: :date + } - opts[:ref] = @commit.id if @filter_ref + opts[:ref] = @commit.id if @filter_ref - Gitlab::Git::Commit.find_all(@repo.raw_repository, opts) + Gitlab::Git::Commit.find_all(@repo.raw_repository, opts) + end end def commits_sort_by_ref diff --git a/app/models/note.rb b/app/models/note.rb index 2df643c46aa..f1760a8dc4a 100644 --- a/app/models/note.rb +++ b/app/models/note.rb @@ -149,7 +149,7 @@ class Note < ApplicationRecord scope :with_api_entity_associations, -> { preload(:note_diff_file, :author) } scope :inc_relations_for_view, ->(noteable = nil) do relations = [{ project: :group }, { author: :status }, :updated_by, :resolved_by, - :award_emoji, { system_note_metadata: :description_version }, :suggestions] + :award_emoji, :note_metadata, { system_note_metadata: :description_version }, :suggestions] if noteable.nil? || DiffNote.noteable_types.include?(noteable.class.name) relations += [:note_diff_file, :diff_note_positions] @@ -197,9 +197,7 @@ class Note < ApplicationRecord # Syncs `confidential` with `internal` as we rename the column. # https://gitlab.com/gitlab-org/gitlab/-/issues/367923 before_create :set_internal_flag - after_destroy :expire_etag_cache after_save :keep_around_commit, if: :for_project_noteable?, unless: -> { importing? || skip_keep_around_commits } - after_save :expire_etag_cache, unless: :importing? after_save :touch_noteable, unless: :importing? after_commit :notify_after_create, on: :create after_commit :notify_after_destroy, on: :destroy @@ -207,6 +205,7 @@ class Note < ApplicationRecord after_commit :trigger_note_subscription_create, on: :create after_commit :trigger_note_subscription_update, on: :update after_commit :trigger_note_subscription_destroy, on: :destroy + after_commit :expire_etag_cache, unless: :importing? def trigger_note_subscription_create return unless trigger_note_subscription? @@ -498,7 +497,7 @@ class Note < ApplicationRecord end def can_be_discussion_note? - self.noteable.supports_discussions? && !part_of_discussion? + self.noteable.supports_discussions? && !part_of_discussion? && !system? end def can_create_todo? @@ -853,7 +852,9 @@ class Note < ApplicationRecord user_visible_reference_count > 0 && user_visible_reference_count == total_reference_count else refs = all_references(user) - refs.all.any? && refs.all_visible? + refs.all + + refs.all_visible? end end diff --git a/app/models/operations/feature_flag.rb b/app/models/operations/feature_flag.rb index 6876af09c2c..01db0a5cf8b 100644 --- a/app/models/operations/feature_flag.rb +++ b/app/models/operations/feature_flag.rb @@ -30,7 +30,9 @@ module Operations length: 2..63, format: { with: Gitlab::Regex.feature_flag_regex, - message: Gitlab::Regex.feature_flag_regex_message + message: ->(_object, _data) { + s_("Validation|can contain only lowercase letters, digits, '_' and '-'. Must start with a letter, and cannot end with '-' or '_'") + } } validates :name, uniqueness: { scope: :project_id } validates :description, allow_blank: true, length: 0..255 diff --git a/app/models/operations/feature_flags/strategy.rb b/app/models/operations/feature_flags/strategy.rb index ed9400dde8f..5dc6de7dfc1 100644 --- a/app/models/operations/feature_flags/strategy.rb +++ b/app/models/operations/feature_flags/strategy.rb @@ -28,7 +28,7 @@ module Operations validates :name, inclusion: { in: STRATEGIES.keys, - message: 'strategy name is invalid' + message: ->(_object, _data) { s_('Validation|strategy name is invalid') } } validate :parameters_validations, if: -> { errors[:name].blank? } @@ -46,7 +46,7 @@ module Operations def same_project_validation unless user_list.project_id == feature_flag.project_id - errors.add(:user_list, 'must belong to the same project') + errors.add(:user_list, s_('Validation|must belong to the same project')) end end @@ -57,13 +57,13 @@ module Operations end def validate_parameters_type - parameters.is_a?(Hash) || parameters_error('parameters are invalid') + parameters.is_a?(Hash) || parameters_error(s_('Validation|parameters are invalid')) end def validate_parameters_keys actual_keys = parameters.keys.sort expected_keys = STRATEGIES[name].sort - expected_keys == actual_keys || parameters_error('parameters are invalid') + expected_keys == actual_keys || parameters_error(s_('Validation|parameters are invalid')) end def validate_parameters_values @@ -89,11 +89,11 @@ module Operations group_id = parameters['groupId'] unless within_range?(percentage, 0, 100) - parameters_error('percentage must be a string between 0 and 100 inclusive') + parameters_error(s_('Validation|percentage must be a string between 0 and 100 inclusive')) end unless group_id.is_a?(String) && group_id.match(/\A[a-z]{1,32}\z/) - parameters_error('groupId parameter is invalid') + parameters_error(s_('Validation|groupId parameter is invalid')) end end @@ -108,11 +108,11 @@ module Operations end unless group_id.is_a?(String) && group_id.match(/\A[a-z]{1,32}\z/) - parameters_error('groupId parameter is invalid') + parameters_error(s_('Validation|groupId parameter is invalid')) end unless within_range?(rollout, 0, 100) - parameters_error('rollout must be a string between 0 and 100 inclusive') + parameters_error(s_('Validation|rollout must be a string between 0 and 100 inclusive')) end end diff --git a/app/models/organizations/organization.rb b/app/models/organizations/organization.rb index 8aeca2eb137..9f2119949fb 100644 --- a/app/models/organizations/organization.rb +++ b/app/models/organizations/organization.rb @@ -37,6 +37,10 @@ module Organizations path end + def user?(user) + users.exists?(user.id) + end + private def check_if_default_organization diff --git a/app/models/packages/nuget/metadatum.rb b/app/models/packages/nuget/metadatum.rb index fae7728cccb..e7cf4528f16 100644 --- a/app/models/packages/nuget/metadatum.rb +++ b/app/models/packages/nuget/metadatum.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Packages::Nuget::Metadatum < ApplicationRecord + include Packages::Nuget::VersionNormalizable + MAX_AUTHORS_LENGTH = 255 MAX_DESCRIPTION_LENGTH = 4000 MAX_URL_LENGTH = 255 @@ -13,9 +15,15 @@ class Packages::Nuget::Metadatum < ApplicationRecord validates :icon_url, public_url: { allow_blank: true }, length: { maximum: MAX_URL_LENGTH } validates :authors, presence: true, length: { maximum: MAX_AUTHORS_LENGTH } validates :description, presence: true, length: { maximum: MAX_DESCRIPTION_LENGTH } + validates :normalized_version, presence: true, + if: -> { Feature.enabled?(:nuget_normalized_version, package&.project) } validate :ensure_nuget_package_type + delegate :version, to: :package, prefix: true + + scope :normalized_version_in, ->(version) { where(normalized_version: version.downcase) } + private def ensure_nuget_package_type diff --git a/app/models/packages/package.rb b/app/models/packages/package.rb index b618c7c20c4..b09911f4216 100644 --- a/app/models/packages/package.rb +++ b/app/models/packages/package.rb @@ -10,6 +10,7 @@ class Packages::Package < ApplicationRecord DISPLAYABLE_STATUSES = [:default, :error].freeze INSTALLABLE_STATUSES = [:default, :hidden].freeze + STATUS_MESSAGE_MAX_LENGTH = 255 enum package_type: { maven: 1, @@ -123,6 +124,22 @@ class Packages::Package < ApplicationRecord where('LOWER(version) = ?', version.downcase) end + scope :with_case_insensitive_name, ->(name) do + where(arel_table[:name].lower.eq(name.downcase)) + end + + scope :with_nuget_version_or_normalized_version, ->(version, with_normalized: true) do + relation = with_case_insensitive_version(version) + + return relation unless with_normalized + + relation + .left_joins(:nuget_metadatum) + .or( + merge(Packages::Nuget::Metadatum.normalized_version_in(version)) + ) + end + scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) } scope :with_version, ->(version) { where(version: version) } scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) } @@ -161,6 +178,14 @@ class Packages::Package < ApplicationRecord scope :preload_pypi_metadatum, -> { preload(:pypi_metadatum) } scope :preload_conan_metadatum, -> { preload(:conan_metadatum) } + scope :with_npm_scope, ->(scope) do + if Feature.enabled?(:npm_package_registry_fix_group_path_validation) + npm.where("position('/' in packages_packages.name) > 0 AND split_part(packages_packages.name, '/', 1) = :package_scope", package_scope: "@#{sanitize_sql_like(scope)}") + else + npm.where("name ILIKE :package_name", package_name: "@#{sanitize_sql_like(scope)}/%") + end + end + scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::TEMPORARY_PACKAGE_NAME) } scope :has_version, -> { where.not(version: nil) } @@ -169,14 +194,12 @@ class Packages::Package < ApplicationRecord scope :preload_pipelines, -> { preload(pipelines: :user) } scope :limit_recent, ->(limit) { order_created_desc.limit(limit) } scope :select_distinct_name, -> { select(:name).distinct } - scope :select_only_first_by_name, -> { select('DISTINCT ON (name) *') } # Sorting scope :order_created, -> { reorder(created_at: :asc) } scope :order_created_desc, -> { reorder(created_at: :desc) } scope :order_name, -> { reorder(name: :asc) } scope :order_name_desc, -> { reorder(name: :desc) } - scope :order_name_desc_version_desc, -> { reorder(name: :desc, version: :desc) } scope :order_version, -> { reorder(version: :asc) } scope :order_version_desc, -> { reorder(version: :desc) } scope :order_type, -> { reorder(package_type: :asc) } @@ -184,7 +207,6 @@ class Packages::Package < ApplicationRecord scope :order_project_name, -> { joins(:project).reorder('projects.name ASC') } scope :order_project_name_desc, -> { joins(:project).reorder('projects.name DESC') } scope :order_by_package_file, -> { joins(:package_files).order('packages_package_files.created_at ASC') } - scope :with_npm_scope, ->(scope) { npm.where("name ILIKE :package_name", package_name: "@#{sanitize_sql_like(scope)}/%") } scope :order_project_path, -> do keyset_order = keyset_pagination_order(join_class: Project, column_name: :path, direction: :asc) @@ -361,6 +383,12 @@ class Packages::Package < ApplicationRecord name.gsub(/#{Gitlab::Regex::Packages::PYPI_NORMALIZED_NAME_REGEX_STRING}/o, '-').downcase end + def normalized_nuget_version + return unless nuget? + + nuget_metadatum&.normalized_version + end + def publish_creation_event ::Gitlab::EventStore.publish( ::Packages::PackageCreatedEvent.new(data: { diff --git a/app/models/pages_deployment.rb b/app/models/pages_deployment.rb index fa29cbf8352..ec2293fa032 100644 --- a/app/models/pages_deployment.rb +++ b/app/models/pages_deployment.rb @@ -29,18 +29,13 @@ class PagesDeployment < ApplicationRecord mount_file_store_uploader ::Pages::DeploymentUploader - skip_callback :save, :after, :store_file!, if: :store_after_commit? - after_commit :store_file_after_commit!, on: [:create, :update], if: :store_after_commit? + skip_callback :save, :after, :store_file! + after_commit :store_file_after_commit!, on: [:create, :update] def migrated? file.filename == MIGRATED_FILE_NAME end - def store_after_commit? - Feature.enabled?(:pages_deploy_upload_file_outside_transaction, project) - end - strong_memoize_attr :store_after_commit? - private def set_size diff --git a/app/models/performance_monitoring/prometheus_dashboard.rb b/app/models/performance_monitoring/prometheus_dashboard.rb deleted file mode 100644 index 6fea3abf3d9..00000000000 --- a/app/models/performance_monitoring/prometheus_dashboard.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module PerformanceMonitoring - class PrometheusDashboard - include ActiveModel::Model - - attr_accessor :dashboard, :panel_groups, :path, :environment, :priority, :templating, :links - - validates :dashboard, presence: true - validates :panel_groups, array_members: { member_class: PerformanceMonitoring::PrometheusPanelGroup } - - class << self - def from_json(json_content) - build_from_hash(json_content).tap(&:validate!) - end - - def find_for(project:, user:, path:, options: {}) - template = { path: path, environment: options[:environment] } - rsp = Gitlab::Metrics::Dashboard::Finder.find(project, user, options.merge(dashboard_path: path)) - - case rsp[:http_status] || rsp[:status] - when :success - new(template.merge(rsp[:dashboard] || {})) # when there is empty dashboard file returned rsp is still a success - when :unprocessable_entity - new(template) # validation error - else - nil # any other error - end - end - - private - - def build_from_hash(attributes) - return new unless attributes.is_a?(Hash) - - new( - dashboard: attributes['dashboard'], - panel_groups: initialize_children_collection(attributes['panel_groups']) - ) - end - - def initialize_children_collection(children) - return unless children.is_a?(Array) - - children.map { |group| PerformanceMonitoring::PrometheusPanelGroup.from_json(group) } - end - end - - def to_yaml - self.as_json(only: yaml_valid_attributes).to_yaml - end - - # This method is planned to be refactored as a part of https://gitlab.com/gitlab-org/gitlab/-/issues/219398 - # implementation. For new existing logic was reused to faster deliver MVC - def schema_validation_warnings - self.class.from_json(reload_schema) - [] - rescue Gitlab::Metrics::Dashboard::Errors::LayoutError => e - [e.message] - rescue ActiveModel::ValidationError => e - e.model.errors.map { |error| "#{error.attribute}: #{error.message}" } - end - - private - - # dashboard finder methods are somehow limited, #find includes checking if - # user is authorised to view selected dashboard, but modifies schema, which in some cases may - # cause false positives returned from validation, and #find_raw does not authorise users - def reload_schema - project = environment&.project - project.nil? ? self.as_json : Gitlab::Metrics::Dashboard::Finder.find_raw(project, dashboard_path: path) - end - - def yaml_valid_attributes - %w(panel_groups panels metrics group priority type title y_label weight id unit label query query_range dashboard) - end - end -end diff --git a/app/models/plan.rb b/app/models/plan.rb index e16ecb4c629..22c1201421c 100644 --- a/app/models/plan.rb +++ b/app/models/plan.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Plan < ApplicationRecord +class Plan < MainClusterwide::ApplicationRecord DEFAULT = 'default' has_one :limits, class_name: 'PlanLimits' diff --git a/app/models/pool_repository.rb b/app/models/pool_repository.rb index f22a63ee980..bc3898fafe7 100644 --- a/app/models/pool_repository.rb +++ b/app/models/pool_repository.rb @@ -12,7 +12,13 @@ class PoolRepository < ApplicationRecord has_many :member_projects, class_name: 'Project' - after_create :correct_disk_path + after_create :set_disk_path + + scope :by_source_project, ->(project) { where(source_project: project) } + scope :by_source_project_and_shard_name, ->(project, shard_name) do + by_source_project(project) + .for_repository_storage(shard_name) + end state_machine :state, initial: :none do state :scheduled @@ -107,8 +113,8 @@ class PoolRepository < ApplicationRecord private - def correct_disk_path - update!(disk_path: storage.disk_path) + def set_disk_path + update!(disk_path: storage.disk_path) if disk_path.blank? end def storage diff --git a/app/models/project.rb b/app/models/project.rb index 8959eccbd1f..ad8757880fd 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -43,6 +43,9 @@ class Project < ApplicationRecord include Subquery include IssueParent include UpdatedAtFilterable + include IgnorableColumns + + ignore_column :emails_disabled, remove_with: '16.3', remove_after: '2023-08-22' extend Gitlab::Cache::RequestCache extend Gitlab::Utils::Override @@ -125,7 +128,6 @@ class Project < ApplicationRecord before_validation :remove_leading_spaces_on_name after_validation :check_pending_delete before_save :ensure_runners_token - before_save :update_new_emails_created_column, if: -> { emails_disabled_changed? } after_create -> { create_or_load_association(:project_feature) } after_create -> { create_or_load_association(:ci_cd_settings) } @@ -165,11 +167,14 @@ class Project < ApplicationRecord belongs_to :namespace # Sync deletion via DB Trigger to ensure we do not have # a project without a project_namespace (or vice-versa) - belongs_to :project_namespace, autosave: true, class_name: 'Namespaces::ProjectNamespace', foreign_key: 'project_namespace_id' + belongs_to :project_namespace, autosave: true, class_name: 'Namespaces::ProjectNamespace', foreign_key: 'project_namespace_id', inverse_of: :project alias_method :parent, :namespace alias_attribute :parent_id, :namespace_id has_one :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :project + has_many :ci_components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :project + has_many :catalog_resource_versions, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :project + has_one :last_event, -> { order 'events.created_at DESC' }, class_name: 'Event' has_many :boards @@ -312,7 +317,8 @@ class Project < ApplicationRecord has_many :designs, inverse_of: :project, class_name: 'DesignManagement::Design' has_many :project_authorizations - has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' + has_many :authorized_users, -> { allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/422045') }, + through: :project_authorizations, source: :user, class_name: 'User' has_many :project_members, -> { where(requested_at: nil) }, as: :source, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent @@ -506,6 +512,7 @@ class Project < ApplicationRecord with_options prefix: :ci do delegate :default_git_depth, :default_git_depth= delegate :forward_deployment_enabled, :forward_deployment_enabled= + delegate :forward_deployment_rollback_allowed, :forward_deployment_rollback_allowed= delegate :inbound_job_token_scope_enabled, :inbound_job_token_scope_enabled= delegate :allow_fork_pipelines_to_run_in_parent_project, :allow_fork_pipelines_to_run_in_parent_project= delegate :separated_caches, :separated_caches= @@ -518,6 +525,7 @@ class Project < ApplicationRecord delegate :has_shimo? delegate :show_diff_preview_in_email, :show_diff_preview_in_email=, :show_diff_preview_in_email? delegate :runner_registration_enabled, :runner_registration_enabled=, :runner_registration_enabled? + delegate :emails_enabled, :emails_enabled=, :emails_enabled? delegate :squash_always?, :squash_never?, :squash_enabled_by_default?, :squash_readonly? delegate :mr_default_target_self, :mr_default_target_self= delegate :previous_default_branch, :previous_default_branch= @@ -585,6 +593,7 @@ class Project < ApplicationRecord scope :pending_delete, -> { where(pending_delete: true) } scope :without_deleted, -> { where(pending_delete: false) } scope :not_hidden, -> { where(hidden: false) } + scope :not_in_groups, ->(groups) { where.not(group: groups) } scope :not_aimed_for_deletion, -> { where(marked_for_deletion_at: nil).without_deleted } scope :with_storage_feature, ->(feature) do @@ -703,6 +712,7 @@ class Project < ApplicationRecord # includes(:route) which we use in ProjectsFinder. joins("INNER JOIN routes rs ON rs.source_id = projects.id AND rs.source_type = 'Project'") .where('rs.path LIKE ?', "#{sanitize_sql_like(path)}/%") + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') end scope :with_feature_enabled, ->(feature) { @@ -932,6 +942,7 @@ class Project < ApplicationRecord if include_namespace joins(:route).fuzzy_search(query, [Route.arel_table[:path], Route.arel_table[:name], :description], use_minimum_char_limit: use_minimum_char_limit) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') else fuzzy_search(query, [:path, :name, :description], use_minimum_char_limit: use_minimum_char_limit) end @@ -1209,14 +1220,8 @@ class Project < ApplicationRecord end def emails_disabled? - strong_memoize(:emails_disabled) do - # disabling in the namespace overrides the project setting - super || namespace.emails_disabled? - end - end - - def emails_enabled? - !emails_disabled? + # disabling in the namespace overrides the project setting + !emails_enabled? end override :lfs_enabled? @@ -1760,7 +1765,8 @@ class Project < ApplicationRecord # rubocop: disable CodeReuse/ServiceClass def create_labels Label.templates.each do |label| - params = label.attributes.except('id', 'template', 'created_at', 'updated_at', 'type') + # slice on column_names to ensure an added DB column will not break a mixed deployment + params = label.attributes.slice(*Label.column_names).except('id', 'template', 'created_at', 'updated_at', 'type') Labels::FindOrCreateService.new(nil, self, params).execute(skip_authorization: true) end end @@ -1951,6 +1957,8 @@ class Project < ApplicationRecord def track_project_repository repository = project_repository || build_project_repository repository.update!(shard_name: repository_storage, disk_path: disk_path) + + cleanup if replicate_object_pool_on_move_ff_enabled? end def create_repository(force: false, default_branch: nil) @@ -2466,7 +2474,7 @@ class Project < ApplicationRecord break unless pages_enabled? variables.append(key: 'CI_PAGES_DOMAIN', value: Gitlab.config.pages.host) - variables.append(key: 'CI_PAGES_URL', value: Gitlab::Pages::UrlBuilder.new(self).pages_url) + variables.append(key: 'CI_PAGES_URL', value: Gitlab::Pages::UrlBuilder.new(self).pages_url(with_unique_domain: true)) end end @@ -2825,8 +2833,26 @@ class Project < ApplicationRecord update_column(:pool_repository_id, nil) end + # After repository is moved from shard to shard, disconnect it from the previous object pool and connect to the new pool + def swap_pool_repository! + return unless replicate_object_pool_on_move_ff_enabled? + return unless repository_exists? + + old_pool_repository = pool_repository + return if old_pool_repository.blank? + return if pool_repository_shard_matches_repository?(old_pool_repository) + + new_pool_repository = PoolRepository.by_source_project_and_shard_name(old_pool_repository.source_project, repository_storage).take! + update!(pool_repository: new_pool_repository) + + old_pool_repository.unlink_repository(repository, disconnect: !pending_delete?) + end + def link_pool_repository - pool_repository&.link_repository(repository) + return unless pool_repository + return if (pool_repository.shard_name != repository.shard) && replicate_object_pool_on_move_ff_enabled? + + pool_repository.link_repository(repository) end def has_pool_repository? @@ -3048,6 +3074,12 @@ class Project < ApplicationRecord ci_cd_settings.forward_deployment_enabled? end + def ci_forward_deployment_rollback_allowed? + return false unless ci_cd_settings + + ci_cd_settings.forward_deployment_rollback_allowed? + end + def ci_allow_fork_pipelines_to_run_in_parent_project? return false unless ci_cd_settings @@ -3151,6 +3183,8 @@ class Project < ApplicationRecord end def created_and_owned_by_banned_user? + return false unless creator + creator.banned? && team.max_member_access(creator.id) == Gitlab::Access::OWNER end @@ -3170,6 +3204,10 @@ class Project < ApplicationRecord group&.work_items_mvc_2_feature_flag_enabled? || Feature.enabled?(:work_items_mvc_2) end + def linked_work_items_feature_flag_enabled? + group&.linked_work_items_feature_flag_enabled? || Feature.enabled?(:linked_work_items, self) + end + def enqueue_record_project_target_platforms return unless Gitlab.com? @@ -3437,7 +3475,7 @@ class Project < ApplicationRecord # create project_namespace when project is created build_project_namespace if project_namespace_creation_enabled? - sync_attributes(project_namespace) if sync_project_namespace? + project_namespace.sync_attributes_from_project(self) if sync_project_namespace? end def project_namespace_creation_enabled? @@ -3448,27 +3486,6 @@ class Project < ApplicationRecord (changes.keys & %w(name path namespace_id namespace visibility_level shared_runners_enabled)).any? && project_namespace.present? end - def sync_attributes(project_namespace) - attributes_to_sync = changes.slice(*%w(name path namespace_id namespace visibility_level shared_runners_enabled)) - .transform_values { |val| val[1] } - - # if visibility_level is not set explicitly for project, it defaults to 0, - # but for namespace visibility_level defaults to 20, - # so it gets out of sync right away if we do not set it explicitly when creating the project namespace - attributes_to_sync['visibility_level'] ||= visibility_level if new_record? - - # when a project is associated with a group while the group is created we need to ensure we associate the new - # group with the project namespace as well. - # E.g. - # project = create(:project) <- project is saved - # create(:group, projects: [project]) <- associate project with a group that is not yet created. - if attributes_to_sync.has_key?('namespace_id') && attributes_to_sync['namespace_id'].blank? && namespace.present? - attributes_to_sync['parent'] = namespace - end - - project_namespace.assign_attributes(attributes_to_sync) - end - def reload_project_namespace_details return unless (previous_changes.keys & %w(description description_html cached_markdown_version)).any? && project_namespace.namespace_details.present? @@ -3511,19 +3528,18 @@ class Project < ApplicationRecord end end - def update_new_emails_created_column - return if project_setting.nil? - return if project_setting.emails_enabled == !emails_disabled + def runners_token_prefix + RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + end - if project_setting.persisted? - project_setting.update!(emails_enabled: !emails_disabled) - elsif project_setting - project_setting.emails_enabled = !emails_disabled - end + def replicate_object_pool_on_move_ff_enabled? + Feature.enabled?(:replicate_object_pool_on_move, self) end - def runners_token_prefix - RunnersTokenPrefixable::RUNNERS_TOKEN_PREFIX + def pool_repository_shard_matches_repository?(pool) + pool_repository_shard = pool.shard.name + + pool_repository_shard == repository_storage end end diff --git a/app/models/project_authorization.rb b/app/models/project_authorization.rb index cb578496f26..99128d3cddf 100644 --- a/app/models/project_authorization.rb +++ b/app/models/project_authorization.rb @@ -1,9 +1,6 @@ # frozen_string_literal: true class ProjectAuthorization < ApplicationRecord - BATCH_SIZE = 1000 - SLEEP_DELAY = 0.1 - extend SuppressCompositePrimaryKeyWarning include FromUnion @@ -28,57 +25,6 @@ class ProjectAuthorization < ApplicationRecord def self.insert_all(attributes) super(attributes, unique_by: connection.schema_cache.primary_keys(table_name)) end - - def self.insert_all_in_batches(attributes, per_batch = BATCH_SIZE) - add_delay = add_delay_between_batches?(entire_size: attributes.size, batch_size: per_batch) - log_details(entire_size: attributes.size, batch_size: per_batch) if add_delay - - attributes.each_slice(per_batch) do |attributes_batch| - insert_all(attributes_batch) - perform_delay if add_delay - end - end - - def self.delete_all_in_batches_for_project(project:, user_ids:, per_batch: BATCH_SIZE) - add_delay = add_delay_between_batches?(entire_size: user_ids.size, batch_size: per_batch) - log_details(entire_size: user_ids.size, batch_size: per_batch) if add_delay - - user_ids.each_slice(per_batch) do |user_ids_batch| - project.project_authorizations.where(user_id: user_ids_batch).delete_all - perform_delay if add_delay - end - end - - def self.delete_all_in_batches_for_user(user:, project_ids:, per_batch: BATCH_SIZE) - add_delay = add_delay_between_batches?(entire_size: project_ids.size, batch_size: per_batch) - log_details(entire_size: project_ids.size, batch_size: per_batch) if add_delay - - project_ids.each_slice(per_batch) do |project_ids_batch| - user.project_authorizations.where(project_id: project_ids_batch).delete_all - perform_delay if add_delay - end - end - - private_class_method def self.add_delay_between_batches?(entire_size:, batch_size:) - # The reason for adding a delay is to give the replica database enough time to - # catch up with the primary when large batches of records are being added/removed. - # Hance, we add a delay only if the GitLab installation has a replica database configured. - entire_size > batch_size && - !::Gitlab::Database::LoadBalancing.primary_only? - end - - private_class_method def self.log_details(entire_size:, batch_size:) - Gitlab::AppLogger.info( - entire_size: entire_size, - total_delay: (entire_size / batch_size.to_f).ceil * SLEEP_DELAY, - message: 'Project authorizations refresh performed with delay', - **Gitlab::ApplicationContext.current - ) - end - - private_class_method def self.perform_delay - sleep(SLEEP_DELAY) - end end ProjectAuthorization.prepend_mod_with('ProjectAuthorization') diff --git a/app/models/project_authorizations/changes.rb b/app/models/project_authorizations/changes.rb new file mode 100644 index 00000000000..1d717950c1c --- /dev/null +++ b/app/models/project_authorizations/changes.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module ProjectAuthorizations + # How to use this class + # authorizations_to_add: + # Rows to insert in the form `[{ user_id: user_id, project_id: project_id, access_level: access_level}, ...] + # + # ProjectAuthorizations::Changes.new do |changes| + # changes.add(authorizations_to_add) + # changes.remove_users_in_project(project, user_ids) + # changes.remove_projects_for_user(user, project_ids) + # end.apply! + class Changes + attr_reader :projects_to_remove, :users_to_remove, :authorizations_to_add + + BATCH_SIZE = 1000 + SLEEP_DELAY = 0.1 + + def initialize + @authorizations_to_add = [] + @affected_project_ids = Set.new + yield self + end + + def add(authorizations_to_add) + @authorizations_to_add += authorizations_to_add + end + + def remove_users_in_project(project, user_ids) + @users_to_remove = { user_ids: user_ids, scope: project } + end + + def remove_projects_for_user(user, project_ids) + @projects_to_remove = { project_ids: project_ids, scope: user } + end + + def apply! + delete_authorizations_for_user if should_delete_authorizations_for_user? + delete_authorizations_for_project if should_delete_authorizations_for_project? + add_authorizations if should_add_authorization? + + publish_events + end + + private + + def should_add_authorization? + authorizations_to_add.present? + end + + def should_delete_authorizations_for_user? + user && project_ids.present? + end + + def should_delete_authorizations_for_project? + project && user_ids.present? + end + + def add_authorizations + insert_all_in_batches(authorizations_to_add) + @affected_project_ids += authorizations_to_add.pluck(:project_id) + end + + def delete_authorizations_for_user + delete_all_in_batches(resource: user, + ids_to_remove: project_ids, + column_name_of_ids_to_remove: :project_id) + @affected_project_ids += project_ids + end + + def delete_authorizations_for_project + delete_all_in_batches(resource: project, + ids_to_remove: user_ids, + column_name_of_ids_to_remove: :user_id) + @affected_project_ids << project.id + end + + def delete_all_in_batches(resource:, ids_to_remove:, column_name_of_ids_to_remove:) + add_delay = add_delay_between_batches?(entire_size: ids_to_remove.size, batch_size: BATCH_SIZE) + log_details(entire_size: ids_to_remove.size, batch_size: BATCH_SIZE) if add_delay + + ids_to_remove.each_slice(BATCH_SIZE) do |ids_batch| + resource.project_authorizations.where(column_name_of_ids_to_remove => ids_batch).delete_all + perform_delay if add_delay + end + end + + def insert_all_in_batches(attributes) + add_delay = add_delay_between_batches?(entire_size: attributes.size, batch_size: BATCH_SIZE) + log_details(entire_size: attributes.size, batch_size: BATCH_SIZE) if add_delay + + attributes.each_slice(BATCH_SIZE) do |attributes_batch| + ProjectAuthorization.insert_all(attributes_batch) + perform_delay if add_delay + end + end + + def add_delay_between_batches?(entire_size:, batch_size:) + # The reason for adding a delay is to give the replica database enough time to + # catch up with the primary when large batches of records are being added/removed. + # Hence, we add a delay only if the GitLab installation has a replica database configured. + entire_size > batch_size && + !::Gitlab::Database::LoadBalancing.primary_only? + end + + def log_details(entire_size:, batch_size:) + Gitlab::AppLogger.info( + entire_size: entire_size, + total_delay: (entire_size / batch_size.to_f).ceil * SLEEP_DELAY, + message: 'Project authorizations refresh performed with delay', + **Gitlab::ApplicationContext.current + ) + end + + def perform_delay + sleep(SLEEP_DELAY) + end + + def user + projects_to_remove&.[](:scope) + end + + def project_ids + projects_to_remove&.[](:project_ids) + end + + def project + users_to_remove&.[](:scope) + end + + def user_ids + users_to_remove&.[](:user_ids) + end + + def publish_events + @affected_project_ids.each do |project_id| + ::Gitlab::EventStore.publish( + ::ProjectAuthorizations::AuthorizationsChangedEvent.new(data: { project_id: project_id }) + ) + end + end + end +end diff --git a/app/models/project_group_link.rb b/app/models/project_group_link.rb index 9f9447c1de2..69d8c0db55b 100644 --- a/app/models/project_group_link.rb +++ b/app/models/project_group_link.rb @@ -3,6 +3,7 @@ class ProjectGroupLink < ApplicationRecord include Expirable include EachBatch + include AfterCommitQueue belongs_to :project belongs_to :group @@ -16,6 +17,7 @@ class ProjectGroupLink < ApplicationRecord scope :non_guests, -> { where('group_access > ?', Gitlab::Access::GUEST) } scope :in_group, -> (group_ids) { where(group_id: group_ids) } + scope :for_projects, -> (project_ids) { where(project_id: project_ids) } alias_method :shared_with_group, :group alias_method :shared_from, :project diff --git a/app/models/project_setting.rb b/app/models/project_setting.rb index aeefa5c8dcd..fec951eb7fe 100644 --- a/app/models/project_setting.rb +++ b/app/models/project_setting.rb @@ -99,6 +99,11 @@ class ProjectSetting < ApplicationRecord Gitlab::CurrentSettings.valid_runner_registrars.include?('project') && read_attribute(:runner_registration_enabled) end + def emails_enabled? + super && project.namespace.emails_enabled? + end + strong_memoize_attr :emails_enabled? + private def validates_mr_default_target_self diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 365bb5237c3..942f20f6e5e 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -19,7 +19,7 @@ class ProjectStatistics < ApplicationRecord Namespaces::ScheduleAggregationWorker.perform_async(project_statistics.namespace_id) end - before_save :update_storage_size + after_commit :refresh_storage_size!, on: :update, if: -> { storage_size_components_changed? } COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count, :snippets_size, :uploads_size, :container_registry_size].freeze INCREMENTABLE_COLUMNS = [ @@ -67,7 +67,7 @@ class ProjectStatistics < ApplicationRecord end def update_repository_size - self.repository_size = project.repository.size * 1.megabyte + self.repository_size = project.repository.recent_objects_size.megabytes end def update_wiki_size @@ -105,19 +105,14 @@ class ProjectStatistics < ApplicationRecord super.to_i end - def update_storage_size - self.storage_size = storage_size_components.sum { |component| method(component).call } - end - + # Since this incremental update method does not update the storage_size directly, + # we have to update the storage_size separately in an after_commit action. def refresh_storage_size! detect_race_on_record(log_fields: { caller: __method__, attributes: :storage_size }) do - update!(storage_size: storage_size_sum) + self.class.where(id: id).update_all("storage_size = #{storage_size_sum}") end end - # Since this incremental update method does not call update_storage_size above through before_save, - # we have to update the storage_size separately. - # # For counter attributes, storage_size will be refreshed after the counter is flushed, # through counter_attribute_after_commit # @@ -169,6 +164,10 @@ class ProjectStatistics < ApplicationRecord Namespaces::ScheduleAggregationWorker.perform_async(project.namespace_id) end end + + def storage_size_components_changed? + (previous_changes.keys & STORAGE_SIZE_COMPONENTS.map(&:to_s)).any? + end end ProjectStatistics.prepend_mod_with('ProjectStatistics') diff --git a/app/models/project_team.rb b/app/models/project_team.rb index fbdc88e7b76..3b9b82ee094 100644 --- a/app/models/project_team.rb +++ b/app/models/project_team.rb @@ -141,12 +141,10 @@ class ProjectTeam end ProjectMember.transaction do - source_members.each do |member| - member.save - end + source_members.each(&:save) end - true + source_members rescue StandardError false end diff --git a/app/models/redirect_route.rb b/app/models/redirect_route.rb index 749f4a87818..54b4c9d0fe1 100644 --- a/app/models/redirect_route.rb +++ b/app/models/redirect_route.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class RedirectRoute < ApplicationRecord +class RedirectRoute < MainClusterwide::ApplicationRecord include CaseSensitivity belongs_to :source, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations diff --git a/app/models/release.rb b/app/models/release.rb index f0ba56390ab..6830f6e8480 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -20,6 +20,8 @@ class Release < ApplicationRecord has_many :milestones, through: :milestone_releases has_many :evidences, inverse_of: :release, class_name: 'Releases::Evidence' + has_one :catalog_resource_version, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :release + accepts_nested_attributes_for :links, allow_destroy: true before_create :set_released_at diff --git a/app/models/repository.rb b/app/models/repository.rb index 1321c9da780..b8a46f80bc7 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -47,7 +47,7 @@ class Repository # # For example, for entry `:commit_count` there's a method called `commit_count` which # stores its data in the `commit_count` cache key. - CACHED_METHODS = %i(size commit_count readme_path contribution_guide + CACHED_METHODS = %i(size recent_objects_size commit_count readme_path contribution_guide changelog license_blob license_gitaly gitignore gitlab_ci_yml branch_names tag_names branch_count tag_count avatar exists? root_ref merged_branch_names @@ -363,7 +363,7 @@ class Repository end def expire_statistics_caches - expire_method_caches(%i(size commit_count)) + expire_method_caches(%i(size recent_objects_size commit_count)) end def expire_all_method_caches @@ -579,6 +579,12 @@ class Repository end cache_method :size, fallback: 0.0 + # The recent objects size of this repository in mebibytes. + def recent_objects_size + exists? ? raw_repository.recent_objects_size : 0.0 + end + cache_method :recent_objects_size, fallback: 0.0 + def commit_count root_ref ? raw_repository.commit_count(root_ref) : 0 end @@ -691,7 +697,7 @@ class Repository @head_tree ||= Tree.new(self, root_ref, nil, skip_flat_paths: skip_flat_paths) end - def tree(sha = :head, path = nil, recursive: false, skip_flat_paths: true, pagination_params: nil, ref_type: nil) + def tree(sha = :head, path = nil, recursive: false, skip_flat_paths: true, pagination_params: nil, ref_type: nil, rescue_not_found: true) if sha == :head return if empty? || root_ref.nil? @@ -703,7 +709,7 @@ class Repository end end - Tree.new(self, sha, path, recursive: recursive, skip_flat_paths: skip_flat_paths, pagination_params: pagination_params, ref_type: ref_type) + Tree.new(self, sha, path, recursive: recursive, skip_flat_paths: skip_flat_paths, pagination_params: pagination_params, ref_type: ref_type, rescue_not_found: rescue_not_found) end def blob_at_branch(branch_name, path) @@ -1242,6 +1248,20 @@ class Repository prohibited_branches.each { |name| raw_repository.delete_branch(name) } end + def get_patch_id(old_revision, new_revision) + raw_repository.get_patch_id(old_revision, new_revision) + end + + def object_pool + gitaly_object_pool = raw.object_pool + + return unless gitaly_object_pool + + source_project = project&.pool_repository&.source_project + + Gitlab::Git::ObjectPool.init_from_gitaly(gitaly_object_pool, source_project&.repository) + end + private def ancestor_cache_key(ancestor_id, descendant_id) diff --git a/app/models/review.rb b/app/models/review.rb index c621da3b03c..d47aaf027ce 100644 --- a/app/models/review.rb +++ b/app/models/review.rb @@ -32,3 +32,5 @@ class Review < ApplicationRecord merge_request.user_mentions.where.not(note_id: nil) end end + +Review.prepend_mod diff --git a/app/models/route.rb b/app/models/route.rb index f2fe1664f9e..652c33a673c 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class Route < ApplicationRecord +class Route < MainClusterwide::ApplicationRecord include CaseSensitivity include Gitlab::SQL::Pattern diff --git a/app/models/sent_notification.rb b/app/models/sent_notification.rb index c2fd8b20942..f3a0479d3b7 100644 --- a/app/models/sent_notification.rb +++ b/app/models/sent_notification.rb @@ -1,10 +1,6 @@ # frozen_string_literal: true class SentNotification < ApplicationRecord - include IgnorableColumns - - ignore_column %i[line_code note_type position], remove_with: '16.3', remove_after: '2023-07-22' - belongs_to :project belongs_to :noteable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations belongs_to :recipient, class_name: "User" diff --git a/app/models/service_desk/custom_email_verification.rb b/app/models/service_desk/custom_email_verification.rb index 482a10447ed..5099cf4c5bb 100644 --- a/app/models/service_desk/custom_email_verification.rb +++ b/app/models/service_desk/custom_email_verification.rb @@ -26,6 +26,8 @@ module ServiceDesk validates :project, presence: true validates :state, presence: true + scope :overdue, -> { where('triggered_at < ?', TIMEFRAME.ago) } + delegate :service_desk_setting, to: :project state_machine :state do diff --git a/app/models/system/broadcast_message.rb b/app/models/system/broadcast_message.rb new file mode 100644 index 00000000000..332baea4449 --- /dev/null +++ b/app/models/system/broadcast_message.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +module System + class BroadcastMessage < MainClusterwide::ApplicationRecord + include CacheMarkdownField + include Sortable + + ALLOWED_TARGET_ACCESS_LEVELS = [ + Gitlab::Access::GUEST, + Gitlab::Access::REPORTER, + Gitlab::Access::DEVELOPER, + Gitlab::Access::MAINTAINER, + Gitlab::Access::OWNER + ].freeze + + cache_markdown_field :message, pipeline: :broadcast_message, whitelisted: true + + validates :message, presence: true + validates :starts_at, presence: true + validates :ends_at, presence: true + validates :broadcast_type, presence: true + validates :target_access_levels, inclusion: { in: ALLOWED_TARGET_ACCESS_LEVELS } + validates :show_in_cli, allow_nil: false, inclusion: { in: [true, false], message: N_('must be a boolean value') } + + validates :color, allow_blank: true, color: true + validates :font, allow_blank: true, color: true + + attribute :color, default: '#E75E40' + attribute :font, default: '#FFFFFF' + + scope :current_and_future_messages, -> { where('ends_at > :now', now: Time.current).order_id_asc } + + CACHE_KEY = 'broadcast_message_current_json' + BANNER_CACHE_KEY = 'broadcast_message_current_banner_json' + NOTIFICATION_CACHE_KEY = 'broadcast_message_current_notification_json' + + after_commit :flush_redis_cache + + enum theme: { + indigo: 0, + 'light-indigo': 1, + blue: 2, + 'light-blue': 3, + green: 4, + 'light-green': 5, + red: 6, + 'light-red': 7, + dark: 8, + light: 9 + }, _default: 0, _prefix: true + + enum broadcast_type: { + banner: 1, + notification: 2 + } + + class << self + def current_banner_messages(current_path: nil, user_access_level: nil) + fetch_messages BANNER_CACHE_KEY, current_path, user_access_level do + current_and_future_messages.banner + end + end + + def current_show_in_cli_banner_messages + current_banner_messages.select(&:show_in_cli?) + end + + def current_notification_messages(current_path: nil, user_access_level: nil) + fetch_messages NOTIFICATION_CACHE_KEY, current_path, user_access_level do + current_and_future_messages.notification + end + end + + def current(current_path: nil, user_access_level: nil) + fetch_messages CACHE_KEY, current_path, user_access_level do + current_and_future_messages + end + end + + def cache + ::Gitlab::SafeRequestStore.fetch(:broadcast_message_json_cache) do + Gitlab::Cache::JsonCaches::JsonKeyed.new + end + end + + def cache_expires_in + 2.weeks + end + + private + + def fetch_messages(cache_key, current_path, user_access_level, &block) + messages = cache.fetch(cache_key, as: System::BroadcastMessage, expires_in: cache_expires_in, &block) + + now_or_future = messages.select(&:now_or_future?) + + # If there are cached entries but they don't match the ones we are + # displaying we'll refresh the cache so we don't need to keep filtering. + cache.expire(cache_key) if now_or_future != messages + + messages = now_or_future.select(&:now?) + messages = messages.select do |message| + message.matches_current_user_access_level?(user_access_level) + end + messages.select do |message| + message.matches_current_path(current_path) + end + end + end + + def active? + started? && !ended? + end + + def started? + Time.current >= starts_at + end + + def ended? + ends_at < Time.current + end + + def now? + (starts_at..ends_at).cover?(Time.current) + end + + def future? + starts_at > Time.current + end + + def now_or_future? + now? || future? + end + + def matches_current_user_access_level?(user_access_level) + return true unless target_access_levels.present? + + target_access_levels.include? user_access_level + end + + def matches_current_path(current_path) + return false if current_path.blank? && target_path.present? + return true if current_path.blank? || target_path.blank? + + # Ensure paths are consistent across callers. + # This fixes a mismatch between requests in the GUI and CLI + # + # This has to be reassigned due to frozen strings being provided. + current_path = "/#{current_path}" unless current_path.start_with?("/") + + escaped = Regexp.escape(target_path).gsub('\\*', '.*') + regexp = Regexp.new "^#{escaped}$", Regexp::IGNORECASE + + regexp.match(current_path) + end + + def flush_redis_cache + [CACHE_KEY, BANNER_CACHE_KEY, NOTIFICATION_CACHE_KEY].each do |key| + self.class.cache.expire(key) + end + end + end +end + +System::BroadcastMessage.prepend_mod_with('System::BroadcastMessage') diff --git a/app/models/todo.rb b/app/models/todo.rb index f202e1a266d..d159b51a0eb 100644 --- a/app/models/todo.rb +++ b/app/models/todo.rb @@ -24,6 +24,7 @@ class Todo < ApplicationRecord MERGE_TRAIN_REMOVED = 8 # This is an EE-only feature REVIEW_REQUESTED = 9 MEMBER_ACCESS_REQUESTED = 10 + REVIEW_SUBMITTED = 11 # This is an EE-only feature ACTION_NAMES = { ASSIGNED => :assigned, @@ -35,7 +36,8 @@ class Todo < ApplicationRecord UNMERGEABLE => :unmergeable, DIRECTLY_ADDRESSED => :directly_addressed, MERGE_TRAIN_REMOVED => :merge_train_removed, - MEMBER_ACCESS_REQUESTED => :member_access_requested + MEMBER_ACCESS_REQUESTED => :member_access_requested, + REVIEW_SUBMITTED => :review_submitted }.freeze ACTIONS_MULTIPLE_ALLOWED = [Todo::MENTIONED, Todo::DIRECTLY_ADDRESSED, Todo::MEMBER_ACCESS_REQUESTED].freeze @@ -223,6 +225,10 @@ class Todo < ApplicationRecord action == MEMBER_ACCESS_REQUESTED end + def review_submitted? + action == REVIEW_SUBMITTED + end + def member_access_type target.class.name.downcase end diff --git a/app/models/tree.rb b/app/models/tree.rb index 8622eb793c1..4d62334800d 100644 --- a/app/models/tree.rb +++ b/app/models/tree.rb @@ -7,7 +7,7 @@ class Tree def initialize( repository, sha, path = '/', recursive: false, skip_flat_paths: true, pagination_params: nil, - ref_type: nil) + ref_type: nil, rescue_not_found: true) path = '/' if path.blank? @repository = repository @@ -18,7 +18,9 @@ class Tree ref = ExtractsRef.qualify_ref(@sha, ref_type) - @entries, @cursor = Gitlab::Git::Tree.where(git_repo, ref, @path, recursive, skip_flat_paths, pagination_params) + @entries, @cursor = Gitlab::Git::Tree.where(git_repo, ref, @path, recursive, skip_flat_paths, rescue_not_found, + pagination_params) + @entries.each do |entry| entry.ref_type = self.ref_type end diff --git a/app/models/user.rb b/app/models/user.rb index 4a57cc2e2e2..9f85d41b133 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -2,7 +2,7 @@ require 'carrierwave/orm/activerecord' -class User < ApplicationRecord +class User < MainClusterwide::ApplicationRecord extend Gitlab::ConfigHelper include Gitlab::ConfigHelper @@ -403,6 +403,7 @@ class User < ApplicationRecord delegate :location, :location=, to: :user_detail, allow_nil: true delegate :organization, :organization=, to: :user_detail, allow_nil: true delegate :discord, :discord=, to: :user_detail, allow_nil: true + delegate :email_reset_offered_at, :email_reset_offered_at=, to: :user_detail, allow_nil: true accepts_nested_attributes_for :user_preference, update_only: true accepts_nested_attributes_for :user_detail, update_only: true @@ -520,7 +521,11 @@ class User < ApplicationRecord scope :active, -> { with_state(:active).non_internal } scope :active_without_ghosts, -> { with_state(:active).without_ghosts } scope :deactivated, -> { with_state(:deactivated).non_internal } - scope :without_projects, -> { joins('LEFT JOIN project_authorizations ON users.id = project_authorizations.user_id').where(project_authorizations: { user_id: nil }) } + scope :without_projects, -> do + joins('LEFT JOIN project_authorizations ON users.id = project_authorizations.user_id') + .where(project_authorizations: { user_id: nil }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/422045') + end scope :by_username, -> (usernames) { iwhere(username: Array(usernames).map(&:to_s)) } scope :by_name, -> (names) { iwhere(name: Array(names)) } scope :by_login, -> (login) do @@ -1765,13 +1770,7 @@ class User < ApplicationRecord def following_users_allowed?(user) return false if self.id == user.id - following_users_enabled? && user.following_users_enabled? - end - - def following_users_enabled? - return true unless ::Feature.enabled?(:disable_follow_users, self) - - enabled_following + enabled_following && user.enabled_following end def forkable_namespaces @@ -2192,14 +2191,6 @@ class User < ApplicationRecord callout_dismissed?(callout, ignore_dismissal_earlier_than) end - def dismissed_callout_before?(feature_name, dismissed_before) - callout = callouts_by_feature_name[feature_name] - - return false unless callout - - callout.dismissed_before?(dismissed_before) - end - def dismissed_callout_for_group?(feature_name:, group:, ignore_dismissal_earlier_than: nil) source_feature_name = "#{feature_name}_#{group.id}" callout = group_callouts_by_feature_name[source_feature_name] diff --git a/app/models/user_detail.rb b/app/models/user_detail.rb index 5c9a73571c0..9ac814eebda 100644 --- a/app/models/user_detail.rb +++ b/app/models/user_detail.rb @@ -1,11 +1,10 @@ # frozen_string_literal: true -class UserDetail < ApplicationRecord +class UserDetail < MainClusterwide::ApplicationRecord include IgnorableColumns extend ::Gitlab::Utils::Override ignore_column :requires_credit_card_verification, remove_with: '16.1', remove_after: '2023-06-22' - ignore_column :provisioned_by_group_at, remove_with: '16.3', remove_after: '2023-07-22' REGISTRATION_OBJECTIVE_PAIRS = { basics: 0, move_repository: 1, code_storage: 2, exploring: 3, ci: 4, other: 5, joining_team: 6 }.freeze diff --git a/app/models/user_preference.rb b/app/models/user_preference.rb index c263d552d40..eac66905d0c 100644 --- a/app/models/user_preference.rb +++ b/app/models/user_preference.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class UserPreference < ApplicationRecord +class UserPreference < MainClusterwide::ApplicationRecord include IgnorableColumns # We could use enums, but Rails 4 doesn't support multiple diff --git a/app/models/user_status.rb b/app/models/user_status.rb index da24ef47a2a..35aa2427442 100644 --- a/app/models/user_status.rb +++ b/app/models/user_status.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class UserStatus < ApplicationRecord +class UserStatus < MainClusterwide::ApplicationRecord include CacheMarkdownField self.primary_key = :user_id diff --git a/app/models/user_synced_attributes_metadata.rb b/app/models/user_synced_attributes_metadata.rb index 6b23bce6406..0856febf3f6 100644 --- a/app/models/user_synced_attributes_metadata.rb +++ b/app/models/user_synced_attributes_metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class UserSyncedAttributesMetadata < ApplicationRecord +class UserSyncedAttributesMetadata < MainClusterwide::ApplicationRecord belongs_to :user validates :user, presence: true diff --git a/app/models/users/callout.rb b/app/models/users/callout.rb index 0d02a3b99aa..0d3262b2474 100644 --- a/app/models/users/callout.rb +++ b/app/models/users/callout.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Users - class Callout < ApplicationRecord + class Callout < MainClusterwide::ApplicationRecord include Users::Calloutable self.table_name = 'user_callouts' diff --git a/app/models/users/calloutable.rb b/app/models/users/calloutable.rb index 483d0d785a5..280a819e4d5 100644 --- a/app/models/users/calloutable.rb +++ b/app/models/users/calloutable.rb @@ -13,9 +13,5 @@ module Users def dismissed_after?(dismissed_after) dismissed_at > dismissed_after end - - def dismissed_before?(dismissed_before) - dismissed_at < dismissed_before - end end end diff --git a/app/models/wiki_page.rb b/app/models/wiki_page.rb index e1468872f52..a7e2be0eae5 100644 --- a/app/models/wiki_page.rb +++ b/app/models/wiki_page.rb @@ -284,10 +284,9 @@ class WikiPage def content_changed? if persisted? - # gollum-lib always converts CRLFs to LFs in Gollum::Wiki#normalize, - # so we need to do the same here. - # Also see https://gitlab.com/gitlab-org/gitlab/-/issues/21431 - raw_content.delete("\r") != page&.text_data + # To avoid end-of-line differences depending if Git is enforcing CRLF or not, + # we compare just the Wiki Content. + raw_content.lines(chomp: true) != page&.text_data&.lines(chomp: true) else raw_content.present? end diff --git a/app/models/work_item.rb b/app/models/work_item.rb index adf424a1d94..73156b2f040 100644 --- a/app/models/work_item.rb +++ b/app/models/work_item.rb @@ -22,6 +22,18 @@ class WorkItem < Issue foreign_key: :work_item_id, source: :work_item scope :inc_relations_for_permission_check, -> { includes(:author, project: :project_feature) } + scope :in_namespaces, ->(namespaces) { where(namespace: namespaces) } + + scope :with_confidentiality_check, ->(user) { + confidential_query = <<~SQL + issues.confidential = FALSE + OR (issues.confidential = TRUE + AND (issues.author_id = :user_id + OR EXISTS (SELECT TRUE FROM issue_assignees WHERE user_id = :user_id AND issue_id = issues.id))) + SQL + + where(confidential_query, user_id: user.id) + } class << self def assignee_association_name @@ -59,6 +71,11 @@ class WorkItem < Issue includes(:parent_link).order(keyset_order) end + + override :related_link_class + def related_link_class + WorkItems::RelatedWorkItemLink + end end def noteable_target_type_name diff --git a/app/models/work_items/parent_link.rb b/app/models/work_items/parent_link.rb index 5dff9e8e8d5..d9e3690b6fc 100644 --- a/app/models/work_items/parent_link.rb +++ b/app/models/work_items/parent_link.rb @@ -19,8 +19,10 @@ module WorkItems validate :validate_same_project validate :validate_max_children validate :validate_confidentiality + validate :check_existing_related_link scope :for_parents, ->(parent_ids) { where(work_item_parent_id: parent_ids) } + scope :for_children, ->(children_ids) { where(work_item: children_ids) } class << self def has_public_children?(parent_id) @@ -109,5 +111,14 @@ module WorkItems errors.add :work_item, _('is already present in ancestors') end end + + def check_existing_related_link + return unless work_item && work_item_parent + + existing_link = WorkItems::RelatedWorkItemLink.for_items(work_item, work_item_parent) + return if existing_link.none? + + errors.add(:work_item, _('cannot assign a linked work item as a parent')) + end end end diff --git a/app/models/work_items/related_work_item_link.rb b/app/models/work_items/related_work_item_link.rb new file mode 100644 index 00000000000..4de197d3d35 --- /dev/null +++ b/app/models/work_items/related_work_item_link.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module WorkItems + class RelatedWorkItemLink < ApplicationRecord + include LinkableItem + + self.table_name = 'issue_links' + + belongs_to :source, class_name: 'WorkItem' + belongs_to :target, class_name: 'WorkItem' + + class << self + extend ::Gitlab::Utils::Override + + # Used as issuable table name for calculating blocked and blocking count in IssuableLink + override :issuable_type + def issuable_type + :issue + end + + override :issuable_name + def issuable_name + 'work item' + end + end + end +end diff --git a/app/models/work_items/type.rb b/app/models/work_items/type.rb index 6a619dbab21..369ffc660aa 100644 --- a/app/models/work_items/type.rb +++ b/app/models/work_items/type.rb @@ -19,7 +19,9 @@ module WorkItems requirement: 'Requirement', task: 'Task', objective: 'Objective', - key_result: 'Key Result' + key_result: 'Key Result', + epic: 'Epic', + ticket: 'Ticket' }.freeze # Base types need to exist on the DB on app startup @@ -32,7 +34,9 @@ module WorkItems requirement: { name: TYPE_NAMES[:requirement], icon_name: 'issue-type-requirements', enum_value: 3 }, ## EE-only task: { name: TYPE_NAMES[:task], icon_name: 'issue-type-task', enum_value: 4 }, objective: { name: TYPE_NAMES[:objective], icon_name: 'issue-type-objective', enum_value: 5 }, ## EE-only - key_result: { name: TYPE_NAMES[:key_result], icon_name: 'issue-type-keyresult', enum_value: 6 } ## EE-only + key_result: { name: TYPE_NAMES[:key_result], icon_name: 'issue-type-keyresult', enum_value: 6 }, ## EE-only + epic: { name: TYPE_NAMES[:epic], icon_name: 'issue-type-epic', enum_value: 7 }, ## EE-only + ticket: { name: TYPE_NAMES[:ticket], icon_name: 'issue-type-issue', enum_value: 8 } }.freeze # A list of types user can change between - both original and new @@ -40,7 +44,7 @@ module WorkItems # where it's possible to switch between issue and incident. CHANGEABLE_BASE_TYPES = %w[issue incident test_case].freeze - WI_TYPES_WITH_CREATED_HEADER = %w[issue incident].freeze + WI_TYPES_WITH_CREATED_HEADER = %w[issue incident ticket].freeze cache_markdown_field :description, pipeline: :single_line @@ -79,7 +83,7 @@ module WorkItems end def self.allowed_types_for_issues - base_types.keys.excluding('task', 'objective', 'key_result') + base_types.keys.excluding('task', 'objective', 'key_result', 'epic', 'ticket') end def default? diff --git a/app/models/work_items/widget_definition.rb b/app/models/work_items/widget_definition.rb index 763b1a79069..f25c951406f 100644 --- a/app/models/work_items/widget_definition.rb +++ b/app/models/work_items/widget_definition.rb @@ -31,7 +31,8 @@ module WorkItems test_reports: 13, # EE-only notifications: 14, current_user_todos: 15, - award_emoji: 16 + award_emoji: 16, + linked_items: 17 } def self.available_widgets diff --git a/app/models/work_items/widgets/linked_items.rb b/app/models/work_items/widgets/linked_items.rb new file mode 100644 index 00000000000..06a0f6db964 --- /dev/null +++ b/app/models/work_items/widgets/linked_items.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module WorkItems + module Widgets + class LinkedItems < Base + delegate :related_issues, to: :work_item + end + end +end |