From db384e6b19af03b4c3c82a5760d83a3fd79f7982 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 18 Aug 2023 10:50:51 +0000 Subject: Add latest changes from gitlab-org/gitlab@16-3-stable-ee --- ..._default_branch_protection_namespace_setting.rb | 131 +++++++++++++++++++++ ...fill_dismissal_reason_in_vulnerability_reads.rb | 19 +++ ...fill_missing_vulnerability_dismissal_details.rb | 17 +++ .../backfill_namespace_id_for_project_route.rb | 1 + ...age_size_without_pipeline_artifacts_size_job.rb | 95 +++++++++++++++ .../cleanup_orphaned_routes.rb | 34 +++--- ..._orphaned_transferred_project_approval_rules.rb | 14 +++ ...descendants_override_disabled_shared_runners.rb | 22 ++++ ...llify_creator_id_column_of_orphaned_projects.rb | 8 +- .../populate_projects_star_count.rb | 31 ++--- 10 files changed, 342 insertions(+), 30 deletions(-) create mode 100644 lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb create mode 100644 lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb create mode 100644 lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb create mode 100644 lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb create mode 100644 lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb create mode 100644 lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb (limited to 'lib/gitlab/background_migration') diff --git a/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb b/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb new file mode 100644 index 00000000000..8da29a61d61 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is used to update the default_branch_protection_defaults column + # for user namespaces of the namespace_settings table. + class BackfillDefaultBranchProtectionNamespaceSetting < BatchedMigrationJob + operation_name :set_default_branch_protection_defaults + feature_category :database + + # Migration only version of `namespaces` table + class Namespace < ::ApplicationRecord + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + has_one :namespace_setting, + class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::NamespaceSetting' + end + + # Migration only version of `namespace_settings` table + class NamespaceSetting < ::ApplicationRecord + self.table_name = 'namespace_settings' + belongs_to :namespace, + class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::Namespace' + end + + # Migration only version of Gitlab::Access:BranchProtection application code. + class BranchProtection + attr_reader :level + + def initialize(level) + @level = level + end + + PROTECTION_NONE = 0 + PROTECTION_DEV_CAN_PUSH = 1 + PROTECTION_FULL = 2 + PROTECTION_DEV_CAN_MERGE = 3 + PROTECTION_DEV_CAN_INITIAL_PUSH = 4 + + DEVELOPER = 30 + MAINTAINER = 40 + + def to_hash + case level + when PROTECTION_NONE + self.class.protection_none + when PROTECTION_DEV_CAN_PUSH + self.class.protection_partial + when PROTECTION_FULL + self.class.protected_fully + when PROTECTION_DEV_CAN_MERGE + self.class.protected_against_developer_pushes + when PROTECTION_DEV_CAN_INITIAL_PUSH + self.class.protected_after_initial_push + end + end + + class << self + def protection_none + { + allowed_to_push: [{ 'access_level' => DEVELOPER }], + allowed_to_merge: [{ 'access_level' => DEVELOPER }], + allow_force_push: true + } + end + + def protection_partial + protection_none.merge(allow_force_push: false) + end + + def protected_fully + { + allowed_to_push: [{ 'access_level' => MAINTAINER }], + allowed_to_merge: [{ 'access_level' => MAINTAINER }], + allow_force_push: false + } + end + + def protected_against_developer_pushes + { + allowed_to_push: [{ 'access_level' => MAINTAINER }], + allowed_to_merge: [{ 'access_level' => DEVELOPER }], + allow_force_push: true + } + end + + def protected_after_initial_push + { + allowed_to_push: [{ 'access_level' => MAINTAINER }], + allowed_to_merge: [{ 'access_level' => DEVELOPER }], + allow_force_push: true, + developer_can_initial_push: true + } + end + end + end + + def perform + each_sub_batch do |sub_batch| + update_default_protection_branch_defaults(sub_batch) + end + end + + private + + def update_default_protection_branch_defaults(batch) + namespace_settings = NamespaceSetting.where(namespace_id: batch.pluck(:namespace_id)).includes(:namespace) + + values_list = namespace_settings.map do |namespace_setting| + level = namespace_setting.namespace.default_branch_protection.to_i + value = BranchProtection.new(level).to_hash.to_json + "(#{namespace_setting.namespace_id}, '#{value}'::jsonb)" + end.join(", ") + + sql = <<~SQL + WITH new_values (namespace_id, default_branch_protection_defaults) AS ( + VALUES + #{values_list} + ) + UPDATE namespace_settings + SET default_branch_protection_defaults = new_values.default_branch_protection_defaults + FROM new_values + WHERE namespace_settings.namespace_id = new_values.namespace_id; + SQL + + connection.execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb new file mode 100644 index 00000000000..d7972a6a7a9 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This batched background migration is EE-only, + # see ee/lib/ee/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb for the actual + # migration code. + # + # This batched background migration will backfill `dismissal_reason` field in `vulnerability_reads` table for + # records with `state: 2` and `dismissal_reason: null`. + class BackfillDismissalReasonInVulnerabilityReads < BatchedMigrationJob + feature_category :vulnerability_management + + def perform; end + end + end +end + +Gitlab::BackgroundMigration::BackfillDismissalReasonInVulnerabilityReads.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb b/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb new file mode 100644 index 00000000000..8399f53b724 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillMissingVulnerabilityDismissalDetails < BatchedMigrationJob + feature_category :vulnerability_management + + def perform + # no-op. The logic is defined in EE module. + end + end + # rubocop: enable Style/Documentation + end +end + +::Gitlab::BackgroundMigration::BackfillMissingVulnerabilityDismissalDetails.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb index 0282531ae17..b0b7882d54d 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb @@ -54,6 +54,7 @@ module Gitlab .where(namespace_id: nil) .where(source_type: 'Project') .where.not(projects: { project_namespace_id: nil }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') .select("routes.id, projects.project_namespace_id") end end diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb new file mode 100644 index 00000000000..88d0f27282a --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Layout/LineLength + class Project < ::ApplicationRecord + self.table_name = 'projects' + + has_one :statistics, class_name: '::Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob::ProjectStatistics' # rubocop:disable Layout/LineLength + end + + class ProjectStatistics < ::ApplicationRecord + include ::EachBatch + + self.table_name = 'project_statistics' + + belongs_to :project, class_name: '::Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob::Project' # rubocop:disable Layout/LineLength + + def update_storage_size(storage_size_components) + new_storage_size = storage_size_components.sum { |component| method(component).call } + + # Only update storage_size if storage_size needs updating + return unless storage_size != new_storage_size + + self.storage_size = new_storage_size + save! + + ::Namespaces::ScheduleAggregationWorker.perform_async(project.namespace_id) + log_with_data('Scheduled Namespaces::ScheduleAggregationWorker') + end + + def wiki_size + super.to_i + end + + def snippets_size + super.to_i + end + + private + + def log_with_data(log_line) + log_info( + log_line, + project_id: project.id, + pipeline_artifacts_size: pipeline_artifacts_size, + storage_size: storage_size, + namespace_id: project.namespace_id + ) + end + + def log_info(message, **extra) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: 'BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob', + message: message, + **extra + ) + end + end + + scope_to ->(relation) { + relation.where.not(pipeline_artifacts_size: 0) + } + operation_name :update_storage_size + feature_category :consumables_cost_management + + def perform + each_sub_batch do |sub_batch| + ProjectStatistics.merge(sub_batch).each do |statistics| + statistics.update_storage_size(storage_size_components) + end + end + end + + private + + # Overridden in EE + def storage_size_components + [ + :repository_size, + :wiki_size, + :lfs_objects_size, + :build_artifacts_size, + :packages_size, + :snippets_size, + :uploads_size + ] + end + end + # rubocop:enable Style/Documentation + end +end + +Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob.prepend_mod diff --git a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb index 5c0ddf0ba8b..c221f8ea411 100644 --- a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb +++ b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb @@ -35,25 +35,31 @@ module Gitlab end def update_namespace_id(batch_column, non_orphaned_namespace_routes, sub_batch_size) - non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:fix_missing_namespace_id) do - ApplicationRecord.connection.execute <<~SQL - WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - #{sub_batch.to_sql} - ) - UPDATE routes - SET namespace_id = route_and_ns.namespace_id - FROM route_and_ns - WHERE id = route_and_ns.route_id - SQL + Gitlab::Database.allow_cross_joins_across_databases( + url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") do + non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:fix_missing_namespace_id) do + ApplicationRecord.connection.execute <<~SQL + WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + #{sub_batch.to_sql} + ) + UPDATE routes + SET namespace_id = route_and_ns.namespace_id + FROM route_and_ns + WHERE id = route_and_ns.route_id + SQL + end end end end def cleanup_relations(batch_column, orphaned_namespace_routes, pause_ms, sub_batch_size) - orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:cleanup_orphaned_routes) do - sub_batch.delete_all + Gitlab::Database.allow_cross_joins_across_databases( + url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") do + orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:cleanup_orphaned_routes) do + sub_batch.delete_all + end end end end diff --git a/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb b/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb new file mode 100644 index 00000000000..c0f87644b59 --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting orphaned approval project rules for projects that were transferred + class DeleteOrphanedTransferredProjectApprovalRules < BatchedMigrationJob + feature_category :security_policy_management + + def perform; end + end + end +end + +Gitlab::BackgroundMigration::DeleteOrphanedTransferredProjectApprovalRules.prepend_mod diff --git a/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb new file mode 100644 index 00000000000..44bda3fe2b6 --- /dev/null +++ b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fixes invalid combination of shared runners being enabled and + # allow_descendants_override = true + # This combination fails validation and doesn't make sense: + # we always allow descendants to disable shared runners + class FixAllowDescendantsOverrideDisabledSharedRunners < BatchedMigrationJob + feature_category :runner_fleet + operation_name :fix_allow_descendants_override_disabled_shared_runners + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(shared_runners_enabled: true, + allow_descendants_override_disabled_shared_runners: true) + .update_all(allow_descendants_override_disabled_shared_runners: false) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb index 74f5bc3f725..e1ea0c66ad4 100644 --- a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb +++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb @@ -9,14 +9,18 @@ module Gitlab relation.where.not(creator_id: nil) .joins('LEFT OUTER JOIN users ON users.id = projects.creator_id') .where(users: { id: nil }) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') end operation_name :update_all feature_category :groups_and_projects def perform - each_sub_batch do |sub_batch| - sub_batch.update_all(creator_id: nil) + ::Gitlab::Database.allow_cross_joins_across_databases(url: + 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') do + each_sub_batch do |sub_batch| + sub_batch.update_all(creator_id: nil) + end end end end diff --git a/lib/gitlab/background_migration/populate_projects_star_count.rb b/lib/gitlab/background_migration/populate_projects_star_count.rb index 8417dc91b1b..0790bd98018 100644 --- a/lib/gitlab/background_migration/populate_projects_star_count.rb +++ b/lib/gitlab/background_migration/populate_projects_star_count.rb @@ -39,20 +39,23 @@ module Gitlab # rubocop:enable Database/RescueStatementTimeout def update_batch(sub_batch) - ApplicationRecord.connection.execute <<~SQL - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql}) - UPDATE projects - SET star_count = ( - SELECT COUNT(*) - FROM users_star_projects - INNER JOIN users - ON users_star_projects.user_id = users.id - WHERE users_star_projects.project_id = batched_relation.id - AND users.state = 'active' - ) - FROM batched_relation - WHERE projects.id = batched_relation.id - SQL + ::Gitlab::Database.allow_cross_joins_across_databases(url: + 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') do + ApplicationRecord.connection.execute <<~SQL + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql}) + UPDATE projects + SET star_count = ( + SELECT COUNT(*) + FROM users_star_projects + INNER JOIN users + ON users_star_projects.user_id = users.id + WHERE users_star_projects.project_id = batched_relation.id + AND users.state = 'active' + ) + FROM batched_relation + WHERE projects.id = batched_relation.id + SQL + end end end end -- cgit v1.2.3