Add latest changes from gitlab-org/gitlab@16-3-stable-eev16.3.0-rc42

10 files changed, 342 insertions, 30 deletions

diff --git a/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb b/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb
new file mode 100644
index 00000000000..8da29a61d61
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_default_branch_protection_namespace_setting.rb
@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class is used to update the default_branch_protection_defaults column
+    # for user namespaces of the namespace_settings table.
+    class BackfillDefaultBranchProtectionNamespaceSetting < BatchedMigrationJob
+      operation_name :set_default_branch_protection_defaults
+      feature_category :database
+
+      # Migration only version of `namespaces` table
+      class Namespace < ::ApplicationRecord
+        self.table_name = 'namespaces'
+        self.inheritance_column = :_type_disabled
+
+        has_one :namespace_setting,
+          class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::NamespaceSetting'
+      end
+
+      # Migration only version of `namespace_settings` table
+      class NamespaceSetting < ::ApplicationRecord
+        self.table_name = 'namespace_settings'
+        belongs_to :namespace,
+          class_name: '::Gitlab::BackgroundMigration::BackfillDefaultBranchProtectionNamespaceSetting::Namespace'
+      end
+
+      # Migration only version of Gitlab::Access:BranchProtection application code.
+      class BranchProtection
+        attr_reader :level
+
+        def initialize(level)
+          @level = level
+        end
+
+        PROTECTION_NONE = 0
+        PROTECTION_DEV_CAN_PUSH = 1
+        PROTECTION_FULL = 2
+        PROTECTION_DEV_CAN_MERGE = 3
+        PROTECTION_DEV_CAN_INITIAL_PUSH = 4
+
+        DEVELOPER = 30
+        MAINTAINER = 40
+
+        def to_hash
+          case level
+          when PROTECTION_NONE
+            self.class.protection_none
+          when PROTECTION_DEV_CAN_PUSH
+            self.class.protection_partial
+          when PROTECTION_FULL
+            self.class.protected_fully
+          when PROTECTION_DEV_CAN_MERGE
+            self.class.protected_against_developer_pushes
+          when PROTECTION_DEV_CAN_INITIAL_PUSH
+            self.class.protected_after_initial_push
+          end
+        end
+
+        class << self
+          def protection_none
+            {
+              allowed_to_push: [{ 'access_level' => DEVELOPER }],
+              allowed_to_merge: [{ 'access_level' => DEVELOPER }],
+              allow_force_push: true
+            }
+          end
+
+          def protection_partial
+            protection_none.merge(allow_force_push: false)
+          end
+
+          def protected_fully
+            {
+              allowed_to_push: [{ 'access_level' => MAINTAINER }],
+              allowed_to_merge: [{ 'access_level' => MAINTAINER }],
+              allow_force_push: false
+            }
+          end
+
+          def protected_against_developer_pushes
+            {
+              allowed_to_push: [{ 'access_level' => MAINTAINER }],
+              allowed_to_merge: [{ 'access_level' => DEVELOPER }],
+              allow_force_push: true
+            }
+          end
+
+          def protected_after_initial_push
+            {
+              allowed_to_push: [{ 'access_level' => MAINTAINER }],
+              allowed_to_merge: [{ 'access_level' => DEVELOPER }],
+              allow_force_push: true,
+              developer_can_initial_push: true
+            }
+          end
+        end
+      end
+
+      def perform
+        each_sub_batch do |sub_batch|
+          update_default_protection_branch_defaults(sub_batch)
+        end
+      end
+
+      private
+
+      def update_default_protection_branch_defaults(batch)
+        namespace_settings = NamespaceSetting.where(namespace_id: batch.pluck(:namespace_id)).includes(:namespace)
+
+        values_list = namespace_settings.map do |namespace_setting|
+          level = namespace_setting.namespace.default_branch_protection.to_i
+          value = BranchProtection.new(level).to_hash.to_json
+          "(#{namespace_setting.namespace_id}, '#{value}'::jsonb)"
+        end.join(", ")
+
+        sql = <<~SQL
+          WITH new_values (namespace_id, default_branch_protection_defaults) AS (
+            VALUES
+              #{values_list}
+          )
+          UPDATE namespace_settings
+          SET default_branch_protection_defaults = new_values.default_branch_protection_defaults
+          FROM new_values
+          WHERE namespace_settings.namespace_id = new_values.namespace_id;
+        SQL
+
+        connection.execute(sql)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb
new file mode 100644
index 00000000000..d7972a6a7a9
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This batched background migration is EE-only,
+    # see ee/lib/ee/gitlab/background_migration/backfill_dismissal_reason_in_vulnerability_reads.rb for the actual
+    # migration code.
+    #
+    # This batched background migration will backfill `dismissal_reason` field in `vulnerability_reads` table for
+    # records with `state: 2` and `dismissal_reason: null`.
+    class BackfillDismissalReasonInVulnerabilityReads < BatchedMigrationJob
+      feature_category :vulnerability_management
+
+      def perform; end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::BackfillDismissalReasonInVulnerabilityReads.prepend_mod
diff --git a/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb b/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb
new file mode 100644
index 00000000000..8399f53b724
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_missing_vulnerability_dismissal_details.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class BackfillMissingVulnerabilityDismissalDetails < BatchedMigrationJob
+      feature_category :vulnerability_management
+
+      def perform
+        # no-op. The logic is defined in EE module.
+      end
+    end
+    # rubocop: enable Style/Documentation
+  end
+end
+
+::Gitlab::BackgroundMigration::BackfillMissingVulnerabilityDismissalDetails.prepend_mod
diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb
index 0282531ae17..b0b7882d54d 100644
--- a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb
+++ b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb
@@ -54,6 +54,7 @@ module Gitlab
           .where(namespace_id: nil)
           .where(source_type: 'Project')
           .where.not(projects: { project_namespace_id: nil })
+          .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843')
           .select("routes.id, projects.project_namespace_id")
       end
     end
diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb
new file mode 100644
index 00000000000..88d0f27282a
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_pipeline_artifacts_size_job.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop:disable Style/Documentation
+    class BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Layout/LineLength
+      class Project < ::ApplicationRecord
+        self.table_name = 'projects'
+
+        has_one :statistics, class_name: '::Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob::ProjectStatistics' # rubocop:disable Layout/LineLength
+      end
+
+      class ProjectStatistics < ::ApplicationRecord
+        include ::EachBatch
+
+        self.table_name = 'project_statistics'
+
+        belongs_to :project, class_name: '::Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob::Project' # rubocop:disable Layout/LineLength
+
+        def update_storage_size(storage_size_components)
+          new_storage_size = storage_size_components.sum { |component| method(component).call }
+
+          # Only update storage_size if storage_size needs updating
+          return unless storage_size != new_storage_size
+
+          self.storage_size = new_storage_size
+          save!
+
+          ::Namespaces::ScheduleAggregationWorker.perform_async(project.namespace_id)
+          log_with_data('Scheduled Namespaces::ScheduleAggregationWorker')
+        end
+
+        def wiki_size
+          super.to_i
+        end
+
+        def snippets_size
+          super.to_i
+        end
+
+        private
+
+        def log_with_data(log_line)
+          log_info(
+            log_line,
+            project_id: project.id,
+            pipeline_artifacts_size: pipeline_artifacts_size,
+            storage_size: storage_size,
+            namespace_id: project.namespace_id
+          )
+        end
+
+        def log_info(message, **extra)
+          ::Gitlab::BackgroundMigration::Logger.info(
+            migrator: 'BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob',
+            message: message,
+            **extra
+          )
+        end
+      end
+
+      scope_to ->(relation) {
+        relation.where.not(pipeline_artifacts_size: 0)
+      }
+      operation_name :update_storage_size
+      feature_category :consumables_cost_management
+
+      def perform
+        each_sub_batch do |sub_batch|
+          ProjectStatistics.merge(sub_batch).each do |statistics|
+            statistics.update_storage_size(storage_size_components)
+          end
+        end
+      end
+
+      private
+
+      # Overridden in EE
+      def storage_size_components
+        [
+          :repository_size,
+          :wiki_size,
+          :lfs_objects_size,
+          :build_artifacts_size,
+          :packages_size,
+          :snippets_size,
+          :uploads_size
+        ]
+      end
+    end
+    # rubocop:enable Style/Documentation
+  end
+end
+
+Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutPipelineArtifactsSizeJob.prepend_mod
diff --git a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb
index 5c0ddf0ba8b..c221f8ea411 100644
--- a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb
+++ b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb
@@ -35,25 +35,31 @@ module Gitlab
       end
 
       def update_namespace_id(batch_column, non_orphaned_namespace_routes, sub_batch_size)
-        non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
-          batch_metrics.time_operation(:fix_missing_namespace_id) do
-            ApplicationRecord.connection.execute <<~SQL
-              WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
-                #{sub_batch.to_sql}
-              )
-              UPDATE routes
-              SET namespace_id = route_and_ns.namespace_id
-              FROM route_and_ns
-              WHERE id = route_and_ns.route_id
-            SQL
+        Gitlab::Database.allow_cross_joins_across_databases(
+          url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") do
+          non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
+            batch_metrics.time_operation(:fix_missing_namespace_id) do
+              ApplicationRecord.connection.execute <<~SQL
+                WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
+                  #{sub_batch.to_sql}
+                )
+                UPDATE routes
+                SET namespace_id = route_and_ns.namespace_id
+                FROM route_and_ns
+                WHERE id = route_and_ns.route_id
+              SQL
+            end
           end
         end
       end
 
       def cleanup_relations(batch_column, orphaned_namespace_routes, pause_ms, sub_batch_size)
-        orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
-          batch_metrics.time_operation(:cleanup_orphaned_routes) do
-            sub_batch.delete_all
+        Gitlab::Database.allow_cross_joins_across_databases(
+          url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") do
+          orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
+            batch_metrics.time_operation(:cleanup_orphaned_routes) do
+              sub_batch.delete_all
+            end
           end
         end
       end
diff --git a/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb b/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb
new file mode 100644
index 00000000000..c0f87644b59
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_orphaned_transferred_project_approval_rules.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Background migration for deleting orphaned approval project rules for projects that were transferred
+    class DeleteOrphanedTransferredProjectApprovalRules < BatchedMigrationJob
+      feature_category :security_policy_management
+
+      def perform; end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::DeleteOrphanedTransferredProjectApprovalRules.prepend_mod
diff --git a/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb
new file mode 100644
index 00000000000..44bda3fe2b6
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_allow_descendants_override_disabled_shared_runners.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Fixes invalid combination of shared runners being enabled and
+    # allow_descendants_override = true
+    # This combination fails validation and doesn't make sense:
+    # we always allow descendants to disable shared runners
+    class FixAllowDescendantsOverrideDisabledSharedRunners < BatchedMigrationJob
+      feature_category :runner_fleet
+      operation_name :fix_allow_descendants_override_disabled_shared_runners
+
+      def perform
+        each_sub_batch do |sub_batch|
+          sub_batch.where(shared_runners_enabled: true,
+            allow_descendants_override_disabled_shared_runners: true)
+            .update_all(allow_descendants_override_disabled_shared_runners: false)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb
index 74f5bc3f725..e1ea0c66ad4 100644
--- a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb
+++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb
@@ -9,14 +9,18 @@ module Gitlab
         relation.where.not(creator_id: nil)
                 .joins('LEFT OUTER JOIN users ON users.id = projects.creator_id')
                 .where(users: { id: nil })
+                .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843')
       end
 
       operation_name :update_all
       feature_category :groups_and_projects
 
       def perform
-        each_sub_batch do |sub_batch|
-          sub_batch.update_all(creator_id: nil)
+        ::Gitlab::Database.allow_cross_joins_across_databases(url:
+          'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') do
+          each_sub_batch do |sub_batch|
+            sub_batch.update_all(creator_id: nil)
+          end
         end
       end
     end
diff --git a/lib/gitlab/background_migration/populate_projects_star_count.rb b/lib/gitlab/background_migration/populate_projects_star_count.rb
index 8417dc91b1b..0790bd98018 100644
--- a/lib/gitlab/background_migration/populate_projects_star_count.rb
+++ b/lib/gitlab/background_migration/populate_projects_star_count.rb
@@ -39,20 +39,23 @@ module Gitlab
       # rubocop:enable Database/RescueStatementTimeout
 
       def update_batch(sub_batch)
-        ApplicationRecord.connection.execute <<~SQL
-          WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql})
-          UPDATE projects
-          SET star_count = (
-            SELECT COUNT(*)
-            FROM users_star_projects
-            INNER JOIN users
-            ON users_star_projects.user_id = users.id
-            WHERE users_star_projects.project_id = batched_relation.id
-            AND users.state = 'active'
-          )
-          FROM batched_relation
-          WHERE projects.id = batched_relation.id
-        SQL
+        ::Gitlab::Database.allow_cross_joins_across_databases(url:
+            'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') do
+          ApplicationRecord.connection.execute <<~SQL
+            WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql})
+            UPDATE projects
+            SET star_count = (
+              SELECT COUNT(*)
+              FROM users_star_projects
+              INNER JOIN users
+              ON users_star_projects.user_id = users.id
+              WHERE users_star_projects.project_id = batched_relation.id
+              AND users.state = 'active'
+            )
+            FROM batched_relation
+            WHERE projects.id = batched_relation.id
+          SQL
+        end
       end
     end
   end