Add latest changes from gitlab-org/gitlab@15-9-stable-eev15.9.0-rc42

11 files changed, 294 insertions, 121 deletions

diff --git a/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb b/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb
new file mode 100644
index 00000000000..46e2d5cb930
--- /dev/null
+++ b/lib/gitlab/background_migration/add_namespaces_emails_enabled_column_data.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Iterates through the namespaces table and attempts to set the
+    # opposite of the value of the column "emails_disabled" to a new
+    # column in namespace_settings called emails_enabled
+    class AddNamespacesEmailsEnabledColumnData < BatchedMigrationJob
+      feature_category :database
+      operation_name :add_namespaces_emails_enabled_column_data
+
+      # Targeted table
+      class NamespaceSetting < ApplicationRecord
+        self.table_name = 'namespace_settings'
+      end
+
+      def perform
+        each_sub_batch do |sub_batch|
+          plucked_list = sub_batch.where('NOT emails_disabled IS NULL').pluck(:id, :emails_disabled)
+          next if plucked_list.empty?
+
+          ApplicationRecord.connection.execute <<~SQL
+                  UPDATE namespace_settings
+                  SET emails_enabled= NOT subquery.emails_enabled
+                  FROM (SELECT * FROM (#{Arel::Nodes::ValuesList.new(plucked_list).to_sql}) AS updates(namespace_id, emails_enabled)) AS subquery
+                  WHERE namespace_settings.namespace_id=subquery.namespace_id
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb b/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb
new file mode 100644
index 00000000000..a0ce5d22597
--- /dev/null
+++ b/lib/gitlab/background_migration/add_projects_emails_enabled_column_data.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Iterates through the Projects table and attempts to set the
+    # opposite of the value of the column "emails_disabled" to a new
+    # column in project_settings called emails_enabled
+    class AddProjectsEmailsEnabledColumnData < BatchedMigrationJob
+      feature_category :database
+      operation_name :add_projects_emails_enabled_column_data
+
+      # Targeted table
+      class ProjectSetting < ApplicationRecord
+        self.table_name = 'project_settings'
+      end
+
+      def perform
+        each_sub_batch do |sub_batch|
+          plucked_list = sub_batch.where('NOT emails_disabled IS NULL').pluck(:id, :emails_disabled)
+          next if plucked_list.empty?
+
+          ApplicationRecord.connection.execute <<~SQL
+                  UPDATE project_settings
+                  SET emails_enabled=NOT subquery.emails_enabled
+                  FROM (SELECT * FROM (#{Arel::Nodes::ValuesList.new(plucked_list).to_sql}) AS updates(project_id, emails_enabled)) AS subquery
+                  WHERE project_settings.project_id=subquery.project_id
+          SQL
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb
deleted file mode 100644
index 669e5338dd1..00000000000
--- a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb
+++ /dev/null
@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2,
-# it's enough at the moment to simply notice if the url is from `atlassian.net`
-module Gitlab
-  module BackgroundMigration
-    # Backfill the deployment_type in jira_tracker_data table
-    class BackfillJiraTrackerDeploymentType2
-      # Migration only version of jira_tracker_data table
-      class JiraTrackerDataTemp < ApplicationRecord
-        self.table_name = 'jira_tracker_data'
-
-        def self.encryption_options
-          {
-            key: Settings.attr_encrypted_db_key_base_32,
-            encode: true,
-            mode: :per_attribute_iv,
-            algorithm: 'aes-256-gcm'
-          }
-        end
-
-        attr_encrypted :url, encryption_options
-        attr_encrypted :api_url, encryption_options
-
-        enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment
-      end
-
-      # Migration only version of services table
-      class JiraServiceTemp < ApplicationRecord
-        self.table_name = 'services'
-        self.inheritance_column = :_type_disabled
-      end
-
-      def perform(start_id, stop_id)
-        @server_ids = []
-        @cloud_ids  = []
-
-        JiraTrackerDataTemp
-          .where(id: start_id..stop_id, deployment_type: 0)
-          .each do |jira_tracker_data|
-            collect_deployment_type(jira_tracker_data)
-          end
-
-        unless cloud_ids.empty?
-          JiraTrackerDataTemp.where(id: cloud_ids)
-            .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud])
-        end
-
-        unless server_ids.empty?
-          JiraTrackerDataTemp.where(id: server_ids)
-            .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server])
-        end
-
-        mark_jobs_as_succeeded(start_id, stop_id)
-      end
-
-      private
-
-      attr_reader :server_ids, :cloud_ids
-
-      def client_url(jira_tracker_data)
-        jira_tracker_data.api_url.presence || jira_tracker_data.url.presence
-      end
-
-      def server_type(url)
-        url.downcase.include?('.atlassian.net') ? :cloud : :server
-      end
-
-      def collect_deployment_type(jira_tracker_data)
-        url = client_url(jira_tracker_data)
-        return unless url
-
-        case server_type(url)
-        when :cloud
-          cloud_ids << jira_tracker_data.id
-        else
-          server_ids << jira_tracker_data.id
-        end
-      end
-
-      def mark_jobs_as_succeeded(*arguments)
-        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name.demodulize, arguments)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb
new file mode 100644
index 00000000000..b6e22e481fa
--- /dev/null
+++ b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Migration to make sure that all the prevously saved tokens have their encrypted values in the db.
+    class EncryptCiTriggerToken < Gitlab::BackgroundMigration::BatchedMigrationJob
+      feature_category :continuous_integration
+      scope_to ->(relation) { relation.where(encrypted_token: nil) }
+      operation_name :update
+      # Class that is imitating Ci::Trigger
+      class CiTrigger < ::Ci::ApplicationRecord
+        ALGORITHM = 'aes-256-gcm'
+
+        self.table_name = 'ci_triggers'
+
+        attr_encrypted :encrypted_token_tmp,
+          attribute: :encrypted_token,
+          mode: :per_attribute_iv,
+          algorithm: 'aes-256-gcm',
+          key: Settings.attr_encrypted_db_key_base_32,
+          encode: false,
+          encode_vi: false
+
+        before_save :copy_token_to_encrypted_token
+
+        def copy_token_to_encrypted_token
+          self.encrypted_token_tmp = token
+        end
+      end
+
+      def perform
+        each_sub_batch do |sub_batch|
+          sub_batch.each do |trigger|
+            Gitlab::BackgroundMigration::EncryptCiTriggerToken::CiTrigger.find(trigger.id).save!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb
index 3843356af69..c9582da2a51 100644
--- a/lib/gitlab/background_migration/encrypt_integration_properties.rb
+++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb
@@ -31,7 +31,7 @@ module Gitlab
         def encrypt_properties
           data = ::Gitlab::Json.parse(properties)
           iv = generate_iv(ALGORITHM)
-          ep = self.class.encrypt(:encrypted_properties_tmp, data, { iv: iv })
+          ep = self.class.attr_encrypt(:encrypted_properties_tmp, data, { iv: iv })
 
           [ep, iv]
         end
diff --git a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb
new file mode 100644
index 00000000000..4b6bb12c91b
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # A migration that will:
+    # * get all project statistics where packages_size is non zero
+    # * check the coherence with the related package files
+    # * fix non coherent packages_size values
+    class FixIncoherentPackagesSizeOnProjectStatistics < BatchedMigrationJob
+      MIGRATOR = 'FixIncoherentPackagesSizeOnProjectStatistics'
+
+      feature_category :package_registry
+
+      operation_name :fix_incorrect_packages_size
+
+      def perform
+        each_sub_batch do |sub_batch|
+          fix_packages_size(sub_batch)
+        end
+      end
+
+      private
+
+      def fix_packages_size(project_statistics)
+        statistics_table = FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics.arel_table
+        from = [
+          statistics_table,
+          FixIncoherentPackagesSizeOnProjectStatistics::PackageFile.sum_query.arel.lateral.as('size_sum')
+        ]
+        size_sum_table = ::Arel::Table.new(:size_sum)
+
+        project_statistics.select(:id, :project_id, :packages_size, size_sum_table[:total])
+          .from(from)
+          .where.not(statistics_table[:packages_size].eq(size_sum_table[:total]))
+          .each do |stat|
+          increment = stat[:total].to_i - stat[:packages_size] - buffered_update(stat)
+          next if increment == 0
+
+          ::Gitlab::BackgroundMigration::Logger.info(
+            migrator: MIGRATOR,
+            project_id: stat[:project_id],
+            old_size: stat[:packages_size],
+            new_size: stat[:total].to_i
+          )
+
+          stat.becomes(FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics) # rubocop:disable Cop/AvoidBecomes
+              .increment(increment)
+        end
+      end
+
+      def buffered_update(stat)
+        key = "project:{#{stat[:project_id]}}:counters:ProjectStatistics:#{stat[:id]}:packages_size"
+
+        Gitlab::Redis::SharedState.with do |redis|
+          redis.get(key).to_i
+        end
+      end
+
+      # rubocop:disable Style/Documentation
+      class ProjectStatistics < ::ApplicationRecord
+        self.table_name = 'project_statistics'
+
+        def increment(amount)
+          FixIncoherentPackagesSizeOnProjectStatistics::BufferedCounter.new(self).increment(amount)
+        end
+      end
+
+      class Package < ::ApplicationRecord
+        self.table_name = 'packages_packages'
+
+        has_many :package_files,
+                 class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' # rubocop:disable Layout/LineLength
+      end
+
+      class PackageFile < ::ApplicationRecord
+        self.table_name = 'packages_package_files'
+
+        belongs_to :package,
+                   class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' # rubocop:disable Layout/LineLength
+
+        def self.sum_query
+          packages = FixIncoherentPackagesSizeOnProjectStatistics::Package.arel_table
+          stats = FixIncoherentPackagesSizeOnProjectStatistics::ProjectStatistics.arel_table
+
+          joins(:package)
+            .where(packages[:project_id].eq(stats[:project_id]))
+            .where.not(size: nil)
+            .select('SUM(packages_package_files.size) as total')
+        end
+      end
+
+      class BufferedCounter
+        WORKER_DELAY = 10.minutes
+
+        def initialize(stat)
+          @stat = stat
+        end
+
+        def key
+          "project:{#{@stat.project_id}}:counters:ProjectStatistics:#{@stat.id}:packages_size"
+        end
+
+        def increment(amount)
+          Gitlab::Redis::SharedState.with do |redis|
+            redis.incrby(key, amount)
+          end
+
+          FlushCounterIncrementsWorker.perform_in(
+            WORKER_DELAY,
+            'ProjectStatistics',
+            @stat.id,
+            :packages_size
+          )
+        end
+      end
+      # rubocop:enable Style/Documentation
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb
index 914ababa5c2..db3f98bc2ba 100644
--- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb
+++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb
@@ -68,7 +68,7 @@ module Gitlab
       def valid_json?(metadata)
         Oj.load(metadata)
         true
-      rescue Oj::ParseError, EncodingError, JSON::ParserError, Encoding::UndefinedConversionError
+      rescue Oj::ParseError, EncodingError, JSON::ParserError, JSON::GeneratorError, Encoding::UndefinedConversionError
         false
       end
 
diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb
new file mode 100644
index 00000000000..592ef3220ff
--- /dev/null
+++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # A job to nullify `projects.creator_id` column of projects who creator
+    # does not exist in `users` table anymore.
+    class NullifyCreatorIdColumnOfOrphanedProjects < BatchedMigrationJob
+      scope_to ->(relation) do
+        relation.where.not(creator_id: nil)
+                .joins('LEFT OUTER JOIN users ON users.id = projects.creator_id')
+                .where(users: { id: nil })
+      end
+
+      operation_name :update_all
+      feature_category :projects
+
+      def perform
+        each_sub_batch do |sub_batch|
+          sub_batch.update_all(creator_id: nil)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/rebalance_partition_id.rb b/lib/gitlab/background_migration/rebalance_partition_id.rb
new file mode 100644
index 00000000000..7000ae5a856
--- /dev/null
+++ b/lib/gitlab/background_migration/rebalance_partition_id.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This rebalances partition_id to fix invalid records in production
+    class RebalancePartitionId < BatchedMigrationJob
+      INVALID_PARTITION_ID = 101
+      VALID_PARTITION_ID = 100
+
+      scope_to ->(relation) { relation.where(partition_id: INVALID_PARTITION_ID) }
+      operation_name :update_all
+      feature_category :continuous_integration
+
+      def perform
+        each_sub_batch do |sub_batch|
+          sub_batch.update_all(partition_id: VALID_PARTITION_ID)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/sanitize_confidential_todos.rb b/lib/gitlab/background_migration/sanitize_confidential_todos.rb
index 2df0b8a4d93..8215e92cbeb 100644
--- a/lib/gitlab/background_migration/sanitize_confidential_todos.rb
+++ b/lib/gitlab/background_migration/sanitize_confidential_todos.rb
@@ -10,43 +10,14 @@ module Gitlab
     # to extract all related logic.
     # Details in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908#note_952459215
     class SanitizeConfidentialTodos < BatchedMigrationJob
-      scope_to ->(relation) { relation.where(confidential: true) }
-
       operation_name :delete_invalid_todos
       feature_category :database
 
       def perform
-        each_sub_batch do |sub_batch|
-          delete_ids = invalid_todo_ids(sub_batch)
-
-          Todo.where(id: delete_ids).delete_all if delete_ids.present?
-        end
-      end
-
-      private
-
-      def invalid_todo_ids(notes_batch)
-        todos = Todo.where(note_id: notes_batch.select(:id)).includes(:note, :user)
-
-        todos.each_with_object([]) do |todo, ids|
-          ids << todo.id if invalid_todo?(todo)
-        end
-      end
-
-      def invalid_todo?(todo)
-        return false unless todo.note
-        return false if Ability.allowed?(todo.user, :read_todo, todo)
-
-        logger.info(
-          message: "#{self.class.name} deleting invalid todo",
-          attributes: todo.attributes
-        )
-
-        true
-      end
-
-      def logger
-        @logger ||= Gitlab::BackgroundMigration::Logger.build
+        # no-op: this BG migration is left here only for compatibility reasons,
+        # but it's not scheduled from any migration anymore.
+        # It was a temporary migration which used not-isolated code.
+        # https://gitlab.com/gitlab-org/gitlab/-/issues/382557
       end
     end
   end
diff --git a/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb b/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb
new file mode 100644
index 00000000000..24080a06c46
--- /dev/null
+++ b/lib/gitlab/background_migration/third_recount_epic_cache_counts.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class ThirdRecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob
+      feature_category :database
+
+      def perform; end
+    end
+    # rubocop: enable Style/Documentation
+  end
+end
+
+# rubocop: disable Layout/LineLength
+# we just want to re-enqueue the previous BackfillEpicCacheCounts migration,
+# because it's a EE-only migation and it's a module, we just prepend new
+# RecountEpicCacheCounts with existing batched migration module (which is same in both cases)
+Gitlab::BackgroundMigration::ThirdRecountEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts')
+# rubocop: enable Layout/LineLength