Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42

8 files changed, 163 insertions, 4 deletions

diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
index 6d1df95c66d..236c6b6eb9a 100644
--- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb
+++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
@@ -73,7 +73,7 @@ module Gitlab
         # violation. We can safely roll-back the nested transaction and perform
         # a lookup instead to retrieve the record.
         def create_record
-          subject.transaction(requires_new: true) do
+          subject.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions
             InternalId.create!(
               **scope,
               usage: usage_value,
diff --git a/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb b/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb
new file mode 100644
index 00000000000..67f4690868e
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_iteration_cadence_id_for_boards.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class BackfillIterationCadenceIdForBoards
+      def perform(*args)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::BackfillIterationCadenceIdForBoards.prepend_mod_with('Gitlab::BackgroundMigration::BackfillIterationCadenceIdForBoards')
diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb
index f5c8796bd18..a9eaeb0562d 100644
--- a/lib/gitlab/background_migration/backfill_project_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_project_repositories.rb
@@ -21,7 +21,7 @@ module Gitlab
           shard_id = shards.fetch(name, nil)
           return shard_id if shard_id.present?
 
-          Shard.transaction(requires_new: true) do
+          Shard.transaction(requires_new: true) do # rubocop:disable Performance/ActiveRecordSubtransactions
             create!(name)
           end
         rescue ActiveRecord::RecordNotUnique
diff --git a/lib/gitlab/background_migration/backfill_projects_with_coverage.rb b/lib/gitlab/background_migration/backfill_projects_with_coverage.rb
new file mode 100644
index 00000000000..ca262c0bd59
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_projects_with_coverage.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfill project_ci_feature_usages for a range of projects with coverage
+    class BackfillProjectsWithCoverage
+      class ProjectCiFeatureUsage < ActiveRecord::Base # rubocop:disable Style/Documentation
+        self.table_name = 'project_ci_feature_usages'
+      end
+
+      COVERAGE_ENUM_VALUE = 1
+      INSERT_DELAY_SECONDS = 0.1
+
+      def perform(start_id, end_id, sub_batch_size)
+        report_results = ActiveRecord::Base.connection.execute <<~SQL
+          SELECT DISTINCT project_id, default_branch
+          FROM ci_daily_build_group_report_results
+          WHERE id BETWEEN #{start_id} AND #{end_id}
+        SQL
+
+        report_results.to_a.in_groups_of(sub_batch_size, false) do |batch|
+          ProjectCiFeatureUsage.insert_all(build_values(batch))
+
+          sleep INSERT_DELAY_SECONDS
+        end
+      end
+
+      private
+
+      def build_values(batch)
+        batch.map do |data|
+          {
+            project_id: data['project_id'],
+            feature: COVERAGE_ENUM_VALUE,
+            default_branch: data['default_branch']
+          }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb
new file mode 100644
index 00000000000..31b5b5cdb73
--- /dev/null
+++ b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # The class to extract the project topics into a separate `topics` table
+    class ExtractProjectTopicsIntoSeparateTable
+      # Temporary AR table for tags
+      class Tag < ActiveRecord::Base
+        self.table_name = 'tags'
+      end
+
+      # Temporary AR table for taggings
+      class Tagging < ActiveRecord::Base
+        self.table_name = 'taggings'
+        belongs_to :tag
+      end
+
+      # Temporary AR table for topics
+      class Topic < ActiveRecord::Base
+        self.table_name = 'topics'
+      end
+
+      # Temporary AR table for project topics
+      class ProjectTopic < ActiveRecord::Base
+        self.table_name = 'project_topics'
+        belongs_to :topic
+      end
+
+      # Temporary AR table for projects
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+      end
+
+      def perform(start_id, stop_id)
+        Tagging.includes(:tag).where(taggable_type: 'Project', id: start_id..stop_id).each do |tagging|
+          if Project.exists?(id: tagging.taggable_id) && tagging.tag
+            begin
+              topic = Topic.find_or_create_by(name: tagging.tag.name)
+              project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic)
+
+              tagging.delete if project_topic.persisted?
+            rescue StandardError => e
+              Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id)
+            end
+          else
+            tagging.delete
+          end
+        end
+
+        mark_job_as_succeeded(start_id, stop_id)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          self.class.name.demodulize,
+          arguments
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb
index c096dae0631..3605b157f4f 100644
--- a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb
+++ b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb
@@ -14,7 +14,7 @@ module Gitlab
           mail(
             template_path: 'unconfirm_mailer',
             template_name: 'unconfirm_notification_email',
-            to: @user.notification_email,
+            to: @user.notification_email_or_default,
             subject: subject('GitLab email verification request')
           )
         end
diff --git a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb
index e694e5359cd..7d150b9cd83 100644
--- a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb
+++ b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb
@@ -14,7 +14,7 @@ module Gitlab
 
       # The number of rows in merge_request_diff_commits to get in a single
       # query.
-      COMMIT_ROWS_PER_QUERY = 10_000
+      COMMIT_ROWS_PER_QUERY = 1_000
 
       # The number of rows in merge_request_diff_commits to update in a single
       # query.
@@ -78,6 +78,8 @@ module Gitlab
       # rubocop: enable Style/Documentation
 
       def perform(start_id, stop_id)
+        return if already_processed?(start_id, stop_id)
+
         # This Hash maps user names + emails to their corresponding rows in
         # merge_request_diff_commit_users.
         user_mapping = {}
@@ -94,6 +96,13 @@ module Gitlab
         )
       end
 
+      def already_processed?(start_id, stop_id)
+        Database::BackgroundMigrationJob
+          .for_migration_execution('MigrateMergeRequestDiffCommitUsers', [start_id, stop_id])
+          .succeeded
+          .any?
+      end
+
       # Returns the data we'll use to determine what merge_request_diff_commits
       # rows to update, and what data to use for populating their
       # commit_author_id and committer_id columns.
diff --git a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb
new file mode 100644
index 00000000000..43a7032e682
--- /dev/null
+++ b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # A background migration that finished any pending
+    # MigrateMergeRequestDiffCommitUsers jobs, and schedules new jobs itself.
+    #
+    # This migration exists so we can bypass rescheduling issues (e.g. jobs
+    # getting dropped after too many retries) that may occur when
+    # MigrateMergeRequestDiffCommitUsers jobs take longer than expected.
+    class StealMigrateMergeRequestDiffCommitUsers
+      def perform(start_id, stop_id)
+        MigrateMergeRequestDiffCommitUsers.new.perform(start_id, stop_id)
+        schedule_next_job
+      end
+
+      def schedule_next_job
+        next_job = Database::BackgroundMigrationJob
+          .for_migration_class('MigrateMergeRequestDiffCommitUsers')
+          .pending
+          .first
+
+        return unless next_job
+
+        BackgroundMigrationWorker.perform_in(
+          5.minutes,
+          'StealMigrateMergeRequestDiffCommitUsers',
+          next_job.arguments
+        )
+      end
+    end
+  end
+end