Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42

7 files changed, 134 insertions, 19 deletions

diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb
new file mode 100644
index 00000000000..61eb3b332de
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Update existent project update_at column after their repository storage was moved
+    class BackfillProjectUpdatedAtAfterRepositoryStorageMove
+      def perform(*project_ids)
+        updated_repository_storages = ProjectRepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id)
+
+        Project.connection.execute <<-SQL
+          WITH repository_storage_cte as (
+            #{updated_repository_storages.to_sql}
+          )
+          UPDATE projects
+          SET updated_at = (repository_storage_cte.updated_at + interval '1 second')
+          FROM repository_storage_cte
+          WHERE projects.id = repository_storage_cte.project_id AND projects.updated_at <= repository_storage_cte.updated_at
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb
new file mode 100644
index 00000000000..de2d9909961
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+module Gitlab
+  module BackgroundMigration
+    # EE-specific migration
+    class MigrateDevopsSegmentsToGroups
+      def perform
+        # no-op for CE
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups')
diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
index b8c14aa2573..091e6660bac 100644
--- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
+++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
@@ -16,26 +16,9 @@ module Gitlab
       def perform(start_id, end_id)
         old_registrations = U2fRegistration.where(id: start_id..end_id)
         old_registrations.each_slice(100) do |slice|
-          now = Time.now
           values = slice.map do |u2f_registration|
-            converted_credential = WebAuthn::U2fMigrator.new(
-              app_id: Gitlab.config.gitlab.url,
-              certificate: u2f_registration.certificate,
-              key_handle: u2f_registration.key_handle,
-              public_key: u2f_registration.public_key,
-              counter: u2f_registration.counter
-            ).credential
-
-            {
-                credential_xid: Base64.strict_encode64(converted_credential.id),
-                public_key: Base64.strict_encode64(converted_credential.public_key),
-                counter: u2f_registration.counter || 0,
-                name: u2f_registration.name || '',
-                user_id: u2f_registration.user_id,
-                u2f_registration_id: u2f_registration.id,
-                created_at: now,
-                updated_at: now
-            }
+            converter = Gitlab::Auth::U2fWebauthnConverter.new(u2f_registration)
+            converter.convert
           end
 
           WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false)
diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb
new file mode 100644
index 00000000000..d6795296fb7
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Class to migrate service_desk_reply_to email addresses to issue_email_participants
+    class PopulateIssueEmailParticipants
+      # rubocop:disable Style/Documentation
+      class TmpIssue < ActiveRecord::Base
+        self.table_name = 'issues'
+      end
+
+      def perform(start_id, stop_id)
+        issues = TmpIssue.select(:id, :service_desk_reply_to, :created_at).where(id: (start_id..stop_id)).where.not(service_desk_reply_to: nil)
+
+        rows = issues.map do |issue|
+          {
+            issue_id: issue.id,
+            email: issue.service_desk_reply_to,
+            created_at: issue.created_at,
+            updated_at: issue.created_at
+          }
+        end
+
+        Gitlab::Database.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb
new file mode 100644
index 00000000000..3d3970f50e1
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop:disable Style/Documentation
+    class PopulateUuidsForSecurityFindings
+      NOP_RELATION = Class.new { def each_batch(*); end }
+
+      def self.security_findings
+        NOP_RELATION.new
+      end
+
+      def perform(_scan_ids); end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_if_ee('::EE::Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings')
diff --git a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb
new file mode 100644
index 00000000000..ca61118a06c
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# rubocop: disable Style/Documentation
+class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings
+  DELETE_BATCH_SIZE = 100
+
+  # rubocop:disable Gitlab/NamespacedClass
+  class VulnerabilitiesFinding < ActiveRecord::Base
+    self.table_name = "vulnerability_occurrences"
+  end
+  # rubocop:enable Gitlab/NamespacedClass
+
+  def perform(start_id, end_id)
+    batch = VulnerabilitiesFinding.where(id: start_id..end_id)
+
+    cte = Gitlab::SQL::CTE.new(:batch, batch.select(:report_type, :location_fingerprint, :primary_identifier_id, :project_id))
+
+    query = VulnerabilitiesFinding
+      .select('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id', 'array_agg(id) as ids')
+      .distinct
+      .with(cte.to_arel)
+      .from(cte.alias_to(Arel.sql('batch')))
+      .joins(
+        %(
+        INNER JOIN
+        vulnerability_occurrences ON
+        vulnerability_occurrences.report_type = batch.report_type AND
+        vulnerability_occurrences.location_fingerprint = batch.location_fingerprint AND
+        vulnerability_occurrences.primary_identifier_id = batch.primary_identifier_id AND
+        vulnerability_occurrences.project_id = batch.project_id
+      )).group('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id')
+        .having('COUNT(*) > 1')
+
+    ids_to_delete = []
+
+    query.to_a.each do |record|
+      # We want to keep the latest finding since it might have recent metadata
+      duplicate_ids = record.ids.uniq.sort
+      duplicate_ids.pop
+      ids_to_delete.concat(duplicate_ids)
+
+      if ids_to_delete.size == DELETE_BATCH_SIZE
+        VulnerabilitiesFinding.where(id: ids_to_delete).delete_all
+        ids_to_delete.clear
+      end
+    end
+
+    VulnerabilitiesFinding.where(id: ids_to_delete).delete_all if ids_to_delete.any?
+  end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
index 6d7b9a86e69..8fa0db5fd4b 100644
--- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
@@ -6,6 +6,7 @@ module Gitlab
       module Models
         # isolated Namespace model
         class Namespace < ApplicationRecord
+          include FeatureGate
           include ::Gitlab::VisibilityLevel
           include ::Gitlab::Utils::StrongMemoize
           include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal