Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42

10 files changed, 165 insertions, 5 deletions

diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb
index a0d0791b6af..b0a8c3a8cbb 100644
--- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb
+++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb
@@ -27,6 +27,17 @@ module Gitlab
         eligible_mrs.each_slice(10) do |slice|
           MergeRequest.where(id: slice).update_all(draft: true)
         end
+
+        mark_job_as_succeeded(start_id, end_id)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          'BackfillDraftStatusOnMergeRequests',
+          arguments
+        )
       end
     end
   end
diff --git a/lib/gitlab/background_migration/backfill_integrations_type_new.rb b/lib/gitlab/background_migration/backfill_integrations_type_new.rb
new file mode 100644
index 00000000000..d1a939af58e
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_integrations_type_new.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfills the new `integrations.type_new` column, which contains
+    # the real class name, rather than the legacy class name in `type`
+    # which is mapped via `Gitlab::Integrations::StiType`.
+    class BackfillIntegrationsTypeNew
+      include Gitlab::Database::DynamicModelHelpers
+
+      def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms)
+        parent_batch_relation = define_batchable_model(batch_table)
+          .where(batch_column => start_id..stop_id)
+
+        parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
+          process_sub_batch(sub_batch)
+
+          sleep(pause_ms * 0.001) if pause_ms > 0
+        end
+      end
+
+      private
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def process_sub_batch(sub_batch)
+        # Extract the start/stop IDs from the current sub-batch
+        sub_start_id, sub_stop_id = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first
+
+        # This matches the mapping from the INSERT trigger added in
+        # db/migrate/20210721135638_add_triggers_to_integrations_type_new.rb
+        connection.execute(<<~SQL)
+          WITH mapping(old_type, new_type) AS (VALUES
+            ('AsanaService',                   'Integrations::Asana'),
+            ('AssemblaService',                'Integrations::Assembla'),
+            ('BambooService',                  'Integrations::Bamboo'),
+            ('BugzillaService',                'Integrations::Bugzilla'),
+            ('BuildkiteService',               'Integrations::Buildkite'),
+            ('CampfireService',                'Integrations::Campfire'),
+            ('ConfluenceService',              'Integrations::Confluence'),
+            ('CustomIssueTrackerService',      'Integrations::CustomIssueTracker'),
+            ('DatadogService',                 'Integrations::Datadog'),
+            ('DiscordService',                 'Integrations::Discord'),
+            ('DroneCiService',                 'Integrations::DroneCi'),
+            ('EmailsOnPushService',            'Integrations::EmailsOnPush'),
+            ('EwmService',                     'Integrations::Ewm'),
+            ('ExternalWikiService',            'Integrations::ExternalWiki'),
+            ('FlowdockService',                'Integrations::Flowdock'),
+            ('HangoutsChatService',            'Integrations::HangoutsChat'),
+            ('IrkerService',                   'Integrations::Irker'),
+            ('JenkinsService',                 'Integrations::Jenkins'),
+            ('JiraService',                    'Integrations::Jira'),
+            ('MattermostService',              'Integrations::Mattermost'),
+            ('MattermostSlashCommandsService', 'Integrations::MattermostSlashCommands'),
+            ('MicrosoftTeamsService',          'Integrations::MicrosoftTeams'),
+            ('MockCiService',                  'Integrations::MockCi'),
+            ('MockMonitoringService',          'Integrations::MockMonitoring'),
+            ('PackagistService',               'Integrations::Packagist'),
+            ('PipelinesEmailService',          'Integrations::PipelinesEmail'),
+            ('PivotaltrackerService',          'Integrations::Pivotaltracker'),
+            ('PrometheusService',              'Integrations::Prometheus'),
+            ('PushoverService',                'Integrations::Pushover'),
+            ('RedmineService',                 'Integrations::Redmine'),
+            ('SlackService',                   'Integrations::Slack'),
+            ('SlackSlashCommandsService',      'Integrations::SlackSlashCommands'),
+            ('TeamcityService',                'Integrations::Teamcity'),
+            ('UnifyCircuitService',            'Integrations::UnifyCircuit'),
+            ('WebexTeamsService',              'Integrations::WebexTeams'),
+            ('YoutrackService',                'Integrations::Youtrack'),
+
+            -- EE-only integrations
+            ('GithubService',                  'Integrations::Github'),
+            ('GitlabSlackApplicationService',  'Integrations::GitlabSlackApplication')
+          )
+
+          UPDATE integrations SET type_new = mapping.new_type
+          FROM mapping
+          WHERE integrations.id BETWEEN #{sub_start_id} AND #{sub_stop_id}
+            AND integrations.type = mapping.old_type
+        SQL
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb
index bc113a1e33d..f5c8796bd18 100644
--- a/lib/gitlab/background_migration/backfill_project_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_project_repositories.rb
@@ -189,7 +189,7 @@ module Gitlab
       end
 
       def perform(start_id, stop_id)
-        Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert
+        Gitlab::Database.main.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert
       end
 
       private
diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
index 6f37f1846d2..b58f0a3a3e0 100644
--- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
@@ -105,7 +105,7 @@ module Gitlab
       end
 
       def commit_attrs
-        @commit_attrs ||= { branch_name: 'master', message: 'Initial commit' }
+        @commit_attrs ||= { branch_name: 'main', message: 'Initial commit' }
       end
 
       def create_commit(snippet)
diff --git a/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb
new file mode 100644
index 00000000000..107ac9b0c3b
--- /dev/null
+++ b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class CopyCiBuildsColumnsToSecurityScans
+      extend ::Gitlab::Utils::Override
+
+      UPDATE_BATCH_SIZE = 500
+
+      def perform(start_id, stop_id)
+        (start_id..stop_id).step(UPDATE_BATCH_SIZE).each do |offset|
+          batch_start = offset
+          batch_stop = offset + UPDATE_BATCH_SIZE - 1
+
+          ActiveRecord::Base.connection.execute <<~SQL
+            UPDATE
+              security_scans
+            SET
+              project_id = ci_builds.project_id,
+              pipeline_id = ci_builds.commit_id
+            FROM ci_builds
+            WHERE ci_builds.type='Ci::Build' 
+              AND ci_builds.id=security_scans.build_id 
+              AND security_scans.id BETWEEN #{Integer(batch_start)} AND #{Integer(batch_stop)}
+          SQL
+        end
+
+        mark_job_as_succeeded(start_id, stop_id)
+      rescue StandardError => error
+        Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
+      end
+
+      private
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          'CopyCiBuildsColumnsToSecurityScans',
+          arguments
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/background_migration/create_security_setting.rb b/lib/gitlab/background_migration/create_security_setting.rb
new file mode 100644
index 00000000000..55b37bb03b5
--- /dev/null
+++ b/lib/gitlab/background_migration/create_security_setting.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class doesn't create SecuritySetting
+    # as this feature exists only in EE
+    class CreateSecuritySetting
+      def perform(_from_id, _to_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::CreateSecuritySetting.prepend_mod_with('Gitlab::BackgroundMigration::CreateSecuritySetting')
diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
index d2a9939b9ee..1c60473750d 100644
--- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
+++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
@@ -34,7 +34,7 @@ module Gitlab
             end
           end
 
-          Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert
 
           execute("ANALYZE #{TEMP_TABLE}")
 
diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
index 2bce5037d03..14c72bb4a72 100644
--- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
+++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
@@ -65,7 +65,7 @@ module Gitlab
           next if service_ids.empty?
 
           migrated_ids += service_ids
-          Gitlab::Database.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert
+          Gitlab::Database.main.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert
         end
 
         return if migrated_ids.empty?
diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb
index d6795296fb7..0a56ac1dae8 100644
--- a/lib/gitlab/background_migration/populate_issue_email_participants.rb
+++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb
@@ -21,7 +21,7 @@ module Gitlab
           }
         end
 
-        Gitlab::Database.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert
+        Gitlab::Database.main.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert
       end
     end
   end
diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
index a00d291245c..84ff7423254 100644
--- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
+++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
@@ -9,6 +9,8 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
   end
 
   class VulnerabilitiesFinding < ActiveRecord::Base
+    include ShaAttribute
+
     self.table_name = "vulnerability_occurrences"
     belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id'
     REPORT_TYPES = {
@@ -21,6 +23,9 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
       api_fuzzing: 6
     }.with_indifferent_access.freeze
     enum report_type: REPORT_TYPES
+
+    sha_attribute :fingerprint
+    sha_attribute :location_fingerprint
   end
 
   class CalculateFindingUUID