diff options
Diffstat (limited to 'lib/gitlab/background_migration')
10 files changed, 165 insertions, 5 deletions
diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb index a0d0791b6af..b0a8c3a8cbb 100644 --- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb @@ -27,6 +27,17 @@ module Gitlab eligible_mrs.each_slice(10) do |slice| MergeRequest.where(id: slice).update_all(draft: true) end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillDraftStatusOnMergeRequests', + arguments + ) end end end diff --git a/lib/gitlab/background_migration/backfill_integrations_type_new.rb b/lib/gitlab/background_migration/backfill_integrations_type_new.rb new file mode 100644 index 00000000000..d1a939af58e --- /dev/null +++ b/lib/gitlab/background_migration/backfill_integrations_type_new.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the new `integrations.type_new` column, which contains + # the real class name, rather than the legacy class name in `type` + # which is mapped via `Gitlab::Integrations::StiType`. + class BackfillIntegrationsTypeNew + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = define_batchable_model(batch_table) + .where(batch_column => start_id..stop_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + process_sub_batch(sub_batch) + + sleep(pause_ms * 0.001) if pause_ms > 0 + end + end + + private + + def connection + ActiveRecord::Base.connection + end + + def process_sub_batch(sub_batch) + # Extract the start/stop IDs from the current sub-batch + sub_start_id, sub_stop_id = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first + + # This matches the mapping from the INSERT trigger added in + # db/migrate/20210721135638_add_triggers_to_integrations_type_new.rb + connection.execute(<<~SQL) + WITH mapping(old_type, new_type) AS (VALUES + ('AsanaService', 'Integrations::Asana'), + ('AssemblaService', 'Integrations::Assembla'), + ('BambooService', 'Integrations::Bamboo'), + ('BugzillaService', 'Integrations::Bugzilla'), + ('BuildkiteService', 'Integrations::Buildkite'), + ('CampfireService', 'Integrations::Campfire'), + ('ConfluenceService', 'Integrations::Confluence'), + ('CustomIssueTrackerService', 'Integrations::CustomIssueTracker'), + ('DatadogService', 'Integrations::Datadog'), + ('DiscordService', 'Integrations::Discord'), + ('DroneCiService', 'Integrations::DroneCi'), + ('EmailsOnPushService', 'Integrations::EmailsOnPush'), + ('EwmService', 'Integrations::Ewm'), + ('ExternalWikiService', 'Integrations::ExternalWiki'), + ('FlowdockService', 'Integrations::Flowdock'), + ('HangoutsChatService', 'Integrations::HangoutsChat'), + ('IrkerService', 'Integrations::Irker'), + ('JenkinsService', 'Integrations::Jenkins'), + ('JiraService', 'Integrations::Jira'), + ('MattermostService', 'Integrations::Mattermost'), + ('MattermostSlashCommandsService', 'Integrations::MattermostSlashCommands'), + ('MicrosoftTeamsService', 'Integrations::MicrosoftTeams'), + ('MockCiService', 'Integrations::MockCi'), + ('MockMonitoringService', 'Integrations::MockMonitoring'), + ('PackagistService', 'Integrations::Packagist'), + ('PipelinesEmailService', 'Integrations::PipelinesEmail'), + ('PivotaltrackerService', 'Integrations::Pivotaltracker'), + ('PrometheusService', 'Integrations::Prometheus'), + ('PushoverService', 'Integrations::Pushover'), + ('RedmineService', 'Integrations::Redmine'), + ('SlackService', 'Integrations::Slack'), + ('SlackSlashCommandsService', 'Integrations::SlackSlashCommands'), + ('TeamcityService', 'Integrations::Teamcity'), + ('UnifyCircuitService', 'Integrations::UnifyCircuit'), + ('WebexTeamsService', 'Integrations::WebexTeams'), + ('YoutrackService', 'Integrations::Youtrack'), + + -- EE-only integrations + ('GithubService', 'Integrations::Github'), + ('GitlabSlackApplicationService', 'Integrations::GitlabSlackApplication') + ) + + UPDATE integrations SET type_new = mapping.new_type + FROM mapping + WHERE integrations.id BETWEEN #{sub_start_id} AND #{sub_stop_id} + AND integrations.type = mapping.old_type + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index bc113a1e33d..f5c8796bd18 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -189,7 +189,7 @@ module Gitlab end def perform(start_id, stop_id) - Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert end private diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index 6f37f1846d2..b58f0a3a3e0 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -105,7 +105,7 @@ module Gitlab end def commit_attrs - @commit_attrs ||= { branch_name: 'master', message: 'Initial commit' } + @commit_attrs ||= { branch_name: 'main', message: 'Initial commit' } end def create_commit(snippet) diff --git a/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb new file mode 100644 index 00000000000..107ac9b0c3b --- /dev/null +++ b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class CopyCiBuildsColumnsToSecurityScans + extend ::Gitlab::Utils::Override + + UPDATE_BATCH_SIZE = 500 + + def perform(start_id, stop_id) + (start_id..stop_id).step(UPDATE_BATCH_SIZE).each do |offset| + batch_start = offset + batch_stop = offset + UPDATE_BATCH_SIZE - 1 + + ActiveRecord::Base.connection.execute <<~SQL + UPDATE + security_scans + SET + project_id = ci_builds.project_id, + pipeline_id = ci_builds.commit_id + FROM ci_builds + WHERE ci_builds.type='Ci::Build' + AND ci_builds.id=security_scans.build_id + AND security_scans.id BETWEEN #{Integer(batch_start)} AND #{Integer(batch_stop)} + SQL + end + + mark_job_as_succeeded(start_id, stop_id) + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'CopyCiBuildsColumnsToSecurityScans', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/create_security_setting.rb b/lib/gitlab/background_migration/create_security_setting.rb new file mode 100644 index 00000000000..55b37bb03b5 --- /dev/null +++ b/lib/gitlab/background_migration/create_security_setting.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't create SecuritySetting + # as this feature exists only in EE + class CreateSecuritySetting + def perform(_from_id, _to_id) + end + end + end +end + +Gitlab::BackgroundMigration::CreateSecuritySetting.prepend_mod_with('Gitlab::BackgroundMigration::CreateSecuritySetting') diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb index d2a9939b9ee..1c60473750d 100644 --- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb +++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb @@ -34,7 +34,7 @@ module Gitlab end end - Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert execute("ANALYZE #{TEMP_TABLE}") diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb index 2bce5037d03..14c72bb4a72 100644 --- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb +++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb @@ -65,7 +65,7 @@ module Gitlab next if service_ids.empty? migrated_ids += service_ids - Gitlab::Database.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert end return if migrated_ids.empty? diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb index d6795296fb7..0a56ac1dae8 100644 --- a/lib/gitlab/background_migration/populate_issue_email_participants.rb +++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb @@ -21,7 +21,7 @@ module Gitlab } end - Gitlab::Database.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index a00d291245c..84ff7423254 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -9,6 +9,8 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end class VulnerabilitiesFinding < ActiveRecord::Base + include ShaAttribute + self.table_name = "vulnerability_occurrences" belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' REPORT_TYPES = { @@ -21,6 +23,9 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid api_fuzzing: 6 }.with_indifferent_access.freeze enum report_type: REPORT_TYPES + + sha_attribute :fingerprint + sha_attribute :location_fingerprint end class CalculateFindingUUID |