diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-09-19 04:45:44 +0300 |
commit | 85dc423f7090da0a52c73eb66faf22ddb20efff9 (patch) | |
tree | 9160f299afd8c80c038f08e1545be119f5e3f1e1 /lib/gitlab/background_migration | |
parent | 15c2c8c66dbe422588e5411eee7e68f1fa440bb8 (diff) |
Add latest changes from gitlab-org/gitlab@13-4-stable-ee
Diffstat (limited to 'lib/gitlab/background_migration')
11 files changed, 136 insertions, 16 deletions
diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb index c912628d0fc..5b9ee8a0ee2 100644 --- a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -9,7 +9,7 @@ module Gitlab end def perform(start_id, stop_id) - Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") update = ' commits_count = ( diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb index e62f5edd0e7..76598f6e2a6 100644 --- a/lib/gitlab/background_migration/calculate_wiki_sizes.rb +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -10,7 +10,7 @@ module Gitlab .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| statistics.refresh!(only: [:wiki_size]) rescue => e - Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" end end end diff --git a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb index 4016b807f21..c0099d44b5a 100644 --- a/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb +++ b/lib/gitlab/background_migration/fill_valid_time_for_pages_domain_certificate.rb @@ -25,7 +25,7 @@ module Gitlab certificate_valid_not_after: domain.x509&.not_after&.iso8601 ) rescue => e - Rails.logger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" # rubocop:disable Gitlab/RailsLogger + Gitlab::AppLogger.error "Failed to update pages domain certificate valid time. id: #{domain.id}, message: #{e.message}" end end end diff --git a/lib/gitlab/background_migration/fix_pages_access_level.rb b/lib/gitlab/background_migration/fix_pages_access_level.rb index 31d2e78b2d2..8e46021bd93 100644 --- a/lib/gitlab/background_migration/fix_pages_access_level.rb +++ b/lib/gitlab/background_migration/fix_pages_access_level.rb @@ -103,8 +103,8 @@ module Gitlab end # Private projects are not allowed to have enabled access level, only `private` and `public` - # If access control is enabled, these projects currently behave as if the have `private` pages_access_level - # if access control is disabled, these projects currently behave as if the have `public` pages_access_level + # If access control is enabled, these projects currently behave as if they have `private` pages_access_level + # if access control is disabled, these projects currently behave as if they have `public` pages_access_level # so we preserve this behaviour for projects with pages already deployed # for project without pages we always set `private` access_level def fix_private_access_level(start_id, stop_id) diff --git a/lib/gitlab/background_migration/migrate_to_hashed_storage.rb b/lib/gitlab/background_migration/migrate_to_hashed_storage.rb new file mode 100644 index 00000000000..4054db4fb87 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_to_hashed_storage.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration to move any legacy project to Hashed Storage + class MigrateToHashedStorage + def perform + batch_size = helper.batch_size + legacy_projects_count = Project.with_unmigrated_storage.count + + if storage_migrator.rollback_pending? + logger.warn( + migrator: 'MigrateToHashedStorage', + message: 'Aborting an storage rollback operation currently in progress' + ) + + storage_migrator.abort_rollback! + end + + if legacy_projects_count == 0 + logger.info( + migrator: 'MigrateToHashedStorage', + message: 'There are no projects requiring migration to Hashed Storage' + ) + + return + end + + logger.info( + migrator: 'MigrateToHashedStorage', + message: "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}" + ) + + helper.project_id_batches_migration do |start, finish| + storage_migrator.bulk_schedule_migration(start: start, finish: finish) + + logger.info( + migrator: 'MigrateToHashedStorage', + message: "Enqueuing migration of projects in batches of #{batch_size} from ID=#{start} to ID=#{finish}", + batch_from: start, + batch_to: finish + ) + end + end + + private + + def helper + Gitlab::HashedStorage::RakeHelper + end + + def storage_migrator + @storage_migrator ||= Gitlab::HashedStorage::Migrator.new + end + + def logger + @logger ||= ::Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb new file mode 100644 index 00000000000..eb72ef1de33 --- /dev/null +++ b/lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class PopulateResolvedOnDefaultBranchColumn + def perform(*); end + end + end +end + +Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn') diff --git a/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb new file mode 100644 index 00000000000..a0c89cc4664 --- /dev/null +++ b/lib/gitlab/background_migration/populate_vulnerability_historical_statistics.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class creates/updates those project historical vulnerability statistics + # that haven't been created nor initialized. It should only be executed in EE. + class PopulateVulnerabilityHistoricalStatistics + def perform(project_ids) + end + end + end +end + +Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateVulnerabilityHistoricalStatistics') diff --git a/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb new file mode 100644 index 00000000000..cc9b0329556 --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicate_cs_findings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RemoveDuplicateCsFindings + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::RemoveDuplicateCsFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicateCsFindings') diff --git a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb index 9f765d03d62..527dd2a0a83 100644 --- a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb +++ b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb @@ -4,13 +4,18 @@ module Gitlab module BackgroundMigration # Sets the MergeRequestDiff#files_count value for old rows class SetMergeRequestDiffFilesCount - COUNT_SUBQUERY = <<~SQL - files_count = ( - SELECT count(*) - FROM merge_request_diff_files - WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id - ) - SQL + # Some historic data has a *lot* of files. Apply a sentinel to these cases + FILES_COUNT_SENTINEL = 2**15 - 1 + + def self.count_subquery + <<~SQL + files_count = ( + SELECT LEAST(#{FILES_COUNT_SENTINEL}, count(*)) + FROM merge_request_diff_files + WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id + ) + SQL + end class MergeRequestDiff < ActiveRecord::Base # rubocop:disable Style/Documentation include EachBatch @@ -20,7 +25,7 @@ module Gitlab def perform(start_id, end_id) MergeRequestDiff.where(id: start_id..end_id).each_batch do |relation| - relation.update_all(COUNT_SUBQUERY) + relation.update_all(self.class.count_subquery) end end end diff --git a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb index 9ac92aab637..c485c23f3be 100644 --- a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb +++ b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb @@ -11,9 +11,11 @@ module Gitlab class SetNullPackageFilesFileStoreToLocalValue LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL - # Temporary AR class for package files - class PackageFile < ActiveRecord::Base - self.table_name = 'packages_package_files' + module Packages + # Temporary AR class for package files + class PackageFile < ActiveRecord::Base + self.table_name = 'packages_package_files' + end end def perform(start_id, stop_id) diff --git a/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb new file mode 100644 index 00000000000..651df36fcfd --- /dev/null +++ b/lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class UpdateLocationFingerprintForContainerScanningFindings + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings') |