diff options
Diffstat (limited to 'lib/gitlab/background_migration')
4 files changed, 179 insertions, 0 deletions
diff --git a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb new file mode 100644 index 00000000000..cb9b0e88ef4 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The migration is used to cleanup orphaned lfs_objects_projects in order to + # introduce valid foreign keys to this table + class CleanupOrphanedLfsObjectsProjects + # A model to access lfs_objects_projects table in migrations + class LfsObjectsProject < ActiveRecord::Base + self.table_name = 'lfs_objects_projects' + + include ::EachBatch + + belongs_to :lfs_object + belongs_to :project + end + + # A model to access lfs_objects table in migrations + class LfsObject < ActiveRecord::Base + self.table_name = 'lfs_objects' + end + + # A model to access projects table in migrations + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + + SUB_BATCH_SIZE = 5000 + CLEAR_CACHE_DELAY = 1.minute + + def perform(start_id, end_id) + cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) + cleanup_lfs_objects_projects_without_project(start_id, end_id) + end + + private + + def cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) + each_record_without_association(start_id, end_id, :lfs_object, :lfs_objects) do |lfs_objects_projects_without_lfs_objects| + projects = Project.where(id: lfs_objects_projects_without_lfs_objects.select(:project_id)) + + if projects.present? + ProjectCacheWorker.bulk_perform_in_with_contexts( + CLEAR_CACHE_DELAY, + projects, + arguments_proc: ->(project) { [project.id, [], [:lfs_objects_size]] }, + context_proc: ->(project) { { project: project } } + ) + end + + lfs_objects_projects_without_lfs_objects.delete_all + end + end + + def cleanup_lfs_objects_projects_without_project(start_id, end_id) + each_record_without_association(start_id, end_id, :project, :projects) do |lfs_objects_projects_without_projects| + lfs_objects_projects_without_projects.delete_all + end + end + + def each_record_without_association(start_id, end_id, association, table_name) + batch = LfsObjectsProject.where(id: start_id..end_id) + + batch.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(lfs_objects_projects.id), max(lfs_objects_projects.id)')).first + + lfs_objects_without_association = + LfsObjectsProject + .unscoped + .left_outer_joins(association) + .where(id: (first..last), table_name => { id: nil }) + + yield lfs_objects_without_association + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb new file mode 100644 index 00000000000..9a88eb8ea06 --- /dev/null +++ b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + BATCH_SIZE = 1000 + + # This background migration disables container expiration policies connected + # to a project that has no container repositories + class DisableExpirationPoliciesLinkedToNoContainerImages + # rubocop: disable Style/Documentation + class ContainerExpirationPolicy < ActiveRecord::Base + include EachBatch + + self.table_name = 'container_expiration_policies' + end + # rubocop: enable Style/Documentation + + def perform(from_id, to_id) + ContainerExpirationPolicy.where(enabled: true, project_id: from_id..to_id).each_batch(of: BATCH_SIZE) do |batch| + sql = <<-SQL + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:project_id).limit(BATCH_SIZE).to_sql}) + UPDATE container_expiration_policies + SET enabled = FALSE + FROM batched_relation + WHERE container_expiration_policies.project_id = batched_relation.project_id + AND NOT EXISTS (SELECT 1 FROM "container_repositories" WHERE container_repositories.project_id = container_expiration_policies.project_id) + SQL + execute(sql) + end + end + + private + + def execute(sql) + ActiveRecord::Base + .connection + .execute(sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 888a12f2330..a00d291245c 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -58,6 +58,13 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) + + logger.info(message: 'RecalculateVulnerabilitiesOccurrencesUuid Migration: recalculation is done for:', + finding_ids: mappings.keys.pluck(:id)) + + mark_job_as_succeeded(start_id, end_id) + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) end private @@ -76,4 +83,15 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid CalculateFindingUUID.call(name) end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'RecalculateVulnerabilitiesOccurrencesUuid', + arguments + ) + end end diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb new file mode 100644 index 00000000000..bba1ca26b35 --- /dev/null +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::UpdateJiraTrackerDataDeploymentTypeBasedOnUrl + # rubocop: disable Gitlab/NamespacedClass + class JiraTrackerData < ActiveRecord::Base + self.table_name = "jira_tracker_data" + self.inheritance_column = :_type_disabled + + include ::Integrations::BaseDataFields + attr_encrypted :url, encryption_options + attr_encrypted :api_url, encryption_options + + enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment + end + # rubocop: enable Gitlab/NamespacedClass + + # https://rubular.com/r/uwgK7k9KH23efa + JIRA_CLOUD_REGEX = %r{^https?://[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.atlassian\.net$}ix.freeze + + # rubocop: disable CodeReuse/ActiveRecord + def perform(start_id, end_id) + trackers_data = JiraTrackerData + .where(deployment_type: 'unknown') + .where(id: start_id..end_id) + + cloud, server = trackers_data.partition { |tracker_data| tracker_data.url.match?(JIRA_CLOUD_REGEX) } + + cloud_mappings = cloud.each_with_object({}) do |tracker_data, hash| + hash[tracker_data] = { deployment_type: 2 } + end + + server_mapppings = server.each_with_object({}) do |tracker_data, hash| + hash[tracker_data] = { deployment_type: 1 } + end + + mappings = cloud_mappings.merge(server_mapppings) + + ::Gitlab::Database::BulkUpdate.execute(%i[deployment_type], mappings) + end + # rubocop: enable CodeReuse/ActiveRecord +end |