diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
| commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
| tree | dceebdc68925362117480a5d672bcff122fb625b /lib/gitlab/background_migration | |
| parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) | |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'lib/gitlab/background_migration')
53 files changed, 882 insertions, 1303 deletions
diff --git a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb index 82e607ac7a7..2127ce5975d 100644 --- a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb +++ b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb @@ -12,14 +12,18 @@ module Gitlab end operation_name :update_all - feature_category :authentication_and_authorization + feature_category :system_access ADMIN_MODE_SCOPE = ['admin_mode'].freeze def perform each_sub_batch do |sub_batch| sub_batch.each do |token| - token.update!(scopes: (YAML.safe_load(token.scopes) + ADMIN_MODE_SCOPE).uniq.to_yaml) + existing_scopes = YAML.safe_load(token.scopes, permitted_classes: [Symbol]) + # making sure scopes are not mixed symbols and strings + stringified_scopes = existing_scopes.map(&:to_s) + + token.update!(scopes: (stringified_scopes + ADMIN_MODE_SCOPE).uniq.to_yaml) end end end diff --git a/lib/gitlab/background_migration/backfill_compliance_violations.rb b/lib/gitlab/background_migration/backfill_compliance_violations.rb new file mode 100644 index 00000000000..131b4a05e41 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_compliance_violations.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillComplianceViolations < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :compliance_management + + def perform + # no-op. The logic is defined in EE module. + end + end + # rubocop: enable Style/Documentation + end +end + +::Gitlab::BackgroundMigration::BackfillComplianceViolations.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_design_management_repositories.rb b/lib/gitlab/background_migration/backfill_design_management_repositories.rb new file mode 100644 index 00000000000..fe57767a693 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_design_management_repositories.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill design_management_repositories table for a range of projects + class BackfillDesignManagementRepositories < BatchedMigrationJob + operation_name :backfill_design_management_repositories + feature_category :geo_replication + + def perform + each_sub_batch do |sub_batch| + backfill_design_management_repositories(sub_batch) + end + end + + def backfill_design_management_repositories(relation) + connection.execute( + <<~SQL + INSERT INTO design_management_repositories (project_id, created_at, updated_at) + SELECT projects.id, now(), now() + FROM projects + WHERE projects.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (project_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb index de52629522b..878f89a8b3d 100644 --- a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb +++ b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb @@ -40,13 +40,13 @@ module Gitlab scope :affected, -> { where(type_new: INTEGRATIONS.keys).where.not(encrypted_properties: nil) } attr_encrypted :properties, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm', - marshal: true, - marshaler: ::Gitlab::Json, - encode: false, - encode_iv: false + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false # Handle assignment of props with symbol keys. # To do this correctly, we need to call the method generated by attr_encrypted. diff --git a/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb b/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb new file mode 100644 index 00000000000..1a5ad1c14a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill container_registry_size for project_statistics + class BackfillNamespaceLdapSettings < Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :backfill_namespace_ldap_settings + feature_category :system_access + + def perform + # no-op in FOSS + end + end + end +end + +Gitlab::BackgroundMigration::BackfillNamespaceLdapSettings.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb deleted file mode 100644 index 3b8a452b855..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to set namespaces.traversal_ids in sub-batches, of all namespaces with - # a parent and not already set. - # rubocop:disable Style/Documentation - class BackfillNamespaceTraversalIdsChildren - class Namespace < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'namespaces' - - scope :base_query, -> { where.not(parent_id: nil) } - end - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - batch_query = Namespace.base_query.where(id: start_id..end_id) - batch_query.each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - ranged_query = Namespace.unscoped.base_query.where(id: first..last) - - update_sql = <<~SQL - UPDATE namespaces - SET traversal_ids = calculated_ids.traversal_ids - FROM #{calculated_traversal_ids(ranged_query)} calculated_ids - WHERE namespaces.id = calculated_ids.id - AND namespaces.traversal_ids = '{}' - SQL - ApplicationRecord.connection.execute(update_sql) - - sleep PAUSE_SECONDS - end - - # We have to add all arguments when marking a job as succeeded as they - # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - # Calculate the ancestor path for a given set of namespaces. - def calculated_traversal_ids(batch) - <<~SQL - ( - WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( - ( - SELECT batch.id, batch.id, batch.parent_id, 1 - FROM (#{batch.to_sql}) AS batch - ) - UNION ALL - ( - SELECT cte.source_id, n.id, n.parent_id, cte.height+1 - FROM namespaces n, cte - WHERE n.id = cte.parent_id - ) - ) - SELECT flat_hierarchy.source_id as id, - array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids - FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy - GROUP BY flat_hierarchy.source_id - ) - SQL - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'BackfillNamespaceTraversalIdsChildren', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb deleted file mode 100644 index c69289fb91f..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to set namespaces.traversal_ids in sub-batches, of all namespaces - # without a parent and not already set. - # rubocop:disable Style/Documentation - class BackfillNamespaceTraversalIdsRoots - class Namespace < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'namespaces' - - scope :base_query, -> { where(parent_id: nil) } - end - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - ranged_query = Namespace.base_query - .where(id: start_id..end_id) - .where("traversal_ids = '{}'") - - ranged_query.each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - - # The query need to be reconstructed because .each_batch modifies the default scope - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 - Namespace.unscoped - .base_query - .where(id: first..last) - .where("traversal_ids = '{}'") - .update_all('traversal_ids = ARRAY[id]') - - sleep PAUSE_SECONDS - end - - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'BackfillNamespaceTraversalIdsRoots', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_partitioned_table.rb b/lib/gitlab/background_migration/backfill_partitioned_table.rb new file mode 100644 index 00000000000..6479d40a930 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_partitioned_table.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration to generically copy data from the given table into its corresponding partitioned table + class BackfillPartitionedTable < BatchedMigrationJob + operation_name :upsert_partitioned_table + feature_category :database + job_arguments :partitioned_table + + def perform + validate_paritition_table! + + bulk_copy = Gitlab::Database::PartitioningMigrationHelpers::BulkCopy.new( + batch_table, + partitioned_table, + batch_column, + connection: connection + ) + + each_sub_batch do |relation| + sub_start_id, sub_stop_id = relation.pick(Arel.sql("MIN(#{batch_column}), MAX(#{batch_column})")) + bulk_copy.copy_between(sub_start_id, sub_stop_id) + end + end + + private + + def validate_paritition_table! + unless connection.table_exists?(partitioned_table) + raise "exiting backfill migration because partitioned table #{partitioned_table} does not exist. " \ + "This could be due to rollback of the migration which created the partitioned table." + end + + # rubocop: disable Style/GuardClause + unless Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(partitioned_table).present? + raise "exiting backfill migration because the given destination table is not partitioned." + end + # rubocop: enable Style/GuardClause + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb b/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb new file mode 100644 index 00000000000..9bf503bd6e7 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill prepared_at for an array of merge requests + class BackfillPreparedAtMergeRequests < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) { relation } + operation_name :update_all + feature_category :code_review_workflow + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(prepared_at: nil).where.not(merge_status: 'preparing').update_all('prepared_at = created_at') + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb new file mode 100644 index 00000000000..8d6df905f15 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill project_wiki_repositories table for a range of projects + class BackfillProjectWikiRepositories < BatchedMigrationJob + operation_name :backfill_project_wiki_repositories + feature_category :geo_replication + + scope_to ->(relation) do + relation + .joins('LEFT OUTER JOIN project_wiki_repositories ON project_wiki_repositories.project_id = projects.id') + .where(project_wiki_repositories: { project_id: nil }) + end + + def perform + each_sub_batch do |sub_batch| + backfill_project_wiki_repositories(sub_batch) + end + end + + def backfill_project_wiki_repositories(relation) + connection.execute( + <<~SQL + INSERT INTO project_wiki_repositories (project_id, created_at, updated_at) + SELECT projects.id, now(), now() + FROM projects + WHERE projects.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (project_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb deleted file mode 100644 index 3bf6bf993dd..00000000000 --- a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will populate the upvotes_count field - # for each issue - class BackfillUpvotesCountOnIssues - BATCH_SIZE = 1_000 - - def perform(start_id, stop_id) - (start_id..stop_id).step(BATCH_SIZE).each do |offset| - update_issue_upvotes_count(offset, offset + BATCH_SIZE) - end - end - - private - - def execute(sql) - @connection ||= ApplicationRecord.connection - @connection.execute(sql) - end - - def update_issue_upvotes_count(batch_start, batch_stop) - execute(<<~SQL) - UPDATE issues - SET upvotes_count = sub_q.count_all - FROM ( - SELECT COUNT(*) AS count_all, e.awardable_id AS issue_id - FROM award_emoji AS e - WHERE e.name = 'thumbsup' AND - e.awardable_type = 'Issue' AND - e.awardable_id BETWEEN #{batch_start} AND #{batch_stop} - GROUP BY issue_id - ) AS sub_q - WHERE sub_q.issue_id = issues.id; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_user_namespace.rb b/lib/gitlab/background_migration/backfill_user_namespace.rb deleted file mode 100644 index df6b1f083c3..00000000000 --- a/lib/gitlab/background_migration/backfill_user_namespace.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfills the `namespaces.type` column, replacing any - # instances of `NULL` with `User` - class BackfillUserNamespace - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all(type: 'User') - end - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def connection - ApplicationRecord.connection - end - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection) - .where(source_key_column => start_id..stop_id) - .where(type: nil) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index fc0d0ce3a57..8e2e588e0cd 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -11,7 +11,7 @@ module Gitlab class MigrationIssue < ApplicationRecord self.table_name = 'issues' - scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) } + scope :base_query, ->(base_type) { where(issue_type: base_type) } end MAX_UPDATE_RETRIES = 3 @@ -24,9 +24,7 @@ module Gitlab operation_name :update_all def perform - each_sub_batch( - batching_scope: -> (relation) { relation.where(work_item_type_id: nil) } - ) do |sub_batch| + each_sub_batch do |sub_batch| first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) # The query need to be reconstructed because .each_batch modifies the default scope diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 4039a79cfa7..952e6d01f1a 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -7,6 +7,8 @@ module Gitlab # # Job arguments needed must be defined explicitly, # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#job-arguments. + # rubocop:disable Metrics/ClassLength + # rubocop:disable Metrics/ParameterLists class BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers include Gitlab::ClassAttributes @@ -60,7 +62,8 @@ module Gitlab end def initialize( - start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: + start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection:, + sub_batch_exception: nil ) @start_id = start_id @@ -71,6 +74,7 @@ module Gitlab @pause_ms = pause_ms @job_arguments = job_arguments @connection = connection + @sub_batch_exception = sub_batch_exception end def filter_batch(relation) @@ -87,7 +91,8 @@ module Gitlab private - attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, :pause_ms, :connection + attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, + :pause_ms, :connection, :sub_batch_exception def each_sub_batch(batching_arguments: {}, batching_scope: nil) all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) @@ -98,6 +103,10 @@ module Gitlab sub_batch_relation.each_batch(**all_batching_arguments) do |relation| batch_metrics.instrument_operation(operation_name) do yield relation + rescue *Gitlab::Database::BackgroundMigration::BatchedJob::TIMEOUT_EXCEPTIONS => exception + exception_class = sub_batch_exception || exception.class + + raise exception_class, exception end sleep([pause_ms, 0].max * 0.001) @@ -137,3 +146,5 @@ module Gitlab end end end +# rubocop:enable Metrics/ClassLength +# rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb deleted file mode 100644 index 4da120769a0..00000000000 --- a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The migration is used to cleanup orphaned lfs_objects_projects in order to - # introduce valid foreign keys to this table - class CleanupOrphanedLfsObjectsProjects - # A model to access lfs_objects_projects table in migrations - class LfsObjectsProject < ActiveRecord::Base - self.table_name = 'lfs_objects_projects' - - include ::EachBatch - - belongs_to :lfs_object - belongs_to :project - end - - # A model to access lfs_objects table in migrations - class LfsObject < ActiveRecord::Base - self.table_name = 'lfs_objects' - end - - # A model to access projects table in migrations - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - SUB_BATCH_SIZE = 5000 - CLEAR_CACHE_DELAY = 1.minute - - def perform(start_id, end_id) - cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) - cleanup_lfs_objects_projects_without_project(start_id, end_id) - end - - private - - def cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) - each_record_without_association(start_id, end_id, :lfs_object, :lfs_objects) do |lfs_objects_projects_without_lfs_objects| - projects = Project.where(id: lfs_objects_projects_without_lfs_objects.select(:project_id)) - - if projects.present? - ProjectCacheWorker.bulk_perform_in_with_contexts( - CLEAR_CACHE_DELAY, - projects, - arguments_proc: ->(project) { [project.id, [], [:lfs_objects_size]] }, - context_proc: ->(project) { { project: project } } - ) - end - - lfs_objects_projects_without_lfs_objects.delete_all - end - end - - def cleanup_lfs_objects_projects_without_project(start_id, end_id) - each_record_without_association(start_id, end_id, :project, :projects) do |lfs_objects_projects_without_projects| - lfs_objects_projects_without_projects.delete_all - end - end - - def each_record_without_association(start_id, end_id, association, table_name) - batch = LfsObjectsProject.where(id: start_id..end_id) - - batch.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(lfs_objects_projects.id), max(lfs_objects_projects.id)')) - - lfs_objects_without_association = - LfsObjectsProject - .unscoped - .left_outer_joins(association) - .where(id: (first..last), table_name => { id: nil }) - - yield lfs_objects_without_association - end - end - end - end -end diff --git a/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb new file mode 100644 index 00000000000..e8ee2a4c251 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Clean up personal access tokens with expires_at value is nil + # and set the value to new default 365 days + class CleanupPersonalAccessTokensWithNilExpiresAt < BatchedMigrationJob + feature_category :system_access + + EXPIRES_AT_DEFAULT = 365.days.from_now + + scope_to ->(relation) { relation.where(expires_at: nil) } + operation_name :update_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(expires_at: EXPIRES_AT_DEFAULT) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/create_vulnerability_links.rb b/lib/gitlab/background_migration/create_vulnerability_links.rb new file mode 100644 index 00000000000..bbc71dfb392 --- /dev/null +++ b/lib/gitlab/background_migration/create_vulnerability_links.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class CreateVulnerabilityLinks < BatchedMigrationJob + feature_category :vulnerability_management + def perform; end + end + end +end + +Gitlab::BackgroundMigration::CreateVulnerabilityLinks.prepend_mod +# rubocop:enable Style/Documentation diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb deleted file mode 100644 index 4a3a12ab53d..00000000000 --- a/lib/gitlab/background_migration/delete_orphaned_deployments.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration for deleting orphaned deployments. - class DeleteOrphanedDeployments - include Database::MigrationHelpers - - def perform(start_id, end_id) - orphaned_deployments - .where(id: start_id..end_id) - .delete_all - - mark_job_as_succeeded(start_id, end_id) - end - - def orphaned_deployments - define_batchable_model('deployments', connection: ApplicationRecord.connection) - .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb b/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb new file mode 100644 index 00000000000..a795300fa9d --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphaned packages_dependencies records that have no packages_dependency_links + class DeleteOrphanedPackagesDependencies < BatchedMigrationJob + operation_name :delete_all + feature_category :package_registry + + scope_to ->(relation) { + relation.where( + <<~SQL.squish + NOT EXISTS ( + SELECT 1 + FROM packages_dependency_links + WHERE packages_dependency_links.dependency_id = packages_dependencies.id + ) + SQL + ) + } + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb deleted file mode 100644 index dad5da875ab..00000000000 --- a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - BATCH_SIZE = 1000 - - # This background migration disables container expiration policies connected - # to a project that has no container repositories - class DisableExpirationPoliciesLinkedToNoContainerImages - # rubocop: disable Style/Documentation - class ContainerExpirationPolicy < ActiveRecord::Base - include EachBatch - - self.table_name = 'container_expiration_policies' - end - # rubocop: enable Style/Documentation - - def perform(from_id, to_id) - ContainerExpirationPolicy.where(enabled: true, project_id: from_id..to_id).each_batch(of: BATCH_SIZE) do |batch| - sql = <<-SQL - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:project_id).limit(BATCH_SIZE).to_sql}) - UPDATE container_expiration_policies - SET enabled = FALSE - FROM batched_relation - WHERE container_expiration_policies.project_id = batched_relation.project_id - AND NOT EXISTS (SELECT 1 FROM "container_repositories" WHERE container_repositories.project_id = container_expiration_policies.project_id) - SQL - execute(sql) - end - end - - private - - def execute(sql) - ApplicationRecord - .connection - .execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb index 2eb7c5230ba..276c7a1c6fa 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb @@ -23,8 +23,9 @@ module Gitlab .joins('LEFT OUTER JOIN project_statistics ON project_statistics.project_id = projects.id') .joins('LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id') .joins('LEFT OUTER JOIN issues ON issues.project_id = projects.id') - .where('project_statistics.repository_size' => 0, - 'project_settings.legacy_open_source_license_available' => true) + .where( + 'project_statistics.repository_size' => 0, + 'project_settings.legacy_open_source_license_available' => true) .group('projects.id') .having('COUNT(issues.id) = 0') diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb index 8953836c705..7661ae4b5ad 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb @@ -23,8 +23,9 @@ module Gitlab .joins('LEFT OUTER JOIN project_statistics ON project_statistics.project_id = projects.id') .joins('LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id') .joins('LEFT OUTER JOIN project_authorizations ON project_authorizations.project_id = projects.id') - .where('project_statistics.repository_size' => 0, - 'project_settings.legacy_open_source_license_available' => true) + .where( + 'project_statistics.repository_size' => 0, + 'project_settings.legacy_open_source_license_available' => true) .group('projects.id') .having('COUNT(project_authorizations.user_id) = 1') diff --git a/lib/gitlab/background_migration/drop_invalid_remediations.rb b/lib/gitlab/background_migration/drop_invalid_remediations.rb deleted file mode 100644 index f0a0de586f5..00000000000 --- a/lib/gitlab/background_migration/drop_invalid_remediations.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class DropInvalidRemediations - def perform(start_id, stop_id) - end - end - # rubocop: enable Style/Documentation - end -end - -Gitlab::BackgroundMigration::DropInvalidRemediations.prepend_mod_with('Gitlab::BackgroundMigration::DropInvalidRemediations') diff --git a/lib/gitlab/background_migration/drop_invalid_security_findings.rb b/lib/gitlab/background_migration/drop_invalid_security_findings.rb deleted file mode 100644 index 000628e109c..00000000000 --- a/lib/gitlab/background_migration/drop_invalid_security_findings.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module BackgroundMigration - # Drop rows from security_findings where the uuid is NULL - class DropInvalidSecurityFindings - # rubocop:disable Style/Documentation - class SecurityFinding < ActiveRecord::Base - include ::EachBatch - self.table_name = 'security_findings' - scope :no_uuid, -> { where(uuid: nil) } - end - # rubocop:enable Style/Documentation - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - ranged_query = SecurityFinding - .where(id: start_id..end_id) - .no_uuid - - ranged_query.each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - - # The query need to be reconstructed because .each_batch modifies the default scope - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 - SecurityFinding.unscoped - .where(id: first..last) - .no_uuid - .delete_all - - sleep PAUSE_SECONDS - end - - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb deleted file mode 100644 index 293530f6536..00000000000 --- a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::DropInvalidVulnerabilities - # rubocop: disable Gitlab/NamespacedClass - class Vulnerability < ActiveRecord::Base - self.table_name = "vulnerabilities" - has_many :findings, class_name: 'VulnerabilitiesFinding', inverse_of: :vulnerability - end - - class VulnerabilitiesFinding < ActiveRecord::Base - self.table_name = "vulnerability_occurrences" - belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' - end - # rubocop: enable Gitlab/NamespacedClass - - # rubocop: disable CodeReuse/ActiveRecord - def perform(start_id, end_id) - Vulnerability - .where(id: start_id..end_id) - .left_joins(:findings) - .where(vulnerability_occurrences: { vulnerability_id: nil }) - .delete_all - - mark_job_as_succeeded(start_id, end_id) - end - # rubocop: enable CodeReuse/ActiveRecord - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'DropInvalidVulnerabilities', - arguments - ) - end -end diff --git a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb index b6e22e481fa..237c655a48a 100644 --- a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb +++ b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb @@ -18,8 +18,7 @@ module Gitlab mode: :per_attribute_iv, algorithm: 'aes-256-gcm', key: Settings.attr_encrypted_db_key_base_32, - encode: false, - encode_vi: false + encode: false before_save :copy_token_to_encrypted_token diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb index c9582da2a51..28c28ae48eb 100644 --- a/lib/gitlab/background_migration/encrypt_integration_properties.rb +++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb @@ -18,14 +18,14 @@ module Gitlab scope :for_batch, ->(range) { where(id: range) } attr_encrypted :encrypted_properties_tmp, - attribute: :encrypted_properties, - mode: :per_attribute_iv, - key: ::Settings.attr_encrypted_db_key_base_32, - algorithm: ALGORITHM, - marshal: true, - marshaler: ::Gitlab::Json, - encode: false, - encode_iv: false + attribute: :encrypted_properties, + mode: :per_attribute_iv, + key: ::Settings.attr_encrypted_db_key_base_32, + algorithm: ALGORITHM, + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false # See 'Integration#reencrypt_properties' def encrypt_properties diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb deleted file mode 100644 index 31b5b5cdb73..00000000000 --- a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to extract the project topics into a separate `topics` table - class ExtractProjectTopicsIntoSeparateTable - # Temporary AR table for tags - class Tag < ActiveRecord::Base - self.table_name = 'tags' - end - - # Temporary AR table for taggings - class Tagging < ActiveRecord::Base - self.table_name = 'taggings' - belongs_to :tag - end - - # Temporary AR table for topics - class Topic < ActiveRecord::Base - self.table_name = 'topics' - end - - # Temporary AR table for project topics - class ProjectTopic < ActiveRecord::Base - self.table_name = 'project_topics' - belongs_to :topic - end - - # Temporary AR table for projects - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - def perform(start_id, stop_id) - Tagging.includes(:tag).where(taggable_type: 'Project', id: start_id..stop_id).each do |tagging| - if Project.exists?(id: tagging.taggable_id) && tagging.tag - begin - topic = Topic.find_or_create_by(name: tagging.tag.name) - project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic) - - tagging.delete if project_topic.persisted? - rescue StandardError => e - Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id) - end - else - tagging.delete - end - end - - mark_job_as_succeeded(start_id, stop_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb index 4b6bb12c91b..afd5e18ed7d 100644 --- a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb +++ b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb @@ -69,14 +69,14 @@ module Gitlab self.table_name = 'packages_packages' has_many :package_files, - class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' # rubocop:disable Layout/LineLength + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' end class PackageFile < ::ApplicationRecord self.table_name = 'packages_package_files' belongs_to :package, - class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' # rubocop:disable Layout/LineLength + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' def self.sum_query packages = FixIncoherentPackagesSizeOnProjectStatistics::Package.arel_table diff --git a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb deleted file mode 100644 index 4df55a7b02a..00000000000 --- a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration for fixing merge_request_diff_commit rows that don't - # have committer/author details due to - # https://gitlab.com/gitlab-org/gitlab/-/issues/344080. - class FixMergeRequestDiffCommitUsers - BATCH_SIZE = 100 - - def initialize - @commits = {} - @users = {} - end - - def perform(project_id) - # No-op, see https://gitlab.com/gitlab-org/gitlab/-/issues/344540 - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb b/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb new file mode 100644 index 00000000000..5b3b5642ba8 --- /dev/null +++ b/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes existing `vulnerability_reads` records which did not have `has_issues` + # correctly set at the time of creation. + class FixVulnerabilityReadsHasIssues < BatchedMigrationJob + operation_name :fix_has_issues + feature_category :vulnerability_management + + # rubocop:disable Style/Documentation + class VulnerabilityRead < ::ApplicationRecord + self.table_name = 'vulnerability_reads' + + scope :with_vulnerability_ids, ->(ids) { where(vulnerability_id: ids) } + scope :without_issues, -> { where(has_issues: false) } + end + # rubocop:enable Style/Documentation + + def perform + each_sub_batch do |sub_batch| + vulnerability_reads_with_issue_links(sub_batch).update_all('has_issues = true') + end + end + + private + + def vulnerability_reads_with_issue_links(sub_batch) + VulnerabilityRead.with_vulnerability_ids(sub_batch.select(:vulnerability_id)).without_issues + end + end + end +end diff --git a/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb new file mode 100644 index 00000000000..21ca4392003 --- /dev/null +++ b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates internal_ids records for `usage: issues` from project to namespace scope. + # For project issues it will be project namespace, for group issues it will be group namespace. + class IssuesInternalIdScopeUpdater < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :issues_internal_id_scope_updater + feature_category :database + + ISSUES_USAGE = 0 # see Enums::InternalId#usage_resources[:issues] + + scope_to ->(relation) do + relation.where(usage: ISSUES_USAGE).where.not(project_id: nil) + end + + def perform + each_sub_batch do |sub_batch| + create_namespace_scoped_records(sub_batch) + delete_project_scoped_records(sub_batch) + end + end + + private + + def delete_project_scoped_records(sub_batch) + # There is no need to keep the project scoped issues usage as we move to scoping issues to namespace. + # Also in case we do decide to move back to scoping issues usage to project, we are better off if the + # project record is not present as that would result in overlapping IIDs because project scoped issues + # usage will have outdated IIDs left in the DB + log_info("Deleted internal_ids records", ids: sub_batch.pluck(:id)) + + connection.execute( + <<~SQL + DELETE FROM internal_ids WHERE id IN (#{sub_batch.select(:id).to_sql}) + SQL + ) + end + + def create_namespace_scoped_records(sub_batch) + # Creates a corresponding namespace scoped record for every `issues` usage scoped to a project. + # On conflict it means the record was already created when a new issue is created with the + # newly namespace scoped Issue model, see Issue#has_internal_id definition. In which case to + # make sure we have the namespace_id scoped record set to the greatest of the two last_values. + created_records_ids = connection.execute( + <<~SQL + INSERT INTO internal_ids (usage, last_value, namespace_id) + SELECT #{ISSUES_USAGE}, last_value, project_namespace_id + FROM internal_ids + INNER JOIN projects ON projects.id = internal_ids.project_id + WHERE internal_ids.id IN(#{sub_batch.select(:id).to_sql}) + ON CONFLICT (usage, namespace_id) WHERE namespace_id IS NOT NULL + DO UPDATE SET last_value = GREATEST(EXCLUDED.last_value, internal_ids.last_value) + RETURNING id; + SQL + ) + + log_info("Created/updated internal_ids records", ids: created_records_ids.field_values('id')) + end + + def log_info(message, **extra) + ::Gitlab::BackgroundMigration::Logger.info(migrator: self.class.to_s, message: message, **extra) + end + end + end +end diff --git a/lib/gitlab/background_migration/logger.rb b/lib/gitlab/background_migration/logger.rb index 4ea89771eff..d338c214140 100644 --- a/lib/gitlab/background_migration/logger.rb +++ b/lib/gitlab/background_migration/logger.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Logger that can be used for migrations logging class Logger < ::Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'migrations' end diff --git a/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb new file mode 100644 index 00000000000..dd9fcf7fcfe --- /dev/null +++ b/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the evidence data into their own records from the json attribute + class MigrateEvidencesForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_evidences_for_vulnerability_findings + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::Finding::Evidence + class Evidence < ApplicationRecord + self.table_name = 'vulnerability_finding_evidences' + + # This data has been already validated when parsed into vulnerability_occurrences.raw_metadata + # Having this validation is a requerment from: + # https://gitlab.com/gitlab-org/gitlab/-/blob/dc3262f850cbd0ac14171d3c389b1258b4749cda/spec/db/schema_spec.rb#L253-265 + validates :data, json_schema: { filename: "filename" }, if: false + end + + def perform + each_sub_batch do |sub_batch| + migrate_evidences(sub_batch) + end + end + + private + + def migrate_evidences(sub_batch) + attrs = sub_batch.filter_map do |finding| + evidence = extract_evidence(finding.raw_metadata) + + next unless evidence + + build_evidence(finding, evidence) + end.compact + + create_evidences(attrs) if attrs.present? + end + + def build_evidence(finding, evidence) + current_time = Time.current + { + vulnerability_occurrence_id: finding.id, + data: evidence, + created_at: current_time, + updated_at: current_time + } + end + + def create_evidences(evidences) + Evidence.upsert_all(evidences, returning: false, unique_by: %i[vulnerability_occurrence_id]) + end + + def extract_evidence(metadata) + # This is required because postgres doesn't support the null unicode character, i.e., \u0000. + # The following is the actual error: + # PG::UntranslatableCharacter: ERROR: unsupported Unicode escape sequence + # DETAIL: \u0000 cannot be converted to text. + return if metadata.include?('\u0000') + + parsed_metadata = Gitlab::Json.parse(metadata) + + parsed_metadata['evidence'] + rescue JSON::ParserError + nil + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_human_user_type.rb b/lib/gitlab/background_migration/migrate_human_user_type.rb new file mode 100644 index 00000000000..2cb27225274 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_human_user_type.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates all users with user_type = nil to user_type = 0 + class MigrateHumanUserType < BatchedMigrationJob + OLD_TYPE_VALUE = nil + NEW_TYPE_VALUE = 0 + + operation_name :migrate_human_user_type + scope_to ->(relation) { relation.where(user_type: OLD_TYPE_VALUE) } + feature_category :user_management + + def perform + cleanup_gin_indexes('users') + + each_sub_batch do |sub_batch| + sub_batch.update_all(user_type: NEW_TYPE_VALUE) + end + end + + private + + def cleanup_gin_indexes(table_name) + sql = <<-SQL + SELECT indexname::text FROM pg_indexes WHERE tablename = '#{table_name}' AND indexdef ILIKE '%using gin%' + SQL + + index_names = ApplicationRecord.connection.select_values(sql) + + index_names.each do |index_name| + ApplicationRecord.connection.execute("SELECT gin_clean_pending_list('#{index_name}')") + end + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb new file mode 100644 index 00000000000..0b79bc143db --- /dev/null +++ b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the link data into their own records from the json attribute + class MigrateLinksForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_links_for_vulnerability_findings + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::FindingLink + class Link < ApplicationRecord + self.table_name = 'vulnerability_finding_links' + end + + def perform + each_sub_batch(batching_scope: ->(relation) { relation.select(:id, :raw_metadata) }) do |findings| + migrate_remediations( + findings, + Link + .where(vulnerability_occurrence_id: findings.map(&:id)) + .group(:vulnerability_occurrence_id, :name, :url) + .count + ) + end + end + + private + + def migrate_remediations(findings, existing_links) + findings.each do |finding| + create_links(build_links_from(finding, existing_links)) + rescue ActiveRecord::StatementInvalid => e + logger.error( + message: e.message, + class: self.class.name, + model_id: finding.id + ) + end + end + + def build_link(finding, link) + current_time = Time.current + { + vulnerability_occurrence_id: finding.id, + name: link['name'], + url: link['url'], + created_at: current_time, + updated_at: current_time + } + end + + def build_links_from(finding, existing_links) + extract_links(finding.raw_metadata).filter_map do |link| + key = [finding.id, link['name'], link['url']] + build_link(finding, link) unless existing_links.key?(key) + end + end + + def create_links(attributes) + return if attributes.empty? + + Link.upsert_all(attributes, returning: false) + end + + def extract_links(metadata) + parsed_metadata = Gitlab::Json.parse(metadata) + parsed_links = Array.wrap(parsed_metadata['links']) + + return [] if parsed_links.blank? + + parsed_links.select { |link| link.try(:[], 'url').present? }.uniq + rescue JSON::ParserError => e + logger.warn( + message: e.message, + class: self.class.name + ) + [] + end + + def logger + @logger ||= ::Gitlab::AppLogger + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb deleted file mode 100644 index 7d150b9cd83..00000000000 --- a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb +++ /dev/null @@ -1,296 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Migrates author and committer names and emails from - # merge_request_diff_commits to two columns that point to - # merge_request_diff_commit_users. - # - # rubocop: disable Metrics/ClassLength - class MigrateMergeRequestDiffCommitUsers - # The number of user rows in merge_request_diff_commit_users to get in a - # single query. - USER_ROWS_PER_QUERY = 1_000 - - # The number of rows in merge_request_diff_commits to get in a single - # query. - COMMIT_ROWS_PER_QUERY = 1_000 - - # The number of rows in merge_request_diff_commits to update in a single - # query. - # - # Tests in staging revealed that increasing the number of updates per - # query translates to a longer total runtime for a migration. For example, - # given the same range of rows to migrate, 1000 updates per query required - # a total of roughly 15 seconds. On the other hand, 5000 updates per query - # required a total of roughly 25 seconds. For this reason, we use a value - # of 1000 rows per update. - UPDATES_PER_QUERY = 1_000 - - # rubocop: disable Style/Documentation - class MergeRequestDiffCommit < ActiveRecord::Base - include FromUnion - extend ::SuppressCompositePrimaryKeyWarning - - self.table_name = 'merge_request_diff_commits' - - # Yields each row to migrate in the given range. - # - # This method uses keyset pagination to ensure we don't retrieve - # potentially tens of thousands (or even hundreds of thousands) of rows - # in a single query. Such queries could time out, or increase the amount - # of memory needed to process the data. - # - # We can't use `EachBatch` and similar approaches, as - # merge_request_diff_commits doesn't have a single monotonically - # increasing primary key. - def self.each_row_to_migrate(start_id, stop_id, &block) - order = Pagination::Keyset::Order.build( - %w[merge_request_diff_id relative_order].map do |col| - Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: col, - order_expression: self.arel_table[col.to_sym].asc, - nullable: :not_nullable, - distinct: false - ) - end - ) - - scope = MergeRequestDiffCommit - .where(merge_request_diff_id: start_id...stop_id) - .order(order) - - Pagination::Keyset::Iterator - .new(scope: scope, use_union_optimization: true) - .each_batch(of: COMMIT_ROWS_PER_QUERY) { |rows| rows.each(&block) } - end - end - # rubocop: enable Style/Documentation - - # rubocop: disable Style/Documentation - class MergeRequestDiffCommitUser < ActiveRecord::Base - self.table_name = 'merge_request_diff_commit_users' - - def self.union(queries) - from("(#{queries.join("\nUNION ALL\n")}) #{table_name}") - end - end - # rubocop: enable Style/Documentation - - def perform(start_id, stop_id) - return if already_processed?(start_id, stop_id) - - # This Hash maps user names + emails to their corresponding rows in - # merge_request_diff_commit_users. - user_mapping = {} - - user_details, diff_rows_to_update = get_data_to_update(start_id, stop_id) - - get_user_rows_in_batches(user_details, user_mapping) - create_missing_users(user_details, user_mapping) - update_commit_rows(diff_rows_to_update, user_mapping) - - Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'MigrateMergeRequestDiffCommitUsers', - [start_id, stop_id] - ) - end - - def already_processed?(start_id, stop_id) - Database::BackgroundMigrationJob - .for_migration_execution('MigrateMergeRequestDiffCommitUsers', [start_id, stop_id]) - .succeeded - .any? - end - - # Returns the data we'll use to determine what merge_request_diff_commits - # rows to update, and what data to use for populating their - # commit_author_id and committer_id columns. - def get_data_to_update(start_id, stop_id) - # This Set is used to retrieve users that already exist in - # merge_request_diff_commit_users. - users = Set.new - - # This Hash maps the primary key of every row in - # merge_request_diff_commits to the (trimmed) author and committer - # details to use for updating the row. - to_update = {} - - MergeRequestDiffCommit.each_row_to_migrate(start_id, stop_id) do |row| - author = [prepare(row.author_name), prepare(row.author_email)] - committer = [prepare(row.committer_name), prepare(row.committer_email)] - - to_update[[row.merge_request_diff_id, row.relative_order]] = - [author, committer] - - users << author if author[0] || author[1] - users << committer if committer[0] || committer[1] - end - - [users, to_update] - end - - # Gets any existing rows in merge_request_diff_commit_users in batches. - # - # This method may end up having to retrieve lots of rows. To reduce the - # overhead, we batch queries into a UNION query. We limit the number of - # queries per UNION so we don't end up sending a single query containing - # too many SELECT statements. - def get_user_rows_in_batches(users, user_mapping) - users.each_slice(USER_ROWS_PER_QUERY) do |pairs| - queries = pairs.map do |(name, email)| - MergeRequestDiffCommitUser.where(name: name, email: email).to_sql - end - - MergeRequestDiffCommitUser.union(queries).each do |row| - user_mapping[[row.name.to_s, row.email.to_s]] = row - end - end - end - - # Creates any users for which no row exists in - # merge_request_diff_commit_users. - # - # Not all users queried may exist yet, so we need to create any missing - # ones; making sure we handle concurrent creations of the same user - def create_missing_users(users, mapping) - create = [] - - users.each do |(name, email)| - create << { name: name, email: email } unless mapping[[name, email]] - end - - return if create.empty? - - MergeRequestDiffCommitUser - .insert_all(create, returning: %w[id name email]) - .each do |row| - mapping[[row['name'], row['email']]] = MergeRequestDiffCommitUser - .new(id: row['id'], name: row['name'], email: row['email']) - end - - # It's possible for (name, email) pairs to be inserted concurrently, - # resulting in the above insert not returning anything. Here we get any - # remaining users that were created concurrently. - get_user_rows_in_batches( - users.reject { |pair| mapping.key?(pair) }, - mapping - ) - end - - # Updates rows in merge_request_diff_commits with their new - # commit_author_id and committer_id values. - def update_commit_rows(to_update, user_mapping) - to_update.each_slice(UPDATES_PER_QUERY) do |slice| - updates = {} - - slice.each do |(diff_id, order), (author, committer)| - author_id = user_mapping[author]&.id - committer_id = user_mapping[committer]&.id - - updates[[diff_id, order]] = [author_id, committer_id] - end - - bulk_update_commit_rows(updates) - end - end - - # Bulk updates rows in the merge_request_diff_commits table with their new - # author and/or committer ID values. - # - # Updates are batched together to reduce the overhead of having to produce - # a single UPDATE for every row, as we may end up having to update - # thousands of rows at once. - # - # The query produced by this method is along the lines of the following: - # - # UPDATE merge_request_diff_commits - # SET commit_author_id = - # CASE - # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN X - # WHEN ... - # END, - # committer_id = - # CASE - # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN Y - # WHEN ... - # END - # WHERE (merge_request_diff_id, relative_order) IN ( (x, y), ... ) - # - # The `mapping` argument is a Hash in the following format: - # - # { [merge_request_diff_id, relative_order] => [author_id, committer_id] } - # - # rubocop: disable Metrics/AbcSize - def bulk_update_commit_rows(mapping) - author_case = Arel::Nodes::Case.new - committer_case = Arel::Nodes::Case.new - primary_values = [] - - mapping.each do |diff_id_and_order, (author_id, committer_id)| - primary_value = Arel::Nodes::Grouping.new(diff_id_and_order) - - primary_values << primary_value - - if author_id - author_case.when(primary_key.eq(primary_value)).then(author_id) - end - - if committer_id - committer_case.when(primary_key.eq(primary_value)).then(committer_id) - end - end - - if author_case.conditions.empty? && committer_case.conditions.empty? - return - end - - fields = [] - - # Statements such as `SET x = CASE END` are not valid SQL statements, so - # we omit setting an ID field if there are no values to populate it - # with. - if author_case.conditions.any? - fields << [arel_table[:commit_author_id], author_case] - end - - if committer_case.conditions.any? - fields << [arel_table[:committer_id], committer_case] - end - - query = Arel::UpdateManager.new - .table(arel_table) - .where(primary_key.in(primary_values)) - .set(fields) - .to_sql - - MergeRequestDiffCommit.connection.execute(query) - end - # rubocop: enable Metrics/AbcSize - - def primary_key - Arel::Nodes::Grouping.new( - [arel_table[:merge_request_diff_id], arel_table[:relative_order]] - ) - end - - def arel_table - MergeRequestDiffCommit.arel_table - end - - # Prepares a value to be inserted into a column in the table - # `merge_request_diff_commit_users`. Values in this table are limited to - # 512 characters. - # - # We treat empty strings as NULL values, as there's no point in (for - # example) storing a row where both the name and Email are an empty - # string. In addition, if we treated them differently we could end up with - # two rows: one where field X is NULL, and one where field X is an empty - # string. This is redundant, so we avoid storing such data. - def prepare(value) - value.present? ? value[0..511] : nil - end - end - # rubocop: enable Metrics/ClassLength - end -end diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb deleted file mode 100644 index 68bbd3cfebb..00000000000 --- a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to migrate the context of project taggings from `tags` to `topics` - class MigrateProjectTaggingsContextFromTagsToTopics - # Temporary AR table for taggings - class Tagging < ActiveRecord::Base - include EachBatch - - self.table_name = 'taggings' - end - - def perform(start_id, stop_id) - Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation| - relation.update_all(context: 'topics') - end - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb new file mode 100644 index 00000000000..9eadef96db6 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +module Vulnerabilities + # The class is mimicking Vulnerabilites::Remediation + class Remediation < ApplicationRecord + include FileStoreMounter + include ShaAttribute + + self.table_name = 'vulnerability_remediations' + + sha_attribute :checksum + + mount_file_store_uploader AttachmentUploader + + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end + end +end + +module Gitlab + module BackgroundMigration + # The class to migrate the remediation data into their own records from the json attribute + class MigrateRemediationsForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_remediations_for_vulnerability_findings + + # The class to encapsulate checksum and file for uploading + class DiffFile < StringIO + # This method is used by the `carrierwave` gem + def original_filename + @original_filename ||= self.class.original_filename(checksum) + end + + def checksum + @checksum ||= self.class.checksum(string) + end + + def self.checksum(value) + Digest::SHA256.hexdigest(value) + end + + def self.original_filename(checksum) + "#{checksum}.diff" + end + end + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::FindingRemediation + class FindingRemediation < ApplicationRecord + self.table_name = 'vulnerability_findings_remediations' + end + + def perform + each_sub_batch do |sub_batch| + migrate_remediations(sub_batch) + end + end + + private + + def migrate_remediations(sub_batch) + sub_batch.each do |finding| + FindingRemediation.transaction do + remediations = append_remediations_diff_checksum(finding.raw_metadata) + + result_ids = create_remediations(finding, remediations) + + create_finding_remediations(finding.id, result_ids) + end + rescue StandardError => e + logger.error( + message: e.message, + class: self.class.name, + model_id: finding.id + ) + end + end + + def create_finding_remediations(finding_id, result_ids) + attrs = result_ids.map do |result_id| + build_finding_remediation_attrs(finding_id, result_id) + end + + return unless attrs.present? + + FindingRemediation.upsert_all( + attrs, + returning: false, + unique_by: [:vulnerability_occurrence_id, :vulnerability_remediation_id] + ) + end + + def create_remediations(finding, remediations) + attrs = remediations.map do |remediation| + build_remediation_attrs(finding, remediation) + end + + return [] unless attrs.present? + + ids_checksums = ::Vulnerabilities::Remediation.upsert_all( + attrs, + returning: %w[id checksum], + unique_by: [:project_id, :checksum] + ) + + ids_checksums.each do |id_checksum| + upload_file(id_checksum['id'], id_checksum['checksum'], remediations) + end + + ids_checksums.pluck('id') + end + + def upload_file(id, checksum, remediations) + deserialized_checksum = Gitlab::Database::ShaAttribute.new.deserialize(checksum) + diff = remediations.find { |rem| rem['checksum'] == deserialized_checksum }["diff"] + file = DiffFile.new(diff) + ::Vulnerabilities::Remediation.find_by(id: id).update!(file: file) + end + + def build_remediation_attrs(finding, remediation) + { + project_id: finding.project_id, + summary: remediation['summary'], + file: DiffFile.original_filename(remediation['checksum']), + checksum: remediation['checksum'], + created_at: Time.current, + updated_at: Time.current + } + end + + def build_finding_remediation_attrs(finding_id, remediation_id) + { + vulnerability_occurrence_id: finding_id, + vulnerability_remediation_id: remediation_id, + created_at: Time.current, + updated_at: Time.current + } + end + + def append_remediations_diff_checksum(metadata) + parsed_metadata = Gitlab::Json.parse(metadata) + + return [] unless parsed_metadata['remediations'] + + parsed_metadata['remediations'].filter_map do |remediation| + next unless remediation && remediation['diff'].present? + + remediation.merge('checksum' => DiffFile.checksum(remediation['diff'])) + end.compact.uniq + end + + def logger + @logger ||= ::Gitlab::AppLogger + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb b/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb new file mode 100644 index 00000000000..6a9f1692b72 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MigrateSharedVulnerabilityIdentifiers < BatchedMigrationJob + # rubocop: enable Style/Documentation + + feature_category :vulnerability_management + + def perform; end + end + end +end + +# rubocop: disable Layout/LineLength +Gitlab::BackgroundMigration::MigrateSharedVulnerabilityIdentifiers.prepend_mod_with("Gitlab::BackgroundMigration::MigrateSharedVulnerabilityIdentifiers") +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb deleted file mode 100644 index 83aa36a11e6..00000000000 --- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateU2fWebauthn - class U2fRegistration < ActiveRecord::Base - self.table_name = 'u2f_registrations' - end - - class WebauthnRegistration < ActiveRecord::Base - self.table_name = 'webauthn_registrations' - end - - def perform(start_id, end_id) - old_registrations = U2fRegistration.where(id: start_id..end_id) - old_registrations.each_slice(100) do |slice| - values = slice.map do |u2f_registration| - converter = Gitlab::Auth::U2fWebauthnConverter.new(u2f_registration) - converter.convert - end - - WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb deleted file mode 100644 index 06422ed282f..00000000000 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration moves projects.container_registry_enabled values to - # project_features.container_registry_access_level for the projects within - # the given range of ids. - class MoveContainerRegistryEnabledToProjectFeature - MAX_BATCH_SIZE = 300 - - ENABLED = 20 - DISABLED = 0 - - def perform(from_id, to_id) - (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch| - process_batch(batch.first, batch.last) - end - - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('MoveContainerRegistryEnabledToProjectFeature', [from_id, to_id]) - end - - private - - def process_batch(from_id, to_id) - ApplicationRecord.connection.execute(update_sql(from_id, to_id)) - - logger.info(message: "#{self.class}: Copied container_registry_enabled values for projects with IDs between #{from_id}..#{to_id}") - end - - # For projects that have a project_feature: - # Set project_features.container_registry_access_level to ENABLED (20) or DISABLED (0) - # depending if container_registry_enabled is true or false. - def update_sql(from_id, to_id) - <<~SQL - UPDATE project_features - SET container_registry_access_level = (CASE p.container_registry_enabled - WHEN true THEN #{ENABLED} - WHEN false THEN #{DISABLED} - ELSE #{DISABLED} - END) - FROM projects p - WHERE project_id = p.id AND - project_id BETWEEN #{from_id} AND #{to_id} - SQL - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb deleted file mode 100644 index 2495cb51364..00000000000 --- a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - SUB_BATCH_SIZE = 1_000 - - # The class to populates the total projects counter cache of topics - class PopulateTopicsTotalProjectsCountCache - # Temporary AR model for topics - class Topic < ActiveRecord::Base - include EachBatch - - self.table_name = 'topics' - end - - def perform(start_id, stop_id) - Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ApplicationRecord.connection.execute(<<~SQL) - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) - UPDATE topics - SET total_projects_count = (SELECT COUNT(*) FROM project_topics WHERE topic_id = batched_relation.id) - FROM batched_relation - WHERE topics.id = batched_relation.id - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb deleted file mode 100644 index 175966b940d..00000000000 --- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class PopulateUuidsForSecurityFindings - NOP_RELATION = Class.new { def each_batch(*); end } - - def self.security_findings - NOP_RELATION.new - end - - def perform(*_scan_ids); end - end - end -end - -Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_mod_with('Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') diff --git a/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb new file mode 100644 index 00000000000..ee0f73cc3de --- /dev/null +++ b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Populates missing dismissal information for vulnerabilities. + class PopulateVulnerabilityDismissalFields < BatchedMigrationJob + feature_category :vulnerability_management + scope_to ->(relation) { relation.where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') } + operation_name :populate_vulnerability_dismissal_fields + + # rubocop:disable Style/Documentation + class Vulnerability < ApplicationRecord + self.table_name = 'vulnerabilities' + + has_one :finding, class_name: 'Finding' + + def copy_dismissal_information + return unless finding&.dismissal_feedback + + update_columns( + dismissed_at: finding.dismissal_feedback.created_at, + dismissed_by_id: finding.dismissal_feedback.author_id + ) + end + end + + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: "filename" } + + def dismissal_feedback + Feedback.dismissal.where(finding_uuid: uuid).first + end + end + + class Feedback < ApplicationRecord + DISMISSAL_TYPE = 0 # dismissal + + self.table_name = 'vulnerability_feedback' + + scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) } + end + # rubocop:enable Style/Documentation + + def perform + each_sub_batch do |sub_batch| + vulnerability_ids = sub_batch.pluck(:id) + Vulnerability.includes(:finding).where(id: vulnerability_ids).each do |vulnerability| + populate_for(vulnerability) + end + + log_info(vulnerability_ids) + end + end + + private + + def populate_for(vulnerability) + log_warning(vulnerability) unless vulnerability.copy_dismissal_information + rescue StandardError => error + log_error(error, vulnerability) + end + + def log_info(vulnerability_ids) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: 'Dismissal information has been copied', + count: vulnerability_ids.length + ) + end + + def log_warning(vulnerability) + ::Gitlab::BackgroundMigration::Logger.warn( + migrator: self.class.name, + message: 'Could not update vulnerability!', + vulnerability_id: vulnerability.id + ) + end + + def log_error(error, vulnerability) + ::Gitlab::BackgroundMigration::Logger.error( + migrator: self.class.name, + message: error.message, + vulnerability_id: vulnerability.id + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb deleted file mode 100644 index 15799659b55..00000000000 --- a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings - DELETE_BATCH_SIZE = 50 - - # rubocop:disable Gitlab/NamespacedClass - class VulnerabilitiesFinding < ActiveRecord::Base - self.table_name = "vulnerability_occurrences" - end - # rubocop:enable Gitlab/NamespacedClass - - # rubocop:disable Gitlab/NamespacedClass - class Vulnerability < ActiveRecord::Base - self.table_name = "vulnerabilities" - end - # rubocop:enable Gitlab/NamespacedClass - - def perform(start_id, end_id) - batch = VulnerabilitiesFinding.where(id: start_id..end_id) - - cte = Gitlab::SQL::CTE.new(:batch, batch.select(:report_type, :location_fingerprint, :primary_identifier_id, :project_id)) - - query = VulnerabilitiesFinding - .select('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id', 'array_agg(id) as ids') - .distinct - .with(cte.to_arel) - .from(cte.alias_to(Arel.sql('batch'))) - .joins( - %( - INNER JOIN - vulnerability_occurrences ON - vulnerability_occurrences.report_type = batch.report_type AND - vulnerability_occurrences.location_fingerprint = batch.location_fingerprint AND - vulnerability_occurrences.primary_identifier_id = batch.primary_identifier_id AND - vulnerability_occurrences.project_id = batch.project_id - )).group('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id') - .having('COUNT(*) > 1') - - ids_to_delete = [] - - query.to_a.each do |record| - # We want to keep the latest finding since it might have recent metadata - duplicate_ids = record.ids.uniq.sort - duplicate_ids.pop - ids_to_delete.concat(duplicate_ids) - - if ids_to_delete.size == DELETE_BATCH_SIZE - delete_findings_and_vulnerabilities(ids_to_delete) - ids_to_delete.clear - end - end - - delete_findings_and_vulnerabilities(ids_to_delete) if ids_to_delete.any? - end - - private - - def delete_findings_and_vulnerabilities(ids) - vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).compact - VulnerabilitiesFinding.where(id: ids).delete_all - Vulnerability.where(id: vulnerability_ids).delete_all - end -end diff --git a/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb index 7fe5a427d10..f4f54e2b2eb 100644 --- a/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb +++ b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb @@ -53,7 +53,7 @@ class Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerab def mark_job_as_succeeded(*arguments) Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( self.class.name.demodulize, - arguments + arguments ) end end diff --git a/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb new file mode 100644 index 00000000000..879e52c96bf --- /dev/null +++ b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to remove `project_group_links` records whose associated group + # does not exist in `namespaces` table anymore. + class RemoveProjectGroupLinkWithMissingGroups < Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) { relation } + operation_name :delete_all + feature_category :subgroups + + def perform + each_sub_batch do |sub_batch| + records = sub_batch.joins( + "LEFT OUTER JOIN namespaces ON namespaces.id = project_group_links.group_id AND namespaces.type = 'Group'" + ).where(namespaces: { id: nil }) + + ids = records.map(&:id) + + next if ids.empty? + + Gitlab::AppLogger.info({ message: 'Removing project group link with non-existent groups', + deleted_count: ids.count, + ids: ids }) + + records.delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb index 0dbe2781327..56506814dc0 100644 --- a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -36,8 +36,8 @@ module Gitlab included do has_one :route, - as: :source, - class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route' + as: :source, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route' end def full_path @@ -67,7 +67,7 @@ module Gitlab self.inheritance_column = :_type_disabled belongs_to :parent, - class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' def self.polymorphic_name 'Namespace' @@ -80,7 +80,7 @@ module Gitlab self.table_name = 'projects' belongs_to :namespace, - class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' alias_method :parent, :namespace alias_attribute :parent_id, :namespace_id @@ -92,7 +92,7 @@ module Gitlab self.table_name = 'container_repositories' belongs_to :project, - class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project' + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project' def tags? result = ContainerRegistry.tags_for(path).any? diff --git a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb deleted file mode 100644 index 43a7032e682..00000000000 --- a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A background migration that finished any pending - # MigrateMergeRequestDiffCommitUsers jobs, and schedules new jobs itself. - # - # This migration exists so we can bypass rescheduling issues (e.g. jobs - # getting dropped after too many retries) that may occur when - # MigrateMergeRequestDiffCommitUsers jobs take longer than expected. - class StealMigrateMergeRequestDiffCommitUsers - def perform(start_id, stop_id) - MigrateMergeRequestDiffCommitUsers.new.perform(start_id, stop_id) - schedule_next_job - end - - def schedule_next_job - next_job = Database::BackgroundMigrationJob - .for_migration_class('MigrateMergeRequestDiffCommitUsers') - .pending - .first - - return unless next_job - - BackgroundMigrationWorker.perform_in( - 5.minutes, - 'StealMigrateMergeRequestDiffCommitUsers', - next_job.arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_timelogs_project_id.rb b/lib/gitlab/background_migration/update_timelogs_project_id.rb deleted file mode 100644 index 69bb5cf6e6d..00000000000 --- a/lib/gitlab/background_migration/update_timelogs_project_id.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class to populate project_id for timelogs - class UpdateTimelogsProjectId - BATCH_SIZE = 1000 - - def perform(start_id, stop_id) - (start_id..stop_id).step(BATCH_SIZE).each do |offset| - update_issue_timelogs(offset, offset + BATCH_SIZE) - update_merge_request_timelogs(offset, offset + BATCH_SIZE) - end - end - - def update_issue_timelogs(batch_start, batch_stop) - execute(<<~SQL) - UPDATE timelogs - SET project_id = issues.project_id - FROM issues - WHERE issues.id = timelogs.issue_id - AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} - AND timelogs.project_id IS NULL; - SQL - end - - def update_merge_request_timelogs(batch_start, batch_stop) - execute(<<~SQL) - UPDATE timelogs - SET project_id = merge_requests.target_project_id - FROM merge_requests - WHERE merge_requests.id = timelogs.merge_request_id - AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} - AND timelogs.project_id IS NULL; - SQL - end - - def execute(sql) - @connection ||= ApplicationRecord.connection - @connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb b/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb deleted file mode 100644 index 10db9f5064a..00000000000 --- a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class UpdateUsersWhereTwoFactorAuthRequiredFromGroup # rubocop:disable Metrics/ClassLength - def perform(start_id, stop_id) - ApplicationRecord.connection.execute <<~SQL - UPDATE - users - SET - require_two_factor_authentication_from_group = TRUE - WHERE - users.id BETWEEN #{start_id} - AND #{stop_id} - AND users.require_two_factor_authentication_from_group = FALSE - AND users.id IN ( - SELECT - DISTINCT users_groups_query.user_id - FROM - ( - SELECT - users.id AS user_id, - members.source_id AS group_ids - FROM - users - LEFT JOIN members ON members.source_type = 'Namespace' - AND members.requested_at IS NULL - AND members.user_id = users.id - AND members.type = 'GroupMember' - WHERE - users.require_two_factor_authentication_from_group = FALSE - AND users.id BETWEEN #{start_id} - AND #{stop_id}) AS users_groups_query - INNER JOIN LATERAL ( - WITH RECURSIVE "base_and_ancestors" AS ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = users_groups_query.group_ids - ) - UNION - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces", - "base_and_ancestors" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = "base_and_ancestors"."parent_id" - ) - ), - "base_and_descendants" AS ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = users_groups_query.group_ids - ) - UNION - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces", - "base_and_descendants" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."parent_id" = "base_and_descendants"."id" - ) - ) - SELECT - "namespaces".* - FROM - ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "base_and_ancestors" AS "namespaces" - WHERE - "namespaces"."type" = 'Group' - ) - UNION - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "base_and_descendants" AS "namespaces" - WHERE - "namespaces"."type" = 'Group' - ) - ) namespaces - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."require_two_factor_authentication" = TRUE - ) AS hierarchy_tree ON TRUE - ); - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb b/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb deleted file mode 100644 index 458e0537f1c..00000000000 --- a/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class UpdateVulnerabilityOccurrencesLocation - def perform(start_id, stop_id) - end - end - # rubocop: enable Style/Documentation - end -end - -Gitlab::BackgroundMigration::UpdateVulnerabilityOccurrencesLocation.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilityOccurrencesLocation') |