diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-20 17:22:11 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-20 17:22:11 +0300 |
commit | 0c872e02b2c822e3397515ec324051ff540f0cd5 (patch) | |
tree | ce2fb6ce7030e4dad0f4118d21ab6453e5938cdd /lib/gitlab/background_migration | |
parent | f7e05a6853b12f02911494c4b3fe53d9540d74fc (diff) |
Add latest changes from gitlab-org/gitlab@15-7-stable-eev15.7.0-rc42
Diffstat (limited to 'lib/gitlab/background_migration')
13 files changed, 369 insertions, 44 deletions
diff --git a/lib/gitlab/background_migration/backfill_environment_tiers.rb b/lib/gitlab/background_migration/backfill_environment_tiers.rb new file mode 100644 index 00000000000..6f381577274 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_environment_tiers.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class backfills the `environments.tier` column by using `guess_tier` logic. + # Environments created after 13.10 already have a value, however, environments created before 13.10 don't. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/300741 for more information. + class BackfillEnvironmentTiers < BatchedMigrationJob + operation_name :backfill_environment_tiers + + # Equivalent to `Environment#guess_tier` pattern matching. + PRODUCTION_TIER = 0 + STAGING_TIER = 1 + TESTING_TIER = 2 + DEVELOPMENT_TIER = 3 + OTHER_TIER = 4 + + TIER_REGEXP_PAIR = [ + { tier: DEVELOPMENT_TIER, regexp: '(dev|review|trunk)' }, + { tier: TESTING_TIER, regexp: '(test|tst|int|ac(ce|)pt|qa|qc|control|quality)' }, + { tier: STAGING_TIER, regexp: '(st(a|)g|mod(e|)l|pre|demo|non)' }, + { tier: PRODUCTION_TIER, regexp: '(pr(o|)d|live)' } + ].freeze + + def perform + TIER_REGEXP_PAIR.each do |pair| + each_sub_batch( + batching_scope: ->(relation) { relation.where(tier: nil).where("name ~* '#{pair[:regexp]}'") } # rubocop:disable GitlabSecurity/SqlInjection + ) do |sub_batch| + sub_batch.update_all(tier: pair[:tier]) + end + end + + each_sub_batch(batching_scope: ->(relation) { relation.where(tier: nil) }) do |sub_batch| + sub_batch.update_all(tier: OTHER_TIER) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_note_discussion_id.rb b/lib/gitlab/background_migration/backfill_note_discussion_id.rb index da2c31ebd11..ce2698b3cb8 100644 --- a/lib/gitlab/background_migration/backfill_note_discussion_id.rb +++ b/lib/gitlab/background_migration/backfill_note_discussion_id.rb @@ -33,8 +33,8 @@ module Gitlab private def update_discussion_ids(notes) - mapping = notes.each_with_object({}) do |note, hash| - hash[note] = { discussion_id: note.generate_discussion_id } + mapping = notes.index_with do |note| + { discussion_id: note.generate_discussion_id } end Gitlab::Database::BulkUpdate.execute(%i(discussion_id), mapping) diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb new file mode 100644 index 00000000000..1a3dd88ea31 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_without_uploads_size.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill storage_size for project_statistics + class BackfillProjectStatisticsStorageSizeWithoutUploadsSize < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + # no-op + end + end + end +end + +Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutUploadsSize.prepend_mod_with('Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithoutUploadsSize') # rubocop:disable Layout/LineLength diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 64401bc0674..973ab20f547 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -9,55 +9,68 @@ module Gitlab # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#job-arguments. class BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers + include Gitlab::ClassAttributes - def initialize( - start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: - ) + DEFAULT_FEATURE_CATEGORY = :database - @start_id = start_id - @end_id = end_id - @batch_table = batch_table - @batch_column = batch_column - @sub_batch_size = sub_batch_size - @pause_ms = pause_ms - @job_arguments = job_arguments - @connection = connection - end + class << self + def generic_instance(batch_table:, batch_column:, job_arguments: [], connection:) + new( + batch_table: batch_table, batch_column: batch_column, + job_arguments: job_arguments, connection: connection, + start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0 + ) + end - def self.generic_instance(batch_table:, batch_column:, job_arguments: [], connection:) - new( - batch_table: batch_table, batch_column: batch_column, - job_arguments: job_arguments, connection: connection, - start_id: 0, end_id: 0, sub_batch_size: 0, pause_ms: 0 - ) - end + def job_arguments_count + 0 + end - def self.job_arguments_count - 0 - end + def operation_name(operation) + define_method('operation_name') do + operation + end + end - def self.operation_name(operation) - define_method('operation_name') do - operation + def job_arguments(*args) + args.each.with_index do |arg, index| + define_method(arg) do + @job_arguments[index] + end + end + + define_singleton_method(:job_arguments_count) do + args.count + end end - end - def self.job_arguments(*args) - args.each.with_index do |arg, index| - define_method(arg) do - @job_arguments[index] + def scope_to(scope) + define_method(:filter_batch) do |relation| + instance_exec(relation, &scope) end end - define_singleton_method(:job_arguments_count) do - args.count + def feature_category(feature_category_name = nil) + if feature_category_name.present? + set_class_attribute(:feature_category, feature_category_name) + else + get_class_attribute(:feature_category) || DEFAULT_FEATURE_CATEGORY + end end end - def self.scope_to(scope) - define_method(:filter_batch) do |relation| - instance_exec(relation, &scope) - end + def initialize( + start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: + ) + + @start_id = start_id + @end_id = end_id + @batch_table = batch_table + @batch_column = batch_column + @sub_batch_size = sub_batch_size + @pause_ms = pause_ms + @job_arguments = job_arguments + @connection = connection end def filter_batch(relation) diff --git a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb new file mode 100644 index 00000000000..4b7b7d42c77 --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphans records whenever report_type equals to scan_finding (i.e., 4) + class DeleteOrphansApprovalMergeRequestRules < BatchedMigrationJob + scope_to ->(relation) { relation.where(report_type: 4) } + + operation_name :delete_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(security_orchestration_policy_configuration_id: nil).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb new file mode 100644 index 00000000000..33aa1a8d29d --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphans_approval_project_rules.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphans records whenever report_type equals to scan_finding (i.e., 4) + class DeleteOrphansApprovalProjectRules < BatchedMigrationJob + operation_name :delete_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(report_type: 4, security_orchestration_policy_configuration_id: nil).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb new file mode 100644 index 00000000000..dcef4f086e2 --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_five_mb.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for projects less than 5 MB + class DisableLegacyOpenSourceLicenseForProjectsLessThanFiveMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) do + relation + .where(legacy_open_source_license_available: true) + end + + operation_name :disable_legacy_open_source_license_for_projects_less_than_five_mb + + def perform + each_sub_batch do |sub_batch| + updates = { legacy_open_source_license_available: false, updated_at: Time.current } + + sub_batch + .joins('INNER JOIN project_statistics ON project_statistics.project_id = project_settings.project_id') + .where('project_statistics.repository_size < ?', 5.megabyte) + .update_all(updates) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb new file mode 100644 index 00000000000..4b283bae79d --- /dev/null +++ b/lib/gitlab/background_migration/fix_approval_project_rules_without_protected_branches.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't update approval project rules + # as this feature exists only in EE + class FixApprovalProjectRulesWithoutProtectedBranches < BatchedMigrationJob + def perform; end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::FixApprovalProjectRulesWithoutProtectedBranches.prepend_mod_with('Gitlab::BackgroundMigration::FixApprovalProjectRulesWithoutProtectedBranches') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/fix_security_scan_statuses.rb b/lib/gitlab/background_migration/fix_security_scan_statuses.rb new file mode 100644 index 00000000000..b60e739f870 --- /dev/null +++ b/lib/gitlab/background_migration/fix_security_scan_statuses.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Fixes the `status` attribute of `security_scans` records + class FixSecurityScanStatuses < BatchedMigrationJob + def perform + # no-op. The logic is defined in EE module. + end + end + end +end + +::Gitlab::BackgroundMigration::FixSecurityScanStatuses.prepend_mod diff --git a/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb new file mode 100644 index 00000000000..81b29b5a6cd --- /dev/null +++ b/lib/gitlab/background_migration/migrate_vulnerabilities_feedback_to_vulnerabilities_state_transition.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition < BatchedMigrationJob + def perform; end + end + end +end + +Gitlab::BackgroundMigration::MigrateVulnerabilitiesFeedbackToVulnerabilitiesStateTransition.prepend_mod +# rubocop:enable Style/Documentation diff --git a/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb new file mode 100644 index 00000000000..a91cda2c427 --- /dev/null +++ b/lib/gitlab/background_migration/prune_stale_project_export_jobs.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for deleting stale project import jobs + class PruneStaleProjectExportJobs < BatchedMigrationJob + EXPIRES_IN = 7.days + + scope_to ->(relation) { relation.where("updated_at < ?", EXPIRES_IN.ago) } + operation_name :delete_all + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb new file mode 100644 index 00000000000..09cd3b1895f --- /dev/null +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job that: + # * pickup container repositories with delete_scheduled status. + # * check if there are tags linked to it. + # * if there are tags, reset the status to nil. + class ResetStatusOnContainerRepositories < BatchedMigrationJob + DELETE_SCHEDULED_STATUS = 0 + DUMMY_TAGS = %w[tag].freeze + MIGRATOR = 'ResetStatusOnContainerRepositories' + + scope_to ->(relation) { relation.where(status: DELETE_SCHEDULED_STATUS) } + operation_name :reset_status_on_container_repositories + + def perform + each_sub_batch do |sub_batch| + reset_status_if_tags(sub_batch) + end + end + + private + + def reset_status_if_tags(container_repositories) + container_repositories_with_tags = container_repositories.select { |cr| cr.becomes(ContainerRepository).tags? } # rubocop:disable Cop/AvoidBecomes + + ContainerRepository.where(id: container_repositories_with_tags.map(&:id)) + .update_all(status: nil) + end + + # rubocop:disable Style/Documentation + module Routable + extend ActiveSupport::Concern + + included do + has_one :route, + as: :source, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route' + end + + def full_path + route&.path || build_full_path + end + + def build_full_path + if parent && path + "#{parent.full_path}/#{path}" + else + path + end + end + end + + class Route < ::ApplicationRecord + self.table_name = 'routes' + end + + class Namespace < ::ApplicationRecord + include ::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Routable + include ::Namespaces::Traversal::Recursive + include ::Namespaces::Traversal::Linear + include ::Gitlab::Utils::StrongMemoize + + self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled + + belongs_to :parent, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + + def self.polymorphic_name + 'Namespace' + end + end + + class Project < ::ApplicationRecord + include ::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Routable + + self.table_name = 'projects' + + belongs_to :namespace, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + + alias_method :parent, :namespace + alias_attribute :parent_id, :namespace_id + + delegate :root_ancestor, to: :namespace, allow_nil: true + end + + class ContainerRepository < ::ApplicationRecord + self.table_name = 'container_repositories' + + belongs_to :project, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project' + + def tags? + result = ContainerRegistry.tags_for(path).any? + ::Gitlab::BackgroundMigration::Logger.info( + migrator: MIGRATOR, + has_tags: result, + container_repository_id: id, + container_repository_path: path + ) + result + end + + def path + @path ||= [project.full_path, name].select(&:present?).join('/').downcase + end + end + + class ContainerRegistry + class << self + def tags_for(path) + response = ContainerRegistryClient.repository_tags(path, page_size: 1) + return DUMMY_TAGS unless response + + response['tags'] || [] + rescue StandardError + DUMMY_TAGS + end + end + end + + class ContainerRegistryClient + def self.repository_tags(path, page_size:) + registry_config = ::Gitlab.config.registry + + return { 'tags' => DUMMY_TAGS } unless registry_config.enabled && registry_config.api_url.present? + + pull_token = ::Auth::ContainerRegistryAuthenticationService.pull_access_token(path) + client = ::ContainerRegistry::Client.new(registry_config.api_url, token: pull_token) + client.repository_tags(path, page_size: page_size) + end + end + # rubocop:enable Style/Documentation + end + end +end diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb index e9a38916999..8aab7d13b45 100644 --- a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -37,16 +37,16 @@ module Gitlab end end - cloud_mappings = cloud.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 2 } + cloud_mappings = cloud.index_with do + { deployment_type: 2 } end - server_mappings = server.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 1 } + server_mappings = server.index_with do + { deployment_type: 1 } end - unknown_mappings = unknown.each_with_object({}) do |tracker_data, hash| - hash[tracker_data] = { deployment_type: 0 } + unknown_mappings = unknown.index_with do + { deployment_type: 0 } end mappings = cloud_mappings.merge(server_mappings, unknown_mappings) |