diff options
Diffstat (limited to 'spec/lib/gitlab/background_migration')
11 files changed, 844 insertions, 2 deletions
diff --git a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb index 7075d4694ae..d2da6867773 100644 --- a/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BackfillAdminModeScopeForPersonalAccessTokens, - :migration, schema: 20221228103133, feature_category: :authentication_and_authorization do + :migration, schema: 20221228103133, feature_category: :system_access do let(:users) { table(:users) } let(:personal_access_tokens) { table(:personal_access_tokens) } diff --git a/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb new file mode 100644 index 00000000000..b33a1a31c40 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_prepared_at_merge_requests_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPreparedAtMergeRequests, :migration, + feature_category: :code_review_workflow, schema: 20230202135758 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:mr_table) { table(:merge_requests) } + + let(:namespace) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:proj_namespace) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace.id) } + let(:project) do + projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace.id, project_namespace_id: proj_namespace.id) + end + + let(:test_worker) do + described_class.new( + start_id: 1, + end_id: 100, + batch_table: :merge_requests, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + connection: ApplicationRecord.connection + ) + end + + it 'updates merge requests with prepared_at nil' do + time = Time.current + + mr_1 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: nil, merge_status: 'checking') + mr_2 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: nil, merge_status: 'preparing') + mr_3 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time) + mr_4 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time, merge_status: 'checking') + mr_5 = mr_table.create!(target_project_id: project.id, source_branch: 'master', target_branch: 'feature', + prepared_at: time, merge_status: 'preparing') + + expect(mr_1.prepared_at).to be_nil + expect(mr_2.prepared_at).to be_nil + expect(mr_3.prepared_at.to_i).to eq(time.to_i) + expect(mr_4.prepared_at.to_i).to eq(time.to_i) + expect(mr_5.prepared_at.to_i).to eq(time.to_i) + + test_worker.perform + + expect(mr_1.reload.prepared_at.to_i).to eq(mr_1.created_at.to_i) + expect(mr_2.reload.prepared_at).to be_nil + expect(mr_3.reload.prepared_at.to_i).to eq(time.to_i) + expect(mr_4.reload.prepared_at.to_i).to eq(time.to_i) + expect(mr_5.reload.prepared_at.to_i).to eq(time.to_i) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb new file mode 100644 index 00000000000..e81bd0604e6 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_wiki_repositories_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe( + Gitlab::BackgroundMigration::BackfillProjectWikiRepositories, + schema: 20230306195007, + feature_category: :geo_replication) do + let!(:namespaces) { table(:namespaces) } + let!(:projects) { table(:projects) } + let!(:project_wiki_repositories) { table(:project_wiki_repositories) } + + subject(:migration) do + described_class.new( + start_id: projects.minimum(:id), + end_id: projects.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ) + end + + describe '#perform' do + it 'creates project_wiki_repositories entries for all projects in range' do + namespace1 = create_namespace('test1') + namespace2 = create_namespace('test2') + project1 = create_project(namespace1, 'test1') + project2 = create_project(namespace2, 'test2') + project_wiki_repositories.create!(project_id: project2.id) + + expect { migration.perform } + .to change { project_wiki_repositories.pluck(:project_id) } + .from([project2.id]) + .to match_array([project1.id, project2.id]) + end + + it 'does nothing if project_id already exist in project_wiki_repositories' do + namespace = create_namespace('test1') + project = create_project(namespace, 'test1') + project_wiki_repositories.create!(project_id: project.id) + + expect { migration.perform } + .not_to change { project_wiki_repositories.pluck(:project_id) } + end + + def create_namespace(name) + namespaces.create!( + name: name, + path: name, + type: 'Project' + ) + end + + def create_project(namespace, name) + projects.create!( + namespace_id: namespace.id, + project_namespace_id: namespace.id, + name: name, + path: name + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb index faaaccfdfaf..781bf93dd85 100644 --- a/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb +++ b/spec/lib/gitlab/background_migration/batched_migration_job_spec.rb @@ -301,6 +301,28 @@ RSpec.describe Gitlab::BackgroundMigration::BatchedMigrationJob do perform_job end + context 'when using a sub batch exception for timeouts' do + let(:job_class) do + Class.new(described_class) do + operation_name :update + + def perform(*_) + each_sub_batch { raise ActiveRecord::StatementTimeout } # rubocop:disable Lint/UnreachableLoop + end + end + end + + let(:job_instance) do + job_class.new(start_id: 1, end_id: 10, batch_table: '_test_table', batch_column: 'id', + sub_batch_size: 2, pause_ms: 1000, connection: connection, + sub_batch_exception: StandardError) + end + + it 'raises the expected error type' do + expect { job_instance.perform }.to raise_error(StandardError) + end + end + context 'when batching_arguments are given' do it 'forwards them for batching' do expect(job_instance).to receive(:base_relation).and_return(test_table) diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb new file mode 100644 index 00000000000..0d82717c7de --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphaned_packages_dependencies_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedPackagesDependencies, schema: 20230303105806, + feature_category: :package_registry do + let!(:migration_attrs) do + { + start_id: 1, + end_id: 1000, + batch_table: :packages_dependencies, + batch_column: :id, + sub_batch_size: 500, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let!(:migration) { described_class.new(**migration_attrs) } + + let(:packages_dependencies) { table(:packages_dependencies) } + + let!(:namespace) { table(:namespaces).create!(name: 'project', path: 'project', type: 'Project') } + let!(:project) do + table(:projects).create!(name: 'project', path: 'project', project_namespace_id: namespace.id, + namespace_id: namespace.id) + end + + let!(:package) do + table(:packages_packages).create!(name: 'test', version: '1.2.3', package_type: 2, project_id: project.id) + end + + let!(:orphan_dependency_1) { packages_dependencies.create!(name: 'dependency 1', version_pattern: '~0.0.1') } + let!(:orphan_dependency_2) { packages_dependencies.create!(name: 'dependency 2', version_pattern: '~0.0.2') } + let!(:orphan_dependency_3) { packages_dependencies.create!(name: 'dependency 3', version_pattern: '~0.0.3') } + let!(:linked_dependency) do + packages_dependencies.create!(name: 'dependency 4', version_pattern: '~0.0.4').tap do |dependency| + table(:packages_dependency_links).create!(package_id: package.id, dependency_id: dependency.id, + dependency_type: 'dependencies') + end + end + + subject(:perform_migration) { migration.perform } + + it 'executes 3 queries' do + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(3) + end + + it 'deletes only orphaned dependencies' do + expect { perform_migration }.to change { packages_dependencies.count }.by(-3) + expect(packages_dependencies.all).to eq([linked_dependency]) + end +end diff --git a/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb new file mode 100644 index 00000000000..9f431c43f39 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::FixVulnerabilityReadsHasIssues, schema: 20230302185739, feature_category: :vulnerability_management do # rubocop:disable Layout/LineLength + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:users) { table(:users) } + let(:scanners) { table(:vulnerability_scanners) } + let(:vulnerabilities) { table(:vulnerabilities) } + let(:vulnerability_reads) { table(:vulnerability_reads) } + let(:work_item_types) { table(:work_item_types) } + let(:issues) { table(:issues) } + let(:vulnerability_issue_links) { table(:vulnerability_issue_links) } + + let(:namespace) { namespaces.create!(name: 'user', path: 'user') } + let(:project) { projects.create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } + let(:user) { users.create!(username: 'john_doe', email: 'johndoe@gitlab.com', projects_limit: 10) } + let(:scanner) { scanners.create!(project_id: project.id, external_id: 'external_id', name: 'Test Scanner') } + let(:work_item_type) { work_item_types.create!(name: 'test') } + + let(:vulnerability_records) do + Array.new(4).map do |_, n| + vulnerabilities.create!( + project_id: project.id, + author_id: user.id, + title: "vulnerability #{n}", + severity: 1, + confidence: 1, + report_type: 1 + ) + end + end + + let(:vulnerabilities_with_issues) { [vulnerability_records.first, vulnerability_records.third] } + let(:vulnerabilities_without_issues) { vulnerability_records - vulnerabilities_with_issues } + + let(:vulnerability_read_records) do + vulnerability_records.map do |vulnerability| + vulnerability_reads.create!( + project_id: project.id, + vulnerability_id: vulnerability.id, + scanner_id: scanner.id, + has_issues: false, + severity: 1, + report_type: 1, + state: 1, + uuid: SecureRandom.uuid + ) + end + end + + let!(:issue_links) do + vulnerabilities_with_issues.map do |vulnerability| + issue = issues.create!( + title: vulnerability.title, + author_id: user.id, + project_id: project.id, + confidential: true, + work_item_type_id: work_item_type.id, + namespace_id: namespace.id + ) + + vulnerability_issue_links.create!( + vulnerability_id: vulnerability.id, + issue_id: issue.id + ) + end + end + + def vulnerability_read_for(vulnerability) + vulnerability_read_records.find { |read| read.vulnerability_id == vulnerability.id } + end + + subject(:perform_migration) do + described_class.new( + start_id: issue_links.first.vulnerability_id, + end_id: issue_links.last.vulnerability_id, + batch_table: :vulnerability_issue_links, + batch_column: :vulnerability_id, + sub_batch_size: issue_links.size, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'only changes records with issue links' do + expect(vulnerability_read_records).to all(have_attributes(has_issues: false)) + + perform_migration + + vulnerabilities_with_issues.each do |vulnerability| + expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(true) + end + + vulnerabilities_without_issues.each do |vulnerability| + expect(vulnerability_read_for(vulnerability).reload.has_issues).to eq(false) + end + end +end diff --git a/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb new file mode 100644 index 00000000000..1adff322b41 --- /dev/null +++ b/spec/lib/gitlab/background_migration/issues_internal_id_scope_updater_spec.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'spec_helper' +# this needs the schema to be before we introduce the not null constraint on routes#namespace_id +# rubocop:disable RSpec/MultipleMemoizedHelpers +RSpec.describe Gitlab::BackgroundMigration::IssuesInternalIdScopeUpdater, feature_category: :team_planning do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:internal_ids) { table(:internal_ids) } + + let(:gr1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:gr2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: gr1.id, path: 'space2') } + + let(:pr_nmsp1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: gr1.id) } + let(:pr_nmsp3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp4) { namespaces.create!(name: 'proj4', path: 'proj4', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp5) { namespaces.create!(name: 'proj5', path: 'proj5', type: 'Project', parent_id: gr2.id) } + let(:pr_nmsp6) { namespaces.create!(name: 'proj6', path: 'proj6', type: 'Project', parent_id: gr2.id) } + + # rubocop:disable Layout/LineLength + let(:p1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: gr1.id, project_namespace_id: pr_nmsp1.id) } + let(:p2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: gr1.id, project_namespace_id: pr_nmsp2.id) } + let(:p3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: gr2.id, project_namespace_id: pr_nmsp3.id) } + let(:p4) { projects.create!(name: 'proj4', path: 'proj4', namespace_id: gr2.id, project_namespace_id: pr_nmsp4.id) } + let(:p5) { projects.create!(name: 'proj5', path: 'proj5', namespace_id: gr2.id, project_namespace_id: pr_nmsp5.id) } + let(:p6) { projects.create!(name: 'proj6', path: 'proj6', namespace_id: gr2.id, project_namespace_id: pr_nmsp6.id) } + # rubocop:enable Layout/LineLength + + # a project that already is covered by a record for its namespace. This should result in no new record added and + # project related record deleted + let!(:issues_internal_ids_p1) { internal_ids.create!(project_id: p1.id, usage: 0, last_value: 100) } + let!(:issues_internal_ids_pr_nmsp1) { internal_ids.create!(namespace_id: pr_nmsp1.id, usage: 0, last_value: 111) } + + # project records that do not have a corresponding namespace record. This should result 2 new records + # scoped to corresponding project namespaces being added and the project related records being deleted. + let!(:issues_internal_ids_p2) { internal_ids.create!(project_id: p2.id, usage: 0, last_value: 200) } + let!(:issues_internal_ids_p3) { internal_ids.create!(project_id: p3.id, usage: 0, last_value: 300) } + + # a project record on a different usage, should not be affected by the migration and + # no new record should be created for this case + let!(:issues_internal_ids_p4) { internal_ids.create!(project_id: p4.id, usage: 4, last_value: 400) } + + # a project namespace scoped record without a corresponding project record, should not affect anything. + let!(:issues_internal_ids_pr_nmsp5) { internal_ids.create!(namespace_id: pr_nmsp5.id, usage: 0, last_value: 500) } + + # a record scoped to a group, should not affect anything. + let!(:issues_internal_ids_gr1) { internal_ids.create!(namespace_id: gr1.id, usage: 0, last_value: 600) } + + # a project that is covered by a record for its namespace, but has a higher last_value, due to updates during rolling + # deploy for instance, see https://gitlab.com/gitlab-com/gl-infra/production/-/issues/8548 + let!(:issues_internal_ids_p6) { internal_ids.create!(project_id: p6.id, usage: 0, last_value: 111) } + let!(:issues_internal_ids_pr_nmsp6) { internal_ids.create!(namespace_id: pr_nmsp6.id, usage: 0, last_value: 100) } + + subject(:perform_migration) do + described_class.new( + start_id: internal_ids.minimum(:id), + end_id: internal_ids.maximum(:id), + batch_table: :internal_ids, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection + ).perform + end + + it 'backfills internal_ids records and removes related project records', :aggregate_failures do + perform_migration + + expected_recs = [pr_nmsp1.id, pr_nmsp2.id, pr_nmsp3.id, pr_nmsp5.id, gr1.id, pr_nmsp6.id] + + # all namespace scoped records for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).count).to eq(6) + # all namespace_ids for issues(0) usage + expect(internal_ids.where.not(namespace_id: nil).where(usage: 0).pluck(:namespace_id)).to match_array(expected_recs) + # this is the record with usage: 4 + expect(internal_ids.where.not(project_id: nil).count).to eq(1) + # no project scoped records for issues usage left + expect(internal_ids.where.not(project_id: nil).where(usage: 0).count).to eq(0) + + # the case when the project_id scoped record had the higher last_value, + # see `issues_internal_ids_p6` and issues_internal_ids_pr_nmsp6 definitions above + expect(internal_ids.where(namespace_id: pr_nmsp6.id).first.last_value).to eq(111) + + # the case when the namespace_id scoped record had the higher last_value, + # see `issues_internal_ids_p1` and issues_internal_ids_pr_nmsp1 definitions above. + expect(internal_ids.where(namespace_id: pr_nmsp1.id).first.last_value).to eq(111) + end +end +# rubocop:enable RSpec/MultipleMemoizedHelpers diff --git a/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..b70044ab2a4 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings_spec.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateEvidencesForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_evidences) { table(:vulnerability_finding_evidences) } + let(:evidence_hash) { { url: 'http://test.com' } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of evidence key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any evidence' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with evidence equals to nil' do + before do + create_finding!(project1.id, scanner1.id, { evidence: nil }) + end + + it 'does not create any evidence' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'with existing evidence within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + + it 'creates new evidence for each finding' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(2) + end + + context 'when create throws exception StandardError' do + before do + allow(migration).to receive(:create_evidences).and_raise(StandardError) + end + + it 'logs StandardError' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s + }) + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + + context 'when parse throws exception JSON::ParserError' do + before do + allow(Gitlab::Json).to receive(:parse).and_raise(JSON::ParserError) + end + + it 'does not log this error nor create new records' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + end + end + + context 'with existing evidence records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { evidence: evidence_hash }) } + + before do + vulnerability_finding_evidences.create!(vulnerability_occurrence_id: finding.id, data: evidence_hash) + end + + it 'does not create new evidence' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_evidences.count } + end + + context 'with non-existing evidence' do + let!(:finding3) { create_finding!(project1.id, scanner1.id, { evidence: { url: 'http://secondary.com' } }) } + + it 'creates a new evidence only to the non-existing evidence' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_evidences.count }.by(1) + end + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..fd2e3ffb670 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings_spec.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateLinksForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_finding_links) { table(:vulnerability_finding_links) } + let(:link_hash) { { url: 'http://test.com' } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of links key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to an array of nil element' do + before do + create_finding!(project1.id, scanner1.id, { links: [nil] }) + end + + it 'does not create any link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'with links equals to an array of duplicated elements' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { links: [link_hash, link_hash] }) + end + + it 'creates one new link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_links.count }.by(1) + end + end + + context 'with existing links within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + + it 'creates new link for each finding' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_finding_links.count }.by(2) + end + + context 'when create throws exception ActiveRecord::RecordNotUnique' do + before do + allow(migration).to receive(:create_links).and_raise(ActiveRecord::RecordNotUnique) + end + + it 'does not log this error nor create new records' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + context 'when create throws exception StandardError' do + before do + allow(migration).to receive(:create_links).and_raise(StandardError) + end + + it 'logs StandardError' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding1.id + }) + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding2.id + }) + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + end + + context 'with existing link records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { links: [link_hash] }) } + + before do + vulnerability_finding_links.create!(vulnerability_occurrence_id: finding.id, url: link_hash[:url]) + end + + it 'does not create new link' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_finding_links.count } + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end +end diff --git a/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb new file mode 100644 index 00000000000..b75c0e61b19 --- /dev/null +++ b/spec/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings_spec.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::MigrateRemediationsForVulnerabilityFindings, + feature_category: :vulnerability_management do + let(:vulnerability_occurrences) { table(:vulnerability_occurrences) } + let(:vulnerability_findings_remediations) { table(:vulnerability_findings_remediations) } + let(:vulnerability_remediations) { table(:vulnerability_remediations) } + let(:remediation_hash) { { summary: 'summary', diff: "ZGlmZiAtLWdp" } } + let(:namespace1) { table(:namespaces).create!(name: 'namespace 1', path: 'namespace1') } + let(:project1) { table(:projects).create!(namespace_id: namespace1.id, project_namespace_id: namespace1.id) } + let(:user) { table(:users).create!(email: 'test1@example.com', projects_limit: 5) } + + let(:scanner1) do + table(:vulnerability_scanners).create!(project_id: project1.id, external_id: 'test 1', name: 'test scanner 1') + end + + let(:stating_id) { vulnerability_occurrences.pluck(:id).min } + let(:end_id) { vulnerability_occurrences.pluck(:id).max } + + let(:migration) do + described_class.new( + start_id: stating_id, + end_id: end_id, + batch_table: :vulnerability_occurrences, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + context 'without the presence of remediation key' do + before do + create_finding!(project1.id, scanner1.id, { other_keys: 'test' }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation equals to an array of nil element' do + before do + create_finding!(project1.id, scanner1.id, { remediations: [nil] }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation with empty string as the diff key' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { remediations: [{ summary: 'summary', diff: '' }] }) + end + + it 'does not create any remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + end + end + + context 'with remediation equals to an array of duplicated elements' do + let!(:finding) do + create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash, remediation_hash] }) + end + + it 'creates new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1) + end + end + + context 'with existing remediations within raw_metadata' do + let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + let!(:finding2) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + + it 'creates new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1) + end + + context 'when create throws exception other than ActiveRecord::RecordNotUnique' do + before do + allow(migration).to receive(:create_finding_remediations).and_raise(StandardError) + end + + it 'rolls back all related transactions' do + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding1.id + }) + expect(Gitlab::AppLogger).to receive(:error).with({ + class: described_class.name, message: StandardError.to_s, model_id: finding2.id + }) + expect { perform_migration }.not_to change { vulnerability_remediations.count } + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(0) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(0) + end + end + end + + context 'with existing remediation records' do + let!(:finding) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + + before do + vulnerability_remediations.create!(project_id: project1.id, summary: remediation_hash[:summary], + checksum: checksum(remediation_hash[:diff]), file: Tempfile.new.path) + end + + it 'does not create new remediation' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.not_to change { vulnerability_remediations.count } + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding.id).length).to eq(1) + end + end + + context 'with same raw_metadata for different projects' do + let(:namespace2) { table(:namespaces).create!(name: 'namespace 2', path: 'namespace2') } + let(:project2) { table(:projects).create!(namespace_id: namespace2.id, project_namespace_id: namespace2.id) } + let(:scanner2) do + table(:vulnerability_scanners).create!(project_id: project2.id, external_id: 'test 2', name: 'test scanner 2') + end + + let!(:finding1) { create_finding!(project1.id, scanner1.id, { remediations: [remediation_hash] }) } + let!(:finding2) { create_finding!(project2.id, scanner2.id, { remediations: [remediation_hash] }) } + + it 'creates new remediation for each project' do + expect(Gitlab::AppLogger).not_to receive(:error) + + expect { perform_migration }.to change { vulnerability_remediations.count }.by(2) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding1.id).length).to eq(1) + expect(vulnerability_findings_remediations.where(vulnerability_occurrence_id: finding2.id).length).to eq(1) + end + end + + private + + def create_finding!(project_id, scanner_id, raw_metadata) + vulnerability = table(:vulnerabilities).create!(project_id: project_id, author_id: user.id, title: 'test', + severity: 4, confidence: 4, report_type: 0) + + identifier = table(:vulnerability_identifiers).create!(project_id: project_id, external_type: 'uuid-v5', + external_id: 'uuid-v5', fingerprint: OpenSSL::Digest::SHA256.hexdigest(vulnerability.id.to_s), + name: 'Identifier for UUIDv5 2 2') + + table(:vulnerability_occurrences).create!( + vulnerability_id: vulnerability.id, project_id: project_id, scanner_id: scanner_id, + primary_identifier_id: identifier.id, name: 'test', severity: 4, confidence: 4, report_type: 0, + uuid: SecureRandom.uuid, project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, + location_fingerprint: 'test', metadata_version: 'test', + raw_metadata: raw_metadata.to_json) + end + + def checksum(value) + sha = Digest::SHA256.hexdigest(value) + Gitlab::Database::ShaAttribute.new.serialize(sha) + end +end diff --git a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb index a8574411957..f671a673a08 100644 --- a/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb +++ b/spec/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects_spec.rb @@ -2,7 +2,8 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects do +RSpec.describe Gitlab::BackgroundMigration::NullifyCreatorIdColumnOfOrphanedProjects, feature_category: :projects, + schema: 20230130073109 do let(:users) { table(:users) } let(:projects) { table(:projects) } let(:namespaces) { table(:namespaces) } |