diff options
Diffstat (limited to 'spec/lib/gitlab/background_migration')
21 files changed, 858 insertions, 455 deletions
diff --git a/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb b/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb new file mode 100644 index 00000000000..3aab0cdf54b --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities_spec.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillClusterAgentsHasVulnerabilities, :migration do # rubocop:disable Layout/LineLength + let(:migration) do + described_class.new(start_id: 1, end_id: 10, + batch_table: table_name, batch_column: batch_column, + sub_batch_size: sub_batch_size, pause_ms: pause_ms, + connection: ApplicationRecord.connection) + end + + let(:users_table) { table(:users) } + let(:vulnerability_reads_table) { table(:vulnerability_reads) } + let(:vulnerability_scanners_table) { table(:vulnerability_scanners) } + let(:vulnerabilities_table) { table(:vulnerabilities) } + let(:namespaces_table) { table(:namespaces) } + let(:projects_table) { table(:projects) } + let(:cluster_agents_table) { table(:cluster_agents) } + + let(:table_name) { 'cluster_agents' } + let(:batch_column) { :id } + let(:sub_batch_size) { 100 } + let(:pause_ms) { 0 } + + subject(:perform_migration) { migration.perform } + + before do + users_table.create!(id: 1, name: 'John Doe', email: 'test@example.com', projects_limit: 5) + + namespaces_table.create!(id: 1, name: 'Namespace 1', path: 'namespace-1') + namespaces_table.create!(id: 2, name: 'Namespace 2', path: 'namespace-2') + namespaces_table.create!(id: 3, name: 'Namespace 3', path: 'namespace-3') + + projects_table.create!(id: 1, namespace_id: 1, name: 'Project 1', path: 'project-1', project_namespace_id: 1) + projects_table.create!(id: 2, namespace_id: 2, name: 'Project 2', path: 'project-2', project_namespace_id: 2) + projects_table.create!(id: 3, namespace_id: 2, name: 'Project 3', path: 'project-3', project_namespace_id: 3) + + cluster_agents_table.create!(id: 1, name: 'Agent 1', project_id: 1) + cluster_agents_table.create!(id: 2, name: 'Agent 2', project_id: 2) + cluster_agents_table.create!(id: 3, name: 'Agent 3', project_id: 1) + cluster_agents_table.create!(id: 4, name: 'Agent 4', project_id: 1) + cluster_agents_table.create!(id: 5, name: 'Agent 5', project_id: 1) + cluster_agents_table.create!(id: 6, name: 'Agent 6', project_id: 1) + cluster_agents_table.create!(id: 7, name: 'Agent 7', project_id: 3) + cluster_agents_table.create!(id: 8, name: 'Agent 8', project_id: 1) + cluster_agents_table.create!(id: 9, name: 'Agent 9', project_id: 1) + cluster_agents_table.create!(id: 10, name: 'Agent 10', project_id: 3) + cluster_agents_table.create!(id: 11, name: 'Agent 11', project_id: 1) + + vulnerability_scanners_table.create!(id: 1, project_id: 1, external_id: 'starboard', name: 'Starboard') + vulnerability_scanners_table.create!(id: 2, project_id: 2, external_id: 'starboard', name: 'Starboard') + vulnerability_scanners_table.create!(id: 3, project_id: 3, external_id: 'starboard', name: 'Starboard') + + add_vulnerability_read!(1, project_id: 1, cluster_agent_id: 1, report_type: 7) + add_vulnerability_read!(2, project_id: 1, cluster_agent_id: nil, report_type: 7) + add_vulnerability_read!(3, project_id: 1, cluster_agent_id: 3, report_type: 7) + add_vulnerability_read!(4, project_id: 1, cluster_agent_id: nil, report_type: 7) + add_vulnerability_read!(5, project_id: 2, cluster_agent_id: 5, report_type: 5) + add_vulnerability_read!(7, project_id: 2, cluster_agent_id: 7, report_type: 7) + add_vulnerability_read!(9, project_id: 3, cluster_agent_id: 9, report_type: 7) + add_vulnerability_read!(10, project_id: 1, cluster_agent_id: 10, report_type: 7) + add_vulnerability_read!(11, project_id: 2, cluster_agent_id: 11, report_type: 7) + end + + it 'backfills `has_vulnerabilities` for the selected records', :aggregate_failures do + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(3) + expect(cluster_agents_table.where(has_vulnerabilities: true).count).to eq 2 + expect(cluster_agents_table.where(has_vulnerabilities: true).pluck(:id)).to match_array([1, 3]) + end + + it 'tracks timings of queries' do + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end + + private + + def add_vulnerability_read!(id, project_id:, cluster_agent_id:, report_type:) + vulnerabilities_table.create!( + id: id, + project_id: project_id, + author_id: 1, + title: "Vulnerability #{id}", + severity: 5, + confidence: 5, + report_type: report_type + ) + + vulnerability_reads_table.create!( + id: id, + uuid: SecureRandom.uuid, + severity: 5, + state: 1, + vulnerability_id: id, + scanner_id: project_id, + casted_cluster_agent_id: cluster_agent_id, + project_id: project_id, + report_type: report_type + ) + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb index e363a5a6b20..8947262ae9e 100644 --- a/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_imported_issue_search_data_spec.rb @@ -14,10 +14,10 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillImportedIssueSearchData, table(:projects) .create!( namespace_id: namespace.id, - creator_id: user.id, - name: 'projecty', - path: 'path', - project_namespace_id: namespace.id) + creator_id: user.id, + name: 'projecty', + path: 'path', + project_namespace_id: namespace.id) end let!(:issue) do diff --git a/spec/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification_spec.rb b/spec/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification_spec.rb index b3825a7c4ea..3c0b7766871 100644 --- a/spec/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification_spec.rb @@ -9,25 +9,35 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsEnableSslVerific before do integrations.create!(id: 1, type_new: 'Integrations::Bamboo') # unaffected integration integrations.create!(id: 2, type_new: 'Integrations::DroneCi') # no properties - integrations.create!(id: 3, type_new: 'Integrations::DroneCi', + integrations.create!( + id: 3, type_new: 'Integrations::DroneCi', properties: {}) # no URL - integrations.create!(id: 4, type_new: 'Integrations::DroneCi', + integrations.create!( + id: 4, type_new: 'Integrations::DroneCi', properties: { 'drone_url' => '' }) # blank URL - integrations.create!(id: 5, type_new: 'Integrations::DroneCi', + integrations.create!( + id: 5, type_new: 'Integrations::DroneCi', properties: { 'drone_url' => 'https://example.com:foo' }) # invalid URL - integrations.create!(id: 6, type_new: 'Integrations::DroneCi', + integrations.create!( + id: 6, type_new: 'Integrations::DroneCi', properties: { 'drone_url' => 'https://example.com' }) # unknown URL - integrations.create!(id: 7, type_new: 'Integrations::DroneCi', + integrations.create!( + id: 7, type_new: 'Integrations::DroneCi', properties: { 'drone_url' => 'http://cloud.drone.io' }) # no HTTPS - integrations.create!(id: 8, type_new: 'Integrations::DroneCi', + integrations.create!( + id: 8, type_new: 'Integrations::DroneCi', properties: { 'drone_url' => 'https://cloud.drone.io' }) # known URL - integrations.create!(id: 9, type_new: 'Integrations::Teamcity', + integrations.create!( + id: 9, type_new: 'Integrations::Teamcity', properties: { 'teamcity_url' => 'https://example.com' }) # unknown URL - integrations.create!(id: 10, type_new: 'Integrations::Teamcity', + integrations.create!( + id: 10, type_new: 'Integrations::Teamcity', properties: { 'teamcity_url' => 'https://foo.bar.teamcity.com' }) # unknown URL - integrations.create!(id: 11, type_new: 'Integrations::Teamcity', + integrations.create!( + id: 11, type_new: 'Integrations::Teamcity', properties: { 'teamcity_url' => 'https://teamcity.com' }) # unknown URL - integrations.create!(id: 12, type_new: 'Integrations::Teamcity', + integrations.create!( + id: 12, type_new: 'Integrations::Teamcity', properties: { 'teamcity_url' => 'https://customer.teamcity.com' }) # known URL end diff --git a/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb new file mode 100644 index 00000000000..29833074109 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_namespace_on_issues_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' +# todo: this will need to specify schema version once we introduce the not null constraint on issues#namespace_id +# https://gitlab.com/gitlab-org/gitlab/-/issues/367835 +RSpec.describe Gitlab::BackgroundMigration::BackfillProjectNamespaceOnIssues do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:issues) { table(:issues) } + + let(:namespace1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:namespace2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: namespace1.id, path: 'space2') } + + let(:proj_namespace1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace1.id) } + let(:proj_namespace2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: namespace2.id) } + + # rubocop:disable Layout/LineLength + let(:proj1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace1.id, project_namespace_id: proj_namespace1.id) } + let(:proj2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: namespace2.id, project_namespace_id: proj_namespace2.id) } + + let!(:proj1_issue_with_namespace) { issues.create!(title: 'issue1', project_id: proj1.id, namespace_id: proj_namespace1.id) } + let!(:proj1_issue_without_namespace1) { issues.create!(title: 'issue2', project_id: proj1.id) } + let!(:proj1_issue_without_namespace2) { issues.create!(title: 'issue3', project_id: proj1.id) } + let!(:proj2_issue_with_namespace) { issues.create!(title: 'issue4', project_id: proj2.id, namespace_id: proj_namespace2.id) } + let!(:proj2_issue_without_namespace1) { issues.create!(title: 'issue5', project_id: proj2.id) } + let!(:proj2_issue_without_namespace2) { issues.create!(title: 'issue6', project_id: proj2.id) } + # rubocop:enable Layout/LineLength + + let(:migration) do + described_class.new( + start_id: proj1_issue_with_namespace.id, + end_id: proj2_issue_without_namespace2.id, + batch_table: :issues, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + it 'backfills namespace_id for the selected records', :aggregate_failures do + perform_migration + + expected_namespaces = [proj_namespace1.id, proj_namespace2.id] + + expect(issues.where.not(namespace_id: nil).count).to eq(6) + expect(issues.where.not(namespace_id: nil).pluck(:namespace_id).uniq).to match_array(expected_namespaces) + end + + it 'tracks timings of queries' do + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb index 6f75d3faef3..1c2e0e991d9 100644 --- a/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_snippet_repositories_spec.rb @@ -14,23 +14,23 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat let!(:user) do users.create!(id: 1, - email: 'user@example.com', - projects_limit: 10, - username: 'test', - name: user_name, - state: user_state, - last_activity_on: 1.minute.ago, - user_type: user_type, - confirmed_at: 1.day.ago) + email: 'user@example.com', + projects_limit: 10, + username: 'test', + name: user_name, + state: user_state, + last_activity_on: 1.minute.ago, + user_type: user_type, + confirmed_at: 1.day.ago) end let!(:migration_bot) do users.create!(id: 100, - email: "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}", - user_type: HasUserType::USER_TYPES[:migration_bot], - name: 'GitLab Migration Bot', - projects_limit: 10, - username: 'bot') + email: "noreply+gitlab-migration-bot%s@#{Settings.gitlab.host}", + user_type: HasUserType::USER_TYPES[:migration_bot], + name: 'GitLab Migration Bot', + projects_limit: 10, + username: 'bot') end let!(:snippet_with_repo) { snippets.create!(id: 1, type: 'PersonalSnippet', author_id: user.id, file_name: file_name, content: content) } @@ -260,14 +260,14 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillSnippetRepositories, :migrat let(:user_name) { '.' } let!(:other_user) do users.create!(id: 2, - email: 'user2@example.com', - projects_limit: 10, - username: 'test2', - name: 'Test2', - state: user_state, - last_activity_on: 1.minute.ago, - user_type: user_type, - confirmed_at: 1.day.ago) + email: 'user2@example.com', + projects_limit: 10, + username: 'test2', + name: 'Test2', + state: user_state, + last_activity_on: 1.minute.ago, + user_type: user_type, + confirmed_at: 1.day.ago) end let!(:invalid_snippet) { snippets.create!(id: 4, type: 'PersonalSnippet', author_id: user.id, file_name: '.', content: content) } diff --git a/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb b/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb index 79699375a8d..f642ec8c20d 100644 --- a/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent_spec.rb @@ -23,8 +23,6 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillVulnerabilityReadsClusterAge let(:sub_batch_size) { 1_000 } let(:pause_ms) { 0 } - subject(:perform_migration) { migration.perform } - before do users_table.create!(id: 1, name: 'John Doe', email: 'test@example.com', projects_limit: 5) @@ -49,20 +47,30 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillVulnerabilityReadsClusterAge add_vulnerability_read!(11, project_id: 1, cluster_agent_id: 1, report_type: 7) end - it 'backfills `casted_cluster_agent_id` for the selected records', :aggregate_failures do - queries = ActiveRecord::QueryRecorder.new do - perform_migration + describe '#filter_batch' do + it 'pick only vulnerability reads where report_type = 7' do + expect(migration.filter_batch(vulnerability_reads_table).pluck(:id)).to contain_exactly(1, 3, 7, 9, 10, 11) end - - expect(queries.count).to eq(3) - expect(vulnerability_reads_table.where.not(casted_cluster_agent_id: nil).count).to eq 3 - expect(vulnerability_reads_table.where.not(casted_cluster_agent_id: nil).pluck(:id)).to match_array([1, 9, 10]) end - it 'tracks timings of queries' do - expect(migration.batch_metrics.timings).to be_empty + describe '#perform' do + subject(:perform_migration) { migration.perform } - expect { perform_migration }.to change { migration.batch_metrics.timings } + it 'backfills `casted_cluster_agent_id` for the selected records', :aggregate_failures do + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(3) + expect(vulnerability_reads_table.where.not(casted_cluster_agent_id: nil).count).to eq 3 + expect(vulnerability_reads_table.where.not(casted_cluster_agent_id: nil).pluck(:id)).to match_array([1, 9, 10]) + end + + it 'tracks timings of queries' do + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end end private diff --git a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb index 8d82c533d20..6ef474ad7f9 100644 --- a/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb +++ b/spec/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues_spec.rb @@ -2,12 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :migration, schema: 20220326161803 do - subject(:migrate) { migration.perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, issue_type_enum[:issue], issue_type.id) } - - let(:migration) { described_class.new } - - let(:batch_table) { 'issues' } +RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :migration, schema: 20220825142324 do let(:batch_column) { 'id' } let(:sub_batch_size) { 2 } let(:pause_ms) { 0 } @@ -15,7 +10,7 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi # let_it_be can't be used in migration specs because all tables but `work_item_types` are deleted after each spec let(:issue_type_enum) { { issue: 0, incident: 1, test_case: 2, requirement: 3, task: 4 } } let(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let(:project) { table(:projects).create!(namespace_id: namespace.id) } + let(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) } let(:issues_table) { table(:issues) } let(:issue_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_type_enum[:issue]) } @@ -32,6 +27,21 @@ RSpec.describe Gitlab::BackgroundMigration::BackfillWorkItemTypeIdForIssues, :mi let(:all_issues) { [issue1, issue2, issue3, incident1, test_case1, requirement1] } + let(:migration) do + described_class.new( + start_id: start_id, + end_id: end_id, + batch_table: :issues, + batch_column: :id, + sub_batch_size: sub_batch_size, + pause_ms: pause_ms, + job_arguments: [issue_type_enum[:issue], issue_type.id], + connection: ApplicationRecord.connection + ) + end + + subject(:migrate) { migration.perform } + it 'sets work_item_type_id only for the given type' do expect(all_issues).to all(have_attributes(work_item_type_id: nil)) diff --git a/spec/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy_spec.rb deleted file mode 100644 index 3cba99bfe51..00000000000 --- a/spec/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy_spec.rb +++ /dev/null @@ -1,135 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::BackfillIssueWorkItemTypeBatchingStrategy, '#next_batch', schema: 20220326161803 do # rubocop:disable Layout/LineLength - # let! can't be used in migration specs because all tables but `work_item_types` are deleted after each spec - let!(:issue_type_enum) { { issue: 0, incident: 1, test_case: 2, requirement: 3, task: 4 } } - let!(:namespace) { table(:namespaces).create!(name: 'namespace', path: 'namespace') } - let!(:project) { table(:projects).create!(namespace_id: namespace.id) } - let!(:issues_table) { table(:issues) } - let!(:task_type) { table(:work_item_types).find_by!(namespace_id: nil, base_type: issue_type_enum[:task]) } - - let!(:issue1) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) } - let!(:task1) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:task]) } - let!(:issue2) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) } - let!(:issue3) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:issue]) } - let!(:task2) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:task]) } - let!(:incident1) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:incident]) } - # test_case is EE only, but enum values exist on the FOSS model - let!(:test_case1) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:test_case]) } - - let!(:task3) do - issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:task], work_item_type_id: task_type.id) - end - - let!(:task4) { issues_table.create!(project_id: project.id, issue_type: issue_type_enum[:task]) } - - let!(:batching_strategy) { described_class.new(connection: ActiveRecord::Base.connection) } - - context 'when issue_type is issue' do - let(:job_arguments) { [issue_type_enum[:issue], 'irrelevant_work_item_id'] } - - context 'when starting on the first batch' do - it 'returns the bounds of the next batch' do - batch_bounds = next_batch(issue1.id, 2) - - expect(batch_bounds).to match_array([issue1.id, issue2.id]) - end - end - - context 'when additional batches remain' do - it 'returns the bounds of the next batch' do - batch_bounds = next_batch(issue2.id, 2) - - expect(batch_bounds).to match_array([issue2.id, issue3.id]) - end - end - - context 'when on the final batch' do - it 'returns the bounds of the next batch' do - batch_bounds = next_batch(issue3.id, 2) - - expect(batch_bounds).to match_array([issue3.id, issue3.id]) - end - end - - context 'when no additional batches remain' do - it 'returns nil' do - batch_bounds = next_batch(issue3.id + 1, 1) - - expect(batch_bounds).to be_nil - end - end - end - - context 'when issue_type is incident' do - let(:job_arguments) { [issue_type_enum[:incident], 'irrelevant_work_item_id'] } - - context 'when starting on the first batch' do - it 'returns the bounds of the next batch with only one element' do - batch_bounds = next_batch(incident1.id, 2) - - expect(batch_bounds).to match_array([incident1.id, incident1.id]) - end - end - end - - context 'when issue_type is requirement and there are no matching records' do - let(:job_arguments) { [issue_type_enum[:requirement], 'irrelevant_work_item_id'] } - - context 'when starting on the first batch' do - it 'returns nil' do - batch_bounds = next_batch(1, 2) - - expect(batch_bounds).to be_nil - end - end - end - - context 'when issue_type is task' do - let(:job_arguments) { [issue_type_enum[:task], 'irrelevant_work_item_id'] } - - context 'when starting on the first batch' do - it 'returns the bounds of the next batch' do - batch_bounds = next_batch(task1.id, 2) - - expect(batch_bounds).to match_array([task1.id, task2.id]) - end - end - - context 'when additional batches remain' do - it 'returns the bounds of the next batch, does not skip records where FK is already set' do - batch_bounds = next_batch(task2.id, 2) - - expect(batch_bounds).to match_array([task2.id, task3.id]) - end - end - - context 'when on the final batch' do - it 'returns the bounds of the next batch' do - batch_bounds = next_batch(task4.id, 2) - - expect(batch_bounds).to match_array([task4.id, task4.id]) - end - end - - context 'when no additional batches remain' do - it 'returns nil' do - batch_bounds = next_batch(task4.id + 1, 1) - - expect(batch_bounds).to be_nil - end - end - end - - def next_batch(min_value, batch_size) - batching_strategy.next_batch( - :issues, - :id, - batch_min_value: min_value, - batch_size: batch_size, - job_arguments: job_arguments - ) - end -end diff --git a/spec/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy_spec.rb index 94e9bcf9207..7076e82ea34 100644 --- a/spec/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy_spec.rb +++ b/spec/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy_spec.rb @@ -2,137 +2,6 @@ require 'spec_helper' -RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::BackfillProjectStatisticsWithContainerRegistrySizeBatchingStrategy, '#next_batch' do # rubocop:disable Layout/LineLength - let(:batching_strategy) { described_class.new(connection: ActiveRecord::Base.connection) } - let(:namespace) { table(:namespaces) } - let(:project) { table(:projects) } - let(:container_repositories) { table(:container_repositories) } - - let!(:group) do - namespace.create!( - name: 'namespace1', type: 'Group', path: 'space1' - ) - end - - let!(:proj_namespace1) do - namespace.create!( - name: 'proj1', path: 'proj1', type: 'Project', parent_id: group.id - ) - end - - let!(:proj_namespace2) do - namespace.create!( - name: 'proj2', path: 'proj2', type: 'Project', parent_id: group.id - ) - end - - let!(:proj_namespace3) do - namespace.create!( - name: 'proj3', path: 'proj3', type: 'Project', parent_id: group.id - ) - end - - let!(:proj1) do - project.create!( - name: 'proj1', path: 'proj1', namespace_id: group.id, project_namespace_id: proj_namespace1.id - ) - end - - let!(:proj2) do - project.create!( - name: 'proj2', path: 'proj2', namespace_id: group.id, project_namespace_id: proj_namespace2.id - ) - end - - let!(:proj3) do - project.create!( - name: 'proj3', path: 'proj3', namespace_id: group.id, project_namespace_id: proj_namespace3.id - ) - end - - let!(:con1) do - container_repositories.create!( - project_id: proj1.id, - name: "ContReg_#{proj1.id}:1", - migration_state: 'import_done', - created_at: Date.new(2022, 01, 20) - ) - end - - let!(:con2) do - container_repositories.create!( - project_id: proj1.id, - name: "ContReg_#{proj1.id}:2", - migration_state: 'import_done', - created_at: Date.new(2022, 01, 20) - ) - end - - let!(:con3) do - container_repositories.create!( - project_id: proj2.id, - name: "ContReg_#{proj2.id}:1", - migration_state: 'import_done', - created_at: Date.new(2022, 01, 20) - ) - end - - let!(:con4) do - container_repositories.create!( - project_id: proj3.id, - name: "ContReg_#{proj3.id}:1", - migration_state: 'default', - created_at: Date.new(2022, 02, 20) - ) - end - - let!(:con5) do - container_repositories.create!( - project_id: proj3.id, - name: "ContReg_#{proj3.id}:2", - migration_state: 'default', - created_at: Date.new(2022, 02, 20) - ) - end - +RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::BackfillProjectStatisticsWithContainerRegistrySizeBatchingStrategy do # rubocop:disable Layout/LineLength it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy } - - context 'when starting on the first batch' do - it 'returns the bounds of the next batch' do - batch_bounds = batching_strategy.next_batch( - :container_repositories, - :project_id, - batch_min_value: con1.project_id, - batch_size: 3, - job_arguments: [] - ) - expect(batch_bounds).to eq([con1.project_id, con4.project_id]) - end - end - - context 'when additional batches remain' do - it 'returns the bounds of the next batch' do - batch_bounds = batching_strategy.next_batch( - :container_repositories, - :project_id, - batch_min_value: con3.project_id, - batch_size: 3, - job_arguments: [] - ) - - expect(batch_bounds).to eq([con3.project_id, con5.project_id]) - end - end - - context 'when no additional batches remain' do - it 'returns nil' do - batch_bounds = batching_strategy.next_batch(:container_repositories, - :project_id, - batch_min_value: con5.project_id + 1, - batch_size: 1, job_arguments: [] - ) - - expect(batch_bounds).to be_nil - end - end end diff --git a/spec/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy_spec.rb index f96c7de50f2..e4bef81e0bd 100644 --- a/spec/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy_spec.rb +++ b/spec/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy_spec.rb @@ -3,117 +3,5 @@ require 'spec_helper' RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::DismissedVulnerabilitiesStrategy, '#next_batch' do - let(:batching_strategy) { described_class.new(connection: ActiveRecord::Base.connection) } - let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') } - let(:users) { table(:users) } - let(:user) { create_user! } - let(:project) do - table(:projects).create!( - namespace_id: namespace.id, - project_namespace_id: namespace.id, - packages_enabled: false) - end - - let(:vulnerabilities) { table(:vulnerabilities) } - - let!(:vulnerability1) do - create_vulnerability!( - project_id: project.id, - author_id: user.id, - dismissed_at: Time.current - ) - end - - let!(:vulnerability2) do - create_vulnerability!( - project_id: project.id, - author_id: user.id, - dismissed_at: Time.current - ) - end - - let!(:vulnerability3) do - create_vulnerability!( - project_id: project.id, - author_id: user.id, - dismissed_at: Time.current - ) - end - - let!(:vulnerability4) do - create_vulnerability!( - project_id: project.id, - author_id: user.id, - dismissed_at: nil - ) - end - it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy } - - context 'when starting on the first batch' do - it 'returns the bounds of the next batch' do - batch_bounds = batching_strategy.next_batch( - :vulnerabilities, - :id, - batch_min_value: vulnerability1.id, - batch_size: 2, - job_arguments: [] - ) - expect(batch_bounds).to eq([vulnerability1.id, vulnerability2.id]) - end - end - - context 'when additional batches remain' do - it 'returns the bounds of the next batch and skips the records that do not have `dismissed_at` set' do - batch_bounds = batching_strategy.next_batch( - :vulnerabilities, - :id, - batch_min_value: vulnerability3.id, - batch_size: 2, - job_arguments: [] - ) - - expect(batch_bounds).to eq([vulnerability3.id, vulnerability3.id]) - end - end - - context 'when no additional batches remain' do - it 'returns nil' do - batch_bounds = batching_strategy.next_batch( - :vulnerabilities, - :id, - batch_min_value: vulnerability4.id + 1, - batch_size: 1, - job_arguments: [] - ) - - expect(batch_bounds).to be_nil - end - end - - private - - def create_vulnerability!( - project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0, state: 1, dismissed_at: nil - ) - vulnerabilities.create!( - project_id: project_id, - author_id: author_id, - title: title, - severity: severity, - confidence: confidence, - report_type: report_type, - state: state, - dismissed_at: dismissed_at - ) - end - - def create_user!(name: "Example User", email: "user@example.com", user_type: nil) - users.create!( - name: name, - email: email, - username: name, - projects_limit: 10 - ) - end end diff --git a/spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb index 9fdd7bf8adc..37fdd209622 100644 --- a/spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb +++ b/spec/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy_spec.rb @@ -60,26 +60,21 @@ RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchi expect(batch_bounds).to eq([namespace4.id, namespace4.id]) end - end - - context 'additional filters' do - let(:strategy_with_filters) do - Class.new(described_class) do - def apply_additional_filters(relation, job_arguments:, job_class: nil) - min_id = job_arguments.first - relation.where.not(type: 'Project').where('id >= ?', min_id) + context 'when scope has a join which makes the column name ambiguous' do + let(:job_class) do + Class.new(Gitlab::BackgroundMigration::BatchedMigrationJob) do + scope_to ->(r) { r.joins('LEFT JOIN users ON users.id = namespaces.owner_id') } end end - end - let(:batching_strategy) { strategy_with_filters.new(connection: ActiveRecord::Base.connection) } - let!(:namespace5) { namespaces.create!(name: 'batchtest5', path: 'batch-test5', type: 'Project') } + it 'executes the correct query' do + expect(job_class).to receive(:generic_instance).and_call_original - it 'applies additional filters' do - batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace4.id, batch_size: 3, job_arguments: [1]) + batch_bounds = batching_strategy.next_batch(:namespaces, :id, batch_min_value: namespace4.id, batch_size: 3, job_arguments: [], job_class: job_class) - expect(batch_bounds).to eq([namespace4.id, namespace4.id]) + expect(batch_bounds).to eq([namespace4.id, namespace4.id]) + end end end end diff --git a/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb new file mode 100644 index 00000000000..e296a46ea2f --- /dev/null +++ b/spec/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BatchingStrategies::RemoveBackfilledJobArtifactsExpireAtBatchingStrategy do # rubocop:disable Layout/LineLength + it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchingStrategies::PrimaryKeyBatchingStrategy } +end diff --git a/spec/lib/gitlab/background_migration/destroy_invalid_group_members_spec.rb b/spec/lib/gitlab/background_migration/destroy_invalid_group_members_spec.rb new file mode 100644 index 00000000000..76a9ea82c76 --- /dev/null +++ b/spec/lib/gitlab/background_migration/destroy_invalid_group_members_spec.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DestroyInvalidGroupMembers, :migration, schema: 20220809002011 do + # rubocop: disable Layout/LineLength + # rubocop: disable RSpec/ScatteredLet + let!(:migration_attrs) do + { + start_id: 1, + end_id: 1000, + batch_table: :members, + batch_column: :id, + sub_batch_size: 100, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let!(:migration) { described_class.new(**migration_attrs) } + + subject(:perform_migration) { migration.perform } + + let(:users_table) { table(:users) } + let(:namespaces_table) { table(:namespaces) } + let(:members_table) { table(:members) } + let(:projects_table) { table(:projects) } + + let(:user1) { users_table.create!(name: 'user1', email: 'user1@example.com', projects_limit: 5) } + let(:user2) { users_table.create!(name: 'user2', email: 'user2@example.com', projects_limit: 5) } + let(:user3) { users_table.create!(name: 'user3', email: 'user3@example.com', projects_limit: 5) } + let(:user4) { users_table.create!(name: 'user4', email: 'user4@example.com', projects_limit: 5) } + let(:user5) { users_table.create!(name: 'user5', email: 'user5@example.com', projects_limit: 5) } + let(:user6) { users_table.create!(name: 'user6', email: 'user6@example.com', projects_limit: 5) } + + let!(:group1) { namespaces_table.create!(name: 'marvellous group 1', path: 'group-path-1', type: 'Group') } + + let!(:group2) { namespaces_table.create!(name: 'outstanding group 2', path: 'group-path-2', type: 'Group') } + + # create group member records, a mix of both valid and invalid + # project members will have already been filtered out. + let!(:group_member1) { create_invalid_group_member(id: 1, user_id: user1.id) } + + let!(:group_member4) { create_valid_group_member(id: 4, user_id: user2.id, group_id: group1.id) } + + let!(:group_member5) { create_valid_group_member(id: 5, user_id: user3.id, group_id: group2.id) } + + let!(:group_member6) { create_invalid_group_member(id: 6, user_id: user4.id) } + + let!(:group_member7) { create_valid_group_member(id: 7, user_id: user5.id, group_id: group1.id) } + + let!(:group_member8) { create_invalid_group_member(id: 8, user_id: user6.id) } + + it 'removes invalid memberships but keeps valid ones', :aggregate_failures do + expect(members_table.where(type: 'GroupMember').count).to eq 6 + + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(4) + expect(members_table.where(type: 'GroupMember').pluck(:id)).to match_array([group_member4, group_member5, group_member7].map(&:id)) + end + + it 'tracks timings of queries' do + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end + + it 'logs IDs of deleted records' do + expect(Gitlab::AppLogger).to receive(:info).with({ message: 'Removing invalid group member records', + deleted_count: 3, ids: [group_member1, group_member6, group_member8].map(&:id) }) + + perform_migration + end + + def create_invalid_group_member(id:, user_id:) + members_table.create!(id: id, user_id: user_id, source_id: non_existing_record_id, access_level: Gitlab::Access::MAINTAINER, + type: "GroupMember", source_type: "Namespace", notification_level: 3, member_namespace_id: nil) + end + + def create_valid_group_member(id:, user_id:, group_id:) + members_table.create!(id: id, user_id: user_id, source_id: group_id, access_level: Gitlab::Access::MAINTAINER, + type: "GroupMember", source_type: "Namespace", member_namespace_id: group_id, notification_level: 3) + end + # rubocop: enable Layout/LineLength + # rubocop: enable RSpec/ScatteredLet +end diff --git a/spec/lib/gitlab/background_migration/destroy_invalid_project_members_spec.rb b/spec/lib/gitlab/background_migration/destroy_invalid_project_members_spec.rb new file mode 100644 index 00000000000..029a6adf831 --- /dev/null +++ b/spec/lib/gitlab/background_migration/destroy_invalid_project_members_spec.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DestroyInvalidProjectMembers, :migration, schema: 20220901035725 do + # rubocop: disable Layout/LineLength + # rubocop: disable RSpec/ScatteredLet + let!(:migration_attrs) do + { + start_id: 1, + end_id: 1000, + batch_table: :members, + batch_column: :id, + sub_batch_size: 100, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + let!(:migration) { described_class.new(**migration_attrs) } + + subject(:perform_migration) { migration.perform } + + let(:users_table) { table(:users) } + let(:namespaces_table) { table(:namespaces) } + let(:members_table) { table(:members) } + let(:projects_table) { table(:projects) } + + let(:user1) { users_table.create!(name: 'user1', email: 'user1@example.com', projects_limit: 5) } + let(:user2) { users_table.create!(name: 'user2', email: 'user2@example.com', projects_limit: 5) } + let(:user3) { users_table.create!(name: 'user3', email: 'user3@example.com', projects_limit: 5) } + let(:user4) { users_table.create!(name: 'user4', email: 'user4@example.com', projects_limit: 5) } + let(:user5) { users_table.create!(name: 'user5', email: 'user5@example.com', projects_limit: 5) } + let(:user6) { users_table.create!(name: 'user6', email: 'user6@example.com', projects_limit: 5) } + + let!(:group1) { namespaces_table.create!(name: 'marvellous group 1', path: 'group-path-1', type: 'Group') } + + let!(:project_namespace1) do + namespaces_table.create!(name: 'fabulous project', path: 'project-path-1', type: 'ProjectNamespace', + parent_id: group1.id) + end + + let!(:project1) do + projects_table.create!(name: 'fabulous project', path: 'project-path-1', project_namespace_id: project_namespace1.id, + namespace_id: group1.id) + end + + let!(:project_namespace2) do + namespaces_table.create!(name: 'splendiferous project', path: 'project-path-2', type: 'ProjectNamespace', + parent_id: group1.id) + end + + let!(:project2) do + projects_table.create!(name: 'splendiferous project', path: 'project-path-2', project_namespace_id: project_namespace2.id, + namespace_id: group1.id) + end + + # create project member records, a mix of both valid and invalid + # group members will have already been filtered out. + let!(:project_member1) { create_invalid_project_member(id: 1, user_id: user1.id) } + let!(:project_member2) { create_valid_project_member(id: 4, user_id: user2.id, project: project1) } + let!(:project_member3) { create_valid_project_member(id: 5, user_id: user3.id, project: project2) } + let!(:project_member4) { create_invalid_project_member(id: 6, user_id: user4.id) } + let!(:project_member5) { create_valid_project_member(id: 7, user_id: user5.id, project: project2) } + let!(:project_member6) { create_invalid_project_member(id: 8, user_id: user6.id) } + + it 'removes invalid memberships but keeps valid ones', :aggregate_failures do + expect(members_table.where(type: 'ProjectMember').count).to eq 6 + + queries = ActiveRecord::QueryRecorder.new do + perform_migration + end + + expect(queries.count).to eq(4) + expect(members_table.where(type: 'ProjectMember')).to match_array([project_member2, project_member3, project_member5]) + end + + it 'tracks timings of queries' do + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end + + it 'logs IDs of deleted records' do + expect(Gitlab::AppLogger).to receive(:info).with({ message: 'Removing invalid project member records', + deleted_count: 3, ids: [project_member1, project_member4, project_member6].map(&:id) }) + + perform_migration + end + + def create_invalid_project_member(id:, user_id:) + members_table.create!(id: id, user_id: user_id, source_id: non_existing_record_id, access_level: Gitlab::Access::MAINTAINER, + type: "ProjectMember", source_type: "Project", notification_level: 3, member_namespace_id: nil) + end + + def create_valid_project_member(id:, user_id:, project:) + members_table.create!(id: id, user_id: user_id, source_id: project.id, access_level: Gitlab::Access::MAINTAINER, + type: "ProjectMember", source_type: "Project", member_namespace_id: project.project_namespace_id, notification_level: 3) + end + # rubocop: enable Layout/LineLength + # rubocop: enable RSpec/ScatteredLet +end diff --git a/spec/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects_spec.rb b/spec/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects_spec.rb new file mode 100644 index 00000000000..7edba8cf524 --- /dev/null +++ b/spec/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DisableLegacyOpenSourceLicenceForRecentPublicProjects, :migration do + let(:namespaces_table) { table(:namespaces) } + let(:namespace_1) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-1') } + let(:project_namespace_2) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-2', type: 'Project') } + let(:project_namespace_3) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-3', type: 'Project') } + let(:project_namespace_4) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-4', type: 'Project') } + let(:project_namespace_5) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-5', type: 'Project') } + let(:project_namespace_6) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-6', type: 'Project') } + let(:project_namespace_7) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-7', type: 'Project') } + let(:project_namespace_8) { namespaces_table.create!(name: 'namespace', path: 'namespace-path-8', type: 'Project') } + + let(:project_1) do + projects_table + .create!( + name: 'proj-1', path: 'path-1', namespace_id: namespace_1.id, + project_namespace_id: project_namespace_2.id, visibility_level: 0 + ) + end + + let(:project_2) do + projects_table + .create!( + name: 'proj-2', path: 'path-2', namespace_id: namespace_1.id, + project_namespace_id: project_namespace_3.id, visibility_level: 10, created_at: '2022-02-22' + ) + end + + let(:project_3) do + projects_table + .create!( + name: 'proj-3', path: 'path-3', namespace_id: namespace_1.id, + project_namespace_id: project_namespace_4.id, visibility_level: 20, created_at: '2022-02-17 09:00:01' + ) + end + + let(:project_4) do + projects_table + .create!( + name: 'proj-4', path: 'path-4', namespace_id: namespace_1.id, + project_namespace_id: project_namespace_5.id, visibility_level: 20, created_at: '2022-02-01' + ) + end + + let(:project_5) do + projects_table + .create!( + name: 'proj-5', path: 'path-5', namespace_id: namespace_1.id, + project_namespace_id: project_namespace_6.id, visibility_level: 20, created_at: '2022-01-04' + ) + end + + let(:project_6) do + projects_table + .create!( + name: 'proj-6', path: 'path-6', namespace_id: namespace_1.id, + project_namespace_id: project_namespace_7.id, visibility_level: 20, created_at: '2022-02-17 08:59:59' + ) + end + + let(:project_7) do + projects_table + .create!( + name: 'proj-7', path: 'path-7', namespace_id: namespace_1.id, + project_namespace_id: project_namespace_8.id, visibility_level: 20, created_at: '2022-02-17 09:00:00' + ) + end + + let(:projects_table) { table(:projects) } + let(:project_settings_table) { table(:project_settings) } + + subject(:perform_migration) do + described_class.new(start_id: projects_table.minimum(:id), + end_id: projects_table.maximum(:id), + batch_table: :projects, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + before do + project_settings_table.create!(project_id: project_1.id, legacy_open_source_license_available: true) + project_settings_table.create!(project_id: project_2.id, legacy_open_source_license_available: true) + project_settings_table.create!(project_id: project_3.id, legacy_open_source_license_available: true) + project_settings_table.create!(project_id: project_4.id, legacy_open_source_license_available: true) + project_settings_table.create!(project_id: project_5.id, legacy_open_source_license_available: false) + project_settings_table.create!(project_id: project_6.id, legacy_open_source_license_available: true) + project_settings_table.create!(project_id: project_7.id, legacy_open_source_license_available: true) + end + + it 'sets `legacy_open_source_license_available` attribute to false for public projects created after threshold time', + :aggregate_failures do + record = ActiveRecord::QueryRecorder.new do + expect { perform_migration } + .to not_change { migrated_attribute(project_1.id) }.from(true) + .and not_change { migrated_attribute(project_2.id) }.from(true) + .and change { migrated_attribute(project_3.id) }.from(true).to(false) + .and not_change { migrated_attribute(project_4.id) }.from(true) + .and not_change { migrated_attribute(project_5.id) }.from(false) + .and not_change { migrated_attribute(project_6.id) }.from(true) + .and change { migrated_attribute(project_7.id) }.from(true).to(false) + end + expect(record.count).to eq(19) + end + + def migrated_attribute(project_id) + project_settings_table.find(project_id).legacy_open_source_license_available + end +end diff --git a/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb_spec.rb b/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb_spec.rb new file mode 100644 index 00000000000..205350f9df4 --- /dev/null +++ b/spec/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb_spec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb, + :migration, + schema: 20220906074449 do + let(:namespaces_table) { table(:namespaces) } + let(:projects_table) { table(:projects) } + let(:project_settings_table) { table(:project_settings) } + let(:project_statistics_table) { table(:project_statistics) } + + subject(:perform_migration) do + described_class.new(start_id: project_settings_table.minimum(:project_id), + end_id: project_settings_table.maximum(:project_id), + batch_table: :project_settings, + batch_column: :project_id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + it 'sets `legacy_open_source_license_available` to false only for projects less than 1 MB', + :aggregate_failures do + project_setting_1_mb = create_legacy_license_project_setting(repo_size: 1) + project_setting_2_mb = create_legacy_license_project_setting(repo_size: 2) + project_setting_quarter_mb = create_legacy_license_project_setting(repo_size: 0.25) + project_setting_half_mb = create_legacy_license_project_setting(repo_size: 0.5) + + queries = ActiveRecord::QueryRecorder.new { perform_migration } + + expect(queries.count).to eq(7) + expect(migrated_attribute(project_setting_1_mb)).to be_truthy + expect(migrated_attribute(project_setting_2_mb)).to be_truthy + expect(migrated_attribute(project_setting_quarter_mb)).to be_falsey + expect(migrated_attribute(project_setting_half_mb)).to be_falsey + end + + private + + # @param repo_size: Repo size in MB + def create_legacy_license_project_setting(repo_size:) + path = "path-for-repo-size-#{repo_size}" + namespace = namespaces_table.create!(name: "namespace-#{path}", path: "namespace-#{path}") + project_namespace = + namespaces_table.create!(name: "-project-namespace-#{path}", path: "project-namespace-#{path}", type: 'Project') + project = projects_table + .create!(name: path, path: path, namespace_id: namespace.id, project_namespace_id: project_namespace.id) + + size_in_bytes = 1.megabyte * repo_size + project_statistics_table.create!(project_id: project.id, namespace_id: namespace.id, repository_size: size_in_bytes) + project_settings_table.create!(project_id: project.id, legacy_open_source_license_available: true) + end + + def migrated_attribute(project_setting) + project_settings_table.find(project_setting.project_id).legacy_open_source_license_available + end +end diff --git a/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb b/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb index 38e8b159e63..c788b701d79 100644 --- a/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb +++ b/spec/lib/gitlab/background_migration/encrypt_integration_properties_spec.rb @@ -40,10 +40,10 @@ RSpec.describe Gitlab::BackgroundMigration::EncryptIntegrationProperties, schema expect(integrations.count).to eq(4) expect(props).to match( - no_properties.id => both(be_nil), + no_properties.id => both(be_nil), with_plaintext_1.id => both(eq some_props(1)), with_plaintext_2.id => both(eq some_props(2)), - with_encrypted.id => match([be_nil, eq(some_props(3))]) + with_encrypted.id => match([be_nil, eq(some_props(3))]) ) end diff --git a/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb new file mode 100644 index 00000000000..41266cb24da --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveBackfilledJobArtifactsExpireAt do + it { expect(described_class).to be < Gitlab::BackgroundMigration::BatchedMigrationJob } + + describe '#perform' do + let(:job_artifact) { table(:ci_job_artifacts, database: :ci) } + + let(:test_worker) do + described_class.new( + start_id: 1, + end_id: 100, + batch_table: :ci_job_artifacts, + batch_column: :id, + sub_batch_size: 10, + pause_ms: 0, + connection: Ci::ApplicationRecord.connection + ) + end + + let_it_be(:namespace) { table(:namespaces).create!(id: 1, name: 'user', path: 'user') } + let_it_be(:project) do + table(:projects).create!( + id: 1, + name: 'gitlab1', + path: 'gitlab1', + project_namespace_id: 1, + namespace_id: namespace.id + ) + end + + subject { test_worker.perform } + + context 'with artifacts that has backfilled expire_at' do + let!(:created_on_00_30_45_minutes_on_21_22_23) do + create_job_artifact(id: 1, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) + create_job_artifact(id: 2, file_type: 1, expire_at: Time.zone.parse('2022-01-21 01:30:00.000')) + create_job_artifact(id: 3, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:00:00.000')) + create_job_artifact(id: 4, file_type: 1, expire_at: Time.zone.parse('2022-01-22 12:30:00.000')) + create_job_artifact(id: 5, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:00:00.000')) + create_job_artifact(id: 6, file_type: 1, expire_at: Time.zone.parse('2022-01-23 23:30:00.000')) + create_job_artifact(id: 7, file_type: 1, expire_at: Time.zone.parse('2022-01-23 06:45:00.000')) + end + + let!(:created_close_to_00_or_30_minutes) do + create_job_artifact(id: 8, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:00:00.001')) + create_job_artifact(id: 9, file_type: 1, expire_at: Time.zone.parse('2022-01-21 00:30:00.999')) + end + + let!(:created_on_00_or_30_minutes_on_other_dates) do + create_job_artifact(id: 10, file_type: 1, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) + create_job_artifact(id: 11, file_type: 1, expire_at: Time.zone.parse('2022-01-19 12:00:00.000')) + create_job_artifact(id: 12, file_type: 1, expire_at: Time.zone.parse('2022-01-24 23:30:00.000')) + end + + let!(:created_at_other_times) do + create_job_artifact(id: 13, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:00:00.000')) + create_job_artifact(id: 14, file_type: 1, expire_at: Time.zone.parse('2022-01-19 00:30:00.000')) + create_job_artifact(id: 15, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:00:00.000')) + create_job_artifact(id: 16, file_type: 1, expire_at: Time.zone.parse('2022-01-24 00:30:00.000')) + end + + it 'removes expire_at on job artifacts that have expire_at on 00, 30 or 45 minute of 21, 22, 23 of the month' do + expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(7) + end + + it 'keeps expire_at on other job artifacts' do + expect { subject }.to change { job_artifact.where.not(expire_at: nil).count }.from(16).to(9) + end + end + + context 'with trace artifacts that has backfilled expire_at' do + let!(:trace_artifacts) do + create_job_artifact(id: 1, file_type: 3, expire_at: Time.zone.parse('2022-01-01 00:00:00.000')) + create_job_artifact(id: 2, file_type: 3, expire_at: Time.zone.parse('2022-01-21 00:00:00.000')) + end + + it 'removes expire_at on trace job artifacts' do + expect { subject }.to change { job_artifact.where(expire_at: nil).count }.from(0).to(2) + end + end + + private + + def create_job_artifact(id:, file_type:, expire_at:) + job = table(:ci_builds, database: :ci).create!(id: id) + job_artifact.create!(id: id, job_id: job.id, expire_at: expire_at, project_id: project.id, file_type: file_type) + end + end +end diff --git a/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb b/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb new file mode 100644 index 00000000000..81927100562 --- /dev/null +++ b/spec/lib/gitlab/background_migration/remove_self_managed_wiki_notes_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RemoveSelfManagedWikiNotes, :migration, schema: 20220601110011 do + let(:notes) { table(:notes) } + + subject(:perform_migration) do + described_class.new(start_id: 1, + end_id: 30, + batch_table: :notes, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 0, + connection: ActiveRecord::Base.connection) + .perform + end + + it 'removes all wiki notes' do + notes.create!(id: 2, note: 'Commit note', noteable_type: 'Commit') + notes.create!(id: 10, note: 'Issue note', noteable_type: 'Issue') + notes.create!(id: 20, note: 'Wiki note', noteable_type: 'Wiki') + notes.create!(id: 30, note: 'MergeRequest note', noteable_type: 'MergeRequest') + + expect(notes.where(noteable_type: 'Wiki').size).to eq(1) + + expect { perform_migration }.to change(notes, :count).by(-1) + + expect(notes.where(noteable_type: 'Wiki').size).to eq(0) + end +end diff --git a/spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb b/spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb new file mode 100644 index 00000000000..45932defaf9 --- /dev/null +++ b/spec/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::RenameTaskSystemNoteToChecklistItem do + let(:notes) { table(:notes) } + let(:projects) { table(:projects) } + + let(:namespace) { table(:namespaces).create!(name: 'batchtest1', type: 'Group', path: 'space1') } + let(:project) { table(:projects).create!(name: 'proj1', path: 'proj1', namespace_id: namespace.id) } + let(:issue) { table(:issues).create!(title: 'Test issue') } + + let!(:note1) do + notes.create!( + note: 'marked the task **Task 1** as complete', noteable_type: 'Issue', noteable_id: issue.id, system: true + ) + end + + let!(:note2) do + notes.create!( + note: 'NO_MATCH marked the task **Task 2** as complete', + noteable_type: 'Issue', + noteable_id: issue.id, + system: true + ) + end + + let!(:note3) do + notes.create!( + note: 'marked the task **Task 3** as incomplete', + noteable_type: 'Issue', + noteable_id: issue.id, + system: true + ) + end + + let!(:note4) do + notes.create!( + note: 'marked the task **Task 4** as incomplete', + noteable_type: 'Issue', + noteable_id: issue.id, + system: true + ) + end + + let!(:metadata1) { table(:system_note_metadata).create!(note_id: note1.id, action: :task) } + let!(:metadata2) { table(:system_note_metadata).create!(note_id: note2.id, action: :task) } + let!(:metadata3) { table(:system_note_metadata).create!(note_id: note3.id, action: :task) } + let!(:metadata4) { table(:system_note_metadata).create!(note_id: note4.id, action: :not_task) } + + let(:migration) do + described_class.new( + start_id: metadata1.id, + end_id: metadata4.id, + batch_table: :system_note_metadata, + batch_column: :id, + sub_batch_size: 2, + pause_ms: 2, + connection: ApplicationRecord.connection + ) + end + + subject(:perform_migration) { migration.perform } + + it 'renames task to checklist item in task system notes that match', :aggregate_failures do + expect do + perform_migration + + note1.reload + note2.reload + note3.reload + note4.reload + end.to change(note1, :note).to('marked the checklist item **Task 1** as complete').and( + not_change(note2, :note).from('NO_MATCH marked the task **Task 2** as complete') + ).and( + change(note3, :note).to('marked the checklist item **Task 3** as incomplete') + ).and( + not_change(note4, :note).from('marked the task **Task 4** as incomplete') + ) + end + + it 'updates in batches' do + expect { perform_migration }.to make_queries_matching(/UPDATE notes/, 2) + end + + it 'tracks timings of queries' do + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end +end diff --git a/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb b/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb index d5b98e49a31..2372ce21c4c 100644 --- a/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb +++ b/spec/lib/gitlab/background_migration/set_correct_vulnerability_state_spec.rb @@ -34,7 +34,7 @@ RSpec.describe Gitlab::BackgroundMigration::SetCorrectVulnerabilityState do let(:detected_state) { 1 } let(:dismissed_state) { 2 } - subject(:perform_migration) do + let(:migration_job) do described_class.new(start_id: vulnerability_with_dismissed_at.id, end_id: vulnerability_without_dismissed_at.id, batch_table: :vulnerabilities, @@ -42,15 +42,24 @@ RSpec.describe Gitlab::BackgroundMigration::SetCorrectVulnerabilityState do sub_batch_size: 1, pause_ms: 0, connection: ActiveRecord::Base.connection) - .perform end - it 'changes vulnerability state to `dismissed` when dismissed_at is not nil' do - expect { perform_migration }.to change { vulnerability_with_dismissed_at.reload.state }.to(dismissed_state) + describe '#filter_batch' do + it 'filters out vulnerabilities where dismissed_at is null' do + expect(migration_job.filter_batch(vulnerabilities)).to contain_exactly(vulnerability_with_dismissed_at) + end end - it 'does not change the state when dismissed_at is nil' do - expect { perform_migration }.not_to change { vulnerability_without_dismissed_at.reload.state } + describe '#perform' do + subject(:perform_migration) { migration_job.perform } + + it 'changes vulnerability state to `dismissed` when dismissed_at is not nil' do + expect { perform_migration }.to change { vulnerability_with_dismissed_at.reload.state }.to(dismissed_state) + end + + it 'does not change the state when dismissed_at is nil' do + expect { perform_migration }.not_to change { vulnerability_without_dismissed_at.reload.state } + end end private |