diff options
Diffstat (limited to 'spec/lib/bulk_imports')
13 files changed, 167 insertions, 69 deletions
diff --git a/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb b/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb index f03a178b993..9ea519d367e 100644 --- a/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb +++ b/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb @@ -13,11 +13,12 @@ RSpec.describe BulkImports::Common::Pipelines::EntityFinisher do expect(logger) .to receive(:info) .with( - bulk_import_id: entity.bulk_import.id, + bulk_import_id: entity.bulk_import_id, bulk_import_entity_id: entity.id, bulk_import_entity_type: entity.source_type, pipeline_class: described_class.name, - message: 'Entity finished' + message: 'Entity finished', + importer: 'gitlab_migration' ) end diff --git a/spec/lib/bulk_imports/common/pipelines/lfs_objects_pipeline_spec.rb b/spec/lib/bulk_imports/common/pipelines/lfs_objects_pipeline_spec.rb index f0b461e518e..5220b9d37e5 100644 --- a/spec/lib/bulk_imports/common/pipelines/lfs_objects_pipeline_spec.rb +++ b/spec/lib/bulk_imports/common/pipelines/lfs_objects_pipeline_spec.rb @@ -7,7 +7,7 @@ RSpec.describe BulkImports::Common::Pipelines::LfsObjectsPipeline do let_it_be(:oid) { 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' } let(:tmpdir) { Dir.mktmpdir } - let(:entity) { create(:bulk_import_entity, :project_entity, project: portable, source_full_path: 'test') } + let(:entity) { create(:bulk_import_entity, :project_entity, project: portable, source_full_path: 'test', source_xid: nil) } let(:tracker) { create(:bulk_import_tracker, entity: entity) } let(:context) { BulkImports::Pipeline::Context.new(tracker) } let(:lfs_dir_path) { tmpdir } diff --git a/spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb b/spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb index f650e931dc7..7a93365d098 100644 --- a/spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb +++ b/spec/lib/bulk_imports/common/pipelines/uploads_pipeline_spec.rb @@ -152,14 +152,14 @@ RSpec.describe BulkImports::Common::Pipelines::UploadsPipeline do context 'when importing to group' do let(:portable) { group } - let(:entity) { create(:bulk_import_entity, :group_entity, group: group, source_full_path: 'test') } + let(:entity) { create(:bulk_import_entity, :group_entity, group: group, source_full_path: 'test', source_xid: nil) } include_examples 'uploads import' end context 'when importing to project' do let(:portable) { project } - let(:entity) { create(:bulk_import_entity, :project_entity, project: project, source_full_path: 'test') } + let(:entity) { create(:bulk_import_entity, :project_entity, project: project, source_full_path: 'test', source_xid: nil) } include_examples 'uploads import' end diff --git a/spec/lib/bulk_imports/common/rest/get_badges_query_spec.rb b/spec/lib/bulk_imports/common/rest/get_badges_query_spec.rb index 0a04c0a2243..fabef50af8b 100644 --- a/spec/lib/bulk_imports/common/rest/get_badges_query_spec.rb +++ b/spec/lib/bulk_imports/common/rest/get_badges_query_spec.rb @@ -9,15 +9,32 @@ RSpec.describe BulkImports::Common::Rest::GetBadgesQuery do let(:context) { BulkImports::Pipeline::Context.new(tracker) } let(:encoded_full_path) { ERB::Util.url_encode(entity.source_full_path) } - it 'returns correct query and page info' do - expected = { - resource: [entity.pluralized_name, encoded_full_path, 'badges'].join('/'), - query: { - page: context.tracker.next_page + context 'when source id is present' do + it 'returns correct query using source id and page info' do + expected = { + resource: [entity.base_resource_path, 'badges'].join('/'), + query: { + page: context.tracker.next_page + } } - } - expect(described_class.to_h(context)).to eq(expected) + expect(described_class.to_h(context)).to eq(expected) + end + end + + context 'when source id is missing' do + it 'returns correct query using source full path' do + entity.update!(source_xid: nil) + + expected = { + resource: ["/#{entity.pluralized_name}", encoded_full_path, 'badges'].join('/'), + query: { + page: context.tracker.next_page + } + } + + expect(described_class.to_h(context)).to eq(expected) + end end end diff --git a/spec/lib/bulk_imports/features_spec.rb b/spec/lib/bulk_imports/features_spec.rb new file mode 100644 index 00000000000..a92e4706bbe --- /dev/null +++ b/spec/lib/bulk_imports/features_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Features do + describe '.project_migration_enabled' do + let_it_be(:top_level_namespace) { create(:group) } + + context 'when bulk_import_projects feature flag is enabled' do + it 'returns true' do + stub_feature_flags(bulk_import_projects: true) + + expect(described_class.project_migration_enabled?).to eq(true) + end + + context 'when feature flag is enabled on root ancestor level' do + it 'returns true' do + stub_feature_flags(bulk_import_projects: top_level_namespace) + + expect(described_class.project_migration_enabled?(top_level_namespace.full_path)).to eq(true) + end + end + + context 'when feature flag is enabled on a different top level namespace' do + it 'returns false' do + stub_feature_flags(bulk_import_projects: top_level_namespace) + + different_namepace = create(:group) + + expect(described_class.project_migration_enabled?(different_namepace.full_path)).to eq(false) + end + end + end + + context 'when bulk_import_projects feature flag is disabled' do + it 'returns false' do + stub_feature_flags(bulk_import_projects: false) + + expect(described_class.project_migration_enabled?(top_level_namespace.full_path)).to eq(false) + end + end + end +end diff --git a/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb index 441a34b0c74..36b425f4f12 100644 --- a/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb @@ -13,7 +13,7 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do :bulk_import_entity, bulk_import: bulk_import, source_full_path: 'source/full/path', - destination_name: 'My Destination Group', + destination_slug: 'my-destination-group', destination_namespace: parent.full_path ) end diff --git a/spec/lib/bulk_imports/groups/pipelines/project_entities_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/project_entities_pipeline_spec.rb index 5b6c93e695f..c07d27e973f 100644 --- a/spec/lib/bulk_imports/groups/pipelines/project_entities_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/project_entities_pipeline_spec.rb @@ -19,6 +19,7 @@ RSpec.describe BulkImports::Groups::Pipelines::ProjectEntitiesPipeline do let(:extracted_data) do BulkImports::Pipeline::ExtractedData.new(data: { + 'id' => 'gid://gitlab/Project/1234567', 'name' => 'project', 'full_path' => 'group/project' }) @@ -44,6 +45,7 @@ RSpec.describe BulkImports::Groups::Pipelines::ProjectEntitiesPipeline do expect(project_entity.source_full_path).to eq('group/project') expect(project_entity.destination_name).to eq('project') expect(project_entity.destination_namespace).to eq(destination_group.full_path) + expect(project_entity.source_xid).to eq(1234567) end end diff --git a/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb b/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb index 896af865c56..32d8dc8e207 100644 --- a/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb +++ b/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb @@ -24,59 +24,67 @@ RSpec.describe BulkImports::Groups::Transformers::GroupAttributesTransformer do let(:data) do { 'name' => 'Source Group Name', + 'description' => 'Source Group Description', 'path' => 'source-group-path', 'full_path' => 'source/full/path', 'visibility' => 'private', 'project_creation_level' => 'developer', - 'subgroup_creation_level' => 'maintainer' + 'subgroup_creation_level' => 'maintainer', + 'emails_disabled' => true, + 'lfs_enabled' => false, + 'mentions_disabled' => true, + 'share_with_group_lock' => false, + 'require_two_factor_authentication' => false, + 'two_factor_grace_period' => 100, + 'request_access_enabled' => false } end subject { described_class.new } it 'returns original data with some keys transformed' do - transformed_data = subject.transform(context, { 'name' => 'Name', 'description' => 'Description' }) + transformed_data = subject.transform(context, data) expect(transformed_data).to eq({ - 'name' => 'Name', - 'description' => 'Description', + 'name' => 'Source Group Name', + 'description' => 'Source Group Description', 'parent_id' => parent.id, - 'path' => 'destination-slug-path' + 'path' => entity.destination_slug, + 'visibility_level' => Gitlab::VisibilityLevel.string_options[data['visibility']], + 'project_creation_level' => Gitlab::Access.project_creation_string_options[data['project_creation_level']], + 'subgroup_creation_level' => Gitlab::Access.subgroup_creation_string_options[data['subgroup_creation_level']], + 'emails_disabled' => true, + 'lfs_enabled' => false, + 'mentions_disabled' => true, + 'share_with_group_lock' => false, + 'require_two_factor_authentication' => false, + 'two_factor_grace_period' => 100, + 'request_access_enabled' => false }) end - it 'transforms path from destination_slug' do - transformed_data = subject.transform(context, data) - - expect(transformed_data['path']).to eq(entity.destination_slug) - end - - it 'removes full path' do - transformed_data = subject.transform(context, data) - - expect(transformed_data).not_to have_key('full_path') - end - - it 'transforms visibility level' do - visibility = data['visibility'] - transformed_data = subject.transform(context, data) - - expect(transformed_data).not_to have_key('visibility') - expect(transformed_data['visibility_level']).to eq(Gitlab::VisibilityLevel.string_options[visibility]) - end - - it 'transforms project creation level' do - level = data['project_creation_level'] - transformed_data = subject.transform(context, data) + context 'when some fields are not present' do + it 'does not include those fields' do + data = { + 'name' => 'Source Group Name', + 'description' => 'Source Group Description', + 'path' => 'source-group-path', + 'full_path' => 'source/full/path' + } - expect(transformed_data['project_creation_level']).to eq(Gitlab::Access.project_creation_string_options[level]) - end - - it 'transforms subgroup creation level' do - level = data['subgroup_creation_level'] - transformed_data = subject.transform(context, data) + transformed_data = subject.transform(context, data) - expect(transformed_data['subgroup_creation_level']).to eq(Gitlab::Access.subgroup_creation_string_options[level]) + expect(transformed_data).to eq({ + 'name' => 'Source Group Name', + 'path' => 'destination-slug-path', + 'description' => 'Source Group Description', + 'parent_id' => parent.id, + 'share_with_group_lock' => nil, + 'emails_disabled' => nil, + 'lfs_enabled' => nil, + 'mentions_disabled' => nil + }) + end end describe 'parent group transformation' do diff --git a/spec/lib/bulk_imports/network_error_spec.rb b/spec/lib/bulk_imports/network_error_spec.rb index 11f555fee09..54d6554df96 100644 --- a/spec/lib/bulk_imports/network_error_spec.rb +++ b/spec/lib/bulk_imports/network_error_spec.rb @@ -46,6 +46,22 @@ RSpec.describe BulkImports::NetworkError, :clean_gitlab_redis_cache do expect(exception.retriable?(tracker)).to eq(false) end end + + context 'when entity is passed' do + it 'increments entity cache key' do + entity = create(:bulk_import_entity) + exception = described_class.new('Error!') + + allow(exception).to receive(:cause).and_return(SocketError.new('Error!')) + + expect(Gitlab::Cache::Import::Caching) + .to receive(:increment) + .with("bulk_imports/#{entity.id}/network_error/SocketError") + .and_call_original + + exception.retriable?(entity) + end + end end describe '#retry_delay' do diff --git a/spec/lib/bulk_imports/pipeline/runner_spec.rb b/spec/lib/bulk_imports/pipeline/runner_spec.rb index 810271818ae..a5a01354d0e 100644 --- a/spec/lib/bulk_imports/pipeline/runner_spec.rb +++ b/spec/lib/bulk_imports/pipeline/runner_spec.rb @@ -60,7 +60,9 @@ RSpec.describe BulkImports::Pipeline::Runner do pipeline_step: :extractor, pipeline_class: 'BulkImports::MyPipeline', exception_class: exception_class, - exception_message: exception_message + exception_message: exception_message, + message: "Pipeline failed", + importer: 'gitlab_migration' ) ) end @@ -89,7 +91,8 @@ RSpec.describe BulkImports::Pipeline::Runner do log_params( context, message: 'Aborting entity migration due to pipeline failure', - pipeline_class: 'BulkImports::MyPipeline' + pipeline_class: 'BulkImports::MyPipeline', + importer: 'gitlab_migration' ) ) end @@ -290,9 +293,10 @@ RSpec.describe BulkImports::Pipeline::Runner do def log_params(context, extra = {}) { - bulk_import_id: context.bulk_import.id, + bulk_import_id: context.bulk_import_id, bulk_import_entity_id: context.entity.id, bulk_import_entity_type: context.entity.source_type, + importer: 'gitlab_migration', context_extra: context.extra }.merge(extra) end diff --git a/spec/lib/bulk_imports/pipeline_spec.rb b/spec/lib/bulk_imports/pipeline_spec.rb index dc169bb8d88..72bc8bd7980 100644 --- a/spec/lib/bulk_imports/pipeline_spec.rb +++ b/spec/lib/bulk_imports/pipeline_spec.rb @@ -20,16 +20,17 @@ RSpec.describe BulkImports::Pipeline do loader BulkImports::Loader, foo: :bar end - stub_const('BulkImports::MyPipeline', klass) + stub_const('BulkImports::TestWikiPipeline', klass) end describe 'pipeline attributes' do describe 'getters' do it 'retrieves class attributes' do - expect(BulkImports::MyPipeline.get_extractor).to eq({ klass: BulkImports::Extractor, options: { foo: :bar } }) - expect(BulkImports::MyPipeline.transformers).to contain_exactly({ klass: BulkImports::Transformer, options: { foo: :bar } }) - expect(BulkImports::MyPipeline.get_loader).to eq({ klass: BulkImports::Loader, options: { foo: :bar } }) - expect(BulkImports::MyPipeline.abort_on_failure?).to eq(true) + expect(BulkImports::TestWikiPipeline.get_extractor).to eq({ klass: BulkImports::Extractor, options: { foo: :bar } }) + expect(BulkImports::TestWikiPipeline.transformers).to contain_exactly({ klass: BulkImports::Transformer, options: { foo: :bar } }) + expect(BulkImports::TestWikiPipeline.get_loader).to eq({ klass: BulkImports::Loader, options: { foo: :bar } }) + expect(BulkImports::TestWikiPipeline.abort_on_failure?).to eq(true) + expect(BulkImports::TestWikiPipeline.relation).to eq('test_wiki') end context 'when extractor and loader are defined within the pipeline' do @@ -59,23 +60,23 @@ RSpec.describe BulkImports::Pipeline do klass = Class.new options = { test: :test } - BulkImports::MyPipeline.extractor(klass, options) - BulkImports::MyPipeline.transformer(klass, options) - BulkImports::MyPipeline.loader(klass, options) - BulkImports::MyPipeline.abort_on_failure! - BulkImports::MyPipeline.file_extraction_pipeline! + BulkImports::TestWikiPipeline.extractor(klass, options) + BulkImports::TestWikiPipeline.transformer(klass, options) + BulkImports::TestWikiPipeline.loader(klass, options) + BulkImports::TestWikiPipeline.abort_on_failure! + BulkImports::TestWikiPipeline.file_extraction_pipeline! - expect(BulkImports::MyPipeline.get_extractor).to eq({ klass: klass, options: options }) + expect(BulkImports::TestWikiPipeline.get_extractor).to eq({ klass: klass, options: options }) - expect(BulkImports::MyPipeline.transformers) + expect(BulkImports::TestWikiPipeline.transformers) .to contain_exactly( { klass: BulkImports::Transformer, options: { foo: :bar } }, { klass: klass, options: options }) - expect(BulkImports::MyPipeline.get_loader).to eq({ klass: klass, options: options }) + expect(BulkImports::TestWikiPipeline.get_loader).to eq({ klass: klass, options: options }) - expect(BulkImports::MyPipeline.abort_on_failure?).to eq(true) - expect(BulkImports::MyPipeline.file_extraction_pipeline?).to eq(true) + expect(BulkImports::TestWikiPipeline.abort_on_failure?).to eq(true) + expect(BulkImports::TestWikiPipeline.file_extraction_pipeline?).to eq(true) end end end @@ -87,7 +88,7 @@ RSpec.describe BulkImports::Pipeline do expect(BulkImports::Transformer).to receive(:new).with(foo: :bar) expect(BulkImports::Loader).to receive(:new).with(foo: :bar) - pipeline = BulkImports::MyPipeline.new(context) + pipeline = BulkImports::TestWikiPipeline.new(context) pipeline.send(:extractor) pipeline.send(:transformers) diff --git a/spec/lib/bulk_imports/projects/pipelines/design_bundle_pipeline_spec.rb b/spec/lib/bulk_imports/projects/pipelines/design_bundle_pipeline_spec.rb index 39b539ece21..6a509ca7f14 100644 --- a/spec/lib/bulk_imports/projects/pipelines/design_bundle_pipeline_spec.rb +++ b/spec/lib/bulk_imports/projects/pipelines/design_bundle_pipeline_spec.rb @@ -8,7 +8,10 @@ RSpec.describe BulkImports::Projects::Pipelines::DesignBundlePipeline do let(:portable) { create(:project) } let(:tmpdir) { Dir.mktmpdir } let(:design_bundle_path) { File.join(tmpdir, 'design.bundle') } - let(:entity) { create(:bulk_import_entity, :project_entity, project: portable, source_full_path: 'test') } + let(:entity) do + create(:bulk_import_entity, :project_entity, project: portable, source_full_path: 'test', source_xid: nil) + end + let(:tracker) { create(:bulk_import_tracker, entity: entity) } let(:context) { BulkImports::Pipeline::Context.new(tracker) } diff --git a/spec/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline_spec.rb b/spec/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline_spec.rb index 712c37ee578..b8c21feb05d 100644 --- a/spec/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline_spec.rb +++ b/spec/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline_spec.rb @@ -8,7 +8,10 @@ RSpec.describe BulkImports::Projects::Pipelines::RepositoryBundlePipeline do let(:portable) { create(:project) } let(:tmpdir) { Dir.mktmpdir } let(:bundle_path) { File.join(tmpdir, 'repository.bundle') } - let(:entity) { create(:bulk_import_entity, :project_entity, project: portable, source_full_path: 'test') } + let(:entity) do + create(:bulk_import_entity, :project_entity, project: portable, source_full_path: 'test', source_xid: nil) + end + let(:tracker) { create(:bulk_import_tracker, entity: entity) } let(:context) { BulkImports::Pipeline::Context.new(tracker) } |