diff options
Diffstat (limited to 'spec/lib/bulk_imports')
23 files changed, 421 insertions, 285 deletions
diff --git a/spec/lib/bulk_imports/common/extractors/rest_extractor_spec.rb b/spec/lib/bulk_imports/common/extractors/rest_extractor_spec.rb new file mode 100644 index 00000000000..721dacbe3f4 --- /dev/null +++ b/spec/lib/bulk_imports/common/extractors/rest_extractor_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Common::Extractors::RestExtractor do + let(:http_client) { instance_double(BulkImports::Clients::Http) } + let(:options) { { query: double(to_h: { resource: nil, query: nil }) } } + let(:response) { double(parsed_response: { 'data' => { 'foo' => 'bar' } }, headers: { 'x-next-page' => '2' }) } + + subject { described_class.new(options) } + + describe '#extract' do + before do + allow(subject).to receive(:http_client).and_return(http_client) + allow(http_client).to receive(:get).and_return(response) + end + + it 'returns instance of ExtractedData' do + entity = create(:bulk_import_entity) + tracker = create(:bulk_import_tracker, entity: entity) + context = BulkImports::Pipeline::Context.new(tracker) + + extracted_data = subject.extract(context) + + expect(extracted_data).to be_instance_of(BulkImports::Pipeline::ExtractedData) + expect(extracted_data.data).to contain_exactly(response.parsed_response) + expect(extracted_data.next_page).to eq(response.headers['x-next-page']) + expect(extracted_data.has_next_page?).to eq(true) + end + end +end diff --git a/spec/lib/bulk_imports/common/transformers/user_reference_transformer_spec.rb b/spec/lib/bulk_imports/common/transformers/user_reference_transformer_spec.rb index ff11a10bfe9..ba74c173794 100644 --- a/spec/lib/bulk_imports/common/transformers/user_reference_transformer_spec.rb +++ b/spec/lib/bulk_imports/common/transformers/user_reference_transformer_spec.rb @@ -8,7 +8,8 @@ RSpec.describe BulkImports::Common::Transformers::UserReferenceTransformer do let_it_be(:group) { create(:group) } let_it_be(:bulk_import) { create(:bulk_import) } let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import, group: group) } - let_it_be(:context) { BulkImports::Pipeline::Context.new(entity) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } let(:hash) do { @@ -51,19 +52,26 @@ RSpec.describe BulkImports::Common::Transformers::UserReferenceTransformer do end context 'when custom reference is provided' do - it 'updates provided reference' do - hash = { - 'author' => { - 'public_email' => user.email + shared_examples 'updates provided reference' do |reference| + let(:hash) do + { + 'author' => { + 'public_email' => user.email + } } - } + end - transformer = described_class.new(reference: 'author') - result = transformer.transform(context, hash) + it 'updates provided reference' do + transformer = described_class.new(reference: reference) + result = transformer.transform(context, hash) - expect(result['author']).to be_nil - expect(result['author_id']).to eq(user.id) + expect(result['author']).to be_nil + expect(result['author_id']).to eq(user.id) + end end + + include_examples 'updates provided reference', 'author' + include_examples 'updates provided reference', :author end end end diff --git a/spec/lib/bulk_imports/groups/extractors/subgroups_extractor_spec.rb b/spec/lib/bulk_imports/groups/extractors/subgroups_extractor_spec.rb index 627247c04ab..ac8786440e9 100644 --- a/spec/lib/bulk_imports/groups/extractors/subgroups_extractor_spec.rb +++ b/spec/lib/bulk_imports/groups/extractors/subgroups_extractor_spec.rb @@ -8,8 +8,9 @@ RSpec.describe BulkImports::Groups::Extractors::SubgroupsExtractor do bulk_import = create(:bulk_import) create(:bulk_import_configuration, bulk_import: bulk_import) entity = create(:bulk_import_entity, bulk_import: bulk_import) + tracker = create(:bulk_import_tracker, entity: entity) response = [{ 'test' => 'group' }] - context = BulkImports::Pipeline::Context.new(entity) + context = BulkImports::Pipeline::Context.new(tracker) allow_next_instance_of(BulkImports::Clients::Http) do |client| allow(client).to receive(:each_page).and_return(response) diff --git a/spec/lib/bulk_imports/groups/graphql/get_group_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_group_query_spec.rb index ef46da7062b..b0f8f74783b 100644 --- a/spec/lib/bulk_imports/groups/graphql/get_group_query_spec.rb +++ b/spec/lib/bulk_imports/groups/graphql/get_group_query_spec.rb @@ -4,10 +4,10 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Graphql::GetGroupQuery do describe '#variables' do - let(:entity) { double(source_full_path: 'test', bulk_import: nil) } - let(:context) { BulkImports::Pipeline::Context.new(entity) } - it 'returns query variables based on entity information' do + entity = double(source_full_path: 'test', bulk_import: nil) + tracker = double(entity: entity) + context = BulkImports::Pipeline::Context.new(tracker) expected = { full_path: entity.source_full_path } expect(described_class.variables(context)).to eq(expected) diff --git a/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb index 85f82be7d18..61db644a372 100644 --- a/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb +++ b/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb @@ -4,8 +4,8 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Graphql::GetLabelsQuery do it 'has a valid query' do - entity = create(:bulk_import_entity) - context = BulkImports::Pipeline::Context.new(entity) + tracker = create(:bulk_import_tracker) + context = BulkImports::Pipeline::Context.new(tracker) query = GraphQL::Query.new( GitlabSchema, diff --git a/spec/lib/bulk_imports/groups/graphql/get_members_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_members_query_spec.rb index 5d05f5a2d30..d0c4bb817b2 100644 --- a/spec/lib/bulk_imports/groups/graphql/get_members_query_spec.rb +++ b/spec/lib/bulk_imports/groups/graphql/get_members_query_spec.rb @@ -4,8 +4,8 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Graphql::GetMembersQuery do it 'has a valid query' do - entity = create(:bulk_import_entity) - context = BulkImports::Pipeline::Context.new(entity) + tracker = create(:bulk_import_tracker) + context = BulkImports::Pipeline::Context.new(tracker) query = GraphQL::Query.new( GitlabSchema, diff --git a/spec/lib/bulk_imports/groups/graphql/get_milestones_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_milestones_query_spec.rb index a38505fbf85..7a0f964c5f3 100644 --- a/spec/lib/bulk_imports/groups/graphql/get_milestones_query_spec.rb +++ b/spec/lib/bulk_imports/groups/graphql/get_milestones_query_spec.rb @@ -4,8 +4,8 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Graphql::GetMilestonesQuery do it 'has a valid query' do - entity = create(:bulk_import_entity) - context = BulkImports::Pipeline::Context.new(entity) + tracker = create(:bulk_import_tracker) + context = BulkImports::Pipeline::Context.new(tracker) query = GraphQL::Query.new( GitlabSchema, diff --git a/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb b/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb index 183292722d2..533955b057c 100644 --- a/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb +++ b/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb @@ -4,12 +4,13 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Loaders::GroupLoader do describe '#load' do - let(:user) { create(:user) } - let(:data) { { foo: :bar } } + let_it_be(:user) { create(:user) } + let_it_be(:bulk_import) { create(:bulk_import, user: user) } + let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } let(:service_double) { instance_double(::Groups::CreateService) } - let(:bulk_import) { create(:bulk_import, user: user) } - let(:entity) { create(:bulk_import_entity, bulk_import: bulk_import) } - let(:context) { BulkImports::Pipeline::Context.new(entity) } + let(:data) { { foo: :bar } } subject { described_class.new } diff --git a/spec/lib/bulk_imports/groups/pipelines/badges_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/badges_pipeline_spec.rb new file mode 100644 index 00000000000..9fa35c4707d --- /dev/null +++ b/spec/lib/bulk_imports/groups/pipelines/badges_pipeline_spec.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Pipelines::BadgesPipeline do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + + let_it_be(:entity) do + create( + :bulk_import_entity, + source_full_path: 'source/full/path', + destination_name: 'My Destination Group', + destination_namespace: group.full_path, + group: group + ) + end + + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } + + subject { described_class.new(context) } + + describe '#run' do + it 'imports a group badge' do + first_page = extracted_data(has_next_page: true) + last_page = extracted_data(name: 'badge2') + + allow_next_instance_of(BulkImports::Common::Extractors::RestExtractor) do |extractor| + allow(extractor) + .to receive(:extract) + .and_return(first_page, last_page) + end + + expect { subject.run }.to change(Badge, :count).by(2) + + badge = group.badges.last + + expect(badge.name).to eq('badge2') + expect(badge.link_url).to eq(badge_data['link_url']) + expect(badge.image_url).to eq(badge_data['image_url']) + end + + describe '#load' do + it 'creates a badge' do + expect { subject.load(context, badge_data) }.to change(Badge, :count).by(1) + + badge = group.badges.first + + badge_data.each do |key, value| + expect(badge[key]).to eq(value) + end + end + + it 'does nothing when the data is blank' do + expect { subject.load(context, nil) }.not_to change(Badge, :count) + end + end + + describe '#transform' do + it 'return transformed badge hash' do + badge = subject.transform(context, badge_data) + + expect(badge[:name]).to eq('badge') + expect(badge[:link_url]).to eq(badge_data['link_url']) + expect(badge[:image_url]).to eq(badge_data['image_url']) + expect(badge.keys).to contain_exactly(:name, :link_url, :image_url) + end + + context 'when data is blank' do + it 'does nothing when the data is blank' do + expect(subject.transform(context, nil)).to be_nil + end + end + end + + describe 'pipeline parts' do + it { expect(described_class).to include_module(BulkImports::Pipeline) } + it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } + + it 'has extractors' do + expect(described_class.get_extractor) + .to eq( + klass: BulkImports::Common::Extractors::RestExtractor, + options: { + query: BulkImports::Groups::Rest::GetBadgesQuery + } + ) + end + + it 'has transformers' do + expect(described_class.transformers) + .to contain_exactly( + { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil } + ) + end + end + + def badge_data(name = 'badge') + { + 'name' => name, + 'link_url' => 'https://gitlab.example.com', + 'image_url' => 'https://gitlab.example.com/image.png' + } + end + + def extracted_data(name: 'badge', has_next_page: false) + page_info = { + 'has_next_page' => has_next_page, + 'next_page' => has_next_page ? '2' : nil + } + + BulkImports::Pipeline::ExtractedData.new(data: [badge_data(name)], page_info: page_info) + end + end +end diff --git a/spec/lib/bulk_imports/groups/pipelines/entity_finisher_spec.rb b/spec/lib/bulk_imports/groups/pipelines/entity_finisher_spec.rb new file mode 100644 index 00000000000..8276349c5f4 --- /dev/null +++ b/spec/lib/bulk_imports/groups/pipelines/entity_finisher_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Pipelines::EntityFinisher do + it 'updates the entity status to finished' do + entity = create(:bulk_import_entity, :started) + pipeline_tracker = create(:bulk_import_tracker, entity: entity) + context = BulkImports::Pipeline::Context.new(pipeline_tracker) + subject = described_class.new(context) + + expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect(logger) + .to receive(:info) + .with( + bulk_import_id: entity.bulk_import.id, + bulk_import_entity_id: entity.id, + bulk_import_entity_type: entity.source_type, + pipeline_class: described_class.name, + message: 'Entity finished' + ) + end + + expect { subject.run } + .to change(entity, :status_name).to(:finished) + end + + it 'does nothing when the entity is already finished' do + entity = create(:bulk_import_entity, :finished) + pipeline_tracker = create(:bulk_import_tracker, entity: entity) + context = BulkImports::Pipeline::Context.new(pipeline_tracker) + subject = described_class.new(context) + + expect { subject.run } + .not_to change(entity, :status_name) + end +end diff --git a/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb index 61950cdd9b0..39e782dc093 100644 --- a/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb @@ -4,10 +4,11 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do describe '#run' do - let(:user) { create(:user) } - let(:parent) { create(:group) } - let(:bulk_import) { create(:bulk_import, user: user) } - let(:entity) do + let_it_be(:user) { create(:user) } + let_it_be(:parent) { create(:group) } + let_it_be(:bulk_import) { create(:bulk_import, user: user) } + + let_it_be(:entity) do create( :bulk_import_entity, bulk_import: bulk_import, @@ -17,7 +18,8 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do ) end - let(:context) { BulkImports::Pipeline::Context.new(entity) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } let(:group_data) do { @@ -37,7 +39,7 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do before do allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| - allow(extractor).to receive(:extract).and_return([group_data]) + allow(extractor).to receive(:extract).and_return(BulkImports::Pipeline::ExtractedData.new(data: group_data)) end parent.add_owner(user) diff --git a/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb index 3327a30f1d5..8af646d1101 100644 --- a/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb @@ -3,11 +3,11 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do - let(:user) { create(:user) } - let(:group) { create(:group) } - let(:cursor) { 'cursor' } - let(:timestamp) { Time.new(2020, 01, 01).utc } - let(:entity) do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:timestamp) { Time.new(2020, 01, 01).utc } + + let_it_be(:entity) do create( :bulk_import_entity, source_full_path: 'source/full/path', @@ -17,33 +17,15 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do ) end - let(:context) { BulkImports::Pipeline::Context.new(entity) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } subject { described_class.new(context) } - def label_data(title) - { - 'title' => title, - 'description' => 'desc', - 'color' => '#428BCA', - 'created_at' => timestamp.to_s, - 'updated_at' => timestamp.to_s - } - end - - def extractor_data(title:, has_next_page:, cursor: nil) - page_info = { - 'end_cursor' => cursor, - 'has_next_page' => has_next_page - } - - BulkImports::Pipeline::ExtractedData.new(data: [label_data(title)], page_info: page_info) - end - describe '#run' do it 'imports a group labels' do - first_page = extractor_data(title: 'label1', has_next_page: true, cursor: cursor) - last_page = extractor_data(title: 'label2', has_next_page: false) + first_page = extracted_data(title: 'label1', has_next_page: true) + last_page = extracted_data(title: 'label2') allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| allow(extractor) @@ -63,38 +45,6 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do end end - describe '#after_run' do - context 'when extracted data has next page' do - it 'updates tracker information and runs pipeline again' do - data = extractor_data(title: 'label', has_next_page: true, cursor: cursor) - - expect(subject).to receive(:run) - - subject.after_run(data) - - tracker = entity.trackers.find_by(relation: :labels) - - expect(tracker.has_next_page).to eq(true) - expect(tracker.next_page).to eq(cursor) - end - end - - context 'when extracted data has no next page' do - it 'updates tracker information and does not run pipeline' do - data = extractor_data(title: 'label', has_next_page: false) - - expect(subject).not_to receive(:run) - - subject.after_run(data) - - tracker = entity.trackers.find_by(relation: :labels) - - expect(tracker.has_next_page).to eq(false) - expect(tracker.next_page).to be_nil - end - end - end - describe '#load' do it 'creates the label' do data = label_data('label') @@ -130,4 +80,23 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do ) end end + + def label_data(title) + { + 'title' => title, + 'description' => 'desc', + 'color' => '#428BCA', + 'created_at' => timestamp.to_s, + 'updated_at' => timestamp.to_s + } + end + + def extracted_data(title:, has_next_page: false) + page_info = { + 'has_next_page' => has_next_page, + 'next_page' => has_next_page ? 'cursor' : nil + } + + BulkImports::Pipeline::ExtractedData.new(data: [label_data(title)], page_info: page_info) + end end diff --git a/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb index 74d3e09d263..d8a667ec92a 100644 --- a/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb @@ -8,17 +8,17 @@ RSpec.describe BulkImports::Groups::Pipelines::MembersPipeline do let_it_be(:user) { create(:user) } let_it_be(:group) { create(:group) } - let_it_be(:cursor) { 'cursor' } let_it_be(:bulk_import) { create(:bulk_import, user: user) } let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import, group: group) } - let_it_be(:context) { BulkImports::Pipeline::Context.new(entity) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } subject { described_class.new(context) } describe '#run' do it 'maps existing users to the imported group' do - first_page = member_data(email: member_user1.email, has_next_page: true, cursor: cursor) - last_page = member_data(email: member_user2.email, has_next_page: false) + first_page = extracted_data(email: member_user1.email, has_next_page: true) + last_page = extracted_data(email: member_user2.email) allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| allow(extractor) @@ -88,7 +88,7 @@ RSpec.describe BulkImports::Groups::Pipelines::MembersPipeline do end end - def member_data(email:, has_next_page:, cursor: nil) + def extracted_data(email:, has_next_page: false) data = { 'created_at' => '2020-01-01T00:00:00Z', 'updated_at' => '2020-01-01T00:00:00Z', @@ -102,8 +102,8 @@ RSpec.describe BulkImports::Groups::Pipelines::MembersPipeline do } page_info = { - 'end_cursor' => cursor, - 'has_next_page' => has_next_page + 'has_next_page' => has_next_page, + 'next_page' => has_next_page ? 'cursor' : nil } BulkImports::Pipeline::ExtractedData.new(data: data, page_info: page_info) diff --git a/spec/lib/bulk_imports/groups/pipelines/milestones_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/milestones_pipeline_spec.rb index f0c34c65257..e5cf75c566b 100644 --- a/spec/lib/bulk_imports/groups/pipelines/milestones_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/milestones_pipeline_spec.rb @@ -5,11 +5,10 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Pipelines::MilestonesPipeline do let_it_be(:user) { create(:user) } let_it_be(:group) { create(:group) } - let_it_be(:cursor) { 'cursor' } let_it_be(:timestamp) { Time.new(2020, 01, 01).utc } let_it_be(:bulk_import) { create(:bulk_import, user: user) } - let(:entity) do + let_it_be(:entity) do create( :bulk_import_entity, bulk_import: bulk_import, @@ -20,39 +19,19 @@ RSpec.describe BulkImports::Groups::Pipelines::MilestonesPipeline do ) end - let(:context) { BulkImports::Pipeline::Context.new(entity) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } subject { described_class.new(context) } - def milestone_data(title) - { - 'title' => title, - 'description' => 'desc', - 'state' => 'closed', - 'start_date' => '2020-10-21', - 'due_date' => '2020-10-22', - 'created_at' => timestamp.to_s, - 'updated_at' => timestamp.to_s - } - end - - def extracted_data(title:, has_next_page:, cursor: nil) - page_info = { - 'end_cursor' => cursor, - 'has_next_page' => has_next_page - } - - BulkImports::Pipeline::ExtractedData.new(data: [milestone_data(title)], page_info: page_info) - end - before do group.add_owner(user) end describe '#run' do it 'imports group milestones' do - first_page = extracted_data(title: 'milestone1', has_next_page: true, cursor: cursor) - last_page = extracted_data(title: 'milestone2', has_next_page: false) + first_page = extracted_data(title: 'milestone1', iid: 1, has_next_page: true) + last_page = extracted_data(title: 'milestone2', iid: 2) allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| allow(extractor) @@ -75,38 +54,6 @@ RSpec.describe BulkImports::Groups::Pipelines::MilestonesPipeline do end end - describe '#after_run' do - context 'when extracted data has next page' do - it 'updates tracker information and runs pipeline again' do - data = extracted_data(title: 'milestone', has_next_page: true, cursor: cursor) - - expect(subject).to receive(:run) - - subject.after_run(data) - - tracker = entity.trackers.find_by(relation: :milestones) - - expect(tracker.has_next_page).to eq(true) - expect(tracker.next_page).to eq(cursor) - end - end - - context 'when extracted data has no next page' do - it 'updates tracker information and does not run pipeline' do - data = extracted_data(title: 'milestone', has_next_page: false) - - expect(subject).not_to receive(:run) - - subject.after_run(data) - - tracker = entity.trackers.find_by(relation: :milestones) - - expect(tracker.has_next_page).to eq(false) - expect(tracker.next_page).to be_nil - end - end - end - describe '#load' do it 'creates the milestone' do data = milestone_data('milestone') @@ -120,7 +67,7 @@ RSpec.describe BulkImports::Groups::Pipelines::MilestonesPipeline do end it 'raises NotAllowedError' do - data = extracted_data(title: 'milestone', has_next_page: false) + data = extracted_data(title: 'milestone') expect { subject.load(context, data) }.to raise_error(::BulkImports::Pipeline::NotAllowedError) end @@ -148,4 +95,29 @@ RSpec.describe BulkImports::Groups::Pipelines::MilestonesPipeline do ) end end + + def milestone_data(title, iid: 1) + { + 'title' => title, + 'description' => 'desc', + 'iid' => iid, + 'state' => 'closed', + 'start_date' => '2020-10-21', + 'due_date' => '2020-10-22', + 'created_at' => timestamp.to_s, + 'updated_at' => timestamp.to_s + } + end + + def extracted_data(title:, iid: 1, has_next_page: false) + page_info = { + 'has_next_page' => has_next_page, + 'next_page' => has_next_page ? 'cursor' : nil + } + + BulkImports::Pipeline::ExtractedData.new( + data: milestone_data(title, iid: iid), + page_info: page_info + ) + end end diff --git a/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb index 2a99646bb4a..e4a41428dd2 100644 --- a/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb @@ -6,31 +6,23 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do let_it_be(:user) { create(:user) } let_it_be(:group) { create(:group, path: 'group') } let_it_be(:parent) { create(:group, name: 'imported-group', path: 'imported-group') } - let(:context) { BulkImports::Pipeline::Context.new(parent_entity) } + let_it_be(:parent_entity) { create(:bulk_import_entity, destination_namespace: parent.full_path, group: parent) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: parent_entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } subject { described_class.new(context) } - describe '#run' do - let!(:parent_entity) do - create( - :bulk_import_entity, - destination_namespace: parent.full_path, - group: parent - ) - end - - let(:subgroup_data) do - [ - { - "name" => "subgroup", - "full_path" => "parent/subgroup" - } - ] - end + let(:extracted_data) do + BulkImports::Pipeline::ExtractedData.new(data: { + 'name' => 'subgroup', + 'full_path' => 'parent/subgroup' + }) + end + describe '#run' do before do allow_next_instance_of(BulkImports::Groups::Extractors::SubgroupsExtractor) do |extractor| - allow(extractor).to receive(:extract).and_return(subgroup_data) + allow(extractor).to receive(:extract).and_return(extracted_data) end parent.add_owner(user) diff --git a/spec/lib/bulk_imports/groups/rest/get_badges_query_spec.rb b/spec/lib/bulk_imports/groups/rest/get_badges_query_spec.rb new file mode 100644 index 00000000000..eef6848e118 --- /dev/null +++ b/spec/lib/bulk_imports/groups/rest/get_badges_query_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Rest::GetBadgesQuery do + describe '.to_h' do + it 'returns query resource and page info' do + entity = create(:bulk_import_entity) + tracker = create(:bulk_import_tracker, entity: entity) + context = BulkImports::Pipeline::Context.new(tracker) + encoded_full_path = ERB::Util.url_encode(entity.source_full_path) + expected = { + resource: ['groups', encoded_full_path, 'badges'].join('/'), + query: { + page: context.tracker.next_page + } + } + + expect(described_class.to_h(context)).to eq(expected) + end + end +end diff --git a/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb b/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb index b3fe8a2ba25..75d8c15088a 100644 --- a/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb +++ b/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb @@ -4,11 +4,12 @@ require 'spec_helper' RSpec.describe BulkImports::Groups::Transformers::GroupAttributesTransformer do describe '#transform' do - let(:user) { create(:user) } - let(:parent) { create(:group) } - let(:group) { create(:group, name: 'My Source Group', parent: parent) } - let(:bulk_import) { create(:bulk_import, user: user) } - let(:entity) do + let_it_be(:user) { create(:user) } + let_it_be(:parent) { create(:group) } + let_it_be(:group) { create(:group, name: 'My Source Group', parent: parent) } + let_it_be(:bulk_import) { create(:bulk_import, user: user) } + + let_it_be(:entity) do create( :bulk_import_entity, bulk_import: bulk_import, @@ -18,7 +19,8 @@ RSpec.describe BulkImports::Groups::Transformers::GroupAttributesTransformer do ) end - let(:context) { BulkImports::Pipeline::Context.new(entity) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } let(:data) do { @@ -82,14 +84,7 @@ RSpec.describe BulkImports::Groups::Transformers::GroupAttributesTransformer do context 'when destination namespace is empty' do it 'does not set parent id' do - entity = create( - :bulk_import_entity, - bulk_import: bulk_import, - source_full_path: 'source/full/path', - destination_name: group.name, - destination_namespace: '' - ) - context = BulkImports::Pipeline::Context.new(entity) + entity.update!(destination_namespace: '') transformed_data = subject.transform(context, data) diff --git a/spec/lib/bulk_imports/groups/transformers/member_attributes_transformer_spec.rb b/spec/lib/bulk_imports/groups/transformers/member_attributes_transformer_spec.rb index f66c67fc6a2..f3905a4b6e4 100644 --- a/spec/lib/bulk_imports/groups/transformers/member_attributes_transformer_spec.rb +++ b/spec/lib/bulk_imports/groups/transformers/member_attributes_transformer_spec.rb @@ -8,7 +8,8 @@ RSpec.describe BulkImports::Groups::Transformers::MemberAttributesTransformer do let_it_be(:group) { create(:group) } let_it_be(:bulk_import) { create(:bulk_import, user: user) } let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import, group: group) } - let_it_be(:context) { BulkImports::Pipeline::Context.new(entity) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) } it 'returns nil when receives no data' do expect(subject.transform(context, nil)).to eq(nil) diff --git a/spec/lib/bulk_imports/importers/group_importer_spec.rb b/spec/lib/bulk_imports/importers/group_importer_spec.rb deleted file mode 100644 index 5d501b49e41..00000000000 --- a/spec/lib/bulk_imports/importers/group_importer_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe BulkImports::Importers::GroupImporter do - let(:user) { create(:user) } - let(:group) { create(:group) } - let(:bulk_import) { create(:bulk_import) } - let(:bulk_import_entity) { create(:bulk_import_entity, :started, bulk_import: bulk_import, group: group) } - let(:bulk_import_configuration) { create(:bulk_import_configuration, bulk_import: bulk_import) } - let(:context) { BulkImports::Pipeline::Context.new(bulk_import_entity) } - - before do - allow(BulkImports::Pipeline::Context).to receive(:new).and_return(context) - end - - subject { described_class.new(bulk_import_entity) } - - describe '#execute' do - it 'starts the entity and run its pipelines' do - expect_to_run_pipeline BulkImports::Groups::Pipelines::GroupPipeline, context: context - expect_to_run_pipeline BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, context: context - expect_to_run_pipeline BulkImports::Groups::Pipelines::MembersPipeline, context: context - expect_to_run_pipeline BulkImports::Groups::Pipelines::LabelsPipeline, context: context - expect_to_run_pipeline BulkImports::Groups::Pipelines::MilestonesPipeline, context: context - - if Gitlab.ee? - expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::EpicsPipeline'.constantize, context: context) - expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::EpicAwardEmojiPipeline'.constantize, context: context) - expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::EpicEventsPipeline'.constantize, context: context) - expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::IterationsPipeline'.constantize, context: context) - end - - subject.execute - - expect(bulk_import_entity.reload).to be_finished - end - - context 'when failed' do - let(:bulk_import_entity) { create(:bulk_import_entity, :failed, bulk_import: bulk_import, group: group) } - - it 'does not transition entity to finished state' do - allow(bulk_import_entity).to receive(:start!) - - subject.execute - - expect(bulk_import_entity.reload).to be_failed - end - end - end - - def expect_to_run_pipeline(klass, context:) - expect_next_instance_of(klass, context) do |pipeline| - expect(pipeline).to receive(:run) - end - end -end diff --git a/spec/lib/bulk_imports/pipeline/context_spec.rb b/spec/lib/bulk_imports/pipeline/context_spec.rb index c8c3fe3a861..5b7711ad5d7 100644 --- a/spec/lib/bulk_imports/pipeline/context_spec.rb +++ b/spec/lib/bulk_imports/pipeline/context_spec.rb @@ -3,29 +3,52 @@ require 'spec_helper' RSpec.describe BulkImports::Pipeline::Context do - let(:group) { instance_double(Group) } - let(:user) { instance_double(User) } - let(:bulk_import) { instance_double(BulkImport, user: user, configuration: :config) } - - let(:entity) do - instance_double( - BulkImports::Entity, - bulk_import: bulk_import, - group: group + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:bulk_import) { create(:bulk_import, user: user) } + + let_it_be(:entity) do + create( + :bulk_import_entity, + source_full_path: 'source/full/path', + destination_name: 'My Destination Group', + destination_namespace: group.full_path, + group: group, + bulk_import: bulk_import + ) + end + + let_it_be(:tracker) do + create( + :bulk_import_tracker, + entity: entity, + pipeline_name: described_class.name ) end - subject { described_class.new(entity) } + subject { described_class.new(tracker, extra: :data) } + + describe '#entity' do + it { expect(subject.entity).to eq(entity) } + end describe '#group' do it { expect(subject.group).to eq(group) } end + describe '#bulk_import' do + it { expect(subject.bulk_import).to eq(bulk_import) } + end + describe '#current_user' do it { expect(subject.current_user).to eq(user) } end - describe '#current_user' do + describe '#configuration' do it { expect(subject.configuration).to eq(bulk_import.configuration) } end + + describe '#extra' do + it { expect(subject.extra).to eq(extra: :data) } + end end diff --git a/spec/lib/bulk_imports/pipeline/extracted_data_spec.rb b/spec/lib/bulk_imports/pipeline/extracted_data_spec.rb index 25c5178227a..9c79b3f4c9e 100644 --- a/spec/lib/bulk_imports/pipeline/extracted_data_spec.rb +++ b/spec/lib/bulk_imports/pipeline/extracted_data_spec.rb @@ -9,7 +9,7 @@ RSpec.describe BulkImports::Pipeline::ExtractedData do let(:page_info) do { 'has_next_page' => has_next_page, - 'end_cursor' => cursor + 'next_page' => cursor } end diff --git a/spec/lib/bulk_imports/pipeline/runner_spec.rb b/spec/lib/bulk_imports/pipeline/runner_spec.rb index 59f01c9caaa..7235b7c95cd 100644 --- a/spec/lib/bulk_imports/pipeline/runner_spec.rb +++ b/spec/lib/bulk_imports/pipeline/runner_spec.rb @@ -38,23 +38,20 @@ RSpec.describe BulkImports::Pipeline::Runner do extractor BulkImports::Extractor transformer BulkImports::Transformer loader BulkImports::Loader - - def after_run(_); end end stub_const('BulkImports::MyPipeline', pipeline) end - let_it_be_with_refind(:entity) { create(:bulk_import_entity) } - let(:context) { BulkImports::Pipeline::Context.new(entity, extra: :data) } + let_it_be_with_reload(:entity) { create(:bulk_import_entity) } + let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) } + let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker, extra: :data) } subject { BulkImports::MyPipeline.new(context) } describe 'pipeline runner' do context 'when entity is not marked as failed' do it 'runs pipeline extractor, transformer, loader' do - extracted_data = BulkImports::Pipeline::ExtractedData.new(data: { foo: :bar }) - expect_next_instance_of(BulkImports::Extractor) do |extractor| expect(extractor) .to receive(:extract) @@ -132,6 +129,22 @@ RSpec.describe BulkImports::Pipeline::Runner do subject.run end + context 'when extracted data has multiple pages' do + it 'updates tracker information and runs pipeline again' do + first_page = extracted_data(has_next_page: true) + last_page = extracted_data + + expect_next_instance_of(BulkImports::Extractor) do |extractor| + expect(extractor) + .to receive(:extract) + .with(context) + .and_return(first_page, last_page) + end + + subject.run + end + end + context 'when exception is raised' do before do allow_next_instance_of(BulkImports::Extractor) do |extractor| @@ -170,12 +183,7 @@ RSpec.describe BulkImports::Pipeline::Runner do BulkImports::MyPipeline.abort_on_failure! end - it 'marks entity as failed' do - expect { subject.run } - .to change(entity, :status_name).to(:failed) - end - - it 'logs warn message' do + it 'logs a warn message and marks entity as failed' do expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect(logger).to receive(:warn) .with( @@ -188,6 +196,9 @@ RSpec.describe BulkImports::Pipeline::Runner do end subject.run + + expect(entity.status_name).to eq(:failed) + expect(tracker.status_name).to eq(:failed) end end @@ -206,11 +217,11 @@ RSpec.describe BulkImports::Pipeline::Runner do entity.fail_op! expect_next_instance_of(Gitlab::Import::Logger) do |logger| - expect(logger).to receive(:info) + expect(logger).to receive(:warn) .with( log_params( context, - message: 'Skipping due to failed pipeline status', + message: 'Skipping pipeline due to failed entity', pipeline_class: 'BulkImports::MyPipeline' ) ) @@ -219,14 +230,24 @@ RSpec.describe BulkImports::Pipeline::Runner do subject.run end end - end - def log_params(context, extra = {}) - { - bulk_import_id: context.bulk_import.id, - bulk_import_entity_id: context.entity.id, - bulk_import_entity_type: context.entity.source_type, - context_extra: context.extra - }.merge(extra) + def log_params(context, extra = {}) + { + bulk_import_id: context.bulk_import.id, + bulk_import_entity_id: context.entity.id, + bulk_import_entity_type: context.entity.source_type, + context_extra: context.extra + }.merge(extra) + end + + def extracted_data(has_next_page: false) + BulkImports::Pipeline::ExtractedData.new( + data: { foo: :bar }, + page_info: { + 'has_next_page' => has_next_page, + 'next_page' => has_next_page ? 'cursor' : nil + } + ) + end end end diff --git a/spec/lib/bulk_imports/pipeline_spec.rb b/spec/lib/bulk_imports/pipeline_spec.rb index c882e3d26ea..dda2e41f06c 100644 --- a/spec/lib/bulk_imports/pipeline_spec.rb +++ b/spec/lib/bulk_imports/pipeline_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe BulkImports::Pipeline do + let(:context) { instance_double(BulkImports::Pipeline::Context, tracker: nil) } + before do stub_const('BulkImports::Extractor', Class.new) stub_const('BulkImports::Transformer', Class.new) @@ -44,7 +46,7 @@ RSpec.describe BulkImports::Pipeline do end it 'returns itself when retrieving extractor & loader' do - pipeline = BulkImports::AnotherPipeline.new(nil) + pipeline = BulkImports::AnotherPipeline.new(context) expect(pipeline.send(:extractor)).to eq(pipeline) expect(pipeline.send(:loader)).to eq(pipeline) @@ -83,7 +85,7 @@ RSpec.describe BulkImports::Pipeline do expect(BulkImports::Transformer).to receive(:new).with(foo: :bar) expect(BulkImports::Loader).to receive(:new).with(foo: :bar) - pipeline = BulkImports::MyPipeline.new(nil) + pipeline = BulkImports::MyPipeline.new(context) pipeline.send(:extractor) pipeline.send(:transformers) @@ -109,7 +111,7 @@ RSpec.describe BulkImports::Pipeline do expect(BulkImports::Transformer).to receive(:new).with(no_args) expect(BulkImports::Loader).to receive(:new).with(no_args) - pipeline = BulkImports::NoOptionsPipeline.new(nil) + pipeline = BulkImports::NoOptionsPipeline.new(context) pipeline.send(:extractor) pipeline.send(:transformers) @@ -135,7 +137,7 @@ RSpec.describe BulkImports::Pipeline do transformer = double allow(BulkImports::Transformer).to receive(:new).and_return(transformer) - pipeline = BulkImports::TransformersPipeline.new(nil) + pipeline = BulkImports::TransformersPipeline.new(context) expect(pipeline.send(:transformers)).to eq([pipeline, transformer]) end |