From d298fad0c0564454271cba11e6f20c19681534ac Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 5 Feb 2021 16:20:45 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-9-stable-ee --- .../common/extractors/graphql_extractor_spec.rb | 66 ++++-------- .../common/loaders/entity_loader_spec.rb | 2 +- .../common/transformers/hash_key_digger_spec.rb | 28 ----- .../underscorify_keys_transformer_spec.rb | 27 ----- .../groups/extractors/subgroups_extractor_spec.rb | 24 +++++ .../groups/graphql/get_group_query_spec.rb | 31 ++++++ .../groups/graphql/get_labels_query_spec.rb | 31 ++++++ .../groups/loaders/group_loader_spec.rb | 17 ++- .../groups/loaders/labels_loader_spec.rb | 30 ++++++ .../groups/pipelines/group_pipeline_spec.rb | 49 ++++----- .../groups/pipelines/labels_pipeline_spec.rb | 116 +++++++++++++++++++++ .../pipelines/subgroup_entities_pipeline_spec.rb | 8 +- .../group_attributes_transformer_spec.rb | 24 ++--- .../bulk_imports/importers/group_importer_spec.rb | 13 +-- spec/lib/bulk_imports/pipeline/context_spec.rb | 38 ++++--- .../bulk_imports/pipeline/extracted_data_spec.rb | 53 ++++++++++ spec/lib/bulk_imports/pipeline/runner_spec.rb | 76 +++++++++----- 17 files changed, 420 insertions(+), 213 deletions(-) delete mode 100644 spec/lib/bulk_imports/common/transformers/hash_key_digger_spec.rb delete mode 100644 spec/lib/bulk_imports/common/transformers/underscorify_keys_transformer_spec.rb create mode 100644 spec/lib/bulk_imports/groups/extractors/subgroups_extractor_spec.rb create mode 100644 spec/lib/bulk_imports/groups/graphql/get_group_query_spec.rb create mode 100644 spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb create mode 100644 spec/lib/bulk_imports/groups/loaders/labels_loader_spec.rb create mode 100644 spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb create mode 100644 spec/lib/bulk_imports/pipeline/extracted_data_spec.rb (limited to 'spec/lib/bulk_imports') diff --git a/spec/lib/bulk_imports/common/extractors/graphql_extractor_spec.rb b/spec/lib/bulk_imports/common/extractors/graphql_extractor_spec.rb index 2abd3df20fd..80607485b6e 100644 --- a/spec/lib/bulk_imports/common/extractors/graphql_extractor_spec.rb +++ b/spec/lib/bulk_imports/common/extractors/graphql_extractor_spec.rb @@ -5,8 +5,18 @@ require 'spec_helper' RSpec.describe BulkImports::Common::Extractors::GraphqlExtractor do let(:graphql_client) { instance_double(BulkImports::Clients::Graphql) } let(:import_entity) { create(:bulk_import_entity) } - let(:response) { double(original_hash: { foo: :bar }) } - let(:query) { { query: double(to_s: 'test', variables: {}) } } + let(:response) { double(original_hash: { 'data' => { 'foo' => 'bar' }, 'page_info' => {} }) } + let(:options) do + { + query: double( + to_s: 'test', + variables: {}, + data_path: %w[data foo], + page_info_path: %w[data page_info] + ) + } + end + let(:context) do instance_double( BulkImports::Pipeline::Context, @@ -14,58 +24,20 @@ RSpec.describe BulkImports::Common::Extractors::GraphqlExtractor do ) end - subject { described_class.new(query) } - - before do - allow(subject).to receive(:graphql_client).and_return(graphql_client) - allow(graphql_client).to receive(:parse) - end + subject { described_class.new(options) } describe '#extract' do before do - allow(subject).to receive(:query_variables).and_return({}) - allow(graphql_client).to receive(:execute).and_return(response) - end - - it 'returns original hash' do - expect(subject.extract(context)).to eq({ foo: :bar }) - end - end - - describe 'query variables' do - before do + allow(subject).to receive(:graphql_client).and_return(graphql_client) + allow(graphql_client).to receive(:parse) allow(graphql_client).to receive(:execute).and_return(response) end - context 'when variables are present' do - let(:variables) { { foo: :bar } } - let(:query) { { query: double(to_s: 'test', variables: variables) } } - - it 'builds graphql query variables for import entity' do - expect(graphql_client).to receive(:execute).with(anything, variables) - - subject.extract(context).first - end - end - - context 'when no variables are present' do - let(:query) { { query: double(to_s: 'test', variables: nil) } } - - it 'returns empty hash' do - expect(graphql_client).to receive(:execute).with(anything, nil) - - subject.extract(context).first - end - end - - context 'when variables are empty hash' do - let(:query) { { query: double(to_s: 'test', variables: {}) } } - - it 'makes graphql request with empty hash' do - expect(graphql_client).to receive(:execute).with(anything, {}) + it 'returns ExtractedData' do + extracted_data = subject.extract(context) - subject.extract(context).first - end + expect(extracted_data).to be_instance_of(BulkImports::Pipeline::ExtractedData) + expect(extracted_data.data).to contain_exactly('bar') end end end diff --git a/spec/lib/bulk_imports/common/loaders/entity_loader_spec.rb b/spec/lib/bulk_imports/common/loaders/entity_loader_spec.rb index 4de7d95172f..57ffdfa9aee 100644 --- a/spec/lib/bulk_imports/common/loaders/entity_loader_spec.rb +++ b/spec/lib/bulk_imports/common/loaders/entity_loader_spec.rb @@ -7,7 +7,7 @@ RSpec.describe BulkImports::Common::Loaders::EntityLoader do it "creates entities for the given data" do group = create(:group, path: "imported-group") parent_entity = create(:bulk_import_entity, group: group, bulk_import: create(:bulk_import)) - context = instance_double(BulkImports::Pipeline::Context, entity: parent_entity) + context = BulkImports::Pipeline::Context.new(parent_entity) data = { source_type: :group_entity, diff --git a/spec/lib/bulk_imports/common/transformers/hash_key_digger_spec.rb b/spec/lib/bulk_imports/common/transformers/hash_key_digger_spec.rb deleted file mode 100644 index 2b33701653e..00000000000 --- a/spec/lib/bulk_imports/common/transformers/hash_key_digger_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe BulkImports::Common::Transformers::HashKeyDigger do - describe '#transform' do - it 'when the key_path is an array' do - data = { foo: { bar: :value } } - key_path = %i[foo bar] - transformed = described_class.new(key_path: key_path).transform(nil, data) - - expect(transformed).to eq(:value) - end - - it 'when the key_path is not an array' do - data = { foo: { bar: :value } } - key_path = :foo - transformed = described_class.new(key_path: key_path).transform(nil, data) - - expect(transformed).to eq({ bar: :value }) - end - - it "when the data is not a hash" do - expect { described_class.new(key_path: nil).transform(nil, nil) } - .to raise_error(ArgumentError, "Given data must be a Hash") - end - end -end diff --git a/spec/lib/bulk_imports/common/transformers/underscorify_keys_transformer_spec.rb b/spec/lib/bulk_imports/common/transformers/underscorify_keys_transformer_spec.rb deleted file mode 100644 index cdffa750694..00000000000 --- a/spec/lib/bulk_imports/common/transformers/underscorify_keys_transformer_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe BulkImports::Common::Transformers::UnderscorifyKeysTransformer do - describe '#transform' do - it 'deep underscorifies hash keys' do - data = { - 'fullPath' => 'Foo', - 'snakeKeys' => { - 'snakeCaseKey' => 'Bar', - 'moreKeys' => { - 'anotherSnakeCaseKey' => 'Test' - } - } - } - - transformed_data = described_class.new.transform(nil, data) - - expect(transformed_data).to have_key('full_path') - expect(transformed_data).to have_key('snake_keys') - expect(transformed_data['snake_keys']).to have_key('snake_case_key') - expect(transformed_data['snake_keys']).to have_key('more_keys') - expect(transformed_data.dig('snake_keys', 'more_keys')).to have_key('another_snake_case_key') - end - end -end diff --git a/spec/lib/bulk_imports/groups/extractors/subgroups_extractor_spec.rb b/spec/lib/bulk_imports/groups/extractors/subgroups_extractor_spec.rb new file mode 100644 index 00000000000..627247c04ab --- /dev/null +++ b/spec/lib/bulk_imports/groups/extractors/subgroups_extractor_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Extractors::SubgroupsExtractor do + describe '#extract' do + it 'returns ExtractedData response' do + bulk_import = create(:bulk_import) + create(:bulk_import_configuration, bulk_import: bulk_import) + entity = create(:bulk_import_entity, bulk_import: bulk_import) + response = [{ 'test' => 'group' }] + context = BulkImports::Pipeline::Context.new(entity) + + allow_next_instance_of(BulkImports::Clients::Http) do |client| + allow(client).to receive(:each_page).and_return(response) + end + + extracted_data = subject.extract(context) + + expect(extracted_data).to be_instance_of(BulkImports::Pipeline::ExtractedData) + expect(extracted_data.data).to eq(response) + end + end +end diff --git a/spec/lib/bulk_imports/groups/graphql/get_group_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_group_query_spec.rb new file mode 100644 index 00000000000..78f99c19346 --- /dev/null +++ b/spec/lib/bulk_imports/groups/graphql/get_group_query_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Graphql::GetGroupQuery do + describe '#variables' do + let(:entity) { double(source_full_path: 'test') } + + it 'returns query variables based on entity information' do + expected = { full_path: entity.source_full_path } + + expect(described_class.variables(entity)).to eq(expected) + end + end + + describe '#data_path' do + it 'returns data path' do + expected = %w[data group] + + expect(described_class.data_path).to eq(expected) + end + end + + describe '#page_info_path' do + it 'returns pagination information path' do + expected = %w[data group page_info] + + expect(described_class.page_info_path).to eq(expected) + end + end +end diff --git a/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb b/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb new file mode 100644 index 00000000000..2d8f4fb399a --- /dev/null +++ b/spec/lib/bulk_imports/groups/graphql/get_labels_query_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Graphql::GetLabelsQuery do + describe '#variables' do + let(:entity) { double(source_full_path: 'test', next_page_for: 'next_page') } + + it 'returns query variables based on entity information' do + expected = { full_path: entity.source_full_path, cursor: entity.next_page_for } + + expect(described_class.variables(entity)).to eq(expected) + end + end + + describe '#data_path' do + it 'returns data path' do + expected = %w[data group labels nodes] + + expect(described_class.data_path).to eq(expected) + end + end + + describe '#page_info_path' do + it 'returns pagination information path' do + expected = %w[data group labels page_info] + + expect(described_class.page_info_path).to eq(expected) + end + end +end diff --git a/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb b/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb index b14dfc615a9..183292722d2 100644 --- a/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb +++ b/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb @@ -7,21 +7,20 @@ RSpec.describe BulkImports::Groups::Loaders::GroupLoader do let(:user) { create(:user) } let(:data) { { foo: :bar } } let(:service_double) { instance_double(::Groups::CreateService) } - let(:entity) { create(:bulk_import_entity) } - let(:context) do - instance_double( - BulkImports::Pipeline::Context, - entity: entity, - current_user: user - ) - end + let(:bulk_import) { create(:bulk_import, user: user) } + let(:entity) { create(:bulk_import_entity, bulk_import: bulk_import) } + let(:context) { BulkImports::Pipeline::Context.new(entity) } subject { described_class.new } context 'when user can create group' do shared_examples 'calls Group Create Service to create a new group' do it 'calls Group Create Service to create a new group' do - expect(::Groups::CreateService).to receive(:new).with(context.current_user, data).and_return(service_double) + expect(::Groups::CreateService) + .to receive(:new) + .with(context.current_user, data) + .and_return(service_double) + expect(service_double).to receive(:execute) expect(entity).to receive(:update!) diff --git a/spec/lib/bulk_imports/groups/loaders/labels_loader_spec.rb b/spec/lib/bulk_imports/groups/loaders/labels_loader_spec.rb new file mode 100644 index 00000000000..ac2f9c8cb1d --- /dev/null +++ b/spec/lib/bulk_imports/groups/loaders/labels_loader_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Loaders::LabelsLoader do + describe '#load' do + let(:user) { create(:user) } + let(:group) { create(:group) } + let(:entity) { create(:bulk_import_entity, group: group) } + let(:context) { BulkImports::Pipeline::Context.new(entity) } + + let(:data) do + { + 'title' => 'label', + 'description' => 'description', + 'color' => '#FFFFFF' + } + end + + it 'creates the label' do + expect { subject.load(context, data) }.to change(Label, :count).by(1) + + label = group.labels.first + + expect(label.title).to eq(data['title']) + expect(label.description).to eq(data['description']) + expect(label.color).to eq(data['color']) + end + end +end diff --git a/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb index 1a91f3d7a78..1a6a955544a 100644 --- a/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/group_pipeline_spec.rb @@ -6,42 +6,33 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do describe '#run' do let(:user) { create(:user) } let(:parent) { create(:group) } + let(:bulk_import) { create(:bulk_import, user: user) } let(:entity) do create( :bulk_import_entity, + bulk_import: bulk_import, source_full_path: 'source/full/path', destination_name: 'My Destination Group', destination_namespace: parent.full_path ) end - let(:context) do - BulkImports::Pipeline::Context.new( - current_user: user, - entity: entity - ) - end + let(:context) { BulkImports::Pipeline::Context.new(entity) } let(:group_data) do { - 'data' => { - 'group' => { - 'name' => 'source_name', - 'fullPath' => 'source/full/path', - 'visibility' => 'private', - 'projectCreationLevel' => 'developer', - 'subgroupCreationLevel' => 'maintainer', - 'description' => 'Group Description', - 'emailsDisabled' => true, - 'lfsEnabled' => false, - 'mentionsDisabled' => true - } - } + 'name' => 'source_name', + 'full_path' => 'source/full/path', + 'visibility' => 'private', + 'project_creation_level' => 'developer', + 'subgroup_creation_level' => 'maintainer', + 'description' => 'Group Description', + 'emails_disabled' => true, + 'lfs_enabled' => false, + 'mentions_disabled' => true } end - subject { described_class.new } - before do allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| allow(extractor).to receive(:extract).and_return([group_data]) @@ -60,13 +51,13 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do expect(imported_group).not_to be_nil expect(imported_group.parent).to eq(parent) expect(imported_group.path).to eq(group_path) - expect(imported_group.description).to eq(group_data.dig('data', 'group', 'description')) - expect(imported_group.visibility).to eq(group_data.dig('data', 'group', 'visibility')) - expect(imported_group.project_creation_level).to eq(Gitlab::Access.project_creation_string_options[group_data.dig('data', 'group', 'projectCreationLevel')]) - expect(imported_group.subgroup_creation_level).to eq(Gitlab::Access.subgroup_creation_string_options[group_data.dig('data', 'group', 'subgroupCreationLevel')]) - expect(imported_group.lfs_enabled?).to eq(group_data.dig('data', 'group', 'lfsEnabled')) - expect(imported_group.emails_disabled?).to eq(group_data.dig('data', 'group', 'emailsDisabled')) - expect(imported_group.mentions_disabled?).to eq(group_data.dig('data', 'group', 'mentionsDisabled')) + expect(imported_group.description).to eq(group_data['description']) + expect(imported_group.visibility).to eq(group_data['visibility']) + expect(imported_group.project_creation_level).to eq(Gitlab::Access.project_creation_string_options[group_data['project_creation_level']]) + expect(imported_group.subgroup_creation_level).to eq(Gitlab::Access.subgroup_creation_string_options[group_data['subgroup_creation_level']]) + expect(imported_group.lfs_enabled?).to eq(group_data['lfs_enabled']) + expect(imported_group.emails_disabled?).to eq(group_data['emails_disabled']) + expect(imported_group.mentions_disabled?).to eq(group_data['mentions_disabled']) end end @@ -87,8 +78,6 @@ RSpec.describe BulkImports::Groups::Pipelines::GroupPipeline do it 'has transformers' do expect(described_class.transformers) .to contain_exactly( - { klass: BulkImports::Common::Transformers::HashKeyDigger, options: { key_path: %w[data group] } }, - { klass: BulkImports::Common::Transformers::UnderscorifyKeysTransformer, options: nil }, { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil }, { klass: BulkImports::Groups::Transformers::GroupAttributesTransformer, options: nil } ) diff --git a/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb new file mode 100644 index 00000000000..7c96967971f --- /dev/null +++ b/spec/lib/bulk_imports/groups/pipelines/labels_pipeline_spec.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do + let(:user) { create(:user) } + let(:group) { create(:group) } + let(:cursor) { 'cursor' } + let(:entity) do + create( + :bulk_import_entity, + source_full_path: 'source/full/path', + destination_name: 'My Destination Group', + destination_namespace: group.full_path, + group: group + ) + end + + let(:context) { BulkImports::Pipeline::Context.new(entity) } + + def extractor_data(title:, has_next_page:, cursor: nil) + data = [ + { + 'title' => title, + 'description' => 'desc', + 'color' => '#428BCA' + } + ] + + page_info = { + 'end_cursor' => cursor, + 'has_next_page' => has_next_page + } + + BulkImports::Pipeline::ExtractedData.new(data: data, page_info: page_info) + end + + describe '#run' do + it 'imports a group labels' do + first_page = extractor_data(title: 'label1', has_next_page: true, cursor: cursor) + last_page = extractor_data(title: 'label2', has_next_page: false) + + allow_next_instance_of(BulkImports::Common::Extractors::GraphqlExtractor) do |extractor| + allow(extractor) + .to receive(:extract) + .and_return(first_page, last_page) + end + + expect { subject.run(context) }.to change(Label, :count).by(2) + + label = group.labels.order(:created_at).last + + expect(label.title).to eq('label2') + expect(label.description).to eq('desc') + expect(label.color).to eq('#428BCA') + end + end + + describe '#after_run' do + context 'when extracted data has next page' do + it 'updates tracker information and runs pipeline again' do + data = extractor_data(title: 'label', has_next_page: true, cursor: cursor) + + expect(subject).to receive(:run).with(context) + + subject.after_run(context, data) + + tracker = entity.trackers.find_by(relation: :labels) + + expect(tracker.has_next_page).to eq(true) + expect(tracker.next_page).to eq(cursor) + end + end + + context 'when extracted data has no next page' do + it 'updates tracker information and does not run pipeline' do + data = extractor_data(title: 'label', has_next_page: false) + + expect(subject).not_to receive(:run).with(context) + + subject.after_run(context, data) + + tracker = entity.trackers.find_by(relation: :labels) + + expect(tracker.has_next_page).to eq(false) + expect(tracker.next_page).to be_nil + end + end + end + + describe 'pipeline parts' do + it { expect(described_class).to include_module(BulkImports::Pipeline) } + it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) } + + it 'has extractors' do + expect(described_class.get_extractor) + .to eq( + klass: BulkImports::Common::Extractors::GraphqlExtractor, + options: { + query: BulkImports::Groups::Graphql::GetLabelsQuery + } + ) + end + + it 'has transformers' do + expect(described_class.transformers) + .to contain_exactly( + { klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil } + ) + end + + it 'has loaders' do + expect(described_class.get_loader).to eq(klass: BulkImports::Groups::Loaders::LabelsLoader, options: nil) + end + end +end diff --git a/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb index e5a8ed7f47d..71d1ffdeba3 100644 --- a/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb +++ b/spec/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline_spec.rb @@ -14,13 +14,7 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do ) end - let(:context) do - instance_double( - BulkImports::Pipeline::Context, - current_user: user, - entity: parent_entity - ) - end + let(:context) { BulkImports::Pipeline::Context.new(parent_entity) } let(:subgroup_data) do [ diff --git a/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb b/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb index 28a7859915d..5a7a51675d6 100644 --- a/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb +++ b/spec/lib/bulk_imports/groups/transformers/group_attributes_transformer_spec.rb @@ -7,22 +7,18 @@ RSpec.describe BulkImports::Groups::Transformers::GroupAttributesTransformer do let(:user) { create(:user) } let(:parent) { create(:group) } let(:group) { create(:group, name: 'My Source Group', parent: parent) } + let(:bulk_import) { create(:bulk_import, user: user) } let(:entity) do - instance_double( - BulkImports::Entity, + create( + :bulk_import_entity, + bulk_import: bulk_import, source_full_path: 'source/full/path', destination_name: group.name, destination_namespace: parent.full_path ) end - let(:context) do - instance_double( - BulkImports::Pipeline::Context, - current_user: user, - entity: entity - ) - end + let(:context) { BulkImports::Pipeline::Context.new(entity) } let(:data) do { @@ -85,16 +81,16 @@ RSpec.describe BulkImports::Groups::Transformers::GroupAttributesTransformer do end context 'when destination namespace is user namespace' do - let(:entity) do - instance_double( - BulkImports::Entity, + it 'does not set parent id' do + entity = create( + :bulk_import_entity, + bulk_import: bulk_import, source_full_path: 'source/full/path', destination_name: group.name, destination_namespace: user.namespace.full_path ) - end + context = BulkImports::Pipeline::Context.new(entity) - it 'does not set parent id' do transformed_data = subject.transform(context, data) expect(transformed_data).not_to have_key('parent_id') diff --git a/spec/lib/bulk_imports/importers/group_importer_spec.rb b/spec/lib/bulk_imports/importers/group_importer_spec.rb index 87baf1b8026..0884a51ce7d 100644 --- a/spec/lib/bulk_imports/importers/group_importer_spec.rb +++ b/spec/lib/bulk_imports/importers/group_importer_spec.rb @@ -7,23 +7,18 @@ RSpec.describe BulkImports::Importers::GroupImporter do let(:bulk_import) { create(:bulk_import) } let(:bulk_import_entity) { create(:bulk_import_entity, :started, bulk_import: bulk_import) } let(:bulk_import_configuration) { create(:bulk_import_configuration, bulk_import: bulk_import) } - let(:context) do - BulkImports::Pipeline::Context.new( - current_user: user, - entity: bulk_import_entity, - configuration: bulk_import_configuration - ) - end - - subject { described_class.new(bulk_import_entity) } + let(:context) { BulkImports::Pipeline::Context.new(bulk_import_entity) } before do allow(BulkImports::Pipeline::Context).to receive(:new).and_return(context) end + subject { described_class.new(bulk_import_entity) } + describe '#execute' do it 'starts the entity and run its pipelines' do expect_to_run_pipeline BulkImports::Groups::Pipelines::GroupPipeline, context: context + expect_to_run_pipeline BulkImports::Groups::Pipelines::LabelsPipeline, context: context expect_to_run_pipeline('EE::BulkImports::Groups::Pipelines::EpicsPipeline'.constantize, context: context) if Gitlab.ee? expect_to_run_pipeline BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, context: context diff --git a/spec/lib/bulk_imports/pipeline/context_spec.rb b/spec/lib/bulk_imports/pipeline/context_spec.rb index e9af6313ca4..c8c3fe3a861 100644 --- a/spec/lib/bulk_imports/pipeline/context_spec.rb +++ b/spec/lib/bulk_imports/pipeline/context_spec.rb @@ -3,25 +3,29 @@ require 'spec_helper' RSpec.describe BulkImports::Pipeline::Context do - describe '#initialize' do - it 'initializes with permitted attributes' do - args = { - current_user: create(:user), - entity: create(:bulk_import_entity), - configuration: create(:bulk_import_configuration) - } + let(:group) { instance_double(Group) } + let(:user) { instance_double(User) } + let(:bulk_import) { instance_double(BulkImport, user: user, configuration: :config) } - context = described_class.new(args) + let(:entity) do + instance_double( + BulkImports::Entity, + bulk_import: bulk_import, + group: group + ) + end + + subject { described_class.new(entity) } - args.each do |k, v| - expect(context.public_send(k)).to eq(v) - end - end + describe '#group' do + it { expect(subject.group).to eq(group) } + end + + describe '#current_user' do + it { expect(subject.current_user).to eq(user) } + end - context 'when invalid argument is passed' do - it 'raises NoMethodError' do - expect { described_class.new(test: 'test').test }.to raise_exception(NoMethodError) - end - end + describe '#current_user' do + it { expect(subject.configuration).to eq(bulk_import.configuration) } end end diff --git a/spec/lib/bulk_imports/pipeline/extracted_data_spec.rb b/spec/lib/bulk_imports/pipeline/extracted_data_spec.rb new file mode 100644 index 00000000000..25c5178227a --- /dev/null +++ b/spec/lib/bulk_imports/pipeline/extracted_data_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe BulkImports::Pipeline::ExtractedData do + let(:data) { 'data' } + let(:has_next_page) { true } + let(:cursor) { 'cursor' } + let(:page_info) do + { + 'has_next_page' => has_next_page, + 'end_cursor' => cursor + } + end + + subject { described_class.new(data: data, page_info: page_info) } + + describe '#has_next_page?' do + context 'when next page is present' do + it 'returns true' do + expect(subject.has_next_page?).to eq(true) + end + end + + context 'when next page is not present' do + let(:has_next_page) { false } + + it 'returns false' do + expect(subject.has_next_page?).to eq(false) + end + end + end + + describe '#next_page' do + it 'returns next page cursor information' do + expect(subject.next_page).to eq(cursor) + end + end + + describe '#each' do + context 'when block is present' do + it 'yields each data item' do + expect { |b| subject.each(&b) }.to yield_control + end + end + + context 'when block is not present' do + it 'returns enumerator' do + expect(subject.each).to be_instance_of(Enumerator) + end + end + end +end diff --git a/spec/lib/bulk_imports/pipeline/runner_spec.rb b/spec/lib/bulk_imports/pipeline/runner_spec.rb index 60833e83dcc..3f6a172beee 100644 --- a/spec/lib/bulk_imports/pipeline/runner_spec.rb +++ b/spec/lib/bulk_imports/pipeline/runner_spec.rb @@ -45,42 +45,71 @@ RSpec.describe BulkImports::Pipeline::Runner do end context 'when entity is not marked as failed' do - let(:context) do - instance_double( - BulkImports::Pipeline::Context, - entity: instance_double(BulkImports::Entity, id: 1, source_type: 'group', failed?: false) - ) - end + let(:entity) { create(:bulk_import_entity) } + let(:context) { BulkImports::Pipeline::Context.new(entity) } it 'runs pipeline extractor, transformer, loader' do - entries = [{ foo: :bar }] + extracted_data = BulkImports::Pipeline::ExtractedData.new(data: { foo: :bar }) expect_next_instance_of(BulkImports::Extractor) do |extractor| - expect(extractor).to receive(:extract).with(context).and_return(entries) + expect(extractor) + .to receive(:extract) + .with(context) + .and_return(extracted_data) end expect_next_instance_of(BulkImports::Transformer) do |transformer| - expect(transformer).to receive(:transform).with(context, entries.first).and_return(entries.first) + expect(transformer) + .to receive(:transform) + .with(context, extracted_data.data.first) + .and_return(extracted_data.data.first) end expect_next_instance_of(BulkImports::Loader) do |loader| - expect(loader).to receive(:load).with(context, entries.first) + expect(loader) + .to receive(:load) + .with(context, extracted_data.data.first) end expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect(logger).to receive(:info) .with( + bulk_import_entity_id: entity.id, + bulk_import_entity_type: 'group_entity', message: 'Pipeline started', + pipeline_class: 'BulkImports::MyPipeline' + ) + expect(logger).to receive(:info) + .with( + bulk_import_entity_id: entity.id, + bulk_import_entity_type: 'group_entity', pipeline_class: 'BulkImports::MyPipeline', - bulk_import_entity_id: 1, - bulk_import_entity_type: 'group' + pipeline_step: :extractor, + step_class: 'BulkImports::Extractor' ) expect(logger).to receive(:info) - .with(bulk_import_entity_id: 1, bulk_import_entity_type: 'group', extractor: 'BulkImports::Extractor') + .with( + bulk_import_entity_id: entity.id, + bulk_import_entity_type: 'group_entity', + pipeline_class: 'BulkImports::MyPipeline', + pipeline_step: :transformer, + step_class: 'BulkImports::Transformer' + ) expect(logger).to receive(:info) - .with(bulk_import_entity_id: 1, bulk_import_entity_type: 'group', transformer: 'BulkImports::Transformer') + .with( + bulk_import_entity_id: entity.id, + bulk_import_entity_type: 'group_entity', + pipeline_class: 'BulkImports::MyPipeline', + pipeline_step: :loader, + step_class: 'BulkImports::Loader' + ) expect(logger).to receive(:info) - .with(bulk_import_entity_id: 1, bulk_import_entity_type: 'group', loader: 'BulkImports::Loader') + .with( + bulk_import_entity_id: entity.id, + bulk_import_entity_type: 'group_entity', + message: 'Pipeline finished', + pipeline_class: 'BulkImports::MyPipeline' + ) end BulkImports::MyPipeline.new.run(context) @@ -88,7 +117,7 @@ RSpec.describe BulkImports::Pipeline::Runner do context 'when exception is raised' do let(:entity) { create(:bulk_import_entity, :created) } - let(:context) { BulkImports::Pipeline::Context.new(entity: entity) } + let(:context) { BulkImports::Pipeline::Context.new(entity) } before do allow_next_instance_of(BulkImports::Extractor) do |extractor| @@ -103,6 +132,7 @@ RSpec.describe BulkImports::Pipeline::Runner do expect(failure).to be_present expect(failure.pipeline_class).to eq('BulkImports::MyPipeline') + expect(failure.pipeline_step).to eq('extractor') expect(failure.exception_class).to eq('StandardError') expect(failure.exception_message).to eq('Error!') end @@ -144,21 +174,19 @@ RSpec.describe BulkImports::Pipeline::Runner do end context 'when entity is marked as failed' do - let(:context) do - instance_double( - BulkImports::Pipeline::Context, - entity: instance_double(BulkImports::Entity, id: 1, source_type: 'group', failed?: true) - ) - end + let(:entity) { create(:bulk_import_entity) } + let(:context) { BulkImports::Pipeline::Context.new(entity) } it 'logs and returns without execution' do + allow(entity).to receive(:failed?).and_return(true) + expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect(logger).to receive(:info) .with( message: 'Skipping due to failed pipeline status', pipeline_class: 'BulkImports::MyPipeline', - bulk_import_entity_id: 1, - bulk_import_entity_type: 'group' + bulk_import_entity_id: entity.id, + bulk_import_entity_type: 'group_entity' ) end -- cgit v1.2.3