Add latest changes from gitlab-org/gitlab@14-1-stable-eev14.1.0-rc42

9 files changed, 271 insertions, 14 deletions

diff --git a/spec/lib/bulk_imports/clients/graphql_spec.rb b/spec/lib/bulk_imports/clients/graphql_spec.rb
new file mode 100644
index 00000000000..2f212458c4a
--- /dev/null
+++ b/spec/lib/bulk_imports/clients/graphql_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::Clients::Graphql do
+  let_it_be(:config) { create(:bulk_import_configuration) }
+
+  subject { described_class.new(url: config.url, token: config.access_token) }
+
+  describe '#execute' do
+    let(:query) { '{ metadata { version } }' }
+    let(:graphql_client_double) { double }
+    let(:response_double) { double }
+
+    before do
+      stub_const('BulkImports::MINIMUM_COMPATIBLE_MAJOR_VERSION', version)
+      allow(graphql_client_double).to receive(:execute)
+      allow(subject).to receive(:client).and_return(graphql_client_double)
+      allow(graphql_client_double).to receive(:execute).with(query).and_return(response_double)
+      allow(response_double).to receive_message_chain(:data, :metadata, :version).and_return(version)
+    end
+
+    context 'when source instance is compatible' do
+      let(:version) { '14.0.0' }
+
+      it 'marks source instance as compatible' do
+        subject.execute('test')
+
+        expect(subject.instance_variable_get(:@compatible_instance_version)).to eq(true)
+      end
+    end
+
+    context 'when source instance is incompatible' do
+      let(:version) { '13.0.0' }
+
+      it 'raises an error' do
+        expect { subject.execute('test') }.to raise_error(::BulkImports::Error, "Unsupported GitLab Version. Minimum Supported Gitlab Version #{BulkImport::MINIMUM_GITLAB_MAJOR_VERSION}.")
+      end
+    end
+  end
+end
diff --git a/spec/lib/bulk_imports/clients/http_spec.rb b/spec/lib/bulk_imports/clients/http_spec.rb
index ac42f12a3d4..c36cb80851a 100644
--- a/spec/lib/bulk_imports/clients/http_spec.rb
+++ b/spec/lib/bulk_imports/clients/http_spec.rb
@@ -5,12 +5,20 @@ require 'spec_helper'
 RSpec.describe BulkImports::Clients::HTTP do
   include ImportSpecHelper
 
-  let(:uri) { 'http://gitlab.example' }
+  let(:url) { 'http://gitlab.example' }
   let(:token) { 'token' }
   let(:resource) { 'resource' }
+  let(:version) { "#{BulkImport::MINIMUM_GITLAB_MAJOR_VERSION}.0.0" }
   let(:response_double) { double(code: 200, success?: true, parsed_response: {}) }
+  let(:version_response) { double(code: 200, success?: true, parsed_response: { 'version' => version }) }
 
-  subject { described_class.new(uri: uri, token: token) }
+  before do
+    allow(Gitlab::HTTP).to receive(:get)
+      .with('http://gitlab.example/api/v4/version', anything)
+      .and_return(version_response)
+  end
+
+  subject { described_class.new(url: url, token: token) }
 
   shared_examples 'performs network request' do
     it 'performs network request' do
@@ -21,20 +29,20 @@ RSpec.describe BulkImports::Clients::HTTP do
 
     context 'error handling' do
       context 'when error occurred' do
-        it 'raises ConnectionError' do
+        it 'raises BulkImports::Error' do
           allow(Gitlab::HTTP).to receive(method).and_raise(Errno::ECONNREFUSED)
 
-          expect { subject.public_send(method, resource) }.to raise_exception(described_class::ConnectionError)
+          expect { subject.public_send(method, resource) }.to raise_exception(BulkImports::Error)
         end
       end
 
       context 'when response is not success' do
-        it 'raises ConnectionError' do
+        it 'raises BulkImports::Error' do
           response_double = double(code: 503, success?: false)
 
           allow(Gitlab::HTTP).to receive(method).and_return(response_double)
 
-          expect { subject.public_send(method, resource) }.to raise_exception(described_class::ConnectionError)
+          expect { subject.public_send(method, resource) }.to raise_exception(BulkImports::Error)
         end
       end
     end
@@ -46,7 +54,7 @@ RSpec.describe BulkImports::Clients::HTTP do
     include_examples 'performs network request' do
       let(:expected_args) do
         [
-          'http://gitlab.example:80/api/v4/resource',
+          'http://gitlab.example/api/v4/resource',
           hash_including(
             follow_redirects: false,
             query: {
@@ -96,7 +104,7 @@ RSpec.describe BulkImports::Clients::HTTP do
       private
 
       def stub_http_get(path, query, response)
-        uri = "http://gitlab.example:80/api/v4/#{path}"
+        uri = "http://gitlab.example/api/v4/#{path}"
         params = {
           follow_redirects: false,
           headers: {
@@ -116,7 +124,7 @@ RSpec.describe BulkImports::Clients::HTTP do
     include_examples 'performs network request' do
       let(:expected_args) do
         [
-          'http://gitlab.example:80/api/v4/resource',
+          'http://gitlab.example/api/v4/resource',
           hash_including(
             body: {},
             follow_redirects: false,
@@ -136,7 +144,7 @@ RSpec.describe BulkImports::Clients::HTTP do
     include_examples 'performs network request' do
       let(:expected_args) do
         [
-          'http://gitlab.example:80/api/v4/resource',
+          'http://gitlab.example/api/v4/resource',
           hash_including(
             follow_redirects: false,
             headers: {
@@ -152,7 +160,7 @@ RSpec.describe BulkImports::Clients::HTTP do
   describe '#stream' do
     it 'performs network request with stream_body option' do
       expected_args = [
-        'http://gitlab.example:80/api/v4/resource',
+        'http://gitlab.example/api/v4/resource',
         hash_including(
           stream_body: true,
           headers: {
@@ -167,4 +175,28 @@ RSpec.describe BulkImports::Clients::HTTP do
       subject.stream(resource)
     end
   end
+
+  context 'when source instance is incompatible' do
+    let(:version) { '13.0.0' }
+
+    it 'raises an error' do
+      expect { subject.get(resource) }.to raise_error(::BulkImports::Error, "Unsupported GitLab Version. Minimum Supported Gitlab Version #{BulkImport::MINIMUM_GITLAB_MAJOR_VERSION}.")
+    end
+  end
+
+  context 'when url is relative' do
+    let(:url) { 'http://website.example/gitlab' }
+
+    before do
+      allow(Gitlab::HTTP).to receive(:get)
+        .with('http://website.example/gitlab/api/v4/version', anything)
+        .and_return(version_response)
+    end
+
+    it 'performs network request to a relative gitlab url' do
+      expect(Gitlab::HTTP).to receive(:get).with('http://website.example/gitlab/api/v4/resource', anything).and_return(response_double)
+
+      subject.get(resource)
+    end
+  end
 end
diff --git a/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb b/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb
index 533955b057c..de0b56045b3 100644
--- a/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb
+++ b/spec/lib/bulk_imports/groups/loaders/group_loader_spec.rb
@@ -9,6 +9,7 @@ RSpec.describe BulkImports::Groups::Loaders::GroupLoader do
     let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import) }
     let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) }
     let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) }
+
     let(:service_double) { instance_double(::Groups::CreateService) }
     let(:data) { { foo: :bar } }
 
diff --git a/spec/lib/bulk_imports/groups/pipelines/group_avatar_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/group_avatar_pipeline_spec.rb
new file mode 100644
index 00000000000..c68284aa580
--- /dev/null
+++ b/spec/lib/bulk_imports/groups/pipelines/group_avatar_pipeline_spec.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::Groups::Pipelines::GroupAvatarPipeline do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:bulk_import) { create(:bulk_import, user: user) }
+
+  let_it_be(:entity) do
+    create(
+      :bulk_import_entity,
+      group: group,
+      bulk_import: bulk_import,
+      source_full_path: 'source/full/path',
+      destination_name: 'My Destination Group',
+      destination_namespace: group.full_path
+    )
+  end
+
+  let_it_be(:tracker) { create(:bulk_import_tracker, entity: entity) }
+  let_it_be(:context) { BulkImports::Pipeline::Context.new(tracker) }
+
+  subject { described_class.new(context) }
+
+  describe '#run' do
+    it 'updates the group avatar' do
+      avatar_path = 'spec/fixtures/dk.png'
+      stub_file_download(
+        avatar_path,
+        configuration: context.configuration,
+        relative_url: "/groups/source%2Ffull%2Fpath/avatar",
+        dir: an_instance_of(String),
+        file_size_limit: Avatarable::MAXIMUM_FILE_SIZE,
+        allowed_content_types: described_class::ALLOWED_AVATAR_DOWNLOAD_TYPES
+      )
+
+      expect { subject.run }.to change(context.group, :avatar)
+
+      expect(context.group.avatar.filename).to eq(File.basename(avatar_path))
+    end
+
+    it 'raises an error when the avatar upload fails' do
+      avatar_path = 'spec/fixtures/aosp_manifest.xml'
+      stub_file_download(
+        avatar_path,
+        configuration: context.configuration,
+        relative_url: "/groups/source%2Ffull%2Fpath/avatar",
+        dir: an_instance_of(String),
+        file_size_limit: Avatarable::MAXIMUM_FILE_SIZE,
+        allowed_content_types: described_class::ALLOWED_AVATAR_DOWNLOAD_TYPES
+      )
+
+      expect_next_instance_of(Gitlab::Import::Logger) do |logger|
+        expect(logger).to receive(:error)
+          .with(
+            bulk_import_id: context.bulk_import.id,
+            bulk_import_entity_id: context.entity.id,
+            bulk_import_entity_type: context.entity.source_type,
+            context_extra: context.extra,
+            exception_class: "BulkImports::Groups::Pipelines::GroupAvatarPipeline::GroupAvatarLoadingError",
+            exception_message: "Avatar file format is not supported. Please try one of the following supported formats: image/png, image/jpeg, image/gif, image/bmp, image/tiff, image/vnd.microsoft.icon",
+            pipeline_class: "BulkImports::Groups::Pipelines::GroupAvatarPipeline",
+            pipeline_step: :loader
+          )
+      end
+
+      expect { subject.run }.to change(BulkImports::Failure, :count)
+    end
+  end
+
+  def stub_file_download(filepath = 'file/path.png', **params)
+    expect_next_instance_of(BulkImports::FileDownloadService, params.presence) do |downloader|
+      expect(downloader).to receive(:execute).and_return(filepath)
+    end
+  end
+end
diff --git a/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb b/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb
index d8a667ec92a..0126acb320b 100644
--- a/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb
+++ b/spec/lib/bulk_imports/groups/pipelines/members_pipeline_spec.rb
@@ -63,6 +63,14 @@ RSpec.describe BulkImports::Groups::Pipelines::MembersPipeline do
       expect(member.updated_at).to eq('2020-01-01T00:00:00Z')
       expect(member.expires_at).to eq(nil)
     end
+
+    context 'when user_id is current user id' do
+      it 'does not create new member' do
+        data = { 'user_id' => user.id }
+
+        expect { subject.load(context, data) }.not_to change(GroupMember, :count)
+      end
+    end
   end
 
   describe 'pipeline parts' do
diff --git a/spec/lib/bulk_imports/groups/transformers/member_attributes_transformer_spec.rb b/spec/lib/bulk_imports/groups/transformers/member_attributes_transformer_spec.rb
index f3905a4b6e4..af99428e0c1 100644
--- a/spec/lib/bulk_imports/groups/transformers/member_attributes_transformer_spec.rb
+++ b/spec/lib/bulk_imports/groups/transformers/member_attributes_transformer_spec.rb
@@ -84,9 +84,34 @@ RSpec.describe BulkImports::Groups::Transformers::MemberAttributesTransformer do
         expect(subject.transform(context, data)).to be_nil
       end
     end
+
+    context 'source user id caching' do
+      context 'when user gid is present' do
+        it 'caches source user id' do
+          gid = 'gid://gitlab/User/7'
+          data = member_data(email: user.email, gid: gid)
+
+          expect_next_instance_of(BulkImports::UsersMapper) do |mapper|
+            expect(mapper).to receive(:cache_source_user_id).with('7', user.id)
+          end
+
+          subject.transform(context, data)
+        end
+      end
+
+      context 'when user gid is missing' do
+        it 'does not use caching' do
+          data = member_data(email: user.email)
+
+          expect(BulkImports::UsersMapper).not_to receive(:new)
+
+          subject.transform(context, data)
+        end
+      end
+    end
   end
 
-  def member_data(email: '', access_level: 30)
+  def member_data(email: '', gid: nil, access_level: 30)
     {
       'created_at' => '2020-01-01T00:00:00Z',
       'updated_at' => '2020-01-01T00:00:00Z',
@@ -95,6 +120,7 @@ RSpec.describe BulkImports::Groups::Transformers::MemberAttributesTransformer do
         'integer_value' => access_level
       },
       'user' => {
+        'user_gid' => gid,
         'public_email' => email
       }
     }
diff --git a/spec/lib/bulk_imports/ndjson_pipeline_spec.rb b/spec/lib/bulk_imports/ndjson_pipeline_spec.rb
index a5d1a5f7fbb..57a258b0d9f 100644
--- a/spec/lib/bulk_imports/ndjson_pipeline_spec.rb
+++ b/spec/lib/bulk_imports/ndjson_pipeline_spec.rb
@@ -106,8 +106,11 @@ RSpec.describe BulkImports::NdjsonPipeline do
       data = [hash, 1]
       user = double
       config = double(relation_excluded_keys: nil, top_relation_tree: [])
-      context = double(portable: group, current_user: user, import_export_config: config)
+      import_double = instance_double(BulkImport, id: 1)
+      entity_double = instance_double(BulkImports::Entity, id: 2)
+      context = double(portable: group, current_user: user, import_export_config: config, bulk_import: import_double, entity: entity_double)
       allow(subject).to receive(:import_export_config).and_return(config)
+      allow(subject).to receive(:context).and_return(context)
 
       expect(Gitlab::ImportExport::Group::RelationFactory)
         .to receive(:create)
@@ -116,7 +119,7 @@ RSpec.describe BulkImports::NdjsonPipeline do
           relation_sym: :test,
           relation_hash: hash,
           importable: group,
-          members_mapper: instance_of(Gitlab::ImportExport::MembersMapper),
+          members_mapper: instance_of(BulkImports::UsersMapper),
           object_builder: Gitlab::ImportExport::Group::ObjectBuilder,
           user: user,
           excluded_keys: nil
diff --git a/spec/lib/bulk_imports/stage_spec.rb b/spec/lib/bulk_imports/stage_spec.rb
index d082faa90bc..4398b00e7e9 100644
--- a/spec/lib/bulk_imports/stage_spec.rb
+++ b/spec/lib/bulk_imports/stage_spec.rb
@@ -6,6 +6,7 @@ RSpec.describe BulkImports::Stage do
   let(:pipelines) do
     [
       [0, BulkImports::Groups::Pipelines::GroupPipeline],
+      [1, BulkImports::Groups::Pipelines::GroupAvatarPipeline],
       [1, BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline],
       [1, BulkImports::Groups::Pipelines::MembersPipeline],
       [1, BulkImports::Groups::Pipelines::LabelsPipeline],
diff --git a/spec/lib/bulk_imports/users_mapper_spec.rb b/spec/lib/bulk_imports/users_mapper_spec.rb
new file mode 100644
index 00000000000..e6357319d05
--- /dev/null
+++ b/spec/lib/bulk_imports/users_mapper_spec.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::UsersMapper do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:import) { create(:bulk_import, user: user) }
+  let_it_be(:entity) { create(:bulk_import_entity, bulk_import: import) }
+
+  let(:context) do
+    instance_double(
+      BulkImports::Pipeline::Context,
+      bulk_import: import,
+      entity: entity,
+      current_user: user
+    )
+  end
+
+  subject { described_class.new(context: context) }
+
+  describe '#map' do
+    context 'when value for specified key exists' do
+      it 'returns a map of source & destination user ids from redis' do
+        allow(Gitlab::Cache::Import::Caching).to receive(:values_from_hash).and_return({ "1" => "2" })
+
+        expect(subject.map).to eq({ 1 => 2 })
+      end
+    end
+
+    context 'when value for specified key does not exist' do
+      it 'returns default value' do
+        expect(subject.map[:non_existent_key]).to eq(user.id)
+      end
+    end
+  end
+
+  describe '#default_user_id' do
+    it 'returns current user id' do
+      expect(subject.default_user_id).to eq(user.id)
+    end
+  end
+
+  describe '#include?' do
+    context 'when source user id is present in the map' do
+      it 'returns true' do
+        allow(subject).to receive(:map).and_return({ 1 => 2 })
+
+        expect(subject.include?(1)).to eq(true)
+      end
+    end
+
+    context 'when source user id is missing in the map' do
+      it 'returns false' do
+        allow(subject).to receive(:map).and_return({})
+
+        expect(subject.include?(1)).to eq(false)
+      end
+    end
+  end
+
+  describe '#cache_source_user_id' do
+    it 'caches provided source & destination user ids in redis' do
+      expect(Gitlab::Cache::Import::Caching).to receive(:hash_add).with("bulk_imports/#{import.id}/#{entity.id}/source_user_ids", 1, 2)
+
+      subject.cache_source_user_id(1, 2)
+    end
+  end
+end