Merge branch '48773-gitlab-project-import-should-use-object-storage' into 'master'

Resolve "GitLab Project Import should use object storage"

Closes #48773

See merge request gitlab-org/gitlab-ce!20773

4 files changed, 209 insertions, 4 deletions

diff --git a/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb b/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb
new file mode 100644
index 00000000000..287745eb40e
--- /dev/null
+++ b/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb
@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::FileImporter do
+  let(:shared) { Gitlab::ImportExport::Shared.new(nil) }
+  let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" }
+  let(:valid_file) { "#{shared.export_path}/valid.json" }
+  let(:symlink_file) { "#{shared.export_path}/invalid.json" }
+  let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" }
+  let(:subfolder_symlink_file) { "#{shared.export_path}/subfolder/invalid.json" }
+  let(:evil_symlink_file) { "#{shared.export_path}/.\nevil" }
+
+  before do
+    stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0)
+    stub_feature_flags(import_export_object_storage: true)
+    stub_uploads_object_storage(FileUploader)
+
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path)
+    allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true)
+    allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test')
+    allow(SecureRandom).to receive(:hex).and_return('abcd')
+    setup_files
+  end
+
+  after do
+    FileUtils.rm_rf(storage_path)
+  end
+
+  context 'normal run' do
+    before do
+      described_class.import(project: build(:project), archive_file: '', shared: shared)
+    end
+
+    it 'removes symlinks in root folder' do
+      expect(File.exist?(symlink_file)).to be false
+    end
+
+    it 'removes hidden symlinks in root folder' do
+      expect(File.exist?(hidden_symlink_file)).to be false
+    end
+
+    it 'removes evil symlinks in root folder' do
+      expect(File.exist?(evil_symlink_file)).to be false
+    end
+
+    it 'removes symlinks in subfolders' do
+      expect(File.exist?(subfolder_symlink_file)).to be false
+    end
+
+    it 'does not remove a valid file' do
+      expect(File.exist?(valid_file)).to be true
+    end
+
+    it 'creates the file in the right subfolder' do
+      expect(shared.export_path).to include('test/abcd')
+    end
+  end
+
+  context 'error' do
+    before do
+      allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
+      described_class.import(project: build(:project), archive_file: '', shared: shared)
+    end
+
+    it 'removes symlinks in root folder' do
+      expect(File.exist?(symlink_file)).to be false
+    end
+
+    it 'removes hidden symlinks in root folder' do
+      expect(File.exist?(hidden_symlink_file)).to be false
+    end
+
+    it 'removes symlinks in subfolders' do
+      expect(File.exist?(subfolder_symlink_file)).to be false
+    end
+
+    it 'does not remove a valid file' do
+      expect(File.exist?(valid_file)).to be true
+    end
+  end
+
+  def setup_files
+    FileUtils.mkdir_p("#{shared.export_path}/subfolder/")
+    FileUtils.touch(valid_file)
+    FileUtils.ln_s(valid_file, symlink_file)
+    FileUtils.ln_s(valid_file, subfolder_symlink_file)
+    FileUtils.ln_s(valid_file, hidden_symlink_file)
+    FileUtils.ln_s(valid_file, evil_symlink_file)
+  end
+end
diff --git a/spec/lib/gitlab/import_export/file_importer_spec.rb b/spec/lib/gitlab/import_export/file_importer_spec.rb
index 265937f899e..78fccdf1dfc 100644
--- a/spec/lib/gitlab/import_export/file_importer_spec.rb
+++ b/spec/lib/gitlab/import_export/file_importer_spec.rb
@@ -24,7 +24,7 @@ describe Gitlab::ImportExport::FileImporter do
 
   context 'normal run' do
     before do
-      described_class.import(archive_file: '', shared: shared)
+      described_class.import(project: nil, archive_file: '', shared: shared)
     end
 
     it 'removes symlinks in root folder' do
@@ -55,7 +55,7 @@ describe Gitlab::ImportExport::FileImporter do
   context 'error' do
     before do
       allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError)
-      described_class.import(archive_file: '', shared: shared)
+      described_class.import(project: nil, archive_file: '', shared: shared)
     end
 
     it 'removes symlinks in root folder' do
diff --git a/spec/lib/gitlab/import_export/importer_object_storage_spec.rb b/spec/lib/gitlab/import_export/importer_object_storage_spec.rb
new file mode 100644
index 00000000000..24a994b3611
--- /dev/null
+++ b/spec/lib/gitlab/import_export/importer_object_storage_spec.rb
@@ -0,0 +1,115 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::Importer do
+  let(:user) { create(:user) }
+  let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
+  let(:shared) { project.import_export_shared }
+  let(:project) { create(:project) }
+  let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') }
+
+  subject(:importer) { described_class.new(project) }
+
+  before do
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
+    allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
+    stub_feature_flags(import_export_object_storage: true)
+    stub_uploads_object_storage(FileUploader)
+
+    FileUtils.mkdir_p(shared.export_path)
+    ImportExportUpload.create(project: project, import_file: import_file)
+  end
+
+  after do
+    FileUtils.rm_rf(test_path)
+  end
+
+  describe '#execute' do
+    it 'succeeds' do
+      importer.execute
+
+      expect(shared.errors).to be_empty
+    end
+
+    it 'extracts the archive'  do
+      expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
+
+      importer.execute
+    end
+
+    it 'checks the version' do
+      expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
+
+      importer.execute
+    end
+
+    context 'all restores are executed' do
+      [
+        Gitlab::ImportExport::AvatarRestorer,
+        Gitlab::ImportExport::RepoRestorer,
+        Gitlab::ImportExport::WikiRestorer,
+        Gitlab::ImportExport::UploadsRestorer,
+        Gitlab::ImportExport::LfsRestorer,
+        Gitlab::ImportExport::StatisticsRestorer
+      ].each do |restorer|
+        it "calls the #{restorer}" do
+          fake_restorer = double(restorer.to_s)
+
+          expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
+          expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
+
+          importer.execute
+        end
+      end
+
+      it 'restores the ProjectTree' do
+        expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
+
+        importer.execute
+      end
+
+      it 'removes the import file' do
+        expect(importer).to receive(:remove_import_file).and_call_original
+
+        importer.execute
+
+        expect(project.import_export_upload.import_file&.file).to be_nil
+      end
+    end
+
+    context 'when project successfully restored' do
+      let!(:existing_project) { create(:project, namespace: user.namespace) }
+      let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') }
+
+      before do
+        restorers = double(:restorers, all?: true)
+
+        allow(subject).to receive(:import_file).and_return(true)
+        allow(subject).to receive(:check_version!).and_return(true)
+        allow(subject).to receive(:restorers).and_return(restorers)
+        allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path }))
+      end
+
+      context 'when import_data' do
+        context 'has original_path' do
+          it 'overwrites existing project' do
+            expect_any_instance_of(::Projects::OverwriteProjectService).to receive(:execute).with(existing_project)
+
+            subject.execute
+          end
+        end
+
+        context 'has not original_path' do
+          before do
+            allow(project).to receive(:import_data).and_return(double(data: {}))
+          end
+
+          it 'does not call the overwrite service' do
+            expect_any_instance_of(::Projects::OverwriteProjectService).not_to receive(:execute).with(existing_project)
+
+            subject.execute
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb
index c074e61da26..f07946824c4 100644
--- a/spec/lib/gitlab/import_export/importer_spec.rb
+++ b/spec/lib/gitlab/import_export/importer_spec.rb
@@ -10,9 +10,10 @@ describe Gitlab::ImportExport::Importer do
 
   before do
     allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
+    allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
+
     FileUtils.mkdir_p(shared.export_path)
     FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path)
-    allow(subject).to receive(:remove_import_file)
   end
 
   after do
@@ -69,7 +70,7 @@ describe Gitlab::ImportExport::Importer do
       let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') }
 
       before do
-        restorers = double
+        restorers = double(:restorers, all?: true)
 
         allow(subject).to receive(:import_file).and_return(true)
         allow(subject).to receive(:check_version!).and_return(true)