Port LFS direct_upload from EE

1 files changed, 328 insertions, 24 deletions

diff --git a/spec/uploaders/object_storage_spec.rb b/spec/uploaders/object_storage_spec.rb
index 489b6707c6e..1d406c71955 100644
--- a/spec/uploaders/object_storage_spec.rb
+++ b/spec/uploaders/object_storage_spec.rb
@@ -21,11 +21,11 @@ describe ObjectStorage do
   let(:object) { build_stubbed(:user) }
   let(:uploader) { uploader_class.new(object, :file) }
 
-  before do
-    allow(uploader_class).to receive(:object_store_enabled?).and_return(true)
-  end
-
   describe '#object_store=' do
+    before do
+      allow(uploader_class).to receive(:object_store_enabled?).and_return(true)
+    end
+
     it "reload the local storage" do
       uploader.object_store = described_class::Store::LOCAL
       expect(uploader.file_storage?).to be_truthy
@@ -35,28 +35,28 @@ describe ObjectStorage do
       uploader.object_store = described_class::Store::REMOTE
       expect(uploader.file_storage?).to be_falsey
     end
-  end
 
-  context 'object_store is Store::LOCAL' do
-    before do
-      uploader.object_store = described_class::Store::LOCAL
-    end
+    context 'object_store is Store::LOCAL' do
+      before do
+        uploader.object_store = described_class::Store::LOCAL
+      end
 
-    describe '#store_dir' do
-      it 'is the composition of (base_dir, dynamic_segment)' do
-        expect(uploader.store_dir).to start_with("uploads/-/system/user/")
+      describe '#store_dir' do
+        it 'is the composition of (base_dir, dynamic_segment)' do
+          expect(uploader.store_dir).to start_with("uploads/-/system/user/")
+        end
       end
     end
-  end
 
-  context 'object_store is Store::REMOTE' do
-    before do
-      uploader.object_store = described_class::Store::REMOTE
-    end
+    context 'object_store is Store::REMOTE' do
+      before do
+        uploader.object_store = described_class::Store::REMOTE
+      end
 
-    describe '#store_dir' do
-      it 'is the composition of (dynamic_segment)' do
-        expect(uploader.store_dir).to start_with("user/")
+      describe '#store_dir' do
+        it 'is the composition of (dynamic_segment)' do
+          expect(uploader.store_dir).to start_with("user/")
+        end
       end
     end
   end
@@ -92,7 +92,7 @@ describe ObjectStorage do
   describe '#file_cache_storage?' do
     context 'when file storage is used' do
       before do
-        uploader_class.cache_storage(:file)
+        expect(uploader_class).to receive(:cache_storage) { CarrierWave::Storage::File }
       end
 
       it { expect(uploader).to be_file_cache_storage }
@@ -100,7 +100,7 @@ describe ObjectStorage do
 
     context 'when is remote storage' do
       before do
-        uploader_class.cache_storage(:fog)
+        expect(uploader_class).to receive(:cache_storage) { CarrierWave::Storage::Fog }
       end
 
       it { expect(uploader).not_to be_file_cache_storage }
@@ -298,7 +298,9 @@ describe ObjectStorage do
     let(:remote_directory) { 'directory' }
 
     before do
-      uploader_class.storage_options double(object_store: double(remote_directory: remote_directory))
+      allow(uploader_class).to receive(:options) do
+        double(object_store: double(remote_directory: remote_directory))
+      end
     end
 
     subject { uploader.fog_directory }
@@ -310,7 +312,9 @@ describe ObjectStorage do
     let(:connection) { Settingslogic.new("provider" => "AWS") }
 
     before do
-      uploader_class.storage_options double(object_store: double(connection: connection))
+      allow(uploader_class).to receive(:options) do
+        double(object_store: double(connection: connection))
+      end
     end
 
     subject { uploader.fog_credentials }
@@ -323,4 +327,304 @@ describe ObjectStorage do
 
     it { is_expected.to eq(false) }
   end
+
+  describe '.workhorse_authorize' do
+    subject { uploader_class.workhorse_authorize }
+
+    before do
+      # ensure that we use regular Fog libraries
+      # other tests might call `Fog.mock!` and
+      # it will make tests to fail
+      Fog.unmock!
+    end
+
+    shared_examples 'uses local storage' do
+      it "returns temporary path" do
+        is_expected.to have_key(:TempPath)
+
+        expect(subject[:TempPath]).to start_with(uploader_class.root)
+        expect(subject[:TempPath]).to include(described_class::TMP_UPLOAD_PATH)
+      end
+
+      it "does not return remote store" do
+        is_expected.not_to have_key('RemoteObject')
+      end
+    end
+
+    shared_examples 'uses remote storage' do
+      it "returns remote store" do
+        is_expected.to have_key(:RemoteObject)
+
+        expect(subject[:RemoteObject]).to have_key(:ID)
+        expect(subject[:RemoteObject]).to have_key(:GetURL)
+        expect(subject[:RemoteObject]).to have_key(:DeleteURL)
+        expect(subject[:RemoteObject]).to have_key(:StoreURL)
+        expect(subject[:RemoteObject][:GetURL]).to include(described_class::TMP_UPLOAD_PATH)
+        expect(subject[:RemoteObject][:DeleteURL]).to include(described_class::TMP_UPLOAD_PATH)
+        expect(subject[:RemoteObject][:StoreURL]).to include(described_class::TMP_UPLOAD_PATH)
+      end
+
+      it "does not return local store" do
+        is_expected.not_to have_key('TempPath')
+      end
+    end
+
+    context 'when object storage is disabled' do
+      before do
+        allow(Gitlab.config.uploads.object_store).to receive(:enabled) { false }
+      end
+
+      it_behaves_like 'uses local storage'
+    end
+
+    context 'when object storage is enabled' do
+      before do
+        allow(Gitlab.config.uploads.object_store).to receive(:enabled) { true }
+      end
+
+      context 'when direct upload is enabled' do
+        before do
+          allow(Gitlab.config.uploads.object_store).to receive(:direct_upload) { true }
+        end
+
+        context 'uses AWS' do
+          before do
+            expect(uploader_class).to receive(:object_store_credentials) do
+              { provider: "AWS",
+                aws_access_key_id: "AWS_ACCESS_KEY_ID",
+                aws_secret_access_key: "AWS_SECRET_ACCESS_KEY",
+                region: "eu-central-1" }
+            end
+          end
+
+          it_behaves_like 'uses remote storage' do
+            let(:storage_url) { "https://uploads.s3-eu-central-1.amazonaws.com/" }
+
+            it 'returns links for S3' do
+              expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+              expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+              expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+            end
+          end
+        end
+
+        context 'uses Google' do
+          before do
+            expect(uploader_class).to receive(:object_store_credentials) do
+              { provider: "Google",
+                google_storage_access_key_id: 'ACCESS_KEY_ID',
+                google_storage_secret_access_key: 'SECRET_ACCESS_KEY' }
+            end
+          end
+
+          it_behaves_like 'uses remote storage' do
+            let(:storage_url) { "https://storage.googleapis.com/uploads/" }
+
+            it 'returns links for Google Cloud' do
+              expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+              expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+              expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+            end
+          end
+        end
+
+        context 'uses GDK/minio' do
+          before do
+            expect(uploader_class).to receive(:object_store_credentials) do
+              { provider: "AWS",
+                aws_access_key_id: "AWS_ACCESS_KEY_ID",
+                aws_secret_access_key: "AWS_SECRET_ACCESS_KEY",
+                endpoint: 'http://127.0.0.1:9000',
+                path_style: true,
+                region: "gdk" }
+            end
+          end
+
+          it_behaves_like 'uses remote storage' do
+            let(:storage_url) { "http://127.0.0.1:9000/uploads/" }
+
+            it 'returns links for S3' do
+              expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
+              expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
+              expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
+            end
+          end
+        end
+      end
+
+      context 'when direct upload is disabled' do
+        before do
+          allow(Gitlab.config.uploads.object_store).to receive(:direct_upload) { false }
+        end
+
+        it_behaves_like 'uses local storage'
+      end
+    end
+  end
+
+  describe '#store_workhorse_file!' do
+    subject do
+      uploader.store_workhorse_file!(params, :file)
+    end
+
+    context 'when local file is used' do
+      context 'when valid file is used' do
+        let(:target_path) do
+          File.join(uploader_class.root, uploader_class::TMP_UPLOAD_PATH)
+        end
+
+        before do
+          FileUtils.mkdir_p(target_path)
+        end
+
+        context 'when no filename is specified' do
+          let(:params) do
+            { "file.path" => "test/file" }
+          end
+
+          it 'raises an error' do
+            expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Missing filename/)
+          end
+        end
+
+        context 'when invalid file is specified' do
+          let(:file_path) do
+            File.join(target_path, "..", "test.file")
+          end
+
+          before do
+            FileUtils.touch(file_path)
+          end
+
+          let(:params) do
+            { "file.path" => file_path,
+              "file.name" => "my_file.txt" }
+          end
+
+          it 'raises an error' do
+            expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Bad file path/)
+          end
+        end
+
+        context 'when filename is specified' do
+          let(:params) do
+            { "file.path" => tmp_file,
+              "file.name" => "my_file.txt" }
+          end
+
+          let(:tmp_file) { Tempfile.new('filename', target_path) }
+
+          before do
+            FileUtils.touch(tmp_file)
+          end
+
+          after do
+            FileUtils.rm_f(tmp_file)
+          end
+
+          it 'succeeds' do
+            expect { subject }.not_to raise_error
+
+            expect(uploader).to be_exists
+          end
+
+          it 'proper path is being used' do
+            subject
+
+            expect(uploader.path).to start_with(uploader_class.root)
+            expect(uploader.path).to end_with("my_file.txt")
+          end
+
+          it 'source file to not exist' do
+            subject
+
+            expect(File.exist?(tmp_file.path)).to be_falsey
+          end
+        end
+      end
+    end
+
+    context 'when remote file is used' do
+      let!(:fog_connection) do
+        stub_uploads_object_storage(uploader_class)
+      end
+
+      context 'when valid file is used' do
+        context 'when no filename is specified' do
+          let(:params) do
+            { "file.remote_id" => "test/123123" }
+          end
+
+          it 'raises an error' do
+            expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Missing filename/)
+          end
+        end
+
+        context 'when invalid file is specified' do
+          let(:params) do
+            { "file.remote_id" => "../test/123123",
+              "file.name" => "my_file.txt" }
+          end
+
+          it 'raises an error' do
+            expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Bad file path/)
+          end
+        end
+
+        context 'when non existing file is specified' do
+          let(:params) do
+            { "file.remote_id" => "test/12312300",
+              "file.name" => "my_file.txt" }
+          end
+
+          it 'raises an error' do
+            expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Missing file/)
+          end
+        end
+
+        context 'when filename is specified' do
+          let(:params) do
+            { "file.remote_id" => "test/123123",
+              "file.name" => "my_file.txt" }
+          end
+
+          let!(:fog_file) do
+            fog_connection.directories.get('uploads').files.create(
+              key: 'tmp/upload/test/123123',
+              body: 'content'
+            )
+          end
+
+          it 'succeeds' do
+            expect { subject }.not_to raise_error
+
+            expect(uploader).to be_exists
+          end
+
+          it 'path to not be temporary' do
+            subject
+
+            expect(uploader.path).not_to be_nil
+            expect(uploader.path).not_to include('tmp/upload')
+            expect(uploader.url).to include('/my_file.txt')
+          end
+
+          it 'url is used' do
+            subject
+
+            expect(uploader.url).not_to be_nil
+            expect(uploader.url).to include('/my_file.txt')
+          end
+        end
+      end
+    end
+
+    context 'when no file is used' do
+      let(:params) { {} }
+
+      it 'raises an error' do
+        expect { subject }.to raise_error(uploader_class::RemoteStoreError, /Bad file/)
+      end
+    end
+  end
 end