Add latest changes from gitlab-org/gitlab@master

5 files changed, 163 insertions, 4 deletions

diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
index ba5132326f6..363fd0598f9 100644
--- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
@@ -24,7 +24,7 @@ module Gitlab
         # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be
         # present in the Gitlab::BackgroundMigration::BatchingStrategies module.
         #
-        # If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper
+        # If migration with same job_class_name, table_name, column_name, and job_arguments already exists, this helper
         # will log an warning and not create a new one.
         #
         # job_class_name - The background migration job class as a string
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index e38496ecf67..34e75755dec 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -12,12 +12,13 @@ module Gitlab
 
         private
 
-        attr_reader :project, :current_user, :lock_file
+        attr_reader :project, :current_user, :lock_file, :logger
 
         public
 
         def initialize(attributes = {})
           @options = attributes
+          @logger = Gitlab::Export::Logger.build
         end
 
         def method_missing(method, *args)
@@ -43,6 +44,10 @@ module Gitlab
 
           true
         rescue StandardError => e
+          payload = { message: "After export strategy failed" }
+          Gitlab::ExceptionLogFormatter.format!(e, payload)
+          log_error(payload)
+
           project.import_export_shared.error(e)
           false
         ensure
@@ -108,6 +113,18 @@ module Gitlab
         def log_validation_errors
           errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
         end
+
+        def log_info(params)
+          logger.info(log_default_params.merge(params))
+        end
+
+        def log_error(params)
+          logger.error(log_default_params.merge(params))
+        end
+
+        def log_default_params
+          { project_name: project.name, project_id: project.id }
+        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb b/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb
index 2e3136936f8..bddbe7862cb 100644
--- a/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb
@@ -5,6 +5,7 @@ module Gitlab
     module AfterExportStrategies
       class MoveFileStrategy < BaseAfterExportStrategy
         def initialize(archive_path:)
+          super
           @archive_path = archive_path
         end
 
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index 78608a946de..6c5fba37d7b 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -23,7 +23,17 @@ module Gitlab
         protected
 
         def strategy_execute
-          handle_response_error(send_file)
+          log_info(message: "Started uploading project", export_size: export_size)
+
+          upload_duration = Benchmark.realtime do
+            if Feature.enabled?(:import_export_web_upload_stream) && !project.export_file.file_storage?
+              upload_project_as_remote_stream
+            else
+              handle_response_error(send_file)
+            end
+          end
+
+          log_info(message: "Finished uploading project", export_size: export_size, upload_duration: upload_duration)
         end
 
         def handle_response_error(response)
@@ -44,8 +54,22 @@ module Gitlab
           export_file.close if export_file
         end
 
+        def upload_project_as_remote_stream
+          Gitlab::ImportExport::RemoteStreamUpload.new(
+            download_url: project.export_file.url,
+            upload_url: url,
+            options: {
+              upload_method: http_method.downcase.to_sym,
+              upload_content_type: 'application/gzip'
+            }).execute
+        rescue Gitlab::ImportExport::RemoteStreamUpload::StreamError => e
+          log_error(message: e.message, response_body: e.response_body.truncate(3000))
+
+          raise
+        end
+
         def export_file
-          project.export_file.open
+          @export_file ||= project.export_file.open
         end
 
         def send_file_options
diff --git a/lib/gitlab/import_export/remote_stream_upload.rb b/lib/gitlab/import_export/remote_stream_upload.rb
new file mode 100644
index 00000000000..f3bd241c0bd
--- /dev/null
+++ b/lib/gitlab/import_export/remote_stream_upload.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+# This class downloads a file from one URL and uploads it to another URL
+# without having to save the file on the disk and loading the whole file in
+# memory. The download and upload are performed in chunks size of
+# `buffer_size`. A chunk is downloaded, then uploaded, then a next chunk is
+# downloaded and uploaded. This repeats until all the file is processed.
+
+module Gitlab
+  module ImportExport
+    class RemoteStreamUpload
+      def initialize(download_url:, upload_url:, options: {})
+        @download_url = download_url
+        @upload_url = upload_url
+        @upload_method = options[:upload_method] || :post
+        @upload_content_type = options[:upload_content_type] || 'application/gzip'
+      end
+
+      def execute
+        receive_data(download_url) do |response, chunks|
+          send_data(upload_url, response.content_length, chunks) do |response|
+            if response.code != '200'
+              raise StreamError.new("Invalid response code while uploading file. Code: #{response.code}", response.body)
+            end
+          end
+        end
+      end
+      class StreamError < StandardError
+        attr_reader :response_body
+
+        def initialize(message, response_body = '')
+          super(message)
+          @response_body = response_body
+        end
+      end
+      class ChunkStream
+        DEFAULT_BUFFER_SIZE = 128.kilobytes
+
+        def initialize(chunks)
+          @chunks = chunks
+          @last_chunk = nil
+          @end_of_chunks = false
+        end
+
+        def read(n1 = nil, n2 = nil)
+          ensure_chunk&.read(n1, n2)
+        end
+
+        private
+
+        def ensure_chunk
+          return @last_chunk if @last_chunk && !@last_chunk.eof?
+          return if @end_of_chunks
+
+          @last_chunk = read_next_chunk
+        end
+
+        def read_next_chunk
+          next_chunk = StringIO.new
+
+          begin
+            next_chunk.write(@chunks.next) until next_chunk.size > DEFAULT_BUFFER_SIZE
+          rescue StopIteration
+            @end_of_chunks = true
+          end
+
+          next_chunk.rewind
+
+          next_chunk
+        end
+      end
+
+      private
+
+      attr_reader :download_url, :upload_url, :upload_method, :upload_content_type, :logger
+
+      def receive_data(uri)
+        http = Gitlab::HTTPConnectionAdapter.new(URI(uri), {}).connection
+
+        http.start do
+          request = Net::HTTP::Get.new(uri)
+          http.request(request) do |response|
+            if response.code == '200'
+              yield(response, response.enum_for(:read_body))
+            else
+              raise StreamError.new(
+                "Invalid response code while downloading file. Code: #{response.code}",
+                response.body
+              )
+            end
+          end
+        end
+      end
+
+      def send_data(uri, content_length, chunks)
+        http = Gitlab::HTTPConnectionAdapter.new(URI(uri), {}).connection
+
+        http.start do
+          request = upload_request_class(upload_method).new(uri)
+          request.body_stream = ChunkStream.new(chunks)
+          request.content_length = content_length
+          request.content_type = upload_content_type
+
+          http.request(request) do |response|
+            yield(response)
+          end
+        end
+      end
+
+      def upload_request_class(upload_method)
+        return Net::HTTP::Put if upload_method == :put
+
+        Net::HTTP::Post
+      end
+    end
+  end
+end