diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-11 09:12:18 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-11 09:12:18 +0300 |
commit | 20d8491f30854d1193e9ae0fc8fa97051354551d (patch) | |
tree | a75662ad4e0e608744cfdb9578dae6f08b433b49 /lib | |
parent | 8754d20bbb9e573d48e80d7f6aed1ded40a40263 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
5 files changed, 163 insertions, 4 deletions
diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index ba5132326f6..363fd0598f9 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -24,7 +24,7 @@ module Gitlab # class must be present in the Gitlab::BackgroundMigration module, and the batch class (if specified) must be # present in the Gitlab::BackgroundMigration::BatchingStrategies module. # - # If migration with same job_class_name, table_name, column_name, and job_aruments already exists, this helper + # If migration with same job_class_name, table_name, column_name, and job_arguments already exists, this helper # will log an warning and not create a new one. # # job_class_name - The background migration job class as a string diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index e38496ecf67..34e75755dec 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -12,12 +12,13 @@ module Gitlab private - attr_reader :project, :current_user, :lock_file + attr_reader :project, :current_user, :lock_file, :logger public def initialize(attributes = {}) @options = attributes + @logger = Gitlab::Export::Logger.build end def method_missing(method, *args) @@ -43,6 +44,10 @@ module Gitlab true rescue StandardError => e + payload = { message: "After export strategy failed" } + Gitlab::ExceptionLogFormatter.format!(e, payload) + log_error(payload) + project.import_export_shared.error(e) false ensure @@ -108,6 +113,18 @@ module Gitlab def log_validation_errors errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end + + def log_info(params) + logger.info(log_default_params.merge(params)) + end + + def log_error(params) + logger.error(log_default_params.merge(params)) + end + + def log_default_params + { project_name: project.name, project_id: project.id } + end end end end diff --git a/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb b/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb index 2e3136936f8..bddbe7862cb 100644 --- a/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/move_file_strategy.rb @@ -5,6 +5,7 @@ module Gitlab module AfterExportStrategies class MoveFileStrategy < BaseAfterExportStrategy def initialize(archive_path:) + super @archive_path = archive_path end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index 78608a946de..6c5fba37d7b 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -23,7 +23,17 @@ module Gitlab protected def strategy_execute - handle_response_error(send_file) + log_info(message: "Started uploading project", export_size: export_size) + + upload_duration = Benchmark.realtime do + if Feature.enabled?(:import_export_web_upload_stream) && !project.export_file.file_storage? + upload_project_as_remote_stream + else + handle_response_error(send_file) + end + end + + log_info(message: "Finished uploading project", export_size: export_size, upload_duration: upload_duration) end def handle_response_error(response) @@ -44,8 +54,22 @@ module Gitlab export_file.close if export_file end + def upload_project_as_remote_stream + Gitlab::ImportExport::RemoteStreamUpload.new( + download_url: project.export_file.url, + upload_url: url, + options: { + upload_method: http_method.downcase.to_sym, + upload_content_type: 'application/gzip' + }).execute + rescue Gitlab::ImportExport::RemoteStreamUpload::StreamError => e + log_error(message: e.message, response_body: e.response_body.truncate(3000)) + + raise + end + def export_file - project.export_file.open + @export_file ||= project.export_file.open end def send_file_options diff --git a/lib/gitlab/import_export/remote_stream_upload.rb b/lib/gitlab/import_export/remote_stream_upload.rb new file mode 100644 index 00000000000..f3bd241c0bd --- /dev/null +++ b/lib/gitlab/import_export/remote_stream_upload.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +# This class downloads a file from one URL and uploads it to another URL +# without having to save the file on the disk and loading the whole file in +# memory. The download and upload are performed in chunks size of +# `buffer_size`. A chunk is downloaded, then uploaded, then a next chunk is +# downloaded and uploaded. This repeats until all the file is processed. + +module Gitlab + module ImportExport + class RemoteStreamUpload + def initialize(download_url:, upload_url:, options: {}) + @download_url = download_url + @upload_url = upload_url + @upload_method = options[:upload_method] || :post + @upload_content_type = options[:upload_content_type] || 'application/gzip' + end + + def execute + receive_data(download_url) do |response, chunks| + send_data(upload_url, response.content_length, chunks) do |response| + if response.code != '200' + raise StreamError.new("Invalid response code while uploading file. Code: #{response.code}", response.body) + end + end + end + end + class StreamError < StandardError + attr_reader :response_body + + def initialize(message, response_body = '') + super(message) + @response_body = response_body + end + end + class ChunkStream + DEFAULT_BUFFER_SIZE = 128.kilobytes + + def initialize(chunks) + @chunks = chunks + @last_chunk = nil + @end_of_chunks = false + end + + def read(n1 = nil, n2 = nil) + ensure_chunk&.read(n1, n2) + end + + private + + def ensure_chunk + return @last_chunk if @last_chunk && !@last_chunk.eof? + return if @end_of_chunks + + @last_chunk = read_next_chunk + end + + def read_next_chunk + next_chunk = StringIO.new + + begin + next_chunk.write(@chunks.next) until next_chunk.size > DEFAULT_BUFFER_SIZE + rescue StopIteration + @end_of_chunks = true + end + + next_chunk.rewind + + next_chunk + end + end + + private + + attr_reader :download_url, :upload_url, :upload_method, :upload_content_type, :logger + + def receive_data(uri) + http = Gitlab::HTTPConnectionAdapter.new(URI(uri), {}).connection + + http.start do + request = Net::HTTP::Get.new(uri) + http.request(request) do |response| + if response.code == '200' + yield(response, response.enum_for(:read_body)) + else + raise StreamError.new( + "Invalid response code while downloading file. Code: #{response.code}", + response.body + ) + end + end + end + end + + def send_data(uri, content_length, chunks) + http = Gitlab::HTTPConnectionAdapter.new(URI(uri), {}).connection + + http.start do + request = upload_request_class(upload_method).new(uri) + request.body_stream = ChunkStream.new(chunks) + request.content_length = content_length + request.content_type = upload_content_type + + http.request(request) do |response| + yield(response) + end + end + end + + def upload_request_class(upload_method) + return Net::HTTP::Put if upload_method == :put + + Net::HTTP::Post + end + end + end +end |