diff options
Diffstat (limited to 'lib/gitlab/import_export')
8 files changed, 31 insertions, 32 deletions
diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index 986191bdb6b..62bd650c1d5 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -15,7 +15,6 @@ module Gitlab include Gitlab::Utils::StrongMemoize BATCH_SIZE = 100 - MIN_RECORDS_SIZE = 1 attr_reader :invalid_subrelations @@ -82,7 +81,7 @@ module Gitlab subrelation = relation_object.public_send(definition) association = relation_object.class.reflect_on_association(definition) - next unless association&.collection? && subrelation.size > MIN_RECORDS_SIZE + next unless association&.collection? collection_subrelations[definition] = subrelation.records diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 924ca4e83ea..dfe0815f0a0 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -37,7 +37,7 @@ module Gitlab def gzip_with_options(dir:, filename:, options: nil) filepath = File.join(dir, filename) - cmd = %W(gzip #{filepath}) + cmd = %W[gzip #{filepath}] cmd << "-#{options}" if options _, status = Gitlab::Popen.popen(cmd) @@ -68,6 +68,8 @@ module Gitlab File.open(upload_path, 'wb') do |file| current_size = 0 + # When migrating from Gitlab::HTTP to Gitlab:HTTP_V2, we need to pass `extra_allowed_uris` as an option + # instead of `allow_object_storage`. Gitlab::HTTP.get(url, stream_body: true, allow_object_storage: true) do |fragment| if [301, 302, 303, 307].include?(fragment.code) Gitlab::Import::Logger.warn(message: "received redirect fragment", fragment_code: fragment.code) @@ -87,12 +89,12 @@ module Gitlab end def tar_with_options(archive:, dir:, options:) - execute_cmd(%W(tar -#{options} #{archive} -C #{dir} .)) + execute_cmd(%W[tar -#{options} #{archive} -C #{dir} .]) end def untar_with_options(archive:, dir:, options:) - execute_cmd(%W(tar -#{options} #{archive} -C #{dir})) - execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir})) + execute_cmd(%W[tar -#{options} #{archive} -C #{dir}]) + execute_cmd(%W[chmod -R #{UNTAR_MASK} #{dir}]) clean_extraction_dir!(dir) end diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 3609df89958..13510cb43ca 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -5,13 +5,10 @@ module Gitlab class DecompressedArchiveSizeValidator include Gitlab::Utils::StrongMemoize - TIMEOUT_LIMIT = 210.seconds - ServiceError = Class.new(StandardError) - def initialize(archive_path:, max_bytes: self.class.max_bytes) + def initialize(archive_path:) @archive_path = archive_path - @max_bytes = max_bytes end def valid? @@ -20,10 +17,6 @@ module Gitlab end end - def self.max_bytes - Gitlab::CurrentSettings.current_application_settings.max_decompressed_archive_size.megabytes - end - private def validate @@ -32,7 +25,7 @@ module Gitlab validate_archive_path - Timeout.timeout(TIMEOUT_LIMIT) do + Timeout.timeout(timeout) do stderr_r, stderr_w = IO.pipe stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w) @@ -51,7 +44,7 @@ module Gitlab if status.success? result = stdout.readline - if @max_bytes > 0 && result.to_i > @max_bytes + if max_bytes > 0 && result.to_i > max_bytes valid_archive = false log_error('Decompressed archive size limit reached') @@ -70,7 +63,7 @@ module Gitlab valid_archive rescue Timeout::Error - log_error('Timeout reached during archive decompression') + log_error("Timeout of #{timeout} seconds reached during archive decompression") pgrps.each { |pgrp| Process.kill(-1, pgrp) } if pgrps @@ -107,6 +100,14 @@ module Gitlab import_upload_archive_size: archive_size ) end + + def timeout + Gitlab::CurrentSettings.current_application_settings.decompress_archive_file_timeout + end + + def max_bytes + Gitlab::CurrentSettings.current_application_settings.max_decompressed_archive_size.megabytes + end end end end diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb index e303ac6eefa..60ae163cad5 100644 --- a/lib/gitlab/import_export/json/ndjson_writer.rb +++ b/lib/gitlab/import_export/json/ndjson_writer.rb @@ -44,12 +44,11 @@ module Gitlab def with_file(*path) file_path = File.join(@dir_path, *path) - raise ArgumentError, "The #{file_path} already exist" if File.exist?(file_path) # ensure that path is created mkdir_p(File.dirname(file_path)) - File.open(file_path, "wb") do |file| + File.open(file_path, "ab+") do |file| yield(file) end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 2c64ca53f76..b0951f24628 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -75,10 +75,12 @@ module Gitlab def serialize_many_relations(key, records, options) log_relation_export(key, records.size) - enumerator = Enumerator.new do |items| - key_preloads = preloads&.dig(key) + key_preloads = preloads&.dig(key) + + batch(records, key) do |batch| + next if batch.empty? - batch(records, key) do |batch| + batch_enumerator = Enumerator.new do |items| batch = batch.preload(key_preloads) if key_preloads batch.each do |record| @@ -91,9 +93,11 @@ module Gitlab after_read_callback(record) end end - end - json_writer.write_relation_array(@exportable_path, key, enumerator) + json_writer.write_relation_array(@exportable_path, key, batch_enumerator) + + Gitlab::SafeRequestStore.clear! + end end def exportable_json_record(record, options, key) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 850c89c1fb1..1b04e55e0c7 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -116,7 +116,6 @@ tree: - :project_badges - :ci_cd_settings - :error_tracking_setting - - :metrics_setting - boards: - lists: - label: @@ -156,10 +155,6 @@ included_attributes: - :group_runners_enabled - :runner_token_expiration_interval - :default_git_depth - metrics_setting: - - :dashboard_timezone - - :external_dashboard_url - - :project_id project_badges: - :created_at - :image_url diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 7af65235492..840621a94a2 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -38,7 +38,6 @@ module Gitlab ci_cd_settings: 'ProjectCiCdSetting', error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', links: 'Releases::Link', - metrics_setting: 'ProjectMetricsSetting', commit_author: 'MergeRequest::DiffCommitUser', committer: 'MergeRequest::DiffCommitUser', merge_request_diff_commits: 'MergeRequestDiffCommit', @@ -180,7 +179,7 @@ module Gitlab # When author is not present for source release set the author as ghost user. if @relation_hash['author_id'].blank? - @relation_hash['author_id'] = User.select(:id).ghost.id + @relation_hash['author_id'] = Users::Internal.ghost.id end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index d7d262501de..1861a92100e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -42,7 +42,7 @@ module Gitlab def ensure_repository_does_not_exist! if repository.exists? shared.logger.info( - message: %{Deleting existing "#{repository.disk_path}" to re-import it.} + message: %(Deleting existing "#{repository.disk_path}" to re-import it.) ) Repositories::DestroyService.new(repository).execute |