diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-04-20 13:00:54 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-04-20 13:00:54 +0300 |
commit | 3cccd102ba543e02725d247893729e5c73b38295 (patch) | |
tree | f36a04ec38517f5deaaacb5acc7d949688d1e187 /lib/gitlab/import_export | |
parent | 205943281328046ef7b4528031b90fbda70c75ac (diff) |
Add latest changes from gitlab-org/gitlab@14-10-stable-eev14.10.0-rc42
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r-- | lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export/avatar_saver.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/import_export/command_line_util.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/duration_measuring.rb | 23 | ||||
-rw-r--r-- | lib/gitlab/import_export/fast_hash_serializer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/streaming_serializer.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/lfs_saver.rb | 19 | ||||
-rw-r--r-- | lib/gitlab/import_export/members_mapper.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_saver.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export/repo_saver.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export/snippets_repo_saver.rb | 15 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_saver.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/import_export/version_saver.rb | 11 |
13 files changed, 90 insertions, 44 deletions
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index b43d0a0c3eb..e38496ecf67 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -17,11 +17,11 @@ module Gitlab public def initialize(attributes = {}) - @options = OpenStruct.new(attributes) + @options = attributes + end - self.class.instance_eval do - def_delegators :@options, *attributes.keys - end + def method_missing(method, *args) + @options[method] end def execute(current_user, project) diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 7534ab5a9ce..db90886ad11 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -3,19 +3,23 @@ module Gitlab module ImportExport class AvatarSaver + include DurationMeasuring + def initialize(project:, shared:) @project = project @shared = shared end def save - return true unless @project.avatar.exists? + with_duration_measuring do + break true unless @project.avatar.exists? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared, - relative_export_path: 'avatar' - ).save + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared, + relative_export_path: 'avatar' + ).save + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2b0467d8779..64ef3dd4830 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -66,7 +66,7 @@ module Gitlab current_size = 0 Gitlab::HTTP.get(url, stream_body: true, allow_object_storage: true) do |fragment| - if [301, 302, 307].include?(fragment.code) + if [301, 302, 303, 307].include?(fragment.code) Gitlab::Import::Logger.warn(message: "received redirect fragment", fragment_code: fragment.code) elsif fragment.code == 200 current_size += fragment.bytesize diff --git a/lib/gitlab/import_export/duration_measuring.rb b/lib/gitlab/import_export/duration_measuring.rb new file mode 100644 index 00000000000..c192be6ae29 --- /dev/null +++ b/lib/gitlab/import_export/duration_measuring.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module DurationMeasuring + extend ActiveSupport::Concern + + included do + attr_reader :duration_s + + def with_duration_measuring + result = nil + + @duration_s = Benchmark.realtime do + result = yield + end + + result + end + end + end + end +end diff --git a/lib/gitlab/import_export/fast_hash_serializer.rb b/lib/gitlab/import_export/fast_hash_serializer.rb index e5d52f945b5..d049609187b 100644 --- a/lib/gitlab/import_export/fast_hash_serializer.rb +++ b/lib/gitlab/import_export/fast_hash_serializer.rb @@ -92,7 +92,7 @@ module Gitlab def simple_serialize subject.as_json( - tree.merge(include: nil, preloads: nil)) + tree.merge(include: nil, preloads: nil, unsafe: true)) end def serialize_includes diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 55b8c1d4531..ebabf537ce5 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -37,7 +37,7 @@ module Gitlab def serialize_root(exportable_path = @exportable_path) attributes = exportable.as_json( - relations_schema.merge(include: nil, preloads: nil)) + relations_schema.merge(include: nil, preloads: nil, unsafe: true)) json_writer.write_attributes(exportable_path, attributes) end @@ -145,8 +145,8 @@ module Gitlab arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert reverse_direction = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_ORDER_DIRECTIONS[direction] reverse_nulls_position = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position] - order_expression = ::Gitlab::Database.nulls_order(column, direction, nulls_position) - reverse_order_expression = ::Gitlab::Database.nulls_order(column, reverse_direction, reverse_nulls_position) + order_expression = arel_table[column].public_send(direction).public_send(nulls_position) # rubocop:disable GitlabSecurity/PublicSend + reverse_order_expression = arel_table[column].public_send(reverse_direction).public_send(reverse_nulls_position) # rubocop:disable GitlabSecurity/PublicSend ::Gitlab::Pagination::Keyset::Order.build([ ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 47acd49d529..22a7a8dd7cd 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class LfsSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring attr_accessor :lfs_json, :project, :shared @@ -16,17 +17,19 @@ module Gitlab end def save - project.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| - batch.each do |lfs_object| - save_lfs_object(lfs_object) - end + with_duration_measuring do + project.lfs_objects.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each do |lfs_object| + save_lfs_object(lfs_object) + end - append_lfs_json_for_batch(batch) - end + append_lfs_json_for_batch(batch) + end - write_lfs_json + write_lfs_json - true + true + end rescue StandardError => e shared.error(e) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index d3b1bb6a57d..b1f2a17d4b7 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -16,7 +16,7 @@ module Gitlab def map @map ||= begin - @exported_members.inject(missing_keys_tracking_hash) do |hash, member| + @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| if member['user'] old_user_id = member['user']['id'] existing_user_id = existing_users_email_map[get_email(member)] @@ -24,8 +24,6 @@ module Gitlab else add_team_member(member) end - - hash end end end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index aafed850afa..63c5afa9595 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -4,6 +4,8 @@ module Gitlab module ImportExport module Project class TreeSaver + include DurationMeasuring + attr_reader :full_path def initialize(project:, current_user:, shared:, params: {}, logger: Gitlab::Import::Logger) @@ -15,9 +17,11 @@ module Gitlab end def save - stream_export + with_duration_measuring do + stream_export - true + true + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index fae07039139..454e84bbc04 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class RepoSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring attr_reader :exportable, :shared @@ -13,9 +14,12 @@ module Gitlab end def save - return true unless repository_exists? # it's ok to have no repo + with_duration_measuring do + # it's ok to have no repo + break true unless repository_exists? - bundle_to_disk + bundle_to_disk + end end def repository diff --git a/lib/gitlab/import_export/snippets_repo_saver.rb b/lib/gitlab/import_export/snippets_repo_saver.rb index d3b0fe1c18c..ca0d38272e5 100644 --- a/lib/gitlab/import_export/snippets_repo_saver.rb +++ b/lib/gitlab/import_export/snippets_repo_saver.rb @@ -4,6 +4,7 @@ module Gitlab module ImportExport class SnippetsRepoSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring def initialize(current_user:, project:, shared:) @project = project @@ -12,13 +13,15 @@ module Gitlab end def save - create_snippets_repo_directory + with_duration_measuring do + create_snippets_repo_directory - @project.snippets.find_each.all? do |snippet| - Gitlab::ImportExport::SnippetRepoSaver.new(project: @project, - shared: @shared, - repository: snippet.repository) - .save + @project.snippets.find_each.all? do |snippet| + Gitlab::ImportExport::SnippetRepoSaver.new(project: @project, + shared: @shared, + repository: snippet.repository) + .save + end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 9f58609fa17..05132fd3edd 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -3,16 +3,20 @@ module Gitlab module ImportExport class UploadsSaver + include DurationMeasuring + def initialize(project:, shared:) @project = project @shared = shared end def save - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).save + with_duration_measuring do + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).save + end rescue StandardError => e @shared.error(e) false diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index e8f68f93af0..db5040ec0f6 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -4,17 +4,20 @@ module Gitlab module ImportExport class VersionSaver include Gitlab::ImportExport::CommandLineUtil + include DurationMeasuring def initialize(shared:) @shared = shared end def save - mkdir_p(@shared.export_path) + with_duration_measuring do + mkdir_p(@shared.export_path) - File.write(version_file, Gitlab::ImportExport.version, mode: 'w') - File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') - File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') + File.write(version_file, Gitlab::ImportExport.version, mode: 'w') + File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') + File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') + end rescue StandardError => e @shared.error(e) false |