diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 10:08:36 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-10-21 10:08:36 +0300 |
commit | 48aff82709769b098321c738f3444b9bdaa694c6 (patch) | |
tree | e00c7c43e2d9b603a5a6af576b1685e400410dee /lib/gitlab/import_export | |
parent | 879f5329ee916a948223f8f43d77fba4da6cd028 (diff) |
Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r-- | lib/gitlab/import_export/attribute_cleaner.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/base/relation_factory.rb | 30 | ||||
-rw-r--r-- | lib/gitlab/import_export/file_importer.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/ndjson_reader.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/lfs_saver.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export/members_mapper.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/import_export.yml | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/sample/date_calculator.rb | 37 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb | 51 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_restorer.rb | 10 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_saver.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/repo_restorer.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/import_export/saver.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_manager.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/version_checker.rb | 4 |
15 files changed, 149 insertions, 39 deletions
diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 018cb36fc58..379a734b19c 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -20,8 +20,8 @@ module Gitlab /\Aremote_\w+_(url|urls|request_header)\Z/ # carrierwave automatically creates these attribute methods for uploads ).freeze - def self.clean(*args) - new(*args).clean + def self.clean(*args, **kwargs) + new(*args, **kwargs).clean end def initialize(relation_hash:, relation_class:, excluded_keys: []) diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 05b69362976..d60bc79df4c 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -31,8 +31,8 @@ module Gitlab TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze - def self.create(*args) - new(*args).create + def self.create(*args, **kwargs) + new(*args, **kwargs).create end def self.relation_class(relation_name) @@ -53,6 +53,7 @@ module Gitlab @importable = importable @imported_object_retries = 0 @relation_hash[importable_column_name] = @importable.id + @original_user = {} # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' @@ -112,6 +113,7 @@ module Gitlab def update_user_references self.class::USER_REFERENCES.each do |reference| if @relation_hash[reference] + @original_user[reference] = @relation_hash[reference] @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]] end end @@ -243,28 +245,20 @@ module Gitlab # will be used. Otherwise, a note stating the original author name # is left. def set_note_author - old_author_id = @relation_hash['author_id'] + old_author_id = @original_user['author_id'] author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) unless has_author?(old_author_id) - end - - def has_author?(old_author_id) - admin_user? && @members_mapper.include?(old_author_id) + unless @members_mapper.include?(old_author_id) + @relation_hash['note'] = "%{note}\n\n %{missing_author_note}" % { + note: @relation_hash['note'].presence || '*Blank note*', + missing_author_note: missing_author_note(@relation_hash['updated_at'], author['name']) + } + end end def missing_author_note(updated_at, author_name) timestamp = updated_at.split('.').first - "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" - end - - def update_note_for_missing_author(author_name) - @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank? - @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}" - end - - def admin_user? - @user.admin? + "*By #{author_name} on #{timestamp} (imported from GitLab)*" end def existing_object? diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 081745a49f4..5a6f6e017d2 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -10,8 +10,8 @@ module Gitlab MAX_RETRIES = 8 IGNORED_FILENAMES = %w(. ..).freeze - def self.import(*args) - new(*args).import + def self.import(*args, **kwargs) + new(*args, **kwargs).import end def initialize(importable:, archive_file:, shared:) diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index e9b05afc7d4..0d9839b86cf 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -35,6 +35,7 @@ module Gitlab # This reads from `tree/project/merge_requests.ndjson` path = file_path(importable_path, "#{key}.ndjson") + next unless File.exist?(path) File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num| @@ -43,6 +44,11 @@ module Gitlab end end + # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) + def clear_consumed_relations + @consumed_relations.clear + end + private def json_decode(string) diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 515fd98630c..4964b8b16f4 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -21,10 +21,10 @@ module Gitlab save_lfs_object(lfs_object) end - append_lfs_json_for_batch(batch) if write_lfs_json_enabled? + append_lfs_json_for_batch(batch) end - write_lfs_json if write_lfs_json_enabled? + write_lfs_json true rescue => e @@ -35,10 +35,6 @@ module Gitlab private - def write_lfs_json_enabled? - ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true) - end - def save_lfs_object(lfs_object) if lfs_object.local_store? copy_file_for_lfs_object(lfs_object) diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 31d1f7b48bd..6b37683ea68 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -34,8 +34,8 @@ module Gitlab @user.id end - def include?(old_author_id) - map.has_key?(old_author_id) && map[old_author_id] != default_user_id + def include?(old_user_id) + map.has_key?(old_user_id) end private @@ -63,6 +63,8 @@ module Gitlab end def add_team_member(member, existing_user = nil) + return true if existing_user && @importable.members.exists?(user_id: existing_user.id) + member['user'] = existing_user member_hash = member_hash(member) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 9ec5df8cde9..a0526ba0414 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -403,9 +403,15 @@ ee: - issues: - epic_issue: - :epic + - :issuable_sla - protected_branches: - :unprotect_access_levels - protected_environments: - :deploy_access_levels - :service_desk_setting - :security_setting + + included_attributes: + issuable_sla: + - :issue + - :due_at diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb new file mode 100644 index 00000000000..2d989d21166 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/date_calculator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class DateCalculator + include Gitlab::Utils::StrongMemoize + + def initialize(dates) + @dates = dates.dup + @dates.flatten! + @dates.compact! + @dates.sort! + @dates.map! { |date| date.to_time.to_f } + end + + def closest_date_to_average + strong_memoize(:closest_date_to_average) do + next if @dates.empty? + + average_date = (@dates.first + @dates.last) / 2.0 + closest_date = @dates.min_by { |date| (date - average_date).abs } + Time.zone.at(closest_date) + end + end + + def calculate_by_closest_date_to_average(date) + return date unless closest_date_to_average && closest_date_to_average < Time.current + + date + (Time.current - closest_date_to_average).seconds + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb new file mode 100644 index 00000000000..b0c3940b5f9 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class SampleDataRelationTreeRestorer < RelationTreeRestorer + DATE_MODELS = %i[issues milestones].freeze + + def initialize(*args) + super + + date_calculator + end + + private + + def build_relation(relation_key, relation_definition, data_hash) + # Override due date attributes in data hash for Sample Data templates + # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import + # TODO: To move this logic to RelationFactory (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465333) + override_date_attributes!(relation_key, data_hash) + super + end + + def override_date_attributes!(relation_key, data_hash) + return unless DATE_MODELS.include?(relation_key.to_sym) + + data_hash['start_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['start_date'].to_time) unless data_hash['start_date'].nil? + data_hash['due_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['due_date'].to_time) unless data_hash['due_date'].nil? + end + + # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) + def dates + unless relation_reader.legacy? + DATE_MODELS.map do |tag| + relation_reader.consume_relation(@importable_path, tag).map { |model| model.first['due_date'] }.tap do + relation_reader.clear_consumed_relations + end + end + end + end + + def date_calculator + @date_calculator ||= Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index a16ffe36054..b1d647281ab 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -70,7 +70,7 @@ module Gitlab end def relation_tree_restorer - @relation_tree_restorer ||= RelationTreeRestorer.new( + @relation_tree_restorer ||= relation_tree_restorer_class.new( user: @user, shared: @shared, relation_reader: relation_reader, @@ -84,6 +84,14 @@ module Gitlab ) end + def relation_tree_restorer_class + sample_data_template? ? Sample::SampleDataRelationTreeRestorer : RelationTreeRestorer + end + + def sample_data_template? + @project&.import_data&.data&.dig('sample_data') + end + def members_mapper @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, user: @user, diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 7cca3596da6..80dacf2eb20 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -36,7 +36,7 @@ module Gitlab end def exportable - @project.present(exportable_params) + @project.present(**exportable_params) end def exportable_params diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 9e10e7aea13..f808e30bd6e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -14,10 +14,10 @@ module Gitlab def restore return true unless File.exist?(path_to_bundle) + ensure_repository_does_not_exist! + repository.create_from_bundle(path_to_bundle) rescue => e - Repositories::DestroyService.new(repository).execute - shared.error(e) false end @@ -25,6 +25,16 @@ module Gitlab private attr_accessor :repository, :path_to_bundle, :shared + + def ensure_repository_does_not_exist! + if repository.exists? + shared.logger.info( + message: %Q{Deleting existing "#{repository.path}" to re-import it.} + ) + + Repositories::DestroyService.new(repository).execute + end + end end end end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index e4724659eff..045ba2495bf 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -5,8 +5,8 @@ module Gitlab class Saver include Gitlab::ImportExport::CommandLineUtil - def self.save(*args) - new(*args).save + def self.save(*args, **kwargs) + new(*args, **kwargs).save end def initialize(exportable:, shared:) diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index dca8e3a7449..26e7d2cf765 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -40,7 +40,7 @@ module Gitlab def add_upload(upload) uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys - UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute.to_h + UploadService.new(@project, File.open(upload, 'r'), FileUploader, **uploader_context).execute.to_h end def copy_project_uploads diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 4154d4fe775..48f5b558e52 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -3,8 +3,8 @@ module Gitlab module ImportExport class VersionChecker - def self.check!(*args) - new(*args).check! + def self.check!(*args, **kwargs) + new(*args, **kwargs).check! end def initialize(shared:) |