diff options
Diffstat (limited to 'lib/gitlab/import_export')
10 files changed, 130 insertions, 67 deletions
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 7b8689069d8..8e78f6e274a 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -55,9 +55,17 @@ module Gitlab end def project_tree - @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user, - shared: shared, - project: project) + @project_tree ||= project_tree_class.new(user: current_user, + shared: shared, + project: project) + end + + def project_tree_class + sample_data_template? ? Gitlab::ImportExport::Project::Sample::TreeRestorer : Gitlab::ImportExport::Project::TreeRestorer + end + + def sample_data_template? + project&.import_data&.data&.dig('sample_data') end def avatar_restorer diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 0d9839b86cf..5c8edd485e5 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -29,9 +29,9 @@ module Gitlab json_decode(data) end - def consume_relation(importable_path, key) + def consume_relation(importable_path, key, mark_as_consumed: true) Enumerator.new do |documents| - next unless @consumed_relations.add?("#{importable_path}/#{key}") + next if mark_as_consumed && !@consumed_relations.add?("#{importable_path}/#{key}") # This reads from `tree/project/merge_requests.ndjson` path = file_path(importable_path, "#{key}.ndjson") @@ -44,11 +44,6 @@ module Gitlab end end - # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) - def clear_consumed_relations - @consumed_relations.clear - end - private def json_decode(string) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index a0526ba0414..ae7ddbc5eba 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -185,6 +185,7 @@ excluded_attributes: - :secret - :encrypted_secret_token - :encrypted_secret_token_iv + - :repository_read_only merge_request_diff: - :external_diff - :stored_externally @@ -410,8 +411,25 @@ ee: - :deploy_access_levels - :service_desk_setting - :security_setting + - :push_rule included_attributes: issuable_sla: - :issue - :due_at + push_rule: + - :force_push_regex + - :delete_branch_regex + - :commit_message_regex + - :author_email_regex + - :file_name_regex + - :branch_name_regex + - :commit_message_negative_regex + - :max_file_size + - :deny_delete_tag + - :member_check + - :is_sample + - :prevent_secrets + - :reject_unsigned_commits + - :commit_committer_check + - :regexp_uses_re2 diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb index 2d989d21166..543fd25d883 100644 --- a/lib/gitlab/import_export/project/sample/date_calculator.rb +++ b/lib/gitlab/import_export/project/sample/date_calculator.rb @@ -9,7 +9,6 @@ module Gitlab def initialize(dates) @dates = dates.dup - @dates.flatten! @dates.compact! @dates.sort! @dates.map! { |date| date.to_time.to_f } diff --git a/lib/gitlab/import_export/project/sample/relation_factory.rb b/lib/gitlab/import_export/project/sample/relation_factory.rb new file mode 100644 index 00000000000..6e59174f9a3 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/relation_factory.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class RelationFactory < Project::RelationFactory + DATE_MODELS = %i[issues milestones].freeze + + def initialize(date_calculator:, **args) + super(**args) + + @date_calculator = date_calculator + end + + private + + def setup_models + super + + # Override due date attributes in data hash for Sample Data templates + # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import + override_date_attributes + end + + def override_date_attributes + return unless DATE_MODELS.include?(@relation_name) + + @relation_hash['start_date'] = calculate_by_closest_date(@relation_hash['start_date']&.to_time) + @relation_hash['due_date'] = calculate_by_closest_date(@relation_hash['due_date']&.to_time) + end + + def calculate_by_closest_date(date) + return unless date + + @date_calculator.calculate_by_closest_date_to_average(date) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb new file mode 100644 index 00000000000..44ccb67a531 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class RelationTreeRestorer < ImportExport::RelationTreeRestorer + def initialize(*args) + super + + @date_calculator = Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates) + end + + private + + def relation_factory_params(*args) + super.merge(date_calculator: @date_calculator) + end + + def dates + return [] if relation_reader.legacy? + + RelationFactory::DATE_MODELS.flat_map do |tag| + relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| + model.first['due_date'] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb deleted file mode 100644 index b0c3940b5f9..00000000000 --- a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Project - module Sample - class SampleDataRelationTreeRestorer < RelationTreeRestorer - DATE_MODELS = %i[issues milestones].freeze - - def initialize(*args) - super - - date_calculator - end - - private - - def build_relation(relation_key, relation_definition, data_hash) - # Override due date attributes in data hash for Sample Data templates - # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import - # TODO: To move this logic to RelationFactory (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465333) - override_date_attributes!(relation_key, data_hash) - super - end - - def override_date_attributes!(relation_key, data_hash) - return unless DATE_MODELS.include?(relation_key.to_sym) - - data_hash['start_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['start_date'].to_time) unless data_hash['start_date'].nil? - data_hash['due_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['due_date'].to_time) unless data_hash['due_date'].nil? - end - - # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330) - def dates - unless relation_reader.legacy? - DATE_MODELS.map do |tag| - relation_reader.consume_relation(@importable_path, tag).map { |model| model.first['due_date'] }.tap do - relation_reader.clear_consumed_relations - end - end - end - end - - def date_calculator - @date_calculator ||= Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates) - end - end - end - end - end -end diff --git a/lib/gitlab/import_export/project/sample/tree_restorer.rb b/lib/gitlab/import_export/project/sample/tree_restorer.rb new file mode 100644 index 00000000000..1d4b5328cb9 --- /dev/null +++ b/lib/gitlab/import_export/project/sample/tree_restorer.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + module Sample + class TreeRestorer < Project::TreeRestorer + def relation_tree_restorer_class + RelationTreeRestorer + end + + def relation_factory + RelationFactory + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index b1d647281ab..fb9e5be1877 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -85,11 +85,7 @@ module Gitlab end def relation_tree_restorer_class - sample_data_template? ? Sample::SampleDataRelationTreeRestorer : RelationTreeRestorer - end - - def sample_data_template? - @project&.import_data&.data&.dig('sample_data') + RelationTreeRestorer end def members_mapper diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 26e7d2cf765..428bcbe8dc5 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -86,6 +86,10 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) download_or_copy_upload(upload, upload_path) + rescue Errno::ENAMETOOLONG => e + # Do not fail entire project export if downloaded file has filename that exceeds 255 characters. + # Ignore raised exception, skip such upload, log the error and keep going with the export instead. + Gitlab::ErrorTracking.log_exception(e, project_id: @project.id) end end end |