diff options
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r-- | lib/gitlab/import_export/attributes_finder.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/attributes_permitter.rb | 105 | ||||
-rw-r--r-- | lib/gitlab/import_export/importer.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/streaming_serializer.rb | 15 | ||||
-rw-r--r-- | lib/gitlab/import_export/legacy_relation_tree_saver.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export/members_mapper.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/merge_request_parser.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/import_export.yml | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/repo_restorer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/saver.rb | 21 | ||||
-rw-r--r-- | lib/gitlab/import_export/snippet_repo_restorer.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/import_export/snippets_repo_restorer.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/version_checker.rb | 6 |
13 files changed, 180 insertions, 28 deletions
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index bab473741b1..1e98595bb07 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,6 +3,8 @@ module Gitlab module ImportExport class AttributesFinder + attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads + def initialize(config:) @tree = config[:tree] || {} @included_attributes = config[:included_attributes] || {} diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb new file mode 100644 index 00000000000..86f51add504 --- /dev/null +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +# AttributesPermitter builds a hash of permitted attributes for +# every model defined in import_export.yml that is used to validate and +# filter out any attributes that are not permitted when doing Project/Group Import +# +# Each model's list includes: +# - attributes defined under included_attributes section +# - associations defined under project/group tree +# - methods defined under methods section +# +# Given the following import_export.yml example: +# ``` +# tree: +# project: +# - labels: +# - :priorities +# included_attributes: +# labels: +# - :title +# - :description +# methods: +# labels: +# - :type +# ``` +# +# Produces a list of permitted attributes: +# ``` +# Gitlab::ImportExport::AttributesPermitter.new.permitted_attributes +# +# => { labels: [:priorities, :title, :description, :type] } +# ``` +# +# Filters out any other attributes from specific relation hash: +# ``` +# Gitlab::ImportExport::AttributesPermitter.new.permit(:labels, {id: 5, type: 'opened', description: 'test', sensitive_attribute: 'my_sensitive_attribute'}) +# +# => {:type=>"opened", :description=>"test"} +# ``` +module Gitlab + module ImportExport + class AttributesPermitter + attr_reader :permitted_attributes + + def initialize(config: ImportExport::Config.new.to_h) + @config = config + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config) + @permitted_attributes = {} + + build_permitted_attributes + end + + def permit(relation_name, relation_hash) + permitted_attributes = permitted_attributes_for(relation_name) + + relation_hash.select do |key, _| + permitted_attributes.include?(key) + end + end + + def permitted_attributes_for(relation_name) + @permitted_attributes[relation_name] || [] + end + + private + + def build_permitted_attributes + build_associations + build_attributes + build_methods + end + + # Deep traverse relations tree to build a list of allowed model relations + def build_associations + stack = @attributes_finder.tree.to_a + + while stack.any? + model_name, relations = stack.pop + + if relations.is_a?(Hash) + add_permitted_attributes(model_name, relations.keys) + + stack.concat(relations.to_a) + end + end + + @permitted_attributes + end + + def build_attributes + @attributes_finder.included_attributes.each(&method(:add_permitted_attributes)) + end + + def build_methods + @attributes_finder.methods.each(&method(:add_permitted_attributes)) + end + + def add_permitted_attributes(model_name, attributes) + @permitted_attributes[model_name] ||= [] + + @permitted_attributes[model_name].concat(attributes) if attributes.any? + end + end + end +end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index b1219384732..7b8689069d8 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -24,8 +24,14 @@ module Gitlab raise Projects::ImportService::Error.new(shared.errors.to_sentence) end rescue => e + # If some exception was raised could mean that the SnippetsRepoRestorer + # was not called. This would leave us with snippets without a repository. + # This is a state we don't want them to be, so we better delete them. + remove_non_migrated_snippets + raise Projects::ImportService::Error.new(e.message) ensure + remove_base_tmp_dir remove_import_file end @@ -148,6 +154,18 @@ module Gitlab ::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") end end + + def remove_base_tmp_dir + FileUtils.rm_rf(@shared.base_path) + end + + def remove_non_migrated_snippets + project + .snippets + .left_joins(:snippet_repository) + .where(snippet_repositories: { snippet_id: nil }) + .delete_all + end end end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 7f55a0a3821..20f9c668b9c 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -7,6 +7,15 @@ module Gitlab include Gitlab::ImportExport::CommandLineUtil BATCH_SIZE = 100 + SMALLER_BATCH_SIZE = 20 + + def self.batch_size(exportable) + if Feature.enabled?(:export_reduce_relation_batch_size, exportable) + SMALLER_BATCH_SIZE + else + BATCH_SIZE + end + end class Raw < String def to_json(*_args) @@ -60,7 +69,7 @@ module Gitlab key_preloads = preloads&.dig(key) records = records.preload(key_preloads) if key_preloads - records.find_each(batch_size: BATCH_SIZE) do |record| + records.find_each(batch_size: batch_size) do |record| items << Raw.new(record.to_json(options)) end end @@ -91,6 +100,10 @@ module Gitlab def preloads relations_schema[:preload] end + + def batch_size + @batch_size ||= self.class.batch_size(@exportable) + end end end end diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index cf75a2c7fa8..f8b8b74ffd7 100644 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -7,7 +7,7 @@ module Gitlab def serialize(exportable, relations_tree) Gitlab::ImportExport::FastHashSerializer - .new(exportable, relations_tree) + .new(exportable, relations_tree, batch_size: batch_size(exportable)) .execute end @@ -18,6 +18,12 @@ module Gitlab File.write(File.join(dir_path, filename), tree_json) end + + private + + def batch_size(exportable) + Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable) + end end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 263c49c509f..31d1f7b48bd 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -49,7 +49,7 @@ module Gitlab def ensure_default_member! return if user_already_member? - @importable.members.destroy_all # rubocop: disable DestroyAll + @importable.members.destroy_all # rubocop: disable Cop/DestroyAll relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true) rescue => e diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index f735b9612aa..4643742b607 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -41,7 +41,13 @@ module Gitlab def create_source_branch @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) rescue => err - Rails.logger.warn("Import/Export warning: Failed to create source branch #{@merge_request.source_branch} => #{@diff_head_sha} for MR #{@merge_request.iid}: #{err}") # rubocop:disable Gitlab/RailsLogger + Gitlab::Import::Logger.warn( + message: 'Import warning: Failed to create source branch', + source_branch: @merge_request.source_branch, + diff_head_sha: @diff_head_sha, + merge_request_iid: @merge_request.iid, + error: err.message + ) end def create_target_branch diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 8851b106ad5..f0b733d7e95 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -312,6 +312,7 @@ excluded_attributes: - :pipeline_schedule_id - :merge_request_id - :external_pull_request_id + - :ci_ref_id stages: - :pipeline_id merge_access_levels: @@ -397,3 +398,4 @@ ee: - protected_environments: - :deploy_access_levels - :service_desk_setting + - :security_setting diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 3123687453f..9e10e7aea13 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -16,6 +16,8 @@ module Gitlab repository.create_from_bundle(path_to_bundle) rescue => e + Repositories::DestroyService.new(repository).execute + shared.error(e) false end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index ae82c380755..e4724659eff 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -11,14 +11,16 @@ module Gitlab def initialize(exportable:, shared:) @exportable = exportable - @shared = shared + @shared = shared end def save if compress_and_save - remove_export_path - - Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger + Gitlab::Export::Logger.info( + message: 'Export archive saved', + exportable_class: @exportable.class.to_s, + archive_file: archive_file + ) save_upload else @@ -29,8 +31,7 @@ module Gitlab @shared.error(e) false ensure - remove_archive - remove_export_path + remove_base_tmp_dir end private @@ -39,12 +40,8 @@ module Gitlab tar_czf(archive: archive_file, dir: @shared.export_path) end - def remove_export_path - FileUtils.rm_rf(@shared.export_path) - end - - def remove_archive - FileUtils.rm_rf(@shared.archive_path) + def remove_base_tmp_dir + FileUtils.rm_rf(@shared.base_path) end def archive_file diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb index b58ea14a3a8..334d13a13ae 100644 --- a/lib/gitlab/import_export/snippet_repo_restorer.rb +++ b/lib/gitlab/import_export/snippet_repo_restorer.rb @@ -5,6 +5,8 @@ module Gitlab class SnippetRepoRestorer < RepoRestorer attr_reader :snippet + SnippetRepositoryError = Class.new(StandardError) + def initialize(snippet:, user:, shared:, path_to_bundle:) @snippet = snippet @user = user @@ -34,14 +36,11 @@ module Gitlab end def create_repository_from_db - snippet.create_repository - - commit_attrs = { - branch_name: 'master', - message: 'Initial commit' - } + Gitlab::BackgroundMigration::BackfillSnippetRepositories.new.perform_by_ids([snippet.id]) - repository.create_file(@user, snippet.file_name, snippet.content, commit_attrs) + unless snippet.reset.snippet_repository + raise SnippetRepositoryError, _("Error creating repository for snippet with id %{snippet_id}") % { snippet_id: snippet.id } + end end end end diff --git a/lib/gitlab/import_export/snippets_repo_restorer.rb b/lib/gitlab/import_export/snippets_repo_restorer.rb index 9ff3e74a6b1..5ab28f8dd83 100644 --- a/lib/gitlab/import_export/snippets_repo_restorer.rb +++ b/lib/gitlab/import_export/snippets_repo_restorer.rb @@ -10,15 +10,13 @@ module Gitlab end def restore - return true unless Dir.exist?(snippets_repo_bundle_path) - - @project.snippets.find_each.all? do |snippet| + @project.snippets.find_each.map do |snippet| Gitlab::ImportExport::SnippetRepoRestorer.new(snippet: snippet, user: @user, shared: @shared, path_to_bundle: snippet_repo_bundle_path(snippet)) .restore - end + end.all?(true) end private diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 86ea7a30e69..4154d4fe775 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -36,7 +36,11 @@ module Gitlab def different_version?(version) Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) rescue => e - Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::Import::Logger.error( + message: 'Import error', + error: e.message + ) + raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') end end |