diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-20 11:43:02 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-10-20 11:43:02 +0300 |
commit | d9ab72d6080f594d0b3cae15f14b3ef2c6c638cb (patch) | |
tree | 2341ef426af70ad1e289c38036737e04b0aa5007 /lib/gitlab/import_export | |
parent | d6e514dd13db8947884cd58fe2a9c2a063400a9b (diff) |
Add latest changes from gitlab-org/gitlab@14-4-stable-eev14.4.0-rc42
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r-- | lib/gitlab/import_export/attributes_permitter.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/base/relation_factory.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/command_line_util.rb | 26 | ||||
-rw-r--r-- | lib/gitlab/import_export/group/relation_factory.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/streaming_serializer.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/import_export/merge_request_parser.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/import_export.yml | 140 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_restorer.rb | 18 |
8 files changed, 187 insertions, 14 deletions
diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index acd03d9ec20..2d8e25a9f70 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -44,7 +44,7 @@ module Gitlab # We want to use AttributesCleaner for these relations instead, in the future this should be removed to make sure # we are using AttributesPermitter for every imported relation. - DISABLED_RELATION_NAMES = %i[user author ci_cd_settings issuable_sla push_rule].freeze + DISABLED_RELATION_NAMES = %i[user author issuable_sla].freeze def initialize(config: ImportExport::Config.new.to_h) @config = config diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index a84efd1d240..6749ef4e276 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -183,7 +183,7 @@ module Gitlab def parsed_relation_hash strong_memoize(:parsed_relation_hash) do - if Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) && attributes_permitter.permitted_attributes_defined?(@relation_sym) + if use_attributes_permitter? && attributes_permitter.permitted_attributes_defined?(@relation_sym) attributes_permitter.permit(@relation_sym, @relation_hash) else Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, relation_class: relation_class) @@ -195,6 +195,10 @@ module Gitlab @attributes_permitter ||= Gitlab::ImportExport::AttributesPermitter.new end + def use_attributes_permitter? + Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 6c0b6de9e85..fdc4c22001f 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -14,6 +14,10 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end + def tar_cf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'cf') + end + def gzip(dir:, filename:) gzip_with_options(dir: dir, filename: filename) end @@ -59,19 +63,29 @@ module Gitlab end def tar_with_options(archive:, dir:, options:) - execute(%W(tar -#{options} #{archive} -C #{dir} .)) + execute_cmd(%W(tar -#{options} #{archive} -C #{dir} .)) end def untar_with_options(archive:, dir:, options:) - execute(%W(tar -#{options} #{archive} -C #{dir})) - execute(%W(chmod -R #{UNTAR_MASK} #{dir})) + execute_cmd(%W(tar -#{options} #{archive} -C #{dir})) + execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir})) end - def execute(cmd) + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def execute_cmd(cmd) output, status = Gitlab::Popen.popen(cmd) - @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status == 0 # rubocop:disable Gitlab/ModuleWithInstanceVariables - status == 0 + + return true if status == 0 + + if @shared.respond_to?(:error) + @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) + + false + else + raise Gitlab::ImportExport::Error, 'System call failed' + end end + # rubocop:enable Gitlab/ModuleWithInstanceVariables def git_bin_path Gitlab.config.git.bin_path diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index 91637161377..adbbd37ce10 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -36,6 +36,10 @@ module Gitlab @relation_hash['group_id'] = @importable.id end + + def use_attributes_permitter? + false + end end end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 9d28e1abeab..9ab8fa68d0e 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -171,7 +171,6 @@ module Gitlab def read_from_replica_if_available(&block) return yield unless ::Feature.enabled?(:load_balancing_for_export_workers, type: :development, default_enabled: :yaml) - return yield unless ::Gitlab::Database::LoadBalancing.enable? ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block) end diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 3910afef108..301e90e3171 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -39,7 +39,9 @@ module Gitlab # created manually. Ignore failures so we get the merge request itself if # the commits are missing. def create_source_branch - @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) + if @merge_request.open? + @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) + end rescue StandardError => err Gitlab::Import::Logger.warn( message: 'Import warning: Failed to create source branch', diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 8046fedc4f3..86fd11cc336 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -131,7 +131,6 @@ included_attributes: - :link_url - :name - :project_id - - :type - :updated_at pipeline_schedules: - :active @@ -155,6 +154,124 @@ included_attributes: - :enabled - :project_id - :updated_at + boards: + - :project_id + - :created_at + - :updated_at + - :group_id + - :weight + - :name + - :hide_backlog_list + - :hide_closed_list + lists: + - :list_type + - :position + - :created_at + - :updated_at + - :user_id + - :max_issue_count + - :max_issue_weight + - :limit_metric + custom_attributes: + - :created_at + - :updated_at + - :project_id + - :key + - :value + label: + - :title + - :color + - :project_id + - :group_id + - :created_at + - :updated_at + - :template + - :description + - :priority + labels: + - :title + - :color + - :project_id + - :group_id + - :created_at + - :updated_at + - :template + - :description + - :priority + priorities: + - :project_id + - :priority + - :created_at + - :updated_at + milestone: + - :iid + - :title + - :project_id + - :group_id + - :description + - :due_date + - :created_at + - :updated_at + - :start_date + - :state + milestones: + - :iid + - :title + - :project_id + - :group_id + - :description + - :due_date + - :created_at + - :updated_at + - :start_date + - :state + protected_branches: + - :project_id + - :name + - :created_at + - :updated_at + - :code_owner_approval_required + - :allow_force_push + protected_tags: + - :project_id + - :name + - :created_at + - :updated_at + create_access_levels: + - :access_level + - :created_at + - :updated_at + - :user_id + - :group_id + merge_access_levels: + - :access_level + - :created_at + - :updated_at + - :user_id + - :group_id + push_access_levels: + - :access_level + - :created_at + - :updated_at + - :user_id + - :group_id + releases: + - :tag + - :description + - :project_id + - :author_id + - :created_at + - :updated_at + - :name + - :sha + - :released_at + links: + - :url + - :name + - :created_at + - :updated_at + - :filepath + - :link_type # Do not include the following attributes for the models specified. excluded_attributes: @@ -498,6 +615,10 @@ ee: - :deploy_access_levels - :security_setting - :push_rule + - boards: + - :milestone + - lists: + - :milestone included_attributes: issuable_sla: @@ -519,3 +640,20 @@ ee: - :reject_unsigned_commits - :commit_committer_check - :regexp_uses_re2 + unprotect_access_levels: + - :access_level + - :user_id + - :group_id + deploy_access_levels: + - :created_at + - :updated_at + - :access_level + - :user_id + - :group_id + protected_environments: + - :project_id + - :group_id + - :name + - :created_at + - :updated_at + diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 8d93098a80a..1eeacafef53 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -37,7 +37,7 @@ module Gitlab ActiveRecord::Base.no_touching do update_params! - BulkInsertableAssociations.with_bulk_insert(enabled: @importable.instance_of?(::Project)) do + BulkInsertableAssociations.with_bulk_insert(enabled: project?) do fix_ci_pipelines_not_sorted_on_legacy_project_json! create_relations! end @@ -55,6 +55,10 @@ module Gitlab private + def project? + @importable.instance_of?(::Project) + end + # Loops through the tree of models defined in import_export.yml and # finds them in the imported JSON so they can be instantiated and saved # in the DB. The structure and relationships between models are guessed from @@ -75,7 +79,7 @@ module Gitlab def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) relation_object = build_relation(relation_key, relation_definition, relation_index, data_hash) return unless relation_object - return if importable_class == ::Project && group_model?(relation_object) + return if project? && group_model?(relation_object) relation_object.assign_attributes(importable_class_sym => @importable) @@ -114,7 +118,8 @@ module Gitlab excluded_keys: excluded_keys_for_relation(importable_class_sym)) @importable.assign_attributes(params) - @importable.drop_visibility_level! if importable_class == ::Project + + modify_attributes Gitlab::Timeless.timeless(@importable) do @importable.save! @@ -141,6 +146,13 @@ module Gitlab end end + def modify_attributes + return unless project? + + @importable.reconcile_shared_runners_setting! + @importable.drop_visibility_level! + end + def build_relations(relation_key, relation_definition, relation_index, data_hashes) data_hashes .map { |data_hash| build_relation(relation_key, relation_definition, relation_index, data_hash) } |