diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/ci/config/entry/environment.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | lib/gitlab/database.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/import_export/file_importer.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/import_export/importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/members_mapper.rb | 33 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 223 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_restorer.rb | 243 |
8 files changed, 315 insertions, 220 deletions
diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 68254552f82..fc62cca58ff 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -10,7 +10,7 @@ module Gitlab class Environment < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name url action on_stop kubernetes].freeze + ALLOWED_KEYS = %i[name url action on_stop auto_stop_in kubernetes].freeze entry :kubernetes, Entry::Kubernetes, description: 'Kubernetes deployment configuration.' @@ -49,6 +49,7 @@ module Gitlab validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true + validates :auto_stop_in, duration: true, allow_nil: true end end @@ -80,6 +81,10 @@ module Gitlab value[:kubernetes] end + def auto_stop_in + value[:auto_stop_in] + end + def value case @config when String then { name: @config, action: 'start' } diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index a37714eeed3..bdbce9edd97 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -50,6 +50,7 @@ dependency_scanning: DS_PIP_DEPENDENCY_PATH \ PIP_INDEX_URL \ PIP_EXTRA_INDEX_URL \ + PIP_REQUIREMENTS_FILE \ MAVEN_CLI_OPTS \ BUNDLER_AUDIT_UPDATE_DISABLED \ ) \ diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 50e23681de0..ceab9322857 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -95,6 +95,10 @@ module Gitlab version.to_f >= 9.6 end + def self.upsert_supported? + version.to_f >= 9.5 + end + # map some of the function names that changed between PostgreSQL 9 and 10 # https://wiki.postgresql.org/wiki/New_in_postgres_10 def self.pg_wal_lsn_diff @@ -158,7 +162,9 @@ module Gitlab # disable_quote - A key or an Array of keys to exclude from quoting (You # become responsible for protection from SQL injection for # these keys!) - def self.bulk_insert(table, rows, return_ids: false, disable_quote: []) + # on_conflict - Defines an upsert. Values can be: :disabled (default) or + # :do_nothing + def self.bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) return if rows.empty? keys = rows.first.keys @@ -176,10 +182,12 @@ module Gitlab VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} EOF - if return_ids - sql = "#{sql}RETURNING id" + if upsert_supported? && on_conflict == :do_nothing + sql = "#{sql} ON CONFLICT DO NOTHING" end + sql = "#{sql} RETURNING id" if return_ids + result = connection.execute(sql) if return_ids diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 2fd12e3aa78..9d04d55770d 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -5,6 +5,8 @@ module Gitlab class FileImporter include Gitlab::ImportExport::CommandLineUtil + ImporterError = Class.new(StandardError) + MAX_RETRIES = 8 IGNORED_FILENAMES = %w(. ..).freeze @@ -12,8 +14,8 @@ module Gitlab new(*args).import end - def initialize(project:, archive_file:, shared:) - @project = project + def initialize(importable:, archive_file:, shared:) + @importable = importable @archive_file = archive_file @shared = shared end @@ -52,7 +54,7 @@ module Gitlab def decompress_archive result = untar_zxf(archive: @archive_file, dir: @shared.export_path) - raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + raise ImporterError.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result result end @@ -60,9 +62,9 @@ module Gitlab def copy_archive return if @archive_file - @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project)) + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable)) - download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) + download_or_copy_upload(@importable.import_export_upload.import_file, @archive_file) end def remove_symlinks diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 62cf6c86906..a6463ed678c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -39,7 +39,7 @@ module Gitlab end def import_file - Gitlab::ImportExport::FileImporter.import(project: project, + Gitlab::ImportExport::FileImporter.import(importable: project, archive_file: archive_file, shared: shared) end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 4e976cfca3a..d2e27388b51 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -3,10 +3,10 @@ module Gitlab module ImportExport class MembersMapper - def initialize(exported_members:, user:, project:) + def initialize(exported_members:, user:, importable:) @exported_members = user.admin? ? exported_members : [] @user = user - @project = project + @importable = importable # This needs to run first, as second call would be from #map # which means project members already exist. @@ -19,7 +19,7 @@ module Gitlab @exported_members.inject(missing_keys_tracking_hash) do |hash, member| if member['user'] old_user_id = member['user']['id'] - existing_user = User.where(find_project_user_query(member)).first + existing_user = User.where(find_user_query(member)).first hash[old_user_id] = existing_user.id if existing_user && add_team_member(member, existing_user) else add_team_member(member) @@ -47,39 +47,48 @@ module Gitlab end def ensure_default_member! - @project.project_members.destroy_all # rubocop: disable DestroyAll + @importable.members.destroy_all # rubocop: disable DestroyAll - ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true) + relation_class.create!(user: @user, access_level: relation_class::MAINTAINER, source_id: @importable.id, importing: true) rescue => e - raise e, "Error adding importer user to project members. #{e.message}" + raise e, "Error adding importer user to #{@importable.class} members. #{e.message}" end def add_team_member(member, existing_user = nil) member['user'] = existing_user - ProjectMember.create(member_hash(member)).persisted? + relation_class.create(member_hash(member)).persisted? end def member_hash(member) parsed_hash(member).merge( - 'source_id' => @project.id, + 'source_id' => @importable.id, 'importing' => true, - 'access_level' => [member['access_level'], ProjectMember::MAINTAINER].min + 'access_level' => [member['access_level'], relation_class::MAINTAINER].min ).except('user_id') end def parsed_hash(member) - Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, - relation_class: ProjectMember) + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + relation_class: relation_class) end - def find_project_user_query(member) + def find_user_query(member) user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username'])) end def user_arel @user_arel ||= User.arel_table end + + def relation_class + case @importable + when Project + ProjectMember + when Group + GroupMember + end + end end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 488e8e0fcea..e274b68a94f 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -3,9 +3,6 @@ module Gitlab module ImportExport class ProjectTreeRestorer - # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone].freeze - attr_reader :user attr_reader :shared attr_reader :project @@ -13,34 +10,23 @@ module Gitlab def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user - @shared = shared + @shared = shared @project = project end def restore - begin - @tree_hash = read_tree_hash - rescue => e - Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - + @tree_hash = read_tree_hash @project_members = @tree_hash.delete('project_members') RelationRenameService.rename(@tree_hash) - ActiveRecord::Base.uncached do - ActiveRecord::Base.no_touching do - update_project_params! - create_project_relations! - post_import! - end - end - - # ensure that we have latest version of the restore - @project.reload # rubocop:disable Cop/ActiveRecordAssociationReload + if relation_tree_restorer.restore + @project.merge_requests.set_latest_merge_request_diff_ids! - true + true + else + false + end rescue => e @shared.error(e) false @@ -51,195 +37,36 @@ module Gitlab def read_tree_hash json = IO.read(@path) ActiveSupport::JSON.decode(json) - end - - def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, - user: @user, - project: @project) - end - - # A Hash of the imported merge request ID -> imported ID. - def merge_requests_mapping - @merge_requests_mapping ||= {} - end - - # Loops through the tree of models defined in import_export.yml and - # finds them in the imported JSON so they can be instantiated and saved - # in the DB. The structure and relationships between models are guessed from - # the configuration yaml file too. - # Finally, it updates each attribute in the newly imported project. - def create_project_relations! - project_relations.each(&method( - :process_project_relation!)) - end - - def post_import! - @project.merge_requests.set_latest_merge_request_diff_ids! - end - - def process_project_relation!(relation_key, relation_definition) - data_hashes = @tree_hash.delete(relation_key) - return unless data_hashes - - # we do not care if we process array or hash - data_hashes = [data_hashes] unless data_hashes.is_a?(Array) - - relation_index = 0 - - # consume and remove objects from memory - while data_hash = data_hashes.shift - process_project_relation_item!(relation_key, relation_definition, relation_index, data_hash) - relation_index += 1 - end - end - - def process_project_relation_item!(relation_key, relation_definition, relation_index, data_hash) - relation_object = build_relation(relation_key, relation_definition, data_hash) - return unless relation_object - return if group_model?(relation_object) - - relation_object.project = @project - relation_object.save! - - save_id_mapping(relation_key, data_hash, relation_object) rescue => e - # re-raise if not enabled - raise e unless Feature.enabled?(:import_graceful_failures, @project.group, default_enabled: true) - - log_import_failure(relation_key, relation_index, e) - end - - def log_import_failure(relation_key, relation_index, exception) - Gitlab::Sentry.track_acceptable_exception(exception, - extra: { project_id: @project.id, relation_key: relation_key, relation_index: relation_index }) - - ImportFailure.create( - project: @project, - relation_key: relation_key, - relation_index: relation_index, - exception_class: exception.class.to_s, - exception_message: exception.message.truncate(255), - correlation_id_value: Labkit::Correlation::CorrelationId.current_id - ) + Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') end - # Older, serialized CI pipeline exports may only have a - # merge_request_id and not the full hash of the merge request. To - # import these pipelines, we need to preserve the mapping between - # the old and new the merge request ID. - def save_id_mapping(relation_key, data_hash, relation_object) - return unless relation_key == 'merge_requests' - - merge_requests_mapping[data_hash['id']] = relation_object.id - end - - def project_relations - @project_relations ||= - reader - .attributes_finder - .find_relations_tree(:project) - .deep_stringify_keys - end - - def update_project_params! - project_params = @tree_hash.reject do |key, value| - project_relations.include?(key) - end - - project_params = project_params.merge( - present_project_override_params) - - # Cleaning all imported and overridden params - project_params = Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: project_params, - relation_class: Project, - excluded_keys: excluded_keys_for_relation(:project)) - - @project.assign_attributes(project_params) - @project.drop_visibility_level! - - Gitlab::Timeless.timeless(@project) do - @project.save! - end - end - - def present_project_override_params - # we filter out the empty strings from the overrides - # keeping the default values configured - project_override_params.transform_values do |value| - value.is_a?(String) ? value.presence : value - end.compact - end - - def project_override_params - @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {} - end - - def build_relations(relation_key, relation_definition, data_hashes) - data_hashes.map do |data_hash| - build_relation(relation_key, relation_definition, data_hash) - end.compact - end - - def build_relation(relation_key, relation_definition, data_hash) - # TODO: This is hack to not create relation for the author - # Rather make `RelationFactory#set_note_author` to take care of that - return data_hash if relation_key == 'author' - - # create relation objects recursively for all sub-objects - relation_definition.each do |sub_relation_key, sub_relation_definition| - transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) - end - - Gitlab::ImportExport::RelationFactory.create( - relation_sym: relation_key.to_sym, - relation_hash: data_hash, - members_mapper: members_mapper, - merge_requests_mapping: merge_requests_mapping, + def relation_tree_restorer + @relation_tree_restorer ||= RelationTreeRestorer.new( user: @user, - project: @project, - excluded_keys: excluded_keys_for_relation(relation_key)) + shared: @shared, + importable: @project, + tree_hash: @tree_hash, + members_mapper: members_mapper, + relation_factory: relation_factory, + reader: reader + ) end - def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) - sub_data_hash = data_hash[sub_relation_key] - return unless sub_data_hash - - # if object is a hash we can create simple object - # as it means that this is 1-to-1 vs 1-to-many - sub_data_hash = - if sub_data_hash.is_a?(Array) - build_relations( - sub_relation_key, - sub_relation_definition, - sub_data_hash).presence - else - build_relation( - sub_relation_key, - sub_relation_definition, - sub_data_hash) - end - - # persist object(s) or delete from relation - if sub_data_hash - data_hash[sub_relation_key] = sub_data_hash - else - data_hash.delete(sub_relation_key) - end + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + importable: @project) end - def group_model?(relation_object) - GROUP_MODELS.include?(relation_object.class) && relation_object.group_id + def relation_factory + Gitlab::ImportExport::RelationFactory end def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end - - def excluded_keys_for_relation(relation) - reader.attributes_finder.find_excluded_keys(relation) - end end end end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb new file mode 100644 index 00000000000..15d1f8a8148 --- /dev/null +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -0,0 +1,243 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class RelationTreeRestorer + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone].freeze + + attr_reader :user + attr_reader :shared + attr_reader :importable + attr_reader :tree_hash + + def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, relation_factory:, reader:) + @user = user + @shared = shared + @importable = importable + @tree_hash = tree_hash + @members_mapper = members_mapper + @relation_factory = relation_factory + @reader = reader + end + + def restore + ActiveRecord::Base.uncached do + ActiveRecord::Base.no_touching do + update_params! + create_relations! + end + end + + # ensure that we have latest version of the restore + @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload + + true + rescue => e + @shared.error(e) + false + end + + private + + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project/group. + def create_relations! + relations.each(&method(:process_relation!)) + end + + def process_relation!(relation_key, relation_definition) + data_hashes = @tree_hash.delete(relation_key) + return unless data_hashes + + # we do not care if we process array or hash + data_hashes = [data_hashes] unless data_hashes.is_a?(Array) + + relation_index = 0 + + # consume and remove objects from memory + while data_hash = data_hashes.shift + process_relation_item!(relation_key, relation_definition, relation_index, data_hash) + relation_index += 1 + end + end + + def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) + relation_object = build_relation(relation_key, relation_definition, data_hash) + return unless relation_object + return if importable_class == Project && group_model?(relation_object) + + relation_object.assign_attributes(importable_class_sym => @importable) + relation_object.save! + + save_id_mapping(relation_key, data_hash, relation_object) + rescue => e + # re-raise if not enabled + raise e unless Feature.enabled?(:import_graceful_failures, @importable.group, default_enabled: true) + + log_import_failure(relation_key, relation_index, e) + end + + def log_import_failure(relation_key, relation_index, exception) + Gitlab::Sentry.track_acceptable_exception( + exception, + extra: { project_id: @importable.id, + relation_key: relation_key, + relation_index: relation_index }) + + ImportFailure.create( + project: @importable, + relation_key: relation_key, + relation_index: relation_index, + exception_class: exception.class.to_s, + exception_message: exception.message.truncate(255), + correlation_id_value: Labkit::Correlation::CorrelationId.current_id + ) + end + + # Older, serialized CI pipeline exports may only have a + # merge_request_id and not the full hash of the merge request. To + # import these pipelines, we need to preserve the mapping between + # the old and new the merge request ID. + def save_id_mapping(relation_key, data_hash, relation_object) + return unless importable_class == Project + return unless relation_key == 'merge_requests' + + merge_requests_mapping[data_hash['id']] = relation_object.id + end + + def relations + @relations ||= + @reader + .attributes_finder + .find_relations_tree(importable_class_sym) + .deep_stringify_keys + end + + def update_params! + params = @tree_hash.reject do |key, _| + relations.include?(key) + end + + params = params.merge(present_override_params) + + # Cleaning all imported and overridden params + params = Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: params, + relation_class: importable_class, + excluded_keys: excluded_keys_for_relation(importable_class_sym)) + + @importable.assign_attributes(params) + @importable.drop_visibility_level! if importable_class == Project + + Gitlab::Timeless.timeless(@importable) do + @importable.save! + end + end + + def present_override_params + # we filter out the empty strings from the overrides + # keeping the default values configured + override_params&.transform_values do |value| + value.is_a?(String) ? value.presence : value + end&.compact + end + + def override_params + @importable_override_params ||= importable_override_params + end + + def importable_override_params + if @importable.respond_to?(:import_data) + @importable.import_data&.data&.fetch('override_params', nil) || {} + else + {} + end + end + + def build_relations(relation_key, relation_definition, data_hashes) + data_hashes.map do |data_hash| + build_relation(relation_key, relation_definition, data_hash) + end.compact + end + + def build_relation(relation_key, relation_definition, data_hash) + # TODO: This is hack to not create relation for the author + # Rather make `RelationFactory#set_note_author` to take care of that + return data_hash if relation_key == 'author' + + # create relation objects recursively for all sub-objects + relation_definition.each do |sub_relation_key, sub_relation_definition| + transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) + end + + @relation_factory.create(relation_factory_params(relation_key, data_hash)) + end + + def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition) + sub_data_hash = data_hash[sub_relation_key] + return unless sub_data_hash + + # if object is a hash we can create simple object + # as it means that this is 1-to-1 vs 1-to-many + sub_data_hash = + if sub_data_hash.is_a?(Array) + build_relations( + sub_relation_key, + sub_relation_definition, + sub_data_hash).presence + else + build_relation( + sub_relation_key, + sub_relation_definition, + sub_data_hash) + end + + # persist object(s) or delete from relation + if sub_data_hash + data_hash[sub_relation_key] = sub_data_hash + else + data_hash.delete(sub_relation_key) + end + end + + def group_model?(relation_object) + GROUP_MODELS.include?(relation_object.class) && relation_object.group_id + end + + def excluded_keys_for_relation(relation) + @reader.attributes_finder.find_excluded_keys(relation) + end + + def importable_class + @importable.class + end + + def importable_class_sym + importable_class.to_s.downcase.to_sym + end + + # A Hash of the imported merge request ID -> imported ID. + def merge_requests_mapping + @merge_requests_mapping ||= {} + end + + def relation_factory_params(relation_key, data_hash) + base_params = { + relation_sym: relation_key.to_sym, + relation_hash: data_hash, + members_mapper: @members_mapper, + user: @user, + excluded_keys: excluded_keys_for_relation(relation_key) + } + + base_params[:merge_requests_mapping] = merge_requests_mapping if importable_class == Project + base_params[importable_class_sym] = @importable + base_params + end + end + end +end |