diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-22 00:08:57 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-22 00:08:57 +0300 |
commit | a6c2be7cd20a9515b347e72d63c5b47bb9b79457 (patch) | |
tree | 568212b4debeb2a35bb1133209b98e1468d9ee85 /lib | |
parent | 74a2d57b337034cfdcd719615e4da06643b69114 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/checks/lfs_integrity.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/database/migration_helpers.rb | 10 | ||||
-rw-r--r-- | lib/gitlab/file_type_detection.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/import_export/base_relation_factory.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/import_export/import_export.yml | 3 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_relation_factory.rb | 24 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_restorer.rb | 23 | ||||
-rw-r--r-- | lib/gitlab/uploads/migration_helper.rb | 6 | ||||
-rw-r--r-- | lib/tasks/gitlab/uploads/migrate.rake | 4 |
9 files changed, 31 insertions, 60 deletions
diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index 1652d5a30a4..e18cf6ff8f2 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -9,7 +9,6 @@ module Gitlab @time_left = time_left end - # rubocop: disable CodeReuse/ActiveRecord def objects_missing? return false unless @newrev && @project.lfs_enabled? @@ -19,12 +18,11 @@ module Gitlab return false unless new_lfs_pointers.present? existing_count = @project.all_lfs_objects - .where(oid: new_lfs_pointers.map(&:lfs_oid)) + .for_oids(new_lfs_pointers.map(&:lfs_oid)) .count existing_count != new_lfs_pointers.count end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 95a562ca1f3..f75e943671b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -235,11 +235,17 @@ module Gitlab # PostgreSQL constraint names have a limit of 63 bytes. The logic used # here is based on Rails' foreign_key_name() method, which unfortunately # is private so we can't rely on it directly. - def concurrent_foreign_key_name(table, column) + # + # prefix: + # - The default prefix is `fk_` for backward compatibility with the existing + # concurrent foreign key helpers. + # - For standard rails foreign keys the prefix is `fk_rails_` + # + def concurrent_foreign_key_name(table, column, prefix: 'fk_') identifier = "#{table}_#{column}_fk" hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) - "fk_#{hashed_identifier}" + "#{prefix}#{hashed_identifier}" end # Long-running migrations may take more than the timeout allowed by diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb index e052792675a..475d50e37bf 100644 --- a/lib/gitlab/file_type_detection.rb +++ b/lib/gitlab/file_type_detection.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# The method `filename` must be defined in classes that use this module. +# The method `filename` must be defined in classes that mix in this module. # # This module is intended to be used as a helper and not a security gate # to validate that a file is safe, as it identifies files only by the @@ -35,6 +35,13 @@ module Gitlab DANGEROUS_VIDEO_EXT = [].freeze # None, yet DANGEROUS_AUDIO_EXT = [].freeze # None, yet + def self.extension_match?(filename, extensions) + return false unless filename.present? + + extension = File.extname(filename).delete('.') + extensions.include?(extension.downcase) + end + def image? extension_match?(SAFE_IMAGE_EXT) end @@ -74,10 +81,7 @@ module Gitlab private def extension_match?(extensions) - return false unless filename - - extension = File.extname(filename).delete('.') - extensions.include?(extension.downcase) + ::Gitlab::FileTypeDetection.extension_match?(filename, extensions) end end end diff --git a/lib/gitlab/import_export/base_relation_factory.rb b/lib/gitlab/import_export/base_relation_factory.rb index d3c8802bcce..fcb516fb3a1 100644 --- a/lib/gitlab/import_export/base_relation_factory.rb +++ b/lib/gitlab/import_export/base_relation_factory.rb @@ -43,12 +43,11 @@ module Gitlab relation_name.to_s.constantize end - def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, merge_requests_mapping: nil, user:, importable:, excluded_keys: []) + def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @object_builder = object_builder - @merge_requests_mapping = merge_requests_mapping @user = user @importable = importable @imported_object_retries = 0 diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index e55ad898263..aa275479e63 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -377,3 +377,6 @@ ee: - protected_environments: - :deploy_access_levels - :service_desk_setting + excluded_attributes: + actions: + - image_v432x230 diff --git a/lib/gitlab/import_export/project_relation_factory.rb b/lib/gitlab/import_export/project_relation_factory.rb index e27bb9d3af1..0e08a66b89c 100644 --- a/lib/gitlab/import_export/project_relation_factory.rb +++ b/lib/gitlab/import_export/project_relation_factory.rb @@ -111,28 +111,6 @@ module Gitlab @relation_hash['group_id'] = @importable.namespace_id end - # This code is a workaround for broken project exports that don't - # export merge requests with CI pipelines (i.e. exports that were - # generated from - # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844). - # This method can be removed in GitLab 12.6. - def update_merge_request_references - # If a merge request was properly created, we don't need to fix - # up this export. - return if @relation_hash['merge_request'] - - merge_request_id = @relation_hash['merge_request_id'] - - return unless merge_request_id - - new_merge_request_id = @merge_requests_mapping[merge_request_id] - - return unless new_merge_request_id - - @relation_hash['merge_request_id'] = new_merge_request_id - parsed_relation_hash['merge_request_id'] = new_merge_request_id - end - def setup_build @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') @@ -147,8 +125,6 @@ module Gitlab end def setup_pipeline - update_merge_request_references - @relation_hash.fetch('stages', []).each do |stage| stage.statuses.each do |status| status.pipeline = imported_object diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index cc01d70db16..9b84ade1525 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -76,8 +76,6 @@ module Gitlab import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do relation_object.save! end - - save_id_mapping(relation_key, data_hash, relation_object) rescue => e import_failure_service.log_import_failure( source: 'process_relation_item!', @@ -90,17 +88,6 @@ module Gitlab @import_failure_service ||= ImportFailureService.new(@importable) end - # Older, serialized CI pipeline exports may only have a - # merge_request_id and not the full hash of the merge request. To - # import these pipelines, we need to preserve the mapping between - # the old and new the merge request ID. - def save_id_mapping(relation_key, data_hash, relation_object) - return unless importable_class == Project - return unless relation_key == 'merge_requests' - - merge_requests_mapping[data_hash['id']] = relation_object.id - end - def relations @relations ||= @reader @@ -219,13 +206,8 @@ module Gitlab importable_class.to_s.downcase.to_sym end - # A Hash of the imported merge request ID -> imported ID. - def merge_requests_mapping - @merge_requests_mapping ||= {} - end - def relation_factory_params(relation_key, data_hash) - base_params = { + { relation_sym: relation_key.to_sym, relation_hash: data_hash, importable: @importable, @@ -234,9 +216,6 @@ module Gitlab user: @user, excluded_keys: excluded_keys_for_relation(relation_key) } - - base_params[:merge_requests_mapping] = merge_requests_mapping if importable_class == Project - base_params end end end diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index 4ff064007f1..96ee6f0e8e6 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -21,6 +21,10 @@ module Gitlab prepare_variables(args, logger) end + def self.categories + CATEGORIES + end + def migrate_to_remote_storage @to_store = ObjectStorage::Store::REMOTE @@ -70,3 +74,5 @@ module Gitlab end end end + +Gitlab::Uploads::MigrationHelper.prepend_if_ee('EE::Gitlab::Uploads::MigrationHelper') diff --git a/lib/tasks/gitlab/uploads/migrate.rake b/lib/tasks/gitlab/uploads/migrate.rake index 44536a447c7..879b07da1df 100644 --- a/lib/tasks/gitlab/uploads/migrate.rake +++ b/lib/tasks/gitlab/uploads/migrate.rake @@ -3,7 +3,7 @@ namespace :gitlab do namespace :migrate do desc "GitLab | Uploads | Migrate all uploaded files to object storage" task all: :environment do - Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| + Gitlab::Uploads::MigrationHelper.categories.each do |args| Rake::Task["gitlab:uploads:migrate"].invoke(*args) Rake::Task["gitlab:uploads:migrate"].reenable end @@ -20,7 +20,7 @@ namespace :gitlab do namespace :migrate_to_local do desc "GitLab | Uploads | Migrate all uploaded files to local storage" task all: :environment do - Gitlab::Uploads::MigrationHelper::CATEGORIES.each do |args| + Gitlab::Uploads::MigrationHelper.categories.each do |args| Rake::Task["gitlab:uploads:migrate_to_local"].invoke(*args) Rake::Task["gitlab:uploads:migrate_to_local"].reenable end |