diff options
author | Douwe Maan <douwe@gitlab.com> | 2018-09-07 17:28:25 +0300 |
---|---|---|
committer | Douwe Maan <douwe@gitlab.com> | 2018-09-07 17:28:25 +0300 |
commit | 1f765ce9ea25b45cd702c25ccb762eacceb9a4ae (patch) | |
tree | 1dc0b5b6141b386d4e6fedce1ffd519e21a60e21 /lib | |
parent | 766bb7fb8ed94b438aac47c0197c96cfff6c789f (diff) | |
parent | 1499d1764dbea4234002ee17fe46c0d6d6772067 (diff) |
Merge branch '48778-remove-old-storage-logic-from-import-export' into 'master'
Resolve "Remove old storage logic from Import/Export"
Closes #51144, #50109, #48778, and #51079
See merge request gitlab-org/gitlab-ce!21409
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/project_export.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/import_export/avatar_restorer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/avatar_saver.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/import_export/importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/saver.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_manager.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_restorer.rb | 24 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_saver.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/template_helper.rb | 17 | ||||
-rw-r--r-- | lib/tasks/gitlab/update_templates.rake | 4 |
14 files changed, 26 insertions, 101 deletions
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 15c57a2fc02..8562ae6d737 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -21,12 +21,8 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - path = user_project.export_project_path - - if path - present_disk_file!(path, File.basename(path), 'application/gzip') - elsif user_project.export_project_object_exists? - present_carrierwave_file!(user_project.import_export_upload.export_file) + if user_project.export_file_exists? + present_carrierwave_file!(user_project.export_file) else render_api_error!('404 Not found or has expired', 404) end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index be3710c5b7f..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -40,10 +40,6 @@ module Gitlab "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end - def object_storage? - Feature.enabled?(:import_export_object_storage) - end - def version VERSION end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index 83134bb0769..7cbf653dd97 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -53,7 +53,7 @@ module Gitlab end def self.lock_file_path(project) - return unless project.export_path || object_storage? + return unless project.export_path || export_file_exists? lock_path = project.import_export_shared.archive_path @@ -83,8 +83,8 @@ module Gitlab errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end - def object_storage? - project.export_project_object_exists? + def export_file_exists? + project.export_file_exists? end end end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index dce8f89c0ab..4f29bdcea2c 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -23,7 +23,7 @@ module Gitlab def strategy_execute handle_response_error(send_file) - project.remove_exported_project_file + project.remove_exports end def handle_response_error(response) @@ -40,15 +40,11 @@ module Gitlab def send_file Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend ensure - export_file.close if export_file && !object_storage? + export_file.close if export_file end def export_file - if object_storage? - project.import_export_upload.export_file.file.open - else - File.open(project.export_project_path) - end + project.export_file.open end def send_file_options @@ -63,11 +59,7 @@ module Gitlab end def export_size - if object_storage? - project.import_export_upload.export_file.file.size - else - File.size(project.export_project_path) - end + project.export_file.file.size end end end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index cfa595629f4..ded05f73cf8 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -19,7 +19,7 @@ module Gitlab private def avatar_export_file - @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first end def avatar_export_path diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 31ef0490cb3..6ffebf83dd2 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class AvatarSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared @@ -14,19 +12,12 @@ module Gitlab Gitlab::ImportExport::UploadsManager.new( project: @project, shared: @shared, - relative_export_path: 'avatar', - from: avatar_path + relative_export_path: 'avatar' ).save rescue => e @shared.error(e) false end - - private - - def avatar_path - @project.avatar.path - end end end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 4e179f63d8c..72d5b9b830c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -92,8 +92,6 @@ module Gitlab end def remove_import_file - return unless Gitlab::ImportExport.object_storage? - upload = @project.import_export_upload return unless upload&.import_file&.file diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index f4106e03a57..00ea4b833e2 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -199,7 +199,7 @@ module Gitlab end def excluded_keys_for_relation(relation) - @reader.attributes_finder.find_excluded_keys(relation) + reader.attributes_finder.find_excluded_keys(relation) end end end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 3cd153a4fd2..59a74083395 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab Rails.logger.info("Saved project export #{archive_file}") - save_on_object_storage if use_object_storage? + save_upload else @shared.error(Gitlab::ImportExport::Error.new(error_message)) false @@ -27,10 +27,8 @@ module Gitlab @shared.error(e) false ensure - if use_object_storage? - remove_archive - remove_export_path - end + remove_archive + remove_export_path end private @@ -51,7 +49,7 @@ module Gitlab @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) end - def save_on_object_storage + def save_upload upload = ImportExportUpload.find_or_initialize_by(project: @project) File.open(archive_file) { |file| upload.export_file = file } @@ -59,12 +57,8 @@ module Gitlab upload.save! end - def use_object_storage? - Gitlab::ImportExport.object_storage? - end - def error_message - "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}" + "Unable to save #{archive_file} into #{@shared.export_path}." end end end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index e0d4235e65b..8511319cb1c 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -5,18 +5,13 @@ module Gitlab UPLOADS_BATCH_SIZE = 100 - def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + def initialize(project:, shared:, relative_export_path: 'uploads') @project = project @shared = shared @relative_export_path = relative_export_path - @from = from || default_uploads_path end def save - if File.file?(@from) && @relative_export_path == 'avatar' - copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) - end - copy_project_uploads true @@ -55,17 +50,11 @@ module Gitlab copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path)) else - next unless Gitlab::ImportExport.object_storage? - download_and_copy(uploader) end end end - def default_uploads_path - FileUploader.absolute_base_dir(@project) - end - def uploads_export_path @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 25f85936227..b4313ff4cb4 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,30 +2,14 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - if Gitlab::ImportExport.object_storage? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).restore - elsif File.directory?(uploads_export_path) - copy_files(uploads_export_path, uploads_path) - - true - else - true # Proceed without uploads - end + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - @uploads_export_path ||= File.join(@shared.export_path, 'uploads') - end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index b3f17af5661..0275f686c5e 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class UploadsSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb index f24a01e6cf5..fc498dde723 100644 --- a/lib/gitlab/template_helper.rb +++ b/lib/gitlab/template_helper.rb @@ -1,24 +1,9 @@ module Gitlab module TemplateHelper - include Gitlab::Utils::StrongMemoize - def prepare_template_environment(file) return unless file - if Gitlab::ImportExport.object_storage? - params[:import_export_upload] = ImportExportUpload.new(import_file: file) - else - FileUtils.mkdir_p(File.dirname(import_upload_path)) - FileUtils.copy_entry(file.path, import_upload_path) - - params[:import_source] = import_upload_path - end - end - - def import_upload_path - strong_memoize(:import_upload_path) do - Gitlab::ImportExport.import_upload_path(filename: tmp_filename) - end + params[:import_export_upload] = ImportExportUpload.new(import_file: file) end def tmp_filename diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index a25f7ce59c7..ef6a32d6730 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -6,6 +6,8 @@ namespace :gitlab do desc "GitLab | Update project templates" task :update_project_templates do + include Gitlab::ImportExport::CommandLineUtil + if Rails.env.production? puts "This rake task is not meant fo production instances".red exit(1) @@ -52,7 +54,7 @@ namespace :gitlab do end Projects::ImportExport::ExportService.new(project, admin).execute - FileUtils.cp(project.export_project_path, template.archive_path) + download_or_copy_upload(project.export_file, template.archive_path) Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end |