diff options
author | Sean McGivern <sean@mcgivern.me.uk> | 2018-07-16 13:28:21 +0300 |
---|---|---|
committer | Sean McGivern <sean@mcgivern.me.uk> | 2018-07-16 13:28:21 +0300 |
commit | 01ad732f8d5d5bef3ee6faf20dfa2110feb344c0 (patch) | |
tree | 961cf73e29fbc1dcfea39facd3c97447e34b7e53 /lib/gitlab/import_export | |
parent | 6c237350b3967881cf9da50ec13a536e1a7e6755 (diff) | |
parent | 97ce0607d57d9a03fc6348a7ce3f9b069ced6c90 (diff) |
Merge branch '48745-project-exports-fail-when-uploads-have-been-migrated-to-object-storage' into 'master'
Resolve "Project exports fail when uploads have been migrated to object storage"
Closes #48745
See merge request gitlab-org/gitlab-ce!20484
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r-- | lib/gitlab/import_export/avatar_saver.rb | 11 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_manager.rb | 101 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_restorer.rb | 21 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_saver.rb | 15 |
4 files changed, 130 insertions, 18 deletions
diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 998c21e2586..31ef0490cb3 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -11,7 +11,12 @@ module Gitlab def save return true unless @project.avatar.exists? - copy_files(avatar_path, avatar_export_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared, + relative_export_path: 'avatar', + from: avatar_path + ).save rescue => e @shared.error(e) false @@ -19,10 +24,6 @@ module Gitlab private - def avatar_export_path - File.join(@shared.export_path, 'avatar', @project.avatar_identifier) - end - def avatar_path @project.avatar.path end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb new file mode 100644 index 00000000000..1110149712d --- /dev/null +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -0,0 +1,101 @@ +module Gitlab + module ImportExport + class UploadsManager + include Gitlab::ImportExport::CommandLineUtil + + UPLOADS_BATCH_SIZE = 100 + + def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + @project = project + @shared = shared + @relative_export_path = relative_export_path + @from = from || default_uploads_path + end + + def save + copy_files(@from, uploads_export_path) if File.directory?(@from) + + if File.file?(@from) && @relative_export_path == 'avatar' + copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) + end + + copy_from_object_storage + + true + rescue => e + @shared.error(e) + false + end + + def restore + Dir["#{uploads_export_path}/**/*"].each do |upload| + next if File.directory?(upload) + + add_upload(upload) + end + + true + rescue => e + @shared.error(e) + false + end + + private + + def add_upload(upload) + uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys + + UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute + end + + def copy_from_object_storage + return unless Gitlab::ImportExport.object_storage? + + each_uploader do |uploader| + next unless uploader.file + next if uploader.upload.local? # Already copied, using the old method + + download_and_copy(uploader) + end + end + + def default_uploads_path + FileUploader.absolute_base_dir(@project) + end + + def uploads_export_path + @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) + end + + def each_uploader + avatar_path = @project.avatar&.upload&.path + + if @relative_export_path == 'avatar' + yield(@project.avatar) + else + project_uploads_except_avatar(avatar_path).find_each(batch_size: UPLOADS_BATCH_SIZE) do |upload| + yield(upload.build_uploader) + end + end + end + + def project_uploads_except_avatar(avatar_path) + return @project.uploads unless avatar_path + + @project.uploads.where("path != ?", avatar_path) + end + + def download_and_copy(upload) + secret = upload.try(:secret) || '' + upload_path = File.join(uploads_export_path, secret, upload.filename) + + mkdir_p(File.join(uploads_export_path, secret)) + + File.open(upload_path, 'w') do |file| + # Download (stream) file from the uploader's location + IO.copy_stream(URI.parse(upload.file.url).open, file) + end + end + end + end +end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index df19354b76e..25f85936227 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,13 +2,30 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - return true unless File.directory?(uploads_export_path) + if Gitlab::ImportExport.object_storage? + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore + elsif File.directory?(uploads_export_path) + copy_files(uploads_export_path, uploads_path) - copy_files(uploads_export_path, uploads_path) + true + else + true # Proceed without uploads + end rescue => e @shared.error(e) false end + + def uploads_path + FileUploader.absolute_base_dir(@project) + end + + def uploads_export_path + @uploads_export_path ||= File.join(@shared.export_path, 'uploads') + end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 2f08dda55fd..b3f17af5661 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -9,21 +9,14 @@ module Gitlab end def save - return true unless File.directory?(uploads_path) - - copy_files(uploads_path, uploads_export_path) + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).save rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - File.join(@shared.export_path, 'uploads') - end end end end |