diff options
author | James Lopez <james@jameslopez.es> | 2018-07-26 13:55:21 +0300 |
---|---|---|
committer | James Lopez <james@jameslopez.es> | 2018-08-02 12:50:19 +0300 |
commit | 07009a1f4893652e152794ae8160a2f46e00772c (patch) | |
tree | a9c03b82c4b28dfa3150cbd4735c07483b9547fb /lib/gitlab | |
parent | 3cc420b8094ec0240b473205fc886bd9c728cc5d (diff) |
Add Object Storage to GitLab project import
- Refactor uploads manager
- Refactor importer, update import spec
- Add more object storage specs
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/import_export/command_line_util.rb | 15 | ||||
-rw-r--r-- | lib/gitlab/import_export/file_importer.rb | 24 | ||||
-rw-r--r-- | lib/gitlab/import_export/importer.rb | 12 | ||||
-rw-r--r-- | lib/gitlab/import_export/uploads_manager.rb | 5 | ||||
-rw-r--r-- | lib/gitlab/template_helper.rb | 14 |
5 files changed, 56 insertions, 14 deletions
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f163db936b..3adc44f8044 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -18,6 +18,21 @@ module Gitlab private + def download_or_copy_upload(uploader, upload_path) + if uploader.upload.local? + copy_files(uploader.path, upload_path) + else + download(uploader.url, upload_path) + end + end + + def download(url, upload_path) + File.open(upload_path, 'w') do |file| + # Download (stream) file from the uploader's location + IO.copy_stream(URI.parse(url).open, file) + end + end + def tar_with_options(archive:, dir:, options:) execute(%W(tar -#{options} #{archive} -C #{dir} .)) end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 4c411f4847e..7fd66b4e244 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -10,15 +10,18 @@ module Gitlab new(*args).import end - def initialize(archive_file:, shared:) + def initialize(project:, archive_file:, shared:) + @project = project @archive_file = archive_file @shared = shared end def import mkdir_p(@shared.export_path) + mkdir_p(@shared.archive_path) - remove_symlinks! + remove_symlinks + copy_archive wait_for_archived_file do decompress_archive @@ -27,7 +30,8 @@ module Gitlab @shared.error(e) false ensure - remove_symlinks! + remove_import_file + remove_symlinks end private @@ -51,7 +55,15 @@ module Gitlab result end - def remove_symlinks! + def copy_archive + return if @archive_file + + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + + download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) + end + + def remove_symlinks extracted_files.each do |path| FileUtils.rm(path) if File.lstat(path).symlink? end @@ -59,6 +71,10 @@ module Gitlab true end + def remove_import_file + FileUtils.rm_rf(@archive_file) + end + def extracted_files Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 63cab07324a..4e179f63d8c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -35,7 +35,8 @@ module Gitlab end def import_file - Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, + Gitlab::ImportExport::FileImporter.import(project: @project, + archive_file: @archive_file, shared: @shared) end @@ -91,7 +92,14 @@ module Gitlab end def remove_import_file - FileUtils.rm_rf(@archive_file) + return unless Gitlab::ImportExport.object_storage? + + upload = @project.import_export_upload + + return unless upload&.import_file&.file + + upload.remove_import_file! + upload.save! end def overwrite_project diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 1110149712d..07875ebb56a 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -91,10 +91,7 @@ module Gitlab mkdir_p(File.join(uploads_export_path, secret)) - File.open(upload_path, 'w') do |file| - # Download (stream) file from the uploader's location - IO.copy_stream(URI.parse(upload.file.url).open, file) - end + download_or_copy_upload(upload, upload_path) end end end diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb index 3b8e45e0688..3918abaab45 100644 --- a/lib/gitlab/template_helper.rb +++ b/lib/gitlab/template_helper.rb @@ -2,11 +2,17 @@ module Gitlab module TemplateHelper include Gitlab::Utils::StrongMemoize - def prepare_template_environment(file_path) - return unless file_path.present? + def prepare_template_environment(file) + return unless file&.path.present? - FileUtils.mkdir_p(File.dirname(import_upload_path)) - FileUtils.copy_entry(file_path, import_upload_path) + if Gitlab::ImportExport.object_storage? + params[:import_export_upload] = ImportExportUpload.new(import_file: file) + else + FileUtils.mkdir_p(File.dirname(import_upload_path)) + FileUtils.copy_entry(file.path, import_upload_path) + + params[:import_source] = import_upload_path + end end def import_upload_path |