From 07009a1f4893652e152794ae8160a2f46e00772c Mon Sep 17 00:00:00 2001
From: James Lopez <james@jameslopez.es>
Date: Thu, 26 Jul 2018 12:55:21 +0200
Subject: Add Object Storage to GitLab project import

- Refactor uploads manager
- Refactor importer, update import spec
- Add more object storage specs
---
 lib/gitlab/import_export/command_line_util.rb | 15 +++++++++++++++
 lib/gitlab/import_export/file_importer.rb     | 24 ++++++++++++++++++++----
 lib/gitlab/import_export/importer.rb          | 12 ++++++++++--
 lib/gitlab/import_export/uploads_manager.rb   |  5 +----
 lib/gitlab/template_helper.rb                 | 14 ++++++++++----
 5 files changed, 56 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index 2f163db936b..3adc44f8044 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -18,6 +18,21 @@ module Gitlab
 
       private
 
+      def download_or_copy_upload(uploader, upload_path)
+        if uploader.upload.local?
+          copy_files(uploader.path, upload_path)
+        else
+          download(uploader.url, upload_path)
+        end
+      end
+
+      def download(url, upload_path)
+        File.open(upload_path, 'w') do |file|
+          # Download (stream) file from the uploader's location
+          IO.copy_stream(URI.parse(url).open, file)
+        end
+      end
+
       def tar_with_options(archive:, dir:, options:)
         execute(%W(tar -#{options} #{archive} -C #{dir} .))
       end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 4c411f4847e..7fd66b4e244 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -10,15 +10,18 @@ module Gitlab
         new(*args).import
       end
 
-      def initialize(archive_file:, shared:)
+      def initialize(project:, archive_file:, shared:)
+        @project = project
         @archive_file = archive_file
         @shared = shared
       end
 
       def import
         mkdir_p(@shared.export_path)
+        mkdir_p(@shared.archive_path)
 
-        remove_symlinks!
+        remove_symlinks
+        copy_archive
 
         wait_for_archived_file do
           decompress_archive
@@ -27,7 +30,8 @@ module Gitlab
         @shared.error(e)
         false
       ensure
-        remove_symlinks!
+        remove_import_file
+        remove_symlinks
       end
 
       private
@@ -51,7 +55,15 @@ module Gitlab
         result
       end
 
-      def remove_symlinks!
+      def copy_archive
+        return if @archive_file
+
+        @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+
+        download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
+      end
+
+      def remove_symlinks
         extracted_files.each do |path|
           FileUtils.rm(path) if File.lstat(path).symlink?
         end
@@ -59,6 +71,10 @@ module Gitlab
         true
       end
 
+      def remove_import_file
+        FileUtils.rm_rf(@archive_file)
+      end
+
       def extracted_files
         Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) }
       end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 63cab07324a..4e179f63d8c 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -35,7 +35,8 @@ module Gitlab
       end
 
       def import_file
-        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+        Gitlab::ImportExport::FileImporter.import(project: @project,
+                                                  archive_file: @archive_file,
                                                   shared: @shared)
       end
 
@@ -91,7 +92,14 @@ module Gitlab
       end
 
       def remove_import_file
-        FileUtils.rm_rf(@archive_file)
+        return unless Gitlab::ImportExport.object_storage?
+
+        upload = @project.import_export_upload
+
+        return unless upload&.import_file&.file
+
+        upload.remove_import_file!
+        upload.save!
       end
 
       def overwrite_project
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index 1110149712d..07875ebb56a 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -91,10 +91,7 @@ module Gitlab
 
         mkdir_p(File.join(uploads_export_path, secret))
 
-        File.open(upload_path, 'w') do |file|
-          # Download (stream) file from the uploader's location
-          IO.copy_stream(URI.parse(upload.file.url).open, file)
-        end
+        download_or_copy_upload(upload, upload_path)
       end
     end
   end
diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb
index 3b8e45e0688..3918abaab45 100644
--- a/lib/gitlab/template_helper.rb
+++ b/lib/gitlab/template_helper.rb
@@ -2,11 +2,17 @@ module Gitlab
   module TemplateHelper
     include Gitlab::Utils::StrongMemoize
 
-    def prepare_template_environment(file_path)
-      return unless file_path.present?
+    def prepare_template_environment(file)
+      return unless file&.path.present?
 
-      FileUtils.mkdir_p(File.dirname(import_upload_path))
-      FileUtils.copy_entry(file_path, import_upload_path)
+      if Gitlab::ImportExport.object_storage?
+        params[:import_export_upload] = ImportExportUpload.new(import_file: file)
+      else
+        FileUtils.mkdir_p(File.dirname(import_upload_path))
+        FileUtils.copy_entry(file.path, import_upload_path)
+
+        params[:import_source] = import_upload_path
+      end
     end
 
     def import_upload_path
-- 
cgit v1.2.3