Merge branch 'master-ce' into artifact-format-v2-with-parser

11 files changed, 105 insertions, 191 deletions

diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index af762db517c..906ed498026 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -1,10 +1,7 @@
 require 'yaml'
-require_relative 'helper'
 
 module Backup
   class Repository
-    include Backup::Helper
-
     attr_reader :progress
 
     def initialize(progress)
@@ -42,131 +39,36 @@ module Backup
     end
 
     def prepare_directories
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        delete_all_repositories(name, repository_storage)
+      Gitlab.config.repositories.storages.each do |name, _repository_storage|
+        Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
       end
     end
 
     def backup_project(project)
-      gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          backup_project_gitaly(project)
-        else
-          backup_project_local(project)
-        end
-      end
-
-      backup_custom_hooks(project)
-    rescue => e
-      progress_warn(project, e, 'Failed to backup repo')
-    end
-
-    def backup_project_gitaly(project)
       path_to_project_bundle = path_to_bundle(project)
       Gitlab::GitalyClient::RepositoryService.new(project.repository)
         .create_bundle(path_to_project_bundle)
-    end
-
-    def backup_project_local(project)
-      path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-        path_to_repo(project)
-      end
-
-      path_to_project_bundle = path_to_bundle(project)
-
-      cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
-      output, status = Gitlab::Popen.popen(cmd)
-      progress_warn(project, cmd.join(' '), output) unless status.zero?
-    end
-
-    def delete_all_repositories(name, repository_storage)
-      gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
-        else
-          local_delete_all_repositories(name, repository_storage)
-        end
-      end
-    end
-
-    def local_delete_all_repositories(name, repository_storage)
-      path = repository_storage.legacy_disk_path
-      return unless File.exist?(path)
-
-      bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
-      FileUtils.mkdir_p(bk_repos_path, mode: 0700)
-      files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
-
-      begin
-        FileUtils.mv(files, bk_repos_path)
-      rescue Errno::EACCES
-        access_denied_error(path)
-      rescue Errno::EBUSY
-        resource_busy_error(path)
-      end
-    end
 
-    def local_restore_custom_hooks(project, dir)
-      path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-        path_to_repo(project)
-      end
-      cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
-      output, status = Gitlab::Popen.popen(cmd)
-      unless status.zero?
-        progress_warn(project, cmd.join(' '), output)
-      end
-    end
-
-    def gitaly_restore_custom_hooks(project, dir)
-      custom_hooks_path = path_to_tars(project, dir)
-      Gitlab::GitalyClient::RepositoryService.new(project.repository)
-        .restore_custom_hooks(custom_hooks_path)
+      backup_custom_hooks(project)
+    rescue => e
+      progress_warn(project, e, 'Failed to backup repo')
     end
 
-    def local_backup_custom_hooks(project)
-      in_path(path_to_tars(project)) do |dir|
-        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          path_to_repo(project)
-        end
-        break unless File.exist?(File.join(path_to_project_repo, dir))
-
-        FileUtils.mkdir_p(path_to_tars(project))
-        cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
-        output, status = Gitlab::Popen.popen(cmd)
-
-        unless status.zero?
-          progress_warn(project, cmd.join(' '), output)
-        end
-      end
-    end
+    def backup_custom_hooks(project)
+      FileUtils.mkdir_p(project_backup_path(project))
 
-    def gitaly_backup_custom_hooks(project)
-      FileUtils.mkdir_p(path_to_tars(project))
-      custom_hooks_path = path_to_tars(project, 'custom_hooks')
+      custom_hooks_path = custom_hooks_tar(project)
       Gitlab::GitalyClient::RepositoryService.new(project.repository)
         .backup_custom_hooks(custom_hooks_path)
     end
 
-    def backup_custom_hooks(project)
-      gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-        if is_enabled
-          gitaly_backup_custom_hooks(project)
-        else
-          local_backup_custom_hooks(project)
-        end
-      end
-    end
-
     def restore_custom_hooks(project)
-      in_path(path_to_tars(project)) do |dir|
-        gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
-          if is_enabled
-            gitaly_restore_custom_hooks(project, dir)
-          else
-            local_restore_custom_hooks(project, dir)
-          end
-        end
-      end
+      return unless Dir.exist?(project_backup_path(project))
+      return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none?
+
+      custom_hooks_path = custom_hooks_tar(project)
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .restore_custom_hooks(custom_hooks_path)
     end
 
     def restore
@@ -181,7 +83,8 @@ module Backup
         restore_repo_success = nil
         if File.exist?(path_to_project_bundle)
           begin
-            project.repository.create_from_bundle path_to_project_bundle
+            project.repository.create_from_bundle(path_to_project_bundle)
+            restore_custom_hooks(project)
             restore_repo_success = true
           rescue => e
             restore_repo_success = false
@@ -197,8 +100,6 @@ module Backup
           progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
         end
 
-        restore_custom_hooks(project)
-
         wiki = ProjectWiki.new(project)
         path_to_wiki_bundle = path_to_bundle(wiki)
 
@@ -219,48 +120,28 @@ module Backup
 
     protected
 
-    def path_to_repo(project)
-      project.repository.path_to_repo
-    end
-
     def path_to_bundle(project)
       File.join(backup_repos_path, project.disk_path + '.bundle')
     end
 
-    def path_to_tars(project, dir = nil)
-      path = File.join(backup_repos_path, project.disk_path)
+    def project_backup_path(project)
+      File.join(backup_repos_path, project.disk_path)
+    end
 
-      if dir
-        File.join(path, "#{dir}.tar")
-      else
-        path
-      end
+    def custom_hooks_tar(project)
+      File.join(project_backup_path(project), "custom_hooks.tar")
     end
 
     def backup_repos_path
       File.join(Gitlab.config.backup.path, 'repositories')
     end
 
-    def in_path(path)
-      return unless Dir.exist?(path)
-
-      dir_entries = Dir.entries(path)
-
-      if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar')
-        yield('custom_hooks')
-      end
-    end
-
     def prepare
       FileUtils.rm_rf(backup_repos_path)
       FileUtils.mkdir_p(Gitlab.config.backup.path)
       FileUtils.mkdir(backup_repos_path, mode: 0700)
     end
 
-    def silent
-      { err: '/dev/null', out: '/dev/null' }
-    end
-
     private
 
     def progress_warn(project, cmd, output)
@@ -273,18 +154,8 @@ module Backup
       project_or_wiki.repository.empty?
     end
 
-    def repository_storage_paths_args
-      Gitlab.config.repositories.storages.values.map { |rs| rs.legacy_disk_path }
-    end
-
     def display_repo_path(project)
       project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
     end
-
-    def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
-      Gitlab::GitalyClient.migrate(method, status: status, &block)
-    rescue GRPC::NotFound, GRPC::BadStatus => e
-      raise Error, e
-    end
   end
 end
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
index 4ca5a78e068..04aa6aab771 100644
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -26,6 +26,12 @@ module Gitlab
         end
       end
 
+      # This is called from within a rake task only used by Admins, so allow writing
+      # to STDOUT
+      def self.log(message)
+        puts message # rubocop:disable Rails/Output
+      end
+
       attr_reader :user, :project_name, :bare_repo
 
       delegate :log, to: :class
@@ -59,11 +65,10 @@ module Gitlab
                                               import_type: 'bare_repository',
                                               namespace_id: group&.id).execute
 
-        if project.persisted? && mv_repo(project)
+        if project.persisted? && mv_repositories(project)
           log " * Created #{project.name} (#{project_full_path})".color(:green)
 
           project.write_repository_config
-          Gitlab::Git::Repository.create_hooks(project.repository.path_to_repo, Gitlab.config.gitlab_shell.hooks_path)
 
           ProjectCacheWorker.perform_async(project.id)
         else
@@ -74,12 +79,11 @@ module Gitlab
         project
       end
 
-      def mv_repo(project)
-        storage_path = storage_path_for_shard(project.repository_storage)
-        FileUtils.mv(repo_path, project.repository.path_to_repo)
+      def mv_repositories(project)
+        mv_repo(bare_repo.repo_path, project.repository)
 
         if bare_repo.wiki_exists?
-          FileUtils.mv(wiki_path, File.join(storage_path, project.disk_path + '.wiki.git'))
+          mv_repo(bare_repo.wiki_path, project.wiki.repository)
         end
 
         true
@@ -89,6 +93,11 @@ module Gitlab
         false
       end
 
+      def mv_repo(path, repository)
+        repository.create_from_bundle(bundle(path))
+        FileUtils.rm_rf(path)
+      end
+
       def storage_path_for_shard(shard)
         Gitlab.config.repositories.storages[shard].legacy_disk_path
       end
@@ -101,10 +110,17 @@ module Gitlab
         Groups::NestedCreateService.new(user, group_path: group_path).execute
       end
 
-      # This is called from within a rake task only used by Admins, so allow writing
-      # to STDOUT
-      def self.log(message)
-        puts message # rubocop:disable Rails/Output
+      def bundle(repo_path)
+        # TODO: we could save some time and disk space by using
+        # `git bundle create - --all` and streaming the bundle directly to
+        # Gitaly, rather than writing it on disk first
+        bundle_path = "#{repo_path}.bundle"
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)
+        output, status = Gitlab::Popen.popen(cmd)
+
+        raise output unless status.zero?
+
+        bundle_path
       end
     end
   end
diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb
index fe267248275..c0c666dfb7b 100644
--- a/lib/gitlab/bare_repository_import/repository.rb
+++ b/lib/gitlab/bare_repository_import/repository.rb
@@ -1,5 +1,3 @@
-# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/953
-#
 module Gitlab
   module BareRepositoryImport
     class Repository
diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb
index b88e00311d5..f55ab535efe 100644
--- a/lib/gitlab/cleanup/project_uploads.rb
+++ b/lib/gitlab/cleanup/project_uploads.rb
@@ -40,7 +40,7 @@ module Gitlab
       # Accepts a path in the form of "#{hex_secret}/#{filename}"
       def find_correct_path(upload_path)
         upload = Upload.find_by(uploader: 'FileUploader', path: upload_path)
-        return unless upload && upload.local?
+        return unless upload && upload.local? && upload.model
 
         upload.absolute_path
       rescue => e
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index eb02c7ac8e1..73151e4a4c5 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -39,19 +39,6 @@ module Gitlab
       ChecksumError = Class.new(StandardError)
 
       class << self
-        # Unlike `new`, `create` takes the repository path
-        def create(repo_path, bare: true, symlink_hooks_to: nil)
-          FileUtils.mkdir_p(repo_path, mode: 0770)
-
-          # Equivalent to `git --git-path=#{repo_path} init [--bare]`
-          repo = Rugged::Repository.init_at(repo_path, bare)
-          repo.close
-
-          create_hooks(repo_path, symlink_hooks_to) if symlink_hooks_to.present?
-
-          true
-        end
-
         def create_hooks(repo_path, global_hooks_path)
           local_hooks_path = File.join(repo_path, 'hooks')
           real_local_hooks_path = :not_found
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index 2f163db936b..3adc44f8044 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -18,6 +18,21 @@ module Gitlab
 
       private
 
+      def download_or_copy_upload(uploader, upload_path)
+        if uploader.upload.local?
+          copy_files(uploader.path, upload_path)
+        else
+          download(uploader.url, upload_path)
+        end
+      end
+
+      def download(url, upload_path)
+        File.open(upload_path, 'w') do |file|
+          # Download (stream) file from the uploader's location
+          IO.copy_stream(URI.parse(url).open, file)
+        end
+      end
+
       def tar_with_options(archive:, dir:, options:)
         execute(%W(tar -#{options} #{archive} -C #{dir} .))
       end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 4c411f4847e..7fd66b4e244 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -10,15 +10,18 @@ module Gitlab
         new(*args).import
       end
 
-      def initialize(archive_file:, shared:)
+      def initialize(project:, archive_file:, shared:)
+        @project = project
         @archive_file = archive_file
         @shared = shared
       end
 
       def import
         mkdir_p(@shared.export_path)
+        mkdir_p(@shared.archive_path)
 
-        remove_symlinks!
+        remove_symlinks
+        copy_archive
 
         wait_for_archived_file do
           decompress_archive
@@ -27,7 +30,8 @@ module Gitlab
         @shared.error(e)
         false
       ensure
-        remove_symlinks!
+        remove_import_file
+        remove_symlinks
       end
 
       private
@@ -51,7 +55,15 @@ module Gitlab
         result
       end
 
-      def remove_symlinks!
+      def copy_archive
+        return if @archive_file
+
+        @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+
+        download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
+      end
+
+      def remove_symlinks
         extracted_files.each do |path|
           FileUtils.rm(path) if File.lstat(path).symlink?
         end
@@ -59,6 +71,10 @@ module Gitlab
         true
       end
 
+      def remove_import_file
+        FileUtils.rm_rf(@archive_file)
+      end
+
       def extracted_files
         Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) }
       end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 63cab07324a..4e179f63d8c 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -35,7 +35,8 @@ module Gitlab
       end
 
       def import_file
-        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+        Gitlab::ImportExport::FileImporter.import(project: @project,
+                                                  archive_file: @archive_file,
                                                   shared: @shared)
       end
 
@@ -91,7 +92,14 @@ module Gitlab
       end
 
       def remove_import_file
-        FileUtils.rm_rf(@archive_file)
+        return unless Gitlab::ImportExport.object_storage?
+
+        upload = @project.import_export_upload
+
+        return unless upload&.import_file&.file
+
+        upload.remove_import_file!
+        upload.save!
       end
 
       def overwrite_project
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index 1110149712d..07875ebb56a 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -91,10 +91,7 @@ module Gitlab
 
         mkdir_p(File.join(uploads_export_path, secret))
 
-        File.open(upload_path, 'w') do |file|
-          # Download (stream) file from the uploader's location
-          IO.copy_stream(URI.parse(upload.file.url).open, file)
-        end
+        download_or_copy_upload(upload, upload_path)
       end
     end
   end
diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb
index 0f0588b8b23..530ccf88053 100644
--- a/lib/gitlab/kubernetes/helm.rb
+++ b/lib/gitlab/kubernetes/helm.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Kubernetes
     module Helm
-      HELM_VERSION = '2.7.0'.freeze
+      HELM_VERSION = '2.7.2'.freeze
       NAMESPACE = 'gitlab-managed-apps'.freeze
     end
   end
diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb
index 3b8e45e0688..3918abaab45 100644
--- a/lib/gitlab/template_helper.rb
+++ b/lib/gitlab/template_helper.rb
@@ -2,11 +2,17 @@ module Gitlab
   module TemplateHelper
     include Gitlab::Utils::StrongMemoize
 
-    def prepare_template_environment(file_path)
-      return unless file_path.present?
+    def prepare_template_environment(file)
+      return unless file&.path.present?
 
-      FileUtils.mkdir_p(File.dirname(import_upload_path))
-      FileUtils.copy_entry(file_path, import_upload_path)
+      if Gitlab::ImportExport.object_storage?
+        params[:import_export_upload] = ImportExportUpload.new(import_file: file)
+      else
+        FileUtils.mkdir_p(File.dirname(import_upload_path))
+        FileUtils.copy_entry(file.path, import_upload_path)
+
+        params[:import_source] = import_upload_path
+      end
     end
 
     def import_upload_path