Improve GitLab Import rake task to work with Hashed Storage and Subgroups

4 files changed, 149 insertions, 105 deletions

diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
new file mode 100644
index 00000000000..196de667805
--- /dev/null
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -0,0 +1,101 @@
+module Gitlab
+  module BareRepositoryImport
+    class Importer
+      NoAdminError = Class.new(StandardError)
+
+      def self.execute(import_path)
+        import_path << '/' unless import_path.ends_with?('/')
+        repos_to_import = Dir.glob(import_path + '**/*.git')
+
+        unless user = User.admins.order_id_asc.first
+          raise NoAdminError.new('No admin user found to import repositories')
+        end
+
+        repos_to_import.each do |repo_path|
+          bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path)
+
+          if bare_repo.hashed? || bare_repo.wiki?
+            log " * Skipping repo #{bare_repo.repo_path}".color(:yellow)
+
+            next
+          end
+
+          log "Processing #{repo_path}".color(:yellow)
+
+          new(user, bare_repo).create_project_if_needed
+        end
+      end
+
+      attr_reader :user, :project_name, :bare_repo
+
+      delegate :log, to: :class
+      delegate :project_name, :project_full_path, :group_path, :repo_path, :wiki_path, to: :bare_repo
+
+      def initialize(user, bare_repo)
+        @user = user
+        @bare_repo = bare_repo
+      end
+
+      def create_project_if_needed
+        if project = Project.find_by_full_path(project_full_path)
+          log " * #{project.name} (#{project_full_path}) exists"
+
+          return project
+        end
+
+        create_project
+      end
+
+      private
+
+      def create_project
+        group = find_or_create_groups
+
+        project = Projects::CreateService.new(user,
+                                              name: project_name,
+                                              path: project_name,
+                                              skip_disk_validation: true,
+                                              namespace_id: group&.id).execute
+
+        if project.persisted? && mv_repo(project)
+          log " * Created #{project.name} (#{project_full_path})".color(:green)
+
+          ProjectCacheWorker.perform_async(project.id)
+        else
+          log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red)
+          log "   Errors: #{project.errors.messages}".color(:red) if project.errors.any?
+        end
+
+        project
+      end
+
+      def mv_repo(project)
+        FileUtils.mv(repo_path, File.join(project.repository_storage_path, project.disk_path + '.git'))
+
+        if bare_repo.wiki_exists?
+          FileUtils.mv(wiki_path, File.join(project.repository_storage_path, project.disk_path + '.wiki.git'))
+        end
+
+        true
+      rescue => e
+        log " * Failed to move repo: #{e.message}".color(:red)
+
+        false
+      end
+
+      def find_or_create_groups
+        return nil unless group_path.present?
+
+        log " * Using namespace: #{group_path}"
+
+        Groups::NestedCreateService.new(user, group_path: group_path).execute
+      end
+
+      # This is called from within a rake task only used by Admins, so allow writing
+      # to STDOUT
+      def self.log(message)
+        puts message # rubocop:disable Rails/Output
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb
new file mode 100644
index 00000000000..8574ac6eb30
--- /dev/null
+++ b/lib/gitlab/bare_repository_import/repository.rb
@@ -0,0 +1,42 @@
+module Gitlab
+  module BareRepositoryImport
+    class Repository
+      attr_reader :group_path, :project_name, :repo_path
+
+      def initialize(root_path, repo_path)
+        @root_path = root_path
+        @repo_path = repo_path
+
+        # Split path into 'all/the/namespaces' and 'project_name'
+        @group_path, _, @project_name = repo_relative_path.rpartition('/')
+      end
+
+      def wiki_exists?
+        File.exist?(wiki_path)
+      end
+
+      def wiki?
+        @wiki ||= repo_path.end_with?('.wiki.git')
+      end
+
+      def wiki_path
+        @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git')
+      end
+
+      def hashed?
+        @hashed ||= group_path.start_with?('@hashed')
+      end
+
+      def project_full_path
+        @project_full_path ||= "#{group_path}/#{project_name}"
+      end
+
+      private
+
+      def repo_relative_path
+        # Remove root path and `.git` at the end
+        repo_path[@root_path.size...-4]
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/bare_repository_importer.rb b/lib/gitlab/bare_repository_importer.rb
deleted file mode 100644
index 1d98d187805..00000000000
--- a/lib/gitlab/bare_repository_importer.rb
+++ /dev/null
@@ -1,97 +0,0 @@
-module Gitlab
-  class BareRepositoryImporter
-    NoAdminError = Class.new(StandardError)
-
-    def self.execute
-      Gitlab.config.repositories.storages.each do |storage_name, repository_storage|
-        git_base_path = repository_storage['path']
-        repos_to_import = Dir.glob(git_base_path + '/**/*.git')
-
-        repos_to_import.each do |repo_path|
-          if repo_path.end_with?('.wiki.git')
-            log " * Skipping wiki repo"
-            next
-          end
-
-          log "Processing #{repo_path}".color(:yellow)
-
-          repo_relative_path = repo_path[repository_storage['path'].length..-1]
-                                 .sub(/^\//, '') # Remove leading `/`
-                                 .sub(/\.git$/, '') # Remove `.git` at the end
-          new(storage_name, repo_relative_path).create_project_if_needed
-        end
-      end
-    end
-
-    attr_reader :storage_name, :full_path, :group_path, :project_path, :user
-    delegate :log, to: :class
-
-    def initialize(storage_name, repo_path)
-      @storage_name = storage_name
-      @full_path = repo_path
-
-      unless @user = User.admins.order_id_asc.first
-        raise NoAdminError.new('No admin user found to import repositories')
-      end
-
-      @group_path, @project_path = File.split(repo_path)
-      @group_path = nil if @group_path == '.'
-    end
-
-    def create_project_if_needed
-      if project = Project.find_by_full_path(full_path)
-        log " * #{project.name} (#{full_path}) exists"
-        return project
-      end
-
-      create_project
-    end
-
-    private
-
-    def create_project
-      group = find_or_create_group
-
-      project_params = {
-        name: project_path,
-        path: project_path,
-        repository_storage: storage_name,
-        namespace_id: group&.id,
-        skip_disk_validation: true
-      }
-
-      project = Projects::CreateService.new(user, project_params).execute
-
-      if project.persisted?
-        log " * Created #{project.name} (#{full_path})".color(:green)
-        ProjectCacheWorker.perform_async(project.id)
-      else
-        log " * Failed trying to create #{project.name} (#{full_path})".color(:red)
-        log "   Errors: #{project.errors.messages}".color(:red)
-      end
-
-      project
-    end
-
-    def find_or_create_group
-      return nil unless group_path
-
-      if namespace = Namespace.find_by_full_path(group_path)
-        log " * Namespace #{group_path} exists.".color(:green)
-        return namespace
-      end
-
-      log " * Creating Group: #{group_path}"
-      Groups::NestedCreateService.new(user, group_path: group_path).execute
-    end
-
-    # This is called from within a rake task only used by Admins, so allow writing
-    # to STDOUT
-    #
-    # rubocop:disable Rails/Output
-    def self.log(message)
-      puts message
-    end
-    # rubocop:enable Rails/Output
-  end
-end
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index d227a0c8bdb..adfcc3cda22 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -2,23 +2,21 @@ namespace :gitlab do
   namespace :import do
     # How to use:
     #
-    #  1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories)
-    #  2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production
+    #  1. copy the bare repos to a specific path that contain the group or subgroups structure as folders
+    #  2. run: bundle exec rake gitlab:import:repos[/path/to/repos] RAILS_ENV=production
     #
     # Notes:
     #  * The project owner will set to the first administator of the system
     #  * Existing projects will be skipped
-    #
-    #
     desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance"
-    task repos: :environment do
-      if Project.current_application_settings.hashed_storage_enabled
-        puts 'Cannot import repositories when Hashed Storage is enabled'.color(:red)
+    task :repos, [:import_path] => :environment do |_t, args|
+      unless args.import_path
+        puts 'Please specify an import path that contains the repositories'.color(:red)
 
         exit 1
       end
 
-      Gitlab::BareRepositoryImporter.execute
+      Gitlab::BareRepositoryImport::Importer.execute(args.import_path)
     end
   end
 end