Import gitlab_projects.rb from gitlab-shell

By importing this Ruby code into gitlab-rails (and gitaly-ruby), we avoid
200ms of startup time for each gitlab_projects subprocess we are eliminating.

By not having a gitlab_projects subprocess between gitlab-rails / sidekiq and
any git subprocesses (e.g. for fork_project, fetch_remote, etc, calls), we can
also manage these git processes more cleanly, and avoid sending SIGKILL to them

1 files changed, 82 insertions, 31 deletions

diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb
index a22a63665be..9cdd3d22f18 100644
--- a/lib/gitlab/shell.rb
+++ b/lib/gitlab/shell.rb
@@ -66,7 +66,7 @@ module Gitlab
     # Init new repository
     #
     # storage - project's storage name
-    # name - project path with namespace
+    # name - project disk path
     #
     # Ex.
     #   add_repository("/path/to/storage", "gitlab/gitlab-ci")
@@ -94,23 +94,28 @@ module Gitlab
     # Import repository
     #
     # storage - project's storage path
-    # name - project path with namespace
+    # name - project disk path
+    # url - URL to import from
     #
     # Ex.
-    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git")
+    #   import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def import_repository(storage, name, url)
       # The timeout ensures the subprocess won't hang forever
-      cmd = [gitlab_shell_projects_path, 'import-project',
-             storage, "#{name}.git", url, "#{Gitlab.config.gitlab_shell.git_timeout}"]
-      gitlab_shell_fast_execute_raise_error(cmd)
+      cmd = gitlab_projects(storage, "#{name}.git")
+      success = cmd.import_project(url, git_timeout)
+
+      raise Error, cmd.output unless success
+
+      success
     end
 
     # Fetch remote for repository
     #
     # repository - an instance of Git::Repository
     # remote - remote name
+    # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication
     # forced - should we use --force flag?
     # no_tags - should we use --no-tags flag?
     #
@@ -131,16 +136,15 @@ module Gitlab
 
     # Move repository
     # storage - project's storage path
-    # path - project path with namespace
-    # new_path - new project path with namespace
+    # path - project disk path
+    # new_path - new project disk path
     #
     # Ex.
     #   mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def mv_repository(storage, path, new_path)
-      gitlab_shell_fast_execute([gitlab_shell_projects_path, 'mv-project',
-                                 storage, "#{path}.git", "#{new_path}.git"])
+      gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git")
     end
 
     # Fork repository to new path
@@ -154,30 +158,21 @@ module Gitlab
     #
     # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one.
     def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path)
-      gitlab_shell_fast_execute(
-        [
-          gitlab_shell_projects_path,
-          'fork-repository',
-          forked_from_storage,
-          "#{forked_from_disk_path}.git",
-          forked_to_storage,
-          "#{forked_to_disk_path}.git"
-        ]
-      )
+      gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git")
+        .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git")
     end
 
     # Remove repository from file system
     #
     # storage - project's storage path
-    # name - project path with namespace
+    # name - project disk path
     #
     # Ex.
     #   remove_repository("/path/to/storage", "gitlab/gitlab-ci")
     #
     # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387
     def remove_repository(storage, name)
-      gitlab_shell_fast_execute([gitlab_shell_projects_path,
-                                 'rm-project', storage, "#{name}.git"])
+      gitlab_projects(storage, "#{name}.git").rm_project
     end
 
     # Add new key to gitlab-shell
@@ -311,6 +306,47 @@ module Gitlab
       end
     end
 
+    # Push branch to remote repository
+    #
+    # storage - project's storage path
+    # project_name - project's disk path
+    # remote_name - remote name
+    # branch_names - remote branch names to push
+    # forced - should we use --force flag
+    #
+    # Ex.
+    #   push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature'])
+    #
+    def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true)
+      cmd = gitlab_projects(storage, "#{project_name}.git")
+
+      success = cmd.push_branches(remote_name, git_timeout, forced, branch_names)
+
+      raise Error, cmd.output unless success
+
+      success
+    end
+
+    # Delete branch from remote repository
+    #
+    # storage - project's storage path
+    # project_name - project's disk path
+    # remote_name - remote name
+    # branch_names - remote branch names
+    #
+    # Ex.
+    #   delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature'])
+    #
+    def delete_remote_branches(storage, project_name, remote_name, branch_names)
+      cmd = gitlab_projects(storage, "#{project_name}.git")
+
+      success = cmd.delete_remote_branches(remote_name, branch_names)
+
+      raise Error, cmd.output unless success
+
+      success
+    end
+
     protected
 
     def gitlab_shell_path
@@ -341,24 +377,35 @@ module Gitlab
 
     private
 
-    def local_fetch_remote(storage, name, remote, ssh_auth: nil, forced: false, no_tags: false)
-      args = [gitlab_shell_projects_path, 'fetch-remote', storage, name, remote, "#{Gitlab.config.gitlab_shell.git_timeout}"]
-      args << '--force' if forced
-      args << '--no-tags' if no_tags
+    def gitlab_projects(shard_path, disk_path)
+      Gitlab::Git::GitlabProjects.new(
+        shard_path,
+        disk_path,
+        global_hooks_path: Gitlab.config.gitlab_shell.hooks_path,
+        logger: Rails.logger
+      )
+    end
 
-      vars = {}
+    def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false)
+      vars = { force: forced, tags: !no_tags }
 
       if ssh_auth&.ssh_import?
         if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?
-          vars['GITLAB_SHELL_SSH_KEY'] = ssh_auth.ssh_private_key
+          vars[:ssh_key] = ssh_auth.ssh_private_key
         end
 
         if ssh_auth.ssh_known_hosts.present?
-          vars['GITLAB_SHELL_KNOWN_HOSTS'] = ssh_auth.ssh_known_hosts
+          vars[:known_hosts] = ssh_auth.ssh_known_hosts
         end
       end
 
-      gitlab_shell_fast_execute_raise_error(args, vars)
+      cmd = gitlab_projects(storage_path, repository_relative_path)
+
+      success = cmd.fetch_remote(remote, git_timeout, vars)
+
+      raise Error, cmd.output unless success
+
+      success
     end
 
     def gitlab_shell_fast_execute(cmd)
@@ -394,6 +441,10 @@ module Gitlab
       Gitlab::GitalyClient::NamespaceService.new(storage)
     end
 
+    def git_timeout
+      Gitlab.config.gitlab_shell.git_timeout
+    end
+
     def gitaly_migrate(method, &block)
       Gitlab::GitalyClient.migrate(method, &block)
     rescue GRPC::NotFound, GRPC::BadStatus => e