git: Implement support for bundled Git binaries

Now that we can build bundled Git binaries, this commit introduces
support into the Git command factory to support them. If a new option
"git.use_bundled_binaries" is set to `true`, then we automatically
derive the Git binary's path from the binary directory of Gitaly.

Note that we must go through some hoops though given that the bundled
binaries have a "gitaly-" prefix. While it's not much of a problem when
executing Git directly, some Git commands require it to find auxiliary
helper binaries like "git-remote-http" or "git-receive-pack". To support
these cases, we thus create a temporary directory and symlink all
binaries Git expects to be present into it. By pointing GIT_EXEC_PATH at
this directory, we can thus trick Git into picking the correct set of
binaries.

This new bundled mode will required configuration changes by admins.

10 files changed, 196 insertions, 14 deletions

diff --git a/Makefile b/Makefile
index 1a0869b1c..a8bba2be8 100644
--- a/Makefile
+++ b/Makefile
@@ -242,7 +242,6 @@ export GOCACHE                   ?= ${BUILD_DIR}/cache
 export GOPROXY                   ?= https://proxy.golang.org
 export PATH                      := ${BUILD_DIR}/bin:${PATH}
 export PKG_CONFIG_PATH           := ${LIBGIT2_INSTALL_DIR}/lib/pkgconfig
-export GITALY_TESTING_GIT_BINARY ?= ${GIT_INSTALL_DIR}/bin/git
 # Allow the linker flag -D_THREAD_SAFE as libgit2 is compiled with it on FreeBSD
 export CGO_LDFLAGS_ALLOW          = -D_THREAD_SAFE
 
@@ -312,16 +311,24 @@ build: $(patsubst %,${BUILD_DIR}/bin/gitaly-%,${GIT_EXECUTABLES})
 
 install: $(patsubst %,${INSTALL_DEST_DIR}/gitaly-%,${GIT_EXECUTABLES})
 
+prepare-tests: $(patsubst %,${BUILD_DIR}/bin/gitaly-%,${GIT_EXECUTABLES})
+
 ${BUILD_DIR}/bin/gitaly-%: ${GIT_SOURCE_DIR}/% | ${BUILD_DIR}/bin
 	${Q}install $< $@
 
 ${INSTALL_DEST_DIR}/gitaly-%: ${BUILD_DIR}/bin/gitaly-%
 	${Q}mkdir -p $(@D)
 	${Q}install $< $@
+
+export GITALY_TESTING_BUNDLED_GIT_PATH ?= ${BUILD_DIR}/bin
+else
+prepare-tests: git
+
+export GITALY_TESTING_GIT_BINARY ?= ${GIT_INSTALL_DIR}/bin/git
 endif
 
 .PHONY: prepare-tests
-prepare-tests: git libgit2 prepare-test-repos ${SOURCE_DIR}/.ruby-bundle
+prepare-tests: libgit2 prepare-test-repos ${SOURCE_DIR}/.ruby-bundle
 
 .PHONY: prepare-test-repos
 prepare-test-repos: ${TEST_REPO} ${TEST_REPO_GIT}
diff --git a/internal/command/command.go b/internal/command/command.go
index 5cb6c7d8f..b7417aa9b 100644
--- a/internal/command/command.go
+++ b/internal/command/command.go
@@ -40,6 +40,11 @@ var exportedEnvVars = []string{
 	"GIT_TRACE_PERFORMANCE",
 	"GIT_TRACE_SETUP",
 
+	// GIT_EXEC_PATH tells Git where to find its binaries. This must be exported especially in
+	// the case where we use bundled Git executables given that we cannot rely on a complete Git
+	// installation in that case.
+	"GIT_EXEC_PATH",
+
 	// Git HTTP proxy settings: https://git-scm.com/docs/git-config#git-config-httpproxy
 	"all_proxy",
 	"http_proxy",
diff --git a/internal/git/command_factory.go b/internal/git/command_factory.go
index 8c8d3e5c6..85cfc2b9d 100644
--- a/internal/git/command_factory.go
+++ b/internal/git/command_factory.go
@@ -128,6 +128,7 @@ func (cf *ExecCommandFactory) newCommand(ctx context.Context, repo repository.Gi
 	}
 
 	env = append(env, command.GitEnv...)
+	env = append(env, cf.cfg.GitExecEnv()...)
 
 	execCommand := exec.Command(cf.gitPath(), args...)
 	execCommand.Dir = dir
diff --git a/internal/git/command_options_test.go b/internal/git/command_options_test.go
index a96d5fde3..a4a44385a 100644
--- a/internal/git/command_options_test.go
+++ b/internal/git/command_options_test.go
@@ -196,7 +196,7 @@ func TestGlobalOption(t *testing.T) {
 }
 
 func TestWithConfig(t *testing.T) {
-	var cfg config.Cfg
+	cfg := config.Cfg{BinDir: testhelper.TempDir(t)}
 	require.NoError(t, cfg.SetGitPath())
 
 	ctx, cancel := testhelper.Context()
@@ -269,7 +269,7 @@ func TestWithConfig(t *testing.T) {
 }
 
 func TestExecCommandFactoryGitalyConfigOverrides(t *testing.T) {
-	var cfg config.Cfg
+	cfg := config.Cfg{BinDir: testhelper.TempDir(t)}
 	require.NoError(t, cfg.SetGitPath())
 
 	cfg.Git.Config = []config.GitConfig{
@@ -295,7 +295,7 @@ func TestExecCommandFactoryGitalyConfigOverrides(t *testing.T) {
 }
 
 func TestWithConfigEnv(t *testing.T) {
-	var cfg config.Cfg
+	cfg := config.Cfg{BinDir: testhelper.TempDir(t)}
 	require.NoError(t, cfg.SetGitPath())
 
 	ctx, cancel := testhelper.Context()
diff --git a/internal/git/gittest/command.go b/internal/git/gittest/command.go
index c32953224..1428959ae 100644
--- a/internal/git/gittest/command.go
+++ b/internal/git/gittest/command.go
@@ -52,6 +52,7 @@ func run(t testing.TB, stdin io.Reader, stdout io.Writer, cfg config.Cfg, args,
 		"GIT_CONFIG_KEY_1=init.templateDir",
 		"GIT_CONFIG_VALUE_1=",
 	)
+	cmd.Env = append(cmd.Env, cfg.GitExecEnv()...)
 	cmd.Env = append(cmd.Env, env...)
 
 	cmd.Stdout = stdout
diff --git a/internal/git/gittest/http_server.go b/internal/git/gittest/http_server.go
index ea3bd8c77..75b63621c 100644
--- a/internal/git/gittest/http_server.go
+++ b/internal/git/gittest/http_server.go
@@ -74,12 +74,12 @@ func GitServer(t testing.TB, cfg config.Cfg, repoPath string, middleware func(ht
 		Path: cfg.Git.BinPath,
 		Dir:  "/",
 		Args: []string{"http-backend"},
-		Env: []string{
+		Env: append([]string{
 			"GIT_PROJECT_ROOT=" + filepath.Dir(repoPath),
 			"GIT_CONFIG_COUNT=1",
 			"GIT_CONFIG_KEY_0=http.receivepack",
 			"GIT_CONFIG_VALUE_0=true",
-		},
+		}, cfg.GitExecEnv()...),
 	}
 	s := http.Server{Handler: gitHTTPBackend}
 
diff --git a/internal/gitaly/config/config.go b/internal/gitaly/config/config.go
index 4b46684e3..ef850404d 100644
--- a/internal/gitaly/config/config.go
+++ b/internal/gitaly/config/config.go
@@ -105,9 +105,12 @@ type HTTPSettings struct {
 
 // Git contains the settings for the Git executable
 type Git struct {
-	BinPath          string      `toml:"bin_path"`
-	CatfileCacheSize int         `toml:"catfile_cache_size"`
-	Config           []GitConfig `toml:"config"`
+	UseBundledBinaries bool        `toml:"use_bundled_binaries"`
+	BinPath            string      `toml:"bin_path"`
+	CatfileCacheSize   int         `toml:"catfile_cache_size"`
+	Config             []GitConfig `toml:"config"`
+
+	execEnv []string
 }
 
 // GitConfig contains a key-value pair which is to be passed to git as configuration.
@@ -370,6 +373,71 @@ func (cfg *Cfg) SetGitPath() error {
 		// Nothing to do.
 	case os.Getenv("GITALY_TESTING_GIT_BINARY") != "":
 		cfg.Git.BinPath = os.Getenv("GITALY_TESTING_GIT_BINARY")
+	case os.Getenv("GITALY_TESTING_BUNDLED_GIT_PATH") != "":
+		if cfg.BinDir == "" {
+			return errors.New("cannot use bundled binaries without bin path being set")
+		}
+
+		// We need to symlink pre-built Git binaries into Gitaly's binary directory.
+		// Normally they would of course already exist there, but in tests we create a new
+		// binary directory for each server and thus need to populate it first.
+		for _, binary := range []string{"gitaly-git", "gitaly-git-remote-http", "gitaly-git-http-backend"} {
+			bundledGitBinary := filepath.Join(os.Getenv("GITALY_TESTING_BUNDLED_GIT_PATH"), binary)
+			if _, err := os.Stat(bundledGitBinary); err != nil {
+				return fmt.Errorf("statting %q: %w", binary, err)
+			}
+
+			if err := os.Symlink(bundledGitBinary, filepath.Join(cfg.BinDir, binary)); err != nil {
+				// While Gitaly's Go tests use a temporary binary directory, Ruby
+				// rspecs set up the binary directory to point to our build
+				// directory. They thus already contain the Git binaries and don't
+				// need symlinking.
+				if errors.Is(err, os.ErrExist) {
+					continue
+				}
+				return fmt.Errorf("symlinking bundled %q: %w", binary, err)
+			}
+		}
+
+		cfg.Git.UseBundledBinaries = true
+
+		fallthrough
+	case cfg.Git.UseBundledBinaries:
+		if cfg.Git.BinPath != "" {
+			return errors.New("cannot set Git path and use bundled binaries")
+		}
+
+		// In order to support having a single Git binary only as compared to a complete Git
+		// installation, we create our own GIT_EXEC_PATH which contains symlinks to the Git
+		// binary for executables which Git expects to be present.
+		gitExecPath, err := os.MkdirTemp("", "gitaly-git-exec-path-*")
+		if err != nil {
+			return fmt.Errorf("creating Git exec path: %w", err)
+		}
+
+		for executable, target := range map[string]string{
+			"git":                "gitaly-git",
+			"git-receive-pack":   "gitaly-git",
+			"git-upload-pack":    "gitaly-git",
+			"git-upload-archive": "gitaly-git",
+			"git-http-backend":   "gitaly-git-http-backend",
+			"git-remote-http":    "gitaly-git-remote-http",
+			"git-remote-https":   "gitaly-git-remote-http",
+			"git-remote-ftp":     "gitaly-git-remote-http",
+			"git-remote-ftps":    "gitaly-git-remote-http",
+		} {
+			if err := os.Symlink(
+				filepath.Join(cfg.BinDir, target),
+				filepath.Join(gitExecPath, executable),
+			); err != nil {
+				return fmt.Errorf("linking Git executable %q: %w", executable, err)
+			}
+		}
+
+		cfg.Git.BinPath = filepath.Join(gitExecPath, "git")
+		cfg.Git.execEnv = []string{
+			"GIT_EXEC_PATH=" + gitExecPath,
+		}
 	default:
 		resolvedPath, err := exec.LookPath("git")
 		if err != nil {
@@ -388,6 +456,11 @@ func (cfg *Cfg) SetGitPath() error {
 	return nil
 }
 
+// GitExecEnv returns environment variables required to be set in the environment when Git executes.
+func (cfg *Cfg) GitExecEnv() []string {
+	return cfg.Git.execEnv
+}
+
 // StoragePath looks up the base path for storageName. The second boolean
 // return value indicates if anything was found.
 func (cfg *Cfg) StoragePath(storageName string) (string, bool) {
diff --git a/internal/gitaly/config/config_test.go b/internal/gitaly/config/config_test.go
index 849ba98f8..ded03340d 100644
--- a/internal/gitaly/config/config_test.go
+++ b/internal/gitaly/config/config_test.go
@@ -432,13 +432,16 @@ func TestSetGitPath(t *testing.T) {
 	cleanup := testhelper.ModifyEnvironment(t, "GITALY_TESTING_GIT_BINARY", "")
 	defer cleanup()
 
+	cleanup = testhelper.ModifyEnvironment(t, "GITALY_TESTING_BUNDLED_GIT_PATH", "")
+	defer cleanup()
+
 	t.Run("set in config", func(t *testing.T) {
 		cfg := Cfg{Git: Git{BinPath: "/path/to/myGit"}}
 		require.NoError(t, cfg.SetGitPath())
 		assert.Equal(t, "/path/to/myGit", cfg.Git.BinPath)
 	})
 
-	t.Run("set using env var", func(t *testing.T) {
+	t.Run("set using GITALY_TESTING_GIT_BINARY", func(t *testing.T) {
 		cleanup := testhelper.ModifyEnvironment(t, "GITALY_TESTING_GIT_BINARY", "/path/to/env_git")
 		defer cleanup()
 
@@ -447,6 +450,78 @@ func TestSetGitPath(t *testing.T) {
 		assert.Equal(t, "/path/to/env_git", cfg.Git.BinPath)
 	})
 
+	t.Run("set using GITALY_TESTING_BUNDLED_GIT_PATH", func(t *testing.T) {
+		bundledGitDir := testhelper.TempDir(t)
+		bundledGitExecutable := filepath.Join(bundledGitDir, "gitaly-git")
+		bundledGitRemoteExecutable := filepath.Join(bundledGitDir, "gitaly-git-remote-http")
+		bundledGitHTTPBackendExecutable := filepath.Join(bundledGitDir, "gitaly-git-http-backend")
+
+		cleanup := testhelper.ModifyEnvironment(t, "GITALY_TESTING_BUNDLED_GIT_PATH", bundledGitDir)
+		defer cleanup()
+
+		t.Run("missing bin_dir", func(t *testing.T) {
+			cfg := Cfg{Git: Git{}}
+			require.Equal(t, errors.New("cannot use bundled binaries without bin path being set"), cfg.SetGitPath())
+		})
+
+		t.Run("missing gitaly-git executable", func(t *testing.T) {
+			cfg := Cfg{BinDir: testhelper.TempDir(t), Git: Git{}}
+			err := cfg.SetGitPath()
+			require.Error(t, err)
+			require.Contains(t, err.Error(), `statting "gitaly-git":`)
+			require.True(t, errors.Is(err, os.ErrNotExist))
+		})
+
+		t.Run("missing git-remote-http executable", func(t *testing.T) {
+			require.NoError(t, os.WriteFile(bundledGitExecutable, []byte{}, 0o777))
+
+			cfg := Cfg{BinDir: testhelper.TempDir(t), Git: Git{}}
+			err := cfg.SetGitPath()
+			require.Error(t, err)
+			require.Contains(t, err.Error(), "statting \"gitaly-git-remote-http\":")
+			require.True(t, errors.Is(err, os.ErrNotExist))
+		})
+
+		t.Run("missing git-http-backend executable", func(t *testing.T) {
+			require.NoError(t, os.WriteFile(bundledGitExecutable, []byte{}, 0o777))
+			require.NoError(t, os.WriteFile(bundledGitRemoteExecutable, []byte{}, 0o777))
+
+			cfg := Cfg{BinDir: testhelper.TempDir(t), Git: Git{}}
+			err := cfg.SetGitPath()
+			require.Error(t, err)
+			require.Contains(t, err.Error(), "statting \"gitaly-git-http-backend\":")
+			require.True(t, errors.Is(err, os.ErrNotExist))
+		})
+
+		t.Run("bin_dir with executables", func(t *testing.T) {
+			require.NoError(t, os.WriteFile(bundledGitExecutable, []byte{}, 0o777))
+			require.NoError(t, os.WriteFile(bundledGitRemoteExecutable, []byte{}, 0o777))
+			require.NoError(t, os.WriteFile(bundledGitHTTPBackendExecutable, []byte{}, 0o777))
+
+			binDir := testhelper.TempDir(t)
+
+			cfg := Cfg{BinDir: binDir, Git: Git{}}
+			require.NoError(t, cfg.SetGitPath())
+
+			for _, executable := range []string{"git", "git-remote-http", "git-http-backend"} {
+				symlinkPath := filepath.Join(filepath.Dir(cfg.Git.BinPath), executable)
+
+				// The symlink in Git's temporary BinPath points to the Git
+				// executable in Gitaly's BinDir.
+				target, err := os.Readlink(symlinkPath)
+				require.NoError(t, err)
+				require.Equal(t, filepath.Join(cfg.BinDir, "gitaly-"+executable), target)
+
+				// And in a test setup, the symlink in Gitaly's BinDir must point to
+				// the Git binary pointed to by the GITALY_TESTING_BUNDLED_GIT_PATH
+				// environment variable.
+				target, err = os.Readlink(target)
+				require.NoError(t, err)
+				require.Equal(t, filepath.Join(bundledGitDir, "gitaly-"+executable), target)
+			}
+		})
+	})
+
 	t.Run("not set, get from system", func(t *testing.T) {
 		resolvedPath, err := exec.LookPath("git")
 		require.NoError(t, err)
@@ -526,7 +601,7 @@ func TestValidateGitConfig(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {
-			cfg := Cfg{Git: Git{Config: tc.configPairs}}
+			cfg := Cfg{BinDir: testhelper.TempDir(t), Git: Git{Config: tc.configPairs}}
 			require.Equal(t, tc.expectedErr, cfg.validateGit())
 		})
 	}
diff --git a/internal/testhelper/configure.go b/internal/testhelper/configure.go
index ec9e49661..82fbbbb3d 100644
--- a/internal/testhelper/configure.go
+++ b/internal/testhelper/configure.go
@@ -163,6 +163,8 @@ func configureGit() error {
 	gitPath := "git"
 	if path, ok := os.LookupEnv("GITALY_TESTING_GIT_BINARY"); ok {
 		gitPath = path
+	} else if path, ok := os.LookupEnv("GITALY_TESTING_BUNDLED_GIT_PATH"); ok {
+		gitPath = filepath.Join(path, "gitaly-git")
 	}
 
 	// Unset environment variables which have an effect on Git itself.
diff --git a/ruby/spec/test_repo_helper.rb b/ruby/spec/test_repo_helper.rb
index 603de4d08..91f46dc30 100644
--- a/ruby/spec/test_repo_helper.rb
+++ b/ruby/spec/test_repo_helper.rb
@@ -6,7 +6,22 @@ require 'spec_helper'
 $:.unshift(File.expand_path('../proto', __dir__))
 require 'gitaly'
 
-Gitlab.config.git.test_global_ivar_override(:bin_path, ENV.fetch('GITALY_TESTING_GIT_BINARY', 'git'))
+if ENV.key?('GITALY_TESTING_GIT_BINARY')
+  GIT_BINARY_PATH = ENV['GITALY_TESTING_GIT_BINARY']
+elsif ENV.key?('GITALY_TESTING_BUNDLED_GIT_PATH')
+  GIT_BINARY_PATH = File.join(ENV['GITALY_TESTING_BUNDLED_GIT_PATH'], 'gitaly-git')
+  GIT_EXEC_PATH = File.join(TMP_DIR, 'git-exec-path')
+
+  # We execute git-clone(1) to set up the test repo, and this requires Git to
+  # find git-upload-pack(1). We thus symlink it into a temporary Git exec path
+  # and make it known to Git where it lives.
+  Dir.mkdir(GIT_EXEC_PATH)
+  File.symlink(GIT_BINARY_PATH, File.join(GIT_EXEC_PATH, 'git-upload-pack'))
+else
+  GIT_BINARY_PATH = 'git'.freeze
+end
+
+Gitlab.config.git.test_global_ivar_override(:bin_path, GIT_BINARY_PATH)
 Gitlab.config.git.test_global_ivar_override(:hooks_directory, File.join(GITALY_RUBY_DIR, "hooks"))
 Gitlab.config.gitaly.test_global_ivar_override(:bin_dir, __dir__)
 
@@ -97,7 +112,10 @@ module TestRepo
   end
 
   def self.clone_new_repo!(origin, destination)
-    return if system(Gitlab.config.git.bin_path, "-c", "init.templateDir=", "clone", "--quiet", "--bare", origin.to_s, destination.to_s)
+    env = {}
+    env['GIT_EXEC_PATH'] = GIT_EXEC_PATH if defined?(GIT_EXEC_PATH)
+
+    return if system(env, Gitlab.config.git.bin_path, "-c", "init.templateDir=", "clone", "--quiet", "--bare", origin.to_s, destination.to_s)
 
     abort "Failed to clone test repo. Try running 'make prepare-tests' and try again."
   end