Merge branch 'smh-pack-aux-binaries' into 'master'

Pin auxiliary binaries the main Gitaly binary invokes

Closes #3253 and #4303

See merge request gitlab-org/gitaly!4670

24 files changed, 203 insertions, 53 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 151cc3cff..10d530d0c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -104,7 +104,7 @@ include:
     # But the actual tests should run unprivileged. This assures that we pay
     # proper attention to permission bits and that we don't modify the source
     # directory.
-    - setpriv --reuid=${TEST_UID} --regid=${TEST_UID} --clear-groups --no-new-privs make ${TEST_TARGET} SKIP_RSPEC_BUILD=YesPlease $(test "${GIT_VERSION}" = default && echo WITH_BUNDLED_GIT=YesPlease)
+    - setpriv --reuid=${TEST_UID} --regid=${TEST_UID} --clear-groups --no-new-privs make ${TEST_TARGET} UNPRIVILEGED_CI_SKIP=YesPlease $(test "${GIT_VERSION}" = default && echo WITH_BUNDLED_GIT=YesPlease)
 
   after_script:
     - |
diff --git a/Makefile b/Makefile
index f08d914bb..3dacc50f2 100644
--- a/Makefile
+++ b/Makefile
@@ -241,10 +241,12 @@ TEST_REPO_MIRROR  := ${TEST_REPO_DIR}/gitlab-test-mirror.git
 TEST_REPO_GIT     := ${TEST_REPO_DIR}/gitlab-git-test.git
 BENCHMARK_REPO    := ${TEST_REPO_DIR}/benchmark.git
 
-# All executables provided by Gitaly
-GITALY_EXECUTABLES    = $(addprefix ${BUILD_DIR}/bin/,$(notdir $(shell find ${SOURCE_DIR}/cmd -mindepth 1 -maxdepth 1 -type d -print)) gitaly-git2go-v14)
+# All executables provided by Gitaly.
+GITALY_EXECUTABLES        = $(addprefix ${BUILD_DIR}/bin/,$(notdir $(shell find ${SOURCE_DIR}/cmd -mindepth 1 -maxdepth 1 -type d -print)) gitaly-git2go-v14)
+# All executables packed inside the Gitaly binary.
+GITALY_PACKED_EXECUTABLES = $(filter %gitaly-hooks %gitaly-git2go-v15 %gitaly-ssh %gitaly-lfs-smudge, ${GITALY_EXECUTABLES})
 # Find all Go source files.
-find_go_sources       = $(shell find ${SOURCE_DIR} -type d \( -name ruby -o -name vendor -o -name testdata -o -name '_*' -o -path '*/proto/go/gitalypb' \) -prune -o -type f -name '*.go' -not -name '*.pb.go' -print | sort -u)
+find_go_sources           = $(shell find ${SOURCE_DIR} -type d \( -name ruby -o -name vendor -o -name testdata -o -name '_*' -o -path '*/proto/go/gitalypb' \) -prune -o -type f -name '*.go' -not -name '*.pb.go' -print | sort -u)
 
 # run_go_tests will execute Go tests with all required parameters. Its
 # behaviour can be modified via the following variables:
@@ -351,6 +353,9 @@ endif
 
 .PHONY: prepare-tests
 prepare-tests: libgit2 prepare-test-repos ${SOURCE_DIR}/.ruby-bundle ${GOTESTSUM}
+ifndef UNPRIVILEGED_CI_SKIP
+prepare-tests: ${GITALY_PACKED_EXECUTABLES}
+endif
 	${Q}mkdir -p "$(dir ${TEST_REPORT})"
 
 .PHONY: prepare-debug
@@ -403,10 +408,10 @@ rspec: prepare-tests
 	${Q}cd ${GITALY_RUBY_DIR} && PATH='${SOURCE_DIR}/internal/testhelper/testdata/home/bin:${PATH}' bundle exec rspec
 
 # This is a workaround for our unprivileged CI builds. We manually execute the
-# build target as privileged user, but then run the rspec target unprivileged.
+# build target as privileged user, but then run other targets unprivileged.
 # We thus cannot rebuild binaries there given that we have no permissions to
 # write into the build directory.
-ifndef SKIP_RSPEC_BUILD
+ifndef UNPRIVILEGED_CI_SKIP
 rspec: build
 endif
 
@@ -422,7 +427,7 @@ check-mod-tidy:
 
 .PHONY: lint
 ## Run Go linter.
-lint: ${GOLANGCI_LINT} libgit2
+lint: ${GOLANGCI_LINT} libgit2 ${GITALY_PACKED_EXECUTABLES}
 	${Q}${GOLANGCI_LINT} run --build-tags "${SERVER_BUILD_TAGS},${GIT2GO_BUILD_TAGS}" --out-format tab --config ${GOLANGCI_LINT_CONFIG} ${GOLANGCI_LINT_OPTIONS}
 
 .PHONY: format
@@ -529,7 +534,7 @@ ${SOURCE_DIR}/.ruby-bundle: ${GITALY_RUBY_DIR}/Gemfile.lock ${GITALY_RUBY_DIR}/G
 ${SOURCE_DIR}/NOTICE: ${BUILD_DIR}/NOTICE
 	${Q}mv $< $@
 
-${BUILD_DIR}/NOTICE: ${GO_LICENSES} clean-ruby-vendor-go
+${BUILD_DIR}/NOTICE: ${GO_LICENSES} clean-ruby-vendor-go ${GITALY_PACKED_EXECUTABLES}
 	${Q}rm -rf ${BUILD_DIR}/licenses
 	${Q}GOOS=linux GOFLAGS="-tags=${SERVER_BUILD_TAGS},${GIT2GO_BUILD_TAGS}" ${GO_LICENSES} save ${SOURCE_DIR}/... --save_path=${BUILD_DIR}/licenses
 	${Q}go run ${SOURCE_DIR}/tools/noticegen/noticegen.go -source ${BUILD_DIR}/licenses -template ${SOURCE_DIR}/tools/noticegen/notice.template > ${BUILD_DIR}/NOTICE
@@ -584,6 +589,7 @@ ${BUILD_DIR}/bin/%: ${BUILD_DIR}/intermediate/% | ${BUILD_DIR}/bin
 	fi
 
 ${BUILD_DIR}/intermediate/gitaly:            GO_BUILD_TAGS = ${SERVER_BUILD_TAGS}
+${BUILD_DIR}/intermediate/gitaly:            ${GITALY_PACKED_EXECUTABLES}
 ${BUILD_DIR}/intermediate/praefect:          GO_BUILD_TAGS = ${SERVER_BUILD_TAGS}
 ${BUILD_DIR}/intermediate/gitaly-git2go-v15: GO_BUILD_TAGS = ${GIT2GO_BUILD_TAGS}
 ${BUILD_DIR}/intermediate/gitaly-git2go-v15: libgit2
diff --git a/cmd/gitaly-hooks/hooks_test.go b/cmd/gitaly-hooks/hooks_test.go
index 109b15502..ef9e0f786 100644
--- a/cmd/gitaly-hooks/hooks_test.go
+++ b/cmd/gitaly-hooks/hooks_test.go
@@ -161,8 +161,6 @@ func testHooksPrePostReceive(t *testing.T, cfg config.Cfg, repo *gitalypb.Reposi
 	cfg.Gitlab.HTTPSettings.User = gitlabUser
 	cfg.Gitlab.HTTPSettings.Password = gitlabPassword
 
-	gitalyHooksPath := filepath.Join(cfg.BinDir, "gitaly-hooks")
-
 	gitlabClient, err := gitlab.NewHTTPClient(logger, cfg.Gitlab, cfg.TLS, prometheus.Config{})
 	require.NoError(t, err)
 
@@ -180,7 +178,7 @@ func testHooksPrePostReceive(t *testing.T, cfg config.Cfg, repo *gitalypb.Reposi
 			var stderr, stdout bytes.Buffer
 			stdin := bytes.NewBuffer([]byte(changes))
 			require.NoError(t, err)
-			cmd := exec.Command(gitalyHooksPath)
+			cmd := exec.Command(cfg.BinaryPath("gitaly-hooks"))
 			cmd.Args = []string{hookName}
 			cmd.Stderr = &stderr
 			cmd.Stdout = &stdout
@@ -278,7 +276,7 @@ func testHooksUpdate(t *testing.T, ctx context.Context, cfg config.Cfg, glValues
 
 	refval, oldval, newval := "refval", strings.Repeat("a", 40), strings.Repeat("b", 40)
 
-	cmd := exec.Command(filepath.Join(cfg.BinDir, "gitaly-hooks"))
+	cmd := exec.Command(cfg.BinaryPath("gitaly-hooks"))
 	cmd.Args = []string{"update", refval, oldval, newval}
 	cmd.Env = envForHooks(t, ctx, cfg, repo, glValues, proxyValues{})
 	cmd.Dir = repoPath
@@ -522,7 +520,7 @@ func TestCheckOK(t *testing.T) {
 
 	configPath := writeTemporaryGitalyConfigFile(t, cfg)
 
-	cmd := exec.Command(filepath.Join(cfg.BinDir, "gitaly-hooks"), "check", configPath)
+	cmd := exec.Command(cfg.BinaryPath("gitaly-hooks"), "check", configPath)
 
 	var stderr, stdout bytes.Buffer
 	cmd.Stderr = &stderr
@@ -568,7 +566,7 @@ func TestCheckBadCreds(t *testing.T) {
 
 	configPath := writeTemporaryGitalyConfigFile(t, cfg)
 
-	cmd := exec.Command(filepath.Join(cfg.BinDir, "gitaly-hooks"), "check", configPath)
+	cmd := exec.Command(cfg.BinaryPath("gitaly-hooks"), "check", configPath)
 
 	var stderr, stdout bytes.Buffer
 	cmd.Stderr = &stderr
@@ -652,7 +650,7 @@ func TestGitalyHooksPackObjects(t *testing.T) {
 	baseArgs := []string{
 		"clone",
 		"-u",
-		"git -c uploadpack.allowFilter -c uploadpack.packObjectsHook=" + cfg.BinDir + "/gitaly-hooks upload-pack",
+		"git -c uploadpack.allowFilter -c uploadpack.packObjectsHook=" + cfg.BinaryPath("gitaly-hooks") + " upload-pack",
 		"--quiet",
 		"--no-local",
 		"--bare",
@@ -702,7 +700,7 @@ func TestRequestedHooks(t *testing.T) {
 				payload, err := git.NewHooksPayload(cfg, &gitalypb.Repository{}, nil, nil, git.AllHooks&^hook, nil).Env()
 				require.NoError(t, err)
 
-				cmd := exec.Command(filepath.Join(cfg.BinDir, "gitaly-hooks"))
+				cmd := exec.Command(cfg.BinaryPath("gitaly-hooks"))
 				cmd.Args = hookArgs
 				cmd.Env = []string{payload}
 				require.NoError(t, cmd.Run())
@@ -716,7 +714,7 @@ func TestRequestedHooks(t *testing.T) {
 				payload, err := git.NewHooksPayload(cfg, &gitalypb.Repository{}, nil, nil, hook, nil).Env()
 				require.NoError(t, err)
 
-				cmd := exec.Command(filepath.Join(cfg.BinDir, "gitaly-hooks"))
+				cmd := exec.Command(cfg.BinaryPath("gitaly-hooks"))
 				cmd.Args = hookArgs
 				cmd.Env = []string{payload}
 
diff --git a/cmd/gitaly-ssh/auth_test.go b/cmd/gitaly-ssh/auth_test.go
index 007522314..3dd9eb1b2 100644
--- a/cmd/gitaly-ssh/auth_test.go
+++ b/cmd/gitaly-ssh/auth_test.go
@@ -122,7 +122,7 @@ func TestConnectivity(t *testing.T) {
 				fmt.Sprintf("GITALY_ADDRESS=%s", addr),
 				fmt.Sprintf("GITALY_WD=%s", cwd),
 				fmt.Sprintf("PATH=.:%s", os.Getenv("PATH")),
-				fmt.Sprintf("GIT_SSH_COMMAND=%s upload-pack", filepath.Join(cfg.BinDir, "gitaly-ssh")),
+				fmt.Sprintf("GIT_SSH_COMMAND=%s upload-pack", cfg.BinaryPath("gitaly-ssh")),
 				fmt.Sprintf("SSL_CERT_FILE=%s", certFile),
 			}
 			if testcase.proxy {
diff --git a/cmd/gitaly-ssh/upload_pack_test.go b/cmd/gitaly-ssh/upload_pack_test.go
index fe85df5f1..fc6a99ce3 100644
--- a/cmd/gitaly-ssh/upload_pack_test.go
+++ b/cmd/gitaly-ssh/upload_pack_test.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -85,7 +84,7 @@ func TestVisibilityOfHiddenRefs(t *testing.T) {
 				fmt.Sprintf("GITALY_ADDRESS=unix:%s", socketPath),
 				fmt.Sprintf("GITALY_WD=%s", wd),
 				fmt.Sprintf("PATH=.:%s", os.Getenv("PATH")),
-				fmt.Sprintf("GIT_SSH_COMMAND=%s upload-pack", filepath.Join(cfg.BinDir, "gitaly-ssh")),
+				fmt.Sprintf("GIT_SSH_COMMAND=%s upload-pack", cfg.BinaryPath("gitaly-ssh")),
 			}
 
 			stdout := &bytes.Buffer{}
diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go
index ad9d143ac..793bb015e 100644
--- a/cmd/gitaly/main.go
+++ b/cmd/gitaly/main.go
@@ -11,6 +11,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	log "github.com/sirupsen/logrus"
+	"gitlab.com/gitlab-org/gitaly/v15"
 	"gitlab.com/gitlab-org/gitaly/v15/client"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/backchannel"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/bootstrap"
@@ -154,6 +155,10 @@ func run(cfg config.Cfg) error {
 		}
 	}()
 
+	if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir); err != nil {
+		return fmt.Errorf("unpack auxiliary binaries: %w", err)
+	}
+
 	b, err := bootstrap.New(promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitaly_connections_total",
diff --git a/internal/git/command_factory.go b/internal/git/command_factory.go
index 9a4c8b881..de4ee857e 100644
--- a/internal/git/command_factory.go
+++ b/internal/git/command_factory.go
@@ -275,11 +275,9 @@ func setupHookDirectories(cfg config.Cfg, factoryCfg execCommandFactoryConfig) (
 		return hookDirectories{}, nil, fmt.Errorf("creating temporary hooks directory: %w", err)
 	}
 
-	gitalyHooksPath := filepath.Join(cfg.BinDir, "gitaly-hooks")
-
 	// And now we symlink all required hooks to the wrapper script.
 	for _, hook := range []string{"pre-receive", "post-receive", "update", "reference-transaction"} {
-		if err := os.Symlink(gitalyHooksPath, filepath.Join(tempHooksPath, hook)); err != nil {
+		if err := os.Symlink(cfg.BinaryPath("gitaly-hooks"), filepath.Join(tempHooksPath, hook)); err != nil {
 			return hookDirectories{}, nil, fmt.Errorf("creating symlink for %s hook: %w", hook, err)
 		}
 	}
diff --git a/internal/git/command_factory_test.go b/internal/git/command_factory_test.go
index 9a60a78c4..de98c6f97 100644
--- a/internal/git/command_factory_test.go
+++ b/internal/git/command_factory_test.go
@@ -347,7 +347,7 @@ func TestExecCommandFactoryHooksPath(t *testing.T) {
 			for _, hook := range []string{"update", "pre-receive", "post-receive", "reference-transaction"} {
 				target, err := os.Readlink(filepath.Join(hooksPath, hook))
 				require.NoError(t, err)
-				require.Equal(t, filepath.Join(cfg.BinDir, "gitaly-hooks"), target)
+				require.Equal(t, cfg.BinaryPath("gitaly-hooks"), target)
 			}
 		})
 
diff --git a/internal/git/command_options.go b/internal/git/command_options.go
index 89e24eabc..d72c8f120 100644
--- a/internal/git/command_options.go
+++ b/internal/git/command_options.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -292,7 +291,7 @@ func withInternalFetch(req repoScopedRequest, withSidechannel bool) func(ctx con
 
 		c.env = append(c.env,
 			fmt.Sprintf("GITALY_PAYLOAD=%s", payload),
-			fmt.Sprintf("GIT_SSH_COMMAND=%s %s", filepath.Join(cfg.BinDir, "gitaly-ssh"), "upload-pack"),
+			fmt.Sprintf("GIT_SSH_COMMAND=%s %s", cfg.BinaryPath("gitaly-ssh"), "upload-pack"),
 			fmt.Sprintf("GITALY_ADDRESS=%s", storageInfo.Address),
 			fmt.Sprintf("GITALY_TOKEN=%s", storageInfo.Token),
 			fmt.Sprintf("GITALY_FEATUREFLAGS=%s", strings.Join(flagsWithValue, ",")),
diff --git a/internal/git/command_options_test.go b/internal/git/command_options_test.go
index 88f41d2d0..e5616d476 100644
--- a/internal/git/command_options_test.go
+++ b/internal/git/command_options_test.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/base64"
 	"fmt"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -466,7 +465,7 @@ func TestWithInternalFetch(t *testing.T) {
 			require.NoError(t, option(ctx, cfg, gitCmdFactory, &commandCfg))
 
 			require.Subset(t, commandCfg.env, []string{
-				fmt.Sprintf("GIT_SSH_COMMAND=%s upload-pack", filepath.Join(cfg.BinDir, "gitaly-ssh")),
+				fmt.Sprintf("GIT_SSH_COMMAND=%s upload-pack", cfg.BinaryPath("gitaly-ssh")),
 				fmt.Sprintf("GITALY_PAYLOAD=%s", tc.expectedPayload),
 				"CORRELATION_ID=correlation-id-1",
 				"GIT_SSH_VARIANT=simple",
diff --git a/internal/git/hooks_options.go b/internal/git/hooks_options.go
index 9a77a2ce8..c99f87f92 100644
--- a/internal/git/hooks_options.go
+++ b/internal/git/hooks_options.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"path/filepath"
 
 	"gitlab.com/gitlab-org/gitaly/v15/internal/git/repository"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config"
@@ -80,7 +79,7 @@ func WithPackObjectsHookEnv(repo *gitalypb.Repository, protocol string) CmdOpt {
 
 		cc.globals = append(cc.globals, ConfigPair{
 			Key:   "uploadpack.packObjectsHook",
-			Value: filepath.Join(cfg.BinDir, "gitaly-hooks"),
+			Value: cfg.BinaryPath("gitaly-hooks"),
 		})
 
 		return nil
diff --git a/internal/git/smudge/config.go b/internal/git/smudge/config.go
index c64c4f643..ad126e447 100644
--- a/internal/git/smudge/config.go
+++ b/internal/git/smudge/config.go
@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"path/filepath"
 
 	"gitlab.com/gitlab-org/gitaly/v15/internal/git"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config"
@@ -100,12 +99,12 @@ func (c Config) GitConfiguration(cfg config.Cfg) (git.ConfigPair, error) {
 	case DriverTypeFilter:
 		return git.ConfigPair{
 			Key:   "filter.lfs.smudge",
-			Value: filepath.Join(cfg.BinDir, "gitaly-lfs-smudge"),
+			Value: cfg.BinaryPath("gitaly-lfs-smudge"),
 		}, nil
 	case DriverTypeProcess:
 		return git.ConfigPair{
 			Key:   "filter.lfs.process",
-			Value: filepath.Join(cfg.BinDir, "gitaly-lfs-smudge"),
+			Value: cfg.BinaryPath("gitaly-lfs-smudge"),
 		}, nil
 	default:
 		return git.ConfigPair{}, fmt.Errorf("unknown driver type: %v", c.DriverType)
diff --git a/internal/git2go/executor.go b/internal/git2go/executor.go
index 5007b6c6f..264f34a56 100644
--- a/internal/git2go/executor.go
+++ b/internal/git2go/executor.go
@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
-	"path/filepath"
 	"strings"
 
 	"gitlab.com/gitlab-org/gitaly/v15/internal/command"
@@ -19,7 +18,6 @@ import (
 	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/storage"
 	glog "gitlab.com/gitlab-org/gitaly/v15/internal/log"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag"
-	"gitlab.com/gitlab-org/gitaly/v15/internal/version"
 	"gitlab.com/gitlab-org/labkit/correlation"
 )
 
@@ -28,7 +26,7 @@ var (
 	ErrInvalidArgument = errors.New("invalid parameters")
 
 	// BinaryName is a binary name with version suffix .
-	BinaryName = "gitaly-git2go-" + version.GetModuleVersion()
+	BinaryName = "gitaly-git2go-v15"
 )
 
 // Executor executes gitaly-git2go.
@@ -43,7 +41,7 @@ type Executor struct {
 // configuration.
 func NewExecutor(cfg config.Cfg, gitCmdFactory git.CommandFactory, locator storage.Locator) *Executor {
 	return &Executor{
-		binaryPath:    filepath.Join(cfg.BinDir, BinaryName),
+		binaryPath:    cfg.BinaryPath(BinaryName),
 		gitCmdFactory: gitCmdFactory,
 		locator:       locator,
 		logFormat:     cfg.Logging.Format,
diff --git a/internal/gitaly/config/config.go b/internal/gitaly/config/config.go
index 6e06ab8a3..b467dd9fb 100644
--- a/internal/gitaly/config/config.go
+++ b/internal/gitaly/config/config.go
@@ -281,6 +281,31 @@ func validateIsDirectory(path, name string) error {
 	return nil
 }
 
+// packedBinaries are the binaries that are packed in the main Gitaly binary. This should always match
+// the actual list in <root>/packed_binaries.go so the binaries are correctly located.
+//
+// Resolving the names automatically from the packed binaries is not possible at the moment due to how
+// the packed binaries themselves depend on this config package. If this config package inspected the
+// packed binaries, there would be a cyclic dependency. Anything that the packed binaries import must
+// not depend on <root>/packed_binaries.go.
+var packedBinaries = map[string]struct{}{
+	"gitaly-hooks":      {},
+	"gitaly-ssh":        {},
+	"gitaly-git2go-v15": {},
+	"gitaly-lfs-smudge": {},
+}
+
+// BinaryPath returns the path to a given binary. BinaryPath does not do any validation, it simply joins the binaryName
+// with the correct base directory depending on whether the binary is a packed binary or not.
+func (cfg *Cfg) BinaryPath(binaryName string) string {
+	baseDirectory := cfg.BinDir
+	if _, ok := packedBinaries[binaryName]; ok {
+		baseDirectory = cfg.RuntimeDir
+	}
+
+	return filepath.Join(baseDirectory, binaryName)
+}
+
 func (cfg *Cfg) validateStorages() error {
 	if len(cfg.Storages) == 0 {
 		return fmt.Errorf("no storage configurations found. Are you using the right format? https://gitlab.com/gitlab-org/gitaly/issues/397")
diff --git a/internal/gitaly/config/config_test.go b/internal/gitaly/config/config_test.go
index 313331ae6..b5690bf98 100644
--- a/internal/gitaly/config/config_test.go
+++ b/internal/gitaly/config/config_test.go
@@ -20,6 +20,17 @@ import (
 	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
 )
 
+func TestBinaryPath(t *testing.T) {
+	cfg := Cfg{
+		BinDir:     "bindir",
+		RuntimeDir: "runtime",
+	}
+
+	require.Equal(t, "runtime/gitaly-hooks", cfg.BinaryPath("gitaly-hooks"))
+	require.Equal(t, "bindir/gitaly-debug", cfg.BinaryPath("gitaly-debug"))
+	require.Equal(t, "bindir", cfg.BinaryPath(""))
+}
+
 func TestLoadBrokenConfig(t *testing.T) {
 	tmpFile := strings.NewReader(`path = "/tmp"\nname="foo"`)
 	_, err := Load(tmpFile)
diff --git a/internal/gitaly/service/repository/replicate_test.go b/internal/gitaly/service/repository/replicate_test.go
index 69c049ea9..1b6d17954 100644
--- a/internal/gitaly/service/repository/replicate_test.go
+++ b/internal/gitaly/service/repository/replicate_test.go
@@ -420,7 +420,7 @@ func listenGitalySSHCalls(t *testing.T, conf config.Cfg) func() gitalySSHParams
 	t.Helper()
 
 	require.NotEmpty(t, conf.BinDir)
-	initialPath := filepath.Join(conf.BinDir, "gitaly-ssh")
+	initialPath := conf.BinaryPath("gitaly-ssh")
 	updatedPath := initialPath + "-actual"
 	require.NoError(t, os.Rename(initialPath, updatedPath))
 
diff --git a/internal/gitaly/service/repository/server.go b/internal/gitaly/service/repository/server.go
index 1c0cb7506..a6d8faa07 100644
--- a/internal/gitaly/service/repository/server.go
+++ b/internal/gitaly/service/repository/server.go
@@ -23,7 +23,6 @@ type server struct {
 	txManager           transaction.Manager
 	gitCmdFactory       git.CommandFactory
 	cfg                 config.Cfg
-	binDir              string
 	loggingCfg          config.Logging
 	catfileCache        catfile.Cache
 	git2goExecutor      *git2go.Executor
@@ -49,7 +48,6 @@ func NewServer(
 		gitCmdFactory:       gitCmdFactory,
 		conns:               connsPool,
 		cfg:                 cfg,
-		binDir:              cfg.BinDir,
 		loggingCfg:          cfg.Logging,
 		catfileCache:        catfileCache,
 		git2goExecutor:      git2goExecutor,
diff --git a/internal/gitaly/service/smarthttp/upload_pack_test.go b/internal/gitaly/service/smarthttp/upload_pack_test.go
index 5678a1866..a9dfad29c 100644
--- a/internal/gitaly/service/smarthttp/upload_pack_test.go
+++ b/internal/gitaly/service/smarthttp/upload_pack_test.go
@@ -287,7 +287,7 @@ func testServerPostUploadPackUsesPackObjectsHook(t *testing.T, ctx context.Conte
 	// out. In the best case we'd have just printed the error to stderr and
 	// check the return error message. But it's unfortunately not
 	// transferred back.
-	testhelper.WriteExecutable(t, filepath.Join(cfg.BinDir, "gitaly-hooks"), []byte(hookScript))
+	testhelper.WriteExecutable(t, cfg.BinaryPath("gitaly-hooks"), []byte(hookScript))
 
 	cfg.SocketPath = runSmartHTTPServer(t, cfg)
 
diff --git a/internal/gitaly/service/ssh/receive_pack_test.go b/internal/gitaly/service/ssh/receive_pack_test.go
index a65c17ef5..342285ccc 100644
--- a/internal/gitaly/service/ssh/receive_pack_test.go
+++ b/internal/gitaly/service/ssh/receive_pack_test.go
@@ -684,7 +684,7 @@ func sshPushCommand(ctx context.Context, t *testing.T, cfg config.Cfg, cloneDeta
 		fmt.Sprintf("GITALY_PAYLOAD=%s", payload),
 		fmt.Sprintf("GITALY_ADDRESS=%s", serverSocketPath),
 		fmt.Sprintf("GITALY_FEATUREFLAGS=%s", strings.Join(flagsWithValues, ",")),
-		fmt.Sprintf("GIT_SSH_COMMAND=%s receive-pack", filepath.Join(cfg.BinDir, "gitaly-ssh")),
+		fmt.Sprintf("GIT_SSH_COMMAND=%s receive-pack", cfg.BinaryPath("gitaly-ssh")),
 	}
 
 	return cmd
diff --git a/internal/gitaly/service/ssh/upload_archive_test.go b/internal/gitaly/service/ssh/upload_archive_test.go
index b9be5035c..1e1a95b83 100644
--- a/internal/gitaly/service/ssh/upload_archive_test.go
+++ b/internal/gitaly/service/ssh/upload_archive_test.go
@@ -3,7 +3,6 @@ package ssh
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -136,7 +135,7 @@ func TestUploadArchiveSuccess(t *testing.T) {
 			fmt.Sprintf("GITALY_ADDRESS=%s", cfg.SocketPath),
 			fmt.Sprintf("GITALY_PAYLOAD=%s", payload),
 			fmt.Sprintf("PATH=%s", ".:"+os.Getenv("PATH")),
-			fmt.Sprintf(`GIT_SSH_COMMAND=%s upload-archive`, filepath.Join(cfg.BinDir, "gitaly-ssh")),
+			fmt.Sprintf(`GIT_SSH_COMMAND=%s upload-archive`, cfg.BinaryPath("gitaly-ssh")),
 		},
 	}, "archive", "master", "--remote=git@localhost:test/test.git")
 }
diff --git a/internal/gitaly/service/ssh/upload_pack_test.go b/internal/gitaly/service/ssh/upload_pack_test.go
index a9d010699..811517e84 100644
--- a/internal/gitaly/service/ssh/upload_pack_test.go
+++ b/internal/gitaly/service/ssh/upload_pack_test.go
@@ -65,7 +65,7 @@ func runClone(
 		fmt.Sprintf("GITALY_ADDRESS=%s", cfg.SocketPath),
 		fmt.Sprintf("GITALY_PAYLOAD=%s", payload),
 		fmt.Sprintf("GITALY_FEATUREFLAGS=%s", strings.Join(flagsWithValues, ",")),
-		fmt.Sprintf(`GIT_SSH_COMMAND=%s upload-pack`, filepath.Join(cfg.BinDir, "gitaly-ssh")),
+		fmt.Sprintf(`GIT_SSH_COMMAND=%s upload-pack`, cfg.BinaryPath("gitaly-ssh")),
 	}
 	if withSidechannel {
 		env = append(env, "GITALY_USE_SIDECHANNEL=1")
@@ -622,7 +622,7 @@ func TestUploadPack_packObjectsHook(t *testing.T) {
 	// custom script which replaces the hook binary. It doesn't do anything
 	// special, but writes an error message and errors out and should thus
 	// cause the clone to fail with this error message.
-	testhelper.WriteExecutable(t, filepath.Join(filterDir, "gitaly-hooks"), []byte(fmt.Sprintf(
+	testhelper.WriteExecutable(t, cfg.BinaryPath("gitaly-hooks"), []byte(fmt.Sprintf(
 		`#!/bin/bash
 		set -eo pipefail
 		echo 'I was invoked' >'%s'
@@ -685,7 +685,7 @@ func testUploadPackWithoutSideband(t *testing.T, opts ...testcfg.Option) {
 	// Those simultaneous writes to both stdout and stderr created a race as we could've invoked
 	// two concurrent `SendMsg`s on the gRPC stream. And given that `SendMsg` is not thread-safe
 	// a deadlock would result.
-	uploadPack := exec.Command(filepath.Join(cfg.BinDir, "gitaly-ssh"), "upload-pack", "dontcare", "dontcare")
+	uploadPack := exec.Command(cfg.BinaryPath("gitaly-ssh"), "upload-pack", "dontcare", "dontcare")
 	uploadPack.Env = []string{
 		fmt.Sprintf("GITALY_ADDRESS=%s", cfg.SocketPath),
 		fmt.Sprintf("GITALY_PAYLOAD=%s", payload),
diff --git a/internal/testhelper/testcfg/build.go b/internal/testhelper/testcfg/build.go
index b49b8f47b..d8a88f0df 100644
--- a/internal/testhelper/testcfg/build.go
+++ b/internal/testhelper/testcfg/build.go
@@ -21,33 +21,39 @@ var buildOnceByName sync.Map
 
 // BuildGitalyGit2Go builds the gitaly-git2go command and installs it into the binary directory.
 func BuildGitalyGit2Go(t testing.TB, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-git2go-v15"))
+	return buildGitalyCommand(t, cfg, "gitaly-git2go-v15")
 }
 
 // BuildGitalyWrapper builds the gitaly-wrapper command and installs it into the binary directory.
 func BuildGitalyWrapper(t *testing.T, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-wrapper"))
+	return buildGitalyCommand(t, cfg, "gitaly-wrapper")
 }
 
 // BuildGitalyLFSSmudge builds the gitaly-lfs-smudge command and installs it into the binary
 // directory.
 func BuildGitalyLFSSmudge(t *testing.T, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-lfs-smudge"))
+	return buildGitalyCommand(t, cfg, "gitaly-lfs-smudge")
 }
 
 // BuildGitalyHooks builds the gitaly-hooks command and installs it into the binary directory.
 func BuildGitalyHooks(t testing.TB, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-hooks"))
+	return buildGitalyCommand(t, cfg, "gitaly-hooks")
 }
 
 // BuildGitalySSH builds the gitaly-ssh command and installs it into the binary directory.
 func BuildGitalySSH(t testing.TB, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-ssh"))
+	return buildGitalyCommand(t, cfg, "gitaly-ssh")
 }
 
 // BuildPraefect builds the praefect command and installs it into the binary directory.
 func BuildPraefect(t testing.TB, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("praefect"))
+	return buildGitalyCommand(t, cfg, "praefect")
+}
+
+// buildGitalyCommand builds an executable and places it in the correct directory depending
+// whether it is packed in the production build or not.
+func buildGitalyCommand(t testing.TB, cfg config.Cfg, executableName string) string {
+	return BuildBinary(t, filepath.Dir(cfg.BinaryPath(executableName)), gitalyCommandPath(executableName))
 }
 
 var (
diff --git a/packed_binaries.go b/packed_binaries.go
new file mode 100644
index 000000000..16c02f2e4
--- /dev/null
+++ b/packed_binaries.go
@@ -0,0 +1,67 @@
+package gitaly
+
+import (
+	"embed"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// buildDir is the directory path where our build target places the built binaries.
+const buildDir = "_build/bin"
+
+//go:embed _build/bin/gitaly-hooks _build/bin/gitaly-ssh _build/bin/gitaly-git2go-v15 _build/bin/gitaly-lfs-smudge
+//
+// packedBinariesFS contains embedded binaries. If you modify the above embeddings, you must also update
+// GITALY_PACKED_EXECUTABLES in Makefile and packedBinaries in internal/gitaly/config/config.go.
+var packedBinariesFS embed.FS
+
+// UnpackAuxiliaryBinaries unpacks the packed auxiliary binaries of Gitaly into destination directory.
+//
+// Gitaly invoking auxiliary binaries across different releases is a source of backwards compatibility issues.
+// The calling protocol may change and cause issues if we don't carefully maintain the compatibility. Major version
+// changing the module path also causes problems for gob encoding as it effectively changes the name of every type.
+// To avoid having to maintain backwards compatibility between the different Gitaly binaries, we want to pin a given
+// gitaly binary to only ever call the auxiliary binaries of the same build. We achieve this by packing the auxiliary
+// binaries in the main gitaly binary and unpacking them on start to a temporary directory we can call them from. This
+// way updating the gitaly binaries on the disk is atomic and a running gitaly can't call auxiliary binaries from a
+// different version.
+func UnpackAuxiliaryBinaries(destinationDir string) error {
+	entries, err := packedBinariesFS.ReadDir(buildDir)
+	if err != nil {
+		return fmt.Errorf("list packed binaries: %w", err)
+	}
+
+	for _, entry := range entries {
+		if err := func() error {
+			packedPath := filepath.Join(buildDir, entry.Name())
+			packedFile, err := packedBinariesFS.Open(packedPath)
+			if err != nil {
+				return fmt.Errorf("open packed binary %q: %w", packedPath, err)
+			}
+			defer packedFile.Close()
+
+			unpackedPath := filepath.Join(destinationDir, entry.Name())
+			unpackedFile, err := os.OpenFile(unpackedPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o700)
+			if err != nil {
+				return err
+			}
+			defer unpackedFile.Close()
+
+			if _, err := io.Copy(unpackedFile, packedFile); err != nil {
+				return fmt.Errorf("unpack %q: %w", unpackedPath, err)
+			}
+
+			if err := unpackedFile.Close(); err != nil {
+				return fmt.Errorf("close %q: %w", unpackedPath, err)
+			}
+
+			return nil
+		}(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
diff --git a/packed_binaries_test.go b/packed_binaries_test.go
new file mode 100644
index 000000000..d904ab28b
--- /dev/null
+++ b/packed_binaries_test.go
@@ -0,0 +1,44 @@
+package gitaly
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnpackAuxiliaryBinaries_success(t *testing.T) {
+	destinationDir := t.TempDir()
+	require.NoError(t, UnpackAuxiliaryBinaries(destinationDir))
+
+	entries, err := os.ReadDir(destinationDir)
+	require.NoError(t, err)
+
+	require.Greater(t, len(entries), 1, "expected multiple packed binaries present")
+
+	for _, entry := range entries {
+		fileInfo, err := entry.Info()
+		require.NoError(t, err)
+		require.Equal(t, fileInfo.Mode(), os.FileMode(0o700), "expected the owner to have rwx permissions on the unpacked binary")
+
+		sourceBinary, err := os.ReadFile(filepath.Join(buildDir, fileInfo.Name()))
+		require.NoError(t, err)
+
+		unpackedBinary, err := os.ReadFile(filepath.Join(destinationDir, fileInfo.Name()))
+		require.NoError(t, err)
+
+		require.Equal(t, sourceBinary, unpackedBinary, "unpacked binary does not match the source binary")
+	}
+}
+
+func TestUnpackAuxiliaryBinaries_alreadyExists(t *testing.T) {
+	destinationDir := t.TempDir()
+
+	existingFile := filepath.Join(destinationDir, "gitaly-hooks")
+	require.NoError(t, os.WriteFile(existingFile, []byte("existing file"), os.ModePerm))
+
+	err := UnpackAuxiliaryBinaries(destinationDir)
+	require.EqualError(t, err, fmt.Sprintf(`open %s: file exists`, existingFile), "expected unpacking to fail if destination binary already existed")
+}