Pin auxiliary binaries the main Gitaly binary invokes

We have to maintain backwards compatibility between Gitaly and its
auxiliary binaries as they may originate from different builds during
an upgrade. This has caused incidents in the past and slows down
iteration velocity. To avoid the incidents and th burden of backwards
compatiblity, it would be great if we could guarantee Gitaly
only ever invokes the auxiliary binaries from the same build. We now
have the infrastructure in place to locate binaries on disk and to
pack the auxiliary binaries in the main Gitaly binary to ship them
as a single deployment. This commit hooks wires up the final changes to
ensure the correct binaries get invoked always.

The Gitaly binary contains the auxiliary binaries packed into itself. On
start up, Gitaly will unpack the binaries into the runtime directory.
BinaryPath helper used to locate binaries is updated to correctly locate
the unpacked binaries from the runtime directory. Since each Gitaly process
has its own runtime directory, the upgrades do not affect the binaries
unpacked there. Replacing the gitaly binary on the disk itself is atomic.
The combination ensures that the correct binaries are always invoked.

The testing infrastructure is updated to also locate the packed binaries
to the runtime directory. The auxiliary binaries are not always built prior
to tests but as a part of them. Most of the tests thus do not exercise
the packing or unpacking but only the fact that certain binaries are used
from the runtime directory.

The packed binaries are still installed so they'll be present during the
upgrade if the old Gitaly uses them. We can stop installing them in the
next release.

Changelog: fixed

5 files changed, 41 insertions, 10 deletions

diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go
index ad9d143ac..793bb015e 100644
--- a/cmd/gitaly/main.go
+++ b/cmd/gitaly/main.go
@@ -11,6 +11,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	log "github.com/sirupsen/logrus"
+	"gitlab.com/gitlab-org/gitaly/v15"
 	"gitlab.com/gitlab-org/gitaly/v15/client"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/backchannel"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/bootstrap"
@@ -154,6 +155,10 @@ func run(cfg config.Cfg) error {
 		}
 	}()
 
+	if err := gitaly.UnpackAuxiliaryBinaries(cfg.RuntimeDir); err != nil {
+		return fmt.Errorf("unpack auxiliary binaries: %w", err)
+	}
+
 	b, err := bootstrap.New(promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gitaly_connections_total",
diff --git a/internal/gitaly/config/config.go b/internal/gitaly/config/config.go
index 38f2cfd04..b467dd9fb 100644
--- a/internal/gitaly/config/config.go
+++ b/internal/gitaly/config/config.go
@@ -281,10 +281,28 @@ func validateIsDirectory(path, name string) error {
 	return nil
 }
 
+// packedBinaries are the binaries that are packed in the main Gitaly binary. This should always match
+// the actual list in <root>/packed_binaries.go so the binaries are correctly located.
+//
+// Resolving the names automatically from the packed binaries is not possible at the moment due to how
+// the packed binaries themselves depend on this config package. If this config package inspected the
+// packed binaries, there would be a cyclic dependency. Anything that the packed binaries import must
+// not depend on <root>/packed_binaries.go.
+var packedBinaries = map[string]struct{}{
+	"gitaly-hooks":      {},
+	"gitaly-ssh":        {},
+	"gitaly-git2go-v15": {},
+	"gitaly-lfs-smudge": {},
+}
+
 // BinaryPath returns the path to a given binary. BinaryPath does not do any validation, it simply joins the binaryName
-// with the correct base directory.
+// with the correct base directory depending on whether the binary is a packed binary or not.
 func (cfg *Cfg) BinaryPath(binaryName string) string {
 	baseDirectory := cfg.BinDir
+	if _, ok := packedBinaries[binaryName]; ok {
+		baseDirectory = cfg.RuntimeDir
+	}
+
 	return filepath.Join(baseDirectory, binaryName)
 }
 
diff --git a/internal/gitaly/config/config_test.go b/internal/gitaly/config/config_test.go
index cfd28b8f2..b5690bf98 100644
--- a/internal/gitaly/config/config_test.go
+++ b/internal/gitaly/config/config_test.go
@@ -22,10 +22,12 @@ import (
 
 func TestBinaryPath(t *testing.T) {
 	cfg := Cfg{
-		BinDir: "bindir",
+		BinDir:     "bindir",
+		RuntimeDir: "runtime",
 	}
 
-	require.Equal(t, "bindir/gitaly-hooks", cfg.BinaryPath("gitaly-hooks"))
+	require.Equal(t, "runtime/gitaly-hooks", cfg.BinaryPath("gitaly-hooks"))
+	require.Equal(t, "bindir/gitaly-debug", cfg.BinaryPath("gitaly-debug"))
 	require.Equal(t, "bindir", cfg.BinaryPath(""))
 }
 
diff --git a/internal/testhelper/testcfg/build.go b/internal/testhelper/testcfg/build.go
index b49b8f47b..d8a88f0df 100644
--- a/internal/testhelper/testcfg/build.go
+++ b/internal/testhelper/testcfg/build.go
@@ -21,33 +21,39 @@ var buildOnceByName sync.Map
 
 // BuildGitalyGit2Go builds the gitaly-git2go command and installs it into the binary directory.
 func BuildGitalyGit2Go(t testing.TB, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-git2go-v15"))
+	return buildGitalyCommand(t, cfg, "gitaly-git2go-v15")
 }
 
 // BuildGitalyWrapper builds the gitaly-wrapper command and installs it into the binary directory.
 func BuildGitalyWrapper(t *testing.T, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-wrapper"))
+	return buildGitalyCommand(t, cfg, "gitaly-wrapper")
 }
 
 // BuildGitalyLFSSmudge builds the gitaly-lfs-smudge command and installs it into the binary
 // directory.
 func BuildGitalyLFSSmudge(t *testing.T, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-lfs-smudge"))
+	return buildGitalyCommand(t, cfg, "gitaly-lfs-smudge")
 }
 
 // BuildGitalyHooks builds the gitaly-hooks command and installs it into the binary directory.
 func BuildGitalyHooks(t testing.TB, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-hooks"))
+	return buildGitalyCommand(t, cfg, "gitaly-hooks")
 }
 
 // BuildGitalySSH builds the gitaly-ssh command and installs it into the binary directory.
 func BuildGitalySSH(t testing.TB, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("gitaly-ssh"))
+	return buildGitalyCommand(t, cfg, "gitaly-ssh")
 }
 
 // BuildPraefect builds the praefect command and installs it into the binary directory.
 func BuildPraefect(t testing.TB, cfg config.Cfg) string {
-	return BuildBinary(t, cfg.BinDir, gitalyCommandPath("praefect"))
+	return buildGitalyCommand(t, cfg, "praefect")
+}
+
+// buildGitalyCommand builds an executable and places it in the correct directory depending
+// whether it is packed in the production build or not.
+func buildGitalyCommand(t testing.TB, cfg config.Cfg, executableName string) string {
+	return BuildBinary(t, filepath.Dir(cfg.BinaryPath(executableName)), gitalyCommandPath(executableName))
 }
 
 var (
diff --git a/packed_binaries.go b/packed_binaries.go
index 1eafb3639..16c02f2e4 100644
--- a/packed_binaries.go
+++ b/packed_binaries.go
@@ -14,7 +14,7 @@ const buildDir = "_build/bin"
 //go:embed _build/bin/gitaly-hooks _build/bin/gitaly-ssh _build/bin/gitaly-git2go-v15 _build/bin/gitaly-lfs-smudge
 //
 // packedBinariesFS contains embedded binaries. If you modify the above embeddings, you must also update
-// GITALY_PACKED_EXECUTABLES in Makefile.
+// GITALY_PACKED_EXECUTABLES in Makefile and packedBinaries in internal/gitaly/config/config.go.
 var packedBinariesFS embed.FS
 
 // UnpackAuxiliaryBinaries unpacks the packed auxiliary binaries of Gitaly into destination directory.