smarthttp: Advertise server-side backups as bundle-URI

Now all is set up to make it possible to advertise server-side bundles
to be used for bundle-URI. Inject all the git configuration into the
git-upload-pack(1) command to advertise these when possible.

Label: feature::addition

4 files changed, 297 insertions, 9 deletions

diff --git a/internal/bundleuri/git_config.go b/internal/bundleuri/git_config.go
index 2c6ab5c02..635031153 100644
--- a/internal/bundleuri/git_config.go
+++ b/internal/bundleuri/git_config.go
@@ -2,6 +2,7 @@ package bundleuri
 
 import (
 	"context"
+	"time"
 
 	"gitlab.com/gitlab-org/gitaly/v16/internal/backup"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/featureflag"
@@ -25,3 +26,56 @@ func InfoRefsGitConfig(ctx context.Context) []git.ConfigPair {
 	}
 }
 
+// UploadPackGitConfig return a slice of git.ConfigPairs you can inject into the
+// call to git-upload-pack(1) to advertise the available bundle to the client
+// who clones/fetches from the repository.
+// In case no backups could be found or something else goes wrong, an empty
+// slice is returned without error.
+func UploadPackGitConfig(
+	ctx context.Context,
+	backupLocator backup.Locator,
+	backupSink backup.Sink,
+	repo storage.Repository,
+) []git.ConfigPair {
+	if backupLocator == nil || backupSink == nil || featureflag.BundleURI.IsDisabled(ctx) {
+		return []git.ConfigPair{}
+	}
+
+	theBackup, err := backupLocator.FindLatest(ctx, repo)
+	if err != nil {
+		return []git.ConfigPair{}
+	}
+
+	for _, step := range theBackup.Steps {
+		// Skip non-existing & incremental backups
+		if len(step.BundlePath) == 0 || len(step.PreviousRefPath) > 0 {
+			continue
+		}
+
+		uri, err := backupSink.SignedURL(ctx, step.BundlePath, 10*time.Minute)
+		if err != nil {
+			return []git.ConfigPair{}
+		}
+
+		return []git.ConfigPair{
+			{
+				Key:   "uploadpack.advertiseBundleURIs",
+				Value: "true",
+			},
+			{
+				Key:   "bundle.version",
+				Value: "1",
+			},
+			{
+				Key:   "bundle.mode",
+				Value: "any",
+			},
+			{
+				Key:   "bundle.some.uri",
+				Value: uri,
+			},
+		}
+	}
+
+	return []git.ConfigPair{}
+}
diff --git a/internal/bundleuri/git_config_test.go b/internal/bundleuri/git_config_test.go
new file mode 100644
index 000000000..c14c3625b
--- /dev/null
+++ b/internal/bundleuri/git_config_test.go
@@ -0,0 +1,203 @@
+package bundleuri
+
+import (
+	"context"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/backup"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/featureflag"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/git"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
+)
+
+func TestUploadPackGitConfig(t *testing.T) {
+	testhelper.NewFeatureSets(featureflag.BundleURI).
+		Run(t, testUploadPackGitConfig)
+}
+
+func testUploadPackGitConfig(t *testing.T, ctx context.Context) {
+	t.Parallel()
+
+	cfg := testcfg.Build(t)
+	repoProto, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
+		SkipCreationViaService: true,
+	})
+	repo := localrepo.NewTestRepo(t, cfg, repoProto)
+
+	for _, tc := range []struct {
+		desc           string
+		findLatestFunc func(context.Context, storage.Repository) (*backup.Backup, error)
+		signedURLFunc  func(context.Context, string, time.Duration) (string, error)
+		expectedConfig []git.ConfigPair
+	}{
+		{
+			desc: "no backup locator nor sink",
+		},
+		{
+			desc: "no backup locator",
+			signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+				return "", structerr.NewNotFound("not signed")
+			},
+		},
+		{
+			desc: "no backup sink",
+			findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+				return nil, structerr.NewNotFound("no backup found")
+			},
+		},
+		{
+			desc: "no backup found",
+			findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+				return nil, structerr.NewNotFound("no backup found")
+			},
+			signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+				return "", structerr.NewNotFound("not signed")
+			},
+		},
+		{
+			desc: "backup has no steps",
+			findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+				return &backup.Backup{}, nil
+			},
+			signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+				return "", structerr.NewNotFound("not signed")
+			},
+		},
+		{
+			desc: "backup step is incremental",
+			findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+				return &backup.Backup{
+					Steps: []backup.Step{{PreviousRefPath: "not-nil"}},
+				}, nil
+			},
+			signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+				return "", structerr.NewNotFound("not signed")
+			},
+		},
+		{
+			desc: "sign failed",
+			findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+				return &backup.Backup{
+					Steps: []backup.Step{{BundlePath: "not-nil"}},
+				}, nil
+			},
+			signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+				return "", structerr.NewNotFound("not signed")
+			},
+		},
+		{
+			desc: "success",
+			findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+				return &backup.Backup{
+					Steps: []backup.Step{{BundlePath: "not-nil"}},
+				}, nil
+			},
+			signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+				return "https://example.com/bundle.git?signed=ok", nil
+			},
+			expectedConfig: []git.ConfigPair{
+				{
+					Key:   "uploadpack.advertiseBundleURIs",
+					Value: "true",
+				},
+				{
+					Key:   "bundle.version",
+					Value: "1",
+				},
+				{
+					Key:   "bundle.mode",
+					Value: "any",
+				},
+				{
+					Key:   "bundle.some.uri",
+					Value: "https://example.com/bundle.git?signed=ok",
+				},
+			},
+		},
+	} {
+		tc := tc
+
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Parallel()
+
+			var locator backup.Locator
+			if tc.findLatestFunc != nil {
+				locator = dummyLocator{findLatestFunc: tc.findLatestFunc}
+			}
+
+			var sink backup.Sink
+			if tc.signedURLFunc != nil {
+				sink = dummySink{signedURLFunc: tc.signedURLFunc}
+			}
+
+			actual := UploadPackGitConfig(ctx, locator, sink, repo)
+
+			if featureflag.BundleURI.IsEnabled(ctx) && tc.expectedConfig != nil {
+				require.Equal(t, tc.expectedConfig, actual)
+			} else {
+				require.Empty(t, actual)
+			}
+		})
+	}
+}
+
+type dummyLocator struct {
+	findLatestFunc func(context.Context, storage.Repository) (*backup.Backup, error)
+}
+
+// BeginFull is not supported by this dummyLocator.
+func (l dummyLocator) BeginFull(ctx context.Context, repo storage.Repository, backupID string) *backup.Backup {
+	return nil
+}
+
+// BeginIncremental is not supported by this dummyLocator.
+func (l dummyLocator) BeginIncremental(ctx context.Context, repo storage.Repository, backupID string) (*backup.Backup, error) {
+	return nil, structerr.NewUnimplemented("BeginIncremental not implemented for dummyLocator")
+}
+
+// Commit is not supported by this dummyLocator.
+func (l dummyLocator) Commit(ctx context.Context, backup *backup.Backup) error {
+	return structerr.NewUnimplemented("Commit not implemented for dummyLocator")
+}
+
+// FindLatest calls the findLatestFunc of the dummyLocator.
+func (l dummyLocator) FindLatest(ctx context.Context, repo storage.Repository) (*backup.Backup, error) {
+	return l.findLatestFunc(ctx, repo)
+}
+
+// Find is not supported by this dummyLocator.
+func (l dummyLocator) Find(ctx context.Context, repo storage.Repository, backupID string) (*backup.Backup, error) {
+	return nil, structerr.NewUnimplemented("Find not implemented for dummyLocator")
+}
+
+type dummySink struct {
+	signedURLFunc func(context.Context, string, time.Duration) (string, error)
+}
+
+// Close the dummySink.
+func (s dummySink) Close() error {
+	return nil
+}
+
+// GetWriter is not supported by this dummySink.
+func (s dummySink) GetWriter(ctx context.Context, relativePath string) (io.WriteCloser, error) {
+	return nil, structerr.NewUnimplemented("GetWriter not implemented for dummySink")
+}
+
+// GetReader is not supported by this dummySink.
+func (s dummySink) GetReader(ctx context.Context, relativePath string) (io.ReadCloser, error) {
+	return nil, structerr.NewUnimplemented("GetReader not implemented for dummySink")
+}
+
+// SignedURL calls the signedURLFunc of the dummySink.
+func (s dummySink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) {
+	return s.signedURLFunc(ctx, relativePath, expiry)
+}
diff --git a/internal/gitaly/service/smarthttp/upload_pack.go b/internal/gitaly/service/smarthttp/upload_pack.go
index 7e571df0d..e16fe92d5 100644
--- a/internal/gitaly/service/smarthttp/upload_pack.go
+++ b/internal/gitaly/service/smarthttp/upload_pack.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 
+	"gitlab.com/gitlab-org/gitaly/v16/internal/bundleuri"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/command"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/git"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/git/stats"
@@ -105,6 +106,9 @@ func (s *server) runUploadPack(ctx context.Context, req *gitalypb.PostUploadPack
 		}
 	}()
 
+	gitConfig = append(gitConfig,
+		bundleuri.UploadPackGitConfig(ctx, s.backupLocator, s.backupSink, req.GetRepository())...)
+
 	commandOpts := []git.CmdOpt{
 		git.WithStdin(stdin),
 		git.WithSetupStdout(),
diff --git a/internal/gitaly/service/smarthttp/upload_pack_test.go b/internal/gitaly/service/smarthttp/upload_pack_test.go
index 509c38e6c..dabb6ae3b 100644
--- a/internal/gitaly/service/smarthttp/upload_pack_test.go
+++ b/internal/gitaly/service/smarthttp/upload_pack_test.go
@@ -57,7 +57,10 @@ func runTestWithAndWithoutConfigOptions(
 func TestServer_PostUploadWithChannel(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadWithChannel)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadWithChannel)
 }
 
 func testServerPostUploadWithChannel(t *testing.T, ctx context.Context) {
@@ -108,7 +111,10 @@ func testServerPostUpload(t *testing.T, ctx context.Context, makeRequest request
 func TestServer_PostUploadPackSidechannel_gitConfigOptions(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackSidechannelGitConfigOptions)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadPackSidechannelGitConfigOptions)
 }
 
 func testServerPostUploadPackSidechannelGitConfigOptions(t *testing.T, ctx context.Context) {
@@ -177,7 +183,10 @@ func testServerPostUploadPackGitConfigOptions(t *testing.T, ctx context.Context,
 func TestServer_PostUploadPackWithSidechannel_gitProtocol(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelGitProtocol)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadPackWithSidechannelGitProtocol)
 }
 
 func testServerPostUploadPackWithSidechannelGitProtocol(t *testing.T, ctx context.Context) {
@@ -220,7 +229,10 @@ func testServerPostUploadPackGitProtocol(t *testing.T, ctx context.Context, make
 func TestServer_PostUploadPackWithSidechannel_suppressDeepenExitError(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelSuppressDeepenExitError)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadPackWithSidechannelSuppressDeepenExitError)
 }
 
 func testServerPostUploadPackWithSidechannelSuppressDeepenExitError(t *testing.T, ctx context.Context) {
@@ -250,7 +262,10 @@ func testServerPostUploadPackSuppressDeepenExitError(t *testing.T, ctx context.C
 func TestServer_PostUploadPackWithSidechannel_usesPackObjectsHook(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelUsesPackObjectsHook)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadPackWithSidechannelUsesPackObjectsHook)
 }
 
 func testServerPostUploadPackWithSidechannelUsesPackObjectsHook(t *testing.T, ctx context.Context) {
@@ -300,7 +315,10 @@ func testServerPostUploadPackUsesPackObjectsHook(t *testing.T, ctx context.Conte
 func TestServer_PostUploadPack_validation(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackValidation)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadPackValidation)
 }
 
 func testServerPostUploadPackValidation(t *testing.T, ctx context.Context) {
@@ -346,7 +364,10 @@ func testServerPostUploadPackValidationRequest(t *testing.T, ctx context.Context
 func TestServer_PostUploadPackSidechannel_validation(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackSidechannelValidation)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadPackSidechannelValidation)
 }
 
 func testServerPostUploadPackSidechannelValidation(t *testing.T, ctx context.Context) {
@@ -426,7 +447,10 @@ func extractPackDataFromResponse(t *testing.T, buf *bytes.Buffer) ([]byte, int,
 func TestServer_PostUploadPackWithSidechannel_partialClone(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelPartialClone)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadPackWithSidechannelPartialClone)
 }
 
 func testServerPostUploadPackWithSidechannelPartialClone(t *testing.T, ctx context.Context) {
@@ -484,7 +508,10 @@ func testServerPostUploadPackPartialClone(t *testing.T, ctx context.Context, mak
 func TestServer_PostUploadPackWithSidechannel_allowAnySHA1InWant(t *testing.T) {
 	t.Parallel()
 
-	testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelAllowAnySHA1InWant)
+	testhelper.NewFeatureSets(
+		featureflag.UploadPackBoundaryBitmapTraversal,
+		featureflag.BundleURI,
+	).Run(t, testServerPostUploadPackWithSidechannelAllowAnySHA1InWant)
 }
 
 func testServerPostUploadPackWithSidechannelAllowAnySHA1InWant(t *testing.T, ctx context.Context) {