Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitaly.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJames Fargher <jfargher@gitlab.com>2023-11-21 04:28:20 +0300
committerJames Fargher <jfargher@gitlab.com>2023-11-21 04:28:20 +0300
commit5f57b0c636f00cdff8049375bdca64ebd7b2710f (patch)
tree0538905caea5ceffc2dff993d7b6d4ac7eccbb44
parent38c9c539bdd89301b900e6d25cf18c7f45d60fdd (diff)
parenta987721bb40590b47ad122c30a2aea43a8f20100 (diff)
Merge branch 'toon-poc-bundle-uri' into 'master'
ssh & smarthttp: Advertise server-side backups as bundle-URI See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/6472 Merged-by: James Fargher <jfargher@gitlab.com> Approved-by: James Fargher <jfargher@gitlab.com> Approved-by: karthik nayak <knayak@gitlab.com> Reviewed-by: James Fargher <jfargher@gitlab.com> Reviewed-by: karthik nayak <knayak@gitlab.com> Co-authored-by: Toon Claes <toon@gitlab.com>
-rw-r--r--internal/backup/backup.go4
-rw-r--r--internal/backup/filesystem_sink.go7
-rw-r--r--internal/backup/sink.go43
-rw-r--r--internal/backup/sink_test.go50
-rw-r--r--internal/bundleuri/doc.go16
-rw-r--r--internal/bundleuri/git_config.go81
-rw-r--r--internal/bundleuri/git_config_test.go203
-rw-r--r--internal/bundleuri/testhelper_test.go11
-rw-r--r--internal/featureflag/ff_bundle_uri.go9
-rw-r--r--internal/gitaly/service/smarthttp/inforefs.go7
-rw-r--r--internal/gitaly/service/smarthttp/inforefs_test.go100
-rw-r--r--internal/gitaly/service/smarthttp/server.go7
-rw-r--r--internal/gitaly/service/smarthttp/upload_pack.go4
-rw-r--r--internal/gitaly/service/smarthttp/upload_pack_test.go45
-rw-r--r--internal/gitaly/service/ssh/server.go5
-rw-r--r--internal/gitaly/service/ssh/upload_pack.go4
-rw-r--r--internal/gitaly/service/ssh/upload_pack_test.go25
17 files changed, 581 insertions, 40 deletions
diff --git a/internal/backup/backup.go b/internal/backup/backup.go
index 9f32ee22e..e58fc8718 100644
--- a/internal/backup/backup.go
+++ b/internal/backup/backup.go
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"strings"
+ "time"
"gitlab.com/gitlab-org/gitaly/v16/internal/git"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
@@ -38,6 +39,9 @@ type Sink interface {
// GetReader returns a reader that servers the data stored by relativePath.
// If relativePath doesn't exists the ErrDoesntExist will be returned.
GetReader(ctx context.Context, relativePath string) (io.ReadCloser, error)
+ // SignedURL returns a URL that can be used to GET the blob for the duration
+ // specified in expiry.
+ SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error)
}
// Backup represents all the information needed to restore a backup for a repository
diff --git a/internal/backup/filesystem_sink.go b/internal/backup/filesystem_sink.go
index 6d68502db..2b4c04944 100644
--- a/internal/backup/filesystem_sink.go
+++ b/internal/backup/filesystem_sink.go
@@ -7,8 +7,10 @@ import (
"io"
"os"
"path/filepath"
+ "time"
"gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
)
// FilesystemSink is a sink for creating and restoring backups from the local filesystem.
@@ -59,3 +61,8 @@ func (fs *FilesystemSink) GetReader(ctx context.Context, relativePath string) (i
func (fs *FilesystemSink) Close() error {
return nil
}
+
+// SignedURL is not supported by FilesystemSink.
+func (fs *FilesystemSink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) {
+ return "", structerr.NewUnimplemented("SignedURL not implemented for FilesystemSink")
+}
diff --git a/internal/backup/sink.go b/internal/backup/sink.go
index 330b1c5b4..9de909024 100644
--- a/internal/backup/sink.go
+++ b/internal/backup/sink.go
@@ -6,17 +6,22 @@ import (
"io"
"net/url"
"strings"
+ "time"
"gocloud.dev/blob"
"gocloud.dev/blob/azureblob"
"gocloud.dev/blob/gcsblob"
+ "gocloud.dev/blob/memblob"
"gocloud.dev/blob/s3blob"
"gocloud.dev/gcerrors"
)
-// ResolveSink returns a sink implementation based on the provided path.
-func ResolveSink(ctx context.Context, path string) (Sink, error) {
- parsed, err := url.Parse(path)
+// ResolveSink returns a sink implementation based on the provided uri.
+// The storage engine is chosen based on the provided uri.
+// It is the caller's responsibility to provide all required environment
+// variables in order to get properly initialized storage engine driver.
+func ResolveSink(ctx context.Context, uri string) (Sink, error) {
+ parsed, err := url.Parse(uri)
if err != nil {
return nil, err
}
@@ -29,11 +34,10 @@ func ResolveSink(ctx context.Context, path string) (Sink, error) {
}
switch scheme {
- case s3blob.Scheme, azureblob.Scheme, gcsblob.Scheme:
- sink, err := NewStorageServiceSink(ctx, path)
- return sink, err
+ case s3blob.Scheme, azureblob.Scheme, gcsblob.Scheme, memblob.Scheme:
+ return newStorageServiceSink(ctx, uri)
default:
- return NewFilesystemSink(path), nil
+ return NewFilesystemSink(uri), nil
}
}
@@ -42,11 +46,8 @@ type StorageServiceSink struct {
bucket *blob.Bucket
}
-// NewStorageServiceSink returns initialized instance of StorageServiceSink instance.
-// The storage engine is chosen based on the provided url value and a set of pre-registered
-// blank imports in that file. It is the caller's responsibility to provide all required environment
-// variables in order to get properly initialized storage engine driver.
-func NewStorageServiceSink(ctx context.Context, url string) (*StorageServiceSink, error) {
+// newStorageServiceSink returns initialized instance of StorageServiceSink instance.
+func newStorageServiceSink(ctx context.Context, url string) (*StorageServiceSink, error) {
bucket, err := blob.OpenBucket(ctx, url)
if err != nil {
return nil, fmt.Errorf("storage service sink: open bucket: %w", err)
@@ -98,3 +99,21 @@ func (s *StorageServiceSink) GetReader(ctx context.Context, relativePath string)
}
return reader, nil
}
+
+// SignedURL returns a URL that can be used to GET the blob for the duration
+// specified in expiry.
+func (s *StorageServiceSink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) {
+ opt := &blob.SignedURLOptions{
+ Expiry: expiry,
+ }
+
+ signed, err := s.bucket.SignedURL(ctx, relativePath, opt)
+ if err != nil {
+ if gcerrors.Code(err) == gcerrors.NotFound {
+ err = ErrDoesntExist
+ }
+ return "", fmt.Errorf("storage service sink: signed URL for %q: %w", relativePath, err)
+ }
+
+ return signed, err
+}
diff --git a/internal/backup/sink_test.go b/internal/backup/sink_test.go
index 0c40ee6ff..9ff6f3f8a 100644
--- a/internal/backup/sink_test.go
+++ b/internal/backup/sink_test.go
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
@@ -115,7 +116,7 @@ func TestStorageServiceSink(t *testing.T) {
t.Parallel()
ctx := testhelper.Context(t)
- sss, err := NewStorageServiceSink(ctx, "mem://test_bucket")
+ sss, err := ResolveSink(ctx, "mem://test_bucket")
require.NoError(t, err)
defer func() { require.NoError(t, sss.Close()) }()
@@ -147,3 +148,50 @@ func TestStorageServiceSink(t *testing.T) {
require.Nil(t, reader)
})
}
+
+func TestStorageServiceSink_SignedURL_notImplemented(t *testing.T) {
+ t.Parallel()
+
+ ctx := testhelper.Context(t)
+ tmpDir := testhelper.TempDir(t)
+
+ for _, tc := range []struct {
+ desc string
+ bucketURL string
+ }{
+ {
+ desc: "memory bucket",
+ bucketURL: "mem://test_bucket",
+ },
+ {
+ desc: "fs bucket",
+ bucketURL: tmpDir,
+ },
+ } {
+ tc := tc
+
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ sss, err := ResolveSink(ctx, tc.bucketURL)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, sss.Close()) })
+
+ const relativePath = "path/to/data"
+
+ data := []byte("test")
+
+ w, err := sss.GetWriter(ctx, relativePath)
+ require.NoError(t, err)
+
+ _, err = io.Copy(w, bytes.NewReader(data))
+ require.NoError(t, err)
+
+ require.NoError(t, w.Close())
+
+ _, err = sss.SignedURL(ctx, relativePath, 10*time.Minute)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "not implemented")
+ })
+ }
+}
diff --git a/internal/bundleuri/doc.go b/internal/bundleuri/doc.go
new file mode 100644
index 000000000..22a0b4957
--- /dev/null
+++ b/internal/bundleuri/doc.go
@@ -0,0 +1,16 @@
+// Package bundleuri is used to enable the use [Bundle-URI] when the client
+// clones/fetches from the repository.
+//
+// Bundle-URI is a concept in Git that allows the server to send one or more
+// URIs where [git bundles] are available. The client can download such bundles
+// to prepopulate the repository before it starts the object negotiation with
+// the server. This reduces the CPU load on the server, and the amount of
+// traffic that has to travel directly from server to client.
+//
+// This feature piggy-backs onto server-side backups. Refer to the
+// [backup documentation] how to create and store bundles on a cloud provider.
+//
+// [Bundle-URI]: https://git-scm.com/docs/bundle-uri
+// [git bundles]: https://git-scm.com/docs/git-bundle
+// [backup documentation]: https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/gitaly-backup.md
+package bundleuri
diff --git a/internal/bundleuri/git_config.go b/internal/bundleuri/git_config.go
new file mode 100644
index 000000000..635031153
--- /dev/null
+++ b/internal/bundleuri/git_config.go
@@ -0,0 +1,81 @@
+package bundleuri
+
+import (
+ "context"
+ "time"
+
+ "gitlab.com/gitlab-org/gitaly/v16/internal/backup"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/git"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+)
+
+// InfoRefsGitConfig return a slice of git.ConfigPairs you can inject into the
+// call to git-upload-pack(1) --advertise-refs, to advertise the use of
+// bundle-URI to the client who clones/fetches from the repository.
+func InfoRefsGitConfig(ctx context.Context) []git.ConfigPair {
+ if featureflag.BundleURI.IsDisabled(ctx) {
+ return []git.ConfigPair{}
+ }
+
+ return []git.ConfigPair{
+ {
+ Key: "uploadpack.advertiseBundleURIs",
+ Value: "true",
+ },
+ }
+}
+
+// UploadPackGitConfig return a slice of git.ConfigPairs you can inject into the
+// call to git-upload-pack(1) to advertise the available bundle to the client
+// who clones/fetches from the repository.
+// In case no backups could be found or something else goes wrong, an empty
+// slice is returned without error.
+func UploadPackGitConfig(
+ ctx context.Context,
+ backupLocator backup.Locator,
+ backupSink backup.Sink,
+ repo storage.Repository,
+) []git.ConfigPair {
+ if backupLocator == nil || backupSink == nil || featureflag.BundleURI.IsDisabled(ctx) {
+ return []git.ConfigPair{}
+ }
+
+ theBackup, err := backupLocator.FindLatest(ctx, repo)
+ if err != nil {
+ return []git.ConfigPair{}
+ }
+
+ for _, step := range theBackup.Steps {
+ // Skip non-existing & incremental backups
+ if len(step.BundlePath) == 0 || len(step.PreviousRefPath) > 0 {
+ continue
+ }
+
+ uri, err := backupSink.SignedURL(ctx, step.BundlePath, 10*time.Minute)
+ if err != nil {
+ return []git.ConfigPair{}
+ }
+
+ return []git.ConfigPair{
+ {
+ Key: "uploadpack.advertiseBundleURIs",
+ Value: "true",
+ },
+ {
+ Key: "bundle.version",
+ Value: "1",
+ },
+ {
+ Key: "bundle.mode",
+ Value: "any",
+ },
+ {
+ Key: "bundle.some.uri",
+ Value: uri,
+ },
+ }
+ }
+
+ return []git.ConfigPair{}
+}
diff --git a/internal/bundleuri/git_config_test.go b/internal/bundleuri/git_config_test.go
new file mode 100644
index 000000000..c14c3625b
--- /dev/null
+++ b/internal/bundleuri/git_config_test.go
@@ -0,0 +1,203 @@
+package bundleuri
+
+import (
+ "context"
+ "io"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/backup"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/git"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
+)
+
+func TestUploadPackGitConfig(t *testing.T) {
+ testhelper.NewFeatureSets(featureflag.BundleURI).
+ Run(t, testUploadPackGitConfig)
+}
+
+func testUploadPackGitConfig(t *testing.T, ctx context.Context) {
+ t.Parallel()
+
+ cfg := testcfg.Build(t)
+ repoProto, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
+ SkipCreationViaService: true,
+ })
+ repo := localrepo.NewTestRepo(t, cfg, repoProto)
+
+ for _, tc := range []struct {
+ desc string
+ findLatestFunc func(context.Context, storage.Repository) (*backup.Backup, error)
+ signedURLFunc func(context.Context, string, time.Duration) (string, error)
+ expectedConfig []git.ConfigPair
+ }{
+ {
+ desc: "no backup locator nor sink",
+ },
+ {
+ desc: "no backup locator",
+ signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+ return "", structerr.NewNotFound("not signed")
+ },
+ },
+ {
+ desc: "no backup sink",
+ findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+ return nil, structerr.NewNotFound("no backup found")
+ },
+ },
+ {
+ desc: "no backup found",
+ findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+ return nil, structerr.NewNotFound("no backup found")
+ },
+ signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+ return "", structerr.NewNotFound("not signed")
+ },
+ },
+ {
+ desc: "backup has no steps",
+ findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+ return &backup.Backup{}, nil
+ },
+ signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+ return "", structerr.NewNotFound("not signed")
+ },
+ },
+ {
+ desc: "backup step is incremental",
+ findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+ return &backup.Backup{
+ Steps: []backup.Step{{PreviousRefPath: "not-nil"}},
+ }, nil
+ },
+ signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+ return "", structerr.NewNotFound("not signed")
+ },
+ },
+ {
+ desc: "sign failed",
+ findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+ return &backup.Backup{
+ Steps: []backup.Step{{BundlePath: "not-nil"}},
+ }, nil
+ },
+ signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+ return "", structerr.NewNotFound("not signed")
+ },
+ },
+ {
+ desc: "success",
+ findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) {
+ return &backup.Backup{
+ Steps: []backup.Step{{BundlePath: "not-nil"}},
+ }, nil
+ },
+ signedURLFunc: func(context.Context, string, time.Duration) (string, error) {
+ return "https://example.com/bundle.git?signed=ok", nil
+ },
+ expectedConfig: []git.ConfigPair{
+ {
+ Key: "uploadpack.advertiseBundleURIs",
+ Value: "true",
+ },
+ {
+ Key: "bundle.version",
+ Value: "1",
+ },
+ {
+ Key: "bundle.mode",
+ Value: "any",
+ },
+ {
+ Key: "bundle.some.uri",
+ Value: "https://example.com/bundle.git?signed=ok",
+ },
+ },
+ },
+ } {
+ tc := tc
+
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+
+ var locator backup.Locator
+ if tc.findLatestFunc != nil {
+ locator = dummyLocator{findLatestFunc: tc.findLatestFunc}
+ }
+
+ var sink backup.Sink
+ if tc.signedURLFunc != nil {
+ sink = dummySink{signedURLFunc: tc.signedURLFunc}
+ }
+
+ actual := UploadPackGitConfig(ctx, locator, sink, repo)
+
+ if featureflag.BundleURI.IsEnabled(ctx) && tc.expectedConfig != nil {
+ require.Equal(t, tc.expectedConfig, actual)
+ } else {
+ require.Empty(t, actual)
+ }
+ })
+ }
+}
+
+type dummyLocator struct {
+ findLatestFunc func(context.Context, storage.Repository) (*backup.Backup, error)
+}
+
+// BeginFull is not supported by this dummyLocator.
+func (l dummyLocator) BeginFull(ctx context.Context, repo storage.Repository, backupID string) *backup.Backup {
+ return nil
+}
+
+// BeginIncremental is not supported by this dummyLocator.
+func (l dummyLocator) BeginIncremental(ctx context.Context, repo storage.Repository, backupID string) (*backup.Backup, error) {
+ return nil, structerr.NewUnimplemented("BeginIncremental not implemented for dummyLocator")
+}
+
+// Commit is not supported by this dummyLocator.
+func (l dummyLocator) Commit(ctx context.Context, backup *backup.Backup) error {
+ return structerr.NewUnimplemented("Commit not implemented for dummyLocator")
+}
+
+// FindLatest calls the findLatestFunc of the dummyLocator.
+func (l dummyLocator) FindLatest(ctx context.Context, repo storage.Repository) (*backup.Backup, error) {
+ return l.findLatestFunc(ctx, repo)
+}
+
+// Find is not supported by this dummyLocator.
+func (l dummyLocator) Find(ctx context.Context, repo storage.Repository, backupID string) (*backup.Backup, error) {
+ return nil, structerr.NewUnimplemented("Find not implemented for dummyLocator")
+}
+
+type dummySink struct {
+ signedURLFunc func(context.Context, string, time.Duration) (string, error)
+}
+
+// Close the dummySink.
+func (s dummySink) Close() error {
+ return nil
+}
+
+// GetWriter is not supported by this dummySink.
+func (s dummySink) GetWriter(ctx context.Context, relativePath string) (io.WriteCloser, error) {
+ return nil, structerr.NewUnimplemented("GetWriter not implemented for dummySink")
+}
+
+// GetReader is not supported by this dummySink.
+func (s dummySink) GetReader(ctx context.Context, relativePath string) (io.ReadCloser, error) {
+ return nil, structerr.NewUnimplemented("GetReader not implemented for dummySink")
+}
+
+// SignedURL calls the signedURLFunc of the dummySink.
+func (s dummySink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) {
+ return s.signedURLFunc(ctx, relativePath, expiry)
+}
diff --git a/internal/bundleuri/testhelper_test.go b/internal/bundleuri/testhelper_test.go
new file mode 100644
index 000000000..b2dd3cfea
--- /dev/null
+++ b/internal/bundleuri/testhelper_test.go
@@ -0,0 +1,11 @@
+package bundleuri
+
+import (
+ "testing"
+
+ "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
+)
+
+func TestMain(m *testing.M) {
+ testhelper.Run(m)
+}
diff --git a/internal/featureflag/ff_bundle_uri.go b/internal/featureflag/ff_bundle_uri.go
new file mode 100644
index 000000000..c912169dc
--- /dev/null
+++ b/internal/featureflag/ff_bundle_uri.go
@@ -0,0 +1,9 @@
+package featureflag
+
+// BundleURI enables the use of git's bundle URI feature
+var BundleURI = NewFeatureFlag(
+ "bundle_uri",
+ "v16.6.0",
+ "https://gitlab.com/gitlab-org/gitaly/-/issues/5656",
+ false,
+)
diff --git a/internal/gitaly/service/smarthttp/inforefs.go b/internal/gitaly/service/smarthttp/inforefs.go
index 2a01868d7..63be73f0f 100644
--- a/internal/gitaly/service/smarthttp/inforefs.go
+++ b/internal/gitaly/service/smarthttp/inforefs.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/bundleuri"
"gitlab.com/gitlab-org/gitaly/v16/internal/git"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/pktline"
"gitlab.com/gitlab-org/gitaly/v16/internal/log"
@@ -62,11 +63,13 @@ func (s *server) handleInfoRefs(ctx context.Context, service, repoPath string, r
cmdOpts = append(cmdOpts, git.WithDisabledHooks())
}
- config, err := git.ConvertConfigOptions(req.GitConfigOptions)
+ gitConfig, err := git.ConvertConfigOptions(req.GitConfigOptions)
if err != nil {
return err
}
- cmdOpts = append(cmdOpts, git.WithConfig(config...))
+ gitConfig = append(gitConfig, bundleuri.InfoRefsGitConfig(ctx)...)
+
+ cmdOpts = append(cmdOpts, git.WithConfig(gitConfig...))
if _, err := pktline.WriteString(w, fmt.Sprintf("# service=git-%s\n", service)); err != nil {
return structerr.NewInternal("pktLine: %w", err)
diff --git a/internal/gitaly/service/smarthttp/inforefs_test.go b/internal/gitaly/service/smarthttp/inforefs_test.go
index ba458bac9..836dfce4a 100644
--- a/internal/gitaly/service/smarthttp/inforefs_test.go
+++ b/internal/gitaly/service/smarthttp/inforefs_test.go
@@ -33,7 +33,10 @@ import (
func TestInfoRefsUploadPack_successful(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackSuccessful)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsUploadPackSuccessful)
}
func testInfoRefsUploadPackSuccessful(t *testing.T, ctx context.Context) {
@@ -63,7 +66,10 @@ func testInfoRefsUploadPackSuccessful(t *testing.T, ctx context.Context) {
func TestInfoRefsUploadPack_internalRefs(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackInternalRefs)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsUploadPackInternalRefs)
}
func testInfoRefsUploadPackInternalRefs(t *testing.T, ctx context.Context) {
@@ -138,7 +144,10 @@ func testInfoRefsUploadPackInternalRefs(t *testing.T, ctx context.Context) {
func TestInfoRefsUploadPack_validate(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackValidate)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsUploadPackValidate)
}
func testInfoRefsUploadPackValidate(t *testing.T, ctx context.Context) {
@@ -176,7 +185,10 @@ func testInfoRefsUploadPackValidate(t *testing.T, ctx context.Context) {
func TestInfoRefsUploadPack_partialClone(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackPartialClone)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsUploadPackPartialClone)
}
func testInfoRefsUploadPackPartialClone(t *testing.T, ctx context.Context) {
@@ -204,7 +216,10 @@ func testInfoRefsUploadPackPartialClone(t *testing.T, ctx context.Context) {
func TestInfoRefsUploadPack_gitConfigOptions(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackGitConfigOptions)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsUploadPackGitConfigOptions)
}
func testInfoRefsUploadPackGitConfigOptions(t *testing.T, ctx context.Context) {
@@ -230,10 +245,45 @@ func testInfoRefsUploadPackGitConfigOptions(t *testing.T, ctx context.Context) {
})
}
+func TestInfoRefsUploadPack_bundleURI(t *testing.T) {
+ t.Parallel()
+
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ ).Run(t, testInfoRefsUploadPackBundleURI)
+}
+
+func testInfoRefsUploadPackBundleURI(t *testing.T, ctx context.Context) {
+ t.Parallel()
+
+ ctx = featureflag.OutgoingCtxWithFeatureFlag(ctx, featureflag.BundleURI, true)
+
+ cfg := testcfg.Build(t)
+ cfg.SocketPath = runSmartHTTPServer(t, cfg)
+
+ repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+
+ gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"), gittest.WithParents())
+
+ rpcRequest := &gitalypb.InfoRefsRequest{
+ Repository: repo,
+ GitProtocol: git.ProtocolV2,
+ GitConfigOptions: []string{"transfer.bundleURI=true"},
+ }
+ response, err := makeInfoRefsUploadPackRequest(t, ctx, cfg.SocketPath, cfg.Auth.Token, rpcRequest)
+ require.NoError(t, err)
+ requireAdvertisedCapabilitiesV2(t, string(response), "git-upload-pack", []string{
+ "bundle-uri",
+ })
+}
+
func TestInfoRefsUploadPack_gitProtocol(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackGitProtocol)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsUploadPackGitProtocol)
}
func testInfoRefsUploadPackGitProtocol(t *testing.T, ctx context.Context) {
@@ -290,7 +340,10 @@ func makeInfoRefsUploadPackRequest(t *testing.T, ctx context.Context, serverSock
func TestInfoRefsReceivePack_successful(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsReceivePackSuccessful)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsReceivePackSuccessful)
}
func testInfoRefsReceivePackSuccessful(t *testing.T, ctx context.Context) {
@@ -322,7 +375,10 @@ func TestInfoRefsReceivePack_hiddenRefs(t *testing.T) {
Object pools are not yet support with WAL. This test is testing with a pooled repository.`)
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsReceivePackHiddenRefs)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsReceivePackHiddenRefs)
}
func testInfoRefsReceivePackHiddenRefs(t *testing.T, ctx context.Context) {
@@ -352,7 +408,10 @@ func testInfoRefsReceivePackHiddenRefs(t *testing.T, ctx context.Context) {
func TestInfoRefsReceivePack_validate(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsReceivePackValidate)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsReceivePackValidate)
}
func testInfoRefsReceivePackValidate(t *testing.T, ctx context.Context) {
@@ -405,6 +464,24 @@ func makeInfoRefsReceivePackRequest(t *testing.T, ctx context.Context, serverSoc
return response, err
}
+func requireAdvertisedCapabilitiesV2(t *testing.T, responseBody, expectedService string, expectedCapabilities []string) {
+ t.Helper()
+
+ responseLines := strings.SplitAfter(responseBody, "\n")
+ require.Greater(t, len(responseLines), 2)
+
+ // The first line contains the service announcement
+ require.Equal(t, gittest.Pktlinef(t, "# service=%s\n", expectedService), responseLines[0])
+
+ // The second line contains the protocol version
+ require.Equal(t, "0000"+gittest.Pktlinef(t, "version %d\n", 2), responseLines[1])
+
+ // The third line and following lines contain capabilities
+ for _, expectedCap := range expectedCapabilities {
+ require.Contains(t, responseLines[2:], gittest.Pktlinef(t, "%s\n", expectedCap))
+ }
+}
+
func requireAdvertisedRefs(t *testing.T, responseBody, expectedService string, expectedRefs []string) {
t.Helper()
@@ -444,7 +521,10 @@ func (ms *mockStreamer) PutStream(ctx context.Context, repo *gitalypb.Repository
func TestInfoRefsUploadPack_cache(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackCache)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testInfoRefsUploadPackCache)
}
func testInfoRefsUploadPackCache(t *testing.T, ctx context.Context) {
diff --git a/internal/gitaly/service/smarthttp/server.go b/internal/gitaly/service/smarthttp/server.go
index e96930535..33f09309f 100644
--- a/internal/gitaly/service/smarthttp/server.go
+++ b/internal/gitaly/service/smarthttp/server.go
@@ -2,6 +2,7 @@ package smarthttp
import (
"github.com/prometheus/client_golang/prometheus"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/backup"
"gitlab.com/gitlab-org/gitaly/v16/internal/git"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
@@ -18,6 +19,8 @@ type server struct {
packfileNegotiationMetrics *prometheus.CounterVec
infoRefCache infoRefCache
txManager transaction.Manager
+ backupLocator backup.Locator
+ backupSink backup.Sink
}
// NewServer creates a new instance of a grpc SmartHTTPServer
@@ -31,7 +34,9 @@ func NewServer(deps *service.Dependencies, serverOpts ...ServerOpt) gitalypb.Sma
prometheus.CounterOpts{},
[]string{"git_negotiation_feature"},
),
- infoRefCache: newInfoRefCache(deps.GetLogger(), deps.GetDiskCache()),
+ infoRefCache: newInfoRefCache(deps.GetLogger(), deps.GetDiskCache()),
+ backupLocator: deps.GetBackupLocator(),
+ backupSink: deps.GetBackupSink(),
}
for _, serverOpt := range serverOpts {
diff --git a/internal/gitaly/service/smarthttp/upload_pack.go b/internal/gitaly/service/smarthttp/upload_pack.go
index 7e571df0d..e16fe92d5 100644
--- a/internal/gitaly/service/smarthttp/upload_pack.go
+++ b/internal/gitaly/service/smarthttp/upload_pack.go
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/bundleuri"
"gitlab.com/gitlab-org/gitaly/v16/internal/command"
"gitlab.com/gitlab-org/gitaly/v16/internal/git"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/stats"
@@ -105,6 +106,9 @@ func (s *server) runUploadPack(ctx context.Context, req *gitalypb.PostUploadPack
}
}()
+ gitConfig = append(gitConfig,
+ bundleuri.UploadPackGitConfig(ctx, s.backupLocator, s.backupSink, req.GetRepository())...)
+
commandOpts := []git.CmdOpt{
git.WithStdin(stdin),
git.WithSetupStdout(),
diff --git a/internal/gitaly/service/smarthttp/upload_pack_test.go b/internal/gitaly/service/smarthttp/upload_pack_test.go
index 509c38e6c..dabb6ae3b 100644
--- a/internal/gitaly/service/smarthttp/upload_pack_test.go
+++ b/internal/gitaly/service/smarthttp/upload_pack_test.go
@@ -57,7 +57,10 @@ func runTestWithAndWithoutConfigOptions(
func TestServer_PostUploadWithChannel(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadWithChannel)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadWithChannel)
}
func testServerPostUploadWithChannel(t *testing.T, ctx context.Context) {
@@ -108,7 +111,10 @@ func testServerPostUpload(t *testing.T, ctx context.Context, makeRequest request
func TestServer_PostUploadPackSidechannel_gitConfigOptions(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackSidechannelGitConfigOptions)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadPackSidechannelGitConfigOptions)
}
func testServerPostUploadPackSidechannelGitConfigOptions(t *testing.T, ctx context.Context) {
@@ -177,7 +183,10 @@ func testServerPostUploadPackGitConfigOptions(t *testing.T, ctx context.Context,
func TestServer_PostUploadPackWithSidechannel_gitProtocol(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelGitProtocol)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadPackWithSidechannelGitProtocol)
}
func testServerPostUploadPackWithSidechannelGitProtocol(t *testing.T, ctx context.Context) {
@@ -220,7 +229,10 @@ func testServerPostUploadPackGitProtocol(t *testing.T, ctx context.Context, make
func TestServer_PostUploadPackWithSidechannel_suppressDeepenExitError(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelSuppressDeepenExitError)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadPackWithSidechannelSuppressDeepenExitError)
}
func testServerPostUploadPackWithSidechannelSuppressDeepenExitError(t *testing.T, ctx context.Context) {
@@ -250,7 +262,10 @@ func testServerPostUploadPackSuppressDeepenExitError(t *testing.T, ctx context.C
func TestServer_PostUploadPackWithSidechannel_usesPackObjectsHook(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelUsesPackObjectsHook)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadPackWithSidechannelUsesPackObjectsHook)
}
func testServerPostUploadPackWithSidechannelUsesPackObjectsHook(t *testing.T, ctx context.Context) {
@@ -300,7 +315,10 @@ func testServerPostUploadPackUsesPackObjectsHook(t *testing.T, ctx context.Conte
func TestServer_PostUploadPack_validation(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackValidation)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadPackValidation)
}
func testServerPostUploadPackValidation(t *testing.T, ctx context.Context) {
@@ -346,7 +364,10 @@ func testServerPostUploadPackValidationRequest(t *testing.T, ctx context.Context
func TestServer_PostUploadPackSidechannel_validation(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackSidechannelValidation)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadPackSidechannelValidation)
}
func testServerPostUploadPackSidechannelValidation(t *testing.T, ctx context.Context) {
@@ -426,7 +447,10 @@ func extractPackDataFromResponse(t *testing.T, buf *bytes.Buffer) ([]byte, int,
func TestServer_PostUploadPackWithSidechannel_partialClone(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelPartialClone)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadPackWithSidechannelPartialClone)
}
func testServerPostUploadPackWithSidechannelPartialClone(t *testing.T, ctx context.Context) {
@@ -484,7 +508,10 @@ func testServerPostUploadPackPartialClone(t *testing.T, ctx context.Context, mak
func TestServer_PostUploadPackWithSidechannel_allowAnySHA1InWant(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelAllowAnySHA1InWant)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testServerPostUploadPackWithSidechannelAllowAnySHA1InWant)
}
func testServerPostUploadPackWithSidechannelAllowAnySHA1InWant(t *testing.T, ctx context.Context) {
diff --git a/internal/gitaly/service/ssh/server.go b/internal/gitaly/service/ssh/server.go
index 65ba2180c..8ad343834 100644
--- a/internal/gitaly/service/ssh/server.go
+++ b/internal/gitaly/service/ssh/server.go
@@ -2,6 +2,7 @@ package ssh
import (
"github.com/prometheus/client_golang/prometheus"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/backup"
"gitlab.com/gitlab-org/gitaly/v16/internal/git"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
@@ -20,6 +21,8 @@ type server struct {
uploadPackRequestTimeoutTickerFactory func() helper.Ticker
uploadArchiveRequestTimeoutTickerFactory func() helper.Ticker
packfileNegotiationMetrics *prometheus.CounterVec
+ backupLocator backup.Locator
+ backupSink backup.Sink
}
// NewServer creates a new instance of a grpc SSHServer
@@ -39,6 +42,8 @@ func NewServer(deps *service.Dependencies, serverOpts ...ServerOpt) gitalypb.SSH
prometheus.CounterOpts{},
[]string{"git_negotiation_feature"},
),
+ backupLocator: deps.GetBackupLocator(),
+ backupSink: deps.GetBackupSink(),
}
for _, serverOpt := range serverOpts {
diff --git a/internal/gitaly/service/ssh/upload_pack.go b/internal/gitaly/service/ssh/upload_pack.go
index 0652d2c1f..1036d8f52 100644
--- a/internal/gitaly/service/ssh/upload_pack.go
+++ b/internal/gitaly/service/ssh/upload_pack.go
@@ -7,6 +7,7 @@ import (
"io"
"sync"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/bundleuri"
"gitlab.com/gitlab-org/gitaly/v16/internal/command"
"gitlab.com/gitlab-org/gitaly/v16/internal/git"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/pktline"
@@ -111,6 +112,9 @@ func (s *server) sshUploadPack(ctx context.Context, req sshUploadPackRequest, st
stats.UpdateMetrics(s.packfileNegotiationMetrics)
}()
+ config = append(config,
+ bundleuri.UploadPackGitConfig(ctx, s.backupLocator, s.backupSink, req.GetRepository())...)
+
commandOpts := []git.CmdOpt{
git.WithGitProtocol(s.logger, req),
git.WithConfig(config...),
diff --git a/internal/gitaly/service/ssh/upload_pack_test.go b/internal/gitaly/service/ssh/upload_pack_test.go
index 580755615..88271f121 100644
--- a/internal/gitaly/service/ssh/upload_pack_test.go
+++ b/internal/gitaly/service/ssh/upload_pack_test.go
@@ -36,7 +36,10 @@ import (
)
func runTestWithAndWithoutConfigOptions(t *testing.T, tf func(t *testing.T, ctx context.Context, opts ...testcfg.Option), opts ...testcfg.Option) {
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, func(t *testing.T, ctx context.Context) {
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, func(t *testing.T, ctx context.Context) {
t.Run("no config options", func(t *testing.T) { tf(t, ctx) })
if len(opts) > 0 {
@@ -160,7 +163,10 @@ func testUploadPackTimeout(t *testing.T, ctx context.Context, opts ...testcfg.Op
func TestUploadPackWithSidechannel_client(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testUploadPackWithSidechannelClient)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testUploadPackWithSidechannelClient)
}
func testUploadPackWithSidechannelClient(t *testing.T, ctx context.Context) {
@@ -475,7 +481,10 @@ func requireFailedSSHStream(t *testing.T, expectedErr error, recv func() (int32,
func TestUploadPack_validation(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testUploadPackValidation)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testUploadPackValidation)
}
func testUploadPackValidation(t *testing.T, ctx context.Context) {
@@ -777,7 +786,10 @@ func testUploadPackWithoutSideband(t *testing.T, ctx context.Context, opts ...te
func TestUploadPack_invalidStorage(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testUploadPackInvalidStorage)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testUploadPackInvalidStorage)
}
func testUploadPackInvalidStorage(t *testing.T, ctx context.Context) {
@@ -806,7 +818,10 @@ func testUploadPackInvalidStorage(t *testing.T, ctx context.Context) {
func TestUploadPack_gitFailure(t *testing.T) {
t.Parallel()
- testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testUploadPackGitFailure)
+ testhelper.NewFeatureSets(
+ featureflag.UploadPackBoundaryBitmapTraversal,
+ featureflag.BundleURI,
+ ).Run(t, testUploadPackGitFailure)
}
func testUploadPackGitFailure(t *testing.T, ctx context.Context) {