diff options
author | James Fargher <jfargher@gitlab.com> | 2023-11-21 04:28:20 +0300 |
---|---|---|
committer | James Fargher <jfargher@gitlab.com> | 2023-11-21 04:28:20 +0300 |
commit | 5f57b0c636f00cdff8049375bdca64ebd7b2710f (patch) | |
tree | 0538905caea5ceffc2dff993d7b6d4ac7eccbb44 | |
parent | 38c9c539bdd89301b900e6d25cf18c7f45d60fdd (diff) | |
parent | a987721bb40590b47ad122c30a2aea43a8f20100 (diff) |
Merge branch 'toon-poc-bundle-uri' into 'master'
ssh & smarthttp: Advertise server-side backups as bundle-URI
See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/6472
Merged-by: James Fargher <jfargher@gitlab.com>
Approved-by: James Fargher <jfargher@gitlab.com>
Approved-by: karthik nayak <knayak@gitlab.com>
Reviewed-by: James Fargher <jfargher@gitlab.com>
Reviewed-by: karthik nayak <knayak@gitlab.com>
Co-authored-by: Toon Claes <toon@gitlab.com>
-rw-r--r-- | internal/backup/backup.go | 4 | ||||
-rw-r--r-- | internal/backup/filesystem_sink.go | 7 | ||||
-rw-r--r-- | internal/backup/sink.go | 43 | ||||
-rw-r--r-- | internal/backup/sink_test.go | 50 | ||||
-rw-r--r-- | internal/bundleuri/doc.go | 16 | ||||
-rw-r--r-- | internal/bundleuri/git_config.go | 81 | ||||
-rw-r--r-- | internal/bundleuri/git_config_test.go | 203 | ||||
-rw-r--r-- | internal/bundleuri/testhelper_test.go | 11 | ||||
-rw-r--r-- | internal/featureflag/ff_bundle_uri.go | 9 | ||||
-rw-r--r-- | internal/gitaly/service/smarthttp/inforefs.go | 7 | ||||
-rw-r--r-- | internal/gitaly/service/smarthttp/inforefs_test.go | 100 | ||||
-rw-r--r-- | internal/gitaly/service/smarthttp/server.go | 7 | ||||
-rw-r--r-- | internal/gitaly/service/smarthttp/upload_pack.go | 4 | ||||
-rw-r--r-- | internal/gitaly/service/smarthttp/upload_pack_test.go | 45 | ||||
-rw-r--r-- | internal/gitaly/service/ssh/server.go | 5 | ||||
-rw-r--r-- | internal/gitaly/service/ssh/upload_pack.go | 4 | ||||
-rw-r--r-- | internal/gitaly/service/ssh/upload_pack_test.go | 25 |
17 files changed, 581 insertions, 40 deletions
diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 9f32ee22e..e58fc8718 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "strings" + "time" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" @@ -38,6 +39,9 @@ type Sink interface { // GetReader returns a reader that servers the data stored by relativePath. // If relativePath doesn't exists the ErrDoesntExist will be returned. GetReader(ctx context.Context, relativePath string) (io.ReadCloser, error) + // SignedURL returns a URL that can be used to GET the blob for the duration + // specified in expiry. + SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) } // Backup represents all the information needed to restore a backup for a repository diff --git a/internal/backup/filesystem_sink.go b/internal/backup/filesystem_sink.go index 6d68502db..2b4c04944 100644 --- a/internal/backup/filesystem_sink.go +++ b/internal/backup/filesystem_sink.go @@ -7,8 +7,10 @@ import ( "io" "os" "path/filepath" + "time" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" ) // FilesystemSink is a sink for creating and restoring backups from the local filesystem. @@ -59,3 +61,8 @@ func (fs *FilesystemSink) GetReader(ctx context.Context, relativePath string) (i func (fs *FilesystemSink) Close() error { return nil } + +// SignedURL is not supported by FilesystemSink. +func (fs *FilesystemSink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) { + return "", structerr.NewUnimplemented("SignedURL not implemented for FilesystemSink") +} diff --git a/internal/backup/sink.go b/internal/backup/sink.go index 330b1c5b4..9de909024 100644 --- a/internal/backup/sink.go +++ b/internal/backup/sink.go @@ -6,17 +6,22 @@ import ( "io" "net/url" "strings" + "time" "gocloud.dev/blob" "gocloud.dev/blob/azureblob" "gocloud.dev/blob/gcsblob" + "gocloud.dev/blob/memblob" "gocloud.dev/blob/s3blob" "gocloud.dev/gcerrors" ) -// ResolveSink returns a sink implementation based on the provided path. -func ResolveSink(ctx context.Context, path string) (Sink, error) { - parsed, err := url.Parse(path) +// ResolveSink returns a sink implementation based on the provided uri. +// The storage engine is chosen based on the provided uri. +// It is the caller's responsibility to provide all required environment +// variables in order to get properly initialized storage engine driver. +func ResolveSink(ctx context.Context, uri string) (Sink, error) { + parsed, err := url.Parse(uri) if err != nil { return nil, err } @@ -29,11 +34,10 @@ func ResolveSink(ctx context.Context, path string) (Sink, error) { } switch scheme { - case s3blob.Scheme, azureblob.Scheme, gcsblob.Scheme: - sink, err := NewStorageServiceSink(ctx, path) - return sink, err + case s3blob.Scheme, azureblob.Scheme, gcsblob.Scheme, memblob.Scheme: + return newStorageServiceSink(ctx, uri) default: - return NewFilesystemSink(path), nil + return NewFilesystemSink(uri), nil } } @@ -42,11 +46,8 @@ type StorageServiceSink struct { bucket *blob.Bucket } -// NewStorageServiceSink returns initialized instance of StorageServiceSink instance. -// The storage engine is chosen based on the provided url value and a set of pre-registered -// blank imports in that file. It is the caller's responsibility to provide all required environment -// variables in order to get properly initialized storage engine driver. -func NewStorageServiceSink(ctx context.Context, url string) (*StorageServiceSink, error) { +// newStorageServiceSink returns initialized instance of StorageServiceSink instance. +func newStorageServiceSink(ctx context.Context, url string) (*StorageServiceSink, error) { bucket, err := blob.OpenBucket(ctx, url) if err != nil { return nil, fmt.Errorf("storage service sink: open bucket: %w", err) @@ -98,3 +99,21 @@ func (s *StorageServiceSink) GetReader(ctx context.Context, relativePath string) } return reader, nil } + +// SignedURL returns a URL that can be used to GET the blob for the duration +// specified in expiry. +func (s *StorageServiceSink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) { + opt := &blob.SignedURLOptions{ + Expiry: expiry, + } + + signed, err := s.bucket.SignedURL(ctx, relativePath, opt) + if err != nil { + if gcerrors.Code(err) == gcerrors.NotFound { + err = ErrDoesntExist + } + return "", fmt.Errorf("storage service sink: signed URL for %q: %w", relativePath, err) + } + + return signed, err +} diff --git a/internal/backup/sink_test.go b/internal/backup/sink_test.go index 0c40ee6ff..9ff6f3f8a 100644 --- a/internal/backup/sink_test.go +++ b/internal/backup/sink_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" @@ -115,7 +116,7 @@ func TestStorageServiceSink(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - sss, err := NewStorageServiceSink(ctx, "mem://test_bucket") + sss, err := ResolveSink(ctx, "mem://test_bucket") require.NoError(t, err) defer func() { require.NoError(t, sss.Close()) }() @@ -147,3 +148,50 @@ func TestStorageServiceSink(t *testing.T) { require.Nil(t, reader) }) } + +func TestStorageServiceSink_SignedURL_notImplemented(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + tmpDir := testhelper.TempDir(t) + + for _, tc := range []struct { + desc string + bucketURL string + }{ + { + desc: "memory bucket", + bucketURL: "mem://test_bucket", + }, + { + desc: "fs bucket", + bucketURL: tmpDir, + }, + } { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + sss, err := ResolveSink(ctx, tc.bucketURL) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sss.Close()) }) + + const relativePath = "path/to/data" + + data := []byte("test") + + w, err := sss.GetWriter(ctx, relativePath) + require.NoError(t, err) + + _, err = io.Copy(w, bytes.NewReader(data)) + require.NoError(t, err) + + require.NoError(t, w.Close()) + + _, err = sss.SignedURL(ctx, relativePath, 10*time.Minute) + require.Error(t, err) + require.Contains(t, err.Error(), "not implemented") + }) + } +} diff --git a/internal/bundleuri/doc.go b/internal/bundleuri/doc.go new file mode 100644 index 000000000..22a0b4957 --- /dev/null +++ b/internal/bundleuri/doc.go @@ -0,0 +1,16 @@ +// Package bundleuri is used to enable the use [Bundle-URI] when the client +// clones/fetches from the repository. +// +// Bundle-URI is a concept in Git that allows the server to send one or more +// URIs where [git bundles] are available. The client can download such bundles +// to prepopulate the repository before it starts the object negotiation with +// the server. This reduces the CPU load on the server, and the amount of +// traffic that has to travel directly from server to client. +// +// This feature piggy-backs onto server-side backups. Refer to the +// [backup documentation] how to create and store bundles on a cloud provider. +// +// [Bundle-URI]: https://git-scm.com/docs/bundle-uri +// [git bundles]: https://git-scm.com/docs/git-bundle +// [backup documentation]: https://gitlab.com/gitlab-org/gitaly/-/blob/master/doc/gitaly-backup.md +package bundleuri diff --git a/internal/bundleuri/git_config.go b/internal/bundleuri/git_config.go new file mode 100644 index 000000000..635031153 --- /dev/null +++ b/internal/bundleuri/git_config.go @@ -0,0 +1,81 @@ +package bundleuri + +import ( + "context" + "time" + + "gitlab.com/gitlab-org/gitaly/v16/internal/backup" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" +) + +// InfoRefsGitConfig return a slice of git.ConfigPairs you can inject into the +// call to git-upload-pack(1) --advertise-refs, to advertise the use of +// bundle-URI to the client who clones/fetches from the repository. +func InfoRefsGitConfig(ctx context.Context) []git.ConfigPair { + if featureflag.BundleURI.IsDisabled(ctx) { + return []git.ConfigPair{} + } + + return []git.ConfigPair{ + { + Key: "uploadpack.advertiseBundleURIs", + Value: "true", + }, + } +} + +// UploadPackGitConfig return a slice of git.ConfigPairs you can inject into the +// call to git-upload-pack(1) to advertise the available bundle to the client +// who clones/fetches from the repository. +// In case no backups could be found or something else goes wrong, an empty +// slice is returned without error. +func UploadPackGitConfig( + ctx context.Context, + backupLocator backup.Locator, + backupSink backup.Sink, + repo storage.Repository, +) []git.ConfigPair { + if backupLocator == nil || backupSink == nil || featureflag.BundleURI.IsDisabled(ctx) { + return []git.ConfigPair{} + } + + theBackup, err := backupLocator.FindLatest(ctx, repo) + if err != nil { + return []git.ConfigPair{} + } + + for _, step := range theBackup.Steps { + // Skip non-existing & incremental backups + if len(step.BundlePath) == 0 || len(step.PreviousRefPath) > 0 { + continue + } + + uri, err := backupSink.SignedURL(ctx, step.BundlePath, 10*time.Minute) + if err != nil { + return []git.ConfigPair{} + } + + return []git.ConfigPair{ + { + Key: "uploadpack.advertiseBundleURIs", + Value: "true", + }, + { + Key: "bundle.version", + Value: "1", + }, + { + Key: "bundle.mode", + Value: "any", + }, + { + Key: "bundle.some.uri", + Value: uri, + }, + } + } + + return []git.ConfigPair{} +} diff --git a/internal/bundleuri/git_config_test.go b/internal/bundleuri/git_config_test.go new file mode 100644 index 000000000..c14c3625b --- /dev/null +++ b/internal/bundleuri/git_config_test.go @@ -0,0 +1,203 @@ +package bundleuri + +import ( + "context" + "io" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/backup" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" +) + +func TestUploadPackGitConfig(t *testing.T) { + testhelper.NewFeatureSets(featureflag.BundleURI). + Run(t, testUploadPackGitConfig) +} + +func testUploadPackGitConfig(t *testing.T, ctx context.Context) { + t.Parallel() + + cfg := testcfg.Build(t) + repoProto, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + repo := localrepo.NewTestRepo(t, cfg, repoProto) + + for _, tc := range []struct { + desc string + findLatestFunc func(context.Context, storage.Repository) (*backup.Backup, error) + signedURLFunc func(context.Context, string, time.Duration) (string, error) + expectedConfig []git.ConfigPair + }{ + { + desc: "no backup locator nor sink", + }, + { + desc: "no backup locator", + signedURLFunc: func(context.Context, string, time.Duration) (string, error) { + return "", structerr.NewNotFound("not signed") + }, + }, + { + desc: "no backup sink", + findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) { + return nil, structerr.NewNotFound("no backup found") + }, + }, + { + desc: "no backup found", + findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) { + return nil, structerr.NewNotFound("no backup found") + }, + signedURLFunc: func(context.Context, string, time.Duration) (string, error) { + return "", structerr.NewNotFound("not signed") + }, + }, + { + desc: "backup has no steps", + findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) { + return &backup.Backup{}, nil + }, + signedURLFunc: func(context.Context, string, time.Duration) (string, error) { + return "", structerr.NewNotFound("not signed") + }, + }, + { + desc: "backup step is incremental", + findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) { + return &backup.Backup{ + Steps: []backup.Step{{PreviousRefPath: "not-nil"}}, + }, nil + }, + signedURLFunc: func(context.Context, string, time.Duration) (string, error) { + return "", structerr.NewNotFound("not signed") + }, + }, + { + desc: "sign failed", + findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) { + return &backup.Backup{ + Steps: []backup.Step{{BundlePath: "not-nil"}}, + }, nil + }, + signedURLFunc: func(context.Context, string, time.Duration) (string, error) { + return "", structerr.NewNotFound("not signed") + }, + }, + { + desc: "success", + findLatestFunc: func(context.Context, storage.Repository) (*backup.Backup, error) { + return &backup.Backup{ + Steps: []backup.Step{{BundlePath: "not-nil"}}, + }, nil + }, + signedURLFunc: func(context.Context, string, time.Duration) (string, error) { + return "https://example.com/bundle.git?signed=ok", nil + }, + expectedConfig: []git.ConfigPair{ + { + Key: "uploadpack.advertiseBundleURIs", + Value: "true", + }, + { + Key: "bundle.version", + Value: "1", + }, + { + Key: "bundle.mode", + Value: "any", + }, + { + Key: "bundle.some.uri", + Value: "https://example.com/bundle.git?signed=ok", + }, + }, + }, + } { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + var locator backup.Locator + if tc.findLatestFunc != nil { + locator = dummyLocator{findLatestFunc: tc.findLatestFunc} + } + + var sink backup.Sink + if tc.signedURLFunc != nil { + sink = dummySink{signedURLFunc: tc.signedURLFunc} + } + + actual := UploadPackGitConfig(ctx, locator, sink, repo) + + if featureflag.BundleURI.IsEnabled(ctx) && tc.expectedConfig != nil { + require.Equal(t, tc.expectedConfig, actual) + } else { + require.Empty(t, actual) + } + }) + } +} + +type dummyLocator struct { + findLatestFunc func(context.Context, storage.Repository) (*backup.Backup, error) +} + +// BeginFull is not supported by this dummyLocator. +func (l dummyLocator) BeginFull(ctx context.Context, repo storage.Repository, backupID string) *backup.Backup { + return nil +} + +// BeginIncremental is not supported by this dummyLocator. +func (l dummyLocator) BeginIncremental(ctx context.Context, repo storage.Repository, backupID string) (*backup.Backup, error) { + return nil, structerr.NewUnimplemented("BeginIncremental not implemented for dummyLocator") +} + +// Commit is not supported by this dummyLocator. +func (l dummyLocator) Commit(ctx context.Context, backup *backup.Backup) error { + return structerr.NewUnimplemented("Commit not implemented for dummyLocator") +} + +// FindLatest calls the findLatestFunc of the dummyLocator. +func (l dummyLocator) FindLatest(ctx context.Context, repo storage.Repository) (*backup.Backup, error) { + return l.findLatestFunc(ctx, repo) +} + +// Find is not supported by this dummyLocator. +func (l dummyLocator) Find(ctx context.Context, repo storage.Repository, backupID string) (*backup.Backup, error) { + return nil, structerr.NewUnimplemented("Find not implemented for dummyLocator") +} + +type dummySink struct { + signedURLFunc func(context.Context, string, time.Duration) (string, error) +} + +// Close the dummySink. +func (s dummySink) Close() error { + return nil +} + +// GetWriter is not supported by this dummySink. +func (s dummySink) GetWriter(ctx context.Context, relativePath string) (io.WriteCloser, error) { + return nil, structerr.NewUnimplemented("GetWriter not implemented for dummySink") +} + +// GetReader is not supported by this dummySink. +func (s dummySink) GetReader(ctx context.Context, relativePath string) (io.ReadCloser, error) { + return nil, structerr.NewUnimplemented("GetReader not implemented for dummySink") +} + +// SignedURL calls the signedURLFunc of the dummySink. +func (s dummySink) SignedURL(ctx context.Context, relativePath string, expiry time.Duration) (string, error) { + return s.signedURLFunc(ctx, relativePath, expiry) +} diff --git a/internal/bundleuri/testhelper_test.go b/internal/bundleuri/testhelper_test.go new file mode 100644 index 000000000..b2dd3cfea --- /dev/null +++ b/internal/bundleuri/testhelper_test.go @@ -0,0 +1,11 @@ +package bundleuri + +import ( + "testing" + + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" +) + +func TestMain(m *testing.M) { + testhelper.Run(m) +} diff --git a/internal/featureflag/ff_bundle_uri.go b/internal/featureflag/ff_bundle_uri.go new file mode 100644 index 000000000..c912169dc --- /dev/null +++ b/internal/featureflag/ff_bundle_uri.go @@ -0,0 +1,9 @@ +package featureflag + +// BundleURI enables the use of git's bundle URI feature +var BundleURI = NewFeatureFlag( + "bundle_uri", + "v16.6.0", + "https://gitlab.com/gitlab-org/gitaly/-/issues/5656", + false, +) diff --git a/internal/gitaly/service/smarthttp/inforefs.go b/internal/gitaly/service/smarthttp/inforefs.go index 2a01868d7..63be73f0f 100644 --- a/internal/gitaly/service/smarthttp/inforefs.go +++ b/internal/gitaly/service/smarthttp/inforefs.go @@ -5,6 +5,7 @@ import ( "fmt" "io" + "gitlab.com/gitlab-org/gitaly/v16/internal/bundleuri" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/pktline" "gitlab.com/gitlab-org/gitaly/v16/internal/log" @@ -62,11 +63,13 @@ func (s *server) handleInfoRefs(ctx context.Context, service, repoPath string, r cmdOpts = append(cmdOpts, git.WithDisabledHooks()) } - config, err := git.ConvertConfigOptions(req.GitConfigOptions) + gitConfig, err := git.ConvertConfigOptions(req.GitConfigOptions) if err != nil { return err } - cmdOpts = append(cmdOpts, git.WithConfig(config...)) + gitConfig = append(gitConfig, bundleuri.InfoRefsGitConfig(ctx)...) + + cmdOpts = append(cmdOpts, git.WithConfig(gitConfig...)) if _, err := pktline.WriteString(w, fmt.Sprintf("# service=git-%s\n", service)); err != nil { return structerr.NewInternal("pktLine: %w", err) diff --git a/internal/gitaly/service/smarthttp/inforefs_test.go b/internal/gitaly/service/smarthttp/inforefs_test.go index ba458bac9..836dfce4a 100644 --- a/internal/gitaly/service/smarthttp/inforefs_test.go +++ b/internal/gitaly/service/smarthttp/inforefs_test.go @@ -33,7 +33,10 @@ import ( func TestInfoRefsUploadPack_successful(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackSuccessful) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsUploadPackSuccessful) } func testInfoRefsUploadPackSuccessful(t *testing.T, ctx context.Context) { @@ -63,7 +66,10 @@ func testInfoRefsUploadPackSuccessful(t *testing.T, ctx context.Context) { func TestInfoRefsUploadPack_internalRefs(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackInternalRefs) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsUploadPackInternalRefs) } func testInfoRefsUploadPackInternalRefs(t *testing.T, ctx context.Context) { @@ -138,7 +144,10 @@ func testInfoRefsUploadPackInternalRefs(t *testing.T, ctx context.Context) { func TestInfoRefsUploadPack_validate(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackValidate) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsUploadPackValidate) } func testInfoRefsUploadPackValidate(t *testing.T, ctx context.Context) { @@ -176,7 +185,10 @@ func testInfoRefsUploadPackValidate(t *testing.T, ctx context.Context) { func TestInfoRefsUploadPack_partialClone(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackPartialClone) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsUploadPackPartialClone) } func testInfoRefsUploadPackPartialClone(t *testing.T, ctx context.Context) { @@ -204,7 +216,10 @@ func testInfoRefsUploadPackPartialClone(t *testing.T, ctx context.Context) { func TestInfoRefsUploadPack_gitConfigOptions(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackGitConfigOptions) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsUploadPackGitConfigOptions) } func testInfoRefsUploadPackGitConfigOptions(t *testing.T, ctx context.Context) { @@ -230,10 +245,45 @@ func testInfoRefsUploadPackGitConfigOptions(t *testing.T, ctx context.Context) { }) } +func TestInfoRefsUploadPack_bundleURI(t *testing.T) { + t.Parallel() + + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + ).Run(t, testInfoRefsUploadPackBundleURI) +} + +func testInfoRefsUploadPackBundleURI(t *testing.T, ctx context.Context) { + t.Parallel() + + ctx = featureflag.OutgoingCtxWithFeatureFlag(ctx, featureflag.BundleURI, true) + + cfg := testcfg.Build(t) + cfg.SocketPath = runSmartHTTPServer(t, cfg) + + repo, repoPath := gittest.CreateRepository(t, ctx, cfg) + + gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"), gittest.WithParents()) + + rpcRequest := &gitalypb.InfoRefsRequest{ + Repository: repo, + GitProtocol: git.ProtocolV2, + GitConfigOptions: []string{"transfer.bundleURI=true"}, + } + response, err := makeInfoRefsUploadPackRequest(t, ctx, cfg.SocketPath, cfg.Auth.Token, rpcRequest) + require.NoError(t, err) + requireAdvertisedCapabilitiesV2(t, string(response), "git-upload-pack", []string{ + "bundle-uri", + }) +} + func TestInfoRefsUploadPack_gitProtocol(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackGitProtocol) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsUploadPackGitProtocol) } func testInfoRefsUploadPackGitProtocol(t *testing.T, ctx context.Context) { @@ -290,7 +340,10 @@ func makeInfoRefsUploadPackRequest(t *testing.T, ctx context.Context, serverSock func TestInfoRefsReceivePack_successful(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsReceivePackSuccessful) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsReceivePackSuccessful) } func testInfoRefsReceivePackSuccessful(t *testing.T, ctx context.Context) { @@ -322,7 +375,10 @@ func TestInfoRefsReceivePack_hiddenRefs(t *testing.T) { Object pools are not yet support with WAL. This test is testing with a pooled repository.`) t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsReceivePackHiddenRefs) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsReceivePackHiddenRefs) } func testInfoRefsReceivePackHiddenRefs(t *testing.T, ctx context.Context) { @@ -352,7 +408,10 @@ func testInfoRefsReceivePackHiddenRefs(t *testing.T, ctx context.Context) { func TestInfoRefsReceivePack_validate(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsReceivePackValidate) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsReceivePackValidate) } func testInfoRefsReceivePackValidate(t *testing.T, ctx context.Context) { @@ -405,6 +464,24 @@ func makeInfoRefsReceivePackRequest(t *testing.T, ctx context.Context, serverSoc return response, err } +func requireAdvertisedCapabilitiesV2(t *testing.T, responseBody, expectedService string, expectedCapabilities []string) { + t.Helper() + + responseLines := strings.SplitAfter(responseBody, "\n") + require.Greater(t, len(responseLines), 2) + + // The first line contains the service announcement + require.Equal(t, gittest.Pktlinef(t, "# service=%s\n", expectedService), responseLines[0]) + + // The second line contains the protocol version + require.Equal(t, "0000"+gittest.Pktlinef(t, "version %d\n", 2), responseLines[1]) + + // The third line and following lines contain capabilities + for _, expectedCap := range expectedCapabilities { + require.Contains(t, responseLines[2:], gittest.Pktlinef(t, "%s\n", expectedCap)) + } +} + func requireAdvertisedRefs(t *testing.T, responseBody, expectedService string, expectedRefs []string) { t.Helper() @@ -444,7 +521,10 @@ func (ms *mockStreamer) PutStream(ctx context.Context, repo *gitalypb.Repository func TestInfoRefsUploadPack_cache(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testInfoRefsUploadPackCache) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testInfoRefsUploadPackCache) } func testInfoRefsUploadPackCache(t *testing.T, ctx context.Context) { diff --git a/internal/gitaly/service/smarthttp/server.go b/internal/gitaly/service/smarthttp/server.go index e96930535..33f09309f 100644 --- a/internal/gitaly/service/smarthttp/server.go +++ b/internal/gitaly/service/smarthttp/server.go @@ -2,6 +2,7 @@ package smarthttp import ( "github.com/prometheus/client_golang/prometheus" + "gitlab.com/gitlab-org/gitaly/v16/internal/backup" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" @@ -18,6 +19,8 @@ type server struct { packfileNegotiationMetrics *prometheus.CounterVec infoRefCache infoRefCache txManager transaction.Manager + backupLocator backup.Locator + backupSink backup.Sink } // NewServer creates a new instance of a grpc SmartHTTPServer @@ -31,7 +34,9 @@ func NewServer(deps *service.Dependencies, serverOpts ...ServerOpt) gitalypb.Sma prometheus.CounterOpts{}, []string{"git_negotiation_feature"}, ), - infoRefCache: newInfoRefCache(deps.GetLogger(), deps.GetDiskCache()), + infoRefCache: newInfoRefCache(deps.GetLogger(), deps.GetDiskCache()), + backupLocator: deps.GetBackupLocator(), + backupSink: deps.GetBackupSink(), } for _, serverOpt := range serverOpts { diff --git a/internal/gitaly/service/smarthttp/upload_pack.go b/internal/gitaly/service/smarthttp/upload_pack.go index 7e571df0d..e16fe92d5 100644 --- a/internal/gitaly/service/smarthttp/upload_pack.go +++ b/internal/gitaly/service/smarthttp/upload_pack.go @@ -6,6 +6,7 @@ import ( "fmt" "io" + "gitlab.com/gitlab-org/gitaly/v16/internal/bundleuri" "gitlab.com/gitlab-org/gitaly/v16/internal/command" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" @@ -105,6 +106,9 @@ func (s *server) runUploadPack(ctx context.Context, req *gitalypb.PostUploadPack } }() + gitConfig = append(gitConfig, + bundleuri.UploadPackGitConfig(ctx, s.backupLocator, s.backupSink, req.GetRepository())...) + commandOpts := []git.CmdOpt{ git.WithStdin(stdin), git.WithSetupStdout(), diff --git a/internal/gitaly/service/smarthttp/upload_pack_test.go b/internal/gitaly/service/smarthttp/upload_pack_test.go index 509c38e6c..dabb6ae3b 100644 --- a/internal/gitaly/service/smarthttp/upload_pack_test.go +++ b/internal/gitaly/service/smarthttp/upload_pack_test.go @@ -57,7 +57,10 @@ func runTestWithAndWithoutConfigOptions( func TestServer_PostUploadWithChannel(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadWithChannel) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadWithChannel) } func testServerPostUploadWithChannel(t *testing.T, ctx context.Context) { @@ -108,7 +111,10 @@ func testServerPostUpload(t *testing.T, ctx context.Context, makeRequest request func TestServer_PostUploadPackSidechannel_gitConfigOptions(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackSidechannelGitConfigOptions) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadPackSidechannelGitConfigOptions) } func testServerPostUploadPackSidechannelGitConfigOptions(t *testing.T, ctx context.Context) { @@ -177,7 +183,10 @@ func testServerPostUploadPackGitConfigOptions(t *testing.T, ctx context.Context, func TestServer_PostUploadPackWithSidechannel_gitProtocol(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelGitProtocol) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadPackWithSidechannelGitProtocol) } func testServerPostUploadPackWithSidechannelGitProtocol(t *testing.T, ctx context.Context) { @@ -220,7 +229,10 @@ func testServerPostUploadPackGitProtocol(t *testing.T, ctx context.Context, make func TestServer_PostUploadPackWithSidechannel_suppressDeepenExitError(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelSuppressDeepenExitError) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadPackWithSidechannelSuppressDeepenExitError) } func testServerPostUploadPackWithSidechannelSuppressDeepenExitError(t *testing.T, ctx context.Context) { @@ -250,7 +262,10 @@ func testServerPostUploadPackSuppressDeepenExitError(t *testing.T, ctx context.C func TestServer_PostUploadPackWithSidechannel_usesPackObjectsHook(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelUsesPackObjectsHook) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadPackWithSidechannelUsesPackObjectsHook) } func testServerPostUploadPackWithSidechannelUsesPackObjectsHook(t *testing.T, ctx context.Context) { @@ -300,7 +315,10 @@ func testServerPostUploadPackUsesPackObjectsHook(t *testing.T, ctx context.Conte func TestServer_PostUploadPack_validation(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackValidation) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadPackValidation) } func testServerPostUploadPackValidation(t *testing.T, ctx context.Context) { @@ -346,7 +364,10 @@ func testServerPostUploadPackValidationRequest(t *testing.T, ctx context.Context func TestServer_PostUploadPackSidechannel_validation(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackSidechannelValidation) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadPackSidechannelValidation) } func testServerPostUploadPackSidechannelValidation(t *testing.T, ctx context.Context) { @@ -426,7 +447,10 @@ func extractPackDataFromResponse(t *testing.T, buf *bytes.Buffer) ([]byte, int, func TestServer_PostUploadPackWithSidechannel_partialClone(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelPartialClone) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadPackWithSidechannelPartialClone) } func testServerPostUploadPackWithSidechannelPartialClone(t *testing.T, ctx context.Context) { @@ -484,7 +508,10 @@ func testServerPostUploadPackPartialClone(t *testing.T, ctx context.Context, mak func TestServer_PostUploadPackWithSidechannel_allowAnySHA1InWant(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testServerPostUploadPackWithSidechannelAllowAnySHA1InWant) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testServerPostUploadPackWithSidechannelAllowAnySHA1InWant) } func testServerPostUploadPackWithSidechannelAllowAnySHA1InWant(t *testing.T, ctx context.Context) { diff --git a/internal/gitaly/service/ssh/server.go b/internal/gitaly/service/ssh/server.go index 65ba2180c..8ad343834 100644 --- a/internal/gitaly/service/ssh/server.go +++ b/internal/gitaly/service/ssh/server.go @@ -2,6 +2,7 @@ package ssh import ( "github.com/prometheus/client_golang/prometheus" + "gitlab.com/gitlab-org/gitaly/v16/internal/backup" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" @@ -20,6 +21,8 @@ type server struct { uploadPackRequestTimeoutTickerFactory func() helper.Ticker uploadArchiveRequestTimeoutTickerFactory func() helper.Ticker packfileNegotiationMetrics *prometheus.CounterVec + backupLocator backup.Locator + backupSink backup.Sink } // NewServer creates a new instance of a grpc SSHServer @@ -39,6 +42,8 @@ func NewServer(deps *service.Dependencies, serverOpts ...ServerOpt) gitalypb.SSH prometheus.CounterOpts{}, []string{"git_negotiation_feature"}, ), + backupLocator: deps.GetBackupLocator(), + backupSink: deps.GetBackupSink(), } for _, serverOpt := range serverOpts { diff --git a/internal/gitaly/service/ssh/upload_pack.go b/internal/gitaly/service/ssh/upload_pack.go index 0652d2c1f..1036d8f52 100644 --- a/internal/gitaly/service/ssh/upload_pack.go +++ b/internal/gitaly/service/ssh/upload_pack.go @@ -7,6 +7,7 @@ import ( "io" "sync" + "gitlab.com/gitlab-org/gitaly/v16/internal/bundleuri" "gitlab.com/gitlab-org/gitaly/v16/internal/command" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/pktline" @@ -111,6 +112,9 @@ func (s *server) sshUploadPack(ctx context.Context, req sshUploadPackRequest, st stats.UpdateMetrics(s.packfileNegotiationMetrics) }() + config = append(config, + bundleuri.UploadPackGitConfig(ctx, s.backupLocator, s.backupSink, req.GetRepository())...) + commandOpts := []git.CmdOpt{ git.WithGitProtocol(s.logger, req), git.WithConfig(config...), diff --git a/internal/gitaly/service/ssh/upload_pack_test.go b/internal/gitaly/service/ssh/upload_pack_test.go index 580755615..88271f121 100644 --- a/internal/gitaly/service/ssh/upload_pack_test.go +++ b/internal/gitaly/service/ssh/upload_pack_test.go @@ -36,7 +36,10 @@ import ( ) func runTestWithAndWithoutConfigOptions(t *testing.T, tf func(t *testing.T, ctx context.Context, opts ...testcfg.Option), opts ...testcfg.Option) { - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, func(t *testing.T, ctx context.Context) { + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, func(t *testing.T, ctx context.Context) { t.Run("no config options", func(t *testing.T) { tf(t, ctx) }) if len(opts) > 0 { @@ -160,7 +163,10 @@ func testUploadPackTimeout(t *testing.T, ctx context.Context, opts ...testcfg.Op func TestUploadPackWithSidechannel_client(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testUploadPackWithSidechannelClient) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testUploadPackWithSidechannelClient) } func testUploadPackWithSidechannelClient(t *testing.T, ctx context.Context) { @@ -475,7 +481,10 @@ func requireFailedSSHStream(t *testing.T, expectedErr error, recv func() (int32, func TestUploadPack_validation(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testUploadPackValidation) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testUploadPackValidation) } func testUploadPackValidation(t *testing.T, ctx context.Context) { @@ -777,7 +786,10 @@ func testUploadPackWithoutSideband(t *testing.T, ctx context.Context, opts ...te func TestUploadPack_invalidStorage(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testUploadPackInvalidStorage) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testUploadPackInvalidStorage) } func testUploadPackInvalidStorage(t *testing.T, ctx context.Context) { @@ -806,7 +818,10 @@ func testUploadPackInvalidStorage(t *testing.T, ctx context.Context) { func TestUploadPack_gitFailure(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.UploadPackBoundaryBitmapTraversal).Run(t, testUploadPackGitFailure) + testhelper.NewFeatureSets( + featureflag.UploadPackBoundaryBitmapTraversal, + featureflag.BundleURI, + ).Run(t, testUploadPackGitFailure) } func testUploadPackGitFailure(t *testing.T, ctx context.Context) { |