diff options
author | Will Chandler <wchandler@gitlab.com> | 2022-07-07 00:35:10 +0300 |
---|---|---|
committer | Will Chandler <wchandler@gitlab.com> | 2022-07-07 00:35:10 +0300 |
commit | 2284c7b59150b6847ae4860744881d91b89a739c (patch) | |
tree | 8d3fa739306d555e03792f61a2dc3f0117aee4ab | |
parent | 6944c95243e2b6ed8f783ae903cb9b03ad50e0f9 (diff) | |
parent | fcd94ab1c42e746c5a83ec8bcae2e3ed93d018e8 (diff) |
Merge branch 'pks-fetch-remote-test-with-alternates' into 'master'
repository: Verify behaviour when fetching into pooled repos
Closes #4304
See merge request gitlab-org/gitaly!4671
13 files changed, 230 insertions, 272 deletions
diff --git a/internal/git/gittest/repo.go b/internal/git/gittest/repo.go index c74412570..c34f24548 100644 --- a/internal/git/gittest/repo.go +++ b/internal/git/gittest/repo.go @@ -131,7 +131,7 @@ func CreateRepository(ctx context.Context, t testing.TB, cfg config.Cfg, configs storage = opts.Storage } - relativePath := newDiskHash(t) + relativePath := NewRepositoryName(t, true) if opts.RelativePath != "" { relativePath = opts.RelativePath } diff --git a/internal/gitaly/service/repository/apply_gitattributes_test.go b/internal/gitaly/service/repository/apply_gitattributes_test.go index c2af2db39..4a684421f 100644 --- a/internal/gitaly/service/repository/apply_gitattributes_test.go +++ b/internal/gitaly/service/repository/apply_gitattributes_test.go @@ -12,11 +12,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v15/internal/backchannel" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v15/internal/metadata" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testcfg" - "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v15/internal/transaction/txinfo" "gitlab.com/gitlab-org/gitaly/v15/internal/transaction/voting" "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb" @@ -92,19 +90,7 @@ func TestApplyGitattributesWithTransaction(t *testing.T) { cfg, repo, repoPath := testcfg.BuildWithRepo(t) transactionServer := &testTransactionServer{} - testserver.RunGitalyServer(t, cfg, nil, func(srv *grpc.Server, deps *service.Dependencies) { - gitalypb.RegisterRepositoryServiceServer(srv, NewServer( - deps.GetCfg(), - deps.GetRubyServer(), - deps.GetLocator(), - deps.GetTxManager(), - deps.GetGitCmdFactory(), - deps.GetCatfileCache(), - deps.GetConnsPool(), - deps.GetGit2goExecutor(), - deps.GetHousekeepingManager(), - )) - }) + runRepositoryService(t, cfg, nil) // We're using internal listener in order to route around // Praefect in our tests. Otherwise Praefect would replace our diff --git a/internal/gitaly/service/repository/archive_test.go b/internal/gitaly/service/repository/archive_test.go index d66944817..41c2cc8dd 100644 --- a/internal/gitaly/service/repository/archive_test.go +++ b/internal/gitaly/service/repository/archive_test.go @@ -186,8 +186,7 @@ func TestGetArchive_includeLfsBlobs(t *testing.T) { }, })) - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil) - client := newRepositoryClient(t, cfg, serverSocketPath) + client, serverSocketPath := runRepositoryService(t, cfg, nil) cfg.SocketPath = serverSocketPath repo, _ := gittest.CreateRepository(ctx, t, cfg, gittest.CreateRepositoryConfig{ @@ -473,11 +472,9 @@ func TestGetArchive_environment(t *testing.T) { testcfg.BuildGitalyHooks(t, cfg) - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil, testserver.WithGitCommandFactory(gitCmdFactory)) + client, serverSocketPath := runRepositoryService(t, cfg, nil, testserver.WithGitCommandFactory(gitCmdFactory)) cfg.SocketPath = serverSocketPath - client := newRepositoryClient(t, cfg, serverSocketPath) - repo, _ := gittest.CloneRepo(t, cfg, cfg.Storages[0]) commitID := "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863" diff --git a/internal/gitaly/service/repository/create_fork_test.go b/internal/gitaly/service/repository/create_fork_test.go index 53651c408..d4ccd2994 100644 --- a/internal/gitaly/service/repository/create_fork_test.go +++ b/internal/gitaly/service/repository/create_fork_test.go @@ -11,31 +11,13 @@ import ( "github.com/stretchr/testify/require" gitalyauth "gitlab.com/gitlab-org/gitaly/v15/auth" "gitlab.com/gitlab-org/gitaly/v15/client" - "gitlab.com/gitlab-org/gitaly/v15/internal/backchannel" - "gitlab.com/gitlab-org/gitaly/v15/internal/cache" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping" - "gitlab.com/gitlab-org/gitaly/v15/internal/git2go" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/hook" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/rubyserver" - gserver "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/server" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/commit" - hookservice "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/hook" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/ref" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/remote" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/ssh" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitlab" "gitlab.com/gitlab-org/gitaly/v15/internal/helper" "gitlab.com/gitlab-org/gitaly/v15/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v15/internal/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v15/internal/praefect/praefectutil" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testcfg" - "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testserver" gitalyx509 "gitlab.com/gitlab-org/gitaly/v15/internal/x509" "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb" "google.golang.org/grpc" @@ -68,20 +50,25 @@ func TestCreateFork_successful(t *testing.T) { testcfg.BuildGitalyHooks(t, cfg) testcfg.BuildGitalySSH(t, cfg) - var ( - client gitalypb.RepositoryServiceClient - conn *grpc.ClientConn - ) - createRepoConfig := gittest.CreateRepositoryConfig{ Seed: gittest.SeedGitLabTest, } + getReplicaPathConfig := gittest.GetReplicaPathConfig{} + + var client gitalypb.RepositoryServiceClient if tt.secure { cfg.TLS = tlsConfig - cfg.TLSListenAddr = runSecureServer(t, cfg, nil) + cfg.TLSListenAddr = "localhost:0" + + _, addr := runRepositoryService(t, cfg, nil) + cfg.TLSListenAddr = addr + + var conn *grpc.ClientConn client, conn = newSecureRepoClient(t, cfg.TLSListenAddr, cfg.Auth.Token, certPool) t.Cleanup(func() { conn.Close() }) + createRepoConfig.ClientConn = conn + getReplicaPathConfig.ClientConn = conn } else { client, cfg.SocketPath = runRepositoryService(t, cfg, nil) } @@ -101,13 +88,7 @@ func TestCreateFork_successful(t *testing.T) { }) require.NoError(t, err) - replicaPath := forkedRepo.GetRelativePath() - if !tt.secure { - // Only the insecure test cases run through Praefect, so we only rewrite the path - // in that case. - replicaPath = gittest.GetReplicaPath(ctx, t, cfg, forkedRepo) - } - + replicaPath := gittest.GetReplicaPath(ctx, t, cfg, forkedRepo, getReplicaPathConfig) forkedRepoPath := filepath.Join(cfg.Storages[0].Path, replicaPath) gittest.Exec(t, cfg, "-C", forkedRepoPath, "fsck") @@ -251,91 +232,6 @@ func injectCustomCATestCerts(t *testing.T) (*x509.CertPool, config.TLS) { return pool, config.TLS{CertPath: certFile, KeyPath: keyFile} } -func runSecureServer(t *testing.T, cfg config.Cfg, rubySrv *rubyserver.Server) string { - t.Helper() - - registry := backchannel.NewRegistry() - locator := config.NewLocator(cfg) - cache := cache.New(cfg, locator) - limitHandler := limithandler.New(cfg, limithandler.LimitConcurrencyByRepo, limithandler.WithConcurrencyLimiters) - server, err := gserver.New(true, cfg, testhelper.NewDiscardingLogEntry(t), registry, cache, []*limithandler.LimiterMiddleware{limitHandler}) - require.NoError(t, err) - listener, addr := testhelper.GetLocalhostListener(t) - - txManager := transaction.NewManager(cfg, registry) - gitCmdFactory := gittest.NewCommandFactory(t, cfg) - hookManager := hook.NewManager(cfg, locator, gittest.NewCommandFactory(t, cfg), txManager, gitlab.NewMockClient( - t, gitlab.MockAllowed, gitlab.MockPreReceive, gitlab.MockPostReceive, - )) - catfileCache := catfile.NewCache(cfg) - t.Cleanup(catfileCache.Stop) - - housekeepingManager := housekeeping.NewManager(cfg.Prometheus, txManager) - - connsPool := client.NewPool() - t.Cleanup(func() { testhelper.MustClose(t, connsPool) }) - - git2goExecutor := git2go.NewExecutor(cfg, gitCmdFactory, locator) - - gitalypb.RegisterRepositoryServiceServer(server, NewServer( - cfg, - rubySrv, - locator, - txManager, - gitCmdFactory, - catfileCache, - connsPool, - git2goExecutor, - housekeepingManager, - )) - gitalypb.RegisterHookServiceServer(server, hookservice.NewServer( - hookManager, - gitCmdFactory, - nil, - )) - gitalypb.RegisterRemoteServiceServer(server, remote.NewServer( - locator, - gitCmdFactory, - catfileCache, - txManager, - connsPool, - )) - gitalypb.RegisterSSHServiceServer(server, ssh.NewServer( - locator, - gitCmdFactory, - txManager, - )) - gitalypb.RegisterRefServiceServer(server, ref.NewServer( - locator, - gitCmdFactory, - txManager, - catfileCache, - )) - gitalypb.RegisterCommitServiceServer(server, commit.NewServer( - locator, - gitCmdFactory, - nil, - catfileCache, - )) - errQ := make(chan error, 1) - - // This creates a secondary GRPC server which isn't "secure". Reusing - // the one created above won't work as its internal socket would be - // protected by the same TLS certificate. - - cfg.TLS.KeyPath = "" - testserver.RunGitalyServer(t, cfg, nil, func(srv *grpc.Server, deps *service.Dependencies) { - gitalypb.RegisterHookServiceServer(srv, hookservice.NewServer(deps.GetHookManager(), deps.GetGitCmdFactory(), deps.GetPackObjectsCache())) - }) - - t.Cleanup(func() { require.NoError(t, <-errQ) }) - - t.Cleanup(server.Stop) - go func() { errQ <- server.Serve(listener) }() - - return "tls://" + addr -} - func newSecureRepoClient(t testing.TB, addr, token string, pool *x509.CertPool) (gitalypb.RepositoryServiceClient, *grpc.ClientConn) { t.Helper() diff --git a/internal/gitaly/service/repository/create_repository_from_snapshot_test.go b/internal/gitaly/service/repository/create_repository_from_snapshot_test.go index 9c7ca3383..cf605fc39 100644 --- a/internal/gitaly/service/repository/create_repository_from_snapshot_test.go +++ b/internal/gitaly/service/repository/create_repository_from_snapshot_test.go @@ -70,8 +70,7 @@ func generateTarFile(t *testing.T, path string) ([]byte, []string) { func createFromSnapshot(t *testing.T, ctx context.Context, req *gitalypb.CreateRepositoryFromSnapshotRequest, cfg config.Cfg) (*gitalypb.CreateRepositoryFromSnapshotResponse, error) { t.Helper() - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil) - client := newRepositoryClient(t, cfg, serverSocketPath) + client, _ := runRepositoryService(t, cfg, nil) return client.CreateRepositoryFromSnapshot(ctx, req) } @@ -106,8 +105,8 @@ func TestCreateRepositoryFromSnapshot_success(t *testing.T) { HttpHost: host, } - cfg.SocketPath = runRepositoryServerWithConfig(t, cfg, nil) - client := newRepositoryClient(t, cfg, cfg.SocketPath) + client, socketPath := runRepositoryService(t, cfg, nil) + cfg.SocketPath = socketPath rsp, err := client.CreateRepositoryFromSnapshot(ctx, req) require.NoError(t, err) diff --git a/internal/gitaly/service/repository/create_repository_test.go b/internal/gitaly/service/repository/create_repository_test.go index a612285fb..61a75db3a 100644 --- a/internal/gitaly/service/repository/create_repository_test.go +++ b/internal/gitaly/service/repository/create_repository_test.go @@ -32,7 +32,7 @@ func TestCreateRepository_missingAuth(t *testing.T) { cfg, repo, _ := testcfg.BuildWithRepo(t, testcfg.WithBase(config.Cfg{Auth: auth.Config{Token: "some"}})) - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil) + _, serverSocketPath := runRepositoryService(t, cfg, nil) client := newRepositoryClient(t, config.Cfg{Auth: auth.Config{Token: ""}}, serverSocketPath) _, err := client.CreateRepository(ctx, &gitalypb.CreateRepositoryRequest{Repository: repo}) diff --git a/internal/gitaly/service/repository/fetch_bundle_test.go b/internal/gitaly/service/repository/fetch_bundle_test.go index 0a45fab43..03a2bd0b8 100644 --- a/internal/gitaly/service/repository/fetch_bundle_test.go +++ b/internal/gitaly/service/repository/fetch_bundle_test.go @@ -10,8 +10,6 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" gitalyhook "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/hook" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/hook" "gitlab.com/gitlab-org/gitaly/v15/internal/helper" "gitlab.com/gitlab-org/gitaly/v15/internal/metadata" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" @@ -20,7 +18,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/transaction/txinfo" "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb" "gitlab.com/gitlab-org/gitaly/v15/streamio" - "google.golang.org/grpc" "google.golang.org/grpc/codes" ) @@ -78,26 +75,7 @@ func TestServer_FetchBundle_transaction(t *testing.T) { testcfg.BuildGitalyHooks(t, cfg) hookManager := &mockHookManager{} - addr := testserver.RunGitalyServer(t, cfg, nil, func(srv *grpc.Server, deps *service.Dependencies) { - gitalypb.RegisterRepositoryServiceServer(srv, NewServer( - deps.GetCfg(), - deps.GetRubyServer(), - deps.GetLocator(), - deps.GetTxManager(), - deps.GetGitCmdFactory(), - deps.GetCatfileCache(), - deps.GetConnsPool(), - deps.GetGit2goExecutor(), - deps.GetHousekeepingManager(), - )) - gitalypb.RegisterHookServiceServer(srv, hook.NewServer( - deps.GetHookManager(), - deps.GetGitCmdFactory(), - deps.GetPackObjectsCache(), - )) - }, testserver.WithHookManager(hookManager), testserver.WithDisablePraefect()) - - client := newRepositoryClient(t, cfg, addr) + client, _ := runRepositoryService(t, cfg, nil, testserver.WithHookManager(hookManager), testserver.WithDisablePraefect()) tmp := testhelper.TempDir(t) bundlePath := filepath.Join(tmp, "test.bundle") diff --git a/internal/gitaly/service/repository/fetch_remote_test.go b/internal/gitaly/service/repository/fetch_remote_test.go index 07fbb35f9..c5e164ff1 100644 --- a/internal/gitaly/service/repository/fetch_remote_test.go +++ b/internal/gitaly/service/repository/fetch_remote_test.go @@ -1,8 +1,10 @@ package repository import ( + "bytes" "context" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -16,7 +18,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/helper/text" "gitlab.com/gitlab-org/gitaly/v15/internal/metadata" @@ -25,38 +26,84 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v15/internal/transaction/txinfo" "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb" - "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) -func TestFetchRemoteSuccess(t *testing.T) { +func TestFetchRemote_checkTagsChanged(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - cfg, _, repoPath, client := setupRepositoryService(ctx, t) + cfg, client := setupRepositoryServiceWithoutRepo(t) - cloneRepo, _ := gittest.CreateRepository(ctx, t, cfg, gittest.CreateRepositoryConfig{ - Seed: gittest.SeedGitLabTest, + _, remoteRepoPath := gittest.CreateRepository(ctx, t, cfg) + + gittest.WriteCommit(t, cfg, remoteRepoPath, gittest.WithParents(), gittest.WithBranch("main")) + + t.Run("check tags without tags", func(t *testing.T) { + repoProto, _ := gittest.CreateRepository(ctx, t, cfg) + + response, err := client.FetchRemote(ctx, &gitalypb.FetchRemoteRequest{ + Repository: repoProto, + RemoteParams: &gitalypb.Remote{ + Url: remoteRepoPath, + }, + CheckTagsChanged: true, + }) + require.NoError(t, err) + testhelper.ProtoEqual(t, &gitalypb.FetchRemoteResponse{}, response) }) - // Ensure there's a new tag to fetch - gittest.WriteTag(t, cfg, repoPath, "testtag", "master") + gittest.WriteTag(t, cfg, remoteRepoPath, "testtag", "main") - req := &gitalypb.FetchRemoteRequest{Repository: cloneRepo, RemoteParams: &gitalypb.Remote{ - Url: repoPath, - }, Timeout: 120, CheckTagsChanged: true} - resp, err := client.FetchRemote(ctx, req) - require.NoError(t, err) - require.NotNil(t, resp) - require.Equal(t, resp.TagsChanged, true) + t.Run("check tags with tags", func(t *testing.T) { + repoProto, _ := gittest.CreateRepository(ctx, t, cfg) - // Ensure that it returns true if we're asked not to check - req.CheckTagsChanged = false - resp, err = client.FetchRemote(ctx, req) - require.NoError(t, err) - require.NotNil(t, resp) - require.Equal(t, resp.TagsChanged, true) + // The first fetch should report that the tags have changed, ... + response, err := client.FetchRemote(ctx, &gitalypb.FetchRemoteRequest{ + Repository: repoProto, + RemoteParams: &gitalypb.Remote{ + Url: remoteRepoPath, + }, + CheckTagsChanged: true, + }) + require.NoError(t, err) + testhelper.ProtoEqual(t, &gitalypb.FetchRemoteResponse{ + TagsChanged: true, + }, response) + + // ... while the second fetch shouldn't fetch it anew, and thus the tag should not + // have changed. + response, err = client.FetchRemote(ctx, &gitalypb.FetchRemoteRequest{ + Repository: repoProto, + RemoteParams: &gitalypb.Remote{ + Url: remoteRepoPath, + }, + CheckTagsChanged: true, + }) + require.NoError(t, err) + testhelper.ProtoEqual(t, &gitalypb.FetchRemoteResponse{}, response) + }) + + t.Run("without checking for changed tags", func(t *testing.T) { + repoProto, _ := gittest.CreateRepository(ctx, t, cfg) + + // We fetch into the same repository multiple times to assert that `TagsChanged` is + // `true` regardless of whether we have the tag locally already or not. + for i := 0; i < 2; i++ { + response, err := client.FetchRemote(ctx, &gitalypb.FetchRemoteRequest{ + Repository: repoProto, + RemoteParams: &gitalypb.Remote{ + Url: remoteRepoPath, + }, + CheckTagsChanged: false, + }) + require.NoError(t, err) + testhelper.ProtoEqual(t, &gitalypb.FetchRemoteResponse{ + TagsChanged: true, + }, response) + } + }) } func TestFetchRemote_sshCommand(t *testing.T) { @@ -173,23 +220,9 @@ func TestFetchRemote_transaction(t *testing.T) { sourceCfg := testcfg.Build(t) txManager := transaction.NewTrackingManager() - addr := testserver.RunGitalyServer(t, sourceCfg, nil, func(srv *grpc.Server, deps *service.Dependencies) { - gitalypb.RegisterRepositoryServiceServer(srv, NewServer( - deps.GetCfg(), - deps.GetRubyServer(), - deps.GetLocator(), - deps.GetTxManager(), - deps.GetGitCmdFactory(), - deps.GetCatfileCache(), - deps.GetConnsPool(), - deps.GetGit2goExecutor(), - deps.GetHousekeepingManager(), - )) - }, testserver.WithTransactionManager(txManager)) + client, addr := runRepositoryService(t, sourceCfg, nil, testserver.WithTransactionManager(txManager)) sourceCfg.SocketPath = addr - client := newRepositoryClient(t, sourceCfg, addr) - ctx := testhelper.Context(t) repo, _ := gittest.CreateRepository(ctx, t, sourceCfg, gittest.CreateRepositoryConfig{ RelativePath: t.Name(), @@ -441,7 +474,7 @@ func TestFetchRemote_force(t *testing.T) { } } -func TestFetchRemoteFailure(t *testing.T) { +func TestFetchRemote_inputValidation(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) @@ -586,11 +619,11 @@ func getRefnames(t *testing.T, cfg config.Cfg, repoPath string) []string { return strings.Split(text.ChompBytes(result), "\n") } -func TestFetchRemoteOverHTTP(t *testing.T) { +func TestFetchRemote_http(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - cfg, _, _, client := setupRepositoryService(ctx, t) + cfg, client := setupRepositoryServiceWithoutRepo(t) testCases := []struct { description string @@ -644,7 +677,7 @@ func TestFetchRemoteOverHTTP(t *testing.T) { } } -func TestFetchRemoteWithPath(t *testing.T) { +func TestFetchRemote_localPath(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) @@ -665,7 +698,7 @@ func TestFetchRemoteWithPath(t *testing.T) { require.Equal(t, getRefnames(t, cfg, sourceRepoPath), getRefnames(t, cfg, mirrorRepoPath)) } -func TestFetchRemoteOverHTTPWithRedirect(t *testing.T) { +func TestFetchRemote_httpWithRedirect(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) @@ -690,7 +723,7 @@ func TestFetchRemoteOverHTTPWithRedirect(t *testing.T) { require.Contains(t, err.Error(), "The requested URL returned error: 303") } -func TestFetchRemoteOverHTTPWithTimeout(t *testing.T) { +func TestFetchRemote_httpWithTimeout(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(testhelper.Context(t)) @@ -721,3 +754,123 @@ func TestFetchRemoteOverHTTPWithTimeout(t *testing.T) { require.Contains(t, err.Error(), "fetch remote: signal: terminated") } + +func TestFetchRemote_pooledRepository(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + + // By default git-fetch(1) will always run with `core.alternateRefsCommand=exit 0 #`, which + // effectively disables use of alternate refs. We can't just unset this value, so instead we + // just write a script that knows to execute git-for-each-ref(1) as expected by this config + // option. + // + // Note that we're using a separate command factory here just to ease the setup because we + // need to recreate the other command factory with the Git configuration specified by the + // test. + alternateRefsCommandFactory := gittest.NewCommandFactory(t, testcfg.Build(t)) + exec := testhelper.WriteExecutable(t, + filepath.Join(testhelper.TempDir(t), "alternate-refs"), + []byte(fmt.Sprintf(`#!/bin/sh + exec %q -C "$1" for-each-ref --format='%%(objectname)' + `, alternateRefsCommandFactory.GetExecutionEnvironment(ctx).BinaryPath)), + ) + + for _, tc := range []struct { + desc string + cfg config.Cfg + shouldAnnouncePooledRefs bool + }{ + { + desc: "with default configuration", + }, + { + desc: "with alternates", + cfg: config.Cfg{ + Git: config.Git{ + Config: []config.GitConfig{ + { + Key: "core.alternateRefsCommand", + Value: exec, + }, + }, + }, + }, + shouldAnnouncePooledRefs: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + cfg := testcfg.Build(t, testcfg.WithBase(tc.cfg)) + gitCmdFactory := gittest.NewCommandFactory(t, cfg) + + client, swocketPath := runRepositoryService(t, cfg, nil, testserver.WithGitCommandFactory(gitCmdFactory)) + cfg.SocketPath = swocketPath + + // Create a repository that emulates an object pool. This object contains a + // single reference with an object that is neither in the pool member nor in + // the remote. If alternate refs are used, then Git will announce it to the + // remote as "have". + _, poolRepoPath := gittest.CreateRepository(ctx, t, cfg) + poolCommitID := gittest.WriteCommit(t, cfg, poolRepoPath, + gittest.WithParents(), + gittest.WithBranch("pooled"), + gittest.WithTreeEntries(gittest.TreeEntry{Path: "pool", Mode: "100644", Content: "pool contents"}), + ) + + // Create the pooled repository and link it to its pool. This is the + // repository we're fetching into. + pooledRepoProto, pooledRepoPath := gittest.CreateRepository(ctx, t, cfg) + require.NoError(t, os.WriteFile(filepath.Join(pooledRepoPath, "objects", "info", "alternates"), []byte(filepath.Join(poolRepoPath, "objects")), 0o644)) + + // And then finally create a third repository that emulates the remote side + // we're fetching from. We need to create at least one reference so that Git + // would actually try to fetch objects. + _, remoteRepoPath := gittest.CreateRepository(ctx, t, cfg) + gittest.WriteCommit(t, cfg, remoteRepoPath, + gittest.WithParents(), + gittest.WithBranch("remote"), + gittest.WithTreeEntries(gittest.TreeEntry{Path: "remote", Mode: "100644", Content: "remote contents"}), + ) + + // Set up an HTTP server and intercept the request. This is done so that we + // can observe the reference negotiation and check whether alternate refs + // are announced or not. + var requestBuffer bytes.Buffer + port, stop := gittest.HTTPServer(ctx, t, gitCmdFactory, remoteRepoPath, func(responseWriter http.ResponseWriter, request *http.Request, handler http.Handler) { + closer := request.Body + defer testhelper.MustClose(t, closer) + + request.Body = io.NopCloser(io.TeeReader(request.Body, &requestBuffer)) + + handler.ServeHTTP(responseWriter, request) + }) + defer func() { + require.NoError(t, stop()) + }() + + // Perform the fetch. + _, err := client.FetchRemote(ctx, &gitalypb.FetchRemoteRequest{ + Repository: pooledRepoProto, + RemoteParams: &gitalypb.Remote{ + Url: fmt.Sprintf("http://127.0.0.1:%d/%s", port, filepath.Base(remoteRepoPath)), + }, + }) + require.NoError(t, err) + + // This should result in the "remote" branch having been fetched into the + // pooled repository. + require.Equal(t, + gittest.ResolveRevision(t, cfg, pooledRepoPath, "refs/heads/remote"), + gittest.ResolveRevision(t, cfg, remoteRepoPath, "refs/heads/remote"), + ) + + // Verify whether alternate refs have been announced as part of the + // reference negotiation phase. + if tc.shouldAnnouncePooledRefs { + require.Contains(t, requestBuffer.String(), fmt.Sprintf("have %s", poolCommitID)) + } else { + require.NotContains(t, requestBuffer.String(), fmt.Sprintf("have %s", poolCommitID)) + } + }) + } +} diff --git a/internal/gitaly/service/repository/replicate_test.go b/internal/gitaly/service/repository/replicate_test.go index f4f4a7030..69c049ea9 100644 --- a/internal/gitaly/service/repository/replicate_test.go +++ b/internal/gitaly/service/repository/replicate_test.go @@ -20,10 +20,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" gitalyhook "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/hook" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/hook" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/ref" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/ssh" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/helper/text" @@ -48,11 +44,9 @@ func TestReplicateRepository(t *testing.T) { testcfg.BuildGitalyHooks(t, cfg) testcfg.BuildGitalySSH(t, cfg) - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil, testserver.WithDisablePraefect()) + client, serverSocketPath := runRepositoryService(t, cfg, nil, testserver.WithDisablePraefect()) cfg.SocketPath = serverSocketPath - client := newRepositoryClient(t, cfg, serverSocketPath) - repo, repoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0]) // create a loose object to ensure snapshot replication is used @@ -121,7 +115,7 @@ func TestReplicateRepositoryTransactional(t *testing.T) { testcfg.BuildGitalyHooks(t, cfg) testcfg.BuildGitalySSH(t, cfg) - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil, testserver.WithDisablePraefect()) + _, serverSocketPath := runRepositoryService(t, cfg, nil, testserver.WithDisablePraefect()) cfg.SocketPath = serverSocketPath sourceRepo, sourceRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[0]) @@ -330,11 +324,9 @@ func TestReplicateRepository_BadRepository(t *testing.T) { testcfg.BuildGitalyHooks(t, cfg) testcfg.BuildGitalySSH(t, cfg) - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil, testserver.WithDisablePraefect()) + client, serverSocketPath := runRepositoryService(t, cfg, nil, testserver.WithDisablePraefect()) cfg.SocketPath = serverSocketPath - client := newRepositoryClient(t, cfg, serverSocketPath) - sourceRepo, _ := gittest.CloneRepo(t, cfg, cfg.Storages[0]) targetRepo, targetRepoPath := gittest.CloneRepo(t, cfg, cfg.Storages[1], gittest.CloneRepoOpts{ RelativePath: sourceRepo.RelativePath, @@ -386,7 +378,8 @@ func TestReplicateRepository_FailedFetchInternalRemote(t *testing.T) { // Our test setup does not allow for Praefects with multiple storages. We thus have to // disable Praefect here. - cfg.SocketPath = runRepositoryServerWithConfig(t, cfg, nil, testserver.WithDisablePraefect()) + client, socketPath := runRepositoryService(t, cfg, nil, testserver.WithDisablePraefect()) + cfg.SocketPath = socketPath targetRepo, _ := gittest.InitRepo(t, cfg, cfg.Storages[1]) @@ -405,9 +398,7 @@ func TestReplicateRepository_FailedFetchInternalRemote(t *testing.T) { ctx = testhelper.MergeOutgoingMetadata(ctx, testcfg.GitalyServersMetadataFromCfg(t, cfg)) - repoClient := newRepositoryClient(t, cfg, cfg.SocketPath) - - _, err = repoClient.ReplicateRepository(ctx, &gitalypb.ReplicateRepositoryRequest{ + _, err = client.ReplicateRepository(ctx, &gitalypb.ReplicateRepositoryRequest{ Repository: targetRepo, Source: sourceRepo, }) @@ -465,19 +456,7 @@ func TestFetchInternalRemote_successful(t *testing.T) { testcfg.BuildGitalyHooks(t, remoteCfg) gittest.WriteCommit(t, remoteCfg, remoteRepoPath, gittest.WithBranch("master")) - remoteAddr := testserver.RunGitalyServer(t, remoteCfg, nil, func(srv *grpc.Server, deps *service.Dependencies) { - gitalypb.RegisterSSHServiceServer(srv, ssh.NewServer( - deps.GetLocator(), - deps.GetGitCmdFactory(), - deps.GetTxManager(), - )) - gitalypb.RegisterRefServiceServer(srv, ref.NewServer( - deps.GetLocator(), - deps.GetGitCmdFactory(), - deps.GetTxManager(), - deps.GetCatfileCache(), - )) - }, testserver.WithDisablePraefect()) + _, remoteAddr := runRepositoryService(t, remoteCfg, nil, testserver.WithDisablePraefect()) localCfg, localRepoProto, localRepoPath := testcfg.BuildWithRepo(t) localRepo := localrepo.NewTestRepo(t, localCfg, localRepoProto) @@ -489,13 +468,7 @@ func TestFetchInternalRemote_successful(t *testing.T) { // We do not require the server's address, but it needs to be around regardless such that // `FetchInternalRemote` can reach the hook service which is injected via the config. - testserver.RunGitalyServer(t, localCfg, nil, func(srv *grpc.Server, deps *service.Dependencies) { - gitalypb.RegisterHookServiceServer(srv, hook.NewServer( - deps.GetHookManager(), - deps.GetGitCmdFactory(), - deps.GetPackObjectsCache(), - )) - }, testserver.WithHookManager(gitalyhook.NewMockManager(t, nil, nil, nil, + runRepositoryService(t, localCfg, nil, testserver.WithHookManager(gitalyhook.NewMockManager(t, nil, nil, nil, func(t *testing.T, _ context.Context, _ gitalyhook.ReferenceTransactionState, _ []string, stdin io.Reader) error { // We need to discard stdin or otherwise the sending Goroutine may return an // EOF error and cause the test to fail. diff --git a/internal/gitaly/service/repository/repository_test.go b/internal/gitaly/service/repository/repository_test.go index f5a26f61e..a834f19dc 100644 --- a/internal/gitaly/service/repository/repository_test.go +++ b/internal/gitaly/service/repository/repository_test.go @@ -21,12 +21,10 @@ func TestRepositoryExists(t *testing.T) { require.NoError(t, os.RemoveAll(cfg.Storages[2].Path), "third storage needs to be invalid") - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil, testserver.WithDisablePraefect()) + client, _ := runRepositoryService(t, cfg, nil, testserver.WithDisablePraefect()) repo, _ := gittest.CloneRepo(t, cfg, cfg.Storages[0]) - client := newRepositoryClient(t, cfg, serverSocketPath) - queries := []struct { desc string request *gitalypb.RepositoryExistsRequest diff --git a/internal/gitaly/service/repository/restore_custom_hooks_test.go b/internal/gitaly/service/repository/restore_custom_hooks_test.go index 965e0d768..caa6d0290 100644 --- a/internal/gitaly/service/repository/restore_custom_hooks_test.go +++ b/internal/gitaly/service/repository/restore_custom_hooks_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/metadata" "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" @@ -21,7 +20,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/transaction/voting" "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb" "gitlab.com/gitlab-org/gitaly/v15/streamio" - "google.golang.org/grpc" "google.golang.org/grpc/codes" ) @@ -38,23 +36,9 @@ func testSuccessfulRestoreCustomHooksRequest(t *testing.T, ctx context.Context) testcfg.BuildGitalyHooks(t, cfg) txManager := transaction.NewTrackingManager() - addr := testserver.RunGitalyServer(t, cfg, nil, func(srv *grpc.Server, deps *service.Dependencies) { - gitalypb.RegisterRepositoryServiceServer(srv, NewServer( - deps.GetCfg(), - deps.GetRubyServer(), - deps.GetLocator(), - deps.GetTxManager(), - deps.GetGitCmdFactory(), - deps.GetCatfileCache(), - deps.GetConnsPool(), - deps.GetGit2goExecutor(), - deps.GetHousekeepingManager(), - )) - }, testserver.WithTransactionManager(txManager)) + client, addr := runRepositoryService(t, cfg, nil, testserver.WithTransactionManager(txManager)) cfg.SocketPath = addr - client := newRepositoryClient(t, cfg, addr) - ctx, err := txinfo.InjectTransaction(ctx, 1, "node", true) require.NoError(t, err) diff --git a/internal/gitaly/service/repository/size_test.go b/internal/gitaly/service/repository/size_test.go index 2ce51466e..be9459f2f 100644 --- a/internal/gitaly/service/repository/size_test.go +++ b/internal/gitaly/service/repository/size_test.go @@ -37,10 +37,9 @@ func TestRepositorySize_SuccessfulRequest(t *testing.T) { func testSuccessfulRepositorySizeRequestPoolMember(t *testing.T, ctx context.Context) { cfg := testcfg.Build(t) - serverSocketPath := runRepositoryServerWithConfig(t, cfg, nil) + repoClient, serverSocketPath := runRepositoryService(t, cfg, nil) cfg.SocketPath = serverSocketPath - repoClient := newRepositoryClient(t, cfg, serverSocketPath) objectPoolClient := newObjectPoolClient(t, cfg, serverSocketPath) repo, repoPath := gittest.CreateRepository(ctx, t, cfg, gittest.CreateRepositoryConfig{ diff --git a/internal/gitaly/service/repository/testhelper_test.go b/internal/gitaly/service/repository/testhelper_test.go index 79fb534d3..4e23841c8 100644 --- a/internal/gitaly/service/repository/testhelper_test.go +++ b/internal/gitaly/service/repository/testhelper_test.go @@ -109,8 +109,8 @@ func assertModTimeAfter(t *testing.T, afterTime time.Time, paths ...string) bool return t.Failed() } -func runRepositoryServerWithConfig(t testing.TB, cfg config.Cfg, rubySrv *rubyserver.Server, opts ...testserver.GitalyServerOpt) string { - return testserver.RunGitalyServer(t, cfg, rubySrv, func(srv *grpc.Server, deps *service.Dependencies) { +func runRepositoryService(t testing.TB, cfg config.Cfg, rubySrv *rubyserver.Server, opts ...testserver.GitalyServerOpt) (gitalypb.RepositoryServiceClient, string) { + serverSocketPath := testserver.RunGitalyServer(t, cfg, rubySrv, func(srv *grpc.Server, deps *service.Dependencies) { gitalypb.RegisterRepositoryServiceServer(srv, NewServer( cfg, deps.GetRubyServer(), @@ -155,13 +155,8 @@ func runRepositoryServerWithConfig(t testing.TB, cfg config.Cfg, rubySrv *rubyse deps.GetHousekeepingManager(), )) }, opts...) -} - -func runRepositoryService(t testing.TB, cfg config.Cfg, rubySrv *rubyserver.Server, opts ...testserver.GitalyServerOpt) (gitalypb.RepositoryServiceClient, string) { - serverSocketPath := runRepositoryServerWithConfig(t, cfg, rubySrv, opts...) - client := newRepositoryClient(t, cfg, serverSocketPath) - return client, serverSocketPath + return newRepositoryClient(t, cfg, serverSocketPath), serverSocketPath } func setupRepositoryService(ctx context.Context, t testing.TB, opts ...testserver.GitalyServerOpt) (config.Cfg, *gitalypb.Repository, string, gitalypb.RepositoryServiceClient) { |