diff options
author | Pavlo Strokov <pstrokov@gitlab.com> | 2023-05-10 12:53:11 +0300 |
---|---|---|
committer | Pavlo Strokov <pstrokov@gitlab.com> | 2023-05-10 12:53:11 +0300 |
commit | 242c75f51dd840799467c41b0fe15a2214ae07c2 (patch) | |
tree | a96a9fe277041c11ea8bfdb849fc779c02e22e1f | |
parent | 0127ee18adf7fffc819662f9598ffd43acc82598 (diff) | |
parent | db4f3082e2280b219c4166edb7e140fcf4a829ac (diff) |
Merge branch 'pks-create-fork-preparatory-refactorings' into 'master'
Preparatory refactorings for linking to pool repos in CreateFork
See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5718
Merged-by: Pavlo Strokov <pstrokov@gitlab.com>
Approved-by: Pavlo Strokov <pstrokov@gitlab.com>
Reviewed-by: Patrick Steinhardt <psteinhardt@gitlab.com>
Reviewed-by: Pavlo Strokov <pstrokov@gitlab.com>
Co-authored-by: Patrick Steinhardt <psteinhardt@gitlab.com>
19 files changed, 186 insertions, 171 deletions
diff --git a/internal/git/gittest/objectpool.go b/internal/git/gittest/objectpool.go new file mode 100644 index 000000000..3374b736b --- /dev/null +++ b/internal/git/gittest/objectpool.go @@ -0,0 +1,91 @@ +package gittest + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb" + "google.golang.org/grpc" +) + +// CreateObjectPoolConfig can be used to tweak how CreateObjectPool behaves. +type CreateObjectPoolConfig struct { + // ClientConn is the connection used to create the repository. If unset, the config is used + // to dial the service. + ClientConn *grpc.ClientConn + // RelativePath sets the relative path of the repository in the storage. If unset, + // the relative path is set to a randomly generated hashed storage path. + RelativePath string + // LinkRepositoryToObjectPool determines whether the repository shall be linked to the object + // pool. + LinkRepositoryToObjectPool bool +} + +// CreateObjectPool creates a new object pool from the given source repository. It returns the +// Protobuf representation used for gRPC calls. This can be passed either no or exactly one +// CreateObjectPoolConfig to influence how the pool will be created. +func CreateObjectPool( + tb testing.TB, + ctx context.Context, + cfg config.Cfg, + source *gitalypb.Repository, + optionalCfg ...CreateObjectPoolConfig, +) (*gitalypb.ObjectPool, string) { + tb.Helper() + + require.LessOrEqual(tb, len(optionalCfg), 1) + var createCfg CreateObjectPoolConfig + if len(optionalCfg) == 1 { + createCfg = optionalCfg[0] + } + + conn := createCfg.ClientConn + if conn == nil { + conn = dialService(tb, ctx, cfg) + defer testhelper.MustClose(tb, conn) + } + client := gitalypb.NewObjectPoolServiceClient(conn) + + // We use the same storage as the source repository. So we need to figure out whether we + // actually have the storage configuration for it. + var storage config.Storage + for _, s := range cfg.Storages { + if s.Name == source.StorageName { + storage = s + break + } + } + require.NotEmpty(tb, storage.Name) + + relativePath := createCfg.RelativePath + if relativePath == "" { + relativePath = NewObjectPoolName(tb) + } + + poolProto := &gitalypb.ObjectPool{ + Repository: &gitalypb.Repository{ + StorageName: storage.Name, + RelativePath: relativePath, + }, + } + + _, err := client.CreateObjectPool(ctx, &gitalypb.CreateObjectPoolRequest{ + ObjectPool: poolProto, + Origin: source, + }) + require.NoError(tb, err) + + if createCfg.LinkRepositoryToObjectPool { + _, err := client.LinkRepositoryToObjectPool(ctx, &gitalypb.LinkRepositoryToObjectPoolRequest{ + ObjectPool: poolProto, + Repository: source, + }) + require.NoError(tb, err) + } + + return poolProto, filepath.Join(storage.Path, getReplicaPath(tb, ctx, conn, poolProto.Repository)) +} diff --git a/internal/gitaly/service/objectpool/alternates_test.go b/internal/gitaly/service/objectpool/alternates_test.go index 02355b70c..e6e58f091 100644 --- a/internal/gitaly/service/objectpool/alternates_test.go +++ b/internal/gitaly/service/objectpool/alternates_test.go @@ -38,7 +38,7 @@ func testDisconnectGitAlternates(t *testing.T, ctx context.Context) { // We create the object pool, link the original repository to it and then repack the pool // member. As the linking step should've pulled all objects into the pool, the repack should // get rid of the now-duplicate objects in the repository in favor of the pooled ones. - _, pool, _ := createObjectPool(t, ctx, cfg, client, repoProto) + _, pool, _ := createObjectPool(t, ctx, cfg, repoProto) require.NoError(t, pool.Link(ctx, repo)) gittest.Exec(t, cfg, "-C", repoPath, "gc") diff --git a/internal/gitaly/service/objectpool/delete_test.go b/internal/gitaly/service/objectpool/delete_test.go index 77b613b25..ed95f2d8c 100644 --- a/internal/gitaly/service/objectpool/delete_test.go +++ b/internal/gitaly/service/objectpool/delete_test.go @@ -21,7 +21,7 @@ func TestDelete(t *testing.T) { repositoryClient := gitalypb.NewRepositoryServiceClient(extractConn(client)) - poolProto, _, _ := createObjectPool(t, ctx, cfg, client, repoProto) + poolProto, _, _ := createObjectPool(t, ctx, cfg, repoProto) validPoolPath := poolProto.GetRepository().GetRelativePath() for _, tc := range []struct { diff --git a/internal/gitaly/service/objectpool/fetch_into_object_pool_test.go b/internal/gitaly/service/objectpool/fetch_into_object_pool_test.go index 93c4a9c74..227f113fa 100644 --- a/internal/gitaly/service/objectpool/fetch_into_object_pool_test.go +++ b/internal/gitaly/service/objectpool/fetch_into_object_pool_test.go @@ -47,7 +47,7 @@ func testFetchIntoObjectPoolSuccess(t *testing.T, ctx context.Context) { parentID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main")) - poolProto, _, poolPath := createObjectPool(t, ctx, cfg, client, repo) + poolProto, _, poolPath := createObjectPool(t, ctx, cfg, repo) // Create a new commit after having created the object pool. This commit exists only in the // pool member, but not in the pool itself. @@ -126,7 +126,7 @@ func testFetchIntoObjectPoolTransactional(t *testing.T, ctx context.Context) { client := gitalypb.NewObjectPoolServiceClient(conn) - poolProto, pool, poolPath := createObjectPool(t, ctx, cfg, client, repo) + poolProto, pool, poolPath := createObjectPool(t, ctx, cfg, repo) // Inject transaction information so that FetchInotObjectPool knows to perform // transactional voting. @@ -230,7 +230,7 @@ func testFetchIntoObjectPoolcollectLogStatistics(t *testing.T, ctx context.Conte t.Cleanup(func() { testhelper.MustClose(t, conn) }) client := gitalypb.NewObjectPoolServiceClient(conn) - poolProto, _, _ := createObjectPool(t, ctx, cfg, client, repo) + poolProto, _, _ := createObjectPool(t, ctx, cfg, repo) req := &gitalypb.FetchIntoObjectPoolRequest{ ObjectPool: poolProto, @@ -254,7 +254,7 @@ func TestFetchIntoObjectPool_Failure(t *testing.T) { ctx := testhelper.Context(t) cfg, repo, _, _, client := setup(t, ctx, testserver.WithDisablePraefect()) - poolProto, _, _ := createObjectPool(t, ctx, cfg, client, repo) + poolProto, _, _ := createObjectPool(t, ctx, cfg, repo) poolWithDifferentStorage := proto.Clone(poolProto).(*gitalypb.ObjectPool) poolWithDifferentStorage.Repository.StorageName = "some other storage" @@ -309,7 +309,7 @@ func testFetchIntoObjectPoolDfConflict(t *testing.T, ctx context.Context) { t.Parallel() cfg, repo, repoPath, _, client := setup(t, ctx) - poolProto, _, poolPath := createObjectPool(t, ctx, cfg, client, repo) + poolProto, _, poolPath := createObjectPool(t, ctx, cfg, repo) gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("branch")) diff --git a/internal/gitaly/service/objectpool/get_test.go b/internal/gitaly/service/objectpool/get_test.go index 86453e711..60afec0e2 100644 --- a/internal/gitaly/service/objectpool/get_test.go +++ b/internal/gitaly/service/objectpool/get_test.go @@ -21,7 +21,7 @@ func TestGetObjectPoolSuccess(t *testing.T) { cfg, repoProto, _, _, client := setup(t, ctx) repo := localrepo.NewTestRepo(t, cfg, repoProto) - _, pool, _ := createObjectPool(t, ctx, cfg, client, repoProto) + _, pool, _ := createObjectPool(t, ctx, cfg, repoProto) relativePoolPath := pool.GetRelativePath() require.NoError(t, pool.Link(ctx, repo)) diff --git a/internal/gitaly/service/objectpool/link_test.go b/internal/gitaly/service/objectpool/link_test.go index 6c4109a69..2a49015e2 100644 --- a/internal/gitaly/service/objectpool/link_test.go +++ b/internal/gitaly/service/objectpool/link_test.go @@ -23,7 +23,7 @@ func TestLink(t *testing.T) { cfg, repo, _, _, client := setup(t, ctx, testserver.WithDisablePraefect()) localRepo := localrepo.NewTestRepo(t, cfg, repo) - poolProto, _, poolPath := createObjectPool(t, ctx, cfg, client, repo) + poolProto, _, poolPath := createObjectPool(t, ctx, cfg, repo) // Mock object in the pool, which should be available to the pool members // after linking @@ -80,7 +80,7 @@ func TestLink_idempotent(t *testing.T) { ctx := testhelper.Context(t) cfg, repoProto, _, _, client := setup(t, ctx) - poolProto, _, _ := createObjectPool(t, ctx, cfg, client, repoProto) + poolProto, _, _ := createObjectPool(t, ctx, cfg, repoProto) request := &gitalypb.LinkRepositoryToObjectPoolRequest{ Repository: repoProto, @@ -99,7 +99,7 @@ func TestLink_noClobber(t *testing.T) { ctx := testhelper.Context(t) cfg, repoProto, repoPath, _, client := setup(t, ctx) - poolProto, _, _ := createObjectPool(t, ctx, cfg, client, repoProto) + poolProto, _, _ := createObjectPool(t, ctx, cfg, repoProto) alternatesFile := filepath.Join(repoPath, "objects/info/alternates") require.NoFileExists(t, alternatesFile) diff --git a/internal/gitaly/service/objectpool/testhelper_test.go b/internal/gitaly/service/objectpool/testhelper_test.go index 8665a01cd..dd78f09b8 100644 --- a/internal/gitaly/service/objectpool/testhelper_test.go +++ b/internal/gitaly/service/objectpool/testhelper_test.go @@ -93,23 +93,11 @@ func createObjectPool( tb testing.TB, ctx context.Context, cfg config.Cfg, - client gitalypb.ObjectPoolServiceClient, source *gitalypb.Repository, ) (*gitalypb.ObjectPool, *objectpool.ObjectPool, string) { tb.Helper() - poolProto := &gitalypb.ObjectPool{ - Repository: &gitalypb.Repository{ - StorageName: source.GetStorageName(), - RelativePath: gittest.NewObjectPoolName(tb), - }, - } - - _, err := client.CreateObjectPool(ctx, &gitalypb.CreateObjectPoolRequest{ - ObjectPool: poolProto, - Origin: source, - }) - require.NoError(tb, err) + poolProto, poolProtoPath := gittest.CreateObjectPool(tb, ctx, cfg, source) txManager := transaction.NewManager(cfg, nil) catfileCache := catfile.NewCache(cfg) @@ -130,5 +118,5 @@ func createObjectPool( ) require.NoError(tb, err) - return poolProto, pool, gittest.RepositoryPath(tb, pool) + return poolProto, pool, poolProtoPath } diff --git a/internal/gitaly/service/repository/create_fork.go b/internal/gitaly/service/repository/create_fork.go index bb172eac5..d10307639 100644 --- a/internal/gitaly/service/repository/create_fork.go +++ b/internal/gitaly/service/repository/create_fork.go @@ -13,16 +13,16 @@ import ( ) func (s *server) CreateFork(ctx context.Context, req *gitalypb.CreateForkRequest) (*gitalypb.CreateForkResponse, error) { - targetRepository := req.Repository - sourceRepository := req.SourceRepository - - if sourceRepository == nil { - return nil, structerr.NewInvalidArgument("empty SourceRepository") + if err := service.ValidateRepository(req.GetSourceRepository()); err != nil { + return nil, structerr.NewInvalidArgument("validating source repository: %w", err) } if err := service.ValidateRepository(req.GetRepository()); err != nil { return nil, structerr.NewInvalidArgument("%w", err) } + targetRepository := req.Repository + sourceRepository := req.SourceRepository + if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, targetRepository, func(repo *gitalypb.Repository) error { targetPath, err := s.locator.GetPath(repo) if err != nil { diff --git a/internal/gitaly/service/repository/create_fork_test.go b/internal/gitaly/service/repository/create_fork_test.go index ff68a9051..a1352e1eb 100644 --- a/internal/gitaly/service/repository/create_fork_test.go +++ b/internal/gitaly/service/repository/create_fork_test.go @@ -283,7 +283,7 @@ func TestCreateFork_validate(t *testing.T) { if testhelper.IsPraefectEnabled() { return status.Error(codes.AlreadyExists, "route repository creation: reserve repository id: repository already exists") } - return status.Error(codes.InvalidArgument, "empty SourceRepository") + return status.Error(codes.InvalidArgument, "validating source repository: empty Repository") }(), }, } { diff --git a/internal/gitaly/service/repository/size_test.go b/internal/gitaly/service/repository/size_test.go index 1143a6b9e..b0e740e8b 100644 --- a/internal/gitaly/service/repository/size_test.go +++ b/internal/gitaly/service/repository/size_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v15/internal/git" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v15/internal/git/quarantine" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" @@ -33,8 +32,6 @@ func TestSuccessfulRepositorySizeRequestPoolMember(t *testing.T) { repoClient, serverSocketPath := runRepositoryService(t, cfg) cfg.SocketPath = serverSocketPath - objectPoolClient := newObjectPoolClient(t, cfg, serverSocketPath) - repo, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ Seed: gittest.SeedGitLabTest, }) @@ -45,32 +42,9 @@ func TestSuccessfulRepositorySizeRequestPoolMember(t *testing.T) { sizeBeforePool := response.GetSize() - catfileCache := catfile.NewCache(cfg) - t.Cleanup(catfileCache.Stop) - - poolProto := &gitalypb.ObjectPool{ - Repository: &gitalypb.Repository{ - StorageName: cfg.Storages[0].Name, - RelativePath: gittest.NewObjectPoolName(t), - }, - } - - _, err = objectPoolClient.CreateObjectPool( - ctx, - &gitalypb.CreateObjectPoolRequest{ - ObjectPool: poolProto, - Origin: repo, - }) - require.NoError(t, err) - - _, err = objectPoolClient.LinkRepositoryToObjectPool( - ctx, - &gitalypb.LinkRepositoryToObjectPoolRequest{ - Repository: repo, - ObjectPool: poolProto, - }, - ) - require.NoError(t, err) + gittest.CreateObjectPool(t, ctx, cfg, repo, gittest.CreateObjectPoolConfig{ + LinkRepositoryToObjectPool: true, + }) gittest.Exec(t, cfg, "-C", repoPath, "gc") diff --git a/internal/gitaly/service/repository/testhelper_test.go b/internal/gitaly/service/repository/testhelper_test.go index 1d82a3710..0bf885d69 100644 --- a/internal/gitaly/service/repository/testhelper_test.go +++ b/internal/gitaly/service/repository/testhelper_test.go @@ -44,14 +44,6 @@ func newRepositoryClient(tb testing.TB, cfg config.Cfg, serverSocketPath string) return gitalypb.NewRepositoryServiceClient(conn) } -func newObjectPoolClient(tb testing.TB, cfg config.Cfg, serverSocketPath string) gitalypb.ObjectPoolServiceClient { - conn, err := gclient.Dial(serverSocketPath, nil) - require.NoError(tb, err) - tb.Cleanup(func() { require.NoError(tb, conn.Close()) }) - - return gitalypb.NewObjectPoolServiceClient(conn) -} - func newMuxedRepositoryClient(t *testing.T, ctx context.Context, cfg config.Cfg, serverSocketPath string, handshaker internalclient.Handshaker) gitalypb.RepositoryServiceClient { conn, err := internalclient.Dial(ctx, serverSocketPath, []grpc.DialOption{ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(cfg.Auth.Token)), diff --git a/internal/gitaly/service/smarthttp/inforefs_test.go b/internal/gitaly/service/smarthttp/inforefs_test.go index 2c08a5e88..a699f5f5c 100644 --- a/internal/gitaly/service/smarthttp/inforefs_test.go +++ b/internal/gitaly/service/smarthttp/inforefs_test.go @@ -12,16 +12,11 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v15/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v15/internal/cache" "gitlab.com/gitlab-org/gitaly/v15/internal/git" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/objectpool" "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" @@ -302,36 +297,16 @@ func TestInfoRefsReceivePack_hiddenRefs(t *testing.T) { ctx := testhelper.Context(t) repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg) - repo := localrepo.NewTestRepo(t, cfg, repoProto) gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main")) - txManager := transaction.NewManager(cfg, backchannel.NewRegistry()) - - pool, err := objectpool.Create( - ctx, - config.NewLocator(cfg), - gittest.NewCommandFactory(t, cfg), - nil, - txManager, - housekeeping.NewManager(cfg.Prometheus, txManager), - &gitalypb.ObjectPool{ - Repository: &gitalypb.Repository{ - StorageName: repo.GetStorageName(), - RelativePath: gittest.NewObjectPoolName(t), - }, - }, - repo, - ) - require.NoError(t, err) - poolPath := gittest.RepositoryPath(t, pool) - + _, poolPath := gittest.CreateObjectPool(t, ctx, cfg, repoProto, gittest.CreateObjectPoolConfig{ + LinkRepositoryToObjectPool: true, + }) commitID := gittest.WriteCommit(t, cfg, poolPath, gittest.WithBranch(t.Name())) - require.NoError(t, pool.Link(ctx, repo)) - - rpcRequest := &gitalypb.InfoRefsRequest{Repository: repoProto} - - response, err := makeInfoRefsReceivePackRequest(t, ctx, cfg.SocketPath, cfg.Auth.Token, rpcRequest) + response, err := makeInfoRefsReceivePackRequest(t, ctx, cfg.SocketPath, cfg.Auth.Token, &gitalypb.InfoRefsRequest{ + Repository: repoProto, + }) require.NoError(t, err) require.NotContains(t, string(response), commitID+" .have") } diff --git a/internal/gitaly/service/smarthttp/testhelper_test.go b/internal/gitaly/service/smarthttp/testhelper_test.go index 55d4429b5..57e206e09 100644 --- a/internal/gitaly/service/smarthttp/testhelper_test.go +++ b/internal/gitaly/service/smarthttp/testhelper_test.go @@ -11,6 +11,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" hookservice "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/hook" + "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/objectpool" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/repository" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testserver" @@ -46,6 +47,13 @@ func startSmartHTTPServerWithOptions(t *testing.T, cfg config.Cfg, opts []Server deps.GetGit2goExecutor(), deps.GetHousekeepingManager(), )) + gitalypb.RegisterObjectPoolServiceServer(srv, objectpool.NewServer( + deps.GetLocator(), + deps.GetGitCmdFactory(), + deps.GetCatfileCache(), + deps.GetTxManager(), + deps.GetHousekeepingManager(), + )) gitalypb.RegisterHookServiceServer(srv, hookservice.NewServer(deps.GetHookManager(), deps.GetGitCmdFactory(), deps.GetPackObjectsCache(), deps.GetPackObjectsConcurrencyTracker(), deps.GetPackObjectsLimiter())) }, serverOpts...) } diff --git a/internal/gitaly/service/ssh/receive_pack_test.go b/internal/gitaly/service/ssh/receive_pack_test.go index 56b836e4a..8966fa76f 100644 --- a/internal/gitaly/service/ssh/receive_pack_test.go +++ b/internal/gitaly/service/ssh/receive_pack_test.go @@ -12,12 +12,9 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v15/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v15/internal/git" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/objectpool" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/gitlab" @@ -421,33 +418,15 @@ func TestReceivePack_hidesObjectPoolReferences(t *testing.T) { testcfg.BuildGitalyHooks(t, cfg) repoProto, _ := gittest.CreateRepository(t, ctx, cfg) - repo := localrepo.NewTestRepo(t, cfg, repoProto) - txManager := transaction.NewManager(cfg, backchannel.NewRegistry()) client := newSSHClient(t, cfg.SocketPath) stream, err := client.SSHReceivePack(ctx) require.NoError(t, err) - pool, err := objectpool.Create( - ctx, - config.NewLocator(cfg), - gittest.NewCommandFactory(t, cfg), - nil, - txManager, - housekeeping.NewManager(cfg.Prometheus, txManager), - &gitalypb.ObjectPool{ - Repository: &gitalypb.Repository{ - StorageName: repo.GetStorageName(), - RelativePath: gittest.NewObjectPoolName(t), - }, - }, - repo, - ) - require.NoError(t, err) - require.NoError(t, pool.Link(ctx, repo)) - poolPath := gittest.RepositoryPath(t, pool) - + _, poolPath := gittest.CreateObjectPool(t, ctx, cfg, repoProto, gittest.CreateObjectPoolConfig{ + LinkRepositoryToObjectPool: true, + }) commitID := gittest.WriteCommit(t, cfg, poolPath, gittest.WithBranch(t.Name())) // First request diff --git a/internal/gitaly/service/ssh/testhelper_test.go b/internal/gitaly/service/ssh/testhelper_test.go index 4c910edc3..dded96cba 100644 --- a/internal/gitaly/service/ssh/testhelper_test.go +++ b/internal/gitaly/service/ssh/testhelper_test.go @@ -7,6 +7,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" hookservice "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/hook" + "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/objectpool" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/repository" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testserver" @@ -51,6 +52,13 @@ func startSSHServerWithOptions(t *testing.T, cfg config.Cfg, opts []ServerOpt, s deps.GetGit2goExecutor(), deps.GetHousekeepingManager(), )) + gitalypb.RegisterObjectPoolServiceServer(srv, objectpool.NewServer( + deps.GetLocator(), + deps.GetGitCmdFactory(), + deps.GetCatfileCache(), + deps.GetTxManager(), + deps.GetHousekeepingManager(), + )) }, serverOpts...) } diff --git a/internal/praefect/replicator_test.go b/internal/praefect/replicator_test.go index 2659eb0a6..80d891037 100644 --- a/internal/praefect/replicator_test.go +++ b/internal/praefect/replicator_test.go @@ -17,15 +17,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gitalyauth "gitlab.com/gitlab-org/gitaly/v15/auth" - "gitlab.com/gitlab-org/gitaly/v15/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/objectpool" - gconfig "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/storage" - "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/helper" "gitlab.com/gitlab-org/gitaly/v15/internal/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v15/internal/praefect/config" @@ -57,7 +51,6 @@ func TestReplMgr_ProcessBacklog(t *testing.T) { Seed: gittest.SeedGitLabTest, }) - testRepo := localrepo.NewTestRepo(t, primaryCfg, testRepoProto) primaryCfg.SocketPath = testserver.RunGitalyServer(t, primaryCfg, setup.RegisterAll, testserver.WithDisablePraefect()) testcfg.BuildGitalySSH(t, primaryCfg) testcfg.BuildGitalyHooks(t, primaryCfg) @@ -83,41 +76,22 @@ func TestReplMgr_ProcessBacklog(t *testing.T) { }}, } - txManager := transaction.NewManager(primaryCfg, backchannel.NewRegistry()) - // create object pool on the source - poolCtx := testhelper.Context(t) - objectPoolPath := gittest.NewObjectPoolName(t) - pool, err := objectpool.Create( - poolCtx, - gconfig.NewLocator(primaryCfg), - gittest.NewCommandFactory(t, primaryCfg), - nil, - txManager, - housekeeping.NewManager(primaryCfg.Prometheus, txManager), - &gitalypb.ObjectPool{ - Repository: &gitalypb.Repository{ - StorageName: testRepoProto.GetStorageName(), - RelativePath: objectPoolPath, - }, - }, - testRepo, - ) - require.NoError(t, err) - require.NoError(t, pool.Link(poolCtx, testRepo)) + poolRepository, _ := gittest.CreateObjectPool(t, ctx, primaryCfg, testRepoProto, gittest.CreateObjectPoolConfig{ + LinkRepositoryToObjectPool: true, + }) // replicate object pool repository to target node - poolRepository := pool.ToProto().GetRepository() - targetObjectPoolRepo := proto.Clone(poolRepository).(*gitalypb.Repository) - targetObjectPoolRepo.StorageName = backupCfg.Storages[0].Name + targetObjectPoolRepo := proto.Clone(poolRepository).(*gitalypb.ObjectPool) + targetObjectPoolRepo.Repository.StorageName = backupCfg.Storages[0].Name ctx, cancel := context.WithCancel(ctx) injectedCtx := metadata.NewOutgoingContext(ctx, testcfg.GitalyServersMetadataFromCfg(t, primaryCfg)) repoClient := newRepositoryClient(t, backupCfg.SocketPath, backupCfg.Auth.Token) - _, err = repoClient.ReplicateRepository(injectedCtx, &gitalypb.ReplicateRepositoryRequest{ - Repository: targetObjectPoolRepo, - Source: poolRepository, + _, err := repoClient.ReplicateRepository(injectedCtx, &gitalypb.ReplicateRepositoryRequest{ + Repository: targetObjectPoolRepo.Repository, + Source: poolRepository.Repository, }) require.NoError(t, err) @@ -142,7 +116,7 @@ func TestReplMgr_ProcessBacklog(t *testing.T) { Change: datastore.UpdateRepo, TargetNodeStorage: secondary.GetStorage(), SourceNodeStorage: shard.Primary.GetStorage(), - RelativePath: testRepo.GetRelativePath(), + RelativePath: testRepoProto.GetRelativePath(), }, State: datastore.JobStateReady, Attempt: 3, @@ -171,7 +145,7 @@ func TestReplMgr_ProcessBacklog(t *testing.T) { db := testdb.New(t) rs := datastore.NewPostgresRepositoryStore(db, conf.StorageNames()) - require.NoError(t, rs.CreateRepository(ctx, repositoryID, conf.VirtualStorages[0].Name, testRepo.GetRelativePath(), testRepo.GetRelativePath(), shard.Primary.GetStorage(), nil, nil, true, false)) + require.NoError(t, rs.CreateRepository(ctx, repositoryID, conf.VirtualStorages[0].Name, testRepoProto.GetRelativePath(), testRepoProto.GetRelativePath(), shard.Primary.GetStorage(), nil, nil, true, false)) replMgr := NewReplMgr( loggerEntry, @@ -213,7 +187,7 @@ func TestReplMgr_ProcessBacklog(t *testing.T) { []interface{}{logEntries[3].Message, logEntries[3].Data["virtual_storage"], logEntries[3].Data["new_state"], logEntries[3].Data[logWithCorrID]}, ) - replicatedPath := filepath.Join(backupCfg.Storages[0].Path, testRepo.GetRelativePath()) + replicatedPath := filepath.Join(backupCfg.Storages[0].Path, testRepoProto.GetRelativePath()) gittest.Exec(t, backupCfg, "-C", replicatedPath, "cat-file", "-e", commitID.String()) gittest.Exec(t, backupCfg, "-C", replicatedPath, "gc") diff --git a/proto/go/gitalypb/repository.pb.go b/proto/go/gitalypb/repository.pb.go index d12d5ee06..144c52b9d 100644 --- a/proto/go/gitalypb/repository.pb.go +++ b/proto/go/gitalypb/repository.pb.go @@ -1829,15 +1829,19 @@ func (x *FindMergeBaseResponse) GetBase() string { return "" } -// This comment is left unintentionally blank. +// CreateForkRequest is a request for the CreateFork RPC. type CreateForkRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // This comment is left unintentionally blank. + // Repository is the repository that shall be created. Repository *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"` - // This comment is left unintentionally blank. + // SourceRepository is the repository that shall be forked. + // + // Note that the source repository is intentionally not marked as an additional repository that is + // to be rewritten by Praefect. This is because Gitaly will use the source repository to perform a + // gRPC call, which must use the original non-rewritten repository. SourceRepository *Repository `protobuf:"bytes,2,opt,name=source_repository,json=sourceRepository,proto3" json:"source_repository,omitempty"` } @@ -1887,7 +1891,7 @@ func (x *CreateForkRequest) GetSourceRepository() *Repository { return nil } -// This comment is left unintentionally blank. +// CreateForkResponse is a response for the CreateFork RPC. type CreateForkResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/proto/go/gitalypb/repository_grpc.pb.go b/proto/go/gitalypb/repository_grpc.pb.go index 73351ab2c..82c7497ba 100644 --- a/proto/go/gitalypb/repository_grpc.pb.go +++ b/proto/go/gitalypb/repository_grpc.pb.go @@ -53,7 +53,13 @@ type RepositoryServiceClient interface { WriteRef(ctx context.Context, in *WriteRefRequest, opts ...grpc.CallOption) (*WriteRefResponse, error) // This comment is left unintentionally blank. FindMergeBase(ctx context.Context, in *FindMergeBaseRequest, opts ...grpc.CallOption) (*FindMergeBaseResponse, error) - // This comment is left unintentionally blank. + // CreateFork creates a new repository from a specific source repository. This new repository will + // have the same branches and tags as the source repository. Internal references will not be + // recreated in the forked repository. + // + // All objects of the source repository will be duplicated, that is there are no space savings by + // creating the repository like this. The newly created repository does not join the object pool + // of the source repository, if there is any. CreateFork(ctx context.Context, in *CreateForkRequest, opts ...grpc.CallOption) (*CreateForkResponse, error) // This comment is left unintentionally blank. CreateRepositoryFromURL(ctx context.Context, in *CreateRepositoryFromURLRequest, opts ...grpc.CallOption) (*CreateRepositoryFromURLResponse, error) @@ -917,7 +923,13 @@ type RepositoryServiceServer interface { WriteRef(context.Context, *WriteRefRequest) (*WriteRefResponse, error) // This comment is left unintentionally blank. FindMergeBase(context.Context, *FindMergeBaseRequest) (*FindMergeBaseResponse, error) - // This comment is left unintentionally blank. + // CreateFork creates a new repository from a specific source repository. This new repository will + // have the same branches and tags as the source repository. Internal references will not be + // recreated in the forked repository. + // + // All objects of the source repository will be duplicated, that is there are no space savings by + // creating the repository like this. The newly created repository does not join the object pool + // of the source repository, if there is any. CreateFork(context.Context, *CreateForkRequest) (*CreateForkResponse, error) // This comment is left unintentionally blank. CreateRepositoryFromURL(context.Context, *CreateRepositoryFromURLRequest) (*CreateRepositoryFromURLResponse, error) diff --git a/proto/repository.proto b/proto/repository.proto index da8027c81..324619d8a 100644 --- a/proto/repository.proto +++ b/proto/repository.proto @@ -106,7 +106,13 @@ service RepositoryService { }; } - // This comment is left unintentionally blank. + // CreateFork creates a new repository from a specific source repository. This new repository will + // have the same branches and tags as the source repository. Internal references will not be + // recreated in the forked repository. + // + // All objects of the source repository will be duplicated, that is there are no space savings by + // creating the repository like this. The newly created repository does not join the object pool + // of the source repository, if there is any. rpc CreateFork(CreateForkRequest) returns (CreateForkResponse) { option (op_type) = { op: MUTATOR @@ -653,15 +659,19 @@ message FindMergeBaseResponse { string base = 1; } -// This comment is left unintentionally blank. +// CreateForkRequest is a request for the CreateFork RPC. message CreateForkRequest { - // This comment is left unintentionally blank. + // Repository is the repository that shall be created. Repository repository = 1 [(target_repository)=true]; - // This comment is left unintentionally blank. + // SourceRepository is the repository that shall be forked. + // + // Note that the source repository is intentionally not marked as an additional repository that is + // to be rewritten by Praefect. This is because Gitaly will use the source repository to perform a + // gRPC call, which must use the original non-rewritten repository. Repository source_repository = 2; } -// This comment is left unintentionally blank. +// CreateForkResponse is a response for the CreateFork RPC. message CreateForkResponse { } |