diff options
author | Patrick Steinhardt <psteinhardt@gitlab.com> | 2022-02-04 13:06:58 +0300 |
---|---|---|
committer | Patrick Steinhardt <psteinhardt@gitlab.com> | 2022-02-07 10:57:15 +0300 |
commit | 6b0b535a4468f5b7bda746534da1f66764f54f28 (patch) | |
tree | e18568583dabe9f9532f79ecbfe2659752a9bf74 | |
parent | 0bc1545fb94d3cf16f14ff657a6083bd635bac8d (diff) |
objectpool: Get repository paths via localrepo interface
Convert the objectpool package to retrieve repository paths via the
localrepo interface.
21 files changed, 156 insertions, 108 deletions
diff --git a/internal/git/objectpool/clone.go b/internal/git/objectpool/clone.go index cdd3e38a2..780afb546 100644 --- a/internal/git/objectpool/clone.go +++ b/internal/git/objectpool/clone.go @@ -6,13 +6,13 @@ import ( "path/filepath" "gitlab.com/gitlab-org/gitaly/v14/internal/git" - "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" ) // clone a repository to a pool, without setting the alternates, is not the // resposibility of this function. -func (o *ObjectPool) clone(ctx context.Context, repo *gitalypb.Repository) error { - repoPath, err := o.locator.GetRepoPath(repo) +func (o *ObjectPool) clone(ctx context.Context, repo *localrepo.Repo) error { + repoPath, err := repo.Path() if err != nil { return err } diff --git a/internal/git/objectpool/clone_test.go b/internal/git/objectpool/clone_test.go index a80580eb7..7b6b22d7a 100644 --- a/internal/git/objectpool/clone_test.go +++ b/internal/git/objectpool/clone_test.go @@ -5,15 +5,17 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" ) func TestClone(t *testing.T) { ctx := testhelper.Context(t) - _, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) - require.NoError(t, pool.clone(ctx, testRepo)) + require.NoError(t, pool.clone(ctx, repo)) defer func() { require.NoError(t, pool.Remove(ctx)) }() @@ -25,13 +27,14 @@ func TestClone(t *testing.T) { func TestCloneExistingPool(t *testing.T) { ctx := testhelper.Context(t) - _, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) - require.NoError(t, pool.clone(ctx, testRepo)) + require.NoError(t, pool.clone(ctx, repo)) defer func() { require.NoError(t, pool.Remove(ctx)) }() // re-clone on the directory - require.Error(t, pool.clone(ctx, testRepo)) + require.Error(t, pool.clone(ctx, repo)) } diff --git a/internal/git/objectpool/fetch.go b/internal/git/objectpool/fetch.go index cbab98fc0..bf03e9496 100644 --- a/internal/git/objectpool/fetch.go +++ b/internal/git/objectpool/fetch.go @@ -16,21 +16,21 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/command" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/housekeeping" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v14/internal/git/updateref" "gitlab.com/gitlab-org/gitaly/v14/internal/helper" - "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) const sourceRefNamespace = "refs/remotes/origin" // FetchFromOrigin initializes the pool and fetches the objects from its origin repository -func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *gitalypb.Repository) error { +func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *localrepo.Repo) error { if err := o.Init(ctx); err != nil { return err } - originPath, err := o.locator.GetPath(origin) + originPath, err := origin.Path() if err != nil { return err } diff --git a/internal/git/objectpool/fetch_test.go b/internal/git/objectpool/fetch_test.go index 79e7d3887..860cf1195 100644 --- a/internal/git/objectpool/fetch_test.go +++ b/internal/git/objectpool/fetch_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v14/internal/helper/text" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" @@ -18,9 +19,10 @@ import ( func TestFetchFromOriginDangling(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) - require.NoError(t, pool.FetchFromOrigin(ctx, testRepo), "seed pool") + require.NoError(t, pool.FetchFromOrigin(ctx, repo), "seed pool") const ( existingTree = "07f8147e8e73aab6c935c296e8cdc5194dee729b" @@ -72,7 +74,7 @@ func TestFetchFromOriginDangling(t *testing.T) { // We expect this second run to convert the dangling objects into // non-dangling objects. - require.NoError(t, pool.FetchFromOrigin(ctx, testRepo), "second fetch") + require.NoError(t, pool.FetchFromOrigin(ctx, repo), "second fetch") refsAfter := gittest.Exec(t, cfg, "-C", pool.FullPath(), "for-each-ref", "--format=%(refname) %(objectname)") refsAfterLines := strings.Split(string(refsAfter), "\n") @@ -84,8 +86,9 @@ func TestFetchFromOriginDangling(t *testing.T) { func TestFetchFromOriginFsck(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, repo := setupObjectPool(t, ctx) - repoPath := filepath.Join(cfg.Storages[0].Path, repo.RelativePath) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) + repoPath := filepath.Join(cfg.Storages[0].Path, repo.GetRelativePath()) require.NoError(t, pool.FetchFromOrigin(ctx, repo), "seed pool") @@ -107,20 +110,21 @@ func TestFetchFromOriginFsck(t *testing.T) { func TestFetchFromOriginDeltaIslands(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, testRepo := setupObjectPool(t, ctx) - testRepoPath := filepath.Join(cfg.Storages[0].Path, testRepo.RelativePath) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) + repoPath := filepath.Join(cfg.Storages[0].Path, repo.GetRelativePath()) - require.NoError(t, pool.FetchFromOrigin(ctx, testRepo), "seed pool") - require.NoError(t, pool.Link(ctx, testRepo)) + require.NoError(t, pool.FetchFromOrigin(ctx, repo), "seed pool") + require.NoError(t, pool.Link(ctx, repo)) - gittest.TestDeltaIslands(t, cfg, testRepoPath, func() error { + gittest.TestDeltaIslands(t, cfg, repoPath, func() error { // This should create a new packfile with good delta chains in the pool - if err := pool.FetchFromOrigin(ctx, testRepo); err != nil { + if err := pool.FetchFromOrigin(ctx, repo); err != nil { return err } // Make sure the old packfile, with bad delta chains, is deleted from the source repo - gittest.Exec(t, cfg, "-C", testRepoPath, "repack", "-ald") + gittest.Exec(t, cfg, "-C", repoPath, "repack", "-ald") return nil }) @@ -129,9 +133,10 @@ func TestFetchFromOriginDeltaIslands(t *testing.T) { func TestFetchFromOriginBitmapHashCache(t *testing.T) { ctx := testhelper.Context(t) - _, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) - require.NoError(t, pool.FetchFromOrigin(ctx, testRepo), "seed pool") + require.NoError(t, pool.FetchFromOrigin(ctx, repo), "seed pool") packDir := filepath.Join(pool.FullPath(), "objects/pack") packEntries, err := os.ReadDir(packDir) @@ -153,12 +158,13 @@ func TestFetchFromOriginBitmapHashCache(t *testing.T) { func TestFetchFromOriginRefUpdates(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, testRepo := setupObjectPool(t, ctx) - testRepoPath := filepath.Join(cfg.Storages[0].Path, testRepo.RelativePath) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) + repoPath := filepath.Join(cfg.Storages[0].Path, repo.GetRelativePath()) poolPath := pool.FullPath() - require.NoError(t, pool.FetchFromOrigin(ctx, testRepo), "seed pool") + require.NoError(t, pool.FetchFromOrigin(ctx, repo), "seed pool") oldRefs := map[string]string{ "heads/csv": "3dd08961455abf80ef9115f4afdc1c6f968b503c", @@ -166,7 +172,7 @@ func TestFetchFromOriginRefUpdates(t *testing.T) { } for ref, oid := range oldRefs { - require.Equal(t, oid, resolveRef(t, cfg, testRepoPath, "refs/"+ref), "look up %q in source", ref) + require.Equal(t, oid, resolveRef(t, cfg, repoPath, "refs/"+ref), "look up %q in source", ref) require.Equal(t, oid, resolveRef(t, cfg, poolPath, "refs/remotes/origin/"+ref), "look up %q in pool", ref) } @@ -180,11 +186,11 @@ func TestFetchFromOriginRefUpdates(t *testing.T) { } for ref, oid := range newRefs { - gittest.Exec(t, cfg, "-C", testRepoPath, "update-ref", "refs/"+ref, oid) - require.Equal(t, oid, resolveRef(t, cfg, testRepoPath, "refs/"+ref), "look up %q in source after update", ref) + gittest.Exec(t, cfg, "-C", repoPath, "update-ref", "refs/"+ref, oid) + require.Equal(t, oid, resolveRef(t, cfg, repoPath, "refs/"+ref), "look up %q in source after update", ref) } - require.NoError(t, pool.FetchFromOrigin(ctx, testRepo), "update pool") + require.NoError(t, pool.FetchFromOrigin(ctx, repo), "update pool") for ref, oid := range newRefs { require.Equal(t, oid, resolveRef(t, cfg, poolPath, "refs/remotes/origin/"+ref), "look up %q in pool after update", ref) @@ -201,7 +207,9 @@ func TestFetchFromOrigin_refs(t *testing.T) { poolPath := pool.FullPath() // Init the source repo with a bunch of refs. - repo, repoPath := gittest.InitRepo(t, cfg, cfg.Storages[0]) + repoProto, repoPath := gittest.InitRepo(t, cfg, cfg.Storages[0]) + repo := localrepo.NewTestRepo(t, cfg, repoProto) + commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(), gittest.WithTreeEntries()) for _, ref := range []string{"refs/heads/master", "refs/environments/1", "refs/tags/lightweight-tag"} { gittest.Exec(t, cfg, "-C", repoPath, "update-ref", ref, commitID.String()) diff --git a/internal/git/objectpool/link.go b/internal/git/objectpool/link.go index 80a4dae60..3527aad33 100644 --- a/internal/git/objectpool/link.go +++ b/internal/git/objectpool/link.go @@ -8,17 +8,16 @@ import ( "path/filepath" "strings" - "gitlab.com/gitlab-org/gitaly/v14/internal/git/repository" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v14/internal/safe" - "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) // Link will link the given repository to the object pool. This is done by writing the object pool's // path relative to the repository into the repository's "alternates" file. This does not trigger // deduplication, which is the responsibility of the caller. -func (o *ObjectPool) Link(ctx context.Context, repo *gitalypb.Repository) (returnedErr error) { - altPath, err := o.locator.InfoAlternatesPath(repo) +func (o *ObjectPool) Link(ctx context.Context, repo *localrepo.Repo) (returnedErr error) { + altPath, err := repo.InfoAlternatesPath() if err != nil { return err } @@ -68,8 +67,8 @@ func (o *ObjectPool) Link(ctx context.Context, repo *gitalypb.Repository) (retur // but none of its members will. With removeMemberBitmaps we try to // change "eventually" to "immediately", so that users won't see the // warning. https://gitlab.com/gitlab-org/gitaly/issues/1728 -func (o *ObjectPool) removeMemberBitmaps(repo repository.GitRepo) error { - poolPath, err := o.locator.GetPath(o) +func (o *ObjectPool) removeMemberBitmaps(repo *localrepo.Repo) error { + poolPath, err := o.Repo.Path() if err != nil { return err } @@ -82,7 +81,7 @@ func (o *ObjectPool) removeMemberBitmaps(repo repository.GitRepo) error { return nil } - repoPath, err := o.locator.GetPath(repo) + repoPath, err := repo.Path() if err != nil { return err } @@ -121,8 +120,8 @@ func getBitmaps(repoPath string) ([]string, error) { return bitmaps, nil } -func (o *ObjectPool) getRelativeObjectPath(repo *gitalypb.Repository) (string, error) { - repoPath, err := o.locator.GetRepoPath(repo) +func (o *ObjectPool) getRelativeObjectPath(repo *localrepo.Repo) (string, error) { + repoPath, err := repo.Path() if err != nil { return "", err } @@ -136,8 +135,8 @@ func (o *ObjectPool) getRelativeObjectPath(repo *gitalypb.Repository) (string, e } // LinkedToRepository tests if a repository is linked to an object pool -func (o *ObjectPool) LinkedToRepository(repo *gitalypb.Repository) (bool, error) { - relPath, err := getAlternateObjectDir(o.locator, repo) +func (o *ObjectPool) LinkedToRepository(repo *localrepo.Repo) (bool, error) { + relPath, err := getAlternateObjectDir(repo) if err != nil { if err == ErrAlternateObjectDirNotExist { return false, nil diff --git a/internal/git/objectpool/link_test.go b/internal/git/objectpool/link_test.go index 2573a7f87..d959c8b18 100644 --- a/internal/git/objectpool/link_test.go +++ b/internal/git/objectpool/link_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/internal/transaction/txinfo" @@ -18,41 +19,43 @@ import ( func TestLink(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) require.NoError(t, pool.Remove(ctx), "make sure pool does not exist prior to creation") - require.NoError(t, pool.Create(ctx, testRepo), "create pool") + require.NoError(t, pool.Create(ctx, repo), "create pool") - altPath, err := pool.locator.InfoAlternatesPath(testRepo) + altPath, err := repo.InfoAlternatesPath() require.NoError(t, err) require.NoFileExists(t, altPath) - require.NoError(t, pool.Link(ctx, testRepo)) + require.NoError(t, pool.Link(ctx, repo)) require.FileExists(t, altPath, "alternates file must exist after Link") content := testhelper.MustReadFile(t, altPath) require.True(t, strings.HasPrefix(string(content), "../"), "expected %q to be relative path", content) - require.NoError(t, pool.Link(ctx, testRepo)) + require.NoError(t, pool.Link(ctx, repo)) newContent := testhelper.MustReadFile(t, altPath) require.Equal(t, content, newContent) - require.False(t, gittest.RemoteExists(t, cfg, pool.FullPath(), testRepo.GetGlRepository()), "pool remotes should not include %v", testRepo) + require.False(t, gittest.RemoteExists(t, cfg, pool.FullPath(), repoProto.GetGlRepository()), "pool remotes should not include %v", repo) } func TestLink_transactional(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - _, pool, poolMember := setupObjectPool(t, ctx) + cfg, pool, poolMemberProto := setupObjectPool(t, ctx) + poolMember := localrepo.NewTestRepo(t, cfg, poolMemberProto) require.NoError(t, pool.Create(ctx, poolMember)) txManager := transaction.NewTrackingManager() pool.txManager = txManager - alternatesPath, err := pool.locator.InfoAlternatesPath(poolMember) + alternatesPath, err := poolMember.InfoAlternatesPath() require.NoError(t, err) require.NoFileExists(t, alternatesPath) @@ -70,10 +73,11 @@ func TestLink_transactional(t *testing.T) { func TestLinkRemoveBitmap(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) require.NoError(t, pool.Init(ctx)) - testRepoPath := filepath.Join(cfg.Storages[0].Path, testRepo.RelativePath) + testRepoPath := filepath.Join(cfg.Storages[0].Path, repo.GetRelativePath()) poolPath := pool.FullPath() gittest.Exec(t, cfg, "-C", poolPath, "fetch", testRepoPath, "+refs/*:refs/*") @@ -86,7 +90,7 @@ func TestLinkRemoveBitmap(t *testing.T) { refsBefore := gittest.Exec(t, cfg, "-C", testRepoPath, "for-each-ref") - require.NoError(t, pool.Link(ctx, testRepo)) + require.NoError(t, pool.Link(ctx, repo)) require.Len(t, listBitmaps(t, pool.FullPath()), 1, "pool bitmaps after") require.Len(t, listBitmaps(t, testRepoPath), 0, "member bitmaps after") @@ -114,21 +118,22 @@ func listBitmaps(t *testing.T, repoPath string) []string { func TestLinkAbsoluteLinkExists(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) - testRepoPath := filepath.Join(cfg.Storages[0].Path, testRepo.RelativePath) + testRepoPath := filepath.Join(cfg.Storages[0].Path, repo.GetRelativePath()) require.NoError(t, pool.Remove(ctx), "make sure pool does not exist prior to creation") - require.NoError(t, pool.Create(ctx, testRepo), "create pool") + require.NoError(t, pool.Create(ctx, repo), "create pool") - altPath, err := pool.locator.InfoAlternatesPath(testRepo) + altPath, err := repo.InfoAlternatesPath() require.NoError(t, err) fullPath := filepath.Join(pool.FullPath(), "objects") require.NoError(t, os.WriteFile(altPath, []byte(fullPath), 0o644)) - require.NoError(t, pool.Link(ctx, testRepo), "we expect this call to change the absolute link to a relative link") + require.NoError(t, pool.Link(ctx, repo), "we expect this call to change the absolute link to a relative link") require.FileExists(t, altPath, "alternates file must exist after Link") @@ -138,5 +143,5 @@ func TestLinkAbsoluteLinkExists(t *testing.T) { testRepoObjectsPath := filepath.Join(testRepoPath, "objects") require.Equal(t, fullPath, filepath.Join(testRepoObjectsPath, string(content)), "the content of the alternates file should be the relative version of the absolute pat") - require.True(t, gittest.RemoteExists(t, cfg, pool.FullPath(), "origin"), "pool remotes should include %v", testRepo) + require.True(t, gittest.RemoteExists(t, cfg, pool.FullPath(), "origin"), "pool remotes should include %v", repo) } diff --git a/internal/git/objectpool/pool.go b/internal/git/objectpool/pool.go index 8f7eaeac2..b251824d9 100644 --- a/internal/git/objectpool/pool.go +++ b/internal/git/objectpool/pool.go @@ -17,7 +17,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) // poolDirRegexp is used to validate object pool directory structure and name. @@ -36,7 +35,6 @@ const ErrInvalidPoolDir errString = "invalid object pool directory" type ObjectPool struct { Repo *localrepo.Repo - locator storage.Locator gitCmdFactory git.CommandFactory txManager transaction.Manager storageName string @@ -67,7 +65,6 @@ func NewObjectPool( } pool := &ObjectPool{ - locator: locator, gitCmdFactory: gitCmdFactory, txManager: txManager, storageName: storageName, @@ -113,7 +110,7 @@ func (o *ObjectPool) IsValid() bool { // Create will create a pool for a repository and pull the required data to this // pool. `repo` that is passed also joins the repository. -func (o *ObjectPool) Create(ctx context.Context, repo *gitalypb.Repository) (err error) { +func (o *ObjectPool) Create(ctx context.Context, repo *localrepo.Repo) (err error) { if err := o.clone(ctx, repo); err != nil { return fmt.Errorf("clone: %v", err) } @@ -164,9 +161,9 @@ func FromRepo( gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, txManager transaction.Manager, - repo *gitalypb.Repository, + repo *localrepo.Repo, ) (*ObjectPool, error) { - dir, err := getAlternateObjectDir(locator, repo) + dir, err := getAlternateObjectDir(repo) if err != nil { return nil, err } @@ -175,7 +172,7 @@ func FromRepo( return nil, nil } - repoPath, err := locator.GetPath(repo) + repoPath, err := repo.Path() if err != nil { return nil, err } @@ -198,8 +195,8 @@ var ( // getAlternateObjectDir returns the entry in the objects/info/attributes file if it exists // it will only return the first line of the file if there are multiple lines. -func getAlternateObjectDir(locator storage.Locator, repo *gitalypb.Repository) (string, error) { - altPath, err := locator.InfoAlternatesPath(repo) +func getAlternateObjectDir(repo *localrepo.Repo) (string, error) { + altPath, err := repo.InfoAlternatesPath() if err != nil { return "", err } diff --git a/internal/git/objectpool/pool_test.go b/internal/git/objectpool/pool_test.go index 345465c43..d28e06fd9 100644 --- a/internal/git/objectpool/pool_test.go +++ b/internal/git/objectpool/pool_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testcfg" @@ -29,12 +30,14 @@ func TestNewObjectPool(t *testing.T) { func TestNewFromRepoSuccess(t *testing.T) { ctx := testhelper.Context(t) - _, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) + locator := config.NewLocator(cfg) - require.NoError(t, pool.Create(ctx, testRepo)) - require.NoError(t, pool.Link(ctx, testRepo)) + require.NoError(t, pool.Create(ctx, repo)) + require.NoError(t, pool.Link(ctx, repo)) - poolFromRepo, err := FromRepo(pool.locator, pool.gitCmdFactory, nil, nil, testRepo) + poolFromRepo, err := FromRepo(locator, pool.gitCmdFactory, nil, nil, repo) require.NoError(t, err) require.Equal(t, pool.relativePath, poolFromRepo.relativePath) require.Equal(t, pool.storageName, poolFromRepo.storageName) @@ -43,12 +46,13 @@ func TestNewFromRepoSuccess(t *testing.T) { func TestNewFromRepoNoObjectPool(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, testRepo := setupObjectPool(t, ctx) - - testRepoPath := filepath.Join(cfg.Storages[0].Path, testRepo.RelativePath) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) + locator := config.NewLocator(cfg) + testRepoPath := filepath.Join(cfg.Storages[0].Path, repo.GetRelativePath()) // no alternates file - poolFromRepo, err := FromRepo(pool.locator, pool.gitCmdFactory, nil, nil, testRepo) + poolFromRepo, err := FromRepo(locator, pool.gitCmdFactory, nil, nil, repo) require.Equal(t, ErrAlternateObjectDirNotExist, err) require.Nil(t, poolFromRepo) @@ -81,7 +85,7 @@ func TestNewFromRepoNoObjectPool(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { alternateFilePath := filepath.Join(testRepoPath, "objects", "info", "alternates") require.NoError(t, os.WriteFile(alternateFilePath, tc.fileContent, 0o644)) - poolFromRepo, err := FromRepo(pool.locator, pool.gitCmdFactory, nil, nil, testRepo) + poolFromRepo, err := FromRepo(locator, pool.gitCmdFactory, nil, nil, repo) require.Equal(t, tc.expectedErr, err) require.Nil(t, poolFromRepo) @@ -93,13 +97,14 @@ func TestNewFromRepoNoObjectPool(t *testing.T) { func TestCreate(t *testing.T) { ctx := testhelper.Context(t) - cfg, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) - testRepoPath := filepath.Join(cfg.Storages[0].Path, testRepo.RelativePath) + testRepoPath := filepath.Join(cfg.Storages[0].Path, repo.GetRelativePath()) masterSha := gittest.Exec(t, cfg, "-C", testRepoPath, "show-ref", "master") - err := pool.Create(ctx, testRepo) + err := pool.Create(ctx, repo) require.NoError(t, err) defer func() { require.NoError(t, pool.Remove(ctx)) @@ -126,24 +131,26 @@ func TestCreate(t *testing.T) { func TestCreateSubDirsExist(t *testing.T) { ctx := testhelper.Context(t) - _, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) - err := pool.Create(ctx, testRepo) + err := pool.Create(ctx, repo) require.NoError(t, err) require.NoError(t, pool.Remove(ctx)) // Recreate pool so the subdirs exist already - err = pool.Create(ctx, testRepo) + err = pool.Create(ctx, repo) require.NoError(t, err) } func TestRemove(t *testing.T) { ctx := testhelper.Context(t) - _, pool, testRepo := setupObjectPool(t, ctx) + cfg, pool, repoProto := setupObjectPool(t, ctx) + repo := localrepo.NewTestRepo(t, cfg, repoProto) - err := pool.Create(ctx, testRepo) + err := pool.Create(ctx, repo) require.NoError(t, err) require.True(t, pool.Exists()) diff --git a/internal/gitaly/service/objectpool/alternates_test.go b/internal/gitaly/service/objectpool/alternates_test.go index 28d249d9b..e8792e5ea 100644 --- a/internal/gitaly/service/objectpool/alternates_test.go +++ b/internal/gitaly/service/objectpool/alternates_test.go @@ -8,13 +8,15 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) func TestDisconnectGitAlternates(t *testing.T) { - cfg, repo, repoPath, locator, client := setup(t) + cfg, repoProto, repoPath, locator, client := setup(t) ctx := testhelper.Context(t) + repo := localrepo.NewTestRepo(t, cfg, repoProto) gitCmdFactory := gittest.NewCommandFactory(t, cfg) pool := initObjectPool(t, cfg, cfg.Storages[0]) @@ -40,7 +42,7 @@ func TestDisconnectGitAlternates(t *testing.T) { // At this point we know that the repository has access to // existingObjectID, but only if objects/info/alternates is in place. - _, err = client.DisconnectGitAlternates(ctx, &gitalypb.DisconnectGitAlternatesRequest{Repository: repo}) + _, err = client.DisconnectGitAlternates(ctx, &gitalypb.DisconnectGitAlternatesRequest{Repository: repoProto}) require.NoError(t, err, "call DisconnectGitAlternates") // Check that the object can still be found, even though diff --git a/internal/gitaly/service/objectpool/create.go b/internal/gitaly/service/objectpool/create.go index 3cdeee1cd..58b9e6480 100644 --- a/internal/gitaly/service/objectpool/create.go +++ b/internal/gitaly/service/objectpool/create.go @@ -31,11 +31,13 @@ func (s *server) CreateObjectPool(ctx context.Context, in *gitalypb.CreateObject return nil, err } + origin := s.localrepo(in.GetOrigin()) + if pool.Exists() { return nil, status.Errorf(codes.FailedPrecondition, "pool already exists at: %v", pool.GetRelativePath()) } - if err := pool.Create(ctx, in.GetOrigin()); err != nil { + if err := pool.Create(ctx, origin); err != nil { return nil, err } diff --git a/internal/gitaly/service/objectpool/create_test.go b/internal/gitaly/service/objectpool/create_test.go index 66a25ad6e..dd14ab48b 100644 --- a/internal/gitaly/service/objectpool/create_test.go +++ b/internal/gitaly/service/objectpool/create_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" @@ -136,8 +137,9 @@ func TestUnsuccessfulCreate(t *testing.T) { } func TestDelete(t *testing.T) { - cfg, repo, _, _, client := setup(t) + cfg, repoProto, _, _, client := setup(t) ctx := testhelper.Context(t) + repo := localrepo.NewTestRepo(t, cfg, repoProto) pool := initObjectPool(t, cfg, cfg.Storages[0]) validPoolPath := pool.GetRelativePath() diff --git a/internal/gitaly/service/objectpool/fetch_into_object_pool.go b/internal/gitaly/service/objectpool/fetch_into_object_pool.go index ef8a7437f..763dc4680 100644 --- a/internal/gitaly/service/objectpool/fetch_into_object_pool.go +++ b/internal/gitaly/service/objectpool/fetch_into_object_pool.go @@ -21,7 +21,9 @@ func (s *server) FetchIntoObjectPool(ctx context.Context, req *gitalypb.FetchInt return nil, helper.ErrInvalidArgument(fmt.Errorf("object pool invalid: %v", err)) } - if err := objectPool.FetchFromOrigin(ctx, req.GetOrigin()); err != nil { + origin := s.localrepo(req.GetOrigin()) + + if err := objectPool.FetchFromOrigin(ctx, origin); err != nil { return nil, helper.ErrInternal(err) } diff --git a/internal/gitaly/service/objectpool/get.go b/internal/gitaly/service/objectpool/get.go index 7fbad62aa..3202838de 100644 --- a/internal/gitaly/service/objectpool/get.go +++ b/internal/gitaly/service/objectpool/get.go @@ -15,7 +15,9 @@ func (s *server) GetObjectPool(ctx context.Context, in *gitalypb.GetObjectPoolRe return nil, helper.ErrInternal(errors.New("repository is empty")) } - objectPool, err := objectpool.FromRepo(s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, in.GetRepository()) + repo := s.localrepo(in.GetRepository()) + + objectPool, err := objectpool.FromRepo(s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, repo) if err != nil { ctxlogrus.Extract(ctx). WithError(err). diff --git a/internal/gitaly/service/objectpool/get_test.go b/internal/gitaly/service/objectpool/get_test.go index 554c3984f..69bc446c3 100644 --- a/internal/gitaly/service/objectpool/get_test.go +++ b/internal/gitaly/service/objectpool/get_test.go @@ -6,12 +6,14 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) func TestGetObjectPoolSuccess(t *testing.T) { - cfg, repo, _, _, client := setup(t) + cfg, repoProto, _, _, client := setup(t) + repo := localrepo.NewTestRepo(t, cfg, repoProto) pool := initObjectPool(t, cfg, cfg.Storages[0]) relativePoolPath := pool.GetRelativePath() @@ -26,7 +28,7 @@ func TestGetObjectPoolSuccess(t *testing.T) { }() resp, err := client.GetObjectPool(ctx, &gitalypb.GetObjectPoolRequest{ - Repository: repo, + Repository: repoProto, }) require.NoError(t, err) diff --git a/internal/gitaly/service/objectpool/link.go b/internal/gitaly/service/objectpool/link.go index 691cd8764..3e378d40b 100644 --- a/internal/gitaly/service/objectpool/link.go +++ b/internal/gitaly/service/objectpool/link.go @@ -19,11 +19,13 @@ func (s *server) LinkRepositoryToObjectPool(ctx context.Context, req *gitalypb.L return nil, err } + repo := s.localrepo(req.GetRepository()) + if err := pool.Init(ctx); err != nil { return nil, helper.ErrInternal(err) } - if err := pool.Link(ctx, req.GetRepository()); err != nil { + if err := pool.Link(ctx, repo); err != nil { return nil, helper.ErrInternal(helper.SanitizeError(err)) } diff --git a/internal/gitaly/service/objectpool/link_test.go b/internal/gitaly/service/objectpool/link_test.go index 79d1a0b75..cfe46dc1c 100644 --- a/internal/gitaly/service/objectpool/link_test.go +++ b/internal/gitaly/service/objectpool/link_test.go @@ -26,7 +26,7 @@ func TestLink(t *testing.T) { pool := initObjectPool(t, cfg, cfg.Storages[0]) require.NoError(t, pool.Remove(ctx), "make sure pool does not exist at start of test") - require.NoError(t, pool.Create(ctx, repo), "create pool") + require.NoError(t, pool.Create(ctx, localRepo), "create pool") // Mock object in the pool, which should be available to the pool members // after linking @@ -84,14 +84,15 @@ func TestLink(t *testing.T) { } func TestLinkIdempotent(t *testing.T) { - cfg, repo, _, _, client := setup(t) + cfg, repoProto, _, _, client := setup(t) ctx := testhelper.Context(t) + repo := localrepo.NewTestRepo(t, cfg, repoProto) pool := initObjectPool(t, cfg, cfg.Storages[0]) require.NoError(t, pool.Create(ctx, repo)) request := &gitalypb.LinkRepositoryToObjectPoolRequest{ - Repository: repo, + Repository: repoProto, ObjectPool: pool.ToProto(), } @@ -103,8 +104,9 @@ func TestLinkIdempotent(t *testing.T) { } func TestLinkNoClobber(t *testing.T) { - cfg, repo, repoPath, _, client := setup(t) + cfg, repoProto, repoPath, _, client := setup(t) ctx := testhelper.Context(t) + repo := localrepo.NewTestRepo(t, cfg, repoProto) pool := initObjectPool(t, cfg, cfg.Storages[0]) require.NoError(t, pool.Create(ctx, repo)) @@ -116,7 +118,7 @@ func TestLinkNoClobber(t *testing.T) { require.NoError(t, os.WriteFile(alternatesFile, []byte(contentBefore), 0o644)) request := &gitalypb.LinkRepositoryToObjectPoolRequest{ - Repository: repo, + Repository: repoProto, ObjectPool: pool.ToProto(), } diff --git a/internal/gitaly/service/objectpool/reduplicate_test.go b/internal/gitaly/service/objectpool/reduplicate_test.go index f87a95d15..4d5ba683f 100644 --- a/internal/gitaly/service/objectpool/reduplicate_test.go +++ b/internal/gitaly/service/objectpool/reduplicate_test.go @@ -7,13 +7,15 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) func TestReduplicate(t *testing.T) { - cfg, repo, repoPath, locator, client := setup(t) + cfg, repoProto, repoPath, locator, client := setup(t) ctx := testhelper.Context(t) + repo := localrepo.NewTestRepo(t, cfg, repoProto) gitCmdFactory := gittest.NewCommandFactory(t, cfg) pool := initObjectPool(t, cfg, cfg.Storages[0]) @@ -36,7 +38,7 @@ func TestReduplicate(t *testing.T) { // Reduplicate and check if the objects appear again require.NoError(t, pool.Link(ctx, repo)) - _, err = client.ReduplicateRepository(ctx, &gitalypb.ReduplicateRepositoryRequest{Repository: repo}) + _, err = client.ReduplicateRepository(ctx, &gitalypb.ReduplicateRepositoryRequest{Repository: repoProto}) require.NoError(t, err) require.NoError(t, os.RemoveAll(altPath)) diff --git a/internal/gitaly/service/objectpool/server.go b/internal/gitaly/service/objectpool/server.go index bdb7091cd..f942ae209 100644 --- a/internal/gitaly/service/objectpool/server.go +++ b/internal/gitaly/service/objectpool/server.go @@ -3,6 +3,8 @@ package objectpool import ( "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/catfile" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" @@ -30,3 +32,7 @@ func NewServer( txManager: txManager, } } + +func (s *server) localrepo(repo repository.GitRepo) *localrepo.Repo { + return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) +} diff --git a/internal/gitaly/service/smarthttp/inforefs_test.go b/internal/gitaly/service/smarthttp/inforefs_test.go index cfd436323..7037f0b34 100644 --- a/internal/gitaly/service/smarthttp/inforefs_test.go +++ b/internal/gitaly/service/smarthttp/inforefs_test.go @@ -16,6 +16,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/cache" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/git/objectpool" "gitlab.com/gitlab-org/gitaly/v14/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config" @@ -215,9 +216,10 @@ func TestObjectPoolRefAdvertisementHiding(t *testing.T) { cfg.SocketPath = runSmartHTTPServer(t, cfg) ctx := testhelper.Context(t) - repo, _ := gittest.CreateRepository(ctx, t, cfg, gittest.CreateRepositoryConfig{ + repoProto, _ := gittest.CreateRepository(ctx, t, cfg, gittest.CreateRepositoryConfig{ Seed: gittest.SeedGitLabTest, }) + repo := localrepo.NewTestRepo(t, cfg, repoProto) pool, err := objectpool.NewObjectPool( config.NewLocator(cfg), @@ -238,7 +240,7 @@ func TestObjectPoolRefAdvertisementHiding(t *testing.T) { require.NoError(t, pool.Link(ctx, repo)) - rpcRequest := &gitalypb.InfoRefsRequest{Repository: repo} + rpcRequest := &gitalypb.InfoRefsRequest{Repository: repoProto} response, err := makeInfoRefsReceivePackRequest(ctx, t, cfg.SocketPath, cfg.Auth.Token, rpcRequest) require.NoError(t, err) diff --git a/internal/gitaly/service/ssh/receive_pack_test.go b/internal/gitaly/service/ssh/receive_pack_test.go index 69a999036..2f64beded 100644 --- a/internal/gitaly/service/ssh/receive_pack_test.go +++ b/internal/gitaly/service/ssh/receive_pack_test.go @@ -246,9 +246,10 @@ func TestObjectPoolRefAdvertisementHidingSSH(t *testing.T) { cfg.SocketPath = runSSHServer(t, cfg) ctx := testhelper.Context(t) - repo, _ := gittest.CreateRepository(testhelper.Context(t), t, cfg, gittest.CreateRepositoryConfig{ + repoProto, _ := gittest.CreateRepository(testhelper.Context(t), t, cfg, gittest.CreateRepositoryConfig{ Seed: gittest.SeedGitLabTest, }) + repo := localrepo.NewTestRepo(t, cfg, repoProto) client, conn := newSSHClient(t, cfg.SocketPath) defer conn.Close() diff --git a/internal/praefect/replicator_test.go b/internal/praefect/replicator_test.go index 05f694fdb..4a409615d 100644 --- a/internal/praefect/replicator_test.go +++ b/internal/praefect/replicator_test.go @@ -17,6 +17,7 @@ import ( gitalyauth "gitlab.com/gitlab-org/gitaly/v14/auth" "gitlab.com/gitlab-org/gitaly/v14/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/git/objectpool" gconfig "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/service" @@ -49,7 +50,8 @@ func TestMain(m *testing.M) { func TestReplMgr_ProcessBacklog(t *testing.T) { t.Parallel() - primaryCfg, testRepo, testRepoPath := testcfg.BuildWithRepo(t, testcfg.WithStorages("primary")) + primaryCfg, testRepoProto, testRepoPath := testcfg.BuildWithRepo(t, testcfg.WithStorages("primary")) + testRepo := localrepo.NewTestRepo(t, primaryCfg, testRepoProto) primaryCfg.SocketPath = testserver.RunGitalyServer(t, primaryCfg, nil, setup.RegisterAll, testserver.WithDisablePraefect()) testcfg.BuildGitalySSH(t, primaryCfg) testcfg.BuildGitalyHooks(t, primaryCfg) @@ -82,7 +84,7 @@ func TestReplMgr_ProcessBacklog(t *testing.T) { gittest.NewCommandFactory(t, primaryCfg), nil, transaction.NewManager(primaryCfg, backchannel.NewRegistry()), - testRepo.GetStorageName(), + testRepoProto.GetStorageName(), objectPoolPath, ) require.NoError(t, err) |