diff options
author | Pavlo Strokov <pstrokov@gitlab.com> | 2022-02-23 12:57:08 +0300 |
---|---|---|
committer | Pavlo Strokov <pstrokov@gitlab.com> | 2022-02-23 12:57:08 +0300 |
commit | 59d695918d269f35a487c00c7b870aee124b9eaa (patch) | |
tree | d1cd4239a86947fcc0065f2c702e70dc2fd691a3 | |
parent | c9d5807defa8f2ea93c1282e552748f076aa392d (diff) | |
parent | 27f942d96c96662b477ef6abf11c9b2003cea796 (diff) |
Merge branch 'ps-cleanup-queue-on-repo-removal' into 'master'
Error is returned in case repository doesn't exist or not provided
See merge request gitlab-org/gitaly!4280
19 files changed, 478 insertions, 224 deletions
diff --git a/internal/gitaly/service/ref/pack_refs.go b/internal/gitaly/service/ref/pack_refs.go index 3d09b8739..0b9ae588c 100644 --- a/internal/gitaly/service/ref/pack_refs.go +++ b/internal/gitaly/service/ref/pack_refs.go @@ -2,9 +2,8 @@ package ref import ( "context" - "errors" - "fmt" + gitalyerrors "gitlab.com/gitlab-org/gitaly/v14/internal/errors" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v14/internal/helper" @@ -25,7 +24,7 @@ func (s *server) PackRefs(ctx context.Context, in *gitalypb.PackRefsRequest) (*g func validatePackRefsRequest(in *gitalypb.PackRefsRequest) error { if in.GetRepository() == nil { - return errors.New("empty repository") + return gitalyerrors.ErrEmptyRepository } return nil } @@ -36,7 +35,7 @@ func (s *server) packRefs(ctx context.Context, repository repository.GitRepo) er Flags: []git.Option{git.Flag{Name: "--all"}}, }) if err != nil { - return fmt.Errorf("initializing pack-refs command: %v", err) + return err } return cmd.Wait() diff --git a/internal/gitaly/service/ref/pack_refs_test.go b/internal/gitaly/service/ref/pack_refs_test.go index 8a7141d42..f16aa8175 100644 --- a/internal/gitaly/service/ref/pack_refs_test.go +++ b/internal/gitaly/service/ref/pack_refs_test.go @@ -15,6 +15,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestPackRefsSuccessfulRequest(t *testing.T) { @@ -62,3 +64,46 @@ func linesInPackfile(t *testing.T, repoPath string) int { } return refs } + +func TestPackRefs_invalidRequest(t *testing.T) { + t.Parallel() + + cfg, client := setupRefServiceWithoutRepo(t) + + tests := []struct { + repo *gitalypb.Repository + err error + desc string + }{ + { + desc: "nil repo", + repo: nil, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect("empty Repository", "repo scoped: empty Repository")), + }, + { + desc: "invalid storage name", + repo: &gitalypb.Repository{StorageName: "foo"}, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: "foo"`, "repo scoped: invalid Repository")), + }, + { + desc: "non-existing repo", + repo: &gitalypb.Repository{StorageName: cfg.Storages[0].Name, RelativePath: "bar"}, + err: status.Error( + codes.NotFound, + gitalyOrPraefect( + fmt.Sprintf(`GetRepoPath: not a git repository: "%s/bar"`, cfg.Storages[0].Path), + `mutator call: route repository mutator: get repository id: repository "default"/"bar" not found`, + ), + ), + }, + } + + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + ctx := testhelper.Context(t) + //nolint:staticcheck + _, err := client.PackRefs(ctx, &gitalypb.PackRefsRequest{Repository: tc.repo}) + testhelper.RequireGrpcError(t, err, tc.err) + }) + } +} diff --git a/internal/gitaly/service/ref/testhelper_test.go b/internal/gitaly/service/ref/testhelper_test.go index 4231b9497..b21d5a61d 100644 --- a/internal/gitaly/service/ref/testhelper_test.go +++ b/internal/gitaly/service/ref/testhelper_test.go @@ -133,3 +133,10 @@ func assertContainsBranch(t *testing.T, branches []*gitalypb.FindAllBranchesResp t.Errorf("Expected to find branch %q in branches %s", branch.Name, branchNames) } + +func gitalyOrPraefect(gitalyMsg, praefectMsg string) string { + if testhelper.IsPraefectEnabled() { + return praefectMsg + } + return gitalyMsg +} diff --git a/internal/gitaly/service/repository/cleanup.go b/internal/gitaly/service/repository/cleanup.go index 83d8514d7..2953e4350 100644 --- a/internal/gitaly/service/repository/cleanup.go +++ b/internal/gitaly/service/repository/cleanup.go @@ -3,11 +3,17 @@ package repository import ( "context" + gitalyerrors "gitlab.com/gitlab-org/gitaly/v14/internal/errors" "gitlab.com/gitlab-org/gitaly/v14/internal/git/housekeeping" + "gitlab.com/gitlab-org/gitaly/v14/internal/helper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) func (s *server) Cleanup(ctx context.Context, in *gitalypb.CleanupRequest) (*gitalypb.CleanupResponse, error) { + if in.GetRepository() == nil { + return nil, helper.ErrInvalidArgument(gitalyerrors.ErrEmptyRepository) + } + repo := s.localrepo(in.GetRepository()) if err := housekeeping.CleanupWorktrees(ctx, repo); err != nil { diff --git a/internal/gitaly/service/repository/cleanup_test.go b/internal/gitaly/service/repository/cleanup_test.go index a9cdaf34c..1802fee12 100644 --- a/internal/gitaly/service/repository/cleanup_test.go +++ b/internal/gitaly/service/repository/cleanup_test.go @@ -1,6 +1,7 @@ package repository import ( + "fmt" "os" "path/filepath" "testing" @@ -11,6 +12,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -145,3 +148,43 @@ func TestCleanupDisconnectedWorktrees(t *testing.T) { // to checkout another worktree at the same path gittest.AddWorktree(t, cfg, repoPath, worktreePath) } + +func TestCleanup_invalidRequest(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + cfg, client := setupRepositoryServiceWithoutRepo(t) + + for _, tc := range []struct { + desc string + in *gitalypb.Repository + err error + }{ + { + desc: "no repository provided", + err: status.Error(codes.InvalidArgument, gitalyOrPraefect("empty Repository", "repo scoped: empty Repository")), + }, + { + desc: "storage doesn't exist", + in: &gitalypb.Repository{StorageName: "stub"}, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: "stub"`, "repo scoped: invalid Repository")), + }, + { + desc: "relative path doesn't exist", + in: &gitalypb.Repository{StorageName: cfg.Storages[0].Name, RelativePath: "so/me/some.git"}, + err: status.Error( + codes.NotFound, + gitalyOrPraefect( + fmt.Sprintf(`GetRepoPath: not a git repository: %q`, filepath.Join(cfg.Storages[0].Path, "so/me/some.git")), + `mutator call: route repository mutator: get repository id: repository "default"/"so/me/some.git" not found`, + ), + ), + }, + } { + t.Run(tc.desc, func(t *testing.T) { + //nolint:staticcheck + _, err := client.Cleanup(ctx, &gitalypb.CleanupRequest{Repository: tc.in}) + testhelper.RequireGrpcError(t, err, tc.err) + }) + } +} diff --git a/internal/gitaly/service/repository/commit_graph.go b/internal/gitaly/service/repository/commit_graph.go index b329bc099..c5db82700 100644 --- a/internal/gitaly/service/repository/commit_graph.go +++ b/internal/gitaly/service/repository/commit_graph.go @@ -3,6 +3,7 @@ package repository import ( "context" + gitalyerrors "gitlab.com/gitlab-org/gitaly/v14/internal/errors" "gitlab.com/gitlab-org/gitaly/v14/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v14/internal/helper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" @@ -13,6 +14,10 @@ func (s *server) WriteCommitGraph( ctx context.Context, in *gitalypb.WriteCommitGraphRequest, ) (*gitalypb.WriteCommitGraphResponse, error) { + if in.GetRepository() == nil { + return nil, helper.ErrInvalidArgument(gitalyerrors.ErrEmptyRepository) + } + repo := s.localrepo(in.GetRepository()) if in.GetSplitStrategy() != gitalypb.WriteCommitGraphRequest_SizeMultiple { diff --git a/internal/gitaly/service/repository/commit_graph_test.go b/internal/gitaly/service/repository/commit_graph_test.go index 7f1bc7a9d..c55c64665 100644 --- a/internal/gitaly/service/repository/commit_graph_test.go +++ b/internal/gitaly/service/repository/commit_graph_test.go @@ -119,7 +119,7 @@ func TestWriteCommitGraph_validationChecks(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - _, repo, _, client := setupRepositoryService(ctx, t, testserver.WithDisablePraefect()) + cfg, repo, _, client := setupRepositoryService(ctx, t, testserver.WithDisablePraefect()) for _, tc := range []struct { desc string @@ -137,13 +137,18 @@ func TestWriteCommitGraph_validationChecks(t *testing.T) { { desc: "no repository", req: &gitalypb.WriteCommitGraphRequest{}, - expErr: status.Error(codes.InvalidArgument, `GetStorageByName: no such storage: ""`), + expErr: status.Error(codes.InvalidArgument, "empty Repository"), }, { desc: "invalid storage", req: &gitalypb.WriteCommitGraphRequest{Repository: &gitalypb.Repository{StorageName: "invalid"}}, expErr: status.Error(codes.InvalidArgument, `GetStorageByName: no such storage: "invalid"`), }, + { + desc: "not existing repository", + req: &gitalypb.WriteCommitGraphRequest{Repository: &gitalypb.Repository{StorageName: repo.StorageName, RelativePath: "invalid"}}, + expErr: status.Error(codes.NotFound, fmt.Sprintf(`GetRepoPath: not a git repository: "%s/invalid"`, cfg.Storages[0].Path)), + }, } { t.Run(tc.desc, func(t *testing.T) { //nolint:staticcheck diff --git a/internal/gitaly/service/repository/gc.go b/internal/gitaly/service/repository/gc.go index b92842d06..6e9d3533f 100644 --- a/internal/gitaly/service/repository/gc.go +++ b/internal/gitaly/service/repository/gc.go @@ -11,6 +11,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" log "github.com/sirupsen/logrus" + gitalyerrors "gitlab.com/gitlab-org/gitaly/v14/internal/errors" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v14/internal/git/housekeeping" @@ -27,6 +28,10 @@ func (s *server) GarbageCollect(ctx context.Context, in *gitalypb.GarbageCollect "WriteBitmaps": in.GetCreateBitmap(), }).Debug("GarbageCollect") + if in.GetRepository() == nil { + return nil, helper.ErrInvalidArgument(gitalyerrors.ErrEmptyRepository) + } + repo := s.localrepo(in.GetRepository()) if err := housekeeping.CleanupWorktrees(ctx, repo); err != nil { diff --git a/internal/gitaly/service/repository/gc_test.go b/internal/gitaly/service/repository/gc_test.go index d72be668b..d3c42e2c5 100644 --- a/internal/gitaly/service/repository/gc_test.go +++ b/internal/gitaly/service/repository/gc_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "time" @@ -18,6 +19,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -356,23 +358,38 @@ func TestGarbageCollectFailure(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - _, repo, _, client := setupRepositoryService(ctx, t) + _, repo, repoPath, client := setupRepositoryService(ctx, t) + storagePath := strings.TrimSuffix(repoPath, "/"+repo.RelativePath) tests := []struct { repo *gitalypb.Repository - code codes.Code + err error }{ - {repo: nil, code: codes.InvalidArgument}, - {repo: &gitalypb.Repository{StorageName: "foo"}, code: codes.InvalidArgument}, - {repo: &gitalypb.Repository{RelativePath: "bar"}, code: codes.InvalidArgument}, - {repo: &gitalypb.Repository{StorageName: repo.GetStorageName(), RelativePath: "bar"}, code: codes.NotFound}, + { + repo: nil, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect("empty Repository", "repo scoped: empty Repository")), + }, + { + repo: &gitalypb.Repository{StorageName: "foo"}, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: "foo"`, "repo scoped: invalid Repository")), + }, + { + repo: &gitalypb.Repository{StorageName: repo.StorageName, RelativePath: "bar"}, + err: status.Error( + codes.NotFound, + gitalyOrPraefect( + fmt.Sprintf(`GetRepoPath: not a git repository: "%s/bar"`, storagePath), + `mutator call: route repository mutator: get repository id: repository "default"/"bar" not found`, + ), + ), + }, } - for _, test := range tests { - t.Run(fmt.Sprintf("%v", test.repo), func(t *testing.T) { + for _, tc := range tests { + t.Run(fmt.Sprintf("%v", tc.repo), func(t *testing.T) { //nolint:staticcheck - _, err := client.GarbageCollect(ctx, &gitalypb.GarbageCollectRequest{Repository: test.repo}) - testhelper.RequireGrpcCode(t, err, test.code) + _, err := client.GarbageCollect(ctx, &gitalypb.GarbageCollectRequest{Repository: tc.repo}) + testhelper.RequireGrpcError(t, err, tc.err) }) } } diff --git a/internal/gitaly/service/repository/midx.go b/internal/gitaly/service/repository/midx.go index 2dc429b70..0540e2ae1 100644 --- a/internal/gitaly/service/repository/midx.go +++ b/internal/gitaly/service/repository/midx.go @@ -9,12 +9,14 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" log "github.com/sirupsen/logrus" + gitalyerrors "gitlab.com/gitlab-org/gitaly/v14/internal/errors" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v14/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v14/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v14/internal/helper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" + "google.golang.org/grpc/status" ) const ( @@ -24,9 +26,16 @@ const ( func (s *server) MidxRepack(ctx context.Context, in *gitalypb.MidxRepackRequest) (*gitalypb.MidxRepackResponse, error) { repoProto := in.GetRepository() + if repoProto == nil { + return nil, helper.ErrInvalidArgument(gitalyerrors.ErrEmptyRepository) + } + repo := s.localrepo(repoProto) if err := repo.SetConfig(ctx, "core.multiPackIndex", "true", s.txManager); err != nil { + if _, ok := status.FromError(err); ok { + return nil, err + } return nil, helper.ErrInternalf("setting config: %w", err) } diff --git a/internal/gitaly/service/repository/midx_test.go b/internal/gitaly/service/repository/midx_test.go index 5abd2fc2f..984f4ee44 100644 --- a/internal/gitaly/service/repository/midx_test.go +++ b/internal/gitaly/service/repository/midx_test.go @@ -21,7 +21,9 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v14/internal/transaction/txinfo" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" + "google.golang.org/grpc/codes" "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" ) func TestMidxWrite(t *testing.T) { @@ -270,3 +272,37 @@ func addPackFiles( } } } + +func TestMidxRepack_validationChecks(t *testing.T) { + t.Parallel() + cfg, client := setupRepositoryServiceWithoutRepo(t, testserver.WithDisablePraefect()) + ctx := testhelper.Context(t) + + for _, tc := range []struct { + desc string + req *gitalypb.MidxRepackRequest + expErr error + }{ + { + desc: "no repository", + req: &gitalypb.MidxRepackRequest{}, + expErr: status.Error(codes.InvalidArgument, "empty Repository"), + }, + { + desc: "invalid storage", + req: &gitalypb.MidxRepackRequest{Repository: &gitalypb.Repository{StorageName: "invalid"}}, + expErr: status.Error(codes.InvalidArgument, `GetStorageByName: no such storage: "invalid"`), + }, + { + desc: "not existing repository", + req: &gitalypb.MidxRepackRequest{Repository: &gitalypb.Repository{StorageName: cfg.Storages[0].Name, RelativePath: "invalid"}}, + expErr: status.Error(codes.NotFound, fmt.Sprintf(`GetRepoPath: not a git repository: "%s/invalid"`, cfg.Storages[0].Path)), + }, + } { + t.Run(tc.desc, func(t *testing.T) { + //nolint:staticcheck + _, err := client.MidxRepack(ctx, tc.req) + testhelper.RequireGrpcError(t, tc.expErr, err) + }) + } +} diff --git a/internal/gitaly/service/repository/optimize.go b/internal/gitaly/service/repository/optimize.go index bbf1ee406..8ea1555ab 100644 --- a/internal/gitaly/service/repository/optimize.go +++ b/internal/gitaly/service/repository/optimize.go @@ -3,6 +3,7 @@ package repository import ( "context" + gitalyerrors "gitlab.com/gitlab-org/gitaly/v14/internal/errors" "gitlab.com/gitlab-org/gitaly/v14/internal/helper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) @@ -23,7 +24,7 @@ func (s *server) OptimizeRepository(ctx context.Context, in *gitalypb.OptimizeRe func (s *server) validateOptimizeRepositoryRequest(in *gitalypb.OptimizeRepositoryRequest) error { if in.GetRepository() == nil { - return helper.ErrInvalidArgumentf("empty repository") + return helper.ErrInvalidArgument(gitalyerrors.ErrEmptyRepository) } _, err := s.locator.GetRepoPath(in.GetRepository()) diff --git a/internal/gitaly/service/repository/optimize_test.go b/internal/gitaly/service/repository/optimize_test.go index c35ffa924..0bc1932d5 100644 --- a/internal/gitaly/service/repository/optimize_test.go +++ b/internal/gitaly/service/repository/optimize_test.go @@ -2,6 +2,7 @@ package repository import ( "bytes" + "fmt" "os" "path/filepath" "testing" @@ -14,6 +15,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func getNewestPackfileModtime(t *testing.T, repoPath string) time.Time { @@ -151,27 +153,33 @@ func TestOptimizeRepositoryValidation(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - _, repo, _, client := setupRepositoryService(ctx, t) + cfg, repo, _, client := setupRepositoryService(ctx, t) testCases := []struct { - desc string - repo *gitalypb.Repository - expectedErrorCode codes.Code + desc string + repo *gitalypb.Repository + exp error }{ { - desc: "empty repository", - repo: nil, - expectedErrorCode: codes.InvalidArgument, + desc: "empty repository", + repo: nil, + exp: status.Error(codes.InvalidArgument, gitalyOrPraefect("empty Repository", "repo scoped: empty Repository")), }, { - desc: "invalid repository storage", - repo: &gitalypb.Repository{StorageName: "non-existent", RelativePath: repo.GetRelativePath()}, - expectedErrorCode: codes.InvalidArgument, + desc: "invalid repository storage", + repo: &gitalypb.Repository{StorageName: "non-existent", RelativePath: repo.GetRelativePath()}, + exp: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: "non-existent"`, "repo scoped: invalid Repository")), }, { - desc: "invalid repository path", - repo: &gitalypb.Repository{StorageName: repo.GetStorageName(), RelativePath: "/path/not/exist"}, - expectedErrorCode: codes.NotFound, + desc: "invalid repository path", + repo: &gitalypb.Repository{StorageName: repo.GetStorageName(), RelativePath: "path/not/exist"}, + exp: status.Error( + codes.NotFound, + gitalyOrPraefect( + fmt.Sprintf(`GetRepoPath: not a git repository: "%s/path/not/exist"`, cfg.Storages[0].Path), + `mutator call: route repository mutator: get repository id: repository "default"/"path/not/exist" not found`, + ), + ), }, } @@ -179,7 +187,7 @@ func TestOptimizeRepositoryValidation(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { _, err := client.OptimizeRepository(ctx, &gitalypb.OptimizeRepositoryRequest{Repository: tc.repo}) require.Error(t, err) - testhelper.RequireGrpcCode(t, err, tc.expectedErrorCode) + testhelper.RequireGrpcError(t, err, tc.exp) }) } diff --git a/internal/gitaly/service/repository/rename.go b/internal/gitaly/service/repository/rename.go index b161b4a9e..9f619e4e2 100644 --- a/internal/gitaly/service/repository/rename.go +++ b/internal/gitaly/service/repository/rename.go @@ -8,6 +8,7 @@ import ( "path/filepath" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" + gitalyerrors "gitlab.com/gitlab-org/gitaly/v14/internal/errors" "gitlab.com/gitlab-org/gitaly/v14/internal/helper" "gitlab.com/gitlab-org/gitaly/v14/internal/safe" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" @@ -97,7 +98,7 @@ func (s *server) renameRepository(ctx context.Context, sourceRepo, targetRepo *g func validateRenameRepositoryRequest(in *gitalypb.RenameRepositoryRequest) error { if in.GetRepository() == nil { - return errors.New("from repository is empty") + return gitalyerrors.ErrEmptyRepository } if in.GetRelativePath() == "" { diff --git a/internal/gitaly/service/repository/rename_test.go b/internal/gitaly/service/repository/rename_test.go index c3528cf1d..16ab49e43 100644 --- a/internal/gitaly/service/repository/rename_test.go +++ b/internal/gitaly/service/repository/rename_test.go @@ -1,8 +1,10 @@ package repository import ( + "fmt" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" @@ -13,6 +15,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestRenameRepository_success(t *testing.T) { @@ -73,30 +76,45 @@ func TestRenameRepository_invalidRequest(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - _, repo, _, client := setupRepositoryService(ctx, t) + _, repo, repoPath, client := setupRepositoryService(ctx, t) + storagePath := strings.TrimSuffix(repoPath, "/"+repo.RelativePath) testCases := []struct { desc string req *gitalypb.RenameRepositoryRequest + exp error }{ { desc: "empty repository", req: &gitalypb.RenameRepositoryRequest{Repository: nil, RelativePath: "/tmp/abc"}, + exp: status.Error(codes.InvalidArgument, gitalyOrPraefect("empty Repository", "repo scoped: empty Repository")), }, { desc: "empty destination relative path", req: &gitalypb.RenameRepositoryRequest{Repository: repo, RelativePath: ""}, + exp: status.Error(codes.InvalidArgument, "destination relative path is empty"), }, { desc: "destination relative path contains path traversal", req: &gitalypb.RenameRepositoryRequest{Repository: repo, RelativePath: "../usr/bin"}, + exp: status.Error(codes.InvalidArgument, "GetRepoPath: relative path escapes root directory"), + }, + { + desc: "repository storage doesn't exist", + req: &gitalypb.RenameRepositoryRequest{Repository: &gitalypb.Repository{StorageName: "stub", RelativePath: repo.RelativePath}, RelativePath: "../usr/bin"}, + exp: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: "stub"`, "repo scoped: invalid Repository")), + }, + { + desc: "repository relative path doesn't exist", + req: &gitalypb.RenameRepositoryRequest{Repository: &gitalypb.Repository{StorageName: repo.StorageName, RelativePath: "stub"}, RelativePath: "../usr/bin"}, + exp: status.Error(codes.NotFound, fmt.Sprintf(`GetRepoPath: not a git repository: "%s/stub"`, storagePath)), }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { _, err := client.RenameRepository(ctx, tc.req) - testhelper.RequireGrpcCode(t, err, codes.InvalidArgument) + testhelper.RequireGrpcError(t, err, tc.exp) }) } } diff --git a/internal/gitaly/service/repository/repack.go b/internal/gitaly/service/repository/repack.go index 416e59d6c..18415d4ba 100644 --- a/internal/gitaly/service/repository/repack.go +++ b/internal/gitaly/service/repository/repack.go @@ -6,6 +6,7 @@ import ( "math" "github.com/prometheus/client_golang/prometheus" + gitalyerrors "gitlab.com/gitlab-org/gitaly/v14/internal/errors" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v14/internal/helper" @@ -33,6 +34,10 @@ func log2Threads(numCPUs int) git.ValueFlag { } func (s *server) RepackFull(ctx context.Context, in *gitalypb.RepackFullRequest) (*gitalypb.RepackFullResponse, error) { + if in.GetRepository() == nil { + return nil, helper.ErrInvalidArgument(gitalyerrors.ErrEmptyRepository) + } + repo := s.localrepo(in.GetRepository()) cfg := housekeeping.RepackObjectsConfig{ FullRepack: true, @@ -49,6 +54,10 @@ func (s *server) RepackFull(ctx context.Context, in *gitalypb.RepackFullRequest) } func (s *server) RepackIncremental(ctx context.Context, in *gitalypb.RepackIncrementalRequest) (*gitalypb.RepackIncrementalResponse, error) { + if in.GetRepository() == nil { + return nil, helper.ErrInvalidArgumentf("empty repository") + } + repo := s.localrepo(in.GetRepository()) cfg := housekeeping.RepackObjectsConfig{ FullRepack: false, diff --git a/internal/gitaly/service/repository/repack_test.go b/internal/gitaly/service/repository/repack_test.go index 2dd6810d3..7e73290a6 100644 --- a/internal/gitaly/service/repository/repack_test.go +++ b/internal/gitaly/service/repository/repack_test.go @@ -1,6 +1,7 @@ package repository import ( + "fmt" "path/filepath" "testing" "time" @@ -15,6 +16,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestRepackIncrementalSuccess(t *testing.T) { @@ -101,21 +103,43 @@ func TestRepackIncrementalFailure(t *testing.T) { tests := []struct { repo *gitalypb.Repository - code codes.Code + err error desc string }{ - {desc: "nil repo", repo: nil, code: codes.InvalidArgument}, - {desc: "invalid storage name", repo: &gitalypb.Repository{StorageName: "foo"}, code: codes.InvalidArgument}, - {desc: "no storage name", repo: &gitalypb.Repository{RelativePath: "bar"}, code: codes.InvalidArgument}, - {desc: "non-existing repo", repo: &gitalypb.Repository{StorageName: cfg.Storages[0].Name, RelativePath: "bar"}, code: codes.NotFound}, + { + desc: "nil repo", + repo: nil, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect("empty repository", "repo scoped: empty Repository")), + }, + { + desc: "invalid storage name", + repo: &gitalypb.Repository{StorageName: "foo"}, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: "foo"`, "repo scoped: invalid Repository")), + }, + { + desc: "no storage name", + repo: &gitalypb.Repository{RelativePath: "bar"}, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: ""`, "repo scoped: invalid Repository")), + }, + { + desc: "non-existing repo", + repo: &gitalypb.Repository{StorageName: cfg.Storages[0].Name, RelativePath: "bar"}, + err: status.Error( + codes.NotFound, + gitalyOrPraefect( + fmt.Sprintf(`GetRepoPath: not a git repository: "%s/bar"`, cfg.Storages[0].Path), + `mutator call: route repository mutator: get repository id: repository "default"/"bar" not found`, + ), + ), + }, } - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { ctx := testhelper.Context(t) //nolint:staticcheck - _, err := client.RepackIncremental(ctx, &gitalypb.RepackIncrementalRequest{Repository: test.repo}) - testhelper.RequireGrpcCode(t, err, test.code) + _, err := client.RepackIncremental(ctx, &gitalypb.RepackIncrementalRequest{Repository: tc.repo}) + testhelper.RequireGrpcError(t, err, tc.err) }) } } @@ -222,22 +246,44 @@ func TestRepackFullFailure(t *testing.T) { cfg, client := setupRepositoryServiceWithoutRepo(t) tests := []struct { - repo *gitalypb.Repository - code codes.Code desc string + repo *gitalypb.Repository + err error }{ - {desc: "nil repo", repo: nil, code: codes.InvalidArgument}, - {desc: "invalid storage name", repo: &gitalypb.Repository{StorageName: "foo"}, code: codes.InvalidArgument}, - {desc: "no storage name", repo: &gitalypb.Repository{RelativePath: "bar"}, code: codes.InvalidArgument}, - {desc: "non-existing repo", repo: &gitalypb.Repository{StorageName: cfg.Storages[0].Name, RelativePath: "bar"}, code: codes.NotFound}, + { + desc: "nil repo", + repo: nil, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect("empty Repository", "repo scoped: empty Repository")), + }, + { + desc: "invalid storage name", + repo: &gitalypb.Repository{StorageName: "foo"}, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: "foo"`, "repo scoped: invalid Repository")), + }, + { + desc: "no storage name", + repo: &gitalypb.Repository{RelativePath: "bar"}, + err: status.Error(codes.InvalidArgument, gitalyOrPraefect(`GetStorageByName: no such storage: ""`, "repo scoped: invalid Repository")), + }, + { + desc: "non-existing repo", + repo: &gitalypb.Repository{StorageName: cfg.Storages[0].Name, RelativePath: "bar"}, + err: status.Error( + codes.NotFound, + gitalyOrPraefect( + fmt.Sprintf(`GetRepoPath: not a git repository: "%s/bar"`, cfg.Storages[0].Path), + `mutator call: route repository mutator: get repository id: repository "default"/"bar" not found`, + ), + ), + }, } - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { ctx := testhelper.Context(t) //nolint:staticcheck - _, err := client.RepackFull(ctx, &gitalypb.RepackFullRequest{Repository: test.repo}) - testhelper.RequireGrpcCode(t, err, test.code) + _, err := client.RepackFull(ctx, &gitalypb.RepackFullRequest{Repository: tc.repo}) + testhelper.RequireGrpcError(t, err, tc.err) }) } } diff --git a/internal/gitaly/service/repository/testhelper_test.go b/internal/gitaly/service/repository/testhelper_test.go index 767eca3eb..df74b6fb6 100644 --- a/internal/gitaly/service/repository/testhelper_test.go +++ b/internal/gitaly/service/repository/testhelper_test.go @@ -192,3 +192,10 @@ func setupRepositoryServiceWithWorktree(ctx context.Context, t testing.TB, opts return cfg, repo, repoPath, client } + +func gitalyOrPraefect(gitalyMsg, praefectMsg string) string { + if testhelper.IsPraefectEnabled() { + return praefectMsg + } + return gitalyMsg +} diff --git a/internal/praefect/datastore/repository_store_test.go b/internal/praefect/datastore/repository_store_test.go index 4c4544d86..e4dd9ac68 100644 --- a/internal/praefect/datastore/repository_store_test.go +++ b/internal/praefect/datastore/repository_store_test.go @@ -34,11 +34,6 @@ type replicaRecord struct { // It structured as virtual-storage->relative_path->storage->replicaRecord type storageState map[string]map[string]map[string]replicaRecord -type ( - requireStateFunc func(t *testing.T, ctx context.Context, vss virtualStorageState, ss storageState) - repositoryStoreFactory func(t *testing.T, storages map[string][]string) (RepositoryStore, requireStateFunc) -) - func requireState(t testing.TB, ctx context.Context, db glsql.Querier, vss virtualStorageState, ss storageState) { t.Helper() @@ -116,119 +111,13 @@ FROM storage_repositories func TestRepositoryStore_Postgres(t *testing.T) { db := testdb.New(t) - testRepositoryStore(t, func(t *testing.T, storages map[string][]string) (RepositoryStore, requireStateFunc) { + + newRepositoryStore := func(t *testing.T, storages map[string][]string) RepositoryStore { db.TruncateAll(t) gs := NewPostgresRepositoryStore(db, storages) - - return gs, func(t *testing.T, ctx context.Context, vss virtualStorageState, ss storageState) { - t.Helper() - requireState(t, ctx, db, vss, ss) - } - }) -} - -func TestRepositoryStore_incrementGenerationConcurrently(t *testing.T) { - db := testdb.New(t) - - type call struct { - primary string - secondaries []string + return gs } - for _, tc := range []struct { - desc string - first call - second call - error error - state storageState - }{ - { - desc: "both successful", - first: call{ - primary: "primary", - secondaries: []string{"secondary"}, - }, - second: call{ - primary: "primary", - secondaries: []string{"secondary"}, - }, - state: storageState{ - "virtual-storage": { - "relative-path": { - "primary": {repositoryID: 1, generation: 2}, - "secondary": {repositoryID: 1, generation: 2}, - }, - }, - }, - }, - { - desc: "second write targeted outdated and up to date nodes", - first: call{ - primary: "primary", - }, - second: call{ - primary: "primary", - secondaries: []string{"secondary"}, - }, - state: storageState{ - "virtual-storage": { - "relative-path": { - "primary": {repositoryID: 1, generation: 2}, - "secondary": {repositoryID: 1, generation: 0}, - }, - }, - }, - }, - { - desc: "second write targeted only outdated nodes", - first: call{ - primary: "primary", - }, - second: call{ - primary: "secondary", - }, - error: errWriteToOutdatedNodes, - state: storageState{ - "virtual-storage": { - "relative-path": { - "primary": {repositoryID: 1, generation: 1}, - "secondary": {repositoryID: 1, generation: 0}, - }, - }, - }, - }, - } { - t.Run(tc.desc, func(t *testing.T) { - ctx := testhelper.Context(t) - - db.TruncateAll(t) - - require.NoError(t, NewPostgresRepositoryStore(db, nil).CreateRepository(ctx, 1, "virtual-storage", "relative-path", "replica-path", "primary", []string{"secondary"}, nil, false, false)) - - firstTx := db.Begin(t) - secondTx := db.Begin(t) - - err := NewPostgresRepositoryStore(firstTx, nil).IncrementGeneration(ctx, 1, tc.first.primary, tc.first.secondaries) - require.NoError(t, err) - - go func() { - testdb.WaitForBlockedQuery(ctx, t, db, "WITH updated_replicas AS (") - firstTx.Commit(t) - }() - - err = NewPostgresRepositoryStore(secondTx, nil).IncrementGeneration(ctx, 1, tc.second.primary, tc.second.secondaries) - require.Equal(t, tc.error, err) - secondTx.Commit(t) - - requireState(t, ctx, db, - virtualStorageState{"virtual-storage": {"relative-path": {repositoryID: 1, replicaPath: "replica-path"}}}, - tc.state, - ) - }) - } -} - -func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { ctx := testhelper.Context(t) const ( @@ -239,20 +128,17 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("IncrementGeneration", func(t *testing.T) { t.Run("doesn't create new records", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.Equal(t, rs.IncrementGeneration(ctx, 1, "primary", []string{"secondary-1"}), commonerr.ErrRepositoryNotFound, ) - requireState(t, ctx, - virtualStorageState{}, - storageState{}, - ) + requireState(t, ctx, db, virtualStorageState{}, storageState{}) }) t.Run("write to outdated nodes", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "latest-node", []string{"outdated-primary", "outdated-secondary"}, nil, false, false)) require.NoError(t, rs.SetGeneration(ctx, 1, "latest-node", repo, 1)) @@ -261,7 +147,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { rs.IncrementGeneration(ctx, 1, "outdated-primary", []string{"outdated-secondary"}), errWriteToOutdatedNodes, ) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -280,7 +166,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("increments generation for up to date nodes", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) for id, pair := range []struct{ virtualStorage, relativePath string }{ {vs, repo}, @@ -294,7 +180,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { require.NoError(t, rs.IncrementGeneration(ctx, 1, "primary", []string{"up-to-date-secondary"})) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path-1"}, @@ -330,7 +216,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { require.NoError(t, rs.IncrementGeneration(ctx, 1, "primary", []string{ "up-to-date-secondary", "outdated-secondary", "non-existing-secondary", })) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path-1"}, @@ -367,11 +253,11 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("SetGeneration", func(t *testing.T) { t.Run("creates a record for the replica", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", stor, nil, nil, false, false)) require.NoError(t, rs.SetGeneration(ctx, 1, "storage-2", repo, 0)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{"virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, }}, @@ -387,10 +273,10 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("updates existing record with the replicated to relative path", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, "original-path", "replica-path", "storage-1", []string{"storage-2"}, nil, true, false)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "original-path": {repositoryID: 1, primary: "storage-1", replicaPath: "replica-path"}, @@ -407,7 +293,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { ) require.NoError(t, rs.RenameRepository(ctx, vs, "original-path", "storage-1", "new-path")) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "new-path": {repositoryID: 1, primary: "storage-1", replicaPath: "new-path"}, @@ -426,7 +312,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { ) require.NoError(t, rs.SetGeneration(ctx, 1, "storage-2", "new-path", 1)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "new-path": {repositoryID: 1, primary: "storage-1", replicaPath: "new-path"}, @@ -444,12 +330,12 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("updates existing record", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "storage-1", nil, nil, false, false)) require.NoError(t, rs.SetGeneration(ctx, 1, stor, repo, 1)) require.NoError(t, rs.SetGeneration(ctx, 1, stor, repo, 0)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -468,7 +354,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("SetAuthoritativeReplica", func(t *testing.T) { t.Run("fails when repository doesnt exist", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.Equal(t, commonerr.NewRepositoryNotFoundError(vs, repo), @@ -477,10 +363,10 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("sets an existing replica as the latest", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "storage-1", []string{"storage-2"}, nil, false, false)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -497,7 +383,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { ) require.NoError(t, rs.SetAuthoritativeReplica(ctx, vs, repo, "storage-1")) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -515,10 +401,10 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("sets a new replica as the latest", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "storage-1", nil, nil, false, false)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -534,7 +420,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { ) require.NoError(t, rs.SetAuthoritativeReplica(ctx, vs, repo, "storage-2")) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -553,7 +439,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("GetGeneration", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) generation, err := rs.GetGeneration(ctx, 1, stor) require.NoError(t, err) @@ -568,7 +454,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("GetReplicatedGeneration", func(t *testing.T) { t.Run("no previous record allowed", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) gen, err := rs.GetReplicatedGeneration(ctx, 1, "source", "target") require.NoError(t, err) @@ -581,7 +467,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("upgrade allowed", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "source", nil, nil, false, false)) require.NoError(t, rs.IncrementGeneration(ctx, 1, "source", nil)) @@ -597,7 +483,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("downgrade prevented", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "target", nil, nil, false, false)) require.NoError(t, rs.IncrementGeneration(ctx, 1, "target", nil)) @@ -678,7 +564,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }, } { t.Run(tc.desc, func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "primary", tc.updatedSecondaries, tc.outdatedSecondaries, tc.storePrimary, tc.storeAssignments)) @@ -694,7 +580,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { expectedStorageState[vs][repo][updatedSecondary] = replicaRecord{repositoryID: 1, generation: 0} } - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ vs: { repo: { @@ -712,7 +598,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("conflict due to virtual storage and relative path", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", stor, nil, nil, false, false)) require.Equal(t, @@ -722,7 +608,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("conflict due to repository id", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, "virtual-storage-1", "relative-path-1", "replica-path", "storage-1", nil, nil, false, false)) require.Equal(t, @@ -734,7 +620,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("DeleteRepository", func(t *testing.T) { t.Run("delete non-existing", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) replicaPath, storages, err := rs.DeleteRepository(ctx, vs, repo) require.Equal(t, commonerr.NewRepositoryNotFoundError(vs, repo), err) @@ -743,13 +629,13 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("delete existing", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, "virtual-storage-1", "repository-1", "replica-path-1", "storage-1", nil, nil, false, false)) require.NoError(t, rs.CreateRepository(ctx, 2, "virtual-storage-2", "repository-1", "replica-path-2", "storage-1", []string{"storage-2"}, nil, false, false)) require.NoError(t, rs.CreateRepository(ctx, 3, "virtual-storage-2", "repository-2", "replica-path-3", "storage-1", nil, nil, false, false)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path-1"}, @@ -782,7 +668,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { require.Equal(t, "replica-path-2", replicaPath) require.Equal(t, []string{"storage-1", "storage-2"}, storages) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path-1"}, @@ -808,7 +694,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("DeleteReplica", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) t.Run("delete non-existing", func(t *testing.T) { require.Equal(t, ErrNoRowsAffected, rs.DeleteReplica(ctx, 1, "storage-1")) @@ -819,7 +705,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { require.NoError(t, rs.CreateRepository(ctx, 2, "virtual-storage-1", "relative-path-2", "replica-path-2", "storage-1", nil, nil, false, false)) require.NoError(t, rs.CreateRepository(ctx, 3, "virtual-storage-2", "relative-path-1", "replica-path-3", "storage-1", nil, nil, false, false)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "relative-path-1": {repositoryID: 1, replicaPath: "replica-path-1"}, @@ -849,7 +735,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { require.NoError(t, rs.DeleteReplica(ctx, 1, "storage-1")) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "relative-path-1": {repositoryID: 1, replicaPath: "replica-path-1"}, @@ -880,7 +766,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("RenameRepository", func(t *testing.T) { t.Run("rename non-existing", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.Equal(t, RepositoryNotExistsError{vs, repo, stor}, @@ -889,12 +775,12 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("rename existing", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, "renamed-all", "replica-path-1", "storage-1", nil, nil, false, false)) require.NoError(t, rs.CreateRepository(ctx, 2, vs, "renamed-some", "replica-path-2", "storage-1", []string{"storage-2"}, nil, false, false)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "renamed-all": {repositoryID: 1, replicaPath: "replica-path-1"}, @@ -917,7 +803,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { require.NoError(t, rs.RenameRepository(ctx, vs, "renamed-all", "storage-1", "renamed-all-new")) require.NoError(t, rs.RenameRepository(ctx, vs, "renamed-some", "storage-1", "renamed-some-new")) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "renamed-all-new": {repositoryID: 1, replicaPath: "renamed-all-new"}, @@ -942,7 +828,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("GetConsistentStorages", func(t *testing.T) { - rs, requireState := newStore(t, map[string][]string{ + rs := newRepositoryStore(t, map[string][]string{ vs: {"primary", "consistent-secondary", "inconsistent-secondary", "no-record"}, }) @@ -961,7 +847,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "primary", []string{"consistent-secondary"}, nil, false, false)) require.NoError(t, rs.IncrementGeneration(ctx, 1, "primary", []string{"consistent-secondary"})) require.NoError(t, rs.SetGeneration(ctx, 1, "inconsistent-secondary", repo, 0)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -1007,7 +893,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("storage with highest generation is not configured", func(t *testing.T) { require.NoError(t, rs.SetGeneration(ctx, 1, "unknown", repo, 2)) require.NoError(t, rs.SetGeneration(ctx, 1, "primary", repo, 1)) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -1039,7 +925,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("returns not found for deleted repositories", func(t *testing.T) { _, _, err := rs.DeleteRepository(ctx, vs, repo) require.NoError(t, err) - requireState(t, ctx, virtualStorageState{}, storageState{}) + requireState(t, ctx, db, virtualStorageState{}, storageState{}) replicaPath, secondaries, err := rs.GetConsistentStorages(ctx, vs, repo) require.Equal(t, commonerr.NewRepositoryNotFoundError(vs, repo), err) @@ -1053,7 +939,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("replicas pending rename are considered outdated", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, "original-path", "replica-path", "storage-1", []string{"storage-2"}, nil, true, false)) replicaPath, storages, err := rs.GetConsistentStorages(ctx, vs, "original-path") @@ -1089,17 +975,17 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { t.Run("DeleteInvalidRepository", func(t *testing.T) { t.Run("only replica", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "invalid-storage", nil, nil, false, false)) require.NoError(t, rs.DeleteInvalidRepository(ctx, 1, "invalid-storage")) - requireState(t, ctx, virtualStorageState{}, storageState{}) + requireState(t, ctx, db, virtualStorageState{}, storageState{}) }) t.Run("another replica", func(t *testing.T) { - rs, requireState := newStore(t, nil) + rs := newRepositoryStore(t, nil) require.NoError(t, rs.CreateRepository(ctx, 1, vs, repo, "replica-path", "invalid-storage", []string{"other-storage"}, nil, false, false)) require.NoError(t, rs.DeleteInvalidRepository(ctx, 1, "invalid-storage")) - requireState(t, ctx, + requireState(t, ctx, db, virtualStorageState{ "virtual-storage-1": { "repository-1": {repositoryID: 1, replicaPath: "replica-path"}, @@ -1117,7 +1003,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("RepositoryExists", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) exists, err := rs.RepositoryExists(ctx, vs, repo) require.NoError(t, err) @@ -1136,7 +1022,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("ReserveRepositoryID", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) id, err := rs.ReserveRepositoryID(ctx, vs, repo) require.NoError(t, err) @@ -1158,7 +1044,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("GetRepositoryID", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) id, err := rs.GetRepositoryID(ctx, vs, repo) require.Equal(t, commonerr.NewRepositoryNotFoundError(vs, repo), err) @@ -1172,7 +1058,7 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) t.Run("GetReplicaPath", func(t *testing.T) { - rs, _ := newStore(t, nil) + rs := newRepositoryStore(t, nil) replicaPath, err := rs.GetReplicaPath(ctx, 1) require.Equal(t, err, commonerr.ErrRepositoryNotFound) @@ -1186,6 +1072,107 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) { }) } +func TestRepositoryStore_incrementGenerationConcurrently(t *testing.T) { + db := testdb.New(t) + + type call struct { + primary string + secondaries []string + } + + for _, tc := range []struct { + desc string + first call + second call + error error + state storageState + }{ + { + desc: "both successful", + first: call{ + primary: "primary", + secondaries: []string{"secondary"}, + }, + second: call{ + primary: "primary", + secondaries: []string{"secondary"}, + }, + state: storageState{ + "virtual-storage": { + "relative-path": { + "primary": {repositoryID: 1, generation: 2}, + "secondary": {repositoryID: 1, generation: 2}, + }, + }, + }, + }, + { + desc: "second write targeted outdated and up to date nodes", + first: call{ + primary: "primary", + }, + second: call{ + primary: "primary", + secondaries: []string{"secondary"}, + }, + state: storageState{ + "virtual-storage": { + "relative-path": { + "primary": {repositoryID: 1, generation: 2}, + "secondary": {repositoryID: 1, generation: 0}, + }, + }, + }, + }, + { + desc: "second write targeted only outdated nodes", + first: call{ + primary: "primary", + }, + second: call{ + primary: "secondary", + }, + error: errWriteToOutdatedNodes, + state: storageState{ + "virtual-storage": { + "relative-path": { + "primary": {repositoryID: 1, generation: 1}, + "secondary": {repositoryID: 1, generation: 0}, + }, + }, + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + ctx := testhelper.Context(t) + + db.TruncateAll(t) + + require.NoError(t, NewPostgresRepositoryStore(db, nil).CreateRepository(ctx, 1, "virtual-storage", "relative-path", "replica-path", "primary", []string{"secondary"}, nil, false, false)) + + firstTx := db.Begin(t) + secondTx := db.Begin(t) + + err := NewPostgresRepositoryStore(firstTx, nil).IncrementGeneration(ctx, 1, tc.first.primary, tc.first.secondaries) + require.NoError(t, err) + + go func() { + testdb.WaitForBlockedQuery(ctx, t, db, "WITH updated_replicas AS (") + firstTx.Commit(t) + }() + + err = NewPostgresRepositoryStore(secondTx, nil).IncrementGeneration(ctx, 1, tc.second.primary, tc.second.secondaries) + require.Equal(t, tc.error, err) + secondTx.Commit(t) + + requireState(t, ctx, db, + virtualStorageState{"virtual-storage": {"relative-path": {repositoryID: 1, replicaPath: "replica-path"}}}, + tc.state, + ) + }) + } +} + func TestPostgresRepositoryStore_GetRepositoryMetadata(t *testing.T) { t.Parallel() db := testdb.New(t) |