diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-05 13:05:28 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-05 13:05:28 +0300 |
commit | bab197494061f0ec76cebea552dfdf85bd75c110 (patch) | |
tree | d7c62d3e46e35414e4bc20eaf4b574ef9b7ffdba | |
parent | a7927b16e6a77aee8e2f1ff26f827d98c633aa15 (diff) | |
parent | a69cf20ceb11365f64d663a8d6028ceb1b79a09c (diff) |
Automatic merge of gitlab-org/gitaly master
40 files changed, 890 insertions, 1060 deletions
@@ -134,6 +134,8 @@ GIT_EXECUTABLES += git-http-backend GIT_VERSION ?= ## The Git version used for bundled Git v2.40. GIT_VERSION_2_40 ?= v2.40.1.gl2 +## The Git version used for bundled Git v2.41. +GIT_VERSION_2_41 ?= v2.41.0 ## Skip overriding the Git version and instead use the Git version as specified ## in the Git sources. This is required when building Git from a version that @@ -332,14 +334,16 @@ install: build .PHONY: build-bundled-git ## Build bundled Git binaries. -build-bundled-git: build-bundled-git-v2.40 +build-bundled-git: build-bundled-git-v2.40 build-bundled-git-v2.41 build-bundled-git-v2.40: $(patsubst %,${BUILD_DIR}/bin/gitaly-%-v2.40,${GIT_EXECUTABLES}) +build-bundled-git-v2.41: $(patsubst %,${BUILD_DIR}/bin/gitaly-%-v2.41,${GIT_EXECUTABLES}) .PHONY: install-bundled-git ## Install bundled Git binaries. The target directory can be modified by ## setting PREFIX and DESTDIR. -install-bundled-git: install-bundled-git-v2.40 +install-bundled-git: install-bundled-git-v2.40 install-bundled-git-v2.41 install-bundled-git-v2.40: $(patsubst %,${INSTALL_DEST_DIR}/gitaly-%-v2.40,${GIT_EXECUTABLES}) +install-bundled-git-v2.41: $(patsubst %,${INSTALL_DEST_DIR}/gitaly-%-v2.41,${GIT_EXECUTABLES}) ifdef WITH_BUNDLED_GIT build: build-bundled-git @@ -565,6 +569,10 @@ ${BUILD_DIR}/bin/gitaly-%-v2.40: override GIT_VERSION = ${GIT_VERSION_2_40} ${BUILD_DIR}/bin/gitaly-%-v2.40: ${DEPENDENCY_DIR}/git-v2.40/% | ${BUILD_DIR}/bin ${Q}install $< $@ +${BUILD_DIR}/bin/gitaly-%-v2.41: override GIT_VERSION = ${GIT_VERSION_2_41} +${BUILD_DIR}/bin/gitaly-%-v2.41: ${DEPENDENCY_DIR}/git-v2.41/% | ${BUILD_DIR}/bin + ${Q}install $< $@ + ${BUILD_DIR}/bin/%: ${BUILD_DIR}/intermediate/% | ${BUILD_DIR}/bin @ # To compute a unique and deterministic value for GNU build-id, we use an @ # intermediate binary which has a fixed build ID of "TEMP_GITALY_BUILD_ID", diff --git a/cmd/gitaly-backup/restore_test.go b/cmd/gitaly-backup/restore_test.go index 037078310..3492c6c15 100644 --- a/cmd/gitaly-backup/restore_test.go +++ b/cmd/gitaly-backup/restore_test.go @@ -76,7 +76,7 @@ func TestRestoreSubcommand(t *testing.T) { require.NoError(t, fs.Parse([]string{"-path", path, "-remove-all-repositories", existingRepo.StorageName})) require.EqualError(t, cmd.Run(ctx, &stdin, io.Discard), - "restore: pipeline: 1 failures encountered:\n - invalid: manager: remove repository: could not dial source: invalid connection string: \"invalid\"\n") + "restore: pipeline: 1 failures encountered:\n - invalid: manager: could not dial source: invalid connection string: \"invalid\"\n") require.NoDirExists(t, existRepoPath) diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 60fa0cd65..85e9a5055 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -12,10 +12,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" - "gitlab.com/gitlab-org/gitaly/v16/streamio" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) @@ -83,6 +81,16 @@ type Repository interface { GetCustomHooks(ctx context.Context, out io.Writer) error // CreateBundle fetches a bundle that contains refs matching patterns. CreateBundle(ctx context.Context, out io.Writer, patterns io.Reader) error + // Remove removes the repository. Does not return an error if the + // repository cannot be found. + Remove(ctx context.Context) error + // Create creates the repository. + Create(ctx context.Context) error + // FetchBundle fetches references from a bundle. Refs will be mirrored to + // the repository. + FetchBundle(ctx context.Context, reader io.Reader) error + // SetCustomHooks updates the custom hooks for the repository. + SetCustomHooks(ctx context.Context, reader io.Reader) error } // ResolveLocator returns a locator implementation based on a locator identifier. @@ -125,6 +133,10 @@ func NewManager(sink Sink, locator Locator, pool *client.Pool, backupID string) locator: locator, backupID: backupID, repositoryFactory: func(ctx context.Context, repo *gitalypb.Repository, server storage.ServerInfo) (Repository, error) { + if err := setContextServerInfo(ctx, &server, repo.GetStorageName()); err != nil { + return nil, err + } + conn, err := pool.Dial(ctx, server.Address, server.Token) if err != nil { return nil, err @@ -136,7 +148,15 @@ func NewManager(sink Sink, locator Locator, pool *client.Pool, backupID string) } // NewManagerLocal creates and returns a *Manager instance for operating on local repositories. -func NewManagerLocal(sink Sink, locator Locator, storageLocator storage.Locator, gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, backupID string) *Manager { +func NewManagerLocal( + sink Sink, + locator Locator, + storageLocator storage.Locator, + gitCmdFactory git.CommandFactory, + catfileCache catfile.Cache, + txManager transaction.Manager, + backupID string, +) *Manager { return &Manager{ sink: sink, conns: nil, // Will be removed once the restore operations are part of the Repository interface. @@ -145,7 +165,7 @@ func NewManagerLocal(sink Sink, locator Locator, storageLocator storage.Locator, repositoryFactory: func(ctx context.Context, repo *gitalypb.Repository, server storage.ServerInfo) (Repository, error) { localRepo := localrepo.New(storageLocator, gitCmdFactory, catfileCache, repo) - return newLocalRepository(storageLocator, localRepo), nil + return newLocalRepository(storageLocator, gitCmdFactory, txManager, localRepo), nil }, } } @@ -185,10 +205,6 @@ type CreateRequest struct { // Create creates a repository backup. func (mgr *Manager) Create(ctx context.Context, req *CreateRequest) error { - if err := setContextServerInfo(ctx, &req.Server, req.Repository.GetStorageName()); err != nil { - return fmt.Errorf("manager: %w", err) - } - repo, err := mgr.repositoryFactory(ctx, req.Repository, req.Server) if err != nil { return fmt.Errorf("manager: %w", err) @@ -241,11 +257,12 @@ type RestoreRequest struct { // Restore restores a repository from a backup. func (mgr *Manager) Restore(ctx context.Context, req *RestoreRequest) error { - if err := setContextServerInfo(ctx, &req.Server, req.Repository.GetStorageName()); err != nil { + repo, err := mgr.repositoryFactory(ctx, req.Repository, req.Server) + if err != nil { return fmt.Errorf("manager: %w", err) } - if err := mgr.removeRepository(ctx, req.Server, req.Repository); err != nil { + if err := repo.Remove(ctx); err != nil { return fmt.Errorf("manager: %w", err) } @@ -254,12 +271,12 @@ func (mgr *Manager) Restore(ctx context.Context, req *RestoreRequest) error { return fmt.Errorf("manager: %w", err) } - if err := mgr.createRepository(ctx, req.Server, req.Repository); err != nil { + if err := repo.Create(ctx); err != nil { return fmt.Errorf("manager: %w", err) } for _, step := range backup.Steps { - if err := mgr.restoreBundle(ctx, step.BundlePath, req.Server, req.Repository); err != nil { + if err := mgr.restoreBundle(ctx, repo, step.BundlePath); err != nil { if step.SkippableOnNotFound && errors.Is(err, ErrDoesntExist) { // For compatibility with existing backups we need to make sure the // repository exists even if there's no bundle for project @@ -271,14 +288,14 @@ func (mgr *Manager) Restore(ctx context.Context, req *RestoreRequest) error { return nil } - if err := mgr.removeRepository(ctx, req.Server, req.Repository); err != nil { + if err := repo.Remove(ctx); err != nil { return fmt.Errorf("manager: remove on skipped: %w", err) } return fmt.Errorf("manager: %w: %s", ErrSkipped, err.Error()) } } - if err := mgr.restoreCustomHooks(ctx, step.CustomHooksPath, req.Server, req.Repository); err != nil { + if err := mgr.restoreCustomHooks(ctx, repo, step.CustomHooksPath); err != nil { return fmt.Errorf("manager: %w", err) } } @@ -300,32 +317,6 @@ func setContextServerInfo(ctx context.Context, server *storage.ServerInfo, stora return nil } -func (mgr *Manager) removeRepository(ctx context.Context, server storage.ServerInfo, repo *gitalypb.Repository) error { - repoClient, err := mgr.newRepoClient(ctx, server) - if err != nil { - return fmt.Errorf("remove repository: %w", err) - } - _, err = repoClient.RemoveRepository(ctx, &gitalypb.RemoveRepositoryRequest{Repository: repo}) - switch { - case status.Code(err) == codes.NotFound: - return nil - case err != nil: - return fmt.Errorf("remove repository: %w", err) - } - return nil -} - -func (mgr *Manager) createRepository(ctx context.Context, server storage.ServerInfo, repo *gitalypb.Repository) error { - repoClient, err := mgr.newRepoClient(ctx, server) - if err != nil { - return fmt.Errorf("create repository: %w", err) - } - if _, err := repoClient.CreateRepository(ctx, &gitalypb.CreateRepositoryRequest{Repository: repo}); err != nil { - return fmt.Errorf("create repository: %w", err) - } - return nil -} - func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step, refs []git.Reference) (returnErr error) { negatedRefs, err := mgr.negatedKnownRefs(ctx, step) if err != nil { @@ -434,37 +425,14 @@ func (s *createBundleFromRefListSender) Send() error { return s.stream.Send(&s.chunk) } -func (mgr *Manager) restoreBundle(ctx context.Context, path string, server storage.ServerInfo, repo *gitalypb.Repository) error { +func (mgr *Manager) restoreBundle(ctx context.Context, repo Repository, path string) error { reader, err := mgr.sink.GetReader(ctx, path) if err != nil { - return fmt.Errorf("restore bundle: %w", err) - } - defer reader.Close() - - repoClient, err := mgr.newRepoClient(ctx, server) - if err != nil { return fmt.Errorf("restore bundle: %q: %w", path, err) } - stream, err := repoClient.FetchBundle(ctx) - if err != nil { - return fmt.Errorf("restore bundle: %q: %w", path, err) - } - request := &gitalypb.FetchBundleRequest{Repository: repo, UpdateHead: true} - bundle := streamio.NewWriter(func(p []byte) error { - request.Data = p - if err := stream.Send(request); err != nil { - return err - } - - // Only set `Repository` on the first `Send` of the stream - request = &gitalypb.FetchBundleRequest{} + defer reader.Close() - return nil - }) - if _, err := io.Copy(bundle, reader); err != nil { - return fmt.Errorf("restore bundle: %q: %w", path, err) - } - if _, err = stream.CloseAndRecv(); err != nil { + if err := repo.FetchBundle(ctx, reader); err != nil { return fmt.Errorf("restore bundle: %q: %w", path, err) } return nil @@ -485,7 +453,7 @@ func (mgr *Manager) writeCustomHooks(ctx context.Context, repo Repository, path return nil } -func (mgr *Manager) restoreCustomHooks(ctx context.Context, path string, server storage.ServerInfo, repo *gitalypb.Repository) error { +func (mgr *Manager) restoreCustomHooks(ctx context.Context, repo Repository, path string) error { reader, err := mgr.sink.GetReader(ctx, path) if err != nil { if errors.Is(err, ErrDoesntExist) { @@ -495,31 +463,7 @@ func (mgr *Manager) restoreCustomHooks(ctx context.Context, path string, server } defer reader.Close() - repoClient, err := mgr.newRepoClient(ctx, server) - if err != nil { - return fmt.Errorf("restore custom hooks, %q: %w", path, err) - } - stream, err := repoClient.SetCustomHooks(ctx) - if err != nil { - return fmt.Errorf("restore custom hooks, %q: %w", path, err) - } - - request := &gitalypb.SetCustomHooksRequest{Repository: repo} - bundle := streamio.NewWriter(func(p []byte) error { - request.Data = p - if err := stream.Send(request); err != nil { - return err - } - - // Only set `Repository` on the first `Send` of the stream - request = &gitalypb.SetCustomHooksRequest{} - - return nil - }) - if _, err := io.Copy(bundle, reader); err != nil { - return fmt.Errorf("restore custom hooks, %q: %w", path, err) - } - if _, err = stream.CloseAndRecv(); err != nil { + if err := repo.SetCustomHooks(ctx, reader); err != nil { return fmt.Errorf("restore custom hooks, %q: %w", path, err) } return nil diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 1341f767c..2472da0a5 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -20,6 +20,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" @@ -94,8 +95,10 @@ func TestManager_Create(t *testing.T) { storageLocator := config.NewLocator(cfg) gitCmdFactory := gittest.NewCommandFactory(tb, cfg) catfileCache := catfile.NewCache(cfg) + tb.Cleanup(catfileCache.Stop) + txManager := transaction.NewTrackingManager() - return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, backupID) + return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, backupID) }, }, } { @@ -243,8 +246,10 @@ func TestManager_Create_incremental(t *testing.T) { storageLocator := config.NewLocator(cfg) gitCmdFactory := gittest.NewCommandFactory(tb, cfg) catfileCache := catfile.NewCache(cfg) + tb.Cleanup(catfileCache.Stop) + txManager := transaction.NewTrackingManager() - return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, backupID) + return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, backupID) }, }, } { @@ -355,232 +360,270 @@ func TestManager_Create_incremental(t *testing.T) { func TestManager_Restore(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) + const backupID = "abc123" + cfg := testcfg.Build(t) testcfg.BuildGitalyHooks(t, cfg) - cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll) - cc, err := client.Dial(cfg.SocketPath, nil) - require.NoError(t, err) - defer testhelper.MustClose(t, cc) - - repoClient := gitalypb.NewRepositoryServiceClient(cc) - - _, repoPath := gittest.CreateRepository(t, ctx, cfg) - commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main")) - gittest.WriteTag(t, cfg, repoPath, "v1.0.0", commitID.Revision()) - repoChecksum := gittest.ChecksumRepo(t, cfg, repoPath) - - backupRoot := testhelper.TempDir(t) - - for _, tc := range []struct { - desc string - locators []string - setup func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) - alwaysCreate bool - expectExists bool - expectedPaths []string - expectedErrAs error + for _, managerTC := range []struct { + desc string + setup func(t testing.TB, sink backup.Sink, locator backup.Locator) *backup.Manager }{ { - desc: "existing repo, without hooks", - locators: []string{"legacy", "pointer"}, - setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { - repo, _ := gittest.CreateRepository(t, ctx, cfg) - - relativePath := stripRelativePath(tb, repo) - require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir)) - bundlePath := filepath.Join(backupRoot, relativePath+".bundle") - gittest.BundleRepo(tb, cfg, repoPath, bundlePath) + desc: "RPC manager", + setup: func(tb testing.TB, sink backup.Sink, locator backup.Locator) *backup.Manager { + pool := client.NewPool() + tb.Cleanup(func() { + testhelper.MustClose(tb, pool) + }) - return repo, repoChecksum - }, - expectExists: true, - }, - { - desc: "existing repo, with hooks", - locators: []string{"legacy", "pointer"}, - setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { - repo, _ := gittest.CreateRepository(t, ctx, cfg) - - relativePath := stripRelativePath(tb, repo) - bundlePath := filepath.Join(backupRoot, relativePath+".bundle") - customHooksPath := filepath.Join(backupRoot, relativePath, "custom_hooks.tar") - require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir)) - gittest.BundleRepo(tb, cfg, repoPath, bundlePath) - testhelper.CopyFile(tb, mustCreateCustomHooksArchive(t, ctx), customHooksPath) - - return repo, repoChecksum - }, - expectedPaths: []string{ - "custom_hooks/pre-commit.sample", - "custom_hooks/prepare-commit-msg.sample", - "custom_hooks/pre-push.sample", - }, - expectExists: true, - }, - { - desc: "missing bundle", - locators: []string{"legacy", "pointer"}, - setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { - repo, _ := gittest.CreateRepository(t, ctx, cfg) - return repo, nil - }, - expectedErrAs: backup.ErrSkipped, - }, - { - desc: "missing bundle, always create", - locators: []string{"legacy", "pointer"}, - setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { - repo, _ := gittest.CreateRepository(t, ctx, cfg) - return repo, new(git.Checksum) + return backup.NewManager(sink, locator, pool, backupID) }, - alwaysCreate: true, - expectExists: true, }, { - desc: "nonexistent repo", - locators: []string{"legacy", "pointer"}, - setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { - repo := &gitalypb.Repository{ - StorageName: "default", - RelativePath: gittest.NewRepositoryName(tb), + desc: "Local manager", + setup: func(tb testing.TB, sink backup.Sink, locator backup.Locator) *backup.Manager { + if testhelper.IsPraefectEnabled() { + tb.Skip("local backup manager expects to operate on the local filesystem so cannot operate through praefect") } - relativePath := stripRelativePath(tb, repo) - require.NoError(tb, os.MkdirAll(filepath.Dir(filepath.Join(backupRoot, relativePath)), perm.PublicDir)) - bundlePath := filepath.Join(backupRoot, relativePath+".bundle") - gittest.BundleRepo(tb, cfg, repoPath, bundlePath) + storageLocator := config.NewLocator(cfg) + gitCmdFactory := gittest.NewCommandFactory(tb, cfg) + catfileCache := catfile.NewCache(cfg) + tb.Cleanup(catfileCache.Stop) + txManager := transaction.NewTrackingManager() - return repo, repoChecksum + return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, backupID) }, - expectExists: true, - }, - { - desc: "single incremental", - locators: []string{"pointer"}, - setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { - const backupID = "abc123" - repo, _ := gittest.CreateRepository(t, ctx, cfg) - repoBackupPath := joinBackupPath(tb, backupRoot, repo) - backupPath := filepath.Join(repoBackupPath, backupID) - require.NoError(tb, os.MkdirAll(backupPath, perm.PublicDir)) - require.NoError(tb, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), perm.PublicFile)) - require.NoError(tb, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("001"), perm.PublicFile)) - bundlePath := filepath.Join(backupPath, "001.bundle") - gittest.BundleRepo(tb, cfg, repoPath, bundlePath) - - return repo, repoChecksum - }, - expectExists: true, - }, - { - desc: "many incrementals", - locators: []string{"pointer"}, - setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { - const backupID = "abc123" - - _, expectedRepoPath := gittest.CreateRepository(t, ctx, cfg) - - repo, _ := gittest.CreateRepository(t, ctx, cfg) - repoBackupPath := joinBackupPath(tb, backupRoot, repo) - backupPath := filepath.Join(repoBackupPath, backupID) - require.NoError(tb, os.MkdirAll(backupPath, perm.PublicDir)) - require.NoError(tb, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), perm.PublicFile)) - require.NoError(tb, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("002"), perm.PublicFile)) - - root := gittest.WriteCommit(tb, cfg, expectedRepoPath, - gittest.WithBranch("master"), - ) - master1 := gittest.WriteCommit(tb, cfg, expectedRepoPath, - gittest.WithBranch("master"), - gittest.WithParents(root), - ) - other := gittest.WriteCommit(tb, cfg, expectedRepoPath, - gittest.WithBranch("other"), - gittest.WithParents(root), - ) - gittest.Exec(tb, cfg, "-C", expectedRepoPath, "symbolic-ref", "HEAD", "refs/heads/master") - bundlePath1 := filepath.Join(backupPath, "001.bundle") - gittest.Exec(tb, cfg, "-C", expectedRepoPath, "bundle", "create", bundlePath1, - "HEAD", - "refs/heads/master", - "refs/heads/other", - ) - - master2 := gittest.WriteCommit(tb, cfg, expectedRepoPath, - gittest.WithBranch("master"), - gittest.WithParents(master1), - ) - bundlePath2 := filepath.Join(backupPath, "002.bundle") - gittest.Exec(tb, cfg, "-C", expectedRepoPath, "bundle", "create", bundlePath2, - "HEAD", - "^"+master1.String(), - "^"+other.String(), - "refs/heads/master", - "refs/heads/other", - ) - - checksum := new(git.Checksum) - checksum.Add(git.NewReference("HEAD", master2.String())) - checksum.Add(git.NewReference("refs/heads/master", master2.String())) - checksum.Add(git.NewReference("refs/heads/other", other.String())) - - return repo, checksum - }, - expectExists: true, }, } { - t.Run(tc.desc, func(t *testing.T) { - require.GreaterOrEqual(t, len(tc.locators), 1, "each test case must specify a locator") - - for _, locatorName := range tc.locators { - t.Run(locatorName, func(t *testing.T) { - repo, expectedChecksum := tc.setup(t) - - pool := client.NewPool() - defer testhelper.MustClose(t, pool) - - sink := backup.NewFilesystemSink(backupRoot) - locator, err := backup.ResolveLocator(locatorName, sink) - require.NoError(t, err) + managerTC := managerTC + + t.Run(managerTC.desc, func(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + + cc, err := client.Dial(cfg.SocketPath, nil) + require.NoError(t, err) + defer testhelper.MustClose(t, cc) + + repoClient := gitalypb.NewRepositoryServiceClient(cc) + + _, repoPath := gittest.CreateRepository(t, ctx, cfg) + commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main")) + gittest.WriteTag(t, cfg, repoPath, "v1.0.0", commitID.Revision()) + repoChecksum := gittest.ChecksumRepo(t, cfg, repoPath) + + backupRoot := testhelper.TempDir(t) + + for _, tc := range []struct { + desc string + locators []string + setup func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) + alwaysCreate bool + expectExists bool + expectedPaths []string + expectedErrAs error + }{ + { + desc: "existing repo, without hooks", + locators: []string{"legacy", "pointer"}, + setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { + repo, _ := gittest.CreateRepository(t, ctx, cfg) + + relativePath := stripRelativePath(tb, repo) + require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir)) + bundlePath := filepath.Join(backupRoot, relativePath+".bundle") + gittest.BundleRepo(tb, cfg, repoPath, bundlePath) + + return repo, repoChecksum + }, + expectExists: true, + }, + { + desc: "existing repo, with hooks", + locators: []string{"legacy", "pointer"}, + setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { + repo, _ := gittest.CreateRepository(t, ctx, cfg) + + relativePath := stripRelativePath(tb, repo) + bundlePath := filepath.Join(backupRoot, relativePath+".bundle") + customHooksPath := filepath.Join(backupRoot, relativePath, "custom_hooks.tar") + require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir)) + gittest.BundleRepo(tb, cfg, repoPath, bundlePath) + testhelper.CopyFile(tb, mustCreateCustomHooksArchive(t, ctx), customHooksPath) + + return repo, repoChecksum + }, + expectedPaths: []string{ + "custom_hooks/pre-commit.sample", + "custom_hooks/prepare-commit-msg.sample", + "custom_hooks/pre-push.sample", + }, + expectExists: true, + }, + { + desc: "missing bundle", + locators: []string{"legacy", "pointer"}, + setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { + repo, _ := gittest.CreateRepository(t, ctx, cfg) + return repo, nil + }, + expectedErrAs: backup.ErrSkipped, + }, + { + desc: "missing bundle, always create", + locators: []string{"legacy", "pointer"}, + setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { + repo, _ := gittest.CreateRepository(t, ctx, cfg) + return repo, new(git.Checksum) + }, + alwaysCreate: true, + expectExists: true, + }, + { + desc: "nonexistent repo", + locators: []string{"legacy", "pointer"}, + setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { + repo := &gitalypb.Repository{ + StorageName: "default", + RelativePath: gittest.NewRepositoryName(tb), + } - fsBackup := backup.NewManager(sink, locator, pool, "unused-backup-id") - err = fsBackup.Restore(ctx, &backup.RestoreRequest{ - Server: storage.ServerInfo{Address: cfg.SocketPath, Token: cfg.Auth.Token}, - Repository: repo, - AlwaysCreate: tc.alwaysCreate, - }) - if tc.expectedErrAs != nil { - require.ErrorAs(t, err, &tc.expectedErrAs) - } else { - require.NoError(t, err) - } + relativePath := stripRelativePath(tb, repo) + require.NoError(tb, os.MkdirAll(filepath.Dir(filepath.Join(backupRoot, relativePath)), perm.PublicDir)) + bundlePath := filepath.Join(backupRoot, relativePath+".bundle") + gittest.BundleRepo(tb, cfg, repoPath, bundlePath) - exists, err := repoClient.RepositoryExists(ctx, &gitalypb.RepositoryExistsRequest{ - Repository: repo, - }) - require.NoError(t, err) - require.Equal(t, tc.expectExists, exists.Exists, "repository exists") - - if expectedChecksum != nil { - checksum, err := repoClient.CalculateChecksum(ctx, &gitalypb.CalculateChecksumRequest{ - Repository: repo, + return repo, repoChecksum + }, + expectExists: true, + }, + { + desc: "single incremental", + locators: []string{"pointer"}, + setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { + const backupID = "abc123" + repo, _ := gittest.CreateRepository(t, ctx, cfg) + repoBackupPath := joinBackupPath(tb, backupRoot, repo) + backupPath := filepath.Join(repoBackupPath, backupID) + require.NoError(tb, os.MkdirAll(backupPath, perm.PublicDir)) + require.NoError(tb, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), perm.PublicFile)) + require.NoError(tb, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("001"), perm.PublicFile)) + bundlePath := filepath.Join(backupPath, "001.bundle") + gittest.BundleRepo(tb, cfg, repoPath, bundlePath) + + return repo, repoChecksum + }, + expectExists: true, + }, + { + desc: "many incrementals", + locators: []string{"pointer"}, + setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) { + const backupID = "abc123" + + _, expectedRepoPath := gittest.CreateRepository(t, ctx, cfg) + + repo, _ := gittest.CreateRepository(t, ctx, cfg) + repoBackupPath := joinBackupPath(tb, backupRoot, repo) + backupPath := filepath.Join(repoBackupPath, backupID) + require.NoError(tb, os.MkdirAll(backupPath, perm.PublicDir)) + require.NoError(tb, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), perm.PublicFile)) + require.NoError(tb, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("002"), perm.PublicFile)) + + root := gittest.WriteCommit(tb, cfg, expectedRepoPath, + gittest.WithBranch("master"), + ) + master1 := gittest.WriteCommit(tb, cfg, expectedRepoPath, + gittest.WithBranch("master"), + gittest.WithParents(root), + ) + other := gittest.WriteCommit(tb, cfg, expectedRepoPath, + gittest.WithBranch("other"), + gittest.WithParents(root), + ) + gittest.Exec(tb, cfg, "-C", expectedRepoPath, "symbolic-ref", "HEAD", "refs/heads/master") + bundlePath1 := filepath.Join(backupPath, "001.bundle") + gittest.Exec(tb, cfg, "-C", expectedRepoPath, "bundle", "create", bundlePath1, + "HEAD", + "refs/heads/master", + "refs/heads/other", + ) + + master2 := gittest.WriteCommit(tb, cfg, expectedRepoPath, + gittest.WithBranch("master"), + gittest.WithParents(master1), + ) + bundlePath2 := filepath.Join(backupPath, "002.bundle") + gittest.Exec(tb, cfg, "-C", expectedRepoPath, "bundle", "create", bundlePath2, + "HEAD", + "^"+master1.String(), + "^"+other.String(), + "refs/heads/master", + "refs/heads/other", + ) + + checksum := new(git.Checksum) + checksum.Add(git.NewReference("HEAD", master2.String())) + checksum.Add(git.NewReference("refs/heads/master", master2.String())) + checksum.Add(git.NewReference("refs/heads/other", other.String())) + + return repo, checksum + }, + expectExists: true, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + require.GreaterOrEqual(t, len(tc.locators), 1, "each test case must specify a locator") + + for _, locatorName := range tc.locators { + t.Run(locatorName, func(t *testing.T) { + repo, expectedChecksum := tc.setup(t) + + sink := backup.NewFilesystemSink(backupRoot) + locator, err := backup.ResolveLocator(locatorName, sink) + require.NoError(t, err) + + fsBackup := managerTC.setup(t, sink, locator) + err = fsBackup.Restore(ctx, &backup.RestoreRequest{ + Server: storage.ServerInfo{Address: cfg.SocketPath, Token: cfg.Auth.Token}, + Repository: repo, + AlwaysCreate: tc.alwaysCreate, + }) + if tc.expectedErrAs != nil { + require.ErrorAs(t, err, &tc.expectedErrAs) + } else { + require.NoError(t, err) + } + + exists, err := repoClient.RepositoryExists(ctx, &gitalypb.RepositoryExistsRequest{ + Repository: repo, + }) + require.NoError(t, err) + require.Equal(t, tc.expectExists, exists.Exists, "repository exists") + + if expectedChecksum != nil { + checksum, err := repoClient.CalculateChecksum(ctx, &gitalypb.CalculateChecksumRequest{ + Repository: repo, + }) + require.NoError(t, err) + + require.Equal(t, expectedChecksum.String(), checksum.GetChecksum()) + } + + if len(tc.expectedPaths) > 0 { + // Restore has to use the rewritten path as the relative path due to the test creating + // the repository through Praefect. In order to get to the correct disk paths, we need + // to get the replica path of the rewritten repository. + repoPath := filepath.Join(cfg.Storages[0].Path, gittest.GetReplicaPath(t, ctx, cfg, repo)) + for _, p := range tc.expectedPaths { + require.FileExists(t, filepath.Join(repoPath, p)) + } + } }) - require.NoError(t, err) - - require.Equal(t, expectedChecksum.String(), checksum.GetChecksum()) - } - - if len(tc.expectedPaths) > 0 { - // Restore has to use the rewritten path as the relative path due to the test creating - // the repository through Praefect. In order to get to the correct disk paths, we need - // to get the replica path of the rewritten repository. - repoPath := filepath.Join(cfg.Storages[0].Path, gittest.GetReplicaPath(t, ctx, cfg, repo)) - for _, p := range tc.expectedPaths { - require.FileExists(t, filepath.Join(repoPath, p)) - } } }) } diff --git a/internal/backup/repository.go b/internal/backup/repository.go index 6c2523908..957c4b64d 100644 --- a/internal/backup/repository.go +++ b/internal/backup/repository.go @@ -12,6 +12,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/repoutil" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/chunk" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" @@ -148,6 +149,89 @@ func (rr *remoteRepository) CreateBundle(ctx context.Context, out io.Writer, pat return nil } +// Remove removes the repository. Does not return an error if the repository +// cannot be found. +func (rr *remoteRepository) Remove(ctx context.Context) error { + repoClient := rr.newRepoClient() + _, err := repoClient.RemoveRepository(ctx, &gitalypb.RemoveRepositoryRequest{ + Repository: rr.repo, + }) + switch { + case status.Code(err) == codes.NotFound: + return nil + case err != nil: + return fmt.Errorf("remote repository: remove: %w", err) + } + return nil +} + +// Create creates the repository. +func (rr *remoteRepository) Create(ctx context.Context) error { + repoClient := rr.newRepoClient() + if _, err := repoClient.CreateRepository(ctx, &gitalypb.CreateRepositoryRequest{Repository: rr.repo}); err != nil { + return fmt.Errorf("remote repository: create: %w", err) + } + return nil +} + +// FetchBundle fetches references from a bundle. Refs will be mirrored to the +// repository. +func (rr *remoteRepository) FetchBundle(ctx context.Context, reader io.Reader) error { + repoClient := rr.newRepoClient() + stream, err := repoClient.FetchBundle(ctx) + if err != nil { + return fmt.Errorf("remote repository: fetch bundle: %w", err) + } + request := &gitalypb.FetchBundleRequest{Repository: rr.repo, UpdateHead: true} + bundle := streamio.NewWriter(func(p []byte) error { + request.Data = p + if err := stream.Send(request); err != nil { + return err + } + + // Only set `Repository` on the first `Send` of the stream + request = &gitalypb.FetchBundleRequest{} + + return nil + }) + if _, err := io.Copy(bundle, reader); err != nil { + return fmt.Errorf("remote repository: fetch bundle: %w", err) + } + if _, err = stream.CloseAndRecv(); err != nil { + return fmt.Errorf("remote repository: fetch bundle: %w", err) + } + return nil +} + +// SetCustomHooks updates the custom hooks for the repository. +func (rr *remoteRepository) SetCustomHooks(ctx context.Context, reader io.Reader) error { + repoClient := rr.newRepoClient() + stream, err := repoClient.SetCustomHooks(ctx) + if err != nil { + return fmt.Errorf("remote repository: set custom hooks: %w", err) + } + + request := &gitalypb.SetCustomHooksRequest{Repository: rr.repo} + bundle := streamio.NewWriter(func(p []byte) error { + request.Data = p + if err := stream.Send(request); err != nil { + return err + } + + // Only set `Repository` on the first `Send` of the stream + request = &gitalypb.SetCustomHooksRequest{} + + return nil + }) + if _, err := io.Copy(bundle, reader); err != nil { + return fmt.Errorf("remote repository: set custom hooks: %w", err) + } + if _, err = stream.CloseAndRecv(); err != nil { + return fmt.Errorf("remote repository: set custom hooks: %w", err) + } + return nil +} + func (rr *remoteRepository) newRepoClient() gitalypb.RepositoryServiceClient { return gitalypb.NewRepositoryServiceClient(rr.conn) } @@ -157,14 +241,23 @@ func (rr *remoteRepository) newRefClient() gitalypb.RefServiceClient { } type localRepository struct { - locator storage.Locator - repo *localrepo.Repo + locator storage.Locator + gitCmdFactory git.CommandFactory + txManager transaction.Manager + repo *localrepo.Repo } -func newLocalRepository(locator storage.Locator, repo *localrepo.Repo) *localRepository { +func newLocalRepository( + locator storage.Locator, + gitCmdFactory git.CommandFactory, + txManager transaction.Manager, + repo *localrepo.Repo, +) *localRepository { return &localRepository{ - locator: locator, - repo: repo, + locator: locator, + gitCmdFactory: gitCmdFactory, + txManager: txManager, + repo: repo, } } @@ -229,3 +322,57 @@ func (r *localRepository) CreateBundle(ctx context.Context, out io.Writer, patte return nil } + +// Remove removes the repository. Does not return an error if the repository +// cannot be found. +func (r *localRepository) Remove(ctx context.Context) error { + err := repoutil.Remove(ctx, r.locator, r.txManager, r.repo) + switch { + case status.Code(err) == codes.NotFound: + return nil + case err != nil: + return fmt.Errorf("local repository: remove: %w", err) + } + return nil +} + +// Create creates the repository. +func (r *localRepository) Create(ctx context.Context) error { + if err := repoutil.Create( + ctx, + r.locator, + r.gitCmdFactory, + r.txManager, + r.repo, + func(repository *gitalypb.Repository) error { return nil }, + ); err != nil { + return fmt.Errorf("local repository: create: %w", err) + } + return nil +} + +// FetchBundle fetches references from a bundle. Refs will be mirrored to the +// repository. +func (r *localRepository) FetchBundle(ctx context.Context, reader io.Reader) error { + err := r.repo.FetchBundle(ctx, r.txManager, reader, &localrepo.FetchBundleOpts{ + UpdateHead: true, + }) + if err != nil { + return fmt.Errorf("local repository: fetch bundle: %w", err) + } + return nil +} + +// SetCustomHooks updates the custom hooks for the repository. +func (r *localRepository) SetCustomHooks(ctx context.Context, reader io.Reader) error { + if err := repoutil.SetCustomHooks( + ctx, + r.locator, + r.txManager, + reader, + r.repo, + ); err != nil { + return fmt.Errorf("local repository: set custom hooks: %w", err) + } + return nil +} diff --git a/internal/cli/praefect/main.go b/internal/cli/praefect/main.go index c9547e895..b1fcf2007 100644 --- a/internal/cli/praefect/main.go +++ b/internal/cli/praefect/main.go @@ -15,12 +15,12 @@ import ( "fmt" "log" "os" - "sort" - "strings" + "os/signal" "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service" "gitlab.com/gitlab-org/gitaly/v16/internal/version" + "golang.org/x/exp/slices" ) func init() { @@ -39,6 +39,8 @@ const ( // NewApp returns a new praefect app. func NewApp() *cli.App { + interrupt := make(chan os.Signal, 1) + return &cli.App{ Name: progname, Usage: "a gitaly proxy", @@ -61,6 +63,7 @@ func NewApp() *cli.App { newListStoragesCommand(), newListUntrackedRepositoriesCommand(), newTrackRepositoryCommand(), + newTrackRepositoriesCommand(), newVerifyCommand(), newMetadataCommand(), newSQLPingCommand(), @@ -79,7 +82,29 @@ func NewApp() *cli.App { Usage: "load configuration from `FILE`", }, }, - CustomAppHelpTemplate: helpTextTemplate(), + Before: func(appCtx *cli.Context) error { + // Praefect service manages os.Interrupt on its own, by making a "table-flip". + // That is why the signal listening is omitted if there are no arguments passed + // (old-fashioned method of starting Praefect service) or 'serve' sub-command + // is invoked. Other sub-commands require signal to be properly handled. + args := appCtx.Args().Slice() + if len(args) == 0 || slices.Contains(args, "serve") { + return nil + } + + signal.Notify(interrupt, os.Interrupt) + go func() { + if _, ok := <-interrupt; ok { + os.Exit(130) // indicates program was interrupted + } + }() + + return nil + }, + After: func(*cli.Context) error { + close(interrupt) + return nil + }, } } @@ -99,22 +124,3 @@ func mustProvideConfigFlag(ctx *cli.Context, command string) string { return pathToConfigFile } - -func helpTextTemplate() string { - var cmds []string - for k := range subcommands(nil) { - cmds = append(cmds, k) - } - sort.Strings(cmds) - - // Because not all sub-commands are registered with the new approach they won't be shown - // with the -help. To have them in the output we inject a simple list of their names into - // the template to have them presented. - return strings.Replace( - cli.AppHelpTemplate, - `COMMANDS:{{template "visibleCommandCategoryTemplate" .}}{{end}}`, - `COMMANDS:{{template "visibleCommandCategoryTemplate" .}}{{end}}`+ - "\n "+strings.Join(cmds, "\n "), - 1, - ) -} diff --git a/internal/cli/praefect/serve.go b/internal/cli/praefect/serve.go index 328459ef6..096db8ad1 100644 --- a/internal/cli/praefect/serve.go +++ b/internal/cli/praefect/serve.go @@ -7,7 +7,6 @@ import ( "fmt" "math/rand" "net/http" - "os" "runtime/debug" "time" @@ -47,26 +46,21 @@ func newServeCommand() *cli.Command { Usage: "launch the server daemon", Action: serveAction, HideHelpCommand: true, + Before: func(context *cli.Context) error { + if context.Args().Present() { + return unexpectedPositionalArgsError{Command: context.Command.Name} + } + return nil + }, } } func serveAction(ctx *cli.Context) error { - logger := log.Default() - // In order to support execution of all sub-commands not yet migrated to use a new cli - // implementation the invocation is done manually here. - subCmd := ctx.Args().First() - if subCmd != "" { - // It doesn't make difference if we provide command name to the invocation below - // or not as there won't be any output printed, because sub-commands are not yet - // registered. - pathToConfigFile := mustProvideConfigFlag(ctx, "") - conf, err := getConfig(logger, pathToConfigFile) - if err != nil { - return err - } - os.Exit(subCommand(conf, logger, subCmd, ctx.Args().Slice()[1:])) + if ctx.Args().Present() { + return unexpectedPositionalArgsError{Command: ctx.Command.Name} } + logger := log.Default() // The ctx.Command.Name can't be used here because if `praefect -config FILE` is used // it will be set to 'praefect' instead of 'serve'. pathToConfigFile := mustProvideConfigFlag(ctx, "serve") diff --git a/internal/cli/praefect/subcmd.go b/internal/cli/praefect/subcmd.go index 457a4bc87..fefcefa8d 100644 --- a/internal/cli/praefect/subcmd.go +++ b/internal/cli/praefect/subcmd.go @@ -4,13 +4,10 @@ import ( "context" "database/sql" "errors" - "flag" "fmt" - "os" - "os/signal" + "io" "time" - "github.com/sirupsen/logrus" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" "gitlab.com/gitlab-org/gitaly/v16/client" internalclient "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/client" @@ -19,11 +16,6 @@ import ( "google.golang.org/grpc" ) -type subcmd interface { - FlagSet() *flag.FlagSet - Exec(flags *flag.FlagSet, config config.Config) error -} - const ( defaultDialTimeout = 10 * time.Second paramVirtualStorage = "virtual-storage" @@ -31,43 +23,6 @@ const ( paramAuthoritativeStorage = "authoritative-storage" ) -func subcommands(logger *logrus.Entry) map[string]subcmd { - return map[string]subcmd{ - trackRepositoriesCmdName: newTrackRepositories(logger, os.Stdout), - } -} - -// subCommand returns an exit code, to be fed into os.Exit. -func subCommand(conf config.Config, logger *logrus.Entry, arg0 string, argRest []string) int { - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - - go func() { - <-interrupt - os.Exit(130) // indicates program was interrupted - }() - - subcmd, ok := subcommands(logger)[arg0] - if !ok { - printfErr("%s: unknown subcommand: %q\n", progname, arg0) - return 1 - } - - flags := subcmd.FlagSet() - - if err := flags.Parse(argRest); err != nil { - printfErr("%s\n", err) - return 1 - } - - if err := subcmd.Exec(flags, conf); err != nil { - printfErr("%s\n", err) - return 1 - } - - return 0 -} - func getNodeAddress(cfg config.Config) (string, error) { switch { case cfg.SocketPath != "": @@ -81,7 +36,7 @@ func getNodeAddress(cfg config.Config) (string, error) { } } -func openDB(conf config.DB) (*sql.DB, func(), error) { +func openDB(conf config.DB, errOut io.Writer) (*sql.DB, func(), error) { ctx := context.Background() openDBCtx, cancel := context.WithTimeout(ctx, 30*time.Second) @@ -93,17 +48,13 @@ func openDB(conf config.DB) (*sql.DB, func(), error) { clean := func() { if err := db.Close(); err != nil { - printfErr("sql close: %v\n", err) + fmt.Fprintf(errOut, "sql close: %v\n", err) } } return db, clean, nil } -func printfErr(format string, a ...interface{}) { - fmt.Fprintf(os.Stderr, format, a...) -} - func subCmdDial(ctx context.Context, addr, token string, timeout time.Duration, opts ...grpc.DialOption) (*grpc.ClientConn, error) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() diff --git a/internal/cli/praefect/subcmd_accept_dataloss_test.go b/internal/cli/praefect/subcmd_accept_dataloss_test.go index f0e78638a..207c21f87 100644 --- a/internal/cli/praefect/subcmd_accept_dataloss_test.go +++ b/internal/cli/praefect/subcmd_accept_dataloss_test.go @@ -1,13 +1,10 @@ package praefect import ( - "bytes" "context" - "io" "testing" "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/info" @@ -138,23 +135,7 @@ func TestAcceptDatalossSubcommand(t *testing.T) { }, } { t.Run(tc.desc, func(t *testing.T) { - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: io.Discard, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newAcceptDatalossCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - - err := app.Run(append([]string{progname, "accept-dataloss"}, tc.args...)) + _, _, err := runApp(append([]string{"-config", confPath, "accept-dataloss"}, tc.args...)) tc.matchError(t, err) for storage, expected := range tc.expectedGenerations { diff --git a/internal/cli/praefect/subcmd_check_test.go b/internal/cli/praefect/subcmd_check_test.go index 68ee4e2aa..c970b8520 100644 --- a/internal/cli/praefect/subcmd_check_test.go +++ b/internal/cli/praefect/subcmd_check_test.go @@ -181,39 +181,42 @@ Checking check 3...Failed (warning) error: i failed but not too badly for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newCheckCommand(tc.checks), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - t.Run("quiet", func(t *testing.T) { - stdout.Reset() - err := app.Run(append([]string{progname, "check", "-q"}, tc.args...)) + var stdout, stderr bytes.Buffer + app := NewApp() + app.Writer = &stdout + app.ErrWriter = &stderr + for i, cmd := range app.Commands { + if cmd.Name == "check" { + app.Commands[i] = newCheckCommand(tc.checks) + break + } + } + err := app.Run(append([]string{progname, "-config", confPath, "check", "-q"}, tc.args...)) assert.Equal(t, tc.expectedError, err) if len(tc.args) == 0 { assert.Equal(t, tc.expectedQuietOutput, stdout.String()) } + assert.Empty(t, stderr) }) t.Run("normal", func(t *testing.T) { - stdout.Reset() - err := app.Run(append([]string{progname, "check"}, tc.args...)) + var stdout, stderr bytes.Buffer + app := NewApp() + app.Writer = &stdout + app.ErrWriter = &stderr + for i, cmd := range app.Commands { + if cmd.Name == "check" { + app.Commands[i] = newCheckCommand(tc.checks) + break + } + } + err := app.Run(append([]string{progname, "-config", confPath, "check"}, tc.args...)) assert.Equal(t, tc.expectedError, err) if len(tc.args) == 0 { assert.Equal(t, tc.expectedOutput, stdout.String()) } + assert.Empty(t, stderr) }) }) } diff --git a/internal/cli/praefect/subcmd_dataloss_test.go b/internal/cli/praefect/subcmd_dataloss_test.go index 4ad1c2059..30a3c49de 100644 --- a/internal/cli/praefect/subcmd_dataloss_test.go +++ b/internal/cli/praefect/subcmd_dataloss_test.go @@ -1,10 +1,9 @@ package praefect import ( - "bytes" - "io" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" @@ -179,28 +178,12 @@ Virtual storage: virtual-storage-2 cfg.VirtualStorages = tc.virtualStorages confPath := writeConfigToFile(t, cfg) - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newDatalossCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - - err := app.Run(append([]string{progname, "dataloss"}, tc.args...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, "dataloss"}, tc.args...)) require.Equal(t, tc.error, err, err) if tc.error == nil { - require.Equal(t, tc.output, stdout.String()) + require.Equal(t, tc.output, stdout) } + assert.Empty(t, stderr) }) } } diff --git a/internal/cli/praefect/subcmd_dial_nodes_test.go b/internal/cli/praefect/subcmd_dial_nodes_test.go index 221b8c78f..37dc49d7a 100644 --- a/internal/cli/praefect/subcmd_dial_nodes_test.go +++ b/internal/cli/praefect/subcmd_dial_nodes_test.go @@ -1,14 +1,13 @@ package praefect import ( - "bytes" "context" "fmt" - "io" "strings" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" @@ -136,27 +135,11 @@ func TestSubCmdDialNodes(t *testing.T) { resp = tt.resp confPath := writeConfigToFile(t, tt.conf) - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newDialNodesCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - - err := app.Run(append([]string{progname, "dial-nodes", "-timeout", time.Second.String()}, tt.args...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, "dial-nodes", "-timeout", time.Second.String()}, tt.args...)) + assert.Empty(t, stderr) if tt.errMsg == "" { require.NoError(t, err) - require.Equal(t, tt.logs, stdout.String()) + require.Equal(t, tt.logs, stdout) return } diff --git a/internal/cli/praefect/subcmd_list_storages_test.go b/internal/cli/praefect/subcmd_list_storages_test.go index 0356b79f3..44bcf9e76 100644 --- a/internal/cli/praefect/subcmd_list_storages_test.go +++ b/internal/cli/praefect/subcmd_list_storages_test.go @@ -2,10 +2,10 @@ package praefect import ( "bytes" - "io" "testing" "github.com/olekukonko/tablewriter" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" @@ -117,27 +117,6 @@ func TestListStoragesSubcommand(t *testing.T) { }, } - exec := func(confPath string, args []string) (string, error) { - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newListStoragesCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, "list-storages"}, args...)) - return stdout.String(), err - } - for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { var expectedOutput bytes.Buffer @@ -161,7 +140,8 @@ func TestListStoragesSubcommand(t *testing.T) { VirtualStorages: tc.virtualStorages, } confPath := writeConfigToFile(t, conf) - stdout, err := exec(confPath, tc.args) + stdout, stderr, err := runApp(append([]string{"-config", confPath, "list-storages"}, tc.args...)) + assert.Empty(t, stderr) require.NoError(t, err) require.Equal(t, expectedOutput.String(), stdout) }) @@ -189,13 +169,15 @@ func TestListStoragesSubcommand(t *testing.T) { confPath := writeConfigToFile(t, conf) t.Run("virtual storage arg matches no virtual storages", func(t *testing.T) { - stdout, err := exec(confPath, []string{"-virtual-storage", "vs-2"}) + stdout, stderr, err := runApp([]string{"-config", confPath, "list-storages", "-virtual-storage", "vs-2"}) + assert.Empty(t, stderr) require.NoError(t, err) require.Equal(t, "No virtual storages named vs-2.\n", stdout) }) t.Run("positional arguments", func(t *testing.T) { - _, err := exec(confPath, []string{"-virtual-storage", "vs-1", "positional-arg"}) + _, stderr, err := runApp([]string{"-config", confPath, "list-storages", "-virtual-storage", "vs-1", "positional-arg"}) + assert.Empty(t, stderr) require.Equal(t, cli.Exit(unexpectedPositionalArgsError{Command: "list-storages"}, 1), err) }) }) diff --git a/internal/cli/praefect/subcmd_list_untracked_repositories_test.go b/internal/cli/praefect/subcmd_list_untracked_repositories_test.go index 5fd7261f9..694f99040 100644 --- a/internal/cli/praefect/subcmd_list_untracked_repositories_test.go +++ b/internal/cli/praefect/subcmd_list_untracked_repositories_test.go @@ -1,15 +1,14 @@ package praefect import ( - "bytes" "context" "fmt" - "io" "os" "strings" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/client" @@ -86,50 +85,31 @@ func TestListUntrackedRepositoriesCommand(t *testing.T) { time.Now().Add(-(timeDelta+1*time.Second)), time.Now().Add(-(timeDelta+1*time.Second)))) - newApp := func() (cli.App, *bytes.Buffer) { - var stdout bytes.Buffer - return cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newListUntrackedRepositoriesCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - }, &stdout - } - t.Run("positional arguments", func(t *testing.T) { - app, _ := newApp() - err := app.Run([]string{progname, "list-untracked-repositories", "positional-arg"}) + stdout, stderr, err := runApp([]string{"-config", confPath, "list-untracked-repositories", "positional-arg"}) require.Equal(t, cli.Exit(unexpectedPositionalArgsError{Command: "list-untracked-repositories"}, 1), err) + assert.NotEmpty(t, stdout, "the help text should be printed") + assert.Empty(t, stderr) }) t.Run("default flag values used", func(t *testing.T) { - app, stdout := newApp() - err := app.Run([]string{progname, "list-untracked-repositories"}) + stdout, stderr, err := runApp([]string{"-config", confPath, "list-untracked-repositories"}) require.NoError(t, err) - require.Empty(t, stdout.String()) + assert.Empty(t, stdout) + assert.Empty(t, stderr) }) t.Run("passed flag values used", func(t *testing.T) { - app, stdout := newApp() - err := app.Run([]string{progname, "list-untracked-repositories", "-older-than", timeDelta.String(), "-delimiter", "~"}) + stdout, stderr, err := runApp([]string{"-config", confPath, "list-untracked-repositories", "-older-than", timeDelta.String(), "-delimiter", "~"}) require.NoError(t, err) - + assert.Empty(t, stderr) exp := []string{ "The following repositories were found on disk, but missing from the tracking database:", fmt.Sprintf(`{"relative_path":%q,"storage":"gitaly-1","virtual_storage":"praefect"}`, repo1.RelativePath), fmt.Sprintf(`{"relative_path":%q,"storage":"gitaly-1","virtual_storage":"praefect"}`, repo2.RelativePath), "", // an empty extra element required as each line ends with "delimiter" and strings.Split returns all parts } - elems := strings.Split(stdout.String(), "~") + elems := strings.Split(stdout, "~") require.Len(t, elems, len(exp)-1) elems = append(elems[1:], strings.Split(elems[0], "\n")...) require.ElementsMatch(t, exp, elems) diff --git a/internal/cli/praefect/subcmd_metadata_test.go b/internal/cli/praefect/subcmd_metadata_test.go index 124960504..5a48f6e97 100644 --- a/internal/cli/praefect/subcmd_metadata_test.go +++ b/internal/cli/praefect/subcmd_metadata_test.go @@ -1,13 +1,12 @@ package praefect import ( - "bytes" "errors" "fmt" - "io" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" @@ -125,23 +124,8 @@ func TestMetadataSubcommand(t *testing.T) { } confPath := writeConfigToFile(t, conf) - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newMetadataCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, "metadata"}, tc.args...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, "metadata"}, tc.args...)) + assert.Empty(t, stderr) require.Equal(t, tc.error, err) if tc.error != nil { return @@ -172,7 +156,7 @@ Replicas: Healthy: false Valid Primary: false Verified At: unverified -`, stdout.String()) +`, stdout) }) } } diff --git a/internal/cli/praefect/subcmd_remove_repository_test.go b/internal/cli/praefect/subcmd_remove_repository_test.go index d57de849c..a89ab27ed 100644 --- a/internal/cli/praefect/subcmd_remove_repository_test.go +++ b/internal/cli/praefect/subcmd_remove_repository_test.go @@ -1,9 +1,7 @@ package praefect import ( - "bytes" "fmt" - "io" "net" "os" "path/filepath" @@ -239,26 +237,11 @@ func TestRemoveRepositorySubcommand(t *testing.T) { } repo := createRepo(t, ctx, repoClient, praefectStorage, t.Name()) replicaPath := gittest.GetReplicaPath(t, ctx, gitalycfg.Cfg{SocketPath: praefectServer.Address()}, repo) - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newRemoveRepositoryCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, "remove-repository"}, tc.args(t, repo, replicaPath)...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, "remove-repository"}, tc.args(t, repo, replicaPath)...)) + assert.Empty(t, stderr) tc.assertError(t, err, repo, replicaPath) if tc.assertOutput != nil { - tc.assertOutput(t, stdout.String(), repo) + tc.assertOutput(t, stdout, repo) } }) } @@ -267,25 +250,10 @@ func TestRemoveRepositorySubcommand(t *testing.T) { repo := createRepo(t, ctx, repoClient, praefectStorage, t.Name()) g2Srv.Shutdown() replicaPath := gittest.GetReplicaPath(t, ctx, gitalycfg.Cfg{SocketPath: praefectServer.Address()}, repo) - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newRemoveRepositoryCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, "remove-repository"}, "-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply")) + stdout, stderr, err := runApp([]string{"-config", confPath, "remove-repository", "-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply"}) + assert.Empty(t, stderr) require.NoError(t, err) - assert.Contains(t, stdout.String(), "Repository removal completed.") + assert.Contains(t, stdout, "Repository removal completed.") require.NoDirExists(t, filepath.Join(g1Cfg.Storages[0].Path, replicaPath)) require.DirExists(t, filepath.Join(g2Cfg.Storages[0].Path, replicaPath)) require.False(t, repositoryExists(t, repo)) diff --git a/internal/cli/praefect/subcmd_set_replication_factor_test.go b/internal/cli/praefect/subcmd_set_replication_factor_test.go index 8b3f57800..f7878575d 100644 --- a/internal/cli/praefect/subcmd_set_replication_factor_test.go +++ b/internal/cli/praefect/subcmd_set_replication_factor_test.go @@ -1,11 +1,10 @@ package praefect import ( - "bytes" "errors" - "io" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect" @@ -115,26 +114,11 @@ func TestSetReplicationFactorSubcommand(t *testing.T) { } confPath := writeConfigToFile(t, conf) - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newSetReplicationFactorCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, setReplicationFactorCmdName}, tc.args...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, setReplicationFactorCmdName}, tc.args...)) + assert.Empty(t, stderr) testhelper.RequireGrpcError(t, tc.error, err) if tc.stdout != "" { - require.Equal(t, tc.stdout, stdout.String()) + require.Equal(t, tc.stdout, stdout) } }) } diff --git a/internal/cli/praefect/subcmd_sql_migrate.go b/internal/cli/praefect/subcmd_sql_migrate.go index 46d941113..2329125bf 100644 --- a/internal/cli/praefect/subcmd_sql_migrate.go +++ b/internal/cli/praefect/subcmd_sql_migrate.go @@ -53,7 +53,7 @@ func sqlMigrateAction(appCtx *cli.Context) error { return err } - db, clean, err := openDB(conf.DB) + db, clean, err := openDB(conf.DB, appCtx.App.ErrWriter) if err != nil { return err } diff --git a/internal/cli/praefect/subcmd_sql_migrate_down_test.go b/internal/cli/praefect/subcmd_sql_migrate_down_test.go index f12e3712c..70983f79a 100644 --- a/internal/cli/praefect/subcmd_sql_migrate_down_test.go +++ b/internal/cli/praefect/subcmd_sql_migrate_down_test.go @@ -1,13 +1,11 @@ package praefect import ( - "bytes" "errors" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore/migrations" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb" @@ -58,28 +56,11 @@ func TestSQLMigrateDownSubcommand(t *testing.T) { }, } { t.Run(tc.desc, func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: &stderr, - HideHelpCommand: true, - Commands: []*cli.Command{ - newSQLMigrateDownCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, sqlMigrateDownCmdName}, tc.args...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, sqlMigrateDownCmdName}, tc.args...)) + assert.Empty(t, stderr) require.Equal(t, tc.expectedErr, err) - assert.Empty(t, stderr.String()) for _, expectedOutput := range tc.expectedOutput { - assert.Contains(t, stdout.String(), expectedOutput) + assert.Contains(t, stdout, expectedOutput) } }) } diff --git a/internal/cli/praefect/subcmd_sql_migrate_status_test.go b/internal/cli/praefect/subcmd_sql_migrate_status_test.go index 5e43fb7eb..a2c978314 100644 --- a/internal/cli/praefect/subcmd_sql_migrate_status_test.go +++ b/internal/cli/praefect/subcmd_sql_migrate_status_test.go @@ -1,7 +1,6 @@ package praefect import ( - "bytes" "testing" "github.com/stretchr/testify/assert" @@ -43,28 +42,11 @@ func TestSQLMigrateStatusSubcommand(t *testing.T) { }, } { t.Run(tc.desc, func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: &stderr, - HideHelpCommand: true, - Commands: []*cli.Command{ - newSQLMigrateStatusCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, sqlMigrateStatusCmdName}, tc.args...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, sqlMigrateStatusCmdName}, tc.args...)) + assert.Empty(t, stderr) require.Equal(t, tc.expectedErr, err) - assert.Empty(t, stderr.String()) for _, expectedOut := range tc.expectedOuts { - assert.Contains(t, stdout.String(), expectedOut) + assert.Contains(t, stdout, expectedOut) } }) } diff --git a/internal/cli/praefect/subcmd_sql_migrate_test.go b/internal/cli/praefect/subcmd_sql_migrate_test.go index 9fae09ca8..880c01555 100644 --- a/internal/cli/praefect/subcmd_sql_migrate_test.go +++ b/internal/cli/praefect/subcmd_sql_migrate_test.go @@ -1,7 +1,6 @@ package praefect import ( - "bytes" "fmt" "testing" @@ -77,29 +76,11 @@ func TestSubCmdSqlMigrate(t *testing.T) { } { t.Run(tc.desc, func(t *testing.T) { testdb.SetMigrations(t, db, cfg, tc.up) - - var stdout bytes.Buffer - var stderr bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: &stderr, - HideHelpCommand: true, - Commands: []*cli.Command{ - newSQLMigrateCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, sqlMigrateCmdName, "-ignore-unknown"}, tc.args...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, sqlMigrateCmdName, "-ignore-unknown"}, tc.args...)) + assert.Empty(t, stderr) require.Equal(t, tc.expectedErr, err) - assert.Empty(t, stderr.String()) for _, out := range tc.expectedOutput { - assert.Contains(t, stdout.String(), out) + assert.Contains(t, stdout, out) } }) } diff --git a/internal/cli/praefect/subcmd_sql_ping.go b/internal/cli/praefect/subcmd_sql_ping.go index 51ac456ef..665d86d0a 100644 --- a/internal/cli/praefect/subcmd_sql_ping.go +++ b/internal/cli/praefect/subcmd_sql_ping.go @@ -36,7 +36,7 @@ func sqlPingAction(appCtx *cli.Context) error { subCmd := progname + " " + appCtx.Command.Name - db, clean, err := openDB(conf.DB) + db, clean, err := openDB(conf.DB, appCtx.App.ErrWriter) if err != nil { return err } diff --git a/internal/cli/praefect/subcmd_sql_ping_test.go b/internal/cli/praefect/subcmd_sql_ping_test.go index c35d99890..22139c55b 100644 --- a/internal/cli/praefect/subcmd_sql_ping_test.go +++ b/internal/cli/praefect/subcmd_sql_ping_test.go @@ -1,13 +1,11 @@ package praefect import ( - "bytes" "errors" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb" ) @@ -56,30 +54,13 @@ func TestSQLPingSubcommand(t *testing.T) { }, } { t.Run(tc.desc, func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: &stderr, - HideHelpCommand: true, - Commands: []*cli.Command{ - newSQLPingCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: tc.confPath(t), - }, - }, - } - err := app.Run(append([]string{progname, sqlPingCmdName}, tc.args...)) + stdout, stderr, err := runApp(append([]string{"-config", tc.confPath(t), sqlPingCmdName}, tc.args...)) + assert.Empty(t, stderr) if tc.expectedErr != nil { require.EqualError(t, err, tc.expectedErr.Error()) } - assert.Empty(t, stderr.String()) if tc.expectedOutput != "" { - assert.Equal(t, tc.expectedOutput, stdout.String()) + assert.Equal(t, tc.expectedOutput, stdout) } }) } diff --git a/internal/cli/praefect/subcmd_test.go b/internal/cli/praefect/subcmd_test.go index aaaa38d5a..6b5604042 100644 --- a/internal/cli/praefect/subcmd_test.go +++ b/internal/cli/praefect/subcmd_test.go @@ -1,6 +1,7 @@ package praefect import ( + "bytes" "fmt" "net" "path/filepath" @@ -68,3 +69,13 @@ func listenAndServe(tb testing.TB, svcs []svcRegistrar) (net.Listener, testhelpe require.NoErrorf(tb, err, "error while stopping server: %q", err) } } + +func runApp(args []string) (string, string, error) { + var stdout, stderr bytes.Buffer + app := NewApp() + app.Writer = &stdout + app.ErrWriter = &stderr + app.Reader = bytes.NewReader(nil) + err := app.Run(append([]string{progname}, args...)) + return stdout.String(), stderr.String(), err +} diff --git a/internal/cli/praefect/subcmd_track_repositories.go b/internal/cli/praefect/subcmd_track_repositories.go index 6130fb1f8..ea61ebe8a 100644 --- a/internal/cli/praefect/subcmd_track_repositories.go +++ b/internal/cli/praefect/subcmd_track_repositories.go @@ -4,16 +4,13 @@ import ( "bufio" "context" "encoding/json" - "flag" "fmt" "io" "os" - "time" - "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" + "github.com/urfave/cli/v2" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore/glsql" "gitlab.com/gitlab-org/labkit/correlation" ) @@ -22,72 +19,63 @@ const ( paramInputPath = "input-path" ) -type invalidRequest struct { - line int - path string - errs []error -} - -type dupPathError struct { - path string - reqNums []int -} - -func (d *dupPathError) Error() string { - return fmt.Sprintf("duplicate entries for relative_path, line %v", d.reqNums) -} - -type trackRepositories struct { - w io.Writer - logger logrus.FieldLogger - inputPath string - replicateImmediately bool -} - -func newTrackRepositories(logger logrus.FieldLogger, w io.Writer) *trackRepositories { - return &trackRepositories{w: w, logger: logger} -} - -func (cmd *trackRepositories) FlagSet() *flag.FlagSet { - fs := flag.NewFlagSet(trackRepositoryCmdName, flag.ExitOnError) - fs.BoolVar(&cmd.replicateImmediately, "replicate-immediately", false, "kick off a replication immediately") - fs.StringVar(&cmd.inputPath, paramInputPath, "", "path to file with details of repositories to track") - fs.Usage = func() { - printfErr("Description:\n" + - " This command allows bulk requests for repositories to be tracked by Praefect.\n" + - " The -input-path flag must be the path of a file containing the details of the repositories\n" + - " to track as a list of newline-delimited JSON objects. Each line must contain the details for\n" + - " one and only one repository. Each item must contain the following keys:\n\n" + - " relative_path - The relative path of the repository on-disk.\n" + - " virtual_storage - The Praefect virtual storage name.\n" + - " authoritative_storage - Which storage to consider as the canonical copy of the repository.\n\n" + - " If -replicate-immediately is used, the command will attempt to replicate the repositories\n" + - " to the secondaries. Otherwise, replication jobs will be created and will be executed\n" + - " eventually by Praefect itself.\n") - fs.PrintDefaults() +func newTrackRepositoriesCommand() *cli.Command { + return &cli.Command{ + Name: trackRepositoriesCmdName, + Usage: "process bulk requests to track repositories in Praefect", + Description: "This command allows bulk requests for repositories to be tracked by Praefect.\n" + + "The -input-path flag must be the path of a file containing the details of the repositories\n" + + "to track as a list of newline-delimited JSON objects. Each line must contain the details for\n" + + "one and only one repository. Each item must contain the following keys:\n\n" + + " relative_path - The relative path of the repository on-disk.\n" + + " virtual_storage - The Praefect virtual storage name.\n" + + " authoritative_storage - Which storage to consider as the canonical copy of the repository.\n\n" + + "If -replicate-immediately is used, the command will attempt to replicate the repositories\n" + + "to the secondaries. Otherwise, replication jobs will be created and will be executed\n" + + "eventually by Praefect itself.\n", + HideHelpCommand: true, + Action: trackRepositoriesAction, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "replicate-immediately", + Usage: "kick off replication jobs immediately", + }, + &cli.StringFlag{ + Name: paramInputPath, + Usage: "path to file with details of repositories to track", + Required: true, + }, + }, + Before: func(ctx *cli.Context) error { + if ctx.Args().Present() { + _ = cli.ShowSubcommandHelp(ctx) + return cli.Exit(unexpectedPositionalArgsError{Command: ctx.Command.Name}, 1) + } + return nil + }, } - return fs } -func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error { - switch { - case flags.NArg() > 0: - return unexpectedPositionalArgsError{Command: flags.Name()} +func trackRepositoriesAction(appCtx *cli.Context) error { + logger := log.Default() + conf, err := getConfig(logger, appCtx.String(configFlagName)) + if err != nil { + return err } - ctx := correlation.ContextWithCorrelation(context.Background(), correlation.SafeRandomID()) - logger := cmd.logger.WithField("correlation_id", correlation.ExtractFromContext(ctx)) - - openDBCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - db, err := glsql.OpenDB(openDBCtx, cfg.DB) + db, clean, err := openDB(conf.DB, appCtx.App.ErrWriter) if err != nil { return fmt.Errorf("connect to database: %w", err) } - defer func() { _ = db.Close() }() - store := datastore.NewPostgresRepositoryStore(db, cfg.StorageNames()) + defer clean() + + ctx := correlation.ContextWithCorrelation(context.Background(), correlation.SafeRandomID()) + logger = logger.WithField("correlation_id", correlation.ExtractFromContext(ctx)) + + store := datastore.NewPostgresRepositoryStore(db, conf.StorageNames()) - f, err := os.Open(cmd.inputPath) + inputPath := appCtx.String(paramInputPath) + f, err := os.Open(inputPath) if err != nil { return fmt.Errorf("open input: %w", err) } @@ -95,7 +83,7 @@ func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error scanner := bufio.NewScanner(f) - fmt.Fprintf(cmd.w, "Validating repository information in %q\n", cmd.inputPath) + fmt.Fprintf(appCtx.App.Writer, "Validating repository information in %q\n", inputPath) var requests []trackRepositoryRequest var line int @@ -164,7 +152,7 @@ func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error continue } - authoritativeRepoExists, err := request.authoritativeRepositoryExists(ctx, cfg, logger, cmd.w, request.AuthoritativeStorage) + authoritativeRepoExists, err := request.authoritativeRepositoryExists(ctx, conf, logger, appCtx.App.Writer, request.AuthoritativeStorage) if err != nil { badReq.errs = append(badReq.errs, fmt.Errorf("checking repository on disk: %w", err)) } else if !authoritativeRepoExists { @@ -179,17 +167,18 @@ func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error } if len(repoErrs) > 0 { - printInvalidRequests(cmd.w, repoErrs, pathLines, cmd.inputPath) + printInvalidRequests(appCtx.App.Writer, repoErrs, pathLines, inputPath) return fmt.Errorf("invalid entries found, aborting") } if len(requests) == 0 { - return fmt.Errorf("no repository information found in %q", cmd.inputPath) + return fmt.Errorf("no repository information found in %q", inputPath) } - fmt.Fprintf(cmd.w, "All repository details are correctly formatted\n") - fmt.Fprintf(cmd.w, "Tracking %v repositories in Praefect DB...\n", line) + fmt.Fprintf(appCtx.App.Writer, "All repository details are correctly formatted\n") + fmt.Fprintf(appCtx.App.Writer, "Tracking %v repositories in Praefect DB...\n", line) + replicateImmediately := appCtx.Bool("replicate-immediately") for _, request := range requests { - if err := request.execRequest(ctx, db, cfg, cmd.w, logger, cmd.replicateImmediately); err != nil { + if err := request.execRequest(ctx, db, conf, appCtx.App.Writer, logger, replicateImmediately); err != nil { return fmt.Errorf("tracking repository %q: %w", request.RelativePath, err) } } @@ -197,6 +186,21 @@ func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error return nil } +type invalidRequest struct { + line int + path string + errs []error +} + +type dupPathError struct { + path string + reqNums []int +} + +func (d *dupPathError) Error() string { + return fmt.Sprintf("duplicate entries for relative_path, line %v", d.reqNums) +} + func printInvalidRequests(w io.Writer, repoErrs []invalidRequest, pathLines map[string][]int, inputPath string) { fmt.Fprintf(w, "Found %v invalid request(s) in %q:\n", len(repoErrs), inputPath) diff --git a/internal/cli/praefect/subcmd_track_repositories_test.go b/internal/cli/praefect/subcmd_track_repositories_test.go index 9c1314894..369bfc125 100644 --- a/internal/cli/praefect/subcmd_track_repositories_test.go +++ b/internal/cli/praefect/subcmd_track_repositories_test.go @@ -1,9 +1,7 @@ package praefect import ( - "bytes" "encoding/json" - "flag" "fmt" "os" "path/filepath" @@ -24,176 +22,10 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "golang.org/x/exp/slices" ) -func TestAddRepositories_FlagSet(t *testing.T) { - t.Parallel() - cmd := &trackRepositories{} - fs := cmd.FlagSet() - require.NoError(t, fs.Parse([]string{"--input-path", "/dev/stdin", "--replicate-immediately", "true"})) - require.Equal(t, "/dev/stdin", cmd.inputPath) - require.Equal(t, true, cmd.replicateImmediately) -} - -func TestAddRepositories_Exec_invalidInput(t *testing.T) { - t.Parallel() - g1Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-1")) - g2Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-2")) - - g1Srv := testserver.StartGitalyServer(t, g1Cfg, setup.RegisterAll, testserver.WithDisablePraefect()) - g2Srv := testserver.StartGitalyServer(t, g2Cfg, setup.RegisterAll, testserver.WithDisablePraefect()) - defer g2Srv.Shutdown() - defer g1Srv.Shutdown() - g1Addr := g1Srv.Address() - - db := testdb.New(t) - dbConf := testdb.GetConfig(t, db.Name) - - virtualStorageName := "praefect" - conf := config.Config{ - AllowLegacyElectors: true, - SocketPath: testhelper.GetTemporaryGitalySocketFileName(t), - VirtualStorages: []*config.VirtualStorage{ - { - Name: virtualStorageName, - Nodes: []*config.Node{ - {Storage: g1Cfg.Storages[0].Name, Address: g1Addr}, - {Storage: g2Cfg.Storages[0].Name, Address: g2Srv.Address()}, - }, - DefaultReplicationFactor: 2, - }, - }, - DB: dbConf, - Failover: config.Failover{ - Enabled: true, - ElectionStrategy: config.ElectionStrategyPerRepository, - }, - } - rs := datastore.NewPostgresRepositoryStore(db, nil) - ctx := testhelper.Context(t) - inputFile := "input_file" - logger := testhelper.NewDiscardingLogger(t) - - trackRepo := func(relativePath string) error { - repositoryID, err := rs.ReserveRepositoryID(ctx, virtualStorageName, relativePath) - if err != nil { - return err - } - return rs.CreateRepository( - ctx, - repositoryID, - virtualStorageName, - relativePath, - relativePath, - g1Cfg.Storages[0].Name, - nil, - nil, - false, - false, - ) - } - - invalidEntryErr := "invalid entries found, aborting" - - testCases := []struct { - input string - desc string - expectedOutput string - expectedError string - trackedPath string - }{ - { - input: "", - desc: "empty input", - expectedError: "no repository information found", - }, - { - input: "@hashed/01/23/01234567890123456789.git", - desc: "invalid JSON", - expectedOutput: "invalid character '@' looking for beginning of value", - expectedError: invalidEntryErr, - }, - { - input: `{"virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, - desc: "missing path", - expectedOutput: `"repository" is a required parameter`, - expectedError: invalidEntryErr, - }, - { - input: `{"virtual_storage":"foo","relative_path":"bar","authoritative_storage":"gitaly-1"}`, - desc: "invalid virtual storage", - expectedOutput: `virtual storage "foo" not found`, - expectedError: invalidEntryErr, - }, - { - input: `{"relative_path":"not_a_repo","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, - desc: "repo does not exist", - expectedOutput: "not a valid git repository", - expectedError: invalidEntryErr, - }, - { - input: `{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"} -{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"}`, - desc: "duplicate path", - expectedOutput: "duplicate entries for relative_path", - expectedError: invalidEntryErr, - }, - { - input: `{"relative_path":"already_tracked","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, - desc: "repo is already tracked", - expectedOutput: "repository is already tracked by Praefect", - expectedError: invalidEntryErr, - trackedPath: "already_tracked", - }, - } - for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { - nodeMgr, err := nodes.NewManager( - testhelper.NewDiscardingLogEntry(t), - conf, - db.DB, - nil, - promtest.NewMockHistogramVec(), - protoregistry.GitalyProtoPreregistered, - nil, - nil, - nil, - ) - require.NoError(t, err) - nodeMgr.Start(0, time.Hour) - defer nodeMgr.Stop() - - tempDir := testhelper.TempDir(t) - f, err := os.Create(filepath.Join(tempDir, inputFile)) - require.NoError(t, err) - _, err = f.Write([]byte(tc.input)) - require.NoError(t, err) - require.NoError(t, f.Close()) - - var stdout bytes.Buffer - - if tc.trackedPath != "" { - require.NoError(t, trackRepo(tc.trackedPath)) - } - - addReposCmd := &trackRepositories{ - inputPath: filepath.Join(tempDir, inputFile), - replicateImmediately: true, - logger: logger, - w: &stdout, - } - err = addReposCmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf) - require.Error(t, err) - - if tc.expectedOutput != "" { - require.Contains(t, stdout.String(), tc.expectedOutput) - } - require.Contains(t, err.Error(), tc.expectedError) - }) - } -} - -func TestAddRepositories_Exec(t *testing.T) { +func TestTrackRepositoriesSubcommand(t *testing.T) { t.Parallel() g1Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-1")) g2Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-2")) @@ -210,7 +42,7 @@ func TestAddRepositories_Exec(t *testing.T) { db := testdb.New(t) dbConf := testdb.GetConfig(t, db.Name) - virtualStorageName := "praefect" + const virtualStorageName = "praefect" conf := config.Config{ AllowLegacyElectors: true, SocketPath: testhelper.GetTemporaryGitalySocketFileName(t), @@ -230,10 +62,11 @@ func TestAddRepositories_Exec(t *testing.T) { ElectionStrategy: config.ElectionStrategyPerRepository, }, } + confPath := writeConfigToFile(t, conf) gitalyCC, err := client.Dial(g1Addr, nil) require.NoError(t, err) - defer func() { require.NoError(t, gitalyCC.Close()) }() + defer testhelper.MustClose(t, gitalyCC) ctx := testhelper.Context(t) gitaly1RepositoryClient := gitalypb.NewRepositoryServiceClient(gitalyCC) @@ -251,53 +84,60 @@ func TestAddRepositories_Exec(t *testing.T) { } authoritativeStorage := g1Cfg.Storages[0].Name - logger := testhelper.NewDiscardingLogger(t) + + nodeMgr, err := nodes.NewManager( + testhelper.NewDiscardingLogEntry(t), + conf, + db.DB, + nil, + promtest.NewMockHistogramVec(), + protoregistry.GitalyProtoPreregistered, + nil, + nil, + nil, + ) + require.NoError(t, err) + nodeMgr.Start(0, time.Hour) + defer nodeMgr.Stop() + + repositoryStore := datastore.NewPostgresRepositoryStore(db, conf.StorageNames()) + assignmentStore := datastore.NewAssignmentStore(db, conf.StorageNames()) t.Run("ok", func(t *testing.T) { testCases := []struct { - relativePaths []string - desc string - replicateImmediately bool - expectedOutput string + desc string + input string + relativePaths []string + args func(inputPath string) []string + expectedOutput string }{ { - relativePaths: []string{"path/to/test/repo1", "path/to/test/repo2"}, - desc: "immediate replication", - replicateImmediately: true, - expectedOutput: "Finished replicating repository to \"gitaly-2\".\n", + desc: "immediate replication", + relativePaths: []string{"path/to/test/repo1", "path/to/test/repo2"}, + args: func(inputPath string) []string { + return []string{"-replicate-immediately", "-input-path=" + inputPath} + }, + expectedOutput: "Finished replicating repository to \"gitaly-2\".\n", }, { - relativePaths: []string{"path/to/test/repo3", "path/to/test/repo4"}, - desc: "no immediate replication", - replicateImmediately: false, - expectedOutput: "Added replication job to replicate repository to \"gitaly-2\".\n", + desc: "no immediate replication", + relativePaths: []string{"path/to/test/repo3", "path/to/test/repo4"}, + args: func(inputPath string) []string { + return []string{"-input-path=" + inputPath} + }, + expectedOutput: "Added replication job to replicate repository to \"gitaly-2\".\n", }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - nodeMgr, err := nodes.NewManager( - testhelper.NewDiscardingLogEntry(t), - conf, - db.DB, - nil, - promtest.NewMockHistogramVec(), - protoregistry.GitalyProtoPreregistered, - nil, - nil, - nil, - ) - require.NoError(t, err) - nodeMgr.Start(0, time.Hour) - defer nodeMgr.Stop() - tempDir := testhelper.TempDir(t) - input, err := os.Create(filepath.Join(tempDir, "input")) + inputPath := filepath.Join(tempDir, "input") + input, err := os.Create(inputPath) require.NoError(t, err) - repoDS := datastore.NewPostgresRepositoryStore(db, conf.StorageNames()) for _, path := range tc.relativePaths { - exists, err := repoDS.RepositoryExists(ctx, virtualStorageName, path) + exists, err := repositoryStore.RepositoryExists(ctx, virtualStorageName, path) require.NoError(t, err) require.False(t, exists) @@ -309,42 +149,36 @@ func TestAddRepositories_Exec(t *testing.T) { // Write repo details to input file repoEntry, err := json.Marshal(trackRepositoryRequest{RelativePath: path, VirtualStorage: virtualStorageName, AuthoritativeStorage: authoritativeStorage}) require.NoError(t, err) - fmt.Fprintf(input, string(repoEntry)+"\n") - } - - var stdout bytes.Buffer - - addRepoCmd := &trackRepositories{ - inputPath: filepath.Join(tempDir, "input"), - replicateImmediately: tc.replicateImmediately, - logger: logger, - w: &stdout, + _, err = fmt.Fprintf(input, string(repoEntry)+"\n") + require.NoError(t, err) } + require.NoError(t, input.Close()) - require.NoError(t, addRepoCmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf)) - assert.Contains(t, stdout.String(), tc.expectedOutput) - - as := datastore.NewAssignmentStore(db, conf.StorageNames()) + stdout, stderr, err := runApp(append([]string{"-config", confPath, trackRepositoriesCmdName}, tc.args(inputPath)...)) + assert.Empty(t, stderr) + require.NoError(t, err) + assert.Contains(t, stdout, tc.expectedOutput) + replicateImmediately := slices.Contains(tc.args(inputPath), "-replicate-immediately") for _, path := range tc.relativePaths { - repositoryID, err := repoDS.GetRepositoryID(ctx, virtualStorageName, path) + repositoryID, err := repositoryStore.GetRepositoryID(ctx, virtualStorageName, path) require.NoError(t, err) - assignments, err := as.GetHostAssignments(ctx, virtualStorageName, repositoryID) + assignments, err := assignmentStore.GetHostAssignments(ctx, virtualStorageName, repositoryID) require.NoError(t, err) require.Len(t, assignments, 2) assert.Contains(t, assignments, g1Cfg.Storages[0].Name) assert.Contains(t, assignments, g2Cfg.Storages[0].Name) - exists, err := repoDS.RepositoryExists(ctx, virtualStorageName, path) + exists, err := repositoryStore.RepositoryExists(ctx, virtualStorageName, path) require.NoError(t, err) assert.True(t, exists) - if !tc.replicateImmediately { + if !replicateImmediately { queue := datastore.NewPostgresReplicationEventQueue(db) events, err := queue.Dequeue(ctx, virtualStorageName, g2Cfg.Storages[0].Name, 1) require.NoError(t, err) - assert.Len(t, events, 1) + require.Len(t, events, 1) assert.Equal(t, path, events[0].Job.RelativePath) } else { require.DirExists(t, filepath.Join(g2Cfg.Storages[0].Path, path)) @@ -353,4 +187,123 @@ func TestAddRepositories_Exec(t *testing.T) { }) } }) + + trackRepo := func(relativePath string) error { + repositoryID, err := repositoryStore.ReserveRepositoryID(ctx, virtualStorageName, relativePath) + if err != nil { + return err + } + return repositoryStore.CreateRepository( + ctx, + repositoryID, + virtualStorageName, + relativePath, + relativePath, + g1Cfg.Storages[0].Name, + nil, + nil, + false, + false, + ) + } + + const invalidEntryErr = "invalid entries found, aborting" + + t.Run("fail", func(t *testing.T) { + for _, tc := range []struct { + desc string + args func(inputPath string) []string + input string + expectedOutput string + expectedError string + trackedPath string + }{ + { + desc: "positional arguments", + args: func(inputPath string) []string { + return []string{"-input-path=" + inputPath, "positional-arg"} + }, + expectedError: "track-repositories doesn't accept positional arguments", + }, + { + desc: "no required flag 'input-path'", + args: func(string) []string { + return nil + }, + expectedError: `Required flag "input-path" not set`, + }, + { + desc: "empty input", + input: "", + expectedError: "no repository information found", + }, + { + desc: "invalid JSON", + input: "@hashed/01/23/01234567890123456789.git", + expectedOutput: "invalid character '@' looking for beginning of value", + expectedError: invalidEntryErr, + }, + { + desc: "missing path", + input: `{"virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, + expectedOutput: `"repository" is a required parameter`, + expectedError: invalidEntryErr, + }, + { + desc: "invalid virtual storage", + input: `{"virtual_storage":"foo","relative_path":"bar","authoritative_storage":"gitaly-1"}`, + expectedOutput: `virtual storage "foo" not found`, + expectedError: invalidEntryErr, + }, + { + desc: "repo does not exist", + input: `{"relative_path":"not_a_repo","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, + expectedOutput: "not a valid git repository", + expectedError: invalidEntryErr, + }, + { + desc: "duplicate path", + input: `{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"} +{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"}`, + expectedOutput: "duplicate entries for relative_path", + expectedError: invalidEntryErr, + }, + { + desc: "repo is already tracked", + input: `{"relative_path":"already_tracked","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, + expectedOutput: "repository is already tracked by Praefect", + expectedError: invalidEntryErr, + trackedPath: "already_tracked", + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + tempDir := testhelper.TempDir(t) + inputPath := filepath.Join(tempDir, "input_file") + f, err := os.Create(inputPath) + require.NoError(t, err) + _, err = f.Write([]byte(tc.input)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + if tc.trackedPath != "" { + require.NoError(t, trackRepo(tc.trackedPath)) + } + + args := []string{"-replicate-immediately", "-input-path=" + inputPath} + if tc.args != nil { + args = tc.args(inputPath) + } + stdout, stderr, err := runApp(append([]string{"-config", confPath, trackRepositoriesCmdName}, args...)) + assert.Empty(t, stderr) + require.Error(t, err) + + if tc.expectedOutput != "" { + require.Contains(t, stdout, tc.expectedOutput) + } + require.Contains(t, err.Error(), tc.expectedError) + }) + } + }) } diff --git a/internal/cli/praefect/subcmd_track_repository_test.go b/internal/cli/praefect/subcmd_track_repository_test.go index d721e51d9..d8e11e6e2 100644 --- a/internal/cli/praefect/subcmd_track_repository_test.go +++ b/internal/cli/praefect/subcmd_track_repository_test.go @@ -1,15 +1,12 @@ package praefect import ( - "bytes" - "io" "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/client" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" @@ -104,27 +101,6 @@ func TestTrackRepositorySubcommand(t *testing.T) { true, )) - runCmd := func(t *testing.T, args []string) (string, error) { - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newTrackRepositoryCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err := app.Run(append([]string{progname, trackRepositoryCmdName}, args...)) - return stdout.String(), err - } - t.Run("fails", func(t *testing.T) { for _, tc := range []struct { name string @@ -171,7 +147,8 @@ func TestTrackRepositorySubcommand(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - _, err := runCmd(t, tc.args) + _, stderr, err := runApp(append([]string{"-config", confPath, trackRepositoryCmdName}, tc.args...)) + assert.Empty(t, stderr) require.EqualError(t, err, tc.errorMsg) }) } @@ -245,7 +222,8 @@ func TestTrackRepositorySubcommand(t *testing.T) { if tc.replicateImmediately { args = append(args, "-replicate-immediately") } - stdout, err := runCmd(t, args) + stdout, stderr, err := runApp(append([]string{"-config", confPath, trackRepositoryCmdName}, args...)) + assert.Empty(t, stderr) require.NoError(t, err) as := datastore.NewAssignmentStore(db, conf.StorageNames()) @@ -288,7 +266,9 @@ func TestTrackRepositorySubcommand(t *testing.T) { require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, relativePath)) require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, relativePath)) - _, err := runCmd(t, []string{ + _, _, err := runApp([]string{ + "-config", confPath, + trackRepositoryCmdName, "-virtual-storage", virtualStorageName, "-repository", relativePath, "-authoritative-storage", authoritativeStorage, @@ -296,12 +276,15 @@ func TestTrackRepositorySubcommand(t *testing.T) { require.NoError(t, err) // running the command twice means we try creating the replication event // again, which should log the duplicate but not break the flow. - stdout, err := runCmd(t, []string{ + stdout, stderr, err := runApp([]string{ + "-config", confPath, + trackRepositoryCmdName, "-virtual-storage", virtualStorageName, "-repository", relativePath, "-authoritative-storage", authoritativeStorage, }) require.NoError(t, err) + assert.Empty(t, stderr) assert.Contains(t, stdout, "replication event queue already has similar entry: replication event \"\" -> \"praefect\" -> \"gitaly-2\" -> \"path/to/test/repo_3\" already exists.") }) } diff --git a/internal/cli/praefect/subcmd_verify_test.go b/internal/cli/praefect/subcmd_verify_test.go index 901579f17..9746f4d05 100644 --- a/internal/cli/praefect/subcmd_verify_test.go +++ b/internal/cli/praefect/subcmd_verify_test.go @@ -1,12 +1,11 @@ package praefect import ( - "bytes" "errors" "fmt" - "io" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" @@ -207,29 +206,14 @@ func TestVerifySubcommand(t *testing.T) { } confPath := writeConfigToFile(t, conf) - var stdout bytes.Buffer - app := cli.App{ - Reader: bytes.NewReader(nil), - Writer: &stdout, - ErrWriter: io.Discard, - HideHelpCommand: true, - Commands: []*cli.Command{ - newVerifyCommand(), - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "config", - Value: confPath, - }, - }, - } - err = app.Run(append([]string{progname, verifyCmdName}, tc.args...)) + stdout, stderr, err := runApp(append([]string{"-config", confPath, verifyCmdName}, tc.args...)) + assert.Empty(t, stderr) testhelper.RequireGrpcError(t, tc.error, err) if tc.error != nil { return } - require.Equal(t, fmt.Sprintf("%d replicas marked unverified\n", tc.replicasMarked), stdout.String()) + require.Equal(t, fmt.Sprintf("%d replicas marked unverified\n", tc.replicasMarked), stdout) actualState := state{} rows, err := db.QueryContext(ctx, ` diff --git a/internal/featureflag/ff_git_v241.go b/internal/featureflag/ff_git_v241.go new file mode 100644 index 000000000..703aaf162 --- /dev/null +++ b/internal/featureflag/ff_git_v241.go @@ -0,0 +1,9 @@ +package featureflag + +// GitV241 enables the use of Git v2.41. +var GitV241 = NewFeatureFlag( + "git_v241", + "v16.1.0", + "https://gitlab.com/gitlab-org/gitaly/-/issues/5336", + false, +) diff --git a/internal/git/execution_environment.go b/internal/git/execution_environment.go index 504dd7c68..2948030f0 100644 --- a/internal/git/execution_environment.go +++ b/internal/git/execution_environment.go @@ -26,6 +26,12 @@ var ( // case `IsEnabled()` returns `false` though. defaultExecutionEnvironmentConstructors = []ExecutionEnvironmentConstructor{ BundledGitEnvironmentConstructor{ + Suffix: "-v2.41", + FeatureFlags: []featureflag.FeatureFlag{ + featureflag.GitV241, + }, + }, + BundledGitEnvironmentConstructor{ Suffix: "-v2.40", }, DistributedGitEnvironmentConstructor{}, diff --git a/internal/gitaly/service/internalgitaly/backup_repos.go b/internal/gitaly/service/internalgitaly/backup_repos.go index ff43b5b5e..7fa56d583 100644 --- a/internal/gitaly/service/internalgitaly/backup_repos.go +++ b/internal/gitaly/service/internalgitaly/backup_repos.go @@ -37,17 +37,14 @@ func (s server) BackupRepos(stream gitalypb.InternalGitaly_BackupReposServer) er return structerr.NewInvalidArgument("backup repos: resolve locator: %w", err) } - manager := backup.NewManagerLocal(sink, locator, s.locator, s.gitCmdFactory, s.catfileCache, backupID) + manager := backup.NewManagerLocal(sink, locator, s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, backupID) pipeline := backup.NewLoggingPipeline(ctxlogrus.Extract(ctx)) for { for _, repo := range request.GetRepositories() { pipeline.Handle(ctx, backup.NewCreateCommand( manager, - // ServerInfo will be removed once restore methods are added to - // backup.Repository. Even though it is unused it must be - // non-zero so that storage.ExtractGitalyServer is not called. - storage.ServerInfo{Address: "unused"}, + storage.ServerInfo{}, repo, false, )) diff --git a/internal/gitaly/service/internalgitaly/backup_repos_test.go b/internal/gitaly/service/internalgitaly/backup_repos_test.go index 6b00d4da0..8adb11524 100644 --- a/internal/gitaly/service/internalgitaly/backup_repos_test.go +++ b/internal/gitaly/service/internalgitaly/backup_repos_test.go @@ -12,6 +12,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" @@ -120,11 +121,14 @@ func TestServerBackupRepos(t *testing.T) { catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) + txManager := transaction.NewTrackingManager() + srv := NewServer( cfg.Storages, config.NewLocator(cfg), gittest.NewCommandFactory(t, cfg), catfileCache, + txManager, ) client := setupInternalGitalyService(t, cfg, srv) diff --git a/internal/gitaly/service/internalgitaly/server.go b/internal/gitaly/service/internalgitaly/server.go index d0ce88352..41231fcb1 100644 --- a/internal/gitaly/service/internalgitaly/server.go +++ b/internal/gitaly/service/internalgitaly/server.go @@ -5,6 +5,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -14,6 +15,7 @@ type server struct { locator storage.Locator gitCmdFactory git.CommandFactory catfileCache catfile.Cache + txManager transaction.Manager } // NewServer return an instance of the Gitaly service. @@ -22,11 +24,13 @@ func NewServer( locator storage.Locator, gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, + txManager transaction.Manager, ) gitalypb.InternalGitalyServer { return &server{ storages: storages, locator: locator, gitCmdFactory: gitCmdFactory, catfileCache: catfileCache, + txManager: txManager, } } diff --git a/internal/gitaly/service/internalgitaly/walkrepos_test.go b/internal/gitaly/service/internalgitaly/walkrepos_test.go index de4e5b23d..a1bfa769a 100644 --- a/internal/gitaly/service/internalgitaly/walkrepos_test.go +++ b/internal/gitaly/service/internalgitaly/walkrepos_test.go @@ -12,6 +12,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" @@ -74,6 +75,8 @@ func TestWalkRepos(t *testing.T) { catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) + txManager := transaction.NewTrackingManager() + // to test a directory being deleted during a walk, we must delete a directory after // the file walk has started. To achieve that, we wrap the server to pass down a wrapped // stream that allows us to hook in to stream responses. We then delete 'b' when @@ -84,6 +87,7 @@ func TestWalkRepos(t *testing.T) { config.NewLocator(cfg), gittest.NewCommandFactory(t, cfg), catfileCache, + txManager, ) wsrv := &serverWrapper{ srv, diff --git a/internal/gitaly/service/setup/register.go b/internal/gitaly/service/setup/register.go index 50f0482c1..0a0177cfe 100644 --- a/internal/gitaly/service/setup/register.go +++ b/internal/gitaly/service/setup/register.go @@ -149,6 +149,7 @@ func RegisterAll(srv *grpc.Server, deps *service.Dependencies) { deps.GetLocator(), deps.GetGitCmdFactory(), deps.GetCatfileCache(), + deps.GetTxManager(), )) healthpb.RegisterHealthServer(srv, health.NewServer()) diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go index af54831d9..c6132a2f6 100644 --- a/internal/testhelper/testhelper.go +++ b/internal/testhelper/testhelper.go @@ -195,6 +195,9 @@ func ContextWithoutCancel(opts ...ContextOpt) context.Context { // Randomly enable the use of the catfile cache in localrepo.ReadObject. ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.LocalrepoReadObjectCached, rnd.Int()%2 == 0) + // Randomly enable either Git v2.40 or Git v2.41. + ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.GitV241, rnd.Int()%2 == 0) + for _, opt := range opts { ctx = opt(ctx) } diff --git a/tools/goimports/go.mod b/tools/goimports/go.mod index ae7ae49e8..35cdca40f 100644 --- a/tools/goimports/go.mod +++ b/tools/goimports/go.mod @@ -2,7 +2,7 @@ module gitlab.com/gitlab-org/gitaly/tools/goimports go 1.19 -require golang.org/x/tools v0.9.1 +require golang.org/x/tools v0.9.3 require ( golang.org/x/mod v0.10.0 // indirect diff --git a/tools/goimports/go.sum b/tools/goimports/go.sum index 7995eec85..5a2e60559 100644 --- a/tools/goimports/go.sum +++ b/tools/goimports/go.sum @@ -2,5 +2,5 @@ golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= diff --git a/tools/golangci-lint/go.mod b/tools/golangci-lint/go.mod index adc35efc7..a3ddbf697 100644 --- a/tools/golangci-lint/go.mod +++ b/tools/golangci-lint/go.mod @@ -6,7 +6,7 @@ require github.com/golangci/golangci-lint v1.52.2 require ( github.com/spf13/viper v1.12.0 - golang.org/x/tools v0.8.0 + golang.org/x/tools v0.9.3 ) require ( @@ -166,8 +166,8 @@ require ( golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.7.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/tools/golangci-lint/go.sum b/tools/golangci-lint/go.sum index d4d9cb469..a7d05da6b 100644 --- a/tools/golangci-lint/go.sum +++ b/tools/golangci-lint/go.sum @@ -649,7 +649,7 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -672,8 +672,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -733,8 +734,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -828,8 +829,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= |