Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitaly.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-06-05 13:05:28 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-06-05 13:05:28 +0300
commitbab197494061f0ec76cebea552dfdf85bd75c110 (patch)
treed7c62d3e46e35414e4bc20eaf4b574ef9b7ffdba
parenta7927b16e6a77aee8e2f1ff26f827d98c633aa15 (diff)
parenta69cf20ceb11365f64d663a8d6028ceb1b79a09c (diff)
Automatic merge of gitlab-org/gitaly master
-rw-r--r--Makefile12
-rw-r--r--cmd/gitaly-backup/restore_test.go2
-rw-r--r--internal/backup/backup.go130
-rw-r--r--internal/backup/backup_test.go461
-rw-r--r--internal/backup/repository.go157
-rw-r--r--internal/cli/praefect/main.go50
-rw-r--r--internal/cli/praefect/serve.go24
-rw-r--r--internal/cli/praefect/subcmd.go55
-rw-r--r--internal/cli/praefect/subcmd_accept_dataloss_test.go21
-rw-r--r--internal/cli/praefect/subcmd_check_test.go45
-rw-r--r--internal/cli/praefect/subcmd_dataloss_test.go25
-rw-r--r--internal/cli/praefect/subcmd_dial_nodes_test.go25
-rw-r--r--internal/cli/praefect/subcmd_list_storages_test.go32
-rw-r--r--internal/cli/praefect/subcmd_list_untracked_repositories_test.go40
-rw-r--r--internal/cli/praefect/subcmd_metadata_test.go24
-rw-r--r--internal/cli/praefect/subcmd_remove_repository_test.go44
-rw-r--r--internal/cli/praefect/subcmd_set_replication_factor_test.go24
-rw-r--r--internal/cli/praefect/subcmd_sql_migrate.go2
-rw-r--r--internal/cli/praefect/subcmd_sql_migrate_down_test.go25
-rw-r--r--internal/cli/praefect/subcmd_sql_migrate_status_test.go24
-rw-r--r--internal/cli/praefect/subcmd_sql_migrate_test.go25
-rw-r--r--internal/cli/praefect/subcmd_sql_ping.go2
-rw-r--r--internal/cli/praefect/subcmd_sql_ping_test.go25
-rw-r--r--internal/cli/praefect/subcmd_test.go11
-rw-r--r--internal/cli/praefect/subcmd_track_repositories.go142
-rw-r--r--internal/cli/praefect/subcmd_track_repositories_test.go397
-rw-r--r--internal/cli/praefect/subcmd_track_repository_test.go39
-rw-r--r--internal/cli/praefect/subcmd_verify_test.go24
-rw-r--r--internal/featureflag/ff_git_v241.go9
-rw-r--r--internal/git/execution_environment.go6
-rw-r--r--internal/gitaly/service/internalgitaly/backup_repos.go7
-rw-r--r--internal/gitaly/service/internalgitaly/backup_repos_test.go4
-rw-r--r--internal/gitaly/service/internalgitaly/server.go4
-rw-r--r--internal/gitaly/service/internalgitaly/walkrepos_test.go4
-rw-r--r--internal/gitaly/service/setup/register.go1
-rw-r--r--internal/testhelper/testhelper.go3
-rw-r--r--tools/goimports/go.mod2
-rw-r--r--tools/goimports/go.sum4
-rw-r--r--tools/golangci-lint/go.mod6
-rw-r--r--tools/golangci-lint/go.sum13
40 files changed, 890 insertions, 1060 deletions
diff --git a/Makefile b/Makefile
index 8d13402f5..ac3258483 100644
--- a/Makefile
+++ b/Makefile
@@ -134,6 +134,8 @@ GIT_EXECUTABLES += git-http-backend
GIT_VERSION ?=
## The Git version used for bundled Git v2.40.
GIT_VERSION_2_40 ?= v2.40.1.gl2
+## The Git version used for bundled Git v2.41.
+GIT_VERSION_2_41 ?= v2.41.0
## Skip overriding the Git version and instead use the Git version as specified
## in the Git sources. This is required when building Git from a version that
@@ -332,14 +334,16 @@ install: build
.PHONY: build-bundled-git
## Build bundled Git binaries.
-build-bundled-git: build-bundled-git-v2.40
+build-bundled-git: build-bundled-git-v2.40 build-bundled-git-v2.41
build-bundled-git-v2.40: $(patsubst %,${BUILD_DIR}/bin/gitaly-%-v2.40,${GIT_EXECUTABLES})
+build-bundled-git-v2.41: $(patsubst %,${BUILD_DIR}/bin/gitaly-%-v2.41,${GIT_EXECUTABLES})
.PHONY: install-bundled-git
## Install bundled Git binaries. The target directory can be modified by
## setting PREFIX and DESTDIR.
-install-bundled-git: install-bundled-git-v2.40
+install-bundled-git: install-bundled-git-v2.40 install-bundled-git-v2.41
install-bundled-git-v2.40: $(patsubst %,${INSTALL_DEST_DIR}/gitaly-%-v2.40,${GIT_EXECUTABLES})
+install-bundled-git-v2.41: $(patsubst %,${INSTALL_DEST_DIR}/gitaly-%-v2.41,${GIT_EXECUTABLES})
ifdef WITH_BUNDLED_GIT
build: build-bundled-git
@@ -565,6 +569,10 @@ ${BUILD_DIR}/bin/gitaly-%-v2.40: override GIT_VERSION = ${GIT_VERSION_2_40}
${BUILD_DIR}/bin/gitaly-%-v2.40: ${DEPENDENCY_DIR}/git-v2.40/% | ${BUILD_DIR}/bin
${Q}install $< $@
+${BUILD_DIR}/bin/gitaly-%-v2.41: override GIT_VERSION = ${GIT_VERSION_2_41}
+${BUILD_DIR}/bin/gitaly-%-v2.41: ${DEPENDENCY_DIR}/git-v2.41/% | ${BUILD_DIR}/bin
+ ${Q}install $< $@
+
${BUILD_DIR}/bin/%: ${BUILD_DIR}/intermediate/% | ${BUILD_DIR}/bin
@ # To compute a unique and deterministic value for GNU build-id, we use an
@ # intermediate binary which has a fixed build ID of "TEMP_GITALY_BUILD_ID",
diff --git a/cmd/gitaly-backup/restore_test.go b/cmd/gitaly-backup/restore_test.go
index 037078310..3492c6c15 100644
--- a/cmd/gitaly-backup/restore_test.go
+++ b/cmd/gitaly-backup/restore_test.go
@@ -76,7 +76,7 @@ func TestRestoreSubcommand(t *testing.T) {
require.NoError(t, fs.Parse([]string{"-path", path, "-remove-all-repositories", existingRepo.StorageName}))
require.EqualError(t,
cmd.Run(ctx, &stdin, io.Discard),
- "restore: pipeline: 1 failures encountered:\n - invalid: manager: remove repository: could not dial source: invalid connection string: \"invalid\"\n")
+ "restore: pipeline: 1 failures encountered:\n - invalid: manager: could not dial source: invalid connection string: \"invalid\"\n")
require.NoDirExists(t, existRepoPath)
diff --git a/internal/backup/backup.go b/internal/backup/backup.go
index 60fa0cd65..85e9a5055 100644
--- a/internal/backup/backup.go
+++ b/internal/backup/backup.go
@@ -12,10 +12,8 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
- "gitlab.com/gitlab-org/gitaly/v16/streamio"
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
@@ -83,6 +81,16 @@ type Repository interface {
GetCustomHooks(ctx context.Context, out io.Writer) error
// CreateBundle fetches a bundle that contains refs matching patterns.
CreateBundle(ctx context.Context, out io.Writer, patterns io.Reader) error
+ // Remove removes the repository. Does not return an error if the
+ // repository cannot be found.
+ Remove(ctx context.Context) error
+ // Create creates the repository.
+ Create(ctx context.Context) error
+ // FetchBundle fetches references from a bundle. Refs will be mirrored to
+ // the repository.
+ FetchBundle(ctx context.Context, reader io.Reader) error
+ // SetCustomHooks updates the custom hooks for the repository.
+ SetCustomHooks(ctx context.Context, reader io.Reader) error
}
// ResolveLocator returns a locator implementation based on a locator identifier.
@@ -125,6 +133,10 @@ func NewManager(sink Sink, locator Locator, pool *client.Pool, backupID string)
locator: locator,
backupID: backupID,
repositoryFactory: func(ctx context.Context, repo *gitalypb.Repository, server storage.ServerInfo) (Repository, error) {
+ if err := setContextServerInfo(ctx, &server, repo.GetStorageName()); err != nil {
+ return nil, err
+ }
+
conn, err := pool.Dial(ctx, server.Address, server.Token)
if err != nil {
return nil, err
@@ -136,7 +148,15 @@ func NewManager(sink Sink, locator Locator, pool *client.Pool, backupID string)
}
// NewManagerLocal creates and returns a *Manager instance for operating on local repositories.
-func NewManagerLocal(sink Sink, locator Locator, storageLocator storage.Locator, gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, backupID string) *Manager {
+func NewManagerLocal(
+ sink Sink,
+ locator Locator,
+ storageLocator storage.Locator,
+ gitCmdFactory git.CommandFactory,
+ catfileCache catfile.Cache,
+ txManager transaction.Manager,
+ backupID string,
+) *Manager {
return &Manager{
sink: sink,
conns: nil, // Will be removed once the restore operations are part of the Repository interface.
@@ -145,7 +165,7 @@ func NewManagerLocal(sink Sink, locator Locator, storageLocator storage.Locator,
repositoryFactory: func(ctx context.Context, repo *gitalypb.Repository, server storage.ServerInfo) (Repository, error) {
localRepo := localrepo.New(storageLocator, gitCmdFactory, catfileCache, repo)
- return newLocalRepository(storageLocator, localRepo), nil
+ return newLocalRepository(storageLocator, gitCmdFactory, txManager, localRepo), nil
},
}
}
@@ -185,10 +205,6 @@ type CreateRequest struct {
// Create creates a repository backup.
func (mgr *Manager) Create(ctx context.Context, req *CreateRequest) error {
- if err := setContextServerInfo(ctx, &req.Server, req.Repository.GetStorageName()); err != nil {
- return fmt.Errorf("manager: %w", err)
- }
-
repo, err := mgr.repositoryFactory(ctx, req.Repository, req.Server)
if err != nil {
return fmt.Errorf("manager: %w", err)
@@ -241,11 +257,12 @@ type RestoreRequest struct {
// Restore restores a repository from a backup.
func (mgr *Manager) Restore(ctx context.Context, req *RestoreRequest) error {
- if err := setContextServerInfo(ctx, &req.Server, req.Repository.GetStorageName()); err != nil {
+ repo, err := mgr.repositoryFactory(ctx, req.Repository, req.Server)
+ if err != nil {
return fmt.Errorf("manager: %w", err)
}
- if err := mgr.removeRepository(ctx, req.Server, req.Repository); err != nil {
+ if err := repo.Remove(ctx); err != nil {
return fmt.Errorf("manager: %w", err)
}
@@ -254,12 +271,12 @@ func (mgr *Manager) Restore(ctx context.Context, req *RestoreRequest) error {
return fmt.Errorf("manager: %w", err)
}
- if err := mgr.createRepository(ctx, req.Server, req.Repository); err != nil {
+ if err := repo.Create(ctx); err != nil {
return fmt.Errorf("manager: %w", err)
}
for _, step := range backup.Steps {
- if err := mgr.restoreBundle(ctx, step.BundlePath, req.Server, req.Repository); err != nil {
+ if err := mgr.restoreBundle(ctx, repo, step.BundlePath); err != nil {
if step.SkippableOnNotFound && errors.Is(err, ErrDoesntExist) {
// For compatibility with existing backups we need to make sure the
// repository exists even if there's no bundle for project
@@ -271,14 +288,14 @@ func (mgr *Manager) Restore(ctx context.Context, req *RestoreRequest) error {
return nil
}
- if err := mgr.removeRepository(ctx, req.Server, req.Repository); err != nil {
+ if err := repo.Remove(ctx); err != nil {
return fmt.Errorf("manager: remove on skipped: %w", err)
}
return fmt.Errorf("manager: %w: %s", ErrSkipped, err.Error())
}
}
- if err := mgr.restoreCustomHooks(ctx, step.CustomHooksPath, req.Server, req.Repository); err != nil {
+ if err := mgr.restoreCustomHooks(ctx, repo, step.CustomHooksPath); err != nil {
return fmt.Errorf("manager: %w", err)
}
}
@@ -300,32 +317,6 @@ func setContextServerInfo(ctx context.Context, server *storage.ServerInfo, stora
return nil
}
-func (mgr *Manager) removeRepository(ctx context.Context, server storage.ServerInfo, repo *gitalypb.Repository) error {
- repoClient, err := mgr.newRepoClient(ctx, server)
- if err != nil {
- return fmt.Errorf("remove repository: %w", err)
- }
- _, err = repoClient.RemoveRepository(ctx, &gitalypb.RemoveRepositoryRequest{Repository: repo})
- switch {
- case status.Code(err) == codes.NotFound:
- return nil
- case err != nil:
- return fmt.Errorf("remove repository: %w", err)
- }
- return nil
-}
-
-func (mgr *Manager) createRepository(ctx context.Context, server storage.ServerInfo, repo *gitalypb.Repository) error {
- repoClient, err := mgr.newRepoClient(ctx, server)
- if err != nil {
- return fmt.Errorf("create repository: %w", err)
- }
- if _, err := repoClient.CreateRepository(ctx, &gitalypb.CreateRepositoryRequest{Repository: repo}); err != nil {
- return fmt.Errorf("create repository: %w", err)
- }
- return nil
-}
-
func (mgr *Manager) writeBundle(ctx context.Context, repo Repository, step *Step, refs []git.Reference) (returnErr error) {
negatedRefs, err := mgr.negatedKnownRefs(ctx, step)
if err != nil {
@@ -434,37 +425,14 @@ func (s *createBundleFromRefListSender) Send() error {
return s.stream.Send(&s.chunk)
}
-func (mgr *Manager) restoreBundle(ctx context.Context, path string, server storage.ServerInfo, repo *gitalypb.Repository) error {
+func (mgr *Manager) restoreBundle(ctx context.Context, repo Repository, path string) error {
reader, err := mgr.sink.GetReader(ctx, path)
if err != nil {
- return fmt.Errorf("restore bundle: %w", err)
- }
- defer reader.Close()
-
- repoClient, err := mgr.newRepoClient(ctx, server)
- if err != nil {
return fmt.Errorf("restore bundle: %q: %w", path, err)
}
- stream, err := repoClient.FetchBundle(ctx)
- if err != nil {
- return fmt.Errorf("restore bundle: %q: %w", path, err)
- }
- request := &gitalypb.FetchBundleRequest{Repository: repo, UpdateHead: true}
- bundle := streamio.NewWriter(func(p []byte) error {
- request.Data = p
- if err := stream.Send(request); err != nil {
- return err
- }
-
- // Only set `Repository` on the first `Send` of the stream
- request = &gitalypb.FetchBundleRequest{}
+ defer reader.Close()
- return nil
- })
- if _, err := io.Copy(bundle, reader); err != nil {
- return fmt.Errorf("restore bundle: %q: %w", path, err)
- }
- if _, err = stream.CloseAndRecv(); err != nil {
+ if err := repo.FetchBundle(ctx, reader); err != nil {
return fmt.Errorf("restore bundle: %q: %w", path, err)
}
return nil
@@ -485,7 +453,7 @@ func (mgr *Manager) writeCustomHooks(ctx context.Context, repo Repository, path
return nil
}
-func (mgr *Manager) restoreCustomHooks(ctx context.Context, path string, server storage.ServerInfo, repo *gitalypb.Repository) error {
+func (mgr *Manager) restoreCustomHooks(ctx context.Context, repo Repository, path string) error {
reader, err := mgr.sink.GetReader(ctx, path)
if err != nil {
if errors.Is(err, ErrDoesntExist) {
@@ -495,31 +463,7 @@ func (mgr *Manager) restoreCustomHooks(ctx context.Context, path string, server
}
defer reader.Close()
- repoClient, err := mgr.newRepoClient(ctx, server)
- if err != nil {
- return fmt.Errorf("restore custom hooks, %q: %w", path, err)
- }
- stream, err := repoClient.SetCustomHooks(ctx)
- if err != nil {
- return fmt.Errorf("restore custom hooks, %q: %w", path, err)
- }
-
- request := &gitalypb.SetCustomHooksRequest{Repository: repo}
- bundle := streamio.NewWriter(func(p []byte) error {
- request.Data = p
- if err := stream.Send(request); err != nil {
- return err
- }
-
- // Only set `Repository` on the first `Send` of the stream
- request = &gitalypb.SetCustomHooksRequest{}
-
- return nil
- })
- if _, err := io.Copy(bundle, reader); err != nil {
- return fmt.Errorf("restore custom hooks, %q: %w", path, err)
- }
- if _, err = stream.CloseAndRecv(); err != nil {
+ if err := repo.SetCustomHooks(ctx, reader); err != nil {
return fmt.Errorf("restore custom hooks, %q: %w", path, err)
}
return nil
diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go
index 1341f767c..2472da0a5 100644
--- a/internal/backup/backup_test.go
+++ b/internal/backup/backup_test.go
@@ -20,6 +20,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
@@ -94,8 +95,10 @@ func TestManager_Create(t *testing.T) {
storageLocator := config.NewLocator(cfg)
gitCmdFactory := gittest.NewCommandFactory(tb, cfg)
catfileCache := catfile.NewCache(cfg)
+ tb.Cleanup(catfileCache.Stop)
+ txManager := transaction.NewTrackingManager()
- return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, backupID)
+ return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, backupID)
},
},
} {
@@ -243,8 +246,10 @@ func TestManager_Create_incremental(t *testing.T) {
storageLocator := config.NewLocator(cfg)
gitCmdFactory := gittest.NewCommandFactory(tb, cfg)
catfileCache := catfile.NewCache(cfg)
+ tb.Cleanup(catfileCache.Stop)
+ txManager := transaction.NewTrackingManager()
- return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, backupID)
+ return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, backupID)
},
},
} {
@@ -355,232 +360,270 @@ func TestManager_Create_incremental(t *testing.T) {
func TestManager_Restore(t *testing.T) {
t.Parallel()
- ctx := testhelper.Context(t)
+ const backupID = "abc123"
+
cfg := testcfg.Build(t)
testcfg.BuildGitalyHooks(t, cfg)
-
cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll)
- cc, err := client.Dial(cfg.SocketPath, nil)
- require.NoError(t, err)
- defer testhelper.MustClose(t, cc)
-
- repoClient := gitalypb.NewRepositoryServiceClient(cc)
-
- _, repoPath := gittest.CreateRepository(t, ctx, cfg)
- commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
- gittest.WriteTag(t, cfg, repoPath, "v1.0.0", commitID.Revision())
- repoChecksum := gittest.ChecksumRepo(t, cfg, repoPath)
-
- backupRoot := testhelper.TempDir(t)
-
- for _, tc := range []struct {
- desc string
- locators []string
- setup func(tb testing.TB) (*gitalypb.Repository, *git.Checksum)
- alwaysCreate bool
- expectExists bool
- expectedPaths []string
- expectedErrAs error
+ for _, managerTC := range []struct {
+ desc string
+ setup func(t testing.TB, sink backup.Sink, locator backup.Locator) *backup.Manager
}{
{
- desc: "existing repo, without hooks",
- locators: []string{"legacy", "pointer"},
- setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
- repo, _ := gittest.CreateRepository(t, ctx, cfg)
-
- relativePath := stripRelativePath(tb, repo)
- require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir))
- bundlePath := filepath.Join(backupRoot, relativePath+".bundle")
- gittest.BundleRepo(tb, cfg, repoPath, bundlePath)
+ desc: "RPC manager",
+ setup: func(tb testing.TB, sink backup.Sink, locator backup.Locator) *backup.Manager {
+ pool := client.NewPool()
+ tb.Cleanup(func() {
+ testhelper.MustClose(tb, pool)
+ })
- return repo, repoChecksum
- },
- expectExists: true,
- },
- {
- desc: "existing repo, with hooks",
- locators: []string{"legacy", "pointer"},
- setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
- repo, _ := gittest.CreateRepository(t, ctx, cfg)
-
- relativePath := stripRelativePath(tb, repo)
- bundlePath := filepath.Join(backupRoot, relativePath+".bundle")
- customHooksPath := filepath.Join(backupRoot, relativePath, "custom_hooks.tar")
- require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir))
- gittest.BundleRepo(tb, cfg, repoPath, bundlePath)
- testhelper.CopyFile(tb, mustCreateCustomHooksArchive(t, ctx), customHooksPath)
-
- return repo, repoChecksum
- },
- expectedPaths: []string{
- "custom_hooks/pre-commit.sample",
- "custom_hooks/prepare-commit-msg.sample",
- "custom_hooks/pre-push.sample",
- },
- expectExists: true,
- },
- {
- desc: "missing bundle",
- locators: []string{"legacy", "pointer"},
- setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
- repo, _ := gittest.CreateRepository(t, ctx, cfg)
- return repo, nil
- },
- expectedErrAs: backup.ErrSkipped,
- },
- {
- desc: "missing bundle, always create",
- locators: []string{"legacy", "pointer"},
- setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
- repo, _ := gittest.CreateRepository(t, ctx, cfg)
- return repo, new(git.Checksum)
+ return backup.NewManager(sink, locator, pool, backupID)
},
- alwaysCreate: true,
- expectExists: true,
},
{
- desc: "nonexistent repo",
- locators: []string{"legacy", "pointer"},
- setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
- repo := &gitalypb.Repository{
- StorageName: "default",
- RelativePath: gittest.NewRepositoryName(tb),
+ desc: "Local manager",
+ setup: func(tb testing.TB, sink backup.Sink, locator backup.Locator) *backup.Manager {
+ if testhelper.IsPraefectEnabled() {
+ tb.Skip("local backup manager expects to operate on the local filesystem so cannot operate through praefect")
}
- relativePath := stripRelativePath(tb, repo)
- require.NoError(tb, os.MkdirAll(filepath.Dir(filepath.Join(backupRoot, relativePath)), perm.PublicDir))
- bundlePath := filepath.Join(backupRoot, relativePath+".bundle")
- gittest.BundleRepo(tb, cfg, repoPath, bundlePath)
+ storageLocator := config.NewLocator(cfg)
+ gitCmdFactory := gittest.NewCommandFactory(tb, cfg)
+ catfileCache := catfile.NewCache(cfg)
+ tb.Cleanup(catfileCache.Stop)
+ txManager := transaction.NewTrackingManager()
- return repo, repoChecksum
+ return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, backupID)
},
- expectExists: true,
- },
- {
- desc: "single incremental",
- locators: []string{"pointer"},
- setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
- const backupID = "abc123"
- repo, _ := gittest.CreateRepository(t, ctx, cfg)
- repoBackupPath := joinBackupPath(tb, backupRoot, repo)
- backupPath := filepath.Join(repoBackupPath, backupID)
- require.NoError(tb, os.MkdirAll(backupPath, perm.PublicDir))
- require.NoError(tb, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), perm.PublicFile))
- require.NoError(tb, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("001"), perm.PublicFile))
- bundlePath := filepath.Join(backupPath, "001.bundle")
- gittest.BundleRepo(tb, cfg, repoPath, bundlePath)
-
- return repo, repoChecksum
- },
- expectExists: true,
- },
- {
- desc: "many incrementals",
- locators: []string{"pointer"},
- setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
- const backupID = "abc123"
-
- _, expectedRepoPath := gittest.CreateRepository(t, ctx, cfg)
-
- repo, _ := gittest.CreateRepository(t, ctx, cfg)
- repoBackupPath := joinBackupPath(tb, backupRoot, repo)
- backupPath := filepath.Join(repoBackupPath, backupID)
- require.NoError(tb, os.MkdirAll(backupPath, perm.PublicDir))
- require.NoError(tb, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), perm.PublicFile))
- require.NoError(tb, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("002"), perm.PublicFile))
-
- root := gittest.WriteCommit(tb, cfg, expectedRepoPath,
- gittest.WithBranch("master"),
- )
- master1 := gittest.WriteCommit(tb, cfg, expectedRepoPath,
- gittest.WithBranch("master"),
- gittest.WithParents(root),
- )
- other := gittest.WriteCommit(tb, cfg, expectedRepoPath,
- gittest.WithBranch("other"),
- gittest.WithParents(root),
- )
- gittest.Exec(tb, cfg, "-C", expectedRepoPath, "symbolic-ref", "HEAD", "refs/heads/master")
- bundlePath1 := filepath.Join(backupPath, "001.bundle")
- gittest.Exec(tb, cfg, "-C", expectedRepoPath, "bundle", "create", bundlePath1,
- "HEAD",
- "refs/heads/master",
- "refs/heads/other",
- )
-
- master2 := gittest.WriteCommit(tb, cfg, expectedRepoPath,
- gittest.WithBranch("master"),
- gittest.WithParents(master1),
- )
- bundlePath2 := filepath.Join(backupPath, "002.bundle")
- gittest.Exec(tb, cfg, "-C", expectedRepoPath, "bundle", "create", bundlePath2,
- "HEAD",
- "^"+master1.String(),
- "^"+other.String(),
- "refs/heads/master",
- "refs/heads/other",
- )
-
- checksum := new(git.Checksum)
- checksum.Add(git.NewReference("HEAD", master2.String()))
- checksum.Add(git.NewReference("refs/heads/master", master2.String()))
- checksum.Add(git.NewReference("refs/heads/other", other.String()))
-
- return repo, checksum
- },
- expectExists: true,
},
} {
- t.Run(tc.desc, func(t *testing.T) {
- require.GreaterOrEqual(t, len(tc.locators), 1, "each test case must specify a locator")
-
- for _, locatorName := range tc.locators {
- t.Run(locatorName, func(t *testing.T) {
- repo, expectedChecksum := tc.setup(t)
-
- pool := client.NewPool()
- defer testhelper.MustClose(t, pool)
-
- sink := backup.NewFilesystemSink(backupRoot)
- locator, err := backup.ResolveLocator(locatorName, sink)
- require.NoError(t, err)
+ managerTC := managerTC
+
+ t.Run(managerTC.desc, func(t *testing.T) {
+ t.Parallel()
+
+ ctx := testhelper.Context(t)
+
+ cc, err := client.Dial(cfg.SocketPath, nil)
+ require.NoError(t, err)
+ defer testhelper.MustClose(t, cc)
+
+ repoClient := gitalypb.NewRepositoryServiceClient(cc)
+
+ _, repoPath := gittest.CreateRepository(t, ctx, cfg)
+ commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
+ gittest.WriteTag(t, cfg, repoPath, "v1.0.0", commitID.Revision())
+ repoChecksum := gittest.ChecksumRepo(t, cfg, repoPath)
+
+ backupRoot := testhelper.TempDir(t)
+
+ for _, tc := range []struct {
+ desc string
+ locators []string
+ setup func(tb testing.TB) (*gitalypb.Repository, *git.Checksum)
+ alwaysCreate bool
+ expectExists bool
+ expectedPaths []string
+ expectedErrAs error
+ }{
+ {
+ desc: "existing repo, without hooks",
+ locators: []string{"legacy", "pointer"},
+ setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
+ repo, _ := gittest.CreateRepository(t, ctx, cfg)
+
+ relativePath := stripRelativePath(tb, repo)
+ require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir))
+ bundlePath := filepath.Join(backupRoot, relativePath+".bundle")
+ gittest.BundleRepo(tb, cfg, repoPath, bundlePath)
+
+ return repo, repoChecksum
+ },
+ expectExists: true,
+ },
+ {
+ desc: "existing repo, with hooks",
+ locators: []string{"legacy", "pointer"},
+ setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
+ repo, _ := gittest.CreateRepository(t, ctx, cfg)
+
+ relativePath := stripRelativePath(tb, repo)
+ bundlePath := filepath.Join(backupRoot, relativePath+".bundle")
+ customHooksPath := filepath.Join(backupRoot, relativePath, "custom_hooks.tar")
+ require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir))
+ gittest.BundleRepo(tb, cfg, repoPath, bundlePath)
+ testhelper.CopyFile(tb, mustCreateCustomHooksArchive(t, ctx), customHooksPath)
+
+ return repo, repoChecksum
+ },
+ expectedPaths: []string{
+ "custom_hooks/pre-commit.sample",
+ "custom_hooks/prepare-commit-msg.sample",
+ "custom_hooks/pre-push.sample",
+ },
+ expectExists: true,
+ },
+ {
+ desc: "missing bundle",
+ locators: []string{"legacy", "pointer"},
+ setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
+ repo, _ := gittest.CreateRepository(t, ctx, cfg)
+ return repo, nil
+ },
+ expectedErrAs: backup.ErrSkipped,
+ },
+ {
+ desc: "missing bundle, always create",
+ locators: []string{"legacy", "pointer"},
+ setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
+ repo, _ := gittest.CreateRepository(t, ctx, cfg)
+ return repo, new(git.Checksum)
+ },
+ alwaysCreate: true,
+ expectExists: true,
+ },
+ {
+ desc: "nonexistent repo",
+ locators: []string{"legacy", "pointer"},
+ setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
+ repo := &gitalypb.Repository{
+ StorageName: "default",
+ RelativePath: gittest.NewRepositoryName(tb),
+ }
- fsBackup := backup.NewManager(sink, locator, pool, "unused-backup-id")
- err = fsBackup.Restore(ctx, &backup.RestoreRequest{
- Server: storage.ServerInfo{Address: cfg.SocketPath, Token: cfg.Auth.Token},
- Repository: repo,
- AlwaysCreate: tc.alwaysCreate,
- })
- if tc.expectedErrAs != nil {
- require.ErrorAs(t, err, &tc.expectedErrAs)
- } else {
- require.NoError(t, err)
- }
+ relativePath := stripRelativePath(tb, repo)
+ require.NoError(tb, os.MkdirAll(filepath.Dir(filepath.Join(backupRoot, relativePath)), perm.PublicDir))
+ bundlePath := filepath.Join(backupRoot, relativePath+".bundle")
+ gittest.BundleRepo(tb, cfg, repoPath, bundlePath)
- exists, err := repoClient.RepositoryExists(ctx, &gitalypb.RepositoryExistsRequest{
- Repository: repo,
- })
- require.NoError(t, err)
- require.Equal(t, tc.expectExists, exists.Exists, "repository exists")
-
- if expectedChecksum != nil {
- checksum, err := repoClient.CalculateChecksum(ctx, &gitalypb.CalculateChecksumRequest{
- Repository: repo,
+ return repo, repoChecksum
+ },
+ expectExists: true,
+ },
+ {
+ desc: "single incremental",
+ locators: []string{"pointer"},
+ setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
+ const backupID = "abc123"
+ repo, _ := gittest.CreateRepository(t, ctx, cfg)
+ repoBackupPath := joinBackupPath(tb, backupRoot, repo)
+ backupPath := filepath.Join(repoBackupPath, backupID)
+ require.NoError(tb, os.MkdirAll(backupPath, perm.PublicDir))
+ require.NoError(tb, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), perm.PublicFile))
+ require.NoError(tb, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("001"), perm.PublicFile))
+ bundlePath := filepath.Join(backupPath, "001.bundle")
+ gittest.BundleRepo(tb, cfg, repoPath, bundlePath)
+
+ return repo, repoChecksum
+ },
+ expectExists: true,
+ },
+ {
+ desc: "many incrementals",
+ locators: []string{"pointer"},
+ setup: func(tb testing.TB) (*gitalypb.Repository, *git.Checksum) {
+ const backupID = "abc123"
+
+ _, expectedRepoPath := gittest.CreateRepository(t, ctx, cfg)
+
+ repo, _ := gittest.CreateRepository(t, ctx, cfg)
+ repoBackupPath := joinBackupPath(tb, backupRoot, repo)
+ backupPath := filepath.Join(repoBackupPath, backupID)
+ require.NoError(tb, os.MkdirAll(backupPath, perm.PublicDir))
+ require.NoError(tb, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), perm.PublicFile))
+ require.NoError(tb, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("002"), perm.PublicFile))
+
+ root := gittest.WriteCommit(tb, cfg, expectedRepoPath,
+ gittest.WithBranch("master"),
+ )
+ master1 := gittest.WriteCommit(tb, cfg, expectedRepoPath,
+ gittest.WithBranch("master"),
+ gittest.WithParents(root),
+ )
+ other := gittest.WriteCommit(tb, cfg, expectedRepoPath,
+ gittest.WithBranch("other"),
+ gittest.WithParents(root),
+ )
+ gittest.Exec(tb, cfg, "-C", expectedRepoPath, "symbolic-ref", "HEAD", "refs/heads/master")
+ bundlePath1 := filepath.Join(backupPath, "001.bundle")
+ gittest.Exec(tb, cfg, "-C", expectedRepoPath, "bundle", "create", bundlePath1,
+ "HEAD",
+ "refs/heads/master",
+ "refs/heads/other",
+ )
+
+ master2 := gittest.WriteCommit(tb, cfg, expectedRepoPath,
+ gittest.WithBranch("master"),
+ gittest.WithParents(master1),
+ )
+ bundlePath2 := filepath.Join(backupPath, "002.bundle")
+ gittest.Exec(tb, cfg, "-C", expectedRepoPath, "bundle", "create", bundlePath2,
+ "HEAD",
+ "^"+master1.String(),
+ "^"+other.String(),
+ "refs/heads/master",
+ "refs/heads/other",
+ )
+
+ checksum := new(git.Checksum)
+ checksum.Add(git.NewReference("HEAD", master2.String()))
+ checksum.Add(git.NewReference("refs/heads/master", master2.String()))
+ checksum.Add(git.NewReference("refs/heads/other", other.String()))
+
+ return repo, checksum
+ },
+ expectExists: true,
+ },
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ require.GreaterOrEqual(t, len(tc.locators), 1, "each test case must specify a locator")
+
+ for _, locatorName := range tc.locators {
+ t.Run(locatorName, func(t *testing.T) {
+ repo, expectedChecksum := tc.setup(t)
+
+ sink := backup.NewFilesystemSink(backupRoot)
+ locator, err := backup.ResolveLocator(locatorName, sink)
+ require.NoError(t, err)
+
+ fsBackup := managerTC.setup(t, sink, locator)
+ err = fsBackup.Restore(ctx, &backup.RestoreRequest{
+ Server: storage.ServerInfo{Address: cfg.SocketPath, Token: cfg.Auth.Token},
+ Repository: repo,
+ AlwaysCreate: tc.alwaysCreate,
+ })
+ if tc.expectedErrAs != nil {
+ require.ErrorAs(t, err, &tc.expectedErrAs)
+ } else {
+ require.NoError(t, err)
+ }
+
+ exists, err := repoClient.RepositoryExists(ctx, &gitalypb.RepositoryExistsRequest{
+ Repository: repo,
+ })
+ require.NoError(t, err)
+ require.Equal(t, tc.expectExists, exists.Exists, "repository exists")
+
+ if expectedChecksum != nil {
+ checksum, err := repoClient.CalculateChecksum(ctx, &gitalypb.CalculateChecksumRequest{
+ Repository: repo,
+ })
+ require.NoError(t, err)
+
+ require.Equal(t, expectedChecksum.String(), checksum.GetChecksum())
+ }
+
+ if len(tc.expectedPaths) > 0 {
+ // Restore has to use the rewritten path as the relative path due to the test creating
+ // the repository through Praefect. In order to get to the correct disk paths, we need
+ // to get the replica path of the rewritten repository.
+ repoPath := filepath.Join(cfg.Storages[0].Path, gittest.GetReplicaPath(t, ctx, cfg, repo))
+ for _, p := range tc.expectedPaths {
+ require.FileExists(t, filepath.Join(repoPath, p))
+ }
+ }
})
- require.NoError(t, err)
-
- require.Equal(t, expectedChecksum.String(), checksum.GetChecksum())
- }
-
- if len(tc.expectedPaths) > 0 {
- // Restore has to use the rewritten path as the relative path due to the test creating
- // the repository through Praefect. In order to get to the correct disk paths, we need
- // to get the replica path of the rewritten repository.
- repoPath := filepath.Join(cfg.Storages[0].Path, gittest.GetReplicaPath(t, ctx, cfg, repo))
- for _, p := range tc.expectedPaths {
- require.FileExists(t, filepath.Join(repoPath, p))
- }
}
})
}
diff --git a/internal/backup/repository.go b/internal/backup/repository.go
index 6c2523908..957c4b64d 100644
--- a/internal/backup/repository.go
+++ b/internal/backup/repository.go
@@ -12,6 +12,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/repoutil"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v16/internal/helper/chunk"
"gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
@@ -148,6 +149,89 @@ func (rr *remoteRepository) CreateBundle(ctx context.Context, out io.Writer, pat
return nil
}
+// Remove removes the repository. Does not return an error if the repository
+// cannot be found.
+func (rr *remoteRepository) Remove(ctx context.Context) error {
+ repoClient := rr.newRepoClient()
+ _, err := repoClient.RemoveRepository(ctx, &gitalypb.RemoveRepositoryRequest{
+ Repository: rr.repo,
+ })
+ switch {
+ case status.Code(err) == codes.NotFound:
+ return nil
+ case err != nil:
+ return fmt.Errorf("remote repository: remove: %w", err)
+ }
+ return nil
+}
+
+// Create creates the repository.
+func (rr *remoteRepository) Create(ctx context.Context) error {
+ repoClient := rr.newRepoClient()
+ if _, err := repoClient.CreateRepository(ctx, &gitalypb.CreateRepositoryRequest{Repository: rr.repo}); err != nil {
+ return fmt.Errorf("remote repository: create: %w", err)
+ }
+ return nil
+}
+
+// FetchBundle fetches references from a bundle. Refs will be mirrored to the
+// repository.
+func (rr *remoteRepository) FetchBundle(ctx context.Context, reader io.Reader) error {
+ repoClient := rr.newRepoClient()
+ stream, err := repoClient.FetchBundle(ctx)
+ if err != nil {
+ return fmt.Errorf("remote repository: fetch bundle: %w", err)
+ }
+ request := &gitalypb.FetchBundleRequest{Repository: rr.repo, UpdateHead: true}
+ bundle := streamio.NewWriter(func(p []byte) error {
+ request.Data = p
+ if err := stream.Send(request); err != nil {
+ return err
+ }
+
+ // Only set `Repository` on the first `Send` of the stream
+ request = &gitalypb.FetchBundleRequest{}
+
+ return nil
+ })
+ if _, err := io.Copy(bundle, reader); err != nil {
+ return fmt.Errorf("remote repository: fetch bundle: %w", err)
+ }
+ if _, err = stream.CloseAndRecv(); err != nil {
+ return fmt.Errorf("remote repository: fetch bundle: %w", err)
+ }
+ return nil
+}
+
+// SetCustomHooks updates the custom hooks for the repository.
+func (rr *remoteRepository) SetCustomHooks(ctx context.Context, reader io.Reader) error {
+ repoClient := rr.newRepoClient()
+ stream, err := repoClient.SetCustomHooks(ctx)
+ if err != nil {
+ return fmt.Errorf("remote repository: set custom hooks: %w", err)
+ }
+
+ request := &gitalypb.SetCustomHooksRequest{Repository: rr.repo}
+ bundle := streamio.NewWriter(func(p []byte) error {
+ request.Data = p
+ if err := stream.Send(request); err != nil {
+ return err
+ }
+
+ // Only set `Repository` on the first `Send` of the stream
+ request = &gitalypb.SetCustomHooksRequest{}
+
+ return nil
+ })
+ if _, err := io.Copy(bundle, reader); err != nil {
+ return fmt.Errorf("remote repository: set custom hooks: %w", err)
+ }
+ if _, err = stream.CloseAndRecv(); err != nil {
+ return fmt.Errorf("remote repository: set custom hooks: %w", err)
+ }
+ return nil
+}
+
func (rr *remoteRepository) newRepoClient() gitalypb.RepositoryServiceClient {
return gitalypb.NewRepositoryServiceClient(rr.conn)
}
@@ -157,14 +241,23 @@ func (rr *remoteRepository) newRefClient() gitalypb.RefServiceClient {
}
type localRepository struct {
- locator storage.Locator
- repo *localrepo.Repo
+ locator storage.Locator
+ gitCmdFactory git.CommandFactory
+ txManager transaction.Manager
+ repo *localrepo.Repo
}
-func newLocalRepository(locator storage.Locator, repo *localrepo.Repo) *localRepository {
+func newLocalRepository(
+ locator storage.Locator,
+ gitCmdFactory git.CommandFactory,
+ txManager transaction.Manager,
+ repo *localrepo.Repo,
+) *localRepository {
return &localRepository{
- locator: locator,
- repo: repo,
+ locator: locator,
+ gitCmdFactory: gitCmdFactory,
+ txManager: txManager,
+ repo: repo,
}
}
@@ -229,3 +322,57 @@ func (r *localRepository) CreateBundle(ctx context.Context, out io.Writer, patte
return nil
}
+
+// Remove removes the repository. Does not return an error if the repository
+// cannot be found.
+func (r *localRepository) Remove(ctx context.Context) error {
+ err := repoutil.Remove(ctx, r.locator, r.txManager, r.repo)
+ switch {
+ case status.Code(err) == codes.NotFound:
+ return nil
+ case err != nil:
+ return fmt.Errorf("local repository: remove: %w", err)
+ }
+ return nil
+}
+
+// Create creates the repository.
+func (r *localRepository) Create(ctx context.Context) error {
+ if err := repoutil.Create(
+ ctx,
+ r.locator,
+ r.gitCmdFactory,
+ r.txManager,
+ r.repo,
+ func(repository *gitalypb.Repository) error { return nil },
+ ); err != nil {
+ return fmt.Errorf("local repository: create: %w", err)
+ }
+ return nil
+}
+
+// FetchBundle fetches references from a bundle. Refs will be mirrored to the
+// repository.
+func (r *localRepository) FetchBundle(ctx context.Context, reader io.Reader) error {
+ err := r.repo.FetchBundle(ctx, r.txManager, reader, &localrepo.FetchBundleOpts{
+ UpdateHead: true,
+ })
+ if err != nil {
+ return fmt.Errorf("local repository: fetch bundle: %w", err)
+ }
+ return nil
+}
+
+// SetCustomHooks updates the custom hooks for the repository.
+func (r *localRepository) SetCustomHooks(ctx context.Context, reader io.Reader) error {
+ if err := repoutil.SetCustomHooks(
+ ctx,
+ r.locator,
+ r.txManager,
+ reader,
+ r.repo,
+ ); err != nil {
+ return fmt.Errorf("local repository: set custom hooks: %w", err)
+ }
+ return nil
+}
diff --git a/internal/cli/praefect/main.go b/internal/cli/praefect/main.go
index c9547e895..b1fcf2007 100644
--- a/internal/cli/praefect/main.go
+++ b/internal/cli/praefect/main.go
@@ -15,12 +15,12 @@ import (
"fmt"
"log"
"os"
- "sort"
- "strings"
+ "os/signal"
"github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service"
"gitlab.com/gitlab-org/gitaly/v16/internal/version"
+ "golang.org/x/exp/slices"
)
func init() {
@@ -39,6 +39,8 @@ const (
// NewApp returns a new praefect app.
func NewApp() *cli.App {
+ interrupt := make(chan os.Signal, 1)
+
return &cli.App{
Name: progname,
Usage: "a gitaly proxy",
@@ -61,6 +63,7 @@ func NewApp() *cli.App {
newListStoragesCommand(),
newListUntrackedRepositoriesCommand(),
newTrackRepositoryCommand(),
+ newTrackRepositoriesCommand(),
newVerifyCommand(),
newMetadataCommand(),
newSQLPingCommand(),
@@ -79,7 +82,29 @@ func NewApp() *cli.App {
Usage: "load configuration from `FILE`",
},
},
- CustomAppHelpTemplate: helpTextTemplate(),
+ Before: func(appCtx *cli.Context) error {
+ // Praefect service manages os.Interrupt on its own, by making a "table-flip".
+ // That is why the signal listening is omitted if there are no arguments passed
+ // (old-fashioned method of starting Praefect service) or 'serve' sub-command
+ // is invoked. Other sub-commands require signal to be properly handled.
+ args := appCtx.Args().Slice()
+ if len(args) == 0 || slices.Contains(args, "serve") {
+ return nil
+ }
+
+ signal.Notify(interrupt, os.Interrupt)
+ go func() {
+ if _, ok := <-interrupt; ok {
+ os.Exit(130) // indicates program was interrupted
+ }
+ }()
+
+ return nil
+ },
+ After: func(*cli.Context) error {
+ close(interrupt)
+ return nil
+ },
}
}
@@ -99,22 +124,3 @@ func mustProvideConfigFlag(ctx *cli.Context, command string) string {
return pathToConfigFile
}
-
-func helpTextTemplate() string {
- var cmds []string
- for k := range subcommands(nil) {
- cmds = append(cmds, k)
- }
- sort.Strings(cmds)
-
- // Because not all sub-commands are registered with the new approach they won't be shown
- // with the -help. To have them in the output we inject a simple list of their names into
- // the template to have them presented.
- return strings.Replace(
- cli.AppHelpTemplate,
- `COMMANDS:{{template "visibleCommandCategoryTemplate" .}}{{end}}`,
- `COMMANDS:{{template "visibleCommandCategoryTemplate" .}}{{end}}`+
- "\n "+strings.Join(cmds, "\n "),
- 1,
- )
-}
diff --git a/internal/cli/praefect/serve.go b/internal/cli/praefect/serve.go
index 328459ef6..096db8ad1 100644
--- a/internal/cli/praefect/serve.go
+++ b/internal/cli/praefect/serve.go
@@ -7,7 +7,6 @@ import (
"fmt"
"math/rand"
"net/http"
- "os"
"runtime/debug"
"time"
@@ -47,26 +46,21 @@ func newServeCommand() *cli.Command {
Usage: "launch the server daemon",
Action: serveAction,
HideHelpCommand: true,
+ Before: func(context *cli.Context) error {
+ if context.Args().Present() {
+ return unexpectedPositionalArgsError{Command: context.Command.Name}
+ }
+ return nil
+ },
}
}
func serveAction(ctx *cli.Context) error {
- logger := log.Default()
- // In order to support execution of all sub-commands not yet migrated to use a new cli
- // implementation the invocation is done manually here.
- subCmd := ctx.Args().First()
- if subCmd != "" {
- // It doesn't make difference if we provide command name to the invocation below
- // or not as there won't be any output printed, because sub-commands are not yet
- // registered.
- pathToConfigFile := mustProvideConfigFlag(ctx, "")
- conf, err := getConfig(logger, pathToConfigFile)
- if err != nil {
- return err
- }
- os.Exit(subCommand(conf, logger, subCmd, ctx.Args().Slice()[1:]))
+ if ctx.Args().Present() {
+ return unexpectedPositionalArgsError{Command: ctx.Command.Name}
}
+ logger := log.Default()
// The ctx.Command.Name can't be used here because if `praefect -config FILE` is used
// it will be set to 'praefect' instead of 'serve'.
pathToConfigFile := mustProvideConfigFlag(ctx, "serve")
diff --git a/internal/cli/praefect/subcmd.go b/internal/cli/praefect/subcmd.go
index 457a4bc87..fefcefa8d 100644
--- a/internal/cli/praefect/subcmd.go
+++ b/internal/cli/praefect/subcmd.go
@@ -4,13 +4,10 @@ import (
"context"
"database/sql"
"errors"
- "flag"
"fmt"
- "os"
- "os/signal"
+ "io"
"time"
- "github.com/sirupsen/logrus"
gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth"
"gitlab.com/gitlab-org/gitaly/v16/client"
internalclient "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/client"
@@ -19,11 +16,6 @@ import (
"google.golang.org/grpc"
)
-type subcmd interface {
- FlagSet() *flag.FlagSet
- Exec(flags *flag.FlagSet, config config.Config) error
-}
-
const (
defaultDialTimeout = 10 * time.Second
paramVirtualStorage = "virtual-storage"
@@ -31,43 +23,6 @@ const (
paramAuthoritativeStorage = "authoritative-storage"
)
-func subcommands(logger *logrus.Entry) map[string]subcmd {
- return map[string]subcmd{
- trackRepositoriesCmdName: newTrackRepositories(logger, os.Stdout),
- }
-}
-
-// subCommand returns an exit code, to be fed into os.Exit.
-func subCommand(conf config.Config, logger *logrus.Entry, arg0 string, argRest []string) int {
- interrupt := make(chan os.Signal, 1)
- signal.Notify(interrupt, os.Interrupt)
-
- go func() {
- <-interrupt
- os.Exit(130) // indicates program was interrupted
- }()
-
- subcmd, ok := subcommands(logger)[arg0]
- if !ok {
- printfErr("%s: unknown subcommand: %q\n", progname, arg0)
- return 1
- }
-
- flags := subcmd.FlagSet()
-
- if err := flags.Parse(argRest); err != nil {
- printfErr("%s\n", err)
- return 1
- }
-
- if err := subcmd.Exec(flags, conf); err != nil {
- printfErr("%s\n", err)
- return 1
- }
-
- return 0
-}
-
func getNodeAddress(cfg config.Config) (string, error) {
switch {
case cfg.SocketPath != "":
@@ -81,7 +36,7 @@ func getNodeAddress(cfg config.Config) (string, error) {
}
}
-func openDB(conf config.DB) (*sql.DB, func(), error) {
+func openDB(conf config.DB, errOut io.Writer) (*sql.DB, func(), error) {
ctx := context.Background()
openDBCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
@@ -93,17 +48,13 @@ func openDB(conf config.DB) (*sql.DB, func(), error) {
clean := func() {
if err := db.Close(); err != nil {
- printfErr("sql close: %v\n", err)
+ fmt.Fprintf(errOut, "sql close: %v\n", err)
}
}
return db, clean, nil
}
-func printfErr(format string, a ...interface{}) {
- fmt.Fprintf(os.Stderr, format, a...)
-}
-
func subCmdDial(ctx context.Context, addr, token string, timeout time.Duration, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
diff --git a/internal/cli/praefect/subcmd_accept_dataloss_test.go b/internal/cli/praefect/subcmd_accept_dataloss_test.go
index f0e78638a..207c21f87 100644
--- a/internal/cli/praefect/subcmd_accept_dataloss_test.go
+++ b/internal/cli/praefect/subcmd_accept_dataloss_test.go
@@ -1,13 +1,10 @@
package praefect
import (
- "bytes"
"context"
- "io"
"testing"
"github.com/stretchr/testify/require"
- "github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/info"
@@ -138,23 +135,7 @@ func TestAcceptDatalossSubcommand(t *testing.T) {
},
} {
t.Run(tc.desc, func(t *testing.T) {
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: io.Discard,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newAcceptDatalossCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
-
- err := app.Run(append([]string{progname, "accept-dataloss"}, tc.args...))
+ _, _, err := runApp(append([]string{"-config", confPath, "accept-dataloss"}, tc.args...))
tc.matchError(t, err)
for storage, expected := range tc.expectedGenerations {
diff --git a/internal/cli/praefect/subcmd_check_test.go b/internal/cli/praefect/subcmd_check_test.go
index 68ee4e2aa..c970b8520 100644
--- a/internal/cli/praefect/subcmd_check_test.go
+++ b/internal/cli/praefect/subcmd_check_test.go
@@ -181,39 +181,42 @@ Checking check 3...Failed (warning) error: i failed but not too badly
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newCheckCommand(tc.checks),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
-
t.Run("quiet", func(t *testing.T) {
- stdout.Reset()
- err := app.Run(append([]string{progname, "check", "-q"}, tc.args...))
+ var stdout, stderr bytes.Buffer
+ app := NewApp()
+ app.Writer = &stdout
+ app.ErrWriter = &stderr
+ for i, cmd := range app.Commands {
+ if cmd.Name == "check" {
+ app.Commands[i] = newCheckCommand(tc.checks)
+ break
+ }
+ }
+ err := app.Run(append([]string{progname, "-config", confPath, "check", "-q"}, tc.args...))
assert.Equal(t, tc.expectedError, err)
if len(tc.args) == 0 {
assert.Equal(t, tc.expectedQuietOutput, stdout.String())
}
+ assert.Empty(t, stderr)
})
t.Run("normal", func(t *testing.T) {
- stdout.Reset()
- err := app.Run(append([]string{progname, "check"}, tc.args...))
+ var stdout, stderr bytes.Buffer
+ app := NewApp()
+ app.Writer = &stdout
+ app.ErrWriter = &stderr
+ for i, cmd := range app.Commands {
+ if cmd.Name == "check" {
+ app.Commands[i] = newCheckCommand(tc.checks)
+ break
+ }
+ }
+ err := app.Run(append([]string{progname, "-config", confPath, "check"}, tc.args...))
assert.Equal(t, tc.expectedError, err)
if len(tc.args) == 0 {
assert.Equal(t, tc.expectedOutput, stdout.String())
}
+ assert.Empty(t, stderr)
})
})
}
diff --git a/internal/cli/praefect/subcmd_dataloss_test.go b/internal/cli/praefect/subcmd_dataloss_test.go
index 4ad1c2059..30a3c49de 100644
--- a/internal/cli/praefect/subcmd_dataloss_test.go
+++ b/internal/cli/praefect/subcmd_dataloss_test.go
@@ -1,10 +1,9 @@
package praefect
import (
- "bytes"
- "io"
"testing"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
@@ -179,28 +178,12 @@ Virtual storage: virtual-storage-2
cfg.VirtualStorages = tc.virtualStorages
confPath := writeConfigToFile(t, cfg)
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newDatalossCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
-
- err := app.Run(append([]string{progname, "dataloss"}, tc.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, "dataloss"}, tc.args...))
require.Equal(t, tc.error, err, err)
if tc.error == nil {
- require.Equal(t, tc.output, stdout.String())
+ require.Equal(t, tc.output, stdout)
}
+ assert.Empty(t, stderr)
})
}
}
diff --git a/internal/cli/praefect/subcmd_dial_nodes_test.go b/internal/cli/praefect/subcmd_dial_nodes_test.go
index 221b8c78f..37dc49d7a 100644
--- a/internal/cli/praefect/subcmd_dial_nodes_test.go
+++ b/internal/cli/praefect/subcmd_dial_nodes_test.go
@@ -1,14 +1,13 @@
package praefect
import (
- "bytes"
"context"
"fmt"
- "io"
"strings"
"testing"
"time"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
@@ -136,27 +135,11 @@ func TestSubCmdDialNodes(t *testing.T) {
resp = tt.resp
confPath := writeConfigToFile(t, tt.conf)
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newDialNodesCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
-
- err := app.Run(append([]string{progname, "dial-nodes", "-timeout", time.Second.String()}, tt.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, "dial-nodes", "-timeout", time.Second.String()}, tt.args...))
+ assert.Empty(t, stderr)
if tt.errMsg == "" {
require.NoError(t, err)
- require.Equal(t, tt.logs, stdout.String())
+ require.Equal(t, tt.logs, stdout)
return
}
diff --git a/internal/cli/praefect/subcmd_list_storages_test.go b/internal/cli/praefect/subcmd_list_storages_test.go
index 0356b79f3..44bcf9e76 100644
--- a/internal/cli/praefect/subcmd_list_storages_test.go
+++ b/internal/cli/praefect/subcmd_list_storages_test.go
@@ -2,10 +2,10 @@ package praefect
import (
"bytes"
- "io"
"testing"
"github.com/olekukonko/tablewriter"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
@@ -117,27 +117,6 @@ func TestListStoragesSubcommand(t *testing.T) {
},
}
- exec := func(confPath string, args []string) (string, error) {
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newListStoragesCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, "list-storages"}, args...))
- return stdout.String(), err
- }
-
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
var expectedOutput bytes.Buffer
@@ -161,7 +140,8 @@ func TestListStoragesSubcommand(t *testing.T) {
VirtualStorages: tc.virtualStorages,
}
confPath := writeConfigToFile(t, conf)
- stdout, err := exec(confPath, tc.args)
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, "list-storages"}, tc.args...))
+ assert.Empty(t, stderr)
require.NoError(t, err)
require.Equal(t, expectedOutput.String(), stdout)
})
@@ -189,13 +169,15 @@ func TestListStoragesSubcommand(t *testing.T) {
confPath := writeConfigToFile(t, conf)
t.Run("virtual storage arg matches no virtual storages", func(t *testing.T) {
- stdout, err := exec(confPath, []string{"-virtual-storage", "vs-2"})
+ stdout, stderr, err := runApp([]string{"-config", confPath, "list-storages", "-virtual-storage", "vs-2"})
+ assert.Empty(t, stderr)
require.NoError(t, err)
require.Equal(t, "No virtual storages named vs-2.\n", stdout)
})
t.Run("positional arguments", func(t *testing.T) {
- _, err := exec(confPath, []string{"-virtual-storage", "vs-1", "positional-arg"})
+ _, stderr, err := runApp([]string{"-config", confPath, "list-storages", "-virtual-storage", "vs-1", "positional-arg"})
+ assert.Empty(t, stderr)
require.Equal(t, cli.Exit(unexpectedPositionalArgsError{Command: "list-storages"}, 1), err)
})
})
diff --git a/internal/cli/praefect/subcmd_list_untracked_repositories_test.go b/internal/cli/praefect/subcmd_list_untracked_repositories_test.go
index 5fd7261f9..694f99040 100644
--- a/internal/cli/praefect/subcmd_list_untracked_repositories_test.go
+++ b/internal/cli/praefect/subcmd_list_untracked_repositories_test.go
@@ -1,15 +1,14 @@
package praefect
import (
- "bytes"
"context"
"fmt"
- "io"
"os"
"strings"
"testing"
"time"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/client"
@@ -86,50 +85,31 @@ func TestListUntrackedRepositoriesCommand(t *testing.T) {
time.Now().Add(-(timeDelta+1*time.Second)),
time.Now().Add(-(timeDelta+1*time.Second))))
- newApp := func() (cli.App, *bytes.Buffer) {
- var stdout bytes.Buffer
- return cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newListUntrackedRepositoriesCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }, &stdout
- }
-
t.Run("positional arguments", func(t *testing.T) {
- app, _ := newApp()
- err := app.Run([]string{progname, "list-untracked-repositories", "positional-arg"})
+ stdout, stderr, err := runApp([]string{"-config", confPath, "list-untracked-repositories", "positional-arg"})
require.Equal(t, cli.Exit(unexpectedPositionalArgsError{Command: "list-untracked-repositories"}, 1), err)
+ assert.NotEmpty(t, stdout, "the help text should be printed")
+ assert.Empty(t, stderr)
})
t.Run("default flag values used", func(t *testing.T) {
- app, stdout := newApp()
- err := app.Run([]string{progname, "list-untracked-repositories"})
+ stdout, stderr, err := runApp([]string{"-config", confPath, "list-untracked-repositories"})
require.NoError(t, err)
- require.Empty(t, stdout.String())
+ assert.Empty(t, stdout)
+ assert.Empty(t, stderr)
})
t.Run("passed flag values used", func(t *testing.T) {
- app, stdout := newApp()
- err := app.Run([]string{progname, "list-untracked-repositories", "-older-than", timeDelta.String(), "-delimiter", "~"})
+ stdout, stderr, err := runApp([]string{"-config", confPath, "list-untracked-repositories", "-older-than", timeDelta.String(), "-delimiter", "~"})
require.NoError(t, err)
-
+ assert.Empty(t, stderr)
exp := []string{
"The following repositories were found on disk, but missing from the tracking database:",
fmt.Sprintf(`{"relative_path":%q,"storage":"gitaly-1","virtual_storage":"praefect"}`, repo1.RelativePath),
fmt.Sprintf(`{"relative_path":%q,"storage":"gitaly-1","virtual_storage":"praefect"}`, repo2.RelativePath),
"", // an empty extra element required as each line ends with "delimiter" and strings.Split returns all parts
}
- elems := strings.Split(stdout.String(), "~")
+ elems := strings.Split(stdout, "~")
require.Len(t, elems, len(exp)-1)
elems = append(elems[1:], strings.Split(elems[0], "\n")...)
require.ElementsMatch(t, exp, elems)
diff --git a/internal/cli/praefect/subcmd_metadata_test.go b/internal/cli/praefect/subcmd_metadata_test.go
index 124960504..5a48f6e97 100644
--- a/internal/cli/praefect/subcmd_metadata_test.go
+++ b/internal/cli/praefect/subcmd_metadata_test.go
@@ -1,13 +1,12 @@
package praefect
import (
- "bytes"
"errors"
"fmt"
- "io"
"testing"
"time"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
@@ -125,23 +124,8 @@ func TestMetadataSubcommand(t *testing.T) {
}
confPath := writeConfigToFile(t, conf)
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newMetadataCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, "metadata"}, tc.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, "metadata"}, tc.args...))
+ assert.Empty(t, stderr)
require.Equal(t, tc.error, err)
if tc.error != nil {
return
@@ -172,7 +156,7 @@ Replicas:
Healthy: false
Valid Primary: false
Verified At: unverified
-`, stdout.String())
+`, stdout)
})
}
}
diff --git a/internal/cli/praefect/subcmd_remove_repository_test.go b/internal/cli/praefect/subcmd_remove_repository_test.go
index d57de849c..a89ab27ed 100644
--- a/internal/cli/praefect/subcmd_remove_repository_test.go
+++ b/internal/cli/praefect/subcmd_remove_repository_test.go
@@ -1,9 +1,7 @@
package praefect
import (
- "bytes"
"fmt"
- "io"
"net"
"os"
"path/filepath"
@@ -239,26 +237,11 @@ func TestRemoveRepositorySubcommand(t *testing.T) {
}
repo := createRepo(t, ctx, repoClient, praefectStorage, t.Name())
replicaPath := gittest.GetReplicaPath(t, ctx, gitalycfg.Cfg{SocketPath: praefectServer.Address()}, repo)
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newRemoveRepositoryCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, "remove-repository"}, tc.args(t, repo, replicaPath)...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, "remove-repository"}, tc.args(t, repo, replicaPath)...))
+ assert.Empty(t, stderr)
tc.assertError(t, err, repo, replicaPath)
if tc.assertOutput != nil {
- tc.assertOutput(t, stdout.String(), repo)
+ tc.assertOutput(t, stdout, repo)
}
})
}
@@ -267,25 +250,10 @@ func TestRemoveRepositorySubcommand(t *testing.T) {
repo := createRepo(t, ctx, repoClient, praefectStorage, t.Name())
g2Srv.Shutdown()
replicaPath := gittest.GetReplicaPath(t, ctx, gitalycfg.Cfg{SocketPath: praefectServer.Address()}, repo)
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newRemoveRepositoryCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, "remove-repository"}, "-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply"))
+ stdout, stderr, err := runApp([]string{"-config", confPath, "remove-repository", "-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply"})
+ assert.Empty(t, stderr)
require.NoError(t, err)
- assert.Contains(t, stdout.String(), "Repository removal completed.")
+ assert.Contains(t, stdout, "Repository removal completed.")
require.NoDirExists(t, filepath.Join(g1Cfg.Storages[0].Path, replicaPath))
require.DirExists(t, filepath.Join(g2Cfg.Storages[0].Path, replicaPath))
require.False(t, repositoryExists(t, repo))
diff --git a/internal/cli/praefect/subcmd_set_replication_factor_test.go b/internal/cli/praefect/subcmd_set_replication_factor_test.go
index 8b3f57800..f7878575d 100644
--- a/internal/cli/praefect/subcmd_set_replication_factor_test.go
+++ b/internal/cli/praefect/subcmd_set_replication_factor_test.go
@@ -1,11 +1,10 @@
package praefect
import (
- "bytes"
"errors"
- "io"
"testing"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect"
@@ -115,26 +114,11 @@ func TestSetReplicationFactorSubcommand(t *testing.T) {
}
confPath := writeConfigToFile(t, conf)
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newSetReplicationFactorCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, setReplicationFactorCmdName}, tc.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, setReplicationFactorCmdName}, tc.args...))
+ assert.Empty(t, stderr)
testhelper.RequireGrpcError(t, tc.error, err)
if tc.stdout != "" {
- require.Equal(t, tc.stdout, stdout.String())
+ require.Equal(t, tc.stdout, stdout)
}
})
}
diff --git a/internal/cli/praefect/subcmd_sql_migrate.go b/internal/cli/praefect/subcmd_sql_migrate.go
index 46d941113..2329125bf 100644
--- a/internal/cli/praefect/subcmd_sql_migrate.go
+++ b/internal/cli/praefect/subcmd_sql_migrate.go
@@ -53,7 +53,7 @@ func sqlMigrateAction(appCtx *cli.Context) error {
return err
}
- db, clean, err := openDB(conf.DB)
+ db, clean, err := openDB(conf.DB, appCtx.App.ErrWriter)
if err != nil {
return err
}
diff --git a/internal/cli/praefect/subcmd_sql_migrate_down_test.go b/internal/cli/praefect/subcmd_sql_migrate_down_test.go
index f12e3712c..70983f79a 100644
--- a/internal/cli/praefect/subcmd_sql_migrate_down_test.go
+++ b/internal/cli/praefect/subcmd_sql_migrate_down_test.go
@@ -1,13 +1,11 @@
package praefect
import (
- "bytes"
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore/migrations"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb"
@@ -58,28 +56,11 @@ func TestSQLMigrateDownSubcommand(t *testing.T) {
},
} {
t.Run(tc.desc, func(t *testing.T) {
- var stdout bytes.Buffer
- var stderr bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: &stderr,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newSQLMigrateDownCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, sqlMigrateDownCmdName}, tc.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, sqlMigrateDownCmdName}, tc.args...))
+ assert.Empty(t, stderr)
require.Equal(t, tc.expectedErr, err)
- assert.Empty(t, stderr.String())
for _, expectedOutput := range tc.expectedOutput {
- assert.Contains(t, stdout.String(), expectedOutput)
+ assert.Contains(t, stdout, expectedOutput)
}
})
}
diff --git a/internal/cli/praefect/subcmd_sql_migrate_status_test.go b/internal/cli/praefect/subcmd_sql_migrate_status_test.go
index 5e43fb7eb..a2c978314 100644
--- a/internal/cli/praefect/subcmd_sql_migrate_status_test.go
+++ b/internal/cli/praefect/subcmd_sql_migrate_status_test.go
@@ -1,7 +1,6 @@
package praefect
import (
- "bytes"
"testing"
"github.com/stretchr/testify/assert"
@@ -43,28 +42,11 @@ func TestSQLMigrateStatusSubcommand(t *testing.T) {
},
} {
t.Run(tc.desc, func(t *testing.T) {
- var stdout bytes.Buffer
- var stderr bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: &stderr,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newSQLMigrateStatusCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, sqlMigrateStatusCmdName}, tc.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, sqlMigrateStatusCmdName}, tc.args...))
+ assert.Empty(t, stderr)
require.Equal(t, tc.expectedErr, err)
- assert.Empty(t, stderr.String())
for _, expectedOut := range tc.expectedOuts {
- assert.Contains(t, stdout.String(), expectedOut)
+ assert.Contains(t, stdout, expectedOut)
}
})
}
diff --git a/internal/cli/praefect/subcmd_sql_migrate_test.go b/internal/cli/praefect/subcmd_sql_migrate_test.go
index 9fae09ca8..880c01555 100644
--- a/internal/cli/praefect/subcmd_sql_migrate_test.go
+++ b/internal/cli/praefect/subcmd_sql_migrate_test.go
@@ -1,7 +1,6 @@
package praefect
import (
- "bytes"
"fmt"
"testing"
@@ -77,29 +76,11 @@ func TestSubCmdSqlMigrate(t *testing.T) {
} {
t.Run(tc.desc, func(t *testing.T) {
testdb.SetMigrations(t, db, cfg, tc.up)
-
- var stdout bytes.Buffer
- var stderr bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: &stderr,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newSQLMigrateCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, sqlMigrateCmdName, "-ignore-unknown"}, tc.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, sqlMigrateCmdName, "-ignore-unknown"}, tc.args...))
+ assert.Empty(t, stderr)
require.Equal(t, tc.expectedErr, err)
- assert.Empty(t, stderr.String())
for _, out := range tc.expectedOutput {
- assert.Contains(t, stdout.String(), out)
+ assert.Contains(t, stdout, out)
}
})
}
diff --git a/internal/cli/praefect/subcmd_sql_ping.go b/internal/cli/praefect/subcmd_sql_ping.go
index 51ac456ef..665d86d0a 100644
--- a/internal/cli/praefect/subcmd_sql_ping.go
+++ b/internal/cli/praefect/subcmd_sql_ping.go
@@ -36,7 +36,7 @@ func sqlPingAction(appCtx *cli.Context) error {
subCmd := progname + " " + appCtx.Command.Name
- db, clean, err := openDB(conf.DB)
+ db, clean, err := openDB(conf.DB, appCtx.App.ErrWriter)
if err != nil {
return err
}
diff --git a/internal/cli/praefect/subcmd_sql_ping_test.go b/internal/cli/praefect/subcmd_sql_ping_test.go
index c35d99890..22139c55b 100644
--- a/internal/cli/praefect/subcmd_sql_ping_test.go
+++ b/internal/cli/praefect/subcmd_sql_ping_test.go
@@ -1,13 +1,11 @@
package praefect
import (
- "bytes"
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb"
)
@@ -56,30 +54,13 @@ func TestSQLPingSubcommand(t *testing.T) {
},
} {
t.Run(tc.desc, func(t *testing.T) {
- var stdout bytes.Buffer
- var stderr bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: &stderr,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newSQLPingCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: tc.confPath(t),
- },
- },
- }
- err := app.Run(append([]string{progname, sqlPingCmdName}, tc.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", tc.confPath(t), sqlPingCmdName}, tc.args...))
+ assert.Empty(t, stderr)
if tc.expectedErr != nil {
require.EqualError(t, err, tc.expectedErr.Error())
}
- assert.Empty(t, stderr.String())
if tc.expectedOutput != "" {
- assert.Equal(t, tc.expectedOutput, stdout.String())
+ assert.Equal(t, tc.expectedOutput, stdout)
}
})
}
diff --git a/internal/cli/praefect/subcmd_test.go b/internal/cli/praefect/subcmd_test.go
index aaaa38d5a..6b5604042 100644
--- a/internal/cli/praefect/subcmd_test.go
+++ b/internal/cli/praefect/subcmd_test.go
@@ -1,6 +1,7 @@
package praefect
import (
+ "bytes"
"fmt"
"net"
"path/filepath"
@@ -68,3 +69,13 @@ func listenAndServe(tb testing.TB, svcs []svcRegistrar) (net.Listener, testhelpe
require.NoErrorf(tb, err, "error while stopping server: %q", err)
}
}
+
+func runApp(args []string) (string, string, error) {
+ var stdout, stderr bytes.Buffer
+ app := NewApp()
+ app.Writer = &stdout
+ app.ErrWriter = &stderr
+ app.Reader = bytes.NewReader(nil)
+ err := app.Run(append([]string{progname}, args...))
+ return stdout.String(), stderr.String(), err
+}
diff --git a/internal/cli/praefect/subcmd_track_repositories.go b/internal/cli/praefect/subcmd_track_repositories.go
index 6130fb1f8..ea61ebe8a 100644
--- a/internal/cli/praefect/subcmd_track_repositories.go
+++ b/internal/cli/praefect/subcmd_track_repositories.go
@@ -4,16 +4,13 @@ import (
"bufio"
"context"
"encoding/json"
- "flag"
"fmt"
"io"
"os"
- "time"
- "github.com/sirupsen/logrus"
- "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
+ "github.com/urfave/cli/v2"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/log"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore"
- "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore/glsql"
"gitlab.com/gitlab-org/labkit/correlation"
)
@@ -22,72 +19,63 @@ const (
paramInputPath = "input-path"
)
-type invalidRequest struct {
- line int
- path string
- errs []error
-}
-
-type dupPathError struct {
- path string
- reqNums []int
-}
-
-func (d *dupPathError) Error() string {
- return fmt.Sprintf("duplicate entries for relative_path, line %v", d.reqNums)
-}
-
-type trackRepositories struct {
- w io.Writer
- logger logrus.FieldLogger
- inputPath string
- replicateImmediately bool
-}
-
-func newTrackRepositories(logger logrus.FieldLogger, w io.Writer) *trackRepositories {
- return &trackRepositories{w: w, logger: logger}
-}
-
-func (cmd *trackRepositories) FlagSet() *flag.FlagSet {
- fs := flag.NewFlagSet(trackRepositoryCmdName, flag.ExitOnError)
- fs.BoolVar(&cmd.replicateImmediately, "replicate-immediately", false, "kick off a replication immediately")
- fs.StringVar(&cmd.inputPath, paramInputPath, "", "path to file with details of repositories to track")
- fs.Usage = func() {
- printfErr("Description:\n" +
- " This command allows bulk requests for repositories to be tracked by Praefect.\n" +
- " The -input-path flag must be the path of a file containing the details of the repositories\n" +
- " to track as a list of newline-delimited JSON objects. Each line must contain the details for\n" +
- " one and only one repository. Each item must contain the following keys:\n\n" +
- " relative_path - The relative path of the repository on-disk.\n" +
- " virtual_storage - The Praefect virtual storage name.\n" +
- " authoritative_storage - Which storage to consider as the canonical copy of the repository.\n\n" +
- " If -replicate-immediately is used, the command will attempt to replicate the repositories\n" +
- " to the secondaries. Otherwise, replication jobs will be created and will be executed\n" +
- " eventually by Praefect itself.\n")
- fs.PrintDefaults()
+func newTrackRepositoriesCommand() *cli.Command {
+ return &cli.Command{
+ Name: trackRepositoriesCmdName,
+ Usage: "process bulk requests to track repositories in Praefect",
+ Description: "This command allows bulk requests for repositories to be tracked by Praefect.\n" +
+ "The -input-path flag must be the path of a file containing the details of the repositories\n" +
+ "to track as a list of newline-delimited JSON objects. Each line must contain the details for\n" +
+ "one and only one repository. Each item must contain the following keys:\n\n" +
+ " relative_path - The relative path of the repository on-disk.\n" +
+ " virtual_storage - The Praefect virtual storage name.\n" +
+ " authoritative_storage - Which storage to consider as the canonical copy of the repository.\n\n" +
+ "If -replicate-immediately is used, the command will attempt to replicate the repositories\n" +
+ "to the secondaries. Otherwise, replication jobs will be created and will be executed\n" +
+ "eventually by Praefect itself.\n",
+ HideHelpCommand: true,
+ Action: trackRepositoriesAction,
+ Flags: []cli.Flag{
+ &cli.BoolFlag{
+ Name: "replicate-immediately",
+ Usage: "kick off replication jobs immediately",
+ },
+ &cli.StringFlag{
+ Name: paramInputPath,
+ Usage: "path to file with details of repositories to track",
+ Required: true,
+ },
+ },
+ Before: func(ctx *cli.Context) error {
+ if ctx.Args().Present() {
+ _ = cli.ShowSubcommandHelp(ctx)
+ return cli.Exit(unexpectedPositionalArgsError{Command: ctx.Command.Name}, 1)
+ }
+ return nil
+ },
}
- return fs
}
-func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error {
- switch {
- case flags.NArg() > 0:
- return unexpectedPositionalArgsError{Command: flags.Name()}
+func trackRepositoriesAction(appCtx *cli.Context) error {
+ logger := log.Default()
+ conf, err := getConfig(logger, appCtx.String(configFlagName))
+ if err != nil {
+ return err
}
- ctx := correlation.ContextWithCorrelation(context.Background(), correlation.SafeRandomID())
- logger := cmd.logger.WithField("correlation_id", correlation.ExtractFromContext(ctx))
-
- openDBCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
- defer cancel()
- db, err := glsql.OpenDB(openDBCtx, cfg.DB)
+ db, clean, err := openDB(conf.DB, appCtx.App.ErrWriter)
if err != nil {
return fmt.Errorf("connect to database: %w", err)
}
- defer func() { _ = db.Close() }()
- store := datastore.NewPostgresRepositoryStore(db, cfg.StorageNames())
+ defer clean()
+
+ ctx := correlation.ContextWithCorrelation(context.Background(), correlation.SafeRandomID())
+ logger = logger.WithField("correlation_id", correlation.ExtractFromContext(ctx))
+
+ store := datastore.NewPostgresRepositoryStore(db, conf.StorageNames())
- f, err := os.Open(cmd.inputPath)
+ inputPath := appCtx.String(paramInputPath)
+ f, err := os.Open(inputPath)
if err != nil {
return fmt.Errorf("open input: %w", err)
}
@@ -95,7 +83,7 @@ func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error
scanner := bufio.NewScanner(f)
- fmt.Fprintf(cmd.w, "Validating repository information in %q\n", cmd.inputPath)
+ fmt.Fprintf(appCtx.App.Writer, "Validating repository information in %q\n", inputPath)
var requests []trackRepositoryRequest
var line int
@@ -164,7 +152,7 @@ func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error
continue
}
- authoritativeRepoExists, err := request.authoritativeRepositoryExists(ctx, cfg, logger, cmd.w, request.AuthoritativeStorage)
+ authoritativeRepoExists, err := request.authoritativeRepositoryExists(ctx, conf, logger, appCtx.App.Writer, request.AuthoritativeStorage)
if err != nil {
badReq.errs = append(badReq.errs, fmt.Errorf("checking repository on disk: %w", err))
} else if !authoritativeRepoExists {
@@ -179,17 +167,18 @@ func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error
}
if len(repoErrs) > 0 {
- printInvalidRequests(cmd.w, repoErrs, pathLines, cmd.inputPath)
+ printInvalidRequests(appCtx.App.Writer, repoErrs, pathLines, inputPath)
return fmt.Errorf("invalid entries found, aborting")
}
if len(requests) == 0 {
- return fmt.Errorf("no repository information found in %q", cmd.inputPath)
+ return fmt.Errorf("no repository information found in %q", inputPath)
}
- fmt.Fprintf(cmd.w, "All repository details are correctly formatted\n")
- fmt.Fprintf(cmd.w, "Tracking %v repositories in Praefect DB...\n", line)
+ fmt.Fprintf(appCtx.App.Writer, "All repository details are correctly formatted\n")
+ fmt.Fprintf(appCtx.App.Writer, "Tracking %v repositories in Praefect DB...\n", line)
+ replicateImmediately := appCtx.Bool("replicate-immediately")
for _, request := range requests {
- if err := request.execRequest(ctx, db, cfg, cmd.w, logger, cmd.replicateImmediately); err != nil {
+ if err := request.execRequest(ctx, db, conf, appCtx.App.Writer, logger, replicateImmediately); err != nil {
return fmt.Errorf("tracking repository %q: %w", request.RelativePath, err)
}
}
@@ -197,6 +186,21 @@ func (cmd trackRepositories) Exec(flags *flag.FlagSet, cfg config.Config) error
return nil
}
+type invalidRequest struct {
+ line int
+ path string
+ errs []error
+}
+
+type dupPathError struct {
+ path string
+ reqNums []int
+}
+
+func (d *dupPathError) Error() string {
+ return fmt.Sprintf("duplicate entries for relative_path, line %v", d.reqNums)
+}
+
func printInvalidRequests(w io.Writer, repoErrs []invalidRequest, pathLines map[string][]int, inputPath string) {
fmt.Fprintf(w, "Found %v invalid request(s) in %q:\n", len(repoErrs), inputPath)
diff --git a/internal/cli/praefect/subcmd_track_repositories_test.go b/internal/cli/praefect/subcmd_track_repositories_test.go
index 9c1314894..369bfc125 100644
--- a/internal/cli/praefect/subcmd_track_repositories_test.go
+++ b/internal/cli/praefect/subcmd_track_repositories_test.go
@@ -1,9 +1,7 @@
package praefect
import (
- "bytes"
"encoding/json"
- "flag"
"fmt"
"os"
"path/filepath"
@@ -24,176 +22,10 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "golang.org/x/exp/slices"
)
-func TestAddRepositories_FlagSet(t *testing.T) {
- t.Parallel()
- cmd := &trackRepositories{}
- fs := cmd.FlagSet()
- require.NoError(t, fs.Parse([]string{"--input-path", "/dev/stdin", "--replicate-immediately", "true"}))
- require.Equal(t, "/dev/stdin", cmd.inputPath)
- require.Equal(t, true, cmd.replicateImmediately)
-}
-
-func TestAddRepositories_Exec_invalidInput(t *testing.T) {
- t.Parallel()
- g1Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-1"))
- g2Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-2"))
-
- g1Srv := testserver.StartGitalyServer(t, g1Cfg, setup.RegisterAll, testserver.WithDisablePraefect())
- g2Srv := testserver.StartGitalyServer(t, g2Cfg, setup.RegisterAll, testserver.WithDisablePraefect())
- defer g2Srv.Shutdown()
- defer g1Srv.Shutdown()
- g1Addr := g1Srv.Address()
-
- db := testdb.New(t)
- dbConf := testdb.GetConfig(t, db.Name)
-
- virtualStorageName := "praefect"
- conf := config.Config{
- AllowLegacyElectors: true,
- SocketPath: testhelper.GetTemporaryGitalySocketFileName(t),
- VirtualStorages: []*config.VirtualStorage{
- {
- Name: virtualStorageName,
- Nodes: []*config.Node{
- {Storage: g1Cfg.Storages[0].Name, Address: g1Addr},
- {Storage: g2Cfg.Storages[0].Name, Address: g2Srv.Address()},
- },
- DefaultReplicationFactor: 2,
- },
- },
- DB: dbConf,
- Failover: config.Failover{
- Enabled: true,
- ElectionStrategy: config.ElectionStrategyPerRepository,
- },
- }
- rs := datastore.NewPostgresRepositoryStore(db, nil)
- ctx := testhelper.Context(t)
- inputFile := "input_file"
- logger := testhelper.NewDiscardingLogger(t)
-
- trackRepo := func(relativePath string) error {
- repositoryID, err := rs.ReserveRepositoryID(ctx, virtualStorageName, relativePath)
- if err != nil {
- return err
- }
- return rs.CreateRepository(
- ctx,
- repositoryID,
- virtualStorageName,
- relativePath,
- relativePath,
- g1Cfg.Storages[0].Name,
- nil,
- nil,
- false,
- false,
- )
- }
-
- invalidEntryErr := "invalid entries found, aborting"
-
- testCases := []struct {
- input string
- desc string
- expectedOutput string
- expectedError string
- trackedPath string
- }{
- {
- input: "",
- desc: "empty input",
- expectedError: "no repository information found",
- },
- {
- input: "@hashed/01/23/01234567890123456789.git",
- desc: "invalid JSON",
- expectedOutput: "invalid character '@' looking for beginning of value",
- expectedError: invalidEntryErr,
- },
- {
- input: `{"virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`,
- desc: "missing path",
- expectedOutput: `"repository" is a required parameter`,
- expectedError: invalidEntryErr,
- },
- {
- input: `{"virtual_storage":"foo","relative_path":"bar","authoritative_storage":"gitaly-1"}`,
- desc: "invalid virtual storage",
- expectedOutput: `virtual storage "foo" not found`,
- expectedError: invalidEntryErr,
- },
- {
- input: `{"relative_path":"not_a_repo","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`,
- desc: "repo does not exist",
- expectedOutput: "not a valid git repository",
- expectedError: invalidEntryErr,
- },
- {
- input: `{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"}
-{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"}`,
- desc: "duplicate path",
- expectedOutput: "duplicate entries for relative_path",
- expectedError: invalidEntryErr,
- },
- {
- input: `{"relative_path":"already_tracked","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`,
- desc: "repo is already tracked",
- expectedOutput: "repository is already tracked by Praefect",
- expectedError: invalidEntryErr,
- trackedPath: "already_tracked",
- },
- }
- for _, tc := range testCases {
- t.Run(tc.desc, func(t *testing.T) {
- nodeMgr, err := nodes.NewManager(
- testhelper.NewDiscardingLogEntry(t),
- conf,
- db.DB,
- nil,
- promtest.NewMockHistogramVec(),
- protoregistry.GitalyProtoPreregistered,
- nil,
- nil,
- nil,
- )
- require.NoError(t, err)
- nodeMgr.Start(0, time.Hour)
- defer nodeMgr.Stop()
-
- tempDir := testhelper.TempDir(t)
- f, err := os.Create(filepath.Join(tempDir, inputFile))
- require.NoError(t, err)
- _, err = f.Write([]byte(tc.input))
- require.NoError(t, err)
- require.NoError(t, f.Close())
-
- var stdout bytes.Buffer
-
- if tc.trackedPath != "" {
- require.NoError(t, trackRepo(tc.trackedPath))
- }
-
- addReposCmd := &trackRepositories{
- inputPath: filepath.Join(tempDir, inputFile),
- replicateImmediately: true,
- logger: logger,
- w: &stdout,
- }
- err = addReposCmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf)
- require.Error(t, err)
-
- if tc.expectedOutput != "" {
- require.Contains(t, stdout.String(), tc.expectedOutput)
- }
- require.Contains(t, err.Error(), tc.expectedError)
- })
- }
-}
-
-func TestAddRepositories_Exec(t *testing.T) {
+func TestTrackRepositoriesSubcommand(t *testing.T) {
t.Parallel()
g1Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-1"))
g2Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-2"))
@@ -210,7 +42,7 @@ func TestAddRepositories_Exec(t *testing.T) {
db := testdb.New(t)
dbConf := testdb.GetConfig(t, db.Name)
- virtualStorageName := "praefect"
+ const virtualStorageName = "praefect"
conf := config.Config{
AllowLegacyElectors: true,
SocketPath: testhelper.GetTemporaryGitalySocketFileName(t),
@@ -230,10 +62,11 @@ func TestAddRepositories_Exec(t *testing.T) {
ElectionStrategy: config.ElectionStrategyPerRepository,
},
}
+ confPath := writeConfigToFile(t, conf)
gitalyCC, err := client.Dial(g1Addr, nil)
require.NoError(t, err)
- defer func() { require.NoError(t, gitalyCC.Close()) }()
+ defer testhelper.MustClose(t, gitalyCC)
ctx := testhelper.Context(t)
gitaly1RepositoryClient := gitalypb.NewRepositoryServiceClient(gitalyCC)
@@ -251,53 +84,60 @@ func TestAddRepositories_Exec(t *testing.T) {
}
authoritativeStorage := g1Cfg.Storages[0].Name
- logger := testhelper.NewDiscardingLogger(t)
+
+ nodeMgr, err := nodes.NewManager(
+ testhelper.NewDiscardingLogEntry(t),
+ conf,
+ db.DB,
+ nil,
+ promtest.NewMockHistogramVec(),
+ protoregistry.GitalyProtoPreregistered,
+ nil,
+ nil,
+ nil,
+ )
+ require.NoError(t, err)
+ nodeMgr.Start(0, time.Hour)
+ defer nodeMgr.Stop()
+
+ repositoryStore := datastore.NewPostgresRepositoryStore(db, conf.StorageNames())
+ assignmentStore := datastore.NewAssignmentStore(db, conf.StorageNames())
t.Run("ok", func(t *testing.T) {
testCases := []struct {
- relativePaths []string
- desc string
- replicateImmediately bool
- expectedOutput string
+ desc string
+ input string
+ relativePaths []string
+ args func(inputPath string) []string
+ expectedOutput string
}{
{
- relativePaths: []string{"path/to/test/repo1", "path/to/test/repo2"},
- desc: "immediate replication",
- replicateImmediately: true,
- expectedOutput: "Finished replicating repository to \"gitaly-2\".\n",
+ desc: "immediate replication",
+ relativePaths: []string{"path/to/test/repo1", "path/to/test/repo2"},
+ args: func(inputPath string) []string {
+ return []string{"-replicate-immediately", "-input-path=" + inputPath}
+ },
+ expectedOutput: "Finished replicating repository to \"gitaly-2\".\n",
},
{
- relativePaths: []string{"path/to/test/repo3", "path/to/test/repo4"},
- desc: "no immediate replication",
- replicateImmediately: false,
- expectedOutput: "Added replication job to replicate repository to \"gitaly-2\".\n",
+ desc: "no immediate replication",
+ relativePaths: []string{"path/to/test/repo3", "path/to/test/repo4"},
+ args: func(inputPath string) []string {
+ return []string{"-input-path=" + inputPath}
+ },
+ expectedOutput: "Added replication job to replicate repository to \"gitaly-2\".\n",
},
}
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
- nodeMgr, err := nodes.NewManager(
- testhelper.NewDiscardingLogEntry(t),
- conf,
- db.DB,
- nil,
- promtest.NewMockHistogramVec(),
- protoregistry.GitalyProtoPreregistered,
- nil,
- nil,
- nil,
- )
- require.NoError(t, err)
- nodeMgr.Start(0, time.Hour)
- defer nodeMgr.Stop()
-
tempDir := testhelper.TempDir(t)
- input, err := os.Create(filepath.Join(tempDir, "input"))
+ inputPath := filepath.Join(tempDir, "input")
+ input, err := os.Create(inputPath)
require.NoError(t, err)
- repoDS := datastore.NewPostgresRepositoryStore(db, conf.StorageNames())
for _, path := range tc.relativePaths {
- exists, err := repoDS.RepositoryExists(ctx, virtualStorageName, path)
+ exists, err := repositoryStore.RepositoryExists(ctx, virtualStorageName, path)
require.NoError(t, err)
require.False(t, exists)
@@ -309,42 +149,36 @@ func TestAddRepositories_Exec(t *testing.T) {
// Write repo details to input file
repoEntry, err := json.Marshal(trackRepositoryRequest{RelativePath: path, VirtualStorage: virtualStorageName, AuthoritativeStorage: authoritativeStorage})
require.NoError(t, err)
- fmt.Fprintf(input, string(repoEntry)+"\n")
- }
-
- var stdout bytes.Buffer
-
- addRepoCmd := &trackRepositories{
- inputPath: filepath.Join(tempDir, "input"),
- replicateImmediately: tc.replicateImmediately,
- logger: logger,
- w: &stdout,
+ _, err = fmt.Fprintf(input, string(repoEntry)+"\n")
+ require.NoError(t, err)
}
+ require.NoError(t, input.Close())
- require.NoError(t, addRepoCmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf))
- assert.Contains(t, stdout.String(), tc.expectedOutput)
-
- as := datastore.NewAssignmentStore(db, conf.StorageNames())
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, trackRepositoriesCmdName}, tc.args(inputPath)...))
+ assert.Empty(t, stderr)
+ require.NoError(t, err)
+ assert.Contains(t, stdout, tc.expectedOutput)
+ replicateImmediately := slices.Contains(tc.args(inputPath), "-replicate-immediately")
for _, path := range tc.relativePaths {
- repositoryID, err := repoDS.GetRepositoryID(ctx, virtualStorageName, path)
+ repositoryID, err := repositoryStore.GetRepositoryID(ctx, virtualStorageName, path)
require.NoError(t, err)
- assignments, err := as.GetHostAssignments(ctx, virtualStorageName, repositoryID)
+ assignments, err := assignmentStore.GetHostAssignments(ctx, virtualStorageName, repositoryID)
require.NoError(t, err)
require.Len(t, assignments, 2)
assert.Contains(t, assignments, g1Cfg.Storages[0].Name)
assert.Contains(t, assignments, g2Cfg.Storages[0].Name)
- exists, err := repoDS.RepositoryExists(ctx, virtualStorageName, path)
+ exists, err := repositoryStore.RepositoryExists(ctx, virtualStorageName, path)
require.NoError(t, err)
assert.True(t, exists)
- if !tc.replicateImmediately {
+ if !replicateImmediately {
queue := datastore.NewPostgresReplicationEventQueue(db)
events, err := queue.Dequeue(ctx, virtualStorageName, g2Cfg.Storages[0].Name, 1)
require.NoError(t, err)
- assert.Len(t, events, 1)
+ require.Len(t, events, 1)
assert.Equal(t, path, events[0].Job.RelativePath)
} else {
require.DirExists(t, filepath.Join(g2Cfg.Storages[0].Path, path))
@@ -353,4 +187,123 @@ func TestAddRepositories_Exec(t *testing.T) {
})
}
})
+
+ trackRepo := func(relativePath string) error {
+ repositoryID, err := repositoryStore.ReserveRepositoryID(ctx, virtualStorageName, relativePath)
+ if err != nil {
+ return err
+ }
+ return repositoryStore.CreateRepository(
+ ctx,
+ repositoryID,
+ virtualStorageName,
+ relativePath,
+ relativePath,
+ g1Cfg.Storages[0].Name,
+ nil,
+ nil,
+ false,
+ false,
+ )
+ }
+
+ const invalidEntryErr = "invalid entries found, aborting"
+
+ t.Run("fail", func(t *testing.T) {
+ for _, tc := range []struct {
+ desc string
+ args func(inputPath string) []string
+ input string
+ expectedOutput string
+ expectedError string
+ trackedPath string
+ }{
+ {
+ desc: "positional arguments",
+ args: func(inputPath string) []string {
+ return []string{"-input-path=" + inputPath, "positional-arg"}
+ },
+ expectedError: "track-repositories doesn't accept positional arguments",
+ },
+ {
+ desc: "no required flag 'input-path'",
+ args: func(string) []string {
+ return nil
+ },
+ expectedError: `Required flag "input-path" not set`,
+ },
+ {
+ desc: "empty input",
+ input: "",
+ expectedError: "no repository information found",
+ },
+ {
+ desc: "invalid JSON",
+ input: "@hashed/01/23/01234567890123456789.git",
+ expectedOutput: "invalid character '@' looking for beginning of value",
+ expectedError: invalidEntryErr,
+ },
+ {
+ desc: "missing path",
+ input: `{"virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`,
+ expectedOutput: `"repository" is a required parameter`,
+ expectedError: invalidEntryErr,
+ },
+ {
+ desc: "invalid virtual storage",
+ input: `{"virtual_storage":"foo","relative_path":"bar","authoritative_storage":"gitaly-1"}`,
+ expectedOutput: `virtual storage "foo" not found`,
+ expectedError: invalidEntryErr,
+ },
+ {
+ desc: "repo does not exist",
+ input: `{"relative_path":"not_a_repo","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`,
+ expectedOutput: "not a valid git repository",
+ expectedError: invalidEntryErr,
+ },
+ {
+ desc: "duplicate path",
+ input: `{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"}
+{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"}`,
+ expectedOutput: "duplicate entries for relative_path",
+ expectedError: invalidEntryErr,
+ },
+ {
+ desc: "repo is already tracked",
+ input: `{"relative_path":"already_tracked","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`,
+ expectedOutput: "repository is already tracked by Praefect",
+ expectedError: invalidEntryErr,
+ trackedPath: "already_tracked",
+ },
+ } {
+ tc := tc
+ t.Run(tc.desc, func(t *testing.T) {
+ t.Parallel()
+ tempDir := testhelper.TempDir(t)
+ inputPath := filepath.Join(tempDir, "input_file")
+ f, err := os.Create(inputPath)
+ require.NoError(t, err)
+ _, err = f.Write([]byte(tc.input))
+ require.NoError(t, err)
+ require.NoError(t, f.Close())
+
+ if tc.trackedPath != "" {
+ require.NoError(t, trackRepo(tc.trackedPath))
+ }
+
+ args := []string{"-replicate-immediately", "-input-path=" + inputPath}
+ if tc.args != nil {
+ args = tc.args(inputPath)
+ }
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, trackRepositoriesCmdName}, args...))
+ assert.Empty(t, stderr)
+ require.Error(t, err)
+
+ if tc.expectedOutput != "" {
+ require.Contains(t, stdout, tc.expectedOutput)
+ }
+ require.Contains(t, err.Error(), tc.expectedError)
+ })
+ }
+ })
}
diff --git a/internal/cli/praefect/subcmd_track_repository_test.go b/internal/cli/praefect/subcmd_track_repository_test.go
index d721e51d9..d8e11e6e2 100644
--- a/internal/cli/praefect/subcmd_track_repository_test.go
+++ b/internal/cli/praefect/subcmd_track_repository_test.go
@@ -1,15 +1,12 @@
package praefect
import (
- "bytes"
- "io"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/client"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
@@ -104,27 +101,6 @@ func TestTrackRepositorySubcommand(t *testing.T) {
true,
))
- runCmd := func(t *testing.T, args []string) (string, error) {
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newTrackRepositoryCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err := app.Run(append([]string{progname, trackRepositoryCmdName}, args...))
- return stdout.String(), err
- }
-
t.Run("fails", func(t *testing.T) {
for _, tc := range []struct {
name string
@@ -171,7 +147,8 @@ func TestTrackRepositorySubcommand(t *testing.T) {
},
} {
t.Run(tc.name, func(t *testing.T) {
- _, err := runCmd(t, tc.args)
+ _, stderr, err := runApp(append([]string{"-config", confPath, trackRepositoryCmdName}, tc.args...))
+ assert.Empty(t, stderr)
require.EqualError(t, err, tc.errorMsg)
})
}
@@ -245,7 +222,8 @@ func TestTrackRepositorySubcommand(t *testing.T) {
if tc.replicateImmediately {
args = append(args, "-replicate-immediately")
}
- stdout, err := runCmd(t, args)
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, trackRepositoryCmdName}, args...))
+ assert.Empty(t, stderr)
require.NoError(t, err)
as := datastore.NewAssignmentStore(db, conf.StorageNames())
@@ -288,7 +266,9 @@ func TestTrackRepositorySubcommand(t *testing.T) {
require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, relativePath))
require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, relativePath))
- _, err := runCmd(t, []string{
+ _, _, err := runApp([]string{
+ "-config", confPath,
+ trackRepositoryCmdName,
"-virtual-storage", virtualStorageName,
"-repository", relativePath,
"-authoritative-storage", authoritativeStorage,
@@ -296,12 +276,15 @@ func TestTrackRepositorySubcommand(t *testing.T) {
require.NoError(t, err)
// running the command twice means we try creating the replication event
// again, which should log the duplicate but not break the flow.
- stdout, err := runCmd(t, []string{
+ stdout, stderr, err := runApp([]string{
+ "-config", confPath,
+ trackRepositoryCmdName,
"-virtual-storage", virtualStorageName,
"-repository", relativePath,
"-authoritative-storage", authoritativeStorage,
})
require.NoError(t, err)
+ assert.Empty(t, stderr)
assert.Contains(t, stdout, "replication event queue already has similar entry: replication event \"\" -> \"praefect\" -> \"gitaly-2\" -> \"path/to/test/repo_3\" already exists.")
})
}
diff --git a/internal/cli/praefect/subcmd_verify_test.go b/internal/cli/praefect/subcmd_verify_test.go
index 901579f17..9746f4d05 100644
--- a/internal/cli/praefect/subcmd_verify_test.go
+++ b/internal/cli/praefect/subcmd_verify_test.go
@@ -1,12 +1,11 @@
package praefect
import (
- "bytes"
"errors"
"fmt"
- "io"
"testing"
+ "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/urfave/cli/v2"
"gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config"
@@ -207,29 +206,14 @@ func TestVerifySubcommand(t *testing.T) {
}
confPath := writeConfigToFile(t, conf)
- var stdout bytes.Buffer
- app := cli.App{
- Reader: bytes.NewReader(nil),
- Writer: &stdout,
- ErrWriter: io.Discard,
- HideHelpCommand: true,
- Commands: []*cli.Command{
- newVerifyCommand(),
- },
- Flags: []cli.Flag{
- &cli.StringFlag{
- Name: "config",
- Value: confPath,
- },
- },
- }
- err = app.Run(append([]string{progname, verifyCmdName}, tc.args...))
+ stdout, stderr, err := runApp(append([]string{"-config", confPath, verifyCmdName}, tc.args...))
+ assert.Empty(t, stderr)
testhelper.RequireGrpcError(t, tc.error, err)
if tc.error != nil {
return
}
- require.Equal(t, fmt.Sprintf("%d replicas marked unverified\n", tc.replicasMarked), stdout.String())
+ require.Equal(t, fmt.Sprintf("%d replicas marked unverified\n", tc.replicasMarked), stdout)
actualState := state{}
rows, err := db.QueryContext(ctx, `
diff --git a/internal/featureflag/ff_git_v241.go b/internal/featureflag/ff_git_v241.go
new file mode 100644
index 000000000..703aaf162
--- /dev/null
+++ b/internal/featureflag/ff_git_v241.go
@@ -0,0 +1,9 @@
+package featureflag
+
+// GitV241 enables the use of Git v2.41.
+var GitV241 = NewFeatureFlag(
+ "git_v241",
+ "v16.1.0",
+ "https://gitlab.com/gitlab-org/gitaly/-/issues/5336",
+ false,
+)
diff --git a/internal/git/execution_environment.go b/internal/git/execution_environment.go
index 504dd7c68..2948030f0 100644
--- a/internal/git/execution_environment.go
+++ b/internal/git/execution_environment.go
@@ -26,6 +26,12 @@ var (
// case `IsEnabled()` returns `false` though.
defaultExecutionEnvironmentConstructors = []ExecutionEnvironmentConstructor{
BundledGitEnvironmentConstructor{
+ Suffix: "-v2.41",
+ FeatureFlags: []featureflag.FeatureFlag{
+ featureflag.GitV241,
+ },
+ },
+ BundledGitEnvironmentConstructor{
Suffix: "-v2.40",
},
DistributedGitEnvironmentConstructor{},
diff --git a/internal/gitaly/service/internalgitaly/backup_repos.go b/internal/gitaly/service/internalgitaly/backup_repos.go
index ff43b5b5e..7fa56d583 100644
--- a/internal/gitaly/service/internalgitaly/backup_repos.go
+++ b/internal/gitaly/service/internalgitaly/backup_repos.go
@@ -37,17 +37,14 @@ func (s server) BackupRepos(stream gitalypb.InternalGitaly_BackupReposServer) er
return structerr.NewInvalidArgument("backup repos: resolve locator: %w", err)
}
- manager := backup.NewManagerLocal(sink, locator, s.locator, s.gitCmdFactory, s.catfileCache, backupID)
+ manager := backup.NewManagerLocal(sink, locator, s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, backupID)
pipeline := backup.NewLoggingPipeline(ctxlogrus.Extract(ctx))
for {
for _, repo := range request.GetRepositories() {
pipeline.Handle(ctx, backup.NewCreateCommand(
manager,
- // ServerInfo will be removed once restore methods are added to
- // backup.Repository. Even though it is unused it must be
- // non-zero so that storage.ExtractGitalyServer is not called.
- storage.ServerInfo{Address: "unused"},
+ storage.ServerInfo{},
repo,
false,
))
diff --git a/internal/gitaly/service/internalgitaly/backup_repos_test.go b/internal/gitaly/service/internalgitaly/backup_repos_test.go
index 6b00d4da0..8adb11524 100644
--- a/internal/gitaly/service/internalgitaly/backup_repos_test.go
+++ b/internal/gitaly/service/internalgitaly/backup_repos_test.go
@@ -12,6 +12,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
@@ -120,11 +121,14 @@ func TestServerBackupRepos(t *testing.T) {
catfileCache := catfile.NewCache(cfg)
t.Cleanup(catfileCache.Stop)
+ txManager := transaction.NewTrackingManager()
+
srv := NewServer(
cfg.Storages,
config.NewLocator(cfg),
gittest.NewCommandFactory(t, cfg),
catfileCache,
+ txManager,
)
client := setupInternalGitalyService(t, cfg, srv)
diff --git a/internal/gitaly/service/internalgitaly/server.go b/internal/gitaly/service/internalgitaly/server.go
index d0ce88352..41231fcb1 100644
--- a/internal/gitaly/service/internalgitaly/server.go
+++ b/internal/gitaly/service/internalgitaly/server.go
@@ -5,6 +5,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
)
@@ -14,6 +15,7 @@ type server struct {
locator storage.Locator
gitCmdFactory git.CommandFactory
catfileCache catfile.Cache
+ txManager transaction.Manager
}
// NewServer return an instance of the Gitaly service.
@@ -22,11 +24,13 @@ func NewServer(
locator storage.Locator,
gitCmdFactory git.CommandFactory,
catfileCache catfile.Cache,
+ txManager transaction.Manager,
) gitalypb.InternalGitalyServer {
return &server{
storages: storages,
locator: locator,
gitCmdFactory: gitCmdFactory,
catfileCache: catfileCache,
+ txManager: txManager,
}
}
diff --git a/internal/gitaly/service/internalgitaly/walkrepos_test.go b/internal/gitaly/service/internalgitaly/walkrepos_test.go
index de4e5b23d..a1bfa769a 100644
--- a/internal/gitaly/service/internalgitaly/walkrepos_test.go
+++ b/internal/gitaly/service/internalgitaly/walkrepos_test.go
@@ -12,6 +12,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest"
"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config"
+ "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
@@ -74,6 +75,8 @@ func TestWalkRepos(t *testing.T) {
catfileCache := catfile.NewCache(cfg)
t.Cleanup(catfileCache.Stop)
+ txManager := transaction.NewTrackingManager()
+
// to test a directory being deleted during a walk, we must delete a directory after
// the file walk has started. To achieve that, we wrap the server to pass down a wrapped
// stream that allows us to hook in to stream responses. We then delete 'b' when
@@ -84,6 +87,7 @@ func TestWalkRepos(t *testing.T) {
config.NewLocator(cfg),
gittest.NewCommandFactory(t, cfg),
catfileCache,
+ txManager,
)
wsrv := &serverWrapper{
srv,
diff --git a/internal/gitaly/service/setup/register.go b/internal/gitaly/service/setup/register.go
index 50f0482c1..0a0177cfe 100644
--- a/internal/gitaly/service/setup/register.go
+++ b/internal/gitaly/service/setup/register.go
@@ -149,6 +149,7 @@ func RegisterAll(srv *grpc.Server, deps *service.Dependencies) {
deps.GetLocator(),
deps.GetGitCmdFactory(),
deps.GetCatfileCache(),
+ deps.GetTxManager(),
))
healthpb.RegisterHealthServer(srv, health.NewServer())
diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go
index af54831d9..c6132a2f6 100644
--- a/internal/testhelper/testhelper.go
+++ b/internal/testhelper/testhelper.go
@@ -195,6 +195,9 @@ func ContextWithoutCancel(opts ...ContextOpt) context.Context {
// Randomly enable the use of the catfile cache in localrepo.ReadObject.
ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.LocalrepoReadObjectCached, rnd.Int()%2 == 0)
+ // Randomly enable either Git v2.40 or Git v2.41.
+ ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.GitV241, rnd.Int()%2 == 0)
+
for _, opt := range opts {
ctx = opt(ctx)
}
diff --git a/tools/goimports/go.mod b/tools/goimports/go.mod
index ae7ae49e8..35cdca40f 100644
--- a/tools/goimports/go.mod
+++ b/tools/goimports/go.mod
@@ -2,7 +2,7 @@ module gitlab.com/gitlab-org/gitaly/tools/goimports
go 1.19
-require golang.org/x/tools v0.9.1
+require golang.org/x/tools v0.9.3
require (
golang.org/x/mod v0.10.0 // indirect
diff --git a/tools/goimports/go.sum b/tools/goimports/go.sum
index 7995eec85..5a2e60559 100644
--- a/tools/goimports/go.sum
+++ b/tools/goimports/go.sum
@@ -2,5 +2,5 @@ golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
-golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
+golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
diff --git a/tools/golangci-lint/go.mod b/tools/golangci-lint/go.mod
index adc35efc7..a3ddbf697 100644
--- a/tools/golangci-lint/go.mod
+++ b/tools/golangci-lint/go.mod
@@ -6,7 +6,7 @@ require github.com/golangci/golangci-lint v1.52.2
require (
github.com/spf13/viper v1.12.0
- golang.org/x/tools v0.8.0
+ golang.org/x/tools v0.9.3
)
require (
@@ -166,8 +166,8 @@ require (
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/exp/typeparams v0.0.0-20230224173230-c95f2b4c22f2 // indirect
golang.org/x/mod v0.10.0 // indirect
- golang.org/x/sync v0.1.0 // indirect
- golang.org/x/sys v0.7.0 // indirect
+ golang.org/x/sync v0.2.0 // indirect
+ golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.7.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
diff --git a/tools/golangci-lint/go.sum b/tools/golangci-lint/go.sum
index d4d9cb469..a7d05da6b 100644
--- a/tools/golangci-lint/go.sum
+++ b/tools/golangci-lint/go.sum
@@ -649,7 +649,7 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -672,8 +672,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
+golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -733,8 +734,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -828,8 +829,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
-golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
+golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM=
+golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=