diff options
author | Patrick Steinhardt <psteinhardt@gitlab.com> | 2023-08-23 14:09:28 +0300 |
---|---|---|
committer | Will Chandler <wchandler@gitlab.com> | 2023-09-06 19:05:58 +0300 |
commit | 23bf5ded84c910e4337b9999e9fec4982fc9a72e (patch) | |
tree | a335de523371ea37b08fc1304478ac8bb75de2f0 | |
parent | 262126b32d3ab7e30671b61516b71b8ec54778a9 (diff) |
Merge branch 'wc/track-repo-replica-path' into 'master'
praefect: Handle replica paths in 'track-repository' and 'track-repositories' subcommands
Closes #5402
See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/6251
Merged-by: Patrick Steinhardt <psteinhardt@gitlab.com>
Approved-by: Patrick Steinhardt <psteinhardt@gitlab.com>
Approved-by: Evan Read <eread@gitlab.com>
Approved-by: Justin Tobler <jtobler@gitlab.com>
Reviewed-by: Patrick Steinhardt <psteinhardt@gitlab.com>
Reviewed-by: Will Chandler <wchandler@gitlab.com>
Reviewed-by: Evan Read <eread@gitlab.com>
Co-authored-by: Will Chandler <wchandler@gitlab.com>
(cherry picked from commit 5649e8da1e31fcaa495da9420a56da09b61236f6)
f05c9a2b praefect: Assert more track-repositories output
02c2260a praefect: Rename 'repository' to 'relative-path'
6efd1c0c praefect: Use replica paths in track-repository
bfbb8f66 praefect: Use replica paths in track-repositories
-rw-r--r-- | internal/cli/praefect/subcmd.go | 3 | ||||
-rw-r--r-- | internal/cli/praefect/subcmd_accept_dataloss_test.go | 16 | ||||
-rw-r--r-- | internal/cli/praefect/subcmd_remove_repository_test.go | 20 | ||||
-rw-r--r-- | internal/cli/praefect/subcmd_set_replication_factor_test.go | 22 | ||||
-rw-r--r-- | internal/cli/praefect/subcmd_track_repositories.go | 19 | ||||
-rw-r--r-- | internal/cli/praefect/subcmd_track_repositories_test.go | 147 | ||||
-rw-r--r-- | internal/cli/praefect/subcmd_track_repository.go | 22 | ||||
-rw-r--r-- | internal/cli/praefect/subcmd_track_repository_test.go | 52 |
8 files changed, 207 insertions, 94 deletions
diff --git a/internal/cli/praefect/subcmd.go b/internal/cli/praefect/subcmd.go index fefcefa8d..aa7a2d623 100644 --- a/internal/cli/praefect/subcmd.go +++ b/internal/cli/praefect/subcmd.go @@ -19,7 +19,8 @@ import ( const ( defaultDialTimeout = 10 * time.Second paramVirtualStorage = "virtual-storage" - paramRelativePath = "repository" + paramRelativePath = "relative-path" + paramReplicaPath = "replica-path" paramAuthoritativeStorage = "authoritative-storage" ) diff --git a/internal/cli/praefect/subcmd_accept_dataloss_test.go b/internal/cli/praefect/subcmd_accept_dataloss_test.go index d537ec1c8..73b239615 100644 --- a/internal/cli/praefect/subcmd_accept_dataloss_test.go +++ b/internal/cli/praefect/subcmd_accept_dataloss_test.go @@ -81,25 +81,25 @@ func TestAcceptDatalossSubcommand(t *testing.T) { }{ { desc: "missing virtual storage", - args: []string{"-repository", "test-repository-1", "-authoritative-storage", "test-physical-storage-2"}, + args: []string{"-relative-path", "test-repository-1", "-authoritative-storage", "test-physical-storage-2"}, matchError: matchErrorMsg(`Required flag "virtual-storage" not set`), expectedGenerations: startingGenerations, }, { desc: "missing repository", args: []string{"-virtual-storage", "test-virtual-storage-1", "-authoritative-storage", "test-physical-storage-2"}, - matchError: matchErrorMsg(`Required flag "repository" not set`), + matchError: matchErrorMsg(`Required flag "relative-path" not set`), expectedGenerations: startingGenerations, }, { desc: "missing authoritative storage", - args: []string{"-virtual-storage", "test-virtual-storage-1", "-repository", "test-repository-1"}, + args: []string{"-virtual-storage", "test-virtual-storage-1", "-relative-path", "test-repository-1"}, matchError: matchErrorMsg(`Required flag "authoritative-storage" not set`), expectedGenerations: startingGenerations, }, { desc: "positional arguments", - args: []string{"-virtual-storage", "test-virtual-storage-1", "-repository", "test-repository-1", "-authoritative-storage", "test-physical-storage-2", "positional-arg"}, + args: []string{"-virtual-storage", "test-virtual-storage-1", "-relative-path", "test-repository-1", "-authoritative-storage", "test-physical-storage-2", "positional-arg"}, matchError: func(t *testing.T, actual error) { t.Helper() require.Equal(t, unexpectedPositionalArgsError{Command: "accept-dataloss"}, actual) @@ -108,25 +108,25 @@ func TestAcceptDatalossSubcommand(t *testing.T) { }, { desc: "non-existent virtual storage", - args: []string{"-virtual-storage", "non-existent", "-repository", "test-repository-1", "-authoritative-storage", "test-physical-storage-2"}, + args: []string{"-virtual-storage", "non-existent", "-relative-path", "test-repository-1", "-authoritative-storage", "test-physical-storage-2"}, matchError: matchErrorMsg(`set authoritative storage: rpc error: code = InvalidArgument desc = unknown virtual storage: "non-existent"`), expectedGenerations: startingGenerations, }, { desc: "non-existent authoritative storage", - args: []string{"-virtual-storage", "test-virtual-storage-1", "-repository", "test-repository-1", "-authoritative-storage", "non-existent"}, + args: []string{"-virtual-storage", "test-virtual-storage-1", "-relative-path", "test-repository-1", "-authoritative-storage", "non-existent"}, matchError: matchErrorMsg(`set authoritative storage: rpc error: code = InvalidArgument desc = unknown authoritative storage: "non-existent"`), expectedGenerations: startingGenerations, }, { desc: "non-existent repository", - args: []string{"-virtual-storage", "test-virtual-storage-1", "-repository", "non-existent", "-authoritative-storage", "test-physical-storage-2"}, + args: []string{"-virtual-storage", "test-virtual-storage-1", "-relative-path", "non-existent", "-authoritative-storage", "test-physical-storage-2"}, matchError: matchErrorMsg(`set authoritative storage: rpc error: code = InvalidArgument desc = repository does not exist on virtual storage`), expectedGenerations: startingGenerations, }, { desc: "success", - args: []string{"-virtual-storage", "test-virtual-storage-1", "-repository", "test-repository-1", "-authoritative-storage", "test-physical-storage-2"}, + args: []string{"-virtual-storage", "test-virtual-storage-1", "-relative-path", "test-repository-1", "-authoritative-storage", "test-physical-storage-2"}, matchError: func(t *testing.T, actual error) { t.Helper() require.NoError(t, actual) diff --git a/internal/cli/praefect/subcmd_remove_repository_test.go b/internal/cli/praefect/subcmd_remove_repository_test.go index 21b88de78..8b872396e 100644 --- a/internal/cli/praefect/subcmd_remove_repository_test.go +++ b/internal/cli/praefect/subcmd_remove_repository_test.go @@ -88,7 +88,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { { desc: "positional arguments", args: func(*testing.T, *gitalypb.Repository, string) []string { - return []string{"-virtual-storage=vs", "-repository=r", "positional-arg"} + return []string{"-virtual-storage=vs", "-relative-path=r", "positional-arg"} }, assertError: func(t *testing.T, err error, _ *gitalypb.Repository, _ string) { assert.Equal(t, cli.Exit(unexpectedPositionalArgsError{Command: "remove-repository"}, 1), err) @@ -97,7 +97,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { { desc: "virtual-storage is not set", args: func(*testing.T, *gitalypb.Repository, string) []string { - return []string{"-repository=r"} + return []string{"-relative-path=r"} }, assertError: func(t *testing.T, err error, _ *gitalypb.Repository, _ string) { assert.EqualError(t, err, `Required flag "virtual-storage" not set`) @@ -109,7 +109,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { return []string{"-virtual-storage=vs"} }, assertError: func(t *testing.T, err error, _ *gitalypb.Repository, _ string) { - assert.EqualError(t, err, `Required flag "repository" not set`) + assert.EqualError(t, err, `Required flag "relative-path" not set`) }, }, { @@ -141,7 +141,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { return writeConfigToFile(t, conf) }, args: func(*testing.T, *gitalypb.Repository, string) []string { - return []string{"-virtual-storage=vs", "-repository=r"} + return []string{"-virtual-storage=vs", "-relative-path=r"} }, assertError: func(t *testing.T, err error, _ *gitalypb.Repository, _ string) { require.Contains(t, err.Error(), "connect to database: send ping: failed to connect to ") @@ -150,7 +150,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { { desc: "dry run", args: func(_ *testing.T, repo *gitalypb.Repository, _ string) []string { - return []string{"-virtual-storage", repo.StorageName, "-repository", repo.RelativePath} + return []string{"-virtual-storage", repo.StorageName, "-relative-path", repo.RelativePath} }, assertError: func(t *testing.T, err error, repo *gitalypb.Repository, replicaPath string) { require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, replicaPath)) @@ -167,7 +167,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { args: func(t *testing.T, repo *gitalypb.Repository, replicaPath string) []string { require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, replicaPath)) require.DirExists(t, filepath.Join(g2Cfg.Storages[0].Path, replicaPath)) - return []string{"-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply"} + return []string{"-virtual-storage", repo.StorageName, "-relative-path", repo.RelativePath, "-apply"} }, assertError: func(t *testing.T, err error, repo *gitalypb.Repository, replicaPath string) { require.NoError(t, err) @@ -184,7 +184,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { { desc: "db only", args: func(t *testing.T, repo *gitalypb.Repository, _ string) []string { - return []string{"-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply", "-db-only"} + return []string{"-virtual-storage", repo.StorageName, "-relative-path", repo.RelativePath, "-apply", "-db-only"} }, assertError: func(t *testing.T, err error, repo *gitalypb.Repository, replicaPath string) { require.NoError(t, err) @@ -209,7 +209,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { desc: "repository doesnt exist on one gitaly", args: func(t *testing.T, repo *gitalypb.Repository, replicaPath string) []string { require.NoError(t, os.RemoveAll(filepath.Join(g2Cfg.Storages[0].Path, replicaPath))) - return []string{"-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply"} + return []string{"-virtual-storage", repo.StorageName, "-relative-path", repo.RelativePath, "-apply"} }, assertError: func(t *testing.T, err error, repo *gitalypb.Repository, replicaPath string) { require.NoDirExists(t, filepath.Join(g1Cfg.Storages[0].Path, replicaPath)) @@ -228,7 +228,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { repoStore := datastore.NewPostgresRepositoryStore(db.DB, nil) _, _, err = repoStore.DeleteRepository(ctx, repo.StorageName, repo.RelativePath) require.NoError(t, err) - return []string{"-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply"} + return []string{"-virtual-storage", repo.StorageName, "-relative-path", repo.RelativePath, "-apply"} }, assertError: func(t *testing.T, err error, repo *gitalypb.Repository, replicaPath string) { require.EqualError(t, err, "repository is not being tracked in Praefect") @@ -258,7 +258,7 @@ func TestRemoveRepositorySubcommand(t *testing.T) { repo := createRepo(t, ctx, repoClient, praefectStorage, t.Name()) g2Srv.Shutdown() replicaPath := gittest.GetReplicaPath(t, ctx, gitalycfg.Cfg{SocketPath: praefectServer.Address()}, repo) - stdout, stderr, err := runApp([]string{"-config", confPath, "remove-repository", "-virtual-storage", repo.StorageName, "-repository", repo.RelativePath, "-apply"}) + stdout, stderr, err := runApp([]string{"-config", confPath, "remove-repository", "-virtual-storage", repo.StorageName, "-relative-path", repo.RelativePath, "-apply"}) assert.Empty(t, stderr) require.NoError(t, err) assert.Contains(t, stdout, "Repository removal completed.") diff --git a/internal/cli/praefect/subcmd_set_replication_factor_test.go b/internal/cli/praefect/subcmd_set_replication_factor_test.go index f7878575d..c1d33f8b6 100644 --- a/internal/cli/praefect/subcmd_set_replication_factor_test.go +++ b/internal/cli/praefect/subcmd_set_replication_factor_test.go @@ -30,53 +30,53 @@ func TestSetReplicationFactorSubcommand(t *testing.T) { }{ { desc: "unexpected positional arguments", - args: []string{"-virtual-storage=virtual-storage", "-repository=relative-path", "-replication-factor=1", "positonal-arg"}, + args: []string{"-virtual-storage=virtual-storage", "-relative-path=relative-path", "-replication-factor=1", "positonal-arg"}, error: cli.Exit(unexpectedPositionalArgsError{Command: "set-replication-factor"}, 1), }, { desc: "missing virtual-storage", - args: []string{"-repository=relative-path", "-replication-factor=1"}, + args: []string{"-relative-path=relative-path", "-replication-factor=1"}, error: errors.New(`Required flag "virtual-storage" not set`), }, { - desc: "missing repository", + desc: "missing relative-path", args: []string{"-virtual-storage=virtual-storage", "-replication-factor=1"}, - error: errors.New(`Required flag "repository" not set`), + error: errors.New(`Required flag "relative-path" not set`), }, { desc: "missing replication-factor", - args: []string{"-virtual-storage=virtual-storage", "-repository=relative-path"}, + args: []string{"-virtual-storage=virtual-storage", "-relative-path=relative-path"}, error: errors.New(`Required flag "replication-factor" not set`), }, { desc: "replication factor too small", - args: []string{"-virtual-storage=virtual-storage", "-repository=relative-path", "-replication-factor=0"}, + args: []string{"-virtual-storage=virtual-storage", "-relative-path=relative-path", "-replication-factor=0"}, error: status.Error(codes.InvalidArgument, "set replication factor: attempted to set replication factor 0 but minimum is 1"), }, { desc: "replication factor too big", - args: []string{"-virtual-storage=virtual-storage", "-repository=relative-path", "-replication-factor=3"}, + args: []string{"-virtual-storage=virtual-storage", "-relative-path=relative-path", "-replication-factor=3"}, error: status.Error(codes.InvalidArgument, "set replication factor: attempted to set replication factor 3 but virtual storage only contains 2 storages"), }, { desc: "virtual storage not found", - args: []string{"-virtual-storage=non-existent", "-repository=relative-path", "-replication-factor=2"}, + args: []string{"-virtual-storage=non-existent", "-relative-path=relative-path", "-replication-factor=2"}, error: status.Error(codes.InvalidArgument, `set replication factor: virtual storage "non-existent" not found`), }, { desc: "repository not found", - args: []string{"-virtual-storage=virtual-storage", "-repository=non-existent", "-replication-factor=2"}, + args: []string{"-virtual-storage=virtual-storage", "-relative-path=non-existent", "-replication-factor=2"}, error: status.Error(codes.InvalidArgument, `set replication factor: repository "virtual-storage"/"non-existent" not found`), }, { desc: "assignments are disabled", - args: []string{"-virtual-storage=virtual-storage", "-repository=relative-path", "-replication-factor=1"}, + args: []string{"-virtual-storage=virtual-storage", "-relative-path=relative-path", "-replication-factor=1"}, store: praefect.NewDisabledAssignmentStore(nil), error: status.Error(codes.Internal, `set replication factor: assignments are disabled`), }, { desc: "successfully set", - args: []string{"-virtual-storage=virtual-storage", "-repository=relative-path", "-replication-factor=2"}, + args: []string{"-virtual-storage=virtual-storage", "-relative-path=relative-path", "-replication-factor=2"}, stdout: "current assignments: primary, secondary\n", }, } { diff --git a/internal/cli/praefect/subcmd_track_repositories.go b/internal/cli/praefect/subcmd_track_repositories.go index ea61ebe8a..9aa5470ec 100644 --- a/internal/cli/praefect/subcmd_track_repositories.go +++ b/internal/cli/praefect/subcmd_track_repositories.go @@ -27,7 +27,8 @@ func newTrackRepositoriesCommand() *cli.Command { "The -input-path flag must be the path of a file containing the details of the repositories\n" + "to track as a list of newline-delimited JSON objects. Each line must contain the details for\n" + "one and only one repository. Each item must contain the following keys:\n\n" + - " relative_path - The relative path of the repository on-disk.\n" + + " relative_path - The relative path on the virtual storage. Usually starts with '@hashed'\n" + + " replica_path - The relative path on physical storage. Can start with '@cluster' or match 'relative_path'.\n" + " virtual_storage - The Praefect virtual storage name.\n" + " authoritative_storage - Which storage to consider as the canonical copy of the repository.\n\n" + "If -replicate-immediately is used, the command will attempt to replicate the repositories\n" + @@ -110,7 +111,12 @@ func trackRepositoriesAction(appCtx *cli.Context) error { if request.RelativePath == "" { badReq.errs = append(badReq.errs, requiredParameterError(paramRelativePath)) } - badReq.path = request.RelativePath + badReq.relativePath = request.RelativePath + + if request.ReplicaPath == "" { + badReq.errs = append(badReq.errs, requiredParameterError(paramReplicaPath)) + } + badReq.replicaPath = request.ReplicaPath if request.VirtualStorage == "" { badReq.errs = append(badReq.errs, requiredParameterError(paramVirtualStorage)) @@ -187,9 +193,10 @@ func trackRepositoriesAction(appCtx *cli.Context) error { } type invalidRequest struct { - line int - path string - errs []error + line int + relativePath string + replicaPath string + errs []error } type dupPathError struct { @@ -205,7 +212,7 @@ func printInvalidRequests(w io.Writer, repoErrs []invalidRequest, pathLines map[ fmt.Fprintf(w, "Found %v invalid request(s) in %q:\n", len(repoErrs), inputPath) for _, l := range repoErrs { - fmt.Fprintf(w, " line %v, relative_path: %q\n", l.line, l.path) + fmt.Fprintf(w, " line %v, relative_path: %q, replica_path: %q\n", l.line, l.relativePath, l.replicaPath) for _, err := range l.errs { if dup, ok := err.(*dupPathError); ok { // The complete set of duplicate reqNums won't be known until input is diff --git a/internal/cli/praefect/subcmd_track_repositories_test.go b/internal/cli/praefect/subcmd_track_repositories_test.go index 369bfc125..ccf060ef0 100644 --- a/internal/cli/praefect/subcmd_track_repositories_test.go +++ b/internal/cli/praefect/subcmd_track_repositories_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "time" @@ -108,10 +109,20 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { desc string input string relativePaths []string + replicaPaths []string args func(inputPath string) []string expectedOutput string }{ { + desc: "replica path differs from relative path", + relativePaths: []string{"path/to/test/diff_repo1", "path/to/test/diff_repo2"}, + replicaPaths: []string{"path/to/replica/diff_repo1", "path/to/replica/diff_repo2"}, + args: func(inputPath string) []string { + return []string{"-replicate-immediately", "-input-path=" + inputPath} + }, + expectedOutput: "Finished replicating repository to \"gitaly-2\".\n", + }, + { desc: "immediate replication", relativePaths: []string{"path/to/test/repo1", "path/to/test/repo2"}, args: func(inputPath string) []string { @@ -136,18 +147,28 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { input, err := os.Create(inputPath) require.NoError(t, err) - for _, path := range tc.relativePaths { - exists, err := repositoryStore.RepositoryExists(ctx, virtualStorageName, path) + if len(tc.replicaPaths) == 0 { + tc.replicaPaths = tc.relativePaths + } + + for i, relPath := range tc.relativePaths { + exists, err := repositoryStore.RepositoryExists(ctx, virtualStorageName, relPath) require.NoError(t, err) require.False(t, exists) + replicaPath := tc.replicaPaths[i] // create the repo on Gitaly without Praefect knowing - require.NoError(t, createRepoThroughGitaly1(path)) - require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, path)) - require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, path)) + require.NoError(t, createRepoThroughGitaly1(replicaPath)) + require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, replicaPath)) + require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, replicaPath)) // Write repo details to input file - repoEntry, err := json.Marshal(trackRepositoryRequest{RelativePath: path, VirtualStorage: virtualStorageName, AuthoritativeStorage: authoritativeStorage}) + repoEntry, err := json.Marshal(trackRepositoryRequest{ + RelativePath: relPath, + ReplicaPath: replicaPath, + VirtualStorage: virtualStorageName, + AuthoritativeStorage: authoritativeStorage, + }) require.NoError(t, err) _, err = fmt.Fprintf(input, string(repoEntry)+"\n") require.NoError(t, err) @@ -160,8 +181,8 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { assert.Contains(t, stdout, tc.expectedOutput) replicateImmediately := slices.Contains(tc.args(inputPath), "-replicate-immediately") - for _, path := range tc.relativePaths { - repositoryID, err := repositoryStore.GetRepositoryID(ctx, virtualStorageName, path) + for i, relPath := range tc.relativePaths { + repositoryID, err := repositoryStore.GetRepositoryID(ctx, virtualStorageName, relPath) require.NoError(t, err) assignments, err := assignmentStore.GetHostAssignments(ctx, virtualStorageName, repositoryID) @@ -170,7 +191,7 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { assert.Contains(t, assignments, g1Cfg.Storages[0].Name) assert.Contains(t, assignments, g2Cfg.Storages[0].Name) - exists, err := repositoryStore.RepositoryExists(ctx, virtualStorageName, path) + exists, err := repositoryStore.RepositoryExists(ctx, virtualStorageName, relPath) require.NoError(t, err) assert.True(t, exists) @@ -179,9 +200,10 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { events, err := queue.Dequeue(ctx, virtualStorageName, g2Cfg.Storages[0].Name, 1) require.NoError(t, err) require.Len(t, events, 1) - assert.Equal(t, path, events[0].Job.RelativePath) + assert.Equal(t, relPath, events[0].Job.RelativePath) } else { - require.DirExists(t, filepath.Join(g2Cfg.Storages[0].Path, path)) + replicaPath := tc.replicaPaths[i] + require.DirExists(t, filepath.Join(g2Cfg.Storages[0].Path, replicaPath)) } } }) @@ -207,6 +229,13 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { ) } + formatOutput := func(inputPath string, requestCt int, lines []string) string { + header := fmt.Sprintf("Validating repository information in %q\n", inputPath) + header += fmt.Sprintf("Found %v invalid request(s) in %q:\n", requestCt, inputPath) + rest := strings.Join(lines, "\n") + return header + rest + "\n" + } + const invalidEntryErr = "invalid entries found, aborting" t.Run("fail", func(t *testing.T) { @@ -214,7 +243,8 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { desc string args func(inputPath string) []string input string - expectedOutput string + requestCt int + expectedOutput []string expectedError string trackedPath string }{ @@ -238,42 +268,78 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { expectedError: "no repository information found", }, { - desc: "invalid JSON", - input: "@hashed/01/23/01234567890123456789.git", - expectedOutput: "invalid character '@' looking for beginning of value", - expectedError: invalidEntryErr, + desc: "invalid JSON", + input: "@hashed/01/23/01234567890123456789.git", + requestCt: 1, + expectedOutput: []string{ + ` line 1, relative_path: "", replica_path: ""`, + " invalid character '@' looking for beginning of value", + }, + expectedError: invalidEntryErr, }, { - desc: "missing path", - input: `{"virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, - expectedOutput: `"repository" is a required parameter`, - expectedError: invalidEntryErr, + desc: "missing relative path", + input: `{"replica_path":"r","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, + requestCt: 1, + expectedOutput: []string{ + ` line 1, relative_path: "", replica_path: "r"`, + ` "relative-path" is a required parameter`, + }, + expectedError: invalidEntryErr, + }, + { + desc: "missing replica path", + input: `{"relative_path":"r","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, + requestCt: 1, + expectedOutput: []string{ + ` line 1, relative_path: "r", replica_path: ""`, + ` "replica-path" is a required parameter`, + }, + expectedError: invalidEntryErr, }, { - desc: "invalid virtual storage", - input: `{"virtual_storage":"foo","relative_path":"bar","authoritative_storage":"gitaly-1"}`, - expectedOutput: `virtual storage "foo" not found`, - expectedError: invalidEntryErr, + desc: "invalid virtual storage", + input: `{"virtual_storage":"foo","relative_path":"bar","replica_path":"bar","authoritative_storage":"gitaly-1"}`, + requestCt: 1, + expectedOutput: []string{ + ` line 1, relative_path: "bar", replica_path: "bar"`, + ` checking repository on disk: virtual storage "foo" not found`, + }, + expectedError: invalidEntryErr, }, { - desc: "repo does not exist", - input: `{"relative_path":"not_a_repo","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, - expectedOutput: "not a valid git repository", - expectedError: invalidEntryErr, + desc: "repo does not exist", + input: `{"relative_path":"not_a_repo","replica_path":"foo","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, + requestCt: 1, + expectedOutput: []string{ + ` line 1, relative_path: "not_a_repo", replica_path: "foo"`, + " not a valid git repository", + }, + expectedError: invalidEntryErr, }, { - desc: "duplicate path", - input: `{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"} -{"virtual_storage":"praefect","relative_path":"duplicate","authoritative_storage":"gitaly-1"}`, - expectedOutput: "duplicate entries for relative_path", - expectedError: invalidEntryErr, + desc: "duplicate relative path", + input: `{"virtual_storage":"praefect","relative_path":"duplicate","replica_path":"foo","authoritative_storage":"gitaly-1"} + {"virtual_storage":"praefect","relative_path":"duplicate","replica_path":"foo","authoritative_storage":"gitaly-1"}`, + requestCt: 2, + expectedOutput: []string{ + ` line 1, relative_path: "duplicate", replica_path: "foo"`, + " not a valid git repository", + ` line 2, relative_path: "duplicate", replica_path: "foo"`, + " duplicate entries for relative_path, line [1 2]", + }, + expectedError: invalidEntryErr, }, { - desc: "repo is already tracked", - input: `{"relative_path":"already_tracked","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, - expectedOutput: "repository is already tracked by Praefect", - expectedError: invalidEntryErr, - trackedPath: "already_tracked", + desc: "repo is already tracked", + input: `{"relative_path":"already_tracked","replica_path":"foo","virtual_storage":"praefect","authoritative_storage":"gitaly-1"}`, + requestCt: 1, + expectedOutput: []string{ + ` line 1, relative_path: "already_tracked", replica_path: "foo"`, + " repository is already tracked by Praefect", + }, + expectedError: invalidEntryErr, + trackedPath: "already_tracked", }, } { tc := tc @@ -299,9 +365,10 @@ func TestTrackRepositoriesSubcommand(t *testing.T) { assert.Empty(t, stderr) require.Error(t, err) - if tc.expectedOutput != "" { - require.Contains(t, stdout, tc.expectedOutput) + if len(tc.expectedOutput) > 0 { + require.Equal(t, formatOutput(inputPath, tc.requestCt, tc.expectedOutput), stdout) } + require.Contains(t, err.Error(), tc.expectedError) }) } diff --git a/internal/cli/praefect/subcmd_track_repository.go b/internal/cli/praefect/subcmd_track_repository.go index 86e461406..bea14388e 100644 --- a/internal/cli/praefect/subcmd_track_repository.go +++ b/internal/cli/praefect/subcmd_track_repository.go @@ -49,12 +49,17 @@ func newTrackRepositoryCommand() *cli.Command { }, &cli.StringFlag{ Name: paramRelativePath, - Usage: "relative path to the repository", + Usage: "relative path of the repository on the virtual storage. Usually starts with '@hashed'", + Required: true, + }, + &cli.StringFlag{ + Name: paramReplicaPath, + Usage: "path of the repository on the physical storage. Can start with '@cluster' or match the relative path", Required: true, }, &cli.StringFlag{ Name: paramAuthoritativeStorage, - Usage: "storage with the repository to consider as authoritative", + Usage: "physical storage to consider as authoritative for the repository", }, &cli.BoolFlag{ Name: "replicate-immediately", @@ -73,6 +78,7 @@ func newTrackRepositoryCommand() *cli.Command { type trackRepositoryRequest struct { RelativePath string `json:"relative_path"` + ReplicaPath string `json:"replica_path"` VirtualStorage string `json:"virtual_storage"` AuthoritativeStorage string `json:"authoritative_storage"` } @@ -88,6 +94,7 @@ func trackRepositoryAction(appCtx *cli.Context) error { virtualStorage := appCtx.String(paramVirtualStorage) relativePath := appCtx.String(paramRelativePath) + replicaPath := appCtx.String(paramReplicaPath) authoritativeStorage := appCtx.String(paramAuthoritativeStorage) replicateImmediately := appCtx.Bool("replicate-immediately") @@ -109,6 +116,7 @@ func trackRepositoryAction(appCtx *cli.Context) error { req := trackRepositoryRequest{ RelativePath: relativePath, + ReplicaPath: replicaPath, AuthoritativeStorage: authoritativeStorage, VirtualStorage: virtualStorage, } @@ -129,6 +137,7 @@ func (req *trackRepositoryRequest) execRequest(ctx context.Context, logger.WithFields(logrus.Fields{ "virtual_storage": req.VirtualStorage, "relative_path": req.RelativePath, + "replica_path": req.ReplicaPath, "authoritative_storage": req.AuthoritativeStorage, }).Debug("track repository") @@ -229,6 +238,7 @@ func (req *trackRepositoryRequest) execRequest(ctx context.Context, RepositoryID: repositoryID, Change: datastore.UpdateRepo, RelativePath: req.RelativePath, + ReplicaPath: req.ReplicaPath, VirtualStorage: req.VirtualStorage, SourceNodeStorage: primary, TargetNodeStorage: secondary, @@ -287,7 +297,7 @@ func (req *trackRepositoryRequest) trackRepository( repositoryID, req.VirtualStorage, req.RelativePath, - req.RelativePath, + req.ReplicaPath, primary, nil, secondaries, @@ -325,14 +335,14 @@ func (req *trackRepositoryRequest) authoritativeRepositoryExists(ctx context.Con for _, node := range vs.Nodes { if node.Storage == nodeName { - logger.Debugf("check if repository %q exists on gitaly %q at %q", req.RelativePath, node.Storage, node.Address) + logger.Debugf("check if repository %q exists on gitaly %q at %q", req.ReplicaPath, node.Storage, node.Address) repo := &gitalypb.Repository{ StorageName: node.Storage, - RelativePath: req.RelativePath, + RelativePath: req.ReplicaPath, } exists, err := repositoryExists(ctx, repo, node.Address, node.Token) if err != nil { - fmt.Fprintf(w, "checking if repository exists %q, %q", node.Storage, req.RelativePath) + fmt.Fprintf(w, "checking if repository exists %q, %q", node.Storage, req.ReplicaPath) return false, nil } return exists, nil diff --git a/internal/cli/praefect/subcmd_track_repository_test.go b/internal/cli/praefect/subcmd_track_repository_test.go index d8e11e6e2..cc7e76cba 100644 --- a/internal/cli/praefect/subcmd_track_repository_test.go +++ b/internal/cli/praefect/subcmd_track_repository_test.go @@ -109,30 +109,42 @@ func TestTrackRepositorySubcommand(t *testing.T) { }{ { name: "positional arguments", - args: []string{"-virtual-storage=v", "-repository=r", "-authoritative-storage=s", "positional-arg"}, + args: []string{"-virtual-storage=v", "-relative-path=r", "-replica-path=r", "-authoritative-storage=s", "positional-arg"}, errorMsg: "track-repository doesn't accept positional arguments", }, { name: "virtual-storage is not set", args: []string{ - "-repository", "path/to/test/repo_1", + "-relative-path", "path/to/test/repo_1", + "-replica-path", "path/to/test/repo_1", "-authoritative-storage", authoritativeStorage, }, errorMsg: `Required flag "virtual-storage" not set`, }, { - name: "repository is not set", + name: "relative-path is not set", args: []string{ "-virtual-storage", virtualStorageName, + "-replica-path", "path/to/test/repo_1", "-authoritative-storage", authoritativeStorage, }, - errorMsg: `Required flag "repository" not set`, + errorMsg: `Required flag "relative-path" not set`, + }, + { + name: "replica-path is not set", + args: []string{ + "-virtual-storage", virtualStorageName, + "-relative-path", "path/to/test/repo_1", + "-authoritative-storage", authoritativeStorage, + }, + errorMsg: `Required flag "replica-path" not set`, }, { name: "authoritative-storage is not set", args: []string{ "-virtual-storage", virtualStorageName, - "-repository", "path/to/test/repo_1", + "-relative-path", "path/to/test/repo_1", + "-replica-path", "path/to/test/repo_1", }, errorMsg: `Required flag "authoritative-storage" not set`, }, @@ -140,7 +152,8 @@ func TestTrackRepositorySubcommand(t *testing.T) { name: "repository does not exist", args: []string{ "-virtual-storage", virtualStorageName, - "-repository", "path/to/test/repo_1", + "-relative-path", "path/to/test/repo_1", + "-replica-path", "path/to/test/repo_1", "-authoritative-storage", authoritativeStorage, }, errorMsg: "attempting to track repository in praefect database: authoritative repository does not exist", @@ -157,12 +170,20 @@ func TestTrackRepositorySubcommand(t *testing.T) { t.Run("ok", func(t *testing.T) { testCases := []struct { relativePath string + replicaPath string desc string replicateImmediately bool repositoryExists bool expectedOutput []string }{ { + desc: "replica path differs from relative path", + relativePath: "path/to/test/diff_repo", + replicaPath: "path/to/replica/diff_repo", + replicateImmediately: true, + expectedOutput: []string{"Finished replicating repository to \"gitaly-2\".\n"}, + }, + { desc: "force replication", relativePath: "path/to/test/repo1", replicateImmediately: true, @@ -203,20 +224,25 @@ func TestTrackRepositorySubcommand(t *testing.T) { nodeMgr.Start(0, time.Hour) defer nodeMgr.Stop() + if tc.replicaPath == "" { + tc.replicaPath = tc.relativePath + } + exists, err := repoDS.RepositoryExists(ctx, virtualStorageName, tc.relativePath) require.NoError(t, err) require.Equal(t, tc.repositoryExists, exists) // create the repo on Gitaly without Praefect knowing if !tc.repositoryExists { - require.NoError(t, createRepoThroughGitaly1(tc.relativePath)) - require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, tc.relativePath)) - require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, tc.relativePath)) + require.NoError(t, createRepoThroughGitaly1(tc.replicaPath)) + require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, tc.replicaPath)) + require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, tc.replicaPath)) } args := []string{ "-virtual-storage", virtualStorageName, - "-repository", tc.relativePath, + "-relative-path", tc.relativePath, + "-replica-path", tc.replicaPath, "-authoritative-storage", authoritativeStorage, } if tc.replicateImmediately { @@ -270,7 +296,8 @@ func TestTrackRepositorySubcommand(t *testing.T) { "-config", confPath, trackRepositoryCmdName, "-virtual-storage", virtualStorageName, - "-repository", relativePath, + "-relative-path", relativePath, + "-replica-path", relativePath, "-authoritative-storage", authoritativeStorage, }) require.NoError(t, err) @@ -280,7 +307,8 @@ func TestTrackRepositorySubcommand(t *testing.T) { "-config", confPath, trackRepositoryCmdName, "-virtual-storage", virtualStorageName, - "-repository", relativePath, + "-relative-path", relativePath, + "-replica-path", relativePath, "-authoritative-storage", authoritativeStorage, }) require.NoError(t, err) |