diff options
author | Patrick Steinhardt <psteinhardt@gitlab.com> | 2023-02-17 15:01:46 +0300 |
---|---|---|
committer | Patrick Steinhardt <psteinhardt@gitlab.com> | 2023-02-17 15:01:46 +0300 |
commit | 220e4027730413cc0fb689fef03f9db9458cc8f8 (patch) | |
tree | 2552c2c731f77338438355a7befa9c7dd4521940 | |
parent | 24b98424df69e1de124662844a7e6df6dccb3768 (diff) | |
parent | 247ffa20e2fbf7d3135ee1905b878f971861b7b8 (diff) |
Merge branch 'jt-sync-custom-hooks' into 'master'
hooks: Implement custom hooks replication
See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5248
Merged-by: Patrick Steinhardt <psteinhardt@gitlab.com>
Approved-by: Patrick Steinhardt <psteinhardt@gitlab.com>
Reviewed-by: Patrick Steinhardt <psteinhardt@gitlab.com>
Reviewed-by: Justin Tobler <jtobler@gitlab.com>
Co-authored-by: Justin Tobler <jtobler@gitlab.com>
-rw-r--r-- | internal/archive/archive.go | 34 | ||||
-rw-r--r-- | internal/backup/backup_test.go | 25 | ||||
-rw-r--r-- | internal/gitaly/service/repository/get_custom_hooks.go | 18 | ||||
-rw-r--r-- | internal/gitaly/service/repository/replicate.go | 33 | ||||
-rw-r--r-- | internal/gitaly/service/repository/replicate_test.go | 105 | ||||
-rw-r--r-- | internal/gitaly/service/repository/set_custom_hooks.go | 36 | ||||
-rw-r--r-- | internal/gitaly/service/repository/set_custom_hooks_test.go | 154 | ||||
-rw-r--r-- | internal/gitaly/service/repository/testdata/corrupted_hooks.tar | 1 | ||||
-rw-r--r-- | internal/gitaly/service/repository/testdata/custom_hooks.tar | bin | 8192 -> 0 bytes | |||
-rw-r--r-- | internal/metadata/featureflag/ff_replicate_repository_hooks.go | 10 | ||||
-rw-r--r-- | internal/praefect/replicator_test.go | 17 |
11 files changed, 390 insertions, 43 deletions
diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 000000000..4d0ef5d9d --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,34 @@ +package archive + +import ( + "context" + "fmt" + "io" + "runtime" + + "gitlab.com/gitlab-org/gitaly/v15/internal/command" +) + +// WriteTarball writes a tarball to an `io.Writer` for the provided path +// containing the specified archive members. Members should be specified +// relative to `path`. +func WriteTarball(ctx context.Context, writer io.Writer, path string, members ...string) error { + cmdArgs := []string{"-c", "-f", "-", "-C", path} + + if runtime.GOOS == "darwin" { + cmdArgs = append(cmdArgs, "--no-mac-metadata") + } + + cmdArgs = append(cmdArgs, members...) + + cmd, err := command.New(ctx, append([]string{"tar"}, cmdArgs...), command.WithStdout(writer)) + if err != nil { + return fmt.Errorf("executing tar command: %w", err) + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("waiting for tar command completion: %w", err) + } + + return nil +} diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 50f25d93e..dd1084275 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v15/client" + "gitlab.com/gitlab-org/gitaly/v15/internal/archive" "gitlab.com/gitlab-org/gitaly/v15/internal/git" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/setup" @@ -324,7 +325,7 @@ func testManagerRestore(t *testing.T, ctx context.Context) { customHooksPath := filepath.Join(backupRoot, relativePath, "custom_hooks.tar") require.NoError(tb, os.MkdirAll(filepath.Join(backupRoot, relativePath), perm.PublicDir)) gittest.BundleRepo(tb, cfg, repoPath, bundlePath) - testhelper.CopyFile(tb, "../gitaly/service/repository/testdata/custom_hooks.tar", customHooksPath) + testhelper.CopyFile(tb, mustCreateCustomHooksArchive(t, ctx), customHooksPath) return repo, repoChecksum }, @@ -671,3 +672,25 @@ func joinBackupPath(tb testing.TB, backupRoot string, repo *gitalypb.Repository, func stripRelativePath(tb testing.TB, repo *gitalypb.Repository) string { return strings.TrimSuffix(repo.GetRelativePath(), ".git") } + +func mustCreateCustomHooksArchive(t *testing.T, ctx context.Context) string { + t.Helper() + + tmpDir := testhelper.TempDir(t) + + hooksDirPath := filepath.Join(tmpDir, "custom_hooks") + require.NoError(t, os.Mkdir(hooksDirPath, os.ModePerm)) + + require.NoError(t, os.WriteFile(filepath.Join(hooksDirPath, "pre-commit.sample"), []byte("foo"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(hooksDirPath, "prepare-commit-msg.sample"), []byte("bar"), os.ModePerm)) + require.NoError(t, os.WriteFile(filepath.Join(hooksDirPath, "pre-push.sample"), []byte("baz"), os.ModePerm)) + + archivePath := filepath.Join(tmpDir, "custom_hooks.tar") + file, err := os.Create(archivePath) + require.NoError(t, err) + defer testhelper.MustClose(t, file) + + require.NoError(t, archive.WriteTarball(ctx, file, tmpDir, "custom_hooks")) + + return archivePath +} diff --git a/internal/gitaly/service/repository/get_custom_hooks.go b/internal/gitaly/service/repository/get_custom_hooks.go index 991b478ab..ab39082ba 100644 --- a/internal/gitaly/service/repository/get_custom_hooks.go +++ b/internal/gitaly/service/repository/get_custom_hooks.go @@ -6,9 +6,8 @@ import ( "io" "os" "path/filepath" - "runtime" - "gitlab.com/gitlab-org/gitaly/v15/internal/command" + "gitlab.com/gitlab-org/gitaly/v15/internal/archive" "gitlab.com/gitlab-org/gitaly/v15/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v15/internal/structerr" @@ -70,19 +69,8 @@ func (s *server) getCustomHooks(ctx context.Context, writer io.Writer, repo repo return nil } - var tar []string - if runtime.GOOS == "darwin" { - tar = []string{"tar", "--no-mac-metadata", "-c", "-f", "-", "-C", repoPath, customHooksDir} - } else { - tar = []string{"tar", "-c", "-f", "-", "-C", repoPath, customHooksDir} - } - cmd, err := command.New(ctx, tar, command.WithStdout(writer)) - if err != nil { - return fmt.Errorf("creating tar command: %w", err) - } - - if err := cmd.Wait(); err != nil { - return fmt.Errorf("waiting for tar command completion: %w", err) + if err := archive.WriteTarball(ctx, writer, repoPath, customHooksDir); err != nil { + return structerr.NewInternal("archiving hooks: %w", err) } return nil diff --git a/internal/gitaly/service/repository/replicate.go b/internal/gitaly/service/repository/replicate.go index 5d442e1a2..79b036188 100644 --- a/internal/gitaly/service/repository/replicate.go +++ b/internal/gitaly/service/repository/replicate.go @@ -22,6 +22,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v15/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v15/internal/safe" "gitlab.com/gitlab-org/gitaly/v15/internal/structerr" "gitlab.com/gitlab-org/gitaly/v15/internal/tempdir" @@ -89,6 +90,12 @@ func (s *server) ReplicateRepository(ctx context.Context, in *gitalypb.Replicate return nil, structerr.NewInternal("synchronizing repository: %w", err) } + if featureflag.ReplicateRepositoryHooks.IsEnabled(ctx) { + if err := s.syncCustomHooks(ctx, in); err != nil { + return nil, structerr.NewInternal("synchronizing custom hooks: %w", err) + } + } + return &gitalypb.ReplicateRepositoryResponse{}, nil } @@ -260,6 +267,32 @@ func fetchInternalRemote( return nil } +// syncCustomHooks replicates custom hooks from a source to a target. +func (s *server) syncCustomHooks(ctx context.Context, in *gitalypb.ReplicateRepositoryRequest) error { + repoClient, err := s.newRepoClient(ctx, in.GetSource().GetStorageName()) + if err != nil { + return fmt.Errorf("creating repo client: %w", err) + } + + stream, err := repoClient.GetCustomHooks(ctx, &gitalypb.GetCustomHooksRequest{ + Repository: in.GetSource(), + }) + if err != nil { + return fmt.Errorf("getting custom hooks: %w", err) + } + + reader := streamio.NewReader(func() ([]byte, error) { + request, err := stream.Recv() + return request.GetData(), err + }) + + if err := s.setCustomHooksTransaction(ctx, reader, in.GetRepository()); err != nil { + return fmt.Errorf("setting custom hooks: %w", err) + } + + return nil +} + func (s *server) syncGitconfig(ctx context.Context, in *gitalypb.ReplicateRepositoryRequest) error { repoClient, err := s.newRepoClient(ctx, in.GetSource().GetStorageName()) if err != nil { diff --git a/internal/gitaly/service/repository/replicate_test.go b/internal/gitaly/service/repository/replicate_test.go index 8e87ae0cd..f15fd5ea2 100644 --- a/internal/gitaly/service/repository/replicate_test.go +++ b/internal/gitaly/service/repository/replicate_test.go @@ -28,6 +28,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v15/internal/helper/text" "gitlab.com/gitlab-org/gitaly/v15/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testserver" @@ -38,10 +39,14 @@ import ( "google.golang.org/protobuf/proto" ) -func TestReplicateRepository(t *testing.T) { +func TestReplicateRepository_success(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) + testhelper.NewFeatureSets(featureflag.ReplicateRepositoryHooks). + Run(t, testReplicateRepositorySuccess) +} + +func testReplicateRepositorySuccess(t *testing.T, ctx context.Context) { cfgBuilder := testcfg.NewGitalyCfgBuilder(testcfg.WithStorages("default", "replica")) cfg := cfgBuilder.Build(t) @@ -115,7 +120,11 @@ func TestReplicateRepository(t *testing.T) { func TestReplicateRepository_hiddenRefs(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) + testhelper.NewFeatureSets(featureflag.ReplicateRepositoryHooks). + Run(t, testReplicateRepositoryHiddenRefs) +} + +func testReplicateRepositoryHiddenRefs(t *testing.T, ctx context.Context) { cfgBuilder := testcfg.NewGitalyCfgBuilder(testcfg.WithStorages("default", "replica")) cfg := cfgBuilder.Build(t) @@ -196,10 +205,13 @@ func TestReplicateRepository_hiddenRefs(t *testing.T) { }) } -func TestReplicateRepositoryTransactional(t *testing.T) { +func TestReplicateRepository_transactional(t *testing.T) { t.Parallel() + testhelper.NewFeatureSets(featureflag.ReplicateRepositoryHooks). + Run(t, testReplicateRepositoryTransactional) +} - ctx := testhelper.Context(t) +func testReplicateRepositoryTransactional(t *testing.T, ctx context.Context) { cfgBuilder := testcfg.NewGitalyCfgBuilder(testcfg.WithStorages("default", "replica")) cfg := cfgBuilder.Build(t) @@ -255,7 +267,9 @@ func TestReplicateRepositoryTransactional(t *testing.T) { // There is a gitconfig though, so the vote should reflect its contents. gitconfigVote := sha1.Sum(testhelper.MustReadFile(t, filepath.Join(sourceRepoPath, "config"))) - require.Equal(t, []string{ + noHooksVote := "fd69c38637bf443296edc19e2b2c649d0502f7c0" + + expectedVotes := []string{ // We cannot easily derive these first two votes: they are based on the complete // hashed contents of the unpacked repository. We thus just only assert that they // are always the first two entries and that they are the same by simply taking the @@ -266,7 +280,13 @@ func TestReplicateRepositoryTransactional(t *testing.T) { hex.EncodeToString(gitconfigVote[:]), hex.EncodeToString(gitattributesVote[:]), hex.EncodeToString(gitattributesVote[:]), - }, votes) + } + + if featureflag.ReplicateRepositoryHooks.IsEnabled(ctx) { + expectedVotes = append(expectedVotes, noHooksVote, noHooksVote) + } + + require.Equal(t, expectedVotes, votes) // We're about to change refs/heads/master, and thus the mirror-fetch will update it. The // vote should reflect that. @@ -286,14 +306,21 @@ func TestReplicateRepositoryTransactional(t *testing.T) { Source: sourceRepo, }) require.NoError(t, err) - require.Equal(t, []string{ + + expectedVotes = []string{ hex.EncodeToString(gitconfigVote[:]), hex.EncodeToString(gitconfigVote[:]), hex.EncodeToString(gitattributesVote[:]), hex.EncodeToString(gitattributesVote[:]), hex.EncodeToString(replicationVote[:]), hex.EncodeToString(replicationVote[:]), - }, votes) + } + + if featureflag.ReplicateRepositoryHooks.IsEnabled(ctx) { + expectedVotes = append(expectedVotes, noHooksVote, noHooksVote) + } + + require.Equal(t, expectedVotes, votes) } func TestReplicateRepositoryInvalidArguments(t *testing.T) { @@ -377,8 +404,11 @@ func TestReplicateRepositoryInvalidArguments(t *testing.T) { func TestReplicateRepository_BadRepository(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) + testhelper.NewFeatureSets(featureflag.ReplicateRepositoryHooks). + Run(t, testReplicateRepositoryBadRepository) +} +func testReplicateRepositoryBadRepository(t *testing.T, ctx context.Context) { for _, tc := range []struct { desc string invalidSource bool @@ -646,3 +676,58 @@ func TestFetchInternalRemote_failure(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "fatal: Could not read from remote repository") } + +func TestReplicateRepository_hooks(t *testing.T) { + t.Parallel() + + testhelper.NewFeatureSets(featureflag.ReplicateRepositoryHooks). + Run(t, testReplicateRepositoryHooks) +} + +func testReplicateRepositoryHooks(t *testing.T, ctx context.Context) { + t.Parallel() + + cfgBuilder := testcfg.NewGitalyCfgBuilder(testcfg.WithStorages("default", "replica")) + cfg := cfgBuilder.Build(t) + + testcfg.BuildGitalyHooks(t, cfg) + testcfg.BuildGitalySSH(t, cfg) + + service, serverSocketPath := runRepositoryService(t, cfg, nil, testserver.WithDisablePraefect()) + cfg.SocketPath = serverSocketPath + + sourceRepo, sourceRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{}) + + // Add custom hooks to source repository. + archivePath := mustCreateCustomHooksArchive(t, ctx, []testFile{ + {name: "pre-commit.sample", content: "foo", mode: 0o755}, + {name: "pre-push.sample", content: "bar", mode: 0o755}, + }, customHooksDir) + + hooks, err := os.Open(archivePath) + require.NoError(t, err) + + err = extractHooks(ctx, hooks, sourceRepoPath) + require.NoError(t, err) + + targetRepo := proto.Clone(sourceRepo).(*gitalypb.Repository) + targetRepo.StorageName = cfg.Storages[1].Name + + ctx = testhelper.MergeOutgoingMetadata(ctx, testcfg.GitalyServersMetadataFromCfg(t, cfg)) + + _, err = service.ReplicateRepository(ctx, &gitalypb.ReplicateRepositoryRequest{ + Repository: targetRepo, + Source: sourceRepo, + }) + require.NoError(t, err) + + targetRepoPath := filepath.Join(cfg.Storages[1].Path, gittest.GetReplicaPath(t, ctx, cfg, targetRepo)) + targetHooksPath := filepath.Join(targetRepoPath, customHooksDir) + + // Make sure target repo contains replicated custom hooks from source repository. + if featureflag.ReplicateRepositoryHooks.IsEnabled(ctx) { + require.FileExists(t, filepath.Join(targetHooksPath, "pre-push.sample")) + } else { + require.NoDirExists(t, targetHooksPath) + } +} diff --git a/internal/gitaly/service/repository/set_custom_hooks.go b/internal/gitaly/service/repository/set_custom_hooks.go index 5e231a6f9..6c8d47444 100644 --- a/internal/gitaly/service/repository/set_custom_hooks.go +++ b/internal/gitaly/service/repository/set_custom_hooks.go @@ -1,6 +1,7 @@ package repository import ( + "bufio" "context" "encoding/binary" "errors" @@ -9,6 +10,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" "gitlab.com/gitlab-org/gitaly/v15/internal/command" @@ -34,12 +36,12 @@ func (s *server) SetCustomHooks(stream gitalypb.RepositoryService_SetCustomHooks firstRequest, err := stream.Recv() if err != nil { - return structerr.NewInternal("first request failed %w", err) + return structerr.NewInternal("getting first request: %w", err) } repo := firstRequest.GetRepository() if err := service.ValidateRepository(repo); err != nil { - return structerr.NewInvalidArgument("%w", err) + return structerr.NewInvalidArgument("validating repo: %w", err) } reader := streamio.NewReader(func() ([]byte, error) { @@ -68,12 +70,12 @@ func (s *server) RestoreCustomHooks(stream gitalypb.RepositoryService_RestoreCus firstRequest, err := stream.Recv() if err != nil { - return structerr.NewInternal("first request failed %w", err) + return structerr.NewInternal("getting first request: %w", err) } repo := firstRequest.GetRepository() if err := service.ValidateRepository(repo); err != nil { - return structerr.NewInvalidArgument("%w", err) + return structerr.NewInvalidArgument("validating repo: %w", err) } reader := streamio.NewReader(func() ([]byte, error) { @@ -97,7 +99,7 @@ func (s *server) RestoreCustomHooks(stream gitalypb.RepositoryService_RestoreCus func (s *server) setCustomHooks(ctx context.Context, reader io.Reader, repo repository.GitRepo) error { if featureflag.TransactionalRestoreCustomHooks.IsEnabled(ctx) { if err := s.setCustomHooksTransaction(ctx, reader, repo); err != nil { - return fmt.Errorf("writing custom hooks: %w", err) + return fmt.Errorf("setting custom hooks: %w", err) } return nil @@ -105,7 +107,7 @@ func (s *server) setCustomHooks(ctx context.Context, reader io.Reader, repo repo repoPath, err := s.locator.GetPath(repo) if err != nil { - return fmt.Errorf("getting repo path failed %w", err) + return fmt.Errorf("getting repo path: %w", err) } if err := extractHooks(ctx, reader, repoPath); err != nil { @@ -291,14 +293,34 @@ func voteCustomHooks( // extractHooks unpacks a tar file containing custom hooks into a `custom_hooks` // directory at the specified path. func extractHooks(ctx context.Context, reader io.Reader, path string) error { + // GNU tar does not accept an empty file as a valid tar archive and produces + // an error. Since an empty hooks tar is symbolic of a repository having no + // hooks, the reader is peeked to check if there is any data present. + buf := bufio.NewReader(reader) + if _, err := buf.Peek(1); err == io.EOF { + return nil + } + cmdArgs := []string{"-xf", "-", "-C", path, customHooksDir} - cmd, err := command.New(ctx, append([]string{"tar"}, cmdArgs...), command.WithStdin(reader)) + var stderrBuilder strings.Builder + cmd, err := command.New(ctx, append([]string{"tar"}, cmdArgs...), + command.WithStdin(buf), + command.WithStderr(&stderrBuilder)) if err != nil { return fmt.Errorf("executing tar command: %w", err) } if err := cmd.Wait(); err != nil { + // GNU and BSD tar versions have differing errors when attempting to + // extract specified members from a valid tar archive. If the tar + // archive is valid the errors for GNU and BSD tar should have the + // same prefix, which can be checked to validate whether the expected + // content is present in the archive for extraction. + if strings.HasPrefix(stderrBuilder.String(), "tar: custom_hooks: Not found in archive") { + return nil + } + return fmt.Errorf("waiting for tar command completion: %w", err) } diff --git a/internal/gitaly/service/repository/set_custom_hooks_test.go b/internal/gitaly/service/repository/set_custom_hooks_test.go index ece277354..040906765 100644 --- a/internal/gitaly/service/repository/set_custom_hooks_test.go +++ b/internal/gitaly/service/repository/set_custom_hooks_test.go @@ -5,6 +5,7 @@ package repository import ( "context" "io" + "io/fs" "os" "path/filepath" "syscall" @@ -12,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v15/internal/archive" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/helper/perm" @@ -113,7 +115,12 @@ func testSuccessfulSetCustomHooksRequest(t *testing.T, ctx context.Context) { writer, closeStream := tc.streamWriter(t, ctx, repo, client) - file, err := os.Open("testdata/custom_hooks.tar") + archivePath := mustCreateCustomHooksArchive(t, ctx, []testFile{ + {name: "pre-commit.sample", content: "foo", mode: 0o755}, + {name: "pre-push.sample", content: "bar", mode: 0o755}, + }, customHooksDir) + + file, err := os.Open(archivePath) require.NoError(t, err) _, err = io.Copy(writer, file) @@ -186,7 +193,7 @@ func testFailedSetCustomHooksDueToValidations(t *testing.T, ctx context.Context) err := tc.streamSender(t, ctx, client) testhelper.RequireGrpcError(t, err, status.Error(codes.InvalidArgument, testhelper.GitalyOrPraefect( - "empty Repository", + "validating repo: empty Repository", "repo scoped: empty Repository", ))) }) @@ -262,7 +269,9 @@ func testFailedSetCustomHooksDueToBadTar(t *testing.T, ctx context.Context) { _, repo, _, client := setupRepositoryService(t, ctx) writer, closeStream := tc.streamWriter(t, ctx, repo, client) - file, err := os.Open("testdata/corrupted_hooks.tar") + archivePath := mustCreateCorruptHooksArchive(t) + + file, err := os.Open(archivePath) require.NoError(t, err) defer testhelper.MustClose(t, file) @@ -330,7 +339,7 @@ func TestNewDirectoryVote(t *testing.T) { }, } { t.Run(tc.desc, func(t *testing.T) { - path := setupTestHooks(t, tc.files) + path := mustWriteCustomHookDirectory(t, tc.files, customHooksDir) voteHash, err := newDirectoryVote(path) require.NoError(t, err) @@ -344,11 +353,11 @@ func TestNewDirectoryVote(t *testing.T) { } } -func setupTestHooks(t *testing.T, files []testFile) string { +func mustWriteCustomHookDirectory(t *testing.T, files []testFile, dirName string) string { t.Helper() tmpDir := testhelper.TempDir(t) - hooksPath := filepath.Join(tmpDir, customHooksDir) + hooksPath := filepath.Join(tmpDir, dirName) err := os.Mkdir(hooksPath, perm.SharedDir) require.NoError(t, err) @@ -360,3 +369,136 @@ func setupTestHooks(t *testing.T, files []testFile) string { return hooksPath } + +func mustCreateCustomHooksArchive(t *testing.T, ctx context.Context, files []testFile, dirName string) string { + t.Helper() + + hooksPath := mustWriteCustomHookDirectory(t, files, dirName) + hooksDir := filepath.Dir(hooksPath) + + tmpDir := testhelper.TempDir(t) + archivePath := filepath.Join(tmpDir, "custom_hooks.tar") + + file, err := os.Create(archivePath) + require.NoError(t, err) + + err = archive.WriteTarball(ctx, file, hooksDir, dirName) + require.NoError(t, err) + + return archivePath +} + +func mustCreateCorruptHooksArchive(t *testing.T) string { + t.Helper() + + tmpDir := testhelper.TempDir(t) + archivePath := filepath.Join(tmpDir, "corrupt_hooks.tar") + + err := os.WriteFile(archivePath, []byte("This is a corrupted tar file"), 0o755) + require.NoError(t, err) + + return archivePath +} + +func TestExtractHooks(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + cfg := testcfg.Build(t) + + hooksFiles := []testFile{ + {name: "pre-commit.sample", content: "foo", mode: 0o755}, + {name: "pre-push.sample", content: "bar", mode: 0o755}, + } + + tmpDir := testhelper.TempDir(t) + emptyTar := filepath.Join(tmpDir, "empty_hooks.tar") + err := os.WriteFile(emptyTar, nil, 0o644) + require.NoError(t, err) + + for _, tc := range []struct { + desc string + path string + expectedFiles []testFile + expectedErr string + }{ + { + desc: "custom hooks tar", + path: mustCreateCustomHooksArchive(t, ctx, hooksFiles, customHooksDir), + expectedFiles: []testFile{ + {name: "custom_hooks"}, + {name: "custom_hooks/pre-commit.sample", content: "foo"}, + {name: "custom_hooks/pre-push.sample", content: "bar"}, + }, + }, + { + desc: "empty custom hooks tar", + path: mustCreateCustomHooksArchive(t, ctx, []testFile{}, customHooksDir), + expectedFiles: []testFile{{name: "custom_hooks"}}, + }, + { + desc: "no hooks tar", + path: mustCreateCustomHooksArchive(t, ctx, hooksFiles, "no_hooks"), + }, + { + desc: "corrupt tar", + path: mustCreateCorruptHooksArchive(t), + expectedErr: "waiting for tar command completion: exit status", + }, + { + desc: "empty tar", + path: emptyTar, + }, + } { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + _, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + + tarball, err := os.Open(tc.path) + require.NoError(t, err) + defer tarball.Close() + + err = extractHooks(ctx, tarball, repoPath) + if tc.expectedErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.expectedErr) + } + + if len(tc.expectedFiles) == 0 { + require.NoDirExists(t, filepath.Join(repoPath, customHooksDir)) + return + } + + var extractedFiles []testFile + + hooksPath := filepath.Join(repoPath, customHooksDir) + err = filepath.WalkDir(hooksPath, func(path string, d fs.DirEntry, err error) error { + require.NoError(t, err) + + relPath, err := filepath.Rel(repoPath, path) + require.NoError(t, err) + + var hookData []byte + if !d.IsDir() { + hookData, err = os.ReadFile(path) + require.NoError(t, err) + } + + extractedFiles = append(extractedFiles, testFile{ + name: relPath, + content: string(hookData), + }) + + return nil + }) + require.NoError(t, err) + require.Equal(t, tc.expectedFiles, extractedFiles) + }) + } +} diff --git a/internal/gitaly/service/repository/testdata/corrupted_hooks.tar b/internal/gitaly/service/repository/testdata/corrupted_hooks.tar deleted file mode 100644 index fe580ca02..000000000 --- a/internal/gitaly/service/repository/testdata/corrupted_hooks.tar +++ /dev/null @@ -1 +0,0 @@ -This is a corrupted tar file diff --git a/internal/gitaly/service/repository/testdata/custom_hooks.tar b/internal/gitaly/service/repository/testdata/custom_hooks.tar Binary files differdeleted file mode 100644 index 9630316d3..000000000 --- a/internal/gitaly/service/repository/testdata/custom_hooks.tar +++ /dev/null diff --git a/internal/metadata/featureflag/ff_replicate_repository_hooks.go b/internal/metadata/featureflag/ff_replicate_repository_hooks.go new file mode 100644 index 000000000..abbf63a43 --- /dev/null +++ b/internal/metadata/featureflag/ff_replicate_repository_hooks.go @@ -0,0 +1,10 @@ +package featureflag + +// ReplicateRepositoryHooks will enable replication of custom hooks with the +// `ReplicateRepository` RPC. +var ReplicateRepositoryHooks = NewFeatureFlag( + "replicate_repository_hooks", + "v15.9.0", + "https://gitlab.com/gitlab-org/gitaly/-/issues/4772", + false, +) diff --git a/internal/praefect/replicator_test.go b/internal/praefect/replicator_test.go index 16604c057..2eb94c0e1 100644 --- a/internal/praefect/replicator_test.go +++ b/internal/praefect/replicator_test.go @@ -27,6 +27,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v15/internal/helper" + "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v15/internal/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v15/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v15/internal/praefect/datastore" @@ -49,8 +50,11 @@ import ( func TestReplMgr_ProcessBacklog(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) + testhelper.NewFeatureSets(featureflag.ReplicateRepositoryHooks). + Run(t, testReplMgrProcessBacklog) +} +func testReplMgrProcessBacklog(t *testing.T, ctx context.Context) { primaryCfg := testcfg.Build(t, testcfg.WithStorages("primary")) testRepoProto, testRepoPath := gittest.CreateRepository(t, ctx, primaryCfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, @@ -355,8 +359,11 @@ func getChecksumFunc(ctx context.Context, client gitalypb.RepositoryServiceClien func TestProcessBacklog_FailedJobs(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) + testhelper.NewFeatureSets(featureflag.ReplicateRepositoryHooks). + Run(t, testProcessBacklogFailedJobs) +} +func testProcessBacklogFailedJobs(t *testing.T, ctx context.Context) { primaryCfg := testcfg.Build(t, testcfg.WithStorages("default")) testRepo, _ := gittest.CreateRepository(t, ctx, primaryCfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, @@ -464,7 +471,11 @@ func TestProcessBacklog_FailedJobs(t *testing.T) { func TestProcessBacklog_Success(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) + testhelper.NewFeatureSets(featureflag.ReplicateRepositoryHooks). + Run(t, testProcessBackLogSuccess) +} + +func testProcessBackLogSuccess(t *testing.T, ctx context.Context) { ctx, cancel := context.WithCancel(ctx) primaryCfg := testcfg.Build(t, testcfg.WithStorages("primary")) |