diff options
author | Christian Couder <chriscool@tuxfamily.org> | 2020-10-15 10:51:29 +0300 |
---|---|---|
committer | Christian Couder <chriscool@tuxfamily.org> | 2020-10-15 10:51:29 +0300 |
commit | 079aff93bb8f6bb525db23b4b20516cf428e1f91 (patch) | |
tree | 67ebdb3295662e9bf565e9481f04ffa6afc12340 | |
parent | 8c7c3fca7a8ee5decd252b16640ac2e6ef8b0026 (diff) | |
parent | 6b3f05f12dc4c30787f2ed88f5040c20b6d1b558 (diff) |
Merge branch 'ps-user-squash-go' into 'master'
Port UserSquash to Go
Closes #3075
See merge request gitlab-org/gitaly!2623
-rw-r--r-- | changelogs/unreleased/ps-user-squash-go.yml | 5 | ||||
-rw-r--r-- | internal/git/command.go | 20 | ||||
-rw-r--r-- | internal/git/safecmd.go | 27 | ||||
-rw-r--r-- | internal/git/safecmd_test.go | 97 | ||||
-rw-r--r-- | internal/gitaly/service/operations/squash.go | 405 | ||||
-rw-r--r-- | internal/gitaly/service/operations/squash_test.go | 132 | ||||
-rw-r--r-- | internal/metadata/featureflag/feature_flags.go | 3 |
7 files changed, 681 insertions, 8 deletions
diff --git a/changelogs/unreleased/ps-user-squash-go.yml b/changelogs/unreleased/ps-user-squash-go.yml new file mode 100644 index 000000000..1f449712d --- /dev/null +++ b/changelogs/unreleased/ps-user-squash-go.yml @@ -0,0 +1,5 @@ +--- +title: Port UserSquash to Go +merge_request: 2623 +author: +type: added diff --git a/internal/git/command.go b/internal/git/command.go index aad8f78bf..dee83e9e5 100644 --- a/internal/git/command.go +++ b/internal/git/command.go @@ -2,6 +2,7 @@ package git import ( "context" + "io" "os/exec" "gitlab.com/gitlab-org/gitaly/internal/command" @@ -34,6 +35,16 @@ func unsafeStdinCmd(ctx context.Context, extraEnv []string, repo repository.GitR return unsafeBareCmd(ctx, CmdStream{In: command.SetupStdin}, env, args...) } +// unsafeStderrCmd runs unsafeBareCmd with stderr directed to passed stderr. +func unsafeStderrCmd(ctx context.Context, stderr io.Writer, repo repository.GitRepo, args ...string) (*command.Command, error) { + args, env, err := argsAndEnv(repo, args...) + if err != nil { + return nil, err + } + + return unsafeBareCmd(ctx, CmdStream{Err: stderr}, env, args...) +} + func argsAndEnv(repo repository.GitRepo, args ...string) ([]string, []string, error) { repoPath, env, err := alternates.PathAndEnv(repo) if err != nil { @@ -52,6 +63,15 @@ func unsafeBareCmd(ctx context.Context, stream CmdStream, env []string, args ... return command.New(ctx, exec.Command(command.GitPath(), args...), stream.In, stream.Out, stream.Err, env...) } +// unsafeBareCmdInDir calls unsafeBareCmd in dir. +func unsafeBareCmdInDir(ctx context.Context, dir string, stream CmdStream, env []string, args ...string) (*command.Command, error) { + env = append(env, command.GitEnv...) + + cmd := exec.Command(command.GitPath(), args...) + cmd.Dir = dir + return command.New(ctx, cmd, stream.In, stream.Out, stream.Err, env...) +} + // unsafeCmdWithoutRepo works like Command but without a git repository func unsafeCmdWithoutRepo(ctx context.Context, stream CmdStream, args ...string) (*command.Command, error) { return unsafeBareCmd(ctx, stream, nil, args...) diff --git a/internal/git/safecmd.go b/internal/git/safecmd.go index 38d8fd0fa..515cda76c 100644 --- a/internal/git/safecmd.go +++ b/internal/git/safecmd.go @@ -62,7 +62,7 @@ func (sc SubCmd) IsCmd() {} func (sc SubCmd) supportsEndOfOptions() bool { switch sc.Name { - case "linguist", "for-each-ref", "archive", "upload-archive", "grep", "clone", "config", "rev-parse", "remote", "blame", "ls-tree": + case "checkout", "linguist", "for-each-ref", "archive", "upload-archive", "grep", "clone", "config", "rev-parse", "remote", "blame", "ls-tree": return false } return true @@ -300,6 +300,20 @@ func SafeBareCmd(ctx context.Context, stream CmdStream, env []string, globals [] return unsafeBareCmd(ctx, stream, append(env, cc.env...), args...) } +// SafeBareCmdInDir runs SafeBareCmd in the dir. +func SafeBareCmdInDir(ctx context.Context, dir string, stream CmdStream, env []string, globals []Option, sc Cmd) (*command.Command, error) { + if dir == "" { + return nil, errors.New("no 'dir' provided") + } + + args, err := combineArgs(globals, sc) + if err != nil { + return nil, err + } + + return unsafeBareCmdInDir(ctx, dir, stream, env, args...) +} + // SafeStdinCmd creates a git.Command with the given args and Repository that is // suitable for Write()ing to. It validates the arguments in the command before // executing. @@ -318,6 +332,17 @@ func SafeStdinCmd(ctx context.Context, repo repository.GitRepo, globals []Option return unsafeStdinCmd(ctx, cc.env, repo, args...) } +// SafeStderrCmd creates a git.Command with the given args and Repository that +// writes its standard error stream into provided stderr. +func SafeStderrCmd(ctx context.Context, stderr io.Writer, repo repository.GitRepo, globals []Option, sc SubCmd) (*command.Command, error) { + args, err := combineArgs(globals, sc) + if err != nil { + return nil, err + } + + return unsafeStderrCmd(ctx, stderr, repo, args...) +} + // SafeCmdWithoutRepo works like Command but without a git repository. It // validates the arguments in the command before executing. func SafeCmdWithoutRepo(ctx context.Context, stream CmdStream, globals []Option, sc SubCmd) (*command.Command, error) { diff --git a/internal/git/safecmd_test.go b/internal/git/safecmd_test.go index 6ab199390..7a0947e67 100644 --- a/internal/git/safecmd_test.go +++ b/internal/git/safecmd_test.go @@ -1,12 +1,15 @@ package git_test import ( + "bytes" "context" + "io/ioutil" "testing" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/internal/git" "gitlab.com/gitlab-org/gitaly/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/internal/helper/text" "gitlab.com/gitlab-org/gitaly/internal/testhelper" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" ) @@ -252,3 +255,97 @@ func disableGitCmd() testhelper.Cleanup { config.Config.Git.BinPath = "/bin/echo" return func() { config.Config.Git.BinPath = oldBinPath } } + +func TestSafeBareCmdInDir(t *testing.T) { + t.Run("no dir specified", func(t *testing.T) { + ctx, cancel := testhelper.Context() + defer cancel() + + _, err := git.SafeBareCmdInDir(ctx, "", git.CmdStream{}, nil, nil, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "no 'dir' provided") + }) + + t.Run("runs in dir", func(t *testing.T) { + _, repoPath, cleanup := testhelper.NewTestRepoWithWorktree(t) + defer cleanup() + + ctx, cancel := testhelper.Context() + defer cancel() + + var stderr bytes.Buffer + cmd, err := git.SafeBareCmdInDir(ctx, repoPath, git.CmdStream{Err: &stderr}, nil, nil, git.SubCmd{ + Name: "rev-parse", + Args: []string{"master"}, + }) + require.NoError(t, err) + + revData, err := ioutil.ReadAll(cmd) + require.NoError(t, err) + + require.NoError(t, cmd.Wait(), stderr.String()) + + require.Equal(t, "1e292f8fedd741b75372e19097c76d327140c312", text.ChompBytes(revData)) + }) + + t.Run("doesn't runs in non existing dir", func(t *testing.T) { + ctx, cancel := testhelper.Context() + defer cancel() + + var stderr bytes.Buffer + _, err := git.SafeBareCmdInDir(ctx, "non-existing-dir", git.CmdStream{Err: &stderr}, nil, nil, git.SubCmd{ + Name: "rev-parse", + Args: []string{"master"}, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "no such file or directory") + }) +} + +func TestSafeStderrCmd(t *testing.T) { + t.Run("stderr is empty if no error", func(t *testing.T) { + repo, _, cleanup := testhelper.NewTestRepoWithWorktree(t) + defer cleanup() + + ctx, cancel := testhelper.Context() + defer cancel() + + var stderr bytes.Buffer + cmd, err := git.SafeStderrCmd(ctx, &stderr, repo, nil, git.SubCmd{ + Name: "rev-parse", + Args: []string{"master"}, + }) + require.NoError(t, err) + + revData, err := ioutil.ReadAll(cmd) + require.NoError(t, err) + + require.NoError(t, cmd.Wait(), stderr.String()) + + require.Equal(t, "1e292f8fedd741b75372e19097c76d327140c312", text.ChompBytes(revData)) + require.Empty(t, stderr.String()) + }) + + t.Run("stderr contains error message on failure", func(t *testing.T) { + repo, _, cleanup := testhelper.NewTestRepoWithWorktree(t) + defer cleanup() + + ctx, cancel := testhelper.Context() + defer cancel() + + var stderr bytes.Buffer + cmd, err := git.SafeStderrCmd(ctx, &stderr, repo, nil, git.SubCmd{ + Name: "rev-parse", + Args: []string{"invalid-ref"}, + }) + require.NoError(t, err) + + revData, err := ioutil.ReadAll(cmd) + require.NoError(t, err) + + require.Error(t, cmd.Wait()) + + require.Equal(t, "invalid-ref", text.ChompBytes(revData)) + require.Contains(t, stderr.String(), "fatal: ambiguous argument 'invalid-ref': unknown revision or path not in the working tree.") + }) +} diff --git a/internal/gitaly/service/operations/squash.go b/internal/gitaly/service/operations/squash.go index e354bea8c..e1e3bfa9d 100644 --- a/internal/gitaly/service/operations/squash.go +++ b/internal/gitaly/service/operations/squash.go @@ -1,20 +1,62 @@ package operations import ( + "bytes" "context" + "errors" "fmt" + "io" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "strings" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" + "github.com/prometheus/client_golang/prometheus" + "gitlab.com/gitlab-org/gitaly/internal/command" + "gitlab.com/gitlab-org/gitaly/internal/git" + "gitlab.com/gitlab-org/gitaly/internal/git/alternates" "gitlab.com/gitlab-org/gitaly/internal/gitaly/rubyserver" + "gitlab.com/gitlab-org/gitaly/internal/helper" + "gitlab.com/gitlab-org/gitaly/internal/helper/text" + "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) +var ( + userSquashImplCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitaly_user_squash_counter", + Help: "Number of calls to UserSquash rpc for each implementation (ruby/go)", + }, + []string{"impl"}, + ) +) + +func init() { + prometheus.MustRegister(userSquashImplCounter) +} + +const ( + squashWorktreePrefix = "squash" + gitlabWorktreesSubDir = "gitlab-worktree" +) + func (s *server) UserSquash(ctx context.Context, req *gitalypb.UserSquashRequest) (*gitalypb.UserSquashResponse, error) { if err := validateUserSquashRequest(req); err != nil { return nil, status.Errorf(codes.InvalidArgument, "UserSquash: %v", err) } + if featureflag.IsEnabled(ctx, featureflag.GoUserSquash) { + userSquashImplCounter.WithLabelValues("go").Inc() + return s.userSquashGo(ctx, req) + } + + userSquashImplCounter.WithLabelValues("ruby").Inc() + client, err := s.ruby.OperationServiceClient(ctx) if err != nil { return nil, err @@ -59,3 +101,366 @@ func validateUserSquashRequest(req *gitalypb.UserSquashRequest) error { return nil } + +type gitError struct { + // ErrMsg error message from 'git' executable if any. + ErrMsg string + // Err is an error that happened during rebase process. + Err error +} + +func (er gitError) Error() string { + return er.ErrMsg + ": " + er.Err.Error() +} + +func (s *server) userSquashGo(ctx context.Context, req *gitalypb.UserSquashRequest) (*gitalypb.UserSquashResponse, error) { + if strings.Contains(req.GetSquashId(), "/") { + return nil, helper.ErrInvalidArgument(errors.New("worktree id can't contain slashes")) + } + + repoPath, env, err := alternates.PathAndEnv(req.GetRepository()) + if err != nil { + return nil, helper.ErrInternal(fmt.Errorf("alternate path: %w", err)) + } + + sha, err := s.runUserSquashGo(ctx, req, env, repoPath) + if err != nil { + var gitErr gitError + if errors.As(err, &gitErr) { + if gitErr.ErrMsg != "" { + // we log an actual error as it would be lost otherwise (it is not sent back to the client) + ctxlogrus.Extract(ctx).WithError(err).Error("user squash") + return &gitalypb.UserSquashResponse{GitError: gitErr.ErrMsg}, nil + } + } + + return nil, helper.ErrInternal(err) + } + + return &gitalypb.UserSquashResponse{SquashSha: sha}, nil +} + +func (s *server) runUserSquashGo(ctx context.Context, req *gitalypb.UserSquashRequest, env []string, repoPath string) (string, error) { + sparseDiffFiles, err := s.diffFiles(ctx, env, repoPath, req) + if err != nil { + return "", fmt.Errorf("define diff files: %w", err) + } + + if len(sparseDiffFiles) == 0 { + sha, err := s.userSquashWithNoDiff(ctx, req, repoPath, env) + if err != nil { + return "", fmt.Errorf("without sparse diff: %w", err) + } + + return sha, nil + } + + sha, err := s.userSquashWithDiffInFiles(ctx, req, repoPath, env, sparseDiffFiles) + if err != nil { + return "", fmt.Errorf("with sparse diff: %w", err) + } + + return sha, nil +} + +func (s *server) diffFiles(ctx context.Context, env []string, repoPath string, req *gitalypb.UserSquashRequest) ([]byte, error) { + var stdout, stderr bytes.Buffer + cmd, err := git.SafeBareCmd(ctx, git.CmdStream{Out: &stdout, Err: &stderr}, env, + []git.Option{git.ValueFlag{Name: "--git-dir", Value: repoPath}}, + git.SubCmd{ + Name: "diff", + Flags: []git.Option{git.Flag{Name: "--name-only"}, git.Flag{Name: "--diff-filter=ar"}, git.Flag{Name: "--binary"}}, + Args: []string{diffRange(req)}, + }, + ) + if err != nil { + return nil, fmt.Errorf("create 'git diff': %w", gitError{ErrMsg: stderr.String(), Err: err}) + } + + if err := cmd.Wait(); err != nil { + return nil, fmt.Errorf("on 'git diff' awaiting: %w", gitError{ErrMsg: stderr.String(), Err: err}) + } + + return stdout.Bytes(), nil +} + +var errNoFilesCheckedOut = errors.New("no files checked out") + +func (s *server) userSquashWithDiffInFiles(ctx context.Context, req *gitalypb.UserSquashRequest, repoPath string, env []string, diffFilesOut []byte) (string, error) { + repo := req.GetRepository() + worktreePath := newSquashWorktreePath(repoPath, req.GetSquashId()) + + if err := s.addWorktree(ctx, repo, worktreePath, ""); err != nil { + return "", fmt.Errorf("add worktree: %w", err) + } + + defer func(worktreeName string) { + if err := s.removeWorktree(ctx, repo, worktreeName); err != nil { + ctxlogrus.Extract(ctx).WithField("worktree_name", worktreeName).WithError(err).Error("failed to remove worktree") + } + }(filepath.Base(worktreePath)) + + worktreeGitPath, err := s.revParseGitDir(ctx, worktreePath) + if err != nil { + return "", fmt.Errorf("define git dir for worktree: %w", err) + } + + if err := runCmd(ctx, repo, "config", []git.Option{git.ConfigPair{Key: "core.sparseCheckout", Value: "true"}}, nil); err != nil { + return "", fmt.Errorf("on 'git config core.sparseCheckout true': %w", err) + } + + if err := s.createSparseCheckoutFile(worktreeGitPath, diffFilesOut); err != nil { + return "", fmt.Errorf("create sparse checkout file: %w", err) + } + + if err := s.checkout(ctx, worktreePath, req); err != nil { + if !errors.Is(err, errNoFilesCheckedOut) { + return "", fmt.Errorf("perform 'git checkout' with core.sparseCheckout true: %w", err) + } + + // try to perform checkout with disabled sparseCheckout feature + if err := runCmd(ctx, repo, "config", []git.Option{git.ConfigPair{Key: "core.sparseCheckout", Value: "false"}}, nil); err != nil { + return "", fmt.Errorf("on 'git config core.sparseCheckout false': %w", err) + } + + if err := s.checkout(ctx, worktreePath, req); err != nil { + return "", fmt.Errorf("perform 'git checkout' with core.sparseCheckout false: %w", err) + } + } + + sha, err := s.applyDiff(ctx, req, worktreePath, env) + if err != nil { + return "", fmt.Errorf("apply diff: %w", err) + } + + return sha, nil +} + +func (s *server) checkout(ctx context.Context, worktreePath string, req *gitalypb.UserSquashRequest) error { + var stderr bytes.Buffer + checkoutCmd, err := git.SafeBareCmdInDir(ctx, worktreePath, git.CmdStream{Err: &stderr}, nil, nil, + git.SubCmd{ + Name: "checkout", + Flags: []git.Option{git.Flag{Name: "--detach"}}, + Args: []string{req.GetStartSha()}, + }, + ) + if err != nil { + return fmt.Errorf("create 'git checkout': %w", gitError{ErrMsg: stderr.String(), Err: err}) + } + + if err = checkoutCmd.Wait(); err != nil { + if strings.Contains(stderr.String(), "error: Sparse checkout leaves no entry on working directory") { + return errNoFilesCheckedOut + } + + return fmt.Errorf("wait for 'git checkout': %w", gitError{ErrMsg: stderr.String(), Err: err}) + } + + return nil +} + +func (s *server) revParseGitDir(ctx context.Context, worktreePath string) (string, error) { + var stdout, stderr bytes.Buffer + cmd, err := git.SafeBareCmdInDir(ctx, worktreePath, git.CmdStream{Out: &stdout, Err: &stderr}, nil, nil, git.SubCmd{ + Name: "rev-parse", + Flags: []git.Option{git.Flag{Name: "--git-dir"}}, + }) + if err != nil { + return "", fmt.Errorf("creation of 'git rev-parse --git-dir': %w", gitError{ErrMsg: stderr.String(), Err: err}) + } + + if err := cmd.Wait(); err != nil { + return "", fmt.Errorf("wait for 'git rev-parse --git-dir': %w", gitError{ErrMsg: stderr.String(), Err: err}) + } + + return text.ChompBytes(stdout.Bytes()), nil +} + +func (s *server) userSquashWithNoDiff(ctx context.Context, req *gitalypb.UserSquashRequest, repoPath string, env []string) (string, error) { + repo := req.GetRepository() + worktreePath := newSquashWorktreePath(repoPath, req.GetSquashId()) + + if err := s.addWorktree(ctx, repo, worktreePath, req.GetStartSha()); err != nil { + return "", fmt.Errorf("add worktree: %w", err) + } + + defer func(worktreeName string) { + if err := s.removeWorktree(ctx, repo, worktreeName); err != nil { + ctxlogrus.Extract(ctx).WithField("worktree_name", worktreeName).WithError(err).Error("failed to remove worktree") + } + }(filepath.Base(worktreePath)) + + sha, err := s.applyDiff(ctx, req, worktreePath, env) + if err != nil { + return "", fmt.Errorf("apply diff: %w", err) + } + + return sha, nil +} + +func (s *server) addWorktree(ctx context.Context, repo *gitalypb.Repository, worktreePath string, committish string) error { + if err := runCmd(ctx, repo, "config", []git.Option{git.ConfigPair{Key: "core.splitIndex", Value: "false"}}, nil); err != nil { + return fmt.Errorf("on 'git config core.splitIndex false': %w", err) + } + + args := []string{worktreePath, committish} + if args[1] == "" { + args = args[:1] + } + + var stderr bytes.Buffer + cmd, err := git.SafeStderrCmd(ctx, &stderr, repo, nil, git.SubCmd{ + Name: "worktree", + Flags: []git.Option{git.SubSubCmd{Name: "add"}, git.Flag{Name: "--detach"}, git.Flag{Name: "--no-checkout"}}, + Args: args, + }) + if err != nil { + return fmt.Errorf("creation of 'git worktree add': %w", gitError{ErrMsg: stderr.String(), Err: err}) + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("wait for 'git worktree add': %w", gitError{ErrMsg: stderr.String(), Err: err}) + } + + return nil +} + +func (s *server) removeWorktree(ctx context.Context, repo *gitalypb.Repository, worktreeName string) error { + cmd, err := git.SafeCmd(ctx, repo, nil, git.SubCmd{ + Name: "worktree", + Flags: []git.Option{git.SubSubCmd{Name: "remove"}, git.Flag{Name: "--force"}}, + Args: []string{worktreeName}, + }) + if err != nil { + return fmt.Errorf("creation of 'worktree remove': %w", err) + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("wait for 'worktree remove': %w", err) + } + + return nil +} + +func (s *server) applyDiff(ctx context.Context, req *gitalypb.UserSquashRequest, worktreePath string, env []string) (string, error) { + diffRange := diffRange(req) + + var diffStderr bytes.Buffer + cmdDiff, err := git.SafeStderrCmd(ctx, &diffStderr, req.GetRepository(), nil, git.SubCmd{ + Name: "diff", + Flags: []git.Option{ + git.Flag{Name: "--binary"}, + }, + Args: []string{diffRange}, + }) + if err != nil { + return "", fmt.Errorf("creation of 'git diff' for range %q: %w", diffRange, gitError{ErrMsg: diffStderr.String(), Err: err}) + } + + var applyStderr bytes.Buffer + cmdApply, err := git.SafeBareCmdInDir(ctx, worktreePath, git.CmdStream{In: command.SetupStdin, Err: &applyStderr}, env, nil, git.SubCmd{ + Name: "apply", + Flags: []git.Option{ + git.Flag{Name: "--index"}, + git.Flag{Name: "--3way"}, + git.Flag{Name: "--whitespace=nowarn"}, + }, + }) + if err != nil { + return "", fmt.Errorf("creation of 'git apply' for range %q: %w", diffRange, gitError{ErrMsg: applyStderr.String(), Err: err}) + } + + if _, err := io.Copy(cmdApply, cmdDiff); err != nil { + return "", fmt.Errorf("piping 'git diff' -> 'git apply' for range %q: %w", diffRange, gitError{ErrMsg: applyStderr.String(), Err: err}) + } + + if err := cmdDiff.Wait(); err != nil { + return "", fmt.Errorf("wait for 'git diff' for range %q: %w", diffRange, gitError{ErrMsg: diffStderr.String(), Err: err}) + } + + if err := cmdApply.Wait(); err != nil { + return "", fmt.Errorf("wait for 'git apply' for range %q: %w", diffRange, gitError{ErrMsg: applyStderr.String(), Err: err}) + } + + commitEnv := append(env, + "GIT_COMMITTER_NAME="+string(req.GetUser().Name), + "GIT_COMMITTER_EMAIL="+string(req.GetUser().Email), + "GIT_AUTHOR_NAME="+string(req.GetAuthor().Name), + "GIT_AUTHOR_EMAIL="+string(req.GetAuthor().Email), + ) + + var commitStderr bytes.Buffer + cmdCommit, err := git.SafeBareCmdInDir(ctx, worktreePath, git.CmdStream{Err: &commitStderr}, commitEnv, nil, git.SubCmd{ + Name: "commit", + Flags: []git.Option{ + git.Flag{Name: "--no-verify"}, + git.Flag{Name: "--quiet"}, + git.ValueFlag{Name: "--message", Value: string(req.GetCommitMessage())}, + }, + }) + if err != nil { + return "", fmt.Errorf("creation of 'git commit': %w", gitError{ErrMsg: commitStderr.String(), Err: err}) + } + + if err := cmdCommit.Wait(); err != nil { + return "", fmt.Errorf("wait for 'git commit': %w", gitError{ErrMsg: commitStderr.String(), Err: err}) + } + + var revParseStdout, revParseStderr bytes.Buffer + revParseCmd, err := git.SafeBareCmdInDir(ctx, worktreePath, git.CmdStream{Out: &revParseStdout, Err: &revParseStderr}, env, nil, git.SubCmd{ + Name: "rev-parse", + Flags: []git.Option{ + git.Flag{Name: "--quiet"}, + git.Flag{Name: "--verify"}, + }, + Args: []string{"HEAD^{commit}"}, + }) + if err != nil { + return "", fmt.Errorf("creation of 'git rev-parse': %w", gitError{ErrMsg: revParseStderr.String(), Err: err}) + } + + if err := revParseCmd.Wait(); err != nil { + return "", fmt.Errorf("wait for 'git rev-parse': %w", gitError{ErrMsg: revParseStderr.String(), Err: err}) + } + + return text.ChompBytes(revParseStdout.Bytes()), nil +} + +func (s *server) createSparseCheckoutFile(worktreeGitPath string, diffFilesOut []byte) error { + if err := os.MkdirAll(filepath.Join(worktreeGitPath, "info"), 0755); err != nil { + return fmt.Errorf("create 'info' dir for worktree %q: %w", worktreeGitPath, err) + } + + if err := ioutil.WriteFile(filepath.Join(worktreeGitPath, "info", "sparse-checkout"), diffFilesOut, 0666); err != nil { + return fmt.Errorf("create 'sparse-checkout' file for worktree %q: %w", worktreeGitPath, err) + } + + return nil +} + +func diffRange(req *gitalypb.UserSquashRequest) string { + return req.GetStartSha() + "..." + req.GetEndSha() +} + +func newSquashWorktreePath(repoPath, squashID string) string { + prefix := []byte("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + rand.Shuffle(len(prefix), func(i, j int) { prefix[i], prefix[j] = prefix[j], prefix[i] }) + + worktreeName := squashWorktreePrefix + "-" + squashID + "-" + string(prefix[:32]) + return filepath.Join(repoPath, gitlabWorktreesSubDir, worktreeName) +} + +func runCmd(ctx context.Context, repo *gitalypb.Repository, cmd string, opts []git.Option, args []string) error { + var stderr bytes.Buffer + safeCmd, err := git.SafeStderrCmd(ctx, &stderr, repo, nil, git.SubCmd{Name: cmd, Flags: opts, Args: args}) + if err != nil { + return fmt.Errorf("create safe cmd %q: %w", cmd, gitError{ErrMsg: stderr.String(), Err: err}) + } + + if err := safeCmd.Wait(); err != nil { + return fmt.Errorf("wait safe cmd %q: %w", cmd, gitError{ErrMsg: stderr.String(), Err: err}) + } + + return nil +} diff --git a/internal/gitaly/service/operations/squash_test.go b/internal/gitaly/service/operations/squash_test.go index 7164f8dc5..989fe23ff 100644 --- a/internal/gitaly/service/operations/squash_test.go +++ b/internal/gitaly/service/operations/squash_test.go @@ -8,12 +8,15 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/internal/git/log" "gitlab.com/gitlab-org/gitaly/internal/helper/text" + "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/internal/testhelper" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) var ( @@ -28,13 +31,31 @@ var ( ) func TestSuccessfulUserSquashRequest(t *testing.T) { - ctx, cancel := testhelper.Context() - defer cancel() + featureSets, err := testhelper.NewFeatureSets([]featureflag.FeatureFlag{featureflag.GoUserSquash}) + require.NoError(t, err) + for _, featureSet := range featureSets { + t.Run("with sparse checkout: disabled "+featureSet.String(), func(t *testing.T) { + ctx, cancel := testhelper.Context() + defer cancel() + + ctx = featureSet.Disable(ctx) + + testSuccessfulUserSquashRequest(t, ctx, startSha, endSha) + }) + + t.Run("without sparse checkout: disabled "+featureSet.String(), func(t *testing.T) { + ctx, cancel := testhelper.Context() + defer cancel() + + ctx = featureSet.Disable(ctx) - testSuccessfulUserSquashRequest(t, ctx) + // there are no files that could be used for sparse checkout for those two commits + testSuccessfulUserSquashRequest(t, ctx, "60ecb67744cb56576c30214ff52294f8ce2def98", "c84ff944ff4529a70788a5e9003c2b7feae29047") + }) + } } -func testSuccessfulUserSquashRequest(t *testing.T, ctx context.Context) { +func testSuccessfulUserSquashRequest(t *testing.T, ctx context.Context, start, end string) { serverSocketPath, stop := runOperationServiceServer(t) defer stop() @@ -50,8 +71,8 @@ func testSuccessfulUserSquashRequest(t *testing.T, ctx context.Context) { SquashId: "1", Author: author, CommitMessage: commitMessage, - StartSha: startSha, - EndSha: endSha, + StartSha: start, + EndSha: end, } response, err := client.UserSquash(ctx, request) @@ -60,7 +81,7 @@ func testSuccessfulUserSquashRequest(t *testing.T, ctx context.Context) { commit, err := log.GetCommit(ctx, testRepo, response.SquashSha) require.NoError(t, err) - require.Equal(t, []string{startSha}, commit.ParentIds) + require.Equal(t, []string{start}, commit.ParentIds) require.Equal(t, author.Name, commit.Author.Name) require.Equal(t, author.Email, commit.Author.Email) require.Equal(t, testhelper.TestUser.Name, commit.Committer.Name) @@ -358,6 +379,19 @@ func TestFailedUserSquashRequestDueToValidations(t *testing.T) { }, code: codes.InvalidArgument, }, + { + desc: "worktree id can't contain slashes", + request: &gitalypb.UserSquashRequest{ + Repository: testRepo, + User: testhelper.TestUser, + SquashId: "1/2", + Author: testhelper.TestUser, + CommitMessage: commitMessage, + StartSha: startSha, + EndSha: endSha, + }, + code: codes.InvalidArgument, + }, } for _, testCase := range testCases { @@ -371,3 +405,87 @@ func TestFailedUserSquashRequestDueToValidations(t *testing.T) { }) } } + +func TestUserSquashWithGitError(t *testing.T) { + serverSocketPath, stop := runOperationServiceServer(t) + defer stop() + + client, conn := newOperationClient(t, serverSocketPath) + defer conn.Close() + + testRepo, _, cleanup := testhelper.NewTestRepo(t) + defer cleanup() + + testCases := []struct { + desc string + request *gitalypb.UserSquashRequest + gitError string + }{ + { + desc: "not existing start SHA", + request: &gitalypb.UserSquashRequest{ + Repository: testRepo, + SquashId: "1", + User: testhelper.TestUser, + Author: testhelper.TestUser, + CommitMessage: commitMessage, + StartSha: "doesntexisting", + EndSha: endSha, + }, + gitError: "fatal: ambiguous argument 'doesntexisting...54cec5282aa9f21856362fe321c800c236a61615'", + }, + { + desc: "not existing end SHA", + request: &gitalypb.UserSquashRequest{ + Repository: testRepo, + SquashId: "1", + User: testhelper.TestUser, + Author: testhelper.TestUser, + CommitMessage: commitMessage, + StartSha: startSha, + EndSha: "doesntexisting", + }, + gitError: "fatal: ambiguous argument 'b83d6e391c22777fca1ed3012fce84f633d7fed0...doesntexisting'", + }, + { + desc: "user has no name set", + request: &gitalypb.UserSquashRequest{ + Repository: testRepo, + SquashId: "1", + User: &gitalypb.User{Email: testhelper.TestUser.Email}, + Author: testhelper.TestUser, + CommitMessage: commitMessage, + StartSha: startSha, + EndSha: endSha, + }, + gitError: "fatal: empty ident name (for <janedoe@gitlab.com>) not allowed", + }, + { + desc: "author has no name set", + request: &gitalypb.UserSquashRequest{ + Repository: testRepo, + SquashId: "1", + User: testhelper.TestUser, + Author: &gitalypb.User{Email: testhelper.TestUser.Email}, + CommitMessage: commitMessage, + StartSha: startSha, + EndSha: endSha, + }, + gitError: "fatal: empty ident name (for <janedoe@gitlab.com>) not allowed", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.desc, func(t *testing.T) { + ctx, cancel := testhelper.Context() + defer cancel() + + resp, err := client.UserSquash(ctx, testCase.request) + s, ok := status.FromError(err) + require.True(t, ok) + assert.Equal(t, codes.OK, s.Code()) + assert.Empty(t, resp.SquashSha) + assert.Contains(t, resp.GitError, testCase.gitError) + }) + } +} diff --git a/internal/metadata/featureflag/feature_flags.go b/internal/metadata/featureflag/feature_flags.go index c1c1a4b63..d0f9f8eb9 100644 --- a/internal/metadata/featureflag/feature_flags.go +++ b/internal/metadata/featureflag/feature_flags.go @@ -34,6 +34,8 @@ var ( GoUserFFBranch = FeatureFlag{Name: "go_user_ff_branch", OnByDefault: false} // GoUserDeleteBranch enables the Go implementation of UserDeleteBranch GoUserDeleteBranch = FeatureFlag{Name: "go_user_delete_branch", OnByDefault: false} + // GoUserSquash enable the Go implementation of UserSquash + GoUserSquash = FeatureFlag{Name: "go_user_squash", OnByDefault: false} ) // All includes all feature flags. @@ -48,6 +50,7 @@ var All = []FeatureFlag{ GoUserMergeToRef, GoUserFFBranch, GoUserDeleteBranch, + GoUserSquash, } const ( |