operations/merge: Merge merge2 stuff into mergeadd-merge2-using-merge-ort

2 files changed, 275 insertions, 24 deletions

diff --git a/internal/git/command_description.go b/internal/git/command_description.go
index faed63717..5c1db3201 100644
--- a/internal/git/command_description.go
+++ b/internal/git/command_description.go
@@ -115,6 +115,9 @@ var commandDescriptions = map[string]commandDescription{
 	"ls-tree": {
 		flags: scNoRefUpdates | scNoEndOfOptions,
 	},
+	"merge": {
+		flags: scNoRefUpdates,
+	},
 	"merge-base": {
 		flags: scNoRefUpdates,
 	},
diff --git a/internal/gitaly/service/operations/merge.go b/internal/gitaly/service/operations/merge.go
index acf13e6ad..9d98e0efe 100644
--- a/internal/gitaly/service/operations/merge.go
+++ b/internal/gitaly/service/operations/merge.go
@@ -1,14 +1,21 @@
 package operations
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/golang/protobuf/ptypes"
+	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/git"
+	"gitlab.com/gitlab-org/gitaly/v14/internal/git/alternates"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/git2go"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/helper"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/metadata/featureflag"
@@ -35,7 +42,10 @@ func validateMergeBranchRequest(request *gitalypb.UserMergeBranchRequest) error
 	return nil
 }
 
-func (s *Server) userMergeBranch(stream gitalypb.OperationService_UserMergeBranchServer) error {
+// UserMergeBranch is using the new merge-ort merge strategy. This is
+// a test to see how it goes and what is needed to make that work
+// wihtout a worktree.
+func (s *Server) UserMergeBranch(stream gitalypb.OperationService_UserMergeBranchServer) error {
 	ctx := stream.Context()
 
 	firstRequest, err := stream.Recv()
@@ -60,37 +70,29 @@ func (s *Server) userMergeBranch(stream gitalypb.OperationService_UserMergeBranc
 		return err
 	}
 
-	authorDate := time.Now()
-	if firstRequest.Timestamp != nil {
-		authorDate, err = ptypes.Timestamp(firstRequest.Timestamp)
-		if err != nil {
-			return helper.ErrInvalidArgument(err)
-		}
-	}
+	env := alternates.Env(repoPath, repo.GetGitObjectDirectory(), repo.GetGitAlternateObjectDirectories())
 
-	merge, err := git2go.MergeCommand{
-		Repository: repoPath,
-		AuthorName: string(firstRequest.User.Name),
-		AuthorMail: string(firstRequest.User.Email),
-		AuthorDate: authorDate,
-		Message:    string(firstRequest.Message),
-		Ours:       revision.String(),
-		Theirs:     firstRequest.CommitId,
-	}.Run(ctx, s.cfg)
+	err = s.userMerge(ctx, referenceName, firstRequest, env, repo, repoPath)
 	if err != nil {
-		if errors.Is(err, git2go.ErrInvalidArgument) {
-			return helper.ErrInvalidArgument(err)
+		var gitErr gitError2
+		if errors.As(err, &gitErr) {
+			if gitErr.ErrMsg != "" {
+				// we log an actual error as it would be lost otherwise (it is not sent back to the client)
+				ctxlogrus.Extract(ctx).WithError(err).Error("user merge")
+				return helper.ErrInternalf("Git error: %w", gitErr)
+			}
 		}
-		return err
+
+		return helper.ErrInternal(err)
 	}
 
-	mergeOID, err := git.NewObjectIDFromHex(merge.CommitID)
+	mergeOID, err := s.localrepo(repo).ResolveRevision(ctx, referenceName.Revision())
 	if err != nil {
-		return helper.ErrInternalf("could not parse merge ID: %w", err)
+		return helper.ErrInternalf("could not resolve merge ID: %w", err)
 	}
 
 	if err := stream.Send(&gitalypb.UserMergeBranchResponse{
-		CommitId: merge.CommitID,
+		CommitId: mergeOID.String(),
 	}); err != nil {
 		return err
 	}
@@ -123,7 +125,7 @@ func (s *Server) userMergeBranch(stream gitalypb.OperationService_UserMergeBranc
 
 	if err := stream.Send(&gitalypb.UserMergeBranchResponse{
 		BranchUpdate: &gitalypb.OperationBranchUpdate{
-			CommitId:      merge.CommitID,
+			CommitId:      mergeOID.String(),
 			RepoCreated:   false,
 			BranchCreated: false,
 		},
@@ -134,6 +136,252 @@ func (s *Server) userMergeBranch(stream gitalypb.OperationService_UserMergeBranc
 	return nil
 }
 
+const (
+	gitlabWorktreesSubDir2 = "gitlab-worktree"
+)
+
+func newWorktreePath(repoPath, prefix, id string) string {
+	suffix := []byte("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	rand.Shuffle(len(suffix), func(i, j int) { suffix[i], suffix[j] = suffix[j], suffix[i] })
+
+	worktreeName := prefix + "-" + id + "-" + string(suffix[:32])
+	return filepath.Join(repoPath, gitlabWorktreesSubDir2, worktreeName)
+}
+
+func (s *Server) addWorktree2(ctx context.Context, repo *gitalypb.Repository, worktreePath string, branch git.ReferenceName) error {
+	if err := s.runCmd2(ctx, repo, "config", []git.Option{git.ConfigPair{Key: "core.splitIndex", Value: "false"}}, nil); err != nil {
+		return fmt.Errorf("on 'git config core.splitIndex false': %w", err)
+	}
+
+	args := []string{worktreePath, branch.String()}
+
+	var stderr bytes.Buffer
+	cmd, err := s.gitCmdFactory.New(ctx, repo,
+		git.SubSubCmd{
+			Name:   "worktree",
+			Action: "add",
+			Args:   args,
+		},
+		git.WithStderr(&stderr),
+		git.WithRefTxHook(ctx, repo, s.cfg),
+	)
+	if err != nil {
+		return fmt.Errorf("creation of 'git worktree add': %w", gitError2{ErrMsg: stderr.String(), Err: err})
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf("wait for 'git worktree add': %w", gitError2{ErrMsg: stderr.String(), Err: err})
+	}
+
+	return nil
+}
+
+func (s *Server) checkout2(ctx context.Context, repo *gitalypb.Repository, worktreePath string, branch git.ReferenceName) error {
+	var stderr bytes.Buffer
+	checkoutCmd, err := s.gitCmdFactory.NewWithDir(ctx, worktreePath,
+		git.SubCmd{
+			Name: "checkout",
+			Args: []string{branch.String()},
+		},
+		git.WithStderr(&stderr),
+		git.WithRefTxHook(ctx, repo, s.cfg),
+	)
+	if err != nil {
+		return fmt.Errorf("create 'git checkout': %w", gitError2{ErrMsg: stderr.String(), Err: err})
+	}
+
+	if err = checkoutCmd.Wait(); err != nil {
+		if strings.Contains(stderr.String(), "error: Sparse checkout leaves no entry on working directory") {
+			return errNoFilesCheckedOut
+		}
+
+		return fmt.Errorf("wait for 'git checkout': %w", gitError2{ErrMsg: stderr.String(), Err: err})
+	}
+
+	return nil
+}
+
+func (s *Server) userMerge(ctx context.Context, referenceName git.ReferenceName, firstRequest *gitalypb.UserMergeBranchRequest,
+	env []string, repo *gitalypb.Repository, repoPath string) error {
+	commitID := firstRequest.CommitId
+	worktreePath := newWorktreePath(repoPath, "merge", commitID)
+
+	if err := s.addWorktree2(ctx, repo, worktreePath, referenceName); err != nil {
+		return fmt.Errorf("add worktree: %w", err)
+	}
+
+	defer func(worktreeName string) {
+		ctx, cancel := context.WithCancel(helper.SuppressCancellation(ctx))
+		defer cancel()
+
+		if err := s.removeWorktree2(ctx, repo, worktreeName); err != nil {
+			ctxlogrus.Extract(ctx).WithField("worktree_name", worktreeName).WithError(err).Error("failed to remove worktree")
+		}
+	}(filepath.Base(worktreePath))
+
+	worktreeGitPath, err := s.revParseGitDir(ctx, worktreePath)
+	if err != nil {
+		return fmt.Errorf("define git dir for worktree: %w", err)
+	}
+
+	if err := s.runCmd2(ctx, repo, "config", []git.Option{git.ConfigPair{Key: "core.sparseCheckout", Value: "true"}}, nil); err != nil {
+		return fmt.Errorf("on 'git config core.sparseCheckout true': %w", err)
+	}
+
+	sparseDiffFiles, err := s.diffFiles2(ctx, env, repoPath, referenceName, commitID)
+	if err != nil {
+		return fmt.Errorf("define diff files: %w", err)
+	}
+
+	if err := s.createSparseCheckoutFile2(worktreeGitPath, sparseDiffFiles); err != nil {
+		return fmt.Errorf("create sparse checkout file: %w", err)
+	}
+
+	if err := s.checkout2(ctx, repo, worktreePath, referenceName); err != nil {
+		if !errors.Is(err, errNoFilesCheckedOut) {
+			return fmt.Errorf("perform 'git checkout' with core.sparseCheckout true: %w", err)
+		}
+
+		// try to perform checkout with disabled sparseCheckout feature
+		if err := s.runCmd2(ctx, repo, "config", []git.Option{git.ConfigPair{Key: "core.sparseCheckout", Value: "false"}}, nil); err != nil {
+			return fmt.Errorf("on 'git config core.sparseCheckout false': %w", err)
+		}
+
+		if err := s.checkout2(ctx, repo, worktreePath, referenceName); err != nil {
+			return fmt.Errorf("perform 'git checkout' with core.sparseCheckout false: %w", err)
+		}
+	}
+
+	// 	Merge params:
+	//               AuthorName: string(firstRequest.User.Name),
+	//               AuthorMail: string(firstRequest.User.Email),
+	//               AuthorDate: authorDate,
+	//               Message:    string(firstRequest.Message),
+	//               Ours:       revision.String(),
+	//               Theirs:     firstRequest.CommitId
+
+	env = append(env, fmt.Sprintf("GIT_AUTHOR_NAME=%s", string(firstRequest.User.Name)))
+	env = append(env, fmt.Sprintf("GIT_AUTHOR_EMAIL=%s", string(firstRequest.User.Email)))
+
+	authorDate := time.Now()
+	if firstRequest.Timestamp != nil {
+		authorDate, err = ptypes.Timestamp(firstRequest.Timestamp)
+		if err != nil {
+			return helper.ErrInvalidArgument(err)
+		}
+	}
+	env = append(env, fmt.Sprintf("GIT_AUTHOR_DATE=%s", authorDate))
+
+	var mergeStderr bytes.Buffer
+	cmdMerge, err := s.gitCmdFactory.NewWithDir(ctx, worktreePath,
+		git.SubCmd{
+			Name: "merge",
+			Flags: []git.Option{
+				git.Flag{Name: "--strategy=ort"},
+				git.ValueFlag{Name: "-m", Value: string(firstRequest.Message)},
+			},
+			Args: []string{commitID},
+		},
+		git.WithEnv(env...),
+		git.WithStderr(&mergeStderr),
+	)
+	if err != nil {
+		return fmt.Errorf("merge for branch: %w", gitError2{ErrMsg: mergeStderr.String(), Err: err})
+	}
+
+	if err := cmdMerge.Wait(); err != nil {
+		return fmt.Errorf("wait for 'git merge': %w", gitError{ErrMsg: mergeStderr.String(), Err: err})
+	}
+
+	return nil
+}
+
+type gitError2 struct {
+	// ErrMsg error message from 'git' executable if any.
+	ErrMsg string
+	// Err is an error that happened during rebase process.
+	Err error
+}
+
+func (er gitError2) Error() string {
+	return er.ErrMsg + ": " + er.Err.Error()
+}
+
+func (s *Server) removeWorktree2(ctx context.Context, repo *gitalypb.Repository, worktreeName string) error {
+	cmd, err := s.gitCmdFactory.New(ctx, repo,
+		git.SubSubCmd{
+			Name:   "worktree",
+			Action: "remove",
+			Flags:  []git.Option{git.Flag{Name: "--force"}},
+			Args:   []string{worktreeName},
+		},
+		git.WithRefTxHook(ctx, repo, s.cfg),
+	)
+	if err != nil {
+		return fmt.Errorf("creation of 'worktree remove': %w", err)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf("wait for 'worktree remove': %w", err)
+	}
+
+	return nil
+}
+
+func (s *Server) createSparseCheckoutFile2(worktreeGitPath string, diffFilesOut []byte) error {
+	if err := os.MkdirAll(filepath.Join(worktreeGitPath, "info"), 0755); err != nil {
+		return fmt.Errorf("create 'info' dir for worktree %q: %w", worktreeGitPath, err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(worktreeGitPath, "info", "sparse-checkout"), diffFilesOut, 0666); err != nil {
+		return fmt.Errorf("create 'sparse-checkout' file for worktree %q: %w", worktreeGitPath, err)
+	}
+
+	return nil
+}
+
+func (s *Server) runCmd2(ctx context.Context, repo *gitalypb.Repository, cmd string, opts []git.Option, args []string) error {
+	var stderr bytes.Buffer
+	safeCmd, err := s.gitCmdFactory.New(ctx, repo, git.SubCmd{Name: cmd, Flags: opts, Args: args}, git.WithStderr(&stderr))
+	if err != nil {
+		return fmt.Errorf("create safe cmd %q: %w", cmd, gitError2{ErrMsg: stderr.String(), Err: err})
+	}
+
+	if err := safeCmd.Wait(); err != nil {
+		return fmt.Errorf("wait safe cmd %q: %w", cmd, gitError2{ErrMsg: stderr.String(), Err: err})
+	}
+
+	return nil
+}
+
+func (s *Server) diffFiles2(ctx context.Context, env []string, repoPath string,
+	referenceName git.ReferenceName, commitID string) ([]byte, error) {
+	var stdout, stderr bytes.Buffer
+	cmd, err := s.gitCmdFactory.NewWithDir(ctx, repoPath,
+		git.SubCmd{
+			Name:  "diff",
+			Flags: []git.Option{git.Flag{Name: "--name-only"}, git.Flag{Name: "--diff-filter=ar"}, git.Flag{Name: "--binary"}},
+			Args:  []string{diffRange2(referenceName, commitID)},
+		},
+		git.WithEnv(env...),
+		git.WithStdout(&stdout),
+		git.WithStderr(&stderr),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("create 'git diff': %w", gitError{ErrMsg: stderr.String(), Err: err})
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return nil, fmt.Errorf("on 'git diff' awaiting: %w", gitError{ErrMsg: stderr.String(), Err: err})
+	}
+
+	return stdout.Bytes(), nil
+}
+
+func diffRange2(referenceName git.ReferenceName, commitID string) string {
+	return referenceName.String() + "..." + commitID
+}
+
 func validateFFRequest(in *gitalypb.UserFFBranchRequest) error {
 	if len(in.Branch) == 0 {
 		return fmt.Errorf("empty branch name")