gitpipe: Add DiffTree that calls git-diff-tree(1)

This change adds a function to get a RevisionIterator from the output of
git-diff-tree. This iterator will loop through all the new objects that
has been introduced between the two given revisions.

2 files changed, 287 insertions, 0 deletions

diff --git a/internal/git/gitpipe/diff_tree.go b/internal/git/gitpipe/diff_tree.go
new file mode 100644
index 000000000..c41e77a6a
--- /dev/null
+++ b/internal/git/gitpipe/diff_tree.go
@@ -0,0 +1,147 @@
+package gitpipe
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"strings"
+
+	"gitlab.com/gitlab-org/gitaly/v15/internal/git"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
+)
+
+// diffTreeConfig is configuration for the DiffTree pipeline step.
+type diffTreeConfig struct {
+	recursive        bool
+	ignoreSubmodules bool
+	skipResult       func(*RevisionResult) bool
+}
+
+// DiffTreeOption is an option for the DiffTree pipeline step.
+type DiffTreeOption func(cfg *diffTreeConfig)
+
+// DiffTreeWithRecursive will make DiffTree recurse into subtrees.
+func DiffTreeWithRecursive() DiffTreeOption {
+	return func(cfg *diffTreeConfig) {
+		cfg.recursive = true
+	}
+}
+
+// DiffTreeWithIgnoreSubmodules causes git-diff-tree(1) to exclude submodule changes.
+func DiffTreeWithIgnoreSubmodules() DiffTreeOption {
+	return func(cfg *diffTreeConfig) {
+		cfg.ignoreSubmodules = true
+	}
+}
+
+// DiffTreeWithSkip will execute the given function for each RevisionResult processed by the
+// pipeline. If the callback returns `true`, then the object will be skipped and not passed down
+// the pipeline.
+func DiffTreeWithSkip(skipResult func(*RevisionResult) bool) DiffTreeOption {
+	return func(cfg *diffTreeConfig) {
+		cfg.skipResult = skipResult
+	}
+}
+
+// DiffTree runs git-diff-tree(1) between the two given revisions. The returned
+// channel will contain the new object IDs listed by this command. For deleted
+// files this would be git.ZeroOID. Cancelling the context will cause the
+// pipeline to be cancelled, too. By default, it will not recurse into subtrees.
+func DiffTree(
+	ctx context.Context,
+	repo *localrepo.Repo,
+	leftRevision, rightRevision string,
+	options ...DiffTreeOption,
+) RevisionIterator {
+	var cfg diffTreeConfig
+	for _, option := range options {
+		option(&cfg)
+	}
+
+	resultChan := make(chan RevisionResult)
+	go func() {
+		defer close(resultChan)
+
+		flags := []git.Option{}
+
+		if cfg.recursive {
+			flags = append(flags, git.Flag{Name: "-r"})
+		}
+		if cfg.ignoreSubmodules {
+			flags = append(flags, git.Flag{Name: "--ignore-submodules"})
+		}
+
+		var stderr strings.Builder
+		cmd, err := repo.Exec(ctx,
+			git.SubCmd{
+				Name:  "diff-tree",
+				Flags: flags,
+				Args:  []string{leftRevision, rightRevision},
+			},
+			git.WithStderr(&stderr),
+		)
+		if err != nil {
+			sendRevisionResult(ctx, resultChan, RevisionResult{
+				err: fmt.Errorf("executing diff-tree: %w", err),
+			})
+			return
+		}
+
+		scanner := bufio.NewScanner(cmd)
+		for scanner.Scan() {
+			// We need to copy the line here because we'll hand it over to the caller
+			// asynchronously, and the next call to `Scan()` will overwrite the buffer.
+			line := make([]byte, len(scanner.Bytes()))
+			copy(line, scanner.Bytes())
+
+			attrsAndFile := bytes.SplitN(line, []byte{'\t'}, 2)
+			if len(attrsAndFile) != 2 {
+				sendRevisionResult(ctx, resultChan, RevisionResult{
+					err: fmt.Errorf("splitting diff-tree attributes and file"),
+				})
+				return
+			}
+
+			attrs := bytes.SplitN(attrsAndFile[0], []byte{' '}, 5)
+			if len(attrs) != 5 {
+				sendRevisionResult(ctx, resultChan, RevisionResult{
+					err: fmt.Errorf("splitting diff-tree attributes"),
+				})
+				return
+			}
+
+			result := RevisionResult{
+				OID:        git.ObjectID(attrs[3]),
+				ObjectName: attrsAndFile[1],
+			}
+
+			if cfg.skipResult != nil && cfg.skipResult(&result) {
+				continue
+			}
+
+			if isDone := sendRevisionResult(ctx, resultChan, result); isDone {
+				return
+			}
+		}
+
+		if err := scanner.Err(); err != nil {
+			sendRevisionResult(ctx, resultChan, RevisionResult{
+				err: fmt.Errorf("scanning diff-tree output: %w", err),
+			})
+			return
+		}
+
+		if err := cmd.Wait(); err != nil {
+			sendRevisionResult(ctx, resultChan, RevisionResult{
+				err: fmt.Errorf("diff-tree pipeline command: %w, stderr: %q", err, stderr.String()),
+			})
+			return
+		}
+	}()
+
+	return &revisionIterator{
+		ctx: ctx,
+		ch:  resultChan,
+	}
+}
diff --git a/internal/git/gitpipe/diff_tree_test.go b/internal/git/gitpipe/diff_tree_test.go
new file mode 100644
index 000000000..700772244
--- /dev/null
+++ b/internal/git/gitpipe/diff_tree_test.go
@@ -0,0 +1,140 @@
+package gitpipe
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testcfg"
+)
+
+func TestDiffTree(t *testing.T) {
+	cfg := testcfg.Build(t)
+
+	repoProto, _ := gittest.CloneRepo(t, cfg, cfg.Storages[0])
+	repo := localrepo.NewTestRepo(t, cfg, repoProto)
+
+	for _, tc := range []struct {
+		desc            string
+		leftRevision    string
+		rightRevision   string
+		options         []DiffTreeOption
+		expectedResults []RevisionResult
+		expectedErr     error
+	}{
+		{
+			desc:          "single file",
+			leftRevision:  "b83d6e391c22777fca1ed3012fce84f633d7fed0",
+			rightRevision: "4a24d82dbca5c11c61556f3b35ca472b7463187e",
+			expectedResults: []RevisionResult{
+				{
+					OID:        "c60514b6d3d6bf4bec1030f70026e34dfbd69ad5",
+					ObjectName: []byte("README.md"),
+				},
+			},
+		},
+		{
+			desc:          "single file in subtree without recursive",
+			leftRevision:  "7975be0116940bf2ad4321f79d02a55c5f7779aa",
+			rightRevision: "1e292f8fedd741b75372e19097c76d327140c312",
+			expectedResults: []RevisionResult{
+				{
+					OID:        "ceb102b8d3f9a95c2eb979213e49f7cc1b23d56e",
+					ObjectName: []byte("files"),
+				},
+			},
+		},
+		{
+			desc:          "single file in subtree with recursive",
+			leftRevision:  "7975be0116940bf2ad4321f79d02a55c5f7779aa",
+			rightRevision: "1e292f8fedd741b75372e19097c76d327140c312",
+			options: []DiffTreeOption{
+				DiffTreeWithRecursive(),
+			},
+			expectedResults: []RevisionResult{
+				{
+					OID:        "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+					ObjectName: []byte("files/flat/path/correct/content.txt"),
+				},
+			},
+		},
+		{
+			desc:          "with submodules",
+			leftRevision:  "570e7b2abdd848b95f2f578043fc23bd6f6fd24d",
+			rightRevision: "5937ac0a7beb003549fc5fd26fc247adbce4a52e",
+			expectedResults: []RevisionResult{
+				{
+					OID:        "efd587ccb47caf5f31fc954edb21f0a713d9ecc3",
+					ObjectName: []byte(".gitmodules"),
+				},
+				{
+					OID:        "645f6c4c82fd3f5e06f67134450a570b795e55a6",
+					ObjectName: []byte("gitlab-grack"),
+				},
+			},
+		},
+		{
+			desc:          "without submodules",
+			leftRevision:  "570e7b2abdd848b95f2f578043fc23bd6f6fd24d",
+			rightRevision: "5937ac0a7beb003549fc5fd26fc247adbce4a52e",
+			options: []DiffTreeOption{
+				DiffTreeWithIgnoreSubmodules(),
+			},
+			expectedResults: []RevisionResult{
+				{
+					OID:        "efd587ccb47caf5f31fc954edb21f0a713d9ecc3",
+					ObjectName: []byte(".gitmodules"),
+				},
+			},
+		},
+		{
+			desc:          "with skip function",
+			leftRevision:  "570e7b2abdd848b95f2f578043fc23bd6f6fd24d",
+			rightRevision: "5937ac0a7beb003549fc5fd26fc247adbce4a52e",
+			options: []DiffTreeOption{
+				DiffTreeWithSkip(func(r *RevisionResult) bool {
+					return r.OID == "efd587ccb47caf5f31fc954edb21f0a713d9ecc3"
+				}),
+			},
+			expectedResults: []RevisionResult{
+				{
+					OID:        "645f6c4c82fd3f5e06f67134450a570b795e55a6",
+					ObjectName: []byte("gitlab-grack"),
+				},
+			},
+		},
+		{
+			desc:          "invalid revision",
+			leftRevision:  "refs/heads/master",
+			rightRevision: "refs/heads/does-not-exist",
+			expectedErr: errors.New("diff-tree pipeline command: exit status 128, stderr: " +
+				"\"fatal: ambiguous argument 'refs/heads/does-not-exist': unknown revision or path not in the working tree.\\n" +
+				"Use '--' to separate paths from revisions, like this:\\n" +
+				"'git <command> [<revision>...] -- [<file>...]'\\n\""),
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			ctx := testhelper.Context(t)
+
+			it := DiffTree(ctx, repo, tc.leftRevision, tc.rightRevision, tc.options...)
+
+			var results []RevisionResult
+			for it.Next() {
+				results = append(results, it.Result())
+			}
+
+			// We're converting the error here to a plain un-nested error such that we
+			// don't have to replicate the complete error's structure.
+			err := it.Err()
+			if err != nil {
+				err = errors.New(err.Error())
+			}
+
+			require.Equal(t, tc.expectedErr, err)
+			require.Equal(t, tc.expectedResults, results)
+		})
+	}
+}