Merge branch 'smh-write-blob' into 'master'

Add WriteBlob and CatFile to repository

See merge request gitlab-org/gitaly!2635

4 files changed, 226 insertions, 11 deletions

diff --git a/internal/git/command.go b/internal/git/command.go
index f89ec74a2..aad8f78bf 100644
--- a/internal/git/command.go
+++ b/internal/git/command.go
@@ -10,7 +10,7 @@ import (
 )
 
 // unsafeCmdWithEnv creates a git.unsafeCmd with the given args, environment, and Repository
-func unsafeCmdWithEnv(ctx context.Context, extraEnv []string, repo repository.GitRepo, args ...string) (*command.Command, error) {
+func unsafeCmdWithEnv(ctx context.Context, extraEnv []string, stream CmdStream, repo repository.GitRepo, args ...string) (*command.Command, error) {
 	args, env, err := argsAndEnv(repo, args...)
 	if err != nil {
 		return nil, err
@@ -18,7 +18,7 @@ func unsafeCmdWithEnv(ctx context.Context, extraEnv []string, repo repository.Gi
 
 	env = append(env, extraEnv...)
 
-	return unsafeBareCmd(ctx, CmdStream{}, env, args...)
+	return unsafeBareCmd(ctx, stream, env, args...)
 }
 
 // unsafeStdinCmd creates a git.Command with the given args and Repository that is
diff --git a/internal/git/repository.go b/internal/git/repository.go
index 1f995f3d5..ab0dce686 100644
--- a/internal/git/repository.go
+++ b/internal/git/repository.go
@@ -12,8 +12,18 @@ import (
 
 	"gitlab.com/gitlab-org/gitaly/internal/command"
 	"gitlab.com/gitlab-org/gitaly/internal/git/repository"
+	"gitlab.com/gitlab-org/gitaly/internal/helper/text"
 )
 
+// InvalidObjectError is returned when trying to get an object id that is invalid or does not exist.
+type InvalidObjectError string
+
+func (err InvalidObjectError) Error() string { return fmt.Sprintf("invalid object %q", string(err)) }
+
+func errorWithStderr(err error, stderr []byte) error {
+	return fmt.Errorf("%w, stderr: %q", err, stderr)
+}
+
 var (
 	ErrReferenceNotFound = errors.New("reference not found")
 )
@@ -53,6 +63,15 @@ type Repository interface {
 	// will be deleted. If oldrev is the zero OID, the reference will
 	// created.
 	UpdateRef(ctx context.Context, reference, newrev, oldrev string) error
+
+	// WriteBlob writes a blob to the repository's object database and
+	// returns its object ID. Path is used by git to decide which filters to
+	// run on the content.
+	WriteBlob(ctx context.Context, path string, content io.Reader) (string, error)
+
+	// CatFile reads an object from the repository's object database. InvalidObjectError
+	// is returned if the oid does not refer to a valid object.
+	CatFile(ctx context.Context, oid string) ([]byte, error)
 }
 
 // localRepository represents a local Git repository.
@@ -70,8 +89,65 @@ func NewRepository(repo repository.GitRepo) Repository {
 // command creates a Git Command with the given args and Repository, executed
 // in the Repository. It validates the arguments in the command before
 // executing.
-func (repo *localRepository) command(ctx context.Context, globals []Option, cmd SubCmd) (*command.Command, error) {
-	return SafeStdinCmd(ctx, repo.repo, globals, cmd)
+func (repo *localRepository) command(ctx context.Context, globals []Option, cmd SubCmd, opts ...CmdOpt) (*command.Command, error) {
+	return SafeCmd(ctx, repo.repo, globals, cmd, opts...)
+}
+
+func (repo *localRepository) WriteBlob(ctx context.Context, path string, content io.Reader) (string, error) {
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+
+	cmd, err := repo.command(ctx, nil,
+		SubCmd{
+			Name: "hash-object",
+			Flags: []Option{
+				ValueFlag{Name: "--path", Value: path},
+				Flag{Name: "--stdin"}, Flag{Name: "-w"},
+			},
+		},
+		WithStdin(content),
+		WithStdout(stdout),
+		WithStderr(stderr),
+	)
+	if err != nil {
+		return "", err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return "", errorWithStderr(err, stderr.Bytes())
+	}
+
+	return text.ChompBytes(stdout.Bytes()), nil
+}
+
+func (repo *localRepository) CatFile(ctx context.Context, oid string) ([]byte, error) {
+	const msgInvalidObject = "fatal: Not a valid object name "
+
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+	cmd, err := repo.command(ctx, nil,
+		SubCmd{
+			Name:  "cat-file",
+			Flags: []Option{Flag{"-p"}},
+			Args:  []string{oid},
+		},
+		WithStdout(stdout),
+		WithStderr(stderr),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		msg := text.ChompBytes(stderr.Bytes())
+		if strings.HasPrefix(msg, msgInvalidObject) {
+			return nil, InvalidObjectError(strings.TrimPrefix(msg, msgInvalidObject))
+		}
+
+		return nil, errorWithStderr(err, stderr.Bytes())
+	}
+
+	return stdout.Bytes(), nil
 }
 
 func (repo *localRepository) ResolveRefish(ctx context.Context, refish string) (string, error) {
@@ -189,15 +265,11 @@ func (repo *localRepository) UpdateRef(ctx context.Context, reference, newrev, o
 	cmd, err := repo.command(ctx, nil, SubCmd{
 		Name:  "update-ref",
 		Flags: []Option{Flag{Name: "-z"}, Flag{Name: "--stdin"}},
-	})
+	}, WithStdin(strings.NewReader(fmt.Sprintf("update %s\x00%s\x00%s\x00", reference, newrev, oldrev))))
 	if err != nil {
 		return err
 	}
 
-	if _, err := fmt.Fprintf(cmd, "update %s\x00%s\x00%s\x00", reference, newrev, oldrev); err != nil {
-		return err
-	}
-
 	if err := cmd.Wait(); err != nil {
 		return fmt.Errorf("UpdateRef: failed updating reference %q from %q to %q: %v", reference, newrev, oldrev, err)
 	}
diff --git a/internal/git/repository_test.go b/internal/git/repository_test.go
index b18509bd7..3b6a2e3e0 100644
--- a/internal/git/repository_test.go
+++ b/internal/git/repository_test.go
@@ -2,9 +2,14 @@ package git
 
 import (
 	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
 )
@@ -252,6 +257,113 @@ func TestLocalRepository_GetReferences(t *testing.T) {
 	}
 }
 
+type ReaderFunc func([]byte) (int, error)
+
+func (fn ReaderFunc) Read(b []byte) (int, error) { return fn(b) }
+
+func TestLocalRepository_WriteBlob(t *testing.T) {
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	pbRepo, repoPath, clean := testhelper.InitBareRepo(t)
+	defer clean()
+
+	// write attributes file so we can verify WriteBlob runs the files through filters as
+	// appropriate
+	require.NoError(t, ioutil.WriteFile(filepath.Join(repoPath, "info", "attributes"), []byte(`
+crlf binary
+lf   text
+	`), os.ModePerm))
+
+	repo := NewRepository(pbRepo)
+
+	for _, tc := range []struct {
+		desc    string
+		path    string
+		input   io.Reader
+		sha     string
+		error   error
+		content string
+	}{
+		{
+			desc:  "error reading",
+			input: ReaderFunc(func([]byte) (int, error) { return 0, assert.AnError }),
+			error: errorWithStderr(assert.AnError, nil),
+		},
+		{
+			desc:    "successful empty blob",
+			input:   strings.NewReader(""),
+			sha:     "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+			content: "",
+		},
+		{
+			desc:    "successful blob",
+			input:   strings.NewReader("some content"),
+			sha:     "f0eec86f614944a81f87d879ebdc9a79aea0d7ea",
+			content: "some content",
+		},
+		{
+			desc:    "line endings not normalized",
+			path:    "crlf",
+			input:   strings.NewReader("\r\n"),
+			sha:     "d3f5a12faa99758192ecc4ed3fc22c9249232e86",
+			content: "\r\n",
+		},
+		{
+			desc:    "line endings normalized",
+			path:    "lf",
+			input:   strings.NewReader("\r\n"),
+			sha:     "8b137891791fe96927ad78e64b0aad7bded08bdc",
+			content: "\n",
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			sha, err := repo.WriteBlob(ctx, tc.path, tc.input)
+			require.Equal(t, tc.error, err)
+			if tc.error != nil {
+				return
+			}
+
+			assert.Equal(t, tc.sha, sha)
+			content, err := repo.CatFile(ctx, sha)
+			require.NoError(t, err)
+			assert.Equal(t, tc.content, string(content))
+		})
+	}
+}
+
+func TestLocalRepository_CatFile(t *testing.T) {
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	repo := NewRepository(testhelper.TestRepository())
+
+	for _, tc := range []struct {
+		desc    string
+		oid     string
+		content string
+		error   error
+	}{
+		{
+			desc:  "invalid object",
+			oid:   NullSHA,
+			error: InvalidObjectError(NullSHA),
+		},
+		{
+			desc: "valid object",
+			// README in gitlab-test
+			oid:     "3742e48c1108ced3bf45ac633b34b65ac3f2af04",
+			content: "Sample repo for testing gitlab features\n",
+		},
+	} {
+		t.Run(tc.desc, func(t *testing.T) {
+			content, err := repo.CatFile(ctx, tc.oid)
+			require.Equal(t, tc.error, err)
+			require.Equal(t, tc.content, string(content))
+		})
+	}
+}
+
 func TestLocalRepository_GetBranches(t *testing.T) {
 	ctx, cancel := testhelper.Context()
 	defer cancel()
diff --git a/internal/git/safecmd.go b/internal/git/safecmd.go
index f305fbfe3..38d8fd0fa 100644
--- a/internal/git/safecmd.go
+++ b/internal/git/safecmd.go
@@ -214,12 +214,39 @@ func ConvertGlobalOptions(options *gitalypb.GlobalOptions) []Option {
 }
 
 type cmdCfg struct {
-	env []string
+	env    []string
+	stdin  io.Reader
+	stdout io.Writer
+	stderr io.Writer
 }
 
 // CmdOpt is an option for running a command
 type CmdOpt func(*cmdCfg) error
 
+// WithStdin sets the command's stdin.
+func WithStdin(r io.Reader) CmdOpt {
+	return func(c *cmdCfg) error {
+		c.stdin = r
+		return nil
+	}
+}
+
+// WithStdout sets the command's stdout.
+func WithStdout(w io.Writer) CmdOpt {
+	return func(c *cmdCfg) error {
+		c.stdout = w
+		return nil
+	}
+}
+
+// WithStderr sets the command's stderr.
+func WithStderr(w io.Writer) CmdOpt {
+	return func(c *cmdCfg) error {
+		c.stderr = w
+		return nil
+	}
+}
+
 func handleOpts(cc *cmdCfg, opts []CmdOpt) error {
 	for _, opt := range opts {
 		if err := opt(cc); err != nil {
@@ -249,7 +276,11 @@ func SafeCmdWithEnv(ctx context.Context, env []string, repo repository.GitRepo,
 		return nil, err
 	}
 
-	return unsafeCmdWithEnv(ctx, append(env, cc.env...), repo, args...)
+	return unsafeCmdWithEnv(ctx, append(env, cc.env...), CmdStream{
+		In:  cc.stdin,
+		Out: cc.stdout,
+		Err: cc.stderr,
+	}, repo, args...)
 }
 
 // SafeBareCmd creates a git.Command with the given args, stream, and env. It