Merge branch '607-server-implementation-repositoryservice-getarchive' into 'master'

Server Implementation RepositoryService::GetArchive

Closes #607

See merge request gitlab-org/gitaly!370

3 files changed, 253 insertions, 0 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 92a154cf0..3eb65f4d8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,8 @@ UNRELEASED
   https://gitlab.com/gitlab-org/gitaly/merge_requests/364
 - Fail harder during startup, fix version string
   https://gitlab.com/gitlab-org/gitaly/merge_requests/379
+- Implement RepositoryService.GetArchive RPC
+  https://gitlab.com/gitlab-org/gitaly/merge_requests/370
 
 v0.42.0
 
diff --git a/internal/service/repository/archive.go b/internal/service/repository/archive.go
new file mode 100644
index 000000000..b7f96c8b6
--- /dev/null
+++ b/internal/service/repository/archive.go
@@ -0,0 +1,72 @@
+package repository
+
+import (
+	"context"
+	"io"
+	"os/exec"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+
+	"gitlab.com/gitlab-org/gitaly/internal/command"
+	"gitlab.com/gitlab-org/gitaly/streamio"
+
+	pb "gitlab.com/gitlab-org/gitaly-proto/go"
+	"gitlab.com/gitlab-org/gitaly/internal/git"
+)
+
+func parseArchiveFormat(format pb.GetArchiveRequest_Format) (*exec.Cmd, string) {
+	switch format {
+	case pb.GetArchiveRequest_TAR:
+		return nil, "tar"
+	case pb.GetArchiveRequest_TAR_GZ:
+		return exec.Command("gzip", "-c", "-n"), "tar"
+	case pb.GetArchiveRequest_TAR_BZ2:
+		return exec.Command("bzip2", "-c"), "tar"
+	case pb.GetArchiveRequest_ZIP:
+		return nil, "zip"
+	}
+
+	return nil, ""
+}
+
+func handleArchive(ctx context.Context, writer io.Writer, repo *pb.Repository,
+	format pb.GetArchiveRequest_Format, prefix, commitID string) error {
+	compressCmd, formatArg := parseArchiveFormat(format)
+	if len(formatArg) == 0 {
+		return grpc.Errorf(codes.InvalidArgument, "invalid format")
+	}
+
+	archiveCommand, err := git.Command(ctx, repo, "archive",
+		"--format="+formatArg, "--prefix="+prefix+"/", commitID)
+	if err != nil {
+		return err
+	}
+
+	if compressCmd != nil {
+		command, err := command.New(ctx, compressCmd, archiveCommand, writer, nil)
+		if err != nil {
+			return err
+		}
+
+		if err := command.Wait(); err != nil {
+			return err
+		}
+	} else if _, err = io.Copy(writer, archiveCommand); err != nil {
+		return err
+	}
+
+	return archiveCommand.Wait()
+}
+
+func (s *server) GetArchive(in *pb.GetArchiveRequest, stream pb.RepositoryService_GetArchiveServer) error {
+	if err := git.ValidateRevision([]byte(in.CommitId)); err != nil {
+		return grpc.Errorf(codes.InvalidArgument, "invalid commitId: %v", err)
+	}
+
+	writer := streamio.NewWriter(func(p []byte) error {
+		return stream.Send(&pb.GetArchiveResponse{Data: p})
+	})
+
+	return handleArchive(stream.Context(), writer, in.Repository, in.Format, in.Prefix, in.CommitId)
+}
diff --git a/internal/service/repository/archive_test.go b/internal/service/repository/archive_test.go
new file mode 100644
index 000000000..bce819c08
--- /dev/null
+++ b/internal/service/repository/archive_test.go
@@ -0,0 +1,179 @@
+package repository
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"google.golang.org/grpc/codes"
+
+	"github.com/stretchr/testify/require"
+
+	pb "gitlab.com/gitlab-org/gitaly-proto/go"
+	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
+	"gitlab.com/gitlab-org/gitaly/streamio"
+)
+
+func TestGetArchiveSuccess(t *testing.T) {
+	server := runRepoServer(t)
+	defer server.Stop()
+
+	client, conn := newRepositoryClient(t)
+	defer conn.Close()
+
+	formats := []pb.GetArchiveRequest_Format{
+		pb.GetArchiveRequest_ZIP,
+		pb.GetArchiveRequest_TAR,
+		pb.GetArchiveRequest_TAR_GZ,
+		pb.GetArchiveRequest_TAR_BZ2,
+	}
+
+	testCases := []struct {
+		desc     string
+		prefix   string
+		commitID string
+	}{
+		{
+			desc:     "without-prefix",
+			commitID: "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863",
+			prefix:   "",
+		},
+		{
+			desc:     "with-prefix",
+			commitID: "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863",
+			prefix:   "my-prefix",
+		},
+	}
+
+	for _, tc := range testCases {
+		// Run test case with each format
+		for _, format := range formats {
+			testCaseName := fmt.Sprintf("%s-%s", tc.desc, format.String())
+			t.Run(testCaseName, func(t *testing.T) {
+				ctx, cancel := testhelper.Context()
+				defer cancel()
+
+				req := &pb.GetArchiveRequest{
+					Repository: testRepo,
+					CommitId:   tc.commitID,
+					Prefix:     tc.prefix,
+					Format:     format,
+				}
+				stream, err := client.GetArchive(ctx, req)
+				require.NoError(t, err)
+
+				data, err := consumeArchive(stream)
+				require.NoError(t, err)
+
+				archiveFile, err := ioutil.TempFile("", "")
+				require.NoError(t, err)
+				defer os.Remove(archiveFile.Name())
+
+				_, err = archiveFile.Write(data)
+				require.NoError(t, err)
+
+				contents := string(compressedFileContents(t, format, archiveFile.Name()))
+
+				require.Contains(t, contents, tc.prefix+"/.gitignore")
+				require.Contains(t, contents, tc.prefix+"/LICENSE")
+				require.Contains(t, contents, tc.prefix+"/README.md")
+			})
+		}
+	}
+}
+
+func TestGetArchiveFailure(t *testing.T) {
+	server := runRepoServer(t)
+	defer server.Stop()
+
+	client, conn := newRepositoryClient(t)
+	defer conn.Close()
+
+	commitID := "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863"
+
+	testCases := []struct {
+		desc     string
+		repo     *pb.Repository
+		prefix   string
+		commitID string
+		format   pb.GetArchiveRequest_Format
+		code     codes.Code
+	}{
+		{
+			desc:     "Repository doesn't exist",
+			repo:     &pb.Repository{StorageName: "fake", RelativePath: "path"},
+			prefix:   "",
+			commitID: commitID,
+			format:   pb.GetArchiveRequest_ZIP,
+			code:     codes.InvalidArgument,
+		},
+		{
+			desc:     "Repository is nil",
+			repo:     nil,
+			prefix:   "",
+			commitID: commitID,
+			format:   pb.GetArchiveRequest_ZIP,
+			code:     codes.InvalidArgument,
+		},
+		{
+			desc:     "CommitId is empty",
+			repo:     testRepo,
+			prefix:   "",
+			commitID: "",
+			format:   pb.GetArchiveRequest_ZIP,
+			code:     codes.InvalidArgument,
+		},
+		{
+			desc:     "Format is invalid",
+			repo:     testRepo,
+			prefix:   "",
+			commitID: "",
+			format:   pb.GetArchiveRequest_Format(-1),
+			code:     codes.InvalidArgument,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			ctx, cancel := testhelper.Context()
+			defer cancel()
+
+			req := &pb.GetArchiveRequest{
+				Repository: tc.repo,
+				CommitId:   tc.commitID,
+				Prefix:     tc.prefix,
+				Format:     tc.format,
+			}
+			stream, err := client.GetArchive(ctx, req)
+			require.NoError(t, err)
+
+			_, err = consumeArchive(stream)
+			testhelper.AssertGrpcError(t, err, tc.code, "")
+		})
+	}
+}
+
+func compressedFileContents(t *testing.T, format pb.GetArchiveRequest_Format, name string) []byte {
+	switch format {
+	case pb.GetArchiveRequest_TAR:
+		return testhelper.MustRunCommand(t, nil, "tar", "tf", name)
+	case pb.GetArchiveRequest_TAR_GZ:
+		return testhelper.MustRunCommand(t, nil, "tar", "ztf", name)
+	case pb.GetArchiveRequest_TAR_BZ2:
+		return testhelper.MustRunCommand(t, nil, "tar", "jtf", name)
+	case pb.GetArchiveRequest_ZIP:
+		return testhelper.MustRunCommand(t, nil, "unzip", "-l", name)
+	}
+
+	return nil
+}
+
+func consumeArchive(stream pb.RepositoryService_GetArchiveClient) ([]byte, error) {
+	reader := streamio.NewReader(func() ([]byte, error) {
+		response, err := stream.Recv()
+		return response.GetData(), err
+	})
+
+	return ioutil.ReadAll(reader)
+}