Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'workhorse/internal/git')
-rw-r--r--workhorse/internal/git/archive.go216
-rw-r--r--workhorse/internal/git/archive_test.go87
-rw-r--r--workhorse/internal/git/blob.go47
-rw-r--r--workhorse/internal/git/blob_test.go17
-rw-r--r--workhorse/internal/git/diff.go48
-rw-r--r--workhorse/internal/git/error.go4
-rw-r--r--workhorse/internal/git/format-patch.go48
-rw-r--r--workhorse/internal/git/git-http.go100
-rw-r--r--workhorse/internal/git/info-refs.go76
-rw-r--r--workhorse/internal/git/pktline.go59
-rw-r--r--workhorse/internal/git/pktline_test.go39
-rw-r--r--workhorse/internal/git/receive-pack.go33
-rw-r--r--workhorse/internal/git/responsewriter.go75
-rw-r--r--workhorse/internal/git/snapshot.go64
-rw-r--r--workhorse/internal/git/upload-pack.go57
-rw-r--r--workhorse/internal/git/upload-pack_test.go85
16 files changed, 1055 insertions, 0 deletions
diff --git a/workhorse/internal/git/archive.go b/workhorse/internal/git/archive.go
new file mode 100644
index 00000000000..b7575be2c02
--- /dev/null
+++ b/workhorse/internal/git/archive.go
@@ -0,0 +1,216 @@
+/*
+In this file we handle 'git archive' downloads
+*/
+
+package git
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "path"
+ "path/filepath"
+ "regexp"
+ "time"
+
+ "github.com/golang/protobuf/proto" //lint:ignore SA1019 https://gitlab.com/gitlab-org/gitlab-workhorse/-/issues/274
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+)
+
+type archive struct{ senddata.Prefix }
+type archiveParams struct {
+ ArchivePath string
+ ArchivePrefix string
+ CommitId string
+ GitalyServer gitaly.Server
+ GitalyRepository gitalypb.Repository
+ DisableCache bool
+ GetArchiveRequest []byte
+}
+
+var (
+ SendArchive = &archive{"git-archive:"}
+ gitArchiveCache = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "gitlab_workhorse_git_archive_cache",
+ Help: "Cache hits and misses for 'git archive' streaming",
+ },
+ []string{"result"},
+ )
+)
+
+func (a *archive) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params archiveParams
+ if err := a.Unpack(&params, sendData); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendArchive: unpack sendData: %v", err))
+ return
+ }
+
+ urlPath := r.URL.Path
+ format, ok := parseBasename(filepath.Base(urlPath))
+ if !ok {
+ helper.Fail500(w, r, fmt.Errorf("SendArchive: invalid format: %s", urlPath))
+ return
+ }
+
+ cacheEnabled := !params.DisableCache
+ archiveFilename := path.Base(params.ArchivePath)
+
+ if cacheEnabled {
+ cachedArchive, err := os.Open(params.ArchivePath)
+ if err == nil {
+ defer cachedArchive.Close()
+ gitArchiveCache.WithLabelValues("hit").Inc()
+ setArchiveHeaders(w, format, archiveFilename)
+ // Even if somebody deleted the cachedArchive from disk since we opened
+ // the file, Unix file semantics guarantee we can still read from the
+ // open file in this process.
+ http.ServeContent(w, r, "", time.Unix(0, 0), cachedArchive)
+ return
+ }
+ }
+
+ gitArchiveCache.WithLabelValues("miss").Inc()
+
+ var tempFile *os.File
+ var err error
+
+ if cacheEnabled {
+ // We assume the tempFile has a unique name so that concurrent requests are
+ // safe. We create the tempfile in the same directory as the final cached
+ // archive we want to create so that we can use an atomic link(2) operation
+ // to finalize the cached archive.
+ tempFile, err = prepareArchiveTempfile(path.Dir(params.ArchivePath), archiveFilename)
+ if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendArchive: create tempfile: %v", err))
+ return
+ }
+ defer tempFile.Close()
+ defer os.Remove(tempFile.Name())
+ }
+
+ var archiveReader io.Reader
+
+ archiveReader, err = handleArchiveWithGitaly(r, params, format)
+ if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("operations.GetArchive: %v", err))
+ return
+ }
+
+ reader := archiveReader
+ if cacheEnabled {
+ reader = io.TeeReader(archiveReader, tempFile)
+ }
+
+ // Start writing the response
+ setArchiveHeaders(w, format, archiveFilename)
+ w.WriteHeader(200) // Don't bother with HTTP 500 from this point on, just return
+ if _, err := io.Copy(w, reader); err != nil {
+ helper.LogError(r, &copyError{fmt.Errorf("SendArchive: copy 'git archive' output: %v", err)})
+ return
+ }
+
+ if cacheEnabled {
+ err := finalizeCachedArchive(tempFile, params.ArchivePath)
+ if err != nil {
+ helper.LogError(r, fmt.Errorf("SendArchive: finalize cached archive: %v", err))
+ return
+ }
+ }
+}
+
+func handleArchiveWithGitaly(r *http.Request, params archiveParams, format gitalypb.GetArchiveRequest_Format) (io.Reader, error) {
+ var request *gitalypb.GetArchiveRequest
+ ctx, c, err := gitaly.NewRepositoryClient(r.Context(), params.GitalyServer)
+ if err != nil {
+ return nil, err
+ }
+
+ if params.GetArchiveRequest != nil {
+ request = &gitalypb.GetArchiveRequest{}
+
+ if err := proto.Unmarshal(params.GetArchiveRequest, request); err != nil {
+ return nil, fmt.Errorf("unmarshal GetArchiveRequest: %v", err)
+ }
+ } else {
+ request = &gitalypb.GetArchiveRequest{
+ Repository: &params.GitalyRepository,
+ CommitId: params.CommitId,
+ Prefix: params.ArchivePrefix,
+ Format: format,
+ }
+ }
+
+ return c.ArchiveReader(ctx, request)
+}
+
+func setArchiveHeaders(w http.ResponseWriter, format gitalypb.GetArchiveRequest_Format, archiveFilename string) {
+ w.Header().Del("Content-Length")
+ w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, archiveFilename))
+ // Caching proxies usually don't cache responses with Set-Cookie header
+ // present because it implies user-specific data, which is not the case
+ // for repository archives.
+ w.Header().Del("Set-Cookie")
+ if format == gitalypb.GetArchiveRequest_ZIP {
+ w.Header().Set("Content-Type", "application/zip")
+ } else {
+ w.Header().Set("Content-Type", "application/octet-stream")
+ }
+ w.Header().Set("Content-Transfer-Encoding", "binary")
+}
+
+func prepareArchiveTempfile(dir string, prefix string) (*os.File, error) {
+ if err := os.MkdirAll(dir, 0700); err != nil {
+ return nil, err
+ }
+ return ioutil.TempFile(dir, prefix)
+}
+
+func finalizeCachedArchive(tempFile *os.File, archivePath string) error {
+ if err := tempFile.Close(); err != nil {
+ return err
+ }
+ if err := os.Link(tempFile.Name(), archivePath); err != nil && !os.IsExist(err) {
+ return err
+ }
+
+ return nil
+}
+
+var (
+ patternZip = regexp.MustCompile(`\.zip$`)
+ patternTar = regexp.MustCompile(`\.tar$`)
+ patternTarGz = regexp.MustCompile(`\.(tar\.gz|tgz|gz)$`)
+ patternTarBz2 = regexp.MustCompile(`\.(tar\.bz2|tbz|tbz2|tb2|bz2)$`)
+)
+
+func parseBasename(basename string) (gitalypb.GetArchiveRequest_Format, bool) {
+ var format gitalypb.GetArchiveRequest_Format
+
+ switch {
+ case (basename == "archive"):
+ format = gitalypb.GetArchiveRequest_TAR_GZ
+ case patternZip.MatchString(basename):
+ format = gitalypb.GetArchiveRequest_ZIP
+ case patternTar.MatchString(basename):
+ format = gitalypb.GetArchiveRequest_TAR
+ case patternTarGz.MatchString(basename):
+ format = gitalypb.GetArchiveRequest_TAR_GZ
+ case patternTarBz2.MatchString(basename):
+ format = gitalypb.GetArchiveRequest_TAR_BZ2
+ default:
+ return format, false
+ }
+
+ return format, true
+}
diff --git a/workhorse/internal/git/archive_test.go b/workhorse/internal/git/archive_test.go
new file mode 100644
index 00000000000..4b0753499e5
--- /dev/null
+++ b/workhorse/internal/git/archive_test.go
@@ -0,0 +1,87 @@
+package git
+
+import (
+ "io/ioutil"
+ "net/http/httptest"
+ "testing"
+
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/testhelper"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestParseBasename(t *testing.T) {
+ for _, testCase := range []struct {
+ in string
+ out gitalypb.GetArchiveRequest_Format
+ }{
+ {"archive", gitalypb.GetArchiveRequest_TAR_GZ},
+ {"master.tar.gz", gitalypb.GetArchiveRequest_TAR_GZ},
+ {"foo-master.tgz", gitalypb.GetArchiveRequest_TAR_GZ},
+ {"foo-v1.2.1.gz", gitalypb.GetArchiveRequest_TAR_GZ},
+ {"foo.tar.bz2", gitalypb.GetArchiveRequest_TAR_BZ2},
+ {"archive.tbz", gitalypb.GetArchiveRequest_TAR_BZ2},
+ {"archive.tbz2", gitalypb.GetArchiveRequest_TAR_BZ2},
+ {"archive.tb2", gitalypb.GetArchiveRequest_TAR_BZ2},
+ {"archive.bz2", gitalypb.GetArchiveRequest_TAR_BZ2},
+ } {
+ basename := testCase.in
+ out, ok := parseBasename(basename)
+ if !ok {
+ t.Fatalf("parseBasename did not recognize %q", basename)
+ }
+
+ if out != testCase.out {
+ t.Fatalf("expected %q, got %q", testCase.out, out)
+ }
+ }
+}
+
+func TestFinalizeArchive(t *testing.T) {
+ tempFile, err := ioutil.TempFile("", "gitlab-workhorse-test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer tempFile.Close()
+
+ // Deliberately cause an EEXIST error: we know tempFile.Name() already exists
+ err = finalizeCachedArchive(tempFile, tempFile.Name())
+ if err != nil {
+ t.Fatalf("expected nil from finalizeCachedArchive, received %v", err)
+ }
+}
+
+func TestSetArchiveHeaders(t *testing.T) {
+ for _, testCase := range []struct {
+ in gitalypb.GetArchiveRequest_Format
+ out string
+ }{
+ {gitalypb.GetArchiveRequest_ZIP, "application/zip"},
+ {gitalypb.GetArchiveRequest_TAR, "application/octet-stream"},
+ {gitalypb.GetArchiveRequest_TAR_GZ, "application/octet-stream"},
+ {gitalypb.GetArchiveRequest_TAR_BZ2, "application/octet-stream"},
+ } {
+ w := httptest.NewRecorder()
+
+ // These should be replaced, not appended to
+ w.Header().Set("Content-Type", "test")
+ w.Header().Set("Content-Length", "test")
+ w.Header().Set("Content-Disposition", "test")
+
+ // This should be deleted
+ w.Header().Set("Set-Cookie", "test")
+
+ // This should be preserved
+ w.Header().Set("Cache-Control", "public, max-age=3600")
+
+ setArchiveHeaders(w, testCase.in, "filename")
+
+ testhelper.RequireResponseHeader(t, w, "Content-Type", testCase.out)
+ testhelper.RequireResponseHeader(t, w, "Content-Length")
+ testhelper.RequireResponseHeader(t, w, "Content-Disposition", `attachment; filename="filename"`)
+ testhelper.RequireResponseHeader(t, w, "Cache-Control", "public, max-age=3600")
+ require.Empty(t, w.Header().Get("Set-Cookie"), "remove Set-Cookie")
+ }
+}
diff --git a/workhorse/internal/git/blob.go b/workhorse/internal/git/blob.go
new file mode 100644
index 00000000000..472f5d0bc96
--- /dev/null
+++ b/workhorse/internal/git/blob.go
@@ -0,0 +1,47 @@
+package git
+
+import (
+ "fmt"
+ "net/http"
+
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+)
+
+type blob struct{ senddata.Prefix }
+type blobParams struct {
+ GitalyServer gitaly.Server
+ GetBlobRequest gitalypb.GetBlobRequest
+}
+
+var SendBlob = &blob{"git-blob:"}
+
+func (b *blob) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params blobParams
+ if err := b.Unpack(&params, sendData); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendBlob: unpack sendData: %v", err))
+ return
+ }
+
+ ctx, blobClient, err := gitaly.NewBlobClient(r.Context(), params.GitalyServer)
+ if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("blob.GetBlob: %v", err))
+ return
+ }
+
+ setBlobHeaders(w)
+ if err := blobClient.SendBlob(ctx, w, &params.GetBlobRequest); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("blob.GetBlob: %v", err))
+ return
+ }
+}
+
+func setBlobHeaders(w http.ResponseWriter) {
+ // Caching proxies usually don't cache responses with Set-Cookie header
+ // present because it implies user-specific data, which is not the case
+ // for blobs.
+ w.Header().Del("Set-Cookie")
+}
diff --git a/workhorse/internal/git/blob_test.go b/workhorse/internal/git/blob_test.go
new file mode 100644
index 00000000000..ec28c2adb2f
--- /dev/null
+++ b/workhorse/internal/git/blob_test.go
@@ -0,0 +1,17 @@
+package git
+
+import (
+ "net/http/httptest"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestSetBlobHeaders(t *testing.T) {
+ w := httptest.NewRecorder()
+ w.Header().Set("Set-Cookie", "gitlab_cookie=123456")
+
+ setBlobHeaders(w)
+
+ require.Empty(t, w.Header().Get("Set-Cookie"), "remove Set-Cookie")
+}
diff --git a/workhorse/internal/git/diff.go b/workhorse/internal/git/diff.go
new file mode 100644
index 00000000000..b1a1c17a650
--- /dev/null
+++ b/workhorse/internal/git/diff.go
@@ -0,0 +1,48 @@
+package git
+
+import (
+ "fmt"
+ "net/http"
+
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+)
+
+type diff struct{ senddata.Prefix }
+type diffParams struct {
+ GitalyServer gitaly.Server
+ RawDiffRequest string
+}
+
+var SendDiff = &diff{"git-diff:"}
+
+func (d *diff) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params diffParams
+ if err := d.Unpack(&params, sendData); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendDiff: unpack sendData: %v", err))
+ return
+ }
+
+ request := &gitalypb.RawDiffRequest{}
+ if err := gitaly.UnmarshalJSON(params.RawDiffRequest, request); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("diff.RawDiff: %v", err))
+ return
+ }
+
+ ctx, diffClient, err := gitaly.NewDiffClient(r.Context(), params.GitalyServer)
+ if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("diff.RawDiff: %v", err))
+ return
+ }
+
+ if err := diffClient.SendRawDiff(ctx, w, request); err != nil {
+ helper.LogError(
+ r,
+ &copyError{fmt.Errorf("diff.RawDiff: request=%v, err=%v", request, err)},
+ )
+ return
+ }
+}
diff --git a/workhorse/internal/git/error.go b/workhorse/internal/git/error.go
new file mode 100644
index 00000000000..2b7cad6bb64
--- /dev/null
+++ b/workhorse/internal/git/error.go
@@ -0,0 +1,4 @@
+package git
+
+// For cosmetic purposes in Sentry
+type copyError struct{ error }
diff --git a/workhorse/internal/git/format-patch.go b/workhorse/internal/git/format-patch.go
new file mode 100644
index 00000000000..db96029b07e
--- /dev/null
+++ b/workhorse/internal/git/format-patch.go
@@ -0,0 +1,48 @@
+package git
+
+import (
+ "fmt"
+ "net/http"
+
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+)
+
+type patch struct{ senddata.Prefix }
+type patchParams struct {
+ GitalyServer gitaly.Server
+ RawPatchRequest string
+}
+
+var SendPatch = &patch{"git-format-patch:"}
+
+func (p *patch) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params patchParams
+ if err := p.Unpack(&params, sendData); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendPatch: unpack sendData: %v", err))
+ return
+ }
+
+ request := &gitalypb.RawPatchRequest{}
+ if err := gitaly.UnmarshalJSON(params.RawPatchRequest, request); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("diff.RawPatch: %v", err))
+ return
+ }
+
+ ctx, diffClient, err := gitaly.NewDiffClient(r.Context(), params.GitalyServer)
+ if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("diff.RawPatch: %v", err))
+ return
+ }
+
+ if err := diffClient.SendRawPatch(ctx, w, request); err != nil {
+ helper.LogError(
+ r,
+ &copyError{fmt.Errorf("diff.RawPatch: request=%v, err=%v", request, err)},
+ )
+ return
+ }
+}
diff --git a/workhorse/internal/git/git-http.go b/workhorse/internal/git/git-http.go
new file mode 100644
index 00000000000..5df20a68bb7
--- /dev/null
+++ b/workhorse/internal/git/git-http.go
@@ -0,0 +1,100 @@
+/*
+In this file we handle the Git 'smart HTTP' protocol
+*/
+
+package git
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "path/filepath"
+ "sync"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+)
+
+const (
+ // We have to use a negative transfer.hideRefs since this is the only way
+ // to undo an already set parameter: https://www.spinics.net/lists/git/msg256772.html
+ GitConfigShowAllRefs = "transfer.hideRefs=!refs"
+)
+
+func ReceivePack(a *api.API) http.Handler {
+ return postRPCHandler(a, "handleReceivePack", handleReceivePack)
+}
+
+func UploadPack(a *api.API) http.Handler {
+ return postRPCHandler(a, "handleUploadPack", handleUploadPack)
+}
+
+func gitConfigOptions(a *api.Response) []string {
+ var out []string
+
+ if a.ShowAllRefs {
+ out = append(out, GitConfigShowAllRefs)
+ }
+
+ return out
+}
+
+func postRPCHandler(a *api.API, name string, handler func(*HttpResponseWriter, *http.Request, *api.Response) error) http.Handler {
+ return repoPreAuthorizeHandler(a, func(rw http.ResponseWriter, r *http.Request, ar *api.Response) {
+ cr := &countReadCloser{ReadCloser: r.Body}
+ r.Body = cr
+
+ w := NewHttpResponseWriter(rw)
+ defer func() {
+ w.Log(r, cr.Count())
+ }()
+
+ if err := handler(w, r, ar); err != nil {
+ // If the handler already wrote a response this WriteHeader call is a
+ // no-op. It never reaches net/http because GitHttpResponseWriter calls
+ // WriteHeader on its underlying ResponseWriter at most once.
+ w.WriteHeader(500)
+ helper.LogError(r, fmt.Errorf("%s: %v", name, err))
+ }
+ })
+}
+
+func repoPreAuthorizeHandler(myAPI *api.API, handleFunc api.HandleFunc) http.Handler {
+ return myAPI.PreAuthorizeHandler(func(w http.ResponseWriter, r *http.Request, a *api.Response) {
+ handleFunc(w, r, a)
+ }, "")
+}
+
+func writePostRPCHeader(w http.ResponseWriter, action string) {
+ w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-result", action))
+ w.Header().Set("Cache-Control", "no-cache")
+}
+
+func getService(r *http.Request) string {
+ if r.Method == "GET" {
+ return r.URL.Query().Get("service")
+ }
+ return filepath.Base(r.URL.Path)
+}
+
+type countReadCloser struct {
+ n int64
+ io.ReadCloser
+ sync.Mutex
+}
+
+func (c *countReadCloser) Read(p []byte) (n int, err error) {
+ n, err = c.ReadCloser.Read(p)
+
+ c.Lock()
+ defer c.Unlock()
+ c.n += int64(n)
+
+ return n, err
+}
+
+func (c *countReadCloser) Count() int64 {
+ c.Lock()
+ defer c.Unlock()
+ return c.n
+}
diff --git a/workhorse/internal/git/info-refs.go b/workhorse/internal/git/info-refs.go
new file mode 100644
index 00000000000..e5491a7b733
--- /dev/null
+++ b/workhorse/internal/git/info-refs.go
@@ -0,0 +1,76 @@
+package git
+
+import (
+ "compress/gzip"
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+
+ "github.com/golang/gddo/httputil"
+
+ "gitlab.com/gitlab-org/labkit/log"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+)
+
+func GetInfoRefsHandler(a *api.API) http.Handler {
+ return repoPreAuthorizeHandler(a, handleGetInfoRefs)
+}
+
+func handleGetInfoRefs(rw http.ResponseWriter, r *http.Request, a *api.Response) {
+ responseWriter := NewHttpResponseWriter(rw)
+ // Log 0 bytes in because we ignore the request body (and there usually is none anyway).
+ defer responseWriter.Log(r, 0)
+
+ rpc := getService(r)
+ if !(rpc == "git-upload-pack" || rpc == "git-receive-pack") {
+ // The 'dumb' Git HTTP protocol is not supported
+ http.Error(responseWriter, "Not Found", 404)
+ return
+ }
+
+ responseWriter.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
+ responseWriter.Header().Set("Cache-Control", "no-cache")
+
+ gitProtocol := r.Header.Get("Git-Protocol")
+
+ offers := []string{"gzip", "identity"}
+ encoding := httputil.NegotiateContentEncoding(r, offers)
+
+ if err := handleGetInfoRefsWithGitaly(r.Context(), responseWriter, a, rpc, gitProtocol, encoding); err != nil {
+ helper.Fail500(responseWriter, r, fmt.Errorf("handleGetInfoRefs: %v", err))
+ }
+}
+
+func handleGetInfoRefsWithGitaly(ctx context.Context, responseWriter *HttpResponseWriter, a *api.Response, rpc, gitProtocol, encoding string) error {
+ ctx, smarthttp, err := gitaly.NewSmartHTTPClient(ctx, a.GitalyServer)
+ if err != nil {
+ return fmt.Errorf("GetInfoRefsHandler: %v", err)
+ }
+
+ infoRefsResponseReader, err := smarthttp.InfoRefsResponseReader(ctx, &a.Repository, rpc, gitConfigOptions(a), gitProtocol)
+ if err != nil {
+ return fmt.Errorf("GetInfoRefsHandler: %v", err)
+ }
+
+ var w io.Writer
+
+ if encoding == "gzip" {
+ gzWriter := gzip.NewWriter(responseWriter)
+ w = gzWriter
+ defer gzWriter.Close()
+
+ responseWriter.Header().Set("Content-Encoding", "gzip")
+ } else {
+ w = responseWriter
+ }
+
+ if _, err = io.Copy(w, infoRefsResponseReader); err != nil {
+ log.WithError(err).Error("GetInfoRefsHandler: error copying gitaly response")
+ }
+
+ return nil
+}
diff --git a/workhorse/internal/git/pktline.go b/workhorse/internal/git/pktline.go
new file mode 100644
index 00000000000..e970f60182d
--- /dev/null
+++ b/workhorse/internal/git/pktline.go
@@ -0,0 +1,59 @@
+package git
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "io"
+ "strconv"
+)
+
+func scanDeepen(body io.Reader) bool {
+ scanner := bufio.NewScanner(body)
+ scanner.Split(pktLineSplitter)
+ for scanner.Scan() {
+ if bytes.HasPrefix(scanner.Bytes(), []byte("deepen")) && scanner.Err() == nil {
+ return true
+ }
+ }
+
+ return false
+}
+
+func pktLineSplitter(data []byte, atEOF bool) (advance int, token []byte, err error) {
+ if len(data) < 4 {
+ if atEOF && len(data) > 0 {
+ return 0, nil, fmt.Errorf("pktLineSplitter: incomplete length prefix on %q", data)
+ }
+ return 0, nil, nil // want more data
+ }
+
+ if bytes.HasPrefix(data, []byte("0000")) {
+ // special case: "0000" terminator packet: return empty token
+ return 4, data[:0], nil
+ }
+
+ // We have at least 4 bytes available so we can decode the 4-hex digit
+ // length prefix of the packet line.
+ pktLength64, err := strconv.ParseInt(string(data[:4]), 16, 0)
+ if err != nil {
+ return 0, nil, fmt.Errorf("pktLineSplitter: decode length: %v", err)
+ }
+
+ // Cast is safe because we requested an int-size number from strconv.ParseInt
+ pktLength := int(pktLength64)
+
+ if pktLength < 0 {
+ return 0, nil, fmt.Errorf("pktLineSplitter: invalid length: %d", pktLength)
+ }
+
+ if len(data) < pktLength {
+ if atEOF {
+ return 0, nil, fmt.Errorf("pktLineSplitter: less than %d bytes in input %q", pktLength, data)
+ }
+ return 0, nil, nil // want more data
+ }
+
+ // return "pkt" token without length prefix
+ return pktLength, data[4:pktLength], nil
+}
diff --git a/workhorse/internal/git/pktline_test.go b/workhorse/internal/git/pktline_test.go
new file mode 100644
index 00000000000..d4be8634538
--- /dev/null
+++ b/workhorse/internal/git/pktline_test.go
@@ -0,0 +1,39 @@
+package git
+
+import (
+ "bytes"
+ "testing"
+)
+
+func TestSuccessfulScanDeepen(t *testing.T) {
+ examples := []struct {
+ input string
+ output bool
+ }{
+ {"000dsomething000cdeepen 10000", true},
+ {"000dsomething0000000cdeepen 1", true},
+ {"000dsomething0000", false},
+ }
+
+ for _, example := range examples {
+ hasDeepen := scanDeepen(bytes.NewReader([]byte(example.input)))
+
+ if hasDeepen != example.output {
+ t.Fatalf("scanDeepen %q: expected %v, got %v", example.input, example.output, hasDeepen)
+ }
+ }
+}
+
+func TestFailedScanDeepen(t *testing.T) {
+ examples := []string{
+ "invalid data",
+ "deepen",
+ "000cdeepen",
+ }
+
+ for _, example := range examples {
+ if scanDeepen(bytes.NewReader([]byte(example))) {
+ t.Fatalf("scanDeepen %q: expected result to be false, got true", example)
+ }
+ }
+}
diff --git a/workhorse/internal/git/receive-pack.go b/workhorse/internal/git/receive-pack.go
new file mode 100644
index 00000000000..e72d8be5174
--- /dev/null
+++ b/workhorse/internal/git/receive-pack.go
@@ -0,0 +1,33 @@
+package git
+
+import (
+ "fmt"
+ "net/http"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+)
+
+// Will not return a non-nil error after the response body has been
+// written to.
+func handleReceivePack(w *HttpResponseWriter, r *http.Request, a *api.Response) error {
+ action := getService(r)
+ writePostRPCHeader(w, action)
+
+ cr, cw := helper.NewWriteAfterReader(r.Body, w)
+ defer cw.Flush()
+
+ gitProtocol := r.Header.Get("Git-Protocol")
+
+ ctx, smarthttp, err := gitaly.NewSmartHTTPClient(r.Context(), a.GitalyServer)
+ if err != nil {
+ return fmt.Errorf("smarthttp.ReceivePack: %v", err)
+ }
+
+ if err := smarthttp.ReceivePack(ctx, &a.Repository, a.GL_ID, a.GL_USERNAME, a.GL_REPOSITORY, a.GitConfigOptions, cr, cw, gitProtocol); err != nil {
+ return fmt.Errorf("smarthttp.ReceivePack: %v", err)
+ }
+
+ return nil
+}
diff --git a/workhorse/internal/git/responsewriter.go b/workhorse/internal/git/responsewriter.go
new file mode 100644
index 00000000000..c4d4ac252d4
--- /dev/null
+++ b/workhorse/internal/git/responsewriter.go
@@ -0,0 +1,75 @@
+package git
+
+import (
+ "net/http"
+ "strconv"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+)
+
+const (
+ directionIn = "in"
+ directionOut = "out"
+)
+
+var (
+ gitHTTPSessionsActive = promauto.NewGauge(prometheus.GaugeOpts{
+ Name: "gitlab_workhorse_git_http_sessions_active",
+ Help: "Number of Git HTTP request-response cycles currently being handled by gitlab-workhorse.",
+ })
+
+ gitHTTPRequests = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "gitlab_workhorse_git_http_requests",
+ Help: "How many Git HTTP requests have been processed by gitlab-workhorse, partitioned by request type and agent.",
+ },
+ []string{"method", "code", "service", "agent"},
+ )
+
+ gitHTTPBytes = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "gitlab_workhorse_git_http_bytes",
+ Help: "How many Git HTTP bytes have been sent by gitlab-workhorse, partitioned by request type, agent and direction.",
+ },
+ []string{"method", "code", "service", "agent", "direction"},
+ )
+)
+
+type HttpResponseWriter struct {
+ helper.CountingResponseWriter
+}
+
+func NewHttpResponseWriter(rw http.ResponseWriter) *HttpResponseWriter {
+ gitHTTPSessionsActive.Inc()
+ return &HttpResponseWriter{
+ CountingResponseWriter: helper.NewCountingResponseWriter(rw),
+ }
+}
+
+func (w *HttpResponseWriter) Log(r *http.Request, writtenIn int64) {
+ service := getService(r)
+ agent := getRequestAgent(r)
+
+ gitHTTPSessionsActive.Dec()
+ gitHTTPRequests.WithLabelValues(r.Method, strconv.Itoa(w.Status()), service, agent).Inc()
+ gitHTTPBytes.WithLabelValues(r.Method, strconv.Itoa(w.Status()), service, agent, directionIn).
+ Add(float64(writtenIn))
+ gitHTTPBytes.WithLabelValues(r.Method, strconv.Itoa(w.Status()), service, agent, directionOut).
+ Add(float64(w.Count()))
+}
+
+func getRequestAgent(r *http.Request) string {
+ u, _, ok := r.BasicAuth()
+ if !ok {
+ return "anonymous"
+ }
+
+ if u == "gitlab-ci-token" {
+ return "gitlab-ci"
+ }
+
+ return "logged"
+}
diff --git a/workhorse/internal/git/snapshot.go b/workhorse/internal/git/snapshot.go
new file mode 100644
index 00000000000..eb38becbd06
--- /dev/null
+++ b/workhorse/internal/git/snapshot.go
@@ -0,0 +1,64 @@
+package git
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
+)
+
+type snapshot struct {
+ senddata.Prefix
+}
+
+type snapshotParams struct {
+ GitalyServer gitaly.Server
+ GetSnapshotRequest string
+}
+
+var (
+ SendSnapshot = &snapshot{"git-snapshot:"}
+)
+
+func (s *snapshot) Inject(w http.ResponseWriter, r *http.Request, sendData string) {
+ var params snapshotParams
+
+ if err := s.Unpack(&params, sendData); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendSnapshot: unpack sendData: %v", err))
+ return
+ }
+
+ request := &gitalypb.GetSnapshotRequest{}
+ if err := gitaly.UnmarshalJSON(params.GetSnapshotRequest, request); err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendSnapshot: unmarshal GetSnapshotRequest: %v", err))
+ return
+ }
+
+ ctx, c, err := gitaly.NewRepositoryClient(r.Context(), params.GitalyServer)
+ if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendSnapshot: gitaly.NewRepositoryClient: %v", err))
+ return
+ }
+
+ reader, err := c.SnapshotReader(ctx, request)
+ if err != nil {
+ helper.Fail500(w, r, fmt.Errorf("SendSnapshot: client.SnapshotReader: %v", err))
+ return
+ }
+
+ w.Header().Del("Content-Length")
+ w.Header().Set("Content-Disposition", `attachment; filename="snapshot.tar"`)
+ w.Header().Set("Content-Type", "application/x-tar")
+ w.Header().Set("Content-Transfer-Encoding", "binary")
+ w.Header().Set("Cache-Control", "private")
+ w.WriteHeader(http.StatusOK) // Errors aren't detectable beyond this point
+
+ if _, err := io.Copy(w, reader); err != nil {
+ helper.LogError(r, fmt.Errorf("SendSnapshot: copy gitaly output: %v", err))
+ }
+}
diff --git a/workhorse/internal/git/upload-pack.go b/workhorse/internal/git/upload-pack.go
new file mode 100644
index 00000000000..a3dbf2f2e02
--- /dev/null
+++ b/workhorse/internal/git/upload-pack.go
@@ -0,0 +1,57 @@
+package git
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "time"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
+)
+
+var (
+ uploadPackTimeout = 10 * time.Minute
+)
+
+// Will not return a non-nil error after the response body has been
+// written to.
+func handleUploadPack(w *HttpResponseWriter, r *http.Request, a *api.Response) error {
+ ctx := r.Context()
+
+ // Prevent the client from holding the connection open indefinitely. A
+ // transfer rate of 17KiB/sec is sufficient to send 10MiB of data in
+ // ten minutes, which seems adequate. Most requests will be much smaller.
+ // This mitigates a use-after-check issue.
+ //
+ // We can't reliably interrupt the read from a http handler, but we can
+ // ensure the request will (eventually) fail: https://github.com/golang/go/issues/16100
+ readerCtx, cancel := context.WithTimeout(ctx, uploadPackTimeout)
+ defer cancel()
+
+ limited := helper.NewContextReader(readerCtx, r.Body)
+ cr, cw := helper.NewWriteAfterReader(limited, w)
+ defer cw.Flush()
+
+ action := getService(r)
+ writePostRPCHeader(w, action)
+
+ gitProtocol := r.Header.Get("Git-Protocol")
+
+ return handleUploadPackWithGitaly(ctx, a, cr, cw, gitProtocol)
+}
+
+func handleUploadPackWithGitaly(ctx context.Context, a *api.Response, clientRequest io.Reader, clientResponse io.Writer, gitProtocol string) error {
+ ctx, smarthttp, err := gitaly.NewSmartHTTPClient(ctx, a.GitalyServer)
+ if err != nil {
+ return fmt.Errorf("smarthttp.UploadPack: %v", err)
+ }
+
+ if err := smarthttp.UploadPack(ctx, &a.Repository, clientRequest, clientResponse, gitConfigOptions(a), gitProtocol); err != nil {
+ return fmt.Errorf("smarthttp.UploadPack: %v", err)
+ }
+
+ return nil
+}
diff --git a/workhorse/internal/git/upload-pack_test.go b/workhorse/internal/git/upload-pack_test.go
new file mode 100644
index 00000000000..c198939d5df
--- /dev/null
+++ b/workhorse/internal/git/upload-pack_test.go
@@ -0,0 +1,85 @@
+package git
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc"
+
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
+ "gitlab.com/gitlab-org/gitlab-workhorse/internal/gitaly"
+)
+
+var (
+ originalUploadPackTimeout = uploadPackTimeout
+)
+
+type fakeReader struct {
+ n int
+ err error
+}
+
+func (f *fakeReader) Read(b []byte) (int, error) {
+ return f.n, f.err
+}
+
+type smartHTTPServiceServer struct {
+ gitalypb.UnimplementedSmartHTTPServiceServer
+ PostUploadPackFunc func(gitalypb.SmartHTTPService_PostUploadPackServer) error
+}
+
+func (srv *smartHTTPServiceServer) PostUploadPack(s gitalypb.SmartHTTPService_PostUploadPackServer) error {
+ return srv.PostUploadPackFunc(s)
+}
+
+func TestUploadPackTimesOut(t *testing.T) {
+ uploadPackTimeout = time.Millisecond
+ defer func() { uploadPackTimeout = originalUploadPackTimeout }()
+
+ addr, cleanUp := startSmartHTTPServer(t, &smartHTTPServiceServer{
+ PostUploadPackFunc: func(stream gitalypb.SmartHTTPService_PostUploadPackServer) error {
+ _, err := stream.Recv() // trigger a read on the client request body
+ require.NoError(t, err)
+ return nil
+ },
+ })
+ defer cleanUp()
+
+ body := &fakeReader{n: 0, err: nil}
+
+ w := httptest.NewRecorder()
+ r := httptest.NewRequest("GET", "/", body)
+ a := &api.Response{GitalyServer: gitaly.Server{Address: addr}}
+
+ err := handleUploadPack(NewHttpResponseWriter(w), r, a)
+ require.EqualError(t, err, "smarthttp.UploadPack: busyReader: context deadline exceeded")
+}
+
+func startSmartHTTPServer(t testing.TB, s gitalypb.SmartHTTPServiceServer) (string, func()) {
+ tmp, err := ioutil.TempDir("", "")
+ require.NoError(t, err)
+
+ socket := filepath.Join(tmp, "gitaly.sock")
+ ln, err := net.Listen("unix", socket)
+ require.NoError(t, err)
+
+ srv := grpc.NewServer()
+ gitalypb.RegisterSmartHTTPServiceServer(srv, s)
+ go func() {
+ require.NoError(t, srv.Serve(ln))
+ }()
+
+ return fmt.Sprintf("%s://%s", ln.Addr().Network(), ln.Addr().String()), func() {
+ srv.GracefulStop()
+ require.NoError(t, os.RemoveAll(tmp), "error removing temp dir %q", tmp)
+ }
+}