Add working example using bundle files

4 files changed, 269 insertions, 82 deletions

diff --git a/glue-packs.go b/glue-packs.go
index b9d7040c3..5d6f896f3 100644
--- a/glue-packs.go
+++ b/glue-packs.go
@@ -1,11 +1,9 @@
 package main
 
 import (
-	"bytes"
 	"crypto/sha1"
 	"encoding/binary"
 	"fmt"
-	"hash"
 	"io"
 	"log"
 	"os"
@@ -72,83 +70,3 @@ func _main() error {
 
 	return nil
 }
-
-const (
-	sumSize        = sha1.Size
-	packBufferSize = 4096
-)
-
-type packReader struct {
-	buf        [packBufferSize]byte
-	avail      []byte
-	reader     io.Reader
-	readErr    error
-	sum        hash.Hash
-	numObjects uint32
-}
-
-const (
-	packMagic      = "PACK\x00\x00\x00\x02"
-	packHeaderSize = 12
-)
-
-// NewPackReader blocks until it has read the packfile header from r.
-func NewPackReader(r io.Reader) (*packReader, error) {
-	pr := &packReader{
-		reader: r,
-		sum:    sha1.New(),
-	}
-
-	header := make([]byte, packHeaderSize)
-	if _, err := io.ReadFull(pr.reader, header); err != nil {
-		return nil, err
-	}
-
-	if magic := string(header[:len(packMagic)]); magic != packMagic {
-		return nil, fmt.Errorf("bad pack header: %q", magic)
-	}
-
-	pr.numObjects = binary.BigEndian.Uint32(header[len(packMagic):])
-
-	if _, err := pr.sum.Write(header); err != nil {
-		return nil, err
-	}
-
-	return pr, nil
-}
-
-func (pr *packReader) NumObjects() uint32 { return pr.numObjects }
-
-func (pr *packReader) numBytesAvailable() int { return len(pr.avail) - sumSize }
-
-func (pr *packReader) Read(p []byte) (int, error) {
-	if pr.numBytesAvailable() <= 0 && pr.readErr == nil {
-		copy(pr.buf[:], pr.avail)
-
-		var nRead int
-		nRead, pr.readErr = pr.reader.Read(pr.buf[len(pr.avail):])
-		if pr.readErr != nil && pr.readErr != io.EOF {
-			return 0, pr.readErr
-		}
-
-		pr.avail = pr.buf[:len(pr.avail)+nRead]
-
-		if n := pr.numBytesAvailable(); n > 0 {
-			if _, err := pr.sum.Write(pr.avail[:n]); err != nil {
-				return 0, err
-			}
-		}
-	}
-
-	if pr.numBytesAvailable() <= 0 {
-		if pr.readErr == io.EOF && !bytes.Equal(pr.sum.Sum(nil), pr.avail) {
-			return 0, fmt.Errorf("packfile checksum mismatch")
-		}
-
-		return 0, pr.readErr
-	}
-
-	nYielded := copy(p, pr.avail[:pr.numBytesAvailable()])
-	pr.avail = pr.avail[nYielded:]
-	return nYielded, nil
-}
diff --git a/pack-objects-clone-bundle/README.md b/pack-objects-clone-bundle/README.md
new file mode 100644
index 000000000..1d6d486fc
--- /dev/null
+++ b/pack-objects-clone-bundle/README.md
@@ -0,0 +1,8 @@
+# Demonstration of concatenating pack files to speed up Git clone
+
+This directory contains code for an executable that can speed up a Git clone when installed on a server. The only type of clone we can speed up is a full clone.
+
+- compile the executable and install at some chosen path, e.g. `go build -o /tmp/pack-objects-clone-bundle`
+- `git  config --global uploadpack.packObjectsHook /tmp/pack-objects-bundle` (has to be global for some reason)
+- in the bare repo you want to speed up, run `git bundle create clone.bundle --branches --tags`
+- now do a full clone from that repo. If it is a local clone, use `git clone --no-local` to see the effect
diff --git a/pack-objects-clone-bundle/pack-objects-bundle.go b/pack-objects-clone-bundle/pack-objects-bundle.go
new file mode 100644
index 000000000..aafc8ece9
--- /dev/null
+++ b/pack-objects-clone-bundle/pack-objects-bundle.go
@@ -0,0 +1,171 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/sha1"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		log.Fatal("not enough argument to pack-objects hook")
+	}
+
+	if err := _main(os.Args[1:]); err != nil {
+		log.Fatal(err)
+	}
+}
+
+var shaRegex = regexp.MustCompile(`\A[0-9a-f]{40}\z`)
+
+func _main(packObjects []string) error {
+	request := &bytes.Buffer{}
+	scanner := bufio.NewScanner(io.TeeReader(os.Stdin, request))
+	seenNot := false
+	isClone := true
+	for scanner.Scan() {
+		if !seenNot && scanner.Text() == "--not" {
+			seenNot = true
+			continue
+		}
+
+		if seenNot && scanner.Text() != "" {
+			isClone = false
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	if !isClone {
+		return fallback(packObjects, request)
+	}
+
+	bundleFile, err := os.Open("clone.bundle")
+	if err != nil {
+		return fallback(packObjects, request)
+	}
+	defer bundleFile.Close()
+
+	bundle := bufio.NewReader(bundleFile)
+	bundleHeader, err := readLine(bundle)
+	if err != nil {
+		return err
+	}
+	if bundleHeader != "# v2 git bundle" {
+		return fmt.Errorf("unexpected bundle header: %q", bundleHeader)
+	}
+
+	request = bytes.NewBuffer(bytes.TrimSpace(request.Bytes()))
+	if _, err := request.WriteString("\n"); err != nil {
+		return err
+	}
+
+	for {
+		refLine, err := readLine(bundle)
+		if err != nil {
+			return err
+		}
+
+		if refLine == "" {
+			break
+		}
+
+		split := strings.SplitN(refLine, " ", 2)
+		if len(split) != 2 {
+			return fmt.Errorf("invalid ref line: %q", refLine)
+		}
+		id := split[0]
+		if !shaRegex.MatchString(id) {
+			return fmt.Errorf("invalid object ID: %q", id)
+		}
+
+		if _, err := fmt.Fprintln(request, id); err != nil {
+			return err
+		}
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, packObjects[0], packObjects[1:]...)
+	cmd.Stdin = request
+	cmd.Stderr = os.Stderr
+
+	packObjectsOut, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+
+	packObjectsReader, err := NewPackReader(packObjectsOut)
+	if err != nil {
+		return err
+	}
+
+	bundleReader, err := NewPackReader(bundle)
+	if err != nil {
+		return err
+	}
+
+	summer := sha1.New()
+	stdout := io.MultiWriter(os.Stdout, summer)
+
+	if _, err := fmt.Fprint(stdout, packMagic); err != nil {
+		return err
+	}
+
+	size := make([]byte, 4)
+	binary.BigEndian.PutUint32(size, packObjectsReader.NumObjects()+bundleReader.NumObjects()) // TODO check for overflow
+	if _, err := stdout.Write(size); err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(stdout, packObjectsReader); err != nil {
+		return err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(stdout, bundleReader); err != nil {
+		return err
+	}
+
+	if _, err := stdout.Write(summer.Sum(nil)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func fallback(packObjects []string, request io.Reader) error {
+	cmd := exec.Command(packObjects[0], packObjects[1:]...)
+	cmd.Stdin = request
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func readLine(r *bufio.Reader) (string, error) {
+	line, err := r.ReadBytes('\n')
+	if err != nil {
+		return "", err
+	}
+
+	return string(line[:len(line)-1]), nil
+}
diff --git a/pack-objects-clone-bundle/packreader.go b/pack-objects-clone-bundle/packreader.go
new file mode 100644
index 000000000..b6154979a
--- /dev/null
+++ b/pack-objects-clone-bundle/packreader.go
@@ -0,0 +1,90 @@
+package main
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/binary"
+	"fmt"
+	"hash"
+	"io"
+)
+
+const (
+	sumSize        = sha1.Size
+	packBufferSize = 4096
+)
+
+type packReader struct {
+	buf        [packBufferSize]byte
+	avail      []byte
+	reader     io.Reader
+	readErr    error
+	sum        hash.Hash
+	numObjects uint32
+}
+
+const (
+	packMagic      = "PACK\x00\x00\x00\x02"
+	packHeaderSize = 12
+)
+
+// NewPackReader blocks until it has read the packfile header from r.
+func NewPackReader(r io.Reader) (*packReader, error) {
+	pr := &packReader{
+		reader: r,
+		sum:    sha1.New(),
+	}
+
+	header := make([]byte, packHeaderSize)
+	if _, err := io.ReadFull(pr.reader, header); err != nil {
+		return nil, err
+	}
+
+	if magic := string(header[:len(packMagic)]); magic != packMagic {
+		return nil, fmt.Errorf("bad pack header: %q", magic)
+	}
+
+	pr.numObjects = binary.BigEndian.Uint32(header[len(packMagic):])
+
+	if _, err := pr.sum.Write(header); err != nil {
+		return nil, err
+	}
+
+	return pr, nil
+}
+
+func (pr *packReader) NumObjects() uint32 { return pr.numObjects }
+
+func (pr *packReader) numBytesAvailable() int { return len(pr.avail) - sumSize }
+
+func (pr *packReader) Read(p []byte) (int, error) {
+	if pr.numBytesAvailable() <= 0 && pr.readErr == nil {
+		copy(pr.buf[:], pr.avail)
+
+		var nRead int
+		nRead, pr.readErr = pr.reader.Read(pr.buf[len(pr.avail):])
+		if pr.readErr != nil && pr.readErr != io.EOF {
+			return 0, pr.readErr
+		}
+
+		pr.avail = pr.buf[:len(pr.avail)+nRead]
+
+		if n := pr.numBytesAvailable(); n > 0 {
+			if _, err := pr.sum.Write(pr.avail[:n]); err != nil {
+				return 0, err
+			}
+		}
+	}
+
+	if pr.numBytesAvailable() <= 0 {
+		if pr.readErr == io.EOF && !bytes.Equal(pr.sum.Sum(nil), pr.avail) {
+			return 0, fmt.Errorf("packfile checksum mismatch")
+		}
+
+		return 0, pr.readErr
+	}
+
+	nYielded := copy(p, pr.avail[:pr.numBytesAvailable()])
+	pr.avail = pr.avail[nYielded:]
+	return nYielded, nil
+}