1 files changed, 295 insertions, 438 deletions

diff --git a/src/indexer.c b/src/indexer.c
index 6f735e651..2cfbd3a5a 100644
--- a/src/indexer.c
+++ b/src/indexer.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2009-2012 the libgit2 contributors
+ * Copyright (C) the libgit2 contributors. All rights reserved.
  *
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
@@ -17,7 +17,7 @@
 #include "posix.h"
 #include "pack.h"
 #include "filebuf.h"
-#include "sha1.h"
+#include "oidmap.h"
 
 #define UINT31_MAX (0x7FFFFFFF)
 
@@ -28,39 +28,32 @@ struct entry {
 	uint64_t offset_long;
 };
 
-struct git_indexer {
-	struct git_pack_file *pack;
-	size_t nr_objects;
-	git_vector objects;
-	git_filebuf file;
-	unsigned int fanout[256];
-	git_oid hash;
-};
-
 struct git_indexer_stream {
 	unsigned int parsed_header :1,
-		opened_pack;
+		opened_pack :1,
+		have_stream :1,
+		have_delta :1;
 	struct git_pack_file *pack;
 	git_filebuf pack_file;
-	git_filebuf index_file;
 	git_off_t off;
+	git_off_t entry_start;
+	git_packfile_stream stream;
 	size_t nr_objects;
 	git_vector objects;
 	git_vector deltas;
 	unsigned int fanout[256];
+	git_hash_ctx hash_ctx;
 	git_oid hash;
+	git_transfer_progress_callback progress_cb;
+	void *progress_payload;
+	char objbuf[8*1024];
 };
 
 struct delta_info {
 	git_off_t delta_off;
 };
 
-const git_oid *git_indexer_hash(git_indexer *idx)
-{
-	return &idx->hash;
-}
-
-const git_oid *git_indexer_stream_hash(git_indexer_stream *idx)
+const git_oid *git_indexer_stream_hash(const git_indexer_stream *idx)
 {
 	return &idx->hash;
 }
@@ -130,23 +123,21 @@ static int objects_cmp(const void *a, const void *b)
 	return git_oid_cmp(&entrya->oid, &entryb->oid);
 }
 
-static int cache_cmp(const void *a, const void *b)
-{
-	const struct git_pack_entry *ea = a;
-	const struct git_pack_entry *eb = b;
-
-	return git_oid_cmp(&ea->sha1, &eb->sha1);
-}
-
-int git_indexer_stream_new(git_indexer_stream **out, const char *prefix)
+int git_indexer_stream_new(
+		git_indexer_stream **out,
+		const char *prefix,
+		git_transfer_progress_callback progress_cb,
+		void *progress_payload)
 {
 	git_indexer_stream *idx;
 	git_buf path = GIT_BUF_INIT;
-	static const char suff[] = "/objects/pack/pack-received";
+	static const char suff[] = "/pack";
 	int error;
 
 	idx = git__calloc(1, sizeof(git_indexer_stream));
 	GITERR_CHECK_ALLOC(idx);
+	idx->progress_cb = progress_cb;
+	idx->progress_payload = progress_payload;
 
 	error = git_buf_joinpath(&path, prefix, suff);
 	if (error < 0)
@@ -171,69 +162,171 @@ cleanup:
 /* Try to store the delta so we can try to resolve it later */
 static int store_delta(git_indexer_stream *idx)
 {
-	git_otype type;
-	git_mwindow *w = NULL;
-	git_mwindow_file *mwf = &idx->pack->mwf;
-	git_off_t entry_start = idx->off;
 	struct delta_info *delta;
-	size_t entry_size;
-	git_rawobj obj;
-	int error;
 
-	/*
-	 * ref-delta objects can refer to object that we haven't
-	 * found yet, so give it another opportunity
-	 */
-	if (git_packfile_unpack_header(&entry_size, &type, mwf, &w, &idx->off) < 0)
+	delta = git__calloc(1, sizeof(struct delta_info));
+	GITERR_CHECK_ALLOC(delta);
+	delta->delta_off = idx->entry_start;
+
+	if (git_vector_insert(&idx->deltas, delta) < 0)
 		return -1;
 
-	git_mwindow_close(&w);
+	return 0;
+}
 
-	/* If it's not a delta, mark it as failure, we can't do anything with it */
-	if (type != GIT_OBJ_REF_DELTA && type != GIT_OBJ_OFS_DELTA)
-		return -1;
+static void hash_header(git_hash_ctx *ctx, git_off_t len, git_otype type)
+{
+	char buffer[64];
+	size_t hdrlen;
+
+	hdrlen = git_odb__format_object_header(buffer, sizeof(buffer), (size_t)len, type);
+	git_hash_update(ctx, buffer, hdrlen);
+}
+
+static int hash_object_stream(git_indexer_stream *idx, git_packfile_stream *stream)
+{
+	ssize_t read;
+
+	assert(idx && stream);
+
+	do {
+		if ((read = git_packfile_stream_read(stream, idx->objbuf, sizeof(idx->objbuf))) < 0)
+			break;
+
+		git_hash_update(&idx->hash_ctx, idx->objbuf, read);
+	} while (read > 0);
+
+	if (read < 0)
+		return (int)read;
+
+	return 0;
+}
+
+/* In order to create the packfile stream, we need to skip over the delta base description */
+static int advance_delta_offset(git_indexer_stream *idx, git_otype type)
+{
+	git_mwindow *w = NULL;
+
+	assert(type == GIT_OBJ_REF_DELTA || type == GIT_OBJ_OFS_DELTA);
 
 	if (type == GIT_OBJ_REF_DELTA) {
 		idx->off += GIT_OID_RAWSZ;
 	} else {
-		git_off_t base_off;
-
-		base_off = get_delta_base(idx->pack, &w, &idx->off, type, entry_start);
+		git_off_t base_off = get_delta_base(idx->pack, &w, &idx->off, type, idx->entry_start);
 		git_mwindow_close(&w);
 		if (base_off < 0)
 			return (int)base_off;
 	}
 
-	error = packfile_unpack_compressed(&obj, idx->pack, &w, &idx->off, entry_size, type);
-	if (error == GIT_EBUFS) {
-		idx->off = entry_start;
-		return GIT_EBUFS;
-	} else if (error < 0){
-		return -1;
+	return 0;
+}
+
+/* Read from the stream and discard any output */
+static int read_object_stream(git_indexer_stream *idx, git_packfile_stream *stream)
+{
+	ssize_t read;
+
+	assert(stream);
+
+	do {
+		read = git_packfile_stream_read(stream, idx->objbuf, sizeof(idx->objbuf));
+	} while (read > 0);
+
+	if (read < 0)
+		return (int)read;
+
+	return 0;
+}
+
+static int crc_object(uint32_t *crc_out, git_mwindow_file *mwf, git_off_t start, git_off_t size)
+{
+	void *ptr;
+	uint32_t crc;
+	unsigned int left, len;
+	git_mwindow *w = NULL;
+
+	crc = crc32(0L, Z_NULL, 0);
+	while (size) {
+		ptr = git_mwindow_open(mwf, &w, start, (size_t)size, &left);
+		if (ptr == NULL)
+			return -1;
+
+		len = min(left, (unsigned int)size);
+		crc = crc32(crc, ptr, len);
+		size -= len;
+		start += len;
+		git_mwindow_close(&w);
 	}
 
-	delta = git__calloc(1, sizeof(struct delta_info));
-	GITERR_CHECK_ALLOC(delta);
-	delta->delta_off = entry_start;
+	*crc_out = htonl(crc);
+	return 0;
+}
+
+static int store_object(git_indexer_stream *idx)
+{
+	int i, error;
+	khiter_t k;
+	git_oid oid;
+	struct entry *entry;
+	git_off_t entry_size;
+	struct git_pack_entry *pentry;
+	git_hash_ctx *ctx = &idx->hash_ctx;
+	git_off_t entry_start = idx->entry_start;
 
-	git__free(obj.data);
+	entry = git__calloc(1, sizeof(*entry));
+	GITERR_CHECK_ALLOC(entry);
 
-	if (git_vector_insert(&idx->deltas, delta) < 0)
-		return -1;
+	pentry = git__malloc(sizeof(struct git_pack_entry));
+	GITERR_CHECK_ALLOC(pentry);
+
+	git_hash_final(&oid, ctx);
+	entry_size = idx->off - entry_start;
+	if (entry_start > UINT31_MAX) {
+		entry->offset = UINT32_MAX;
+		entry->offset_long = entry_start;
+	} else {
+		entry->offset = (uint32_t)entry_start;
+	}
+
+	git_oid_cpy(&pentry->sha1, &oid);
+	pentry->offset = entry_start;
+
+	k = kh_put(oid, idx->pack->idx_cache, &pentry->sha1, &error);
+	if (!error) {
+		git__free(pentry);
+		goto on_error;
+	}
+
+	kh_value(idx->pack->idx_cache, k) = pentry;
+
+	git_oid_cpy(&entry->oid, &oid);
+
+	if (crc_object(&entry->crc, &idx->pack->mwf, entry_start, entry_size) < 0)
+		goto on_error;
+
+	/* Add the object to the list */
+	if (git_vector_insert(&idx->objects, entry) < 0)
+		goto on_error;
+
+	for (i = oid.id[0]; i < 256; ++i) {
+		idx->fanout[i]++;
+	}
 
 	return 0;
+
+on_error:
+	git__free(entry);
+
+	return -1;
 }
 
 static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t entry_start)
 {
-	int i;
+	int i, error;
+	khiter_t k;
 	git_oid oid;
-	void *packed;
 	size_t entry_size;
-	unsigned int left;
 	struct entry *entry;
-	git_mwindow *w = NULL;
-	git_mwindow_file *mwf = &idx->pack->mwf;
 	struct git_pack_entry *pentry;
 
 	entry = git__calloc(1, sizeof(*entry));
@@ -257,20 +350,21 @@ static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t ent
 
 	git_oid_cpy(&pentry->sha1, &oid);
 	pentry->offset = entry_start;
-	if (git_vector_insert(&idx->pack->cache, pentry) < 0)
+	k = kh_put(oid, idx->pack->idx_cache, &pentry->sha1, &error);
+	if (!error) {
+		git__free(pentry);
 		goto on_error;
+	}
+
+	kh_value(idx->pack->idx_cache, k) = pentry;
 
 	git_oid_cpy(&entry->oid, &oid);
 	entry->crc = crc32(0L, Z_NULL, 0);
 
 	entry_size = (size_t)(idx->off - entry_start);
-	packed = git_mwindow_open(mwf, &w, entry_start, entry_size, &left);
-	if (packed == NULL)
+	if (crc_object(&entry->crc, &idx->pack->mwf, entry_start, entry_size) < 0)
 		goto on_error;
 
-	entry->crc = htonl(crc32(entry->crc, packed, (uInt)entry_size));
-	git_mwindow_close(&w);
-
 	/* Add the object to the list */
 	if (git_vector_insert(&idx->objects, entry) < 0)
 		goto on_error;
@@ -283,20 +377,27 @@ static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t ent
 
 on_error:
 	git__free(entry);
-	git__free(pentry);
 	git__free(obj->data);
 	return -1;
 }
 
-int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats)
+static int do_progress_callback(git_indexer_stream *idx, git_transfer_progress *stats)
 {
-	int error;
+	if (!idx->progress_cb) return 0;
+	return idx->progress_cb(stats, idx->progress_payload);
+}
+
+int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_transfer_progress *stats)
+{
+	int error = -1;
 	struct git_pack_header hdr;
-	size_t processed = stats->processed;
+	size_t processed; 
 	git_mwindow_file *mwf = &idx->pack->mwf;
 
 	assert(idx && data && stats);
 
+	processed = stats->indexed_objects;
+
 	if (git_filebuf_write(&idx->pack_file, data, size) < 0)
 		return -1;
 
@@ -311,11 +412,11 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 		mwf = &idx->pack->mwf;
 		if (git_mwindow_file_register(&idx->pack->mwf) < 0)
 			return -1;
-
-		return 0;
 	}
 
 	if (!idx->parsed_header) {
+		unsigned int total_objects;
+
 		if ((unsigned)idx->pack->mwf.size < sizeof(hdr))
 			return 0;
 
@@ -328,19 +429,25 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 
 		/* for now, limit to 2^32 objects */
 		assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
+		if (idx->nr_objects == (size_t)((unsigned int)idx->nr_objects))
+			total_objects = (unsigned int)idx->nr_objects;
+		else
+			total_objects = UINT_MAX;
 
-		if (git_vector_init(&idx->pack->cache, (unsigned int)idx->nr_objects, cache_cmp) < 0)
-			return -1;
+		idx->pack->idx_cache = git_oidmap_alloc();
+		GITERR_CHECK_ALLOC(idx->pack->idx_cache);
 
 		idx->pack->has_cache = 1;
-		if (git_vector_init(&idx->objects, (unsigned int)idx->nr_objects, objects_cmp) < 0)
+		if (git_vector_init(&idx->objects, total_objects, objects_cmp) < 0)
 			return -1;
 
-		if (git_vector_init(&idx->deltas, (unsigned int)(idx->nr_objects / 2), NULL) < 0)
+		if (git_vector_init(&idx->deltas, total_objects / 2, NULL) < 0)
 			return -1;
 
-		stats->total = (unsigned int)idx->nr_objects;
-		stats->processed = 0;
+		stats->received_objects = 0;
+		processed = stats->indexed_objects = 0;
+		stats->total_objects = total_objects;
+		do_progress_callback(idx, stats);
 	}
 
 	/* Now that we have data in the pack, let's try to parse it */
@@ -348,42 +455,91 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 	/* As the file grows any windows we try to use will be out of date */
 	git_mwindow_free_all(mwf);
 	while (processed < idx->nr_objects) {
-		git_rawobj obj;
+		git_packfile_stream *stream = &idx->stream;
 		git_off_t entry_start = idx->off;
+		size_t entry_size;
+		git_otype type;
+		git_mwindow *w = NULL;
 
 		if (idx->pack->mwf.size <= idx->off + 20)
 			return 0;
 
-		error = git_packfile_unpack(&obj, idx->pack, &idx->off);
-		if (error == GIT_EBUFS) {
-			idx->off = entry_start;
-			return 0;
-		}
-
-		if (error < 0) {
-			idx->off = entry_start;
-			error = store_delta(idx);
-			if (error == GIT_EBUFS)
+		if (!idx->have_stream) {
+			error = git_packfile_unpack_header(&entry_size, &type, mwf, &w, &idx->off);
+			if (error == GIT_EBUFS) {
+				idx->off = entry_start;
 				return 0;
+			}
 			if (error < 0)
-				return error;
+				return -1;
+
+			git_mwindow_close(&w);
+			idx->entry_start = entry_start;
+			git_hash_ctx_init(&idx->hash_ctx);
+
+			if (type == GIT_OBJ_REF_DELTA || type == GIT_OBJ_OFS_DELTA) {
+				error = advance_delta_offset(idx, type);
+				if (error == GIT_EBUFS) {
+					idx->off = entry_start;
+					return 0;
+				}
+				if (error < 0)
+					return -1;
+
+				idx->have_delta = 1;
+			} else {
+				idx->have_delta = 0;
+				hash_header(&idx->hash_ctx, entry_size, type);
+			}
+
+			idx->have_stream = 1;
+			if (git_packfile_stream_open(stream, idx->pack, idx->off) < 0)
+				goto on_error;
 
-			continue;
 		}
 
-		if (hash_and_save(idx, &obj, entry_start) < 0)
+		if (idx->have_delta) {
+			error = read_object_stream(idx, stream);
+		} else {
+			error = hash_object_stream(idx, stream);
+		}
+
+		idx->off = stream->curpos;
+		if (error == GIT_EBUFS)
+			return 0;
+
+		/* We want to free the stream reasorces no matter what here */
+		idx->have_stream = 0;
+		git_packfile_stream_free(stream);
+
+		if (error < 0)
+			goto on_error;
+
+		if (idx->have_delta) {
+			error = store_delta(idx);
+		} else {
+			error = store_object(idx);
+		}
+
+		if (error < 0)
 			goto on_error;
 
-		git__free(obj.data);
+		if (!idx->have_delta) {
+			stats->indexed_objects = (unsigned int)++processed;
+		}
+		stats->received_objects++;
 
-		stats->processed = (unsigned int)++processed;
+		if (do_progress_callback(idx, stats) != 0) {
+			error = GIT_EUSER;
+			goto on_error;
+		}
 	}
 
 	return 0;
 
 on_error:
 	git_mwindow_free_all(mwf);
-	return -1;
+	return error;
 }
 
 static int index_path_stream(git_buf *path, git_indexer_stream *idx, const char *suffix)
@@ -408,7 +564,7 @@ static int index_path_stream(git_buf *path, git_indexer_stream *idx, const char
 	return git_buf_oom(path) ? -1 : 0;
 }
 
-static int resolve_deltas(git_indexer_stream *idx, git_indexer_stats *stats)
+static int resolve_deltas(git_indexer_stream *idx, git_transfer_progress *stats)
 {
 	unsigned int i;
 	struct delta_info *delta;
@@ -424,13 +580,14 @@ static int resolve_deltas(git_indexer_stream *idx, git_indexer_stats *stats)
 			return -1;
 
 		git__free(obj.data);
-		stats->processed++;
+		stats->indexed_objects++;
+		do_progress_callback(idx, stats);
 	}
 
 	return 0;
 }
 
-int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stats)
+int git_indexer_stream_finalize(git_indexer_stream *idx, git_transfer_progress *stats)
 {
 	git_mwindow *w = NULL;
 	unsigned int i, long_offsets = 0, left;
@@ -439,11 +596,15 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stat
 	struct entry *entry;
 	void *packfile_hash;
 	git_oid file_hash;
-	SHA_CTX ctx;
+	git_hash_ctx ctx;
+	git_filebuf index_file = {0};
+
+	if (git_hash_ctx_init(&ctx) < 0)
+		return -1;
 
 	/* Test for this before resolve_deltas(), as it plays with idx->off */
 	if (idx->off < idx->pack->mwf.size - GIT_OID_RAWSZ) {
-		giterr_set(GITERR_INDEXER, "Indexing error: junk at the end of the pack");
+		giterr_set(GITERR_INDEXER, "Indexing error: unexpected data at the end of the pack");
 		return -1;
 	}
 
@@ -451,7 +612,7 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stat
 		if (resolve_deltas(idx, stats) < 0)
 			return -1;
 
-	if (stats->processed != stats->total) {
+	if (stats->indexed_objects != stats->total_objects) {
 		giterr_set(GITERR_INDEXER, "Indexing error: early EOF");
 		return -1;
 	}
@@ -464,31 +625,30 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stat
 	if (git_buf_oom(&filename))
 		return -1;
 
-	if (git_filebuf_open(&idx->index_file, filename.ptr, GIT_FILEBUF_HASH_CONTENTS) < 0)
+	if (git_filebuf_open(&index_file, filename.ptr, GIT_FILEBUF_HASH_CONTENTS) < 0)
 		goto on_error;
 
 	/* Write out the header */
 	hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
 	hdr.idx_version = htonl(2);
-	git_filebuf_write(&idx->index_file, &hdr, sizeof(hdr));
+	git_filebuf_write(&index_file, &hdr, sizeof(hdr));
 
 	/* Write out the fanout table */
 	for (i = 0; i < 256; ++i) {
 		uint32_t n = htonl(idx->fanout[i]);
-		git_filebuf_write(&idx->index_file, &n, sizeof(n));
+		git_filebuf_write(&index_file, &n, sizeof(n));
 	}
 
 	/* Write out the object names (SHA-1 hashes) */
-	SHA1_Init(&ctx);
 	git_vector_foreach(&idx->objects, i, entry) {
-		git_filebuf_write(&idx->index_file, &entry->oid, sizeof(git_oid));
-		SHA1_Update(&ctx, &entry->oid, GIT_OID_RAWSZ);
+		git_filebuf_write(&index_file, &entry->oid, sizeof(git_oid));
+		git_hash_update(&ctx, &entry->oid, GIT_OID_RAWSZ);
 	}
-	SHA1_Final(idx->hash.id, &ctx);
+	git_hash_final(&idx->hash, &ctx);
 
 	/* Write out the CRC32 values */
 	git_vector_foreach(&idx->objects, i, entry) {
-		git_filebuf_write(&idx->index_file, &entry->crc, sizeof(uint32_t));
+		git_filebuf_write(&index_file, &entry->crc, sizeof(uint32_t));
 	}
 
 	/* Write out the offsets */
@@ -500,7 +660,7 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stat
 		else
 			n = htonl(entry->offset);
 
-		git_filebuf_write(&idx->index_file, &n, sizeof(uint32_t));
+		git_filebuf_write(&index_file, &n, sizeof(uint32_t));
 	}
 
 	/* Write out the long offsets */
@@ -513,7 +673,7 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stat
 		split[0] = htonl(entry->offset_long >> 32);
 		split[1] = htonl(entry->offset_long & 0xffffffff);
 
-		git_filebuf_write(&idx->index_file, &split, sizeof(uint32_t) * 2);
+		git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2);
 	}
 
 	/* Write out the packfile trailer */
@@ -526,24 +686,26 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stat
 	memcpy(&file_hash, packfile_hash, GIT_OID_RAWSZ);
 	git_mwindow_close(&w);
 
-	git_filebuf_write(&idx->index_file, &file_hash, sizeof(git_oid));
+	git_filebuf_write(&index_file, &file_hash, sizeof(git_oid));
 
 	/* Write out the packfile trailer to the idx file as well */
-	if (git_filebuf_hash(&file_hash, &idx->index_file) < 0)
+	if (git_filebuf_hash(&file_hash, &index_file) < 0)
 		goto on_error;
 
-	git_filebuf_write(&idx->index_file, &file_hash, sizeof(git_oid));
+	git_filebuf_write(&index_file, &file_hash, sizeof(git_oid));
 
 	/* Figure out what the final name should be */
 	if (index_path_stream(&filename, idx, ".idx") < 0)
 		goto on_error;
 
 	/* Commit file */
-	if (git_filebuf_commit_at(&idx->index_file, filename.ptr, GIT_PACK_FILE_MODE) < 0)
+	if (git_filebuf_commit_at(&index_file, filename.ptr, GIT_PACK_FILE_MODE) < 0)
 		goto on_error;
 
 	git_mwindow_free_all(&idx->pack->mwf);
+	/* We need to close the descriptor here so Windows doesn't choke on commit_at */
 	p_close(idx->pack->mwf.fd);
+	idx->pack->mwf.fd = -1;
 
 	if (index_path_stream(&filename, idx, ".pack") < 0)
 		goto on_error;
@@ -556,17 +718,17 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stat
 
 on_error:
 	git_mwindow_free_all(&idx->pack->mwf);
-	p_close(idx->pack->mwf.fd);
-	git_filebuf_cleanup(&idx->index_file);
+	git_filebuf_cleanup(&index_file);
 	git_buf_free(&filename);
+	git_hash_ctx_cleanup(&ctx);
 	return -1;
 }
 
 void git_indexer_stream_free(git_indexer_stream *idx)
 {
+	khiter_t k;
 	unsigned int i;
 	struct entry *e;
-	struct git_pack_entry *pe;
 	struct delta_info *delta;
 
 	if (idx == NULL)
@@ -575,325 +737,20 @@ void git_indexer_stream_free(git_indexer_stream *idx)
 	git_vector_foreach(&idx->objects, i, e)
 		git__free(e);
 	git_vector_free(&idx->objects);
-	git_vector_foreach(&idx->pack->cache, i, pe)
-		git__free(pe);
-	git_vector_free(&idx->pack->cache);
-	git_vector_foreach(&idx->deltas, i, delta)
-		git__free(delta);
-	git_vector_free(&idx->deltas);
-	git__free(idx->pack);
-	git__free(idx);
-}
-
-int git_indexer_new(git_indexer **out, const char *packname)
-{
-	git_indexer *idx;
-	struct git_pack_header hdr;
-	int error;
-
-	assert(out && packname);
-
-	if (git_path_root(packname) < 0) {
-		giterr_set(GITERR_INDEXER, "Path is not absolute");
-		return -1;
-	}
-
-	idx = git__calloc(1, sizeof(git_indexer));
-	GITERR_CHECK_ALLOC(idx);
-
-	open_pack(&idx->pack, packname);
-
-	if ((error = parse_header(&hdr, idx->pack)) < 0)
-		goto cleanup;
-
-	idx->nr_objects = ntohl(hdr.hdr_entries);
-
-	/* for now, limit to 2^32 objects */
-	assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
-
-	error = git_vector_init(&idx->pack->cache, (unsigned int)idx->nr_objects, cache_cmp);
-	if (error < 0)
-		goto cleanup;
-
-	idx->pack->has_cache = 1;
-	error = git_vector_init(&idx->objects, (unsigned int)idx->nr_objects, objects_cmp);
-	if (error < 0)
-		goto cleanup;
-
-	*out = idx;
-
-	return 0;
 
-cleanup:
-	git_indexer_free(idx);
-
-	return -1;
-}
-
-static int index_path(git_buf *path, git_indexer *idx)
-{
-	const char prefix[] = "pack-", suffix[] = ".idx";
-	size_t slash = (size_t)path->size;
-
-	/* search backwards for '/' */
-	while (slash > 0 && path->ptr[slash - 1] != '/')
-		slash--;
-
-	if (git_buf_grow(path, slash + 1 + strlen(prefix) +
-					 GIT_OID_HEXSZ + strlen(suffix) + 1) < 0)
-		return -1;
-
-	git_buf_truncate(path, slash);
-	git_buf_puts(path, prefix);
-	git_oid_fmt(path->ptr + git_buf_len(path), &idx->hash);
-	path->size += GIT_OID_HEXSZ;
-	git_buf_puts(path, suffix);
-
-	return git_buf_oom(path) ? -1 : 0;
-}
-
-int git_indexer_write(git_indexer *idx)
-{
-	git_mwindow *w = NULL;
-	int error;
-	unsigned int i, long_offsets = 0, left;
-	struct git_pack_idx_header hdr;
-	git_buf filename = GIT_BUF_INIT;
-	struct entry *entry;
-	void *packfile_hash;
-	git_oid file_hash;
-	SHA_CTX ctx;
-
-	git_vector_sort(&idx->objects);
-
-	git_buf_sets(&filename, idx->pack->pack_name);
-	git_buf_truncate(&filename, filename.size - strlen("pack"));
-	git_buf_puts(&filename, "idx");
-	if (git_buf_oom(&filename))
-		return -1;
-
-	error = git_filebuf_open(&idx->file, filename.ptr, GIT_FILEBUF_HASH_CONTENTS);
-	if (error < 0)
-		goto cleanup;
-
-	/* Write out the header */
-	hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
-	hdr.idx_version = htonl(2);
-	error = git_filebuf_write(&idx->file, &hdr, sizeof(hdr));
-	if (error < 0)
-		goto cleanup;
-
-	/* Write out the fanout table */
-	for (i = 0; i < 256; ++i) {
-		uint32_t n = htonl(idx->fanout[i]);
-		error = git_filebuf_write(&idx->file, &n, sizeof(n));
-		if (error < 0)
-			goto cleanup;
-	}
-
-	/* Write out the object names (SHA-1 hashes) */
-	SHA1_Init(&ctx);
-	git_vector_foreach(&idx->objects, i, entry) {
-		error = git_filebuf_write(&idx->file, &entry->oid, sizeof(git_oid));
-		SHA1_Update(&ctx, &entry->oid, GIT_OID_RAWSZ);
-		if (error < 0)
-			goto cleanup;
-	}
-	SHA1_Final(idx->hash.id, &ctx);
-
-	/* Write out the CRC32 values */
-	git_vector_foreach(&idx->objects, i, entry) {
-		error = git_filebuf_write(&idx->file, &entry->crc, sizeof(uint32_t));
-		if (error < 0)
-			goto cleanup;
-	}
-
-	/* Write out the offsets */
-	git_vector_foreach(&idx->objects, i, entry) {
-		uint32_t n;
-
-		if (entry->offset == UINT32_MAX)
-			n = htonl(0x80000000 | long_offsets++);
-		else
-			n = htonl(entry->offset);
-
-		error = git_filebuf_write(&idx->file, &n, sizeof(uint32_t));
-		if (error < 0)
-			goto cleanup;
-	}
-
-	/* Write out the long offsets */
-	git_vector_foreach(&idx->objects, i, entry) {
-		uint32_t split[2];
-
-		if (entry->offset != UINT32_MAX)
-			continue;
-
-		split[0] = htonl(entry->offset_long >> 32);
-		split[1] = htonl(entry->offset_long & 0xffffffff);
-
-		error = git_filebuf_write(&idx->file, &split, sizeof(uint32_t) * 2);
-		if (error < 0)
-			goto cleanup;
-	}
-
-	/* Write out the packfile trailer */
-
-	packfile_hash = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
-	git_mwindow_close(&w);
-	if (packfile_hash == NULL) {
-		error = -1;
-		goto cleanup;
-	}
-
-	memcpy(&file_hash, packfile_hash, GIT_OID_RAWSZ);
-
-	git_mwindow_close(&w);
-
-	error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid));
-	if (error < 0)
-		goto cleanup;
-
-	/* Write out the index sha */
-	error = git_filebuf_hash(&file_hash, &idx->file);
-	if (error < 0)
-		goto cleanup;
-
-	error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid));
-	if (error < 0)
-		goto cleanup;
-
-	/* Figure out what the final name should be */
-	error = index_path(&filename, idx);
-	if (error < 0)
-		goto cleanup;
-
-	/* Commit file */
-	error = git_filebuf_commit_at(&idx->file, filename.ptr, GIT_PACK_FILE_MODE);
-
-cleanup:
-	git_mwindow_free_all(&idx->pack->mwf);
-	if (error < 0)
-		git_filebuf_cleanup(&idx->file);
-	git_buf_free(&filename);
-
-	return error;
-}
-
-int git_indexer_run(git_indexer *idx, git_indexer_stats *stats)
-{
-	git_mwindow_file *mwf;
-	git_off_t off = sizeof(struct git_pack_header);
-	int error;
-	struct entry *entry;
-	unsigned int left, processed;
-
-	assert(idx && stats);
-
-	mwf = &idx->pack->mwf;
-	error = git_mwindow_file_register(mwf);
-	if (error < 0)
-		return error;
-
-	stats->total = (unsigned int)idx->nr_objects;
-	stats->processed = processed = 0;
-
-	while (processed < idx->nr_objects) {
-		git_rawobj obj;
-		git_oid oid;
-		struct git_pack_entry *pentry;
-		git_mwindow *w = NULL;
-		int i;
-		git_off_t entry_start = off;
-		void *packed;
-		size_t entry_size;
-		char fmt[GIT_OID_HEXSZ] = {0};
-
-		entry = git__calloc(1, sizeof(*entry));
-		GITERR_CHECK_ALLOC(entry);
-
-		if (off > UINT31_MAX) {
-			entry->offset = UINT32_MAX;
-			entry->offset_long = off;
-		} else {
-			entry->offset = (uint32_t)off;
-		}
-
-		error = git_packfile_unpack(&obj, idx->pack, &off);
-		if (error < 0)
-			goto cleanup;
-
-		/* FIXME: Parse the object instead of hashing it */
-		error = git_odb__hashobj(&oid, &obj);
-		if (error < 0) {
-			giterr_set(GITERR_INDEXER, "Failed to hash object");
-			goto cleanup;
+	if (idx->pack) {
+		for (k = kh_begin(idx->pack->idx_cache); k != kh_end(idx->pack->idx_cache); k++) {
+			if (kh_exist(idx->pack->idx_cache, k))
+				git__free(kh_value(idx->pack->idx_cache, k));
 		}
 
-		pentry = git__malloc(sizeof(struct git_pack_entry));
-		if (pentry == NULL) {
-			error = -1;
-			goto cleanup;
-		}
-
-		git_oid_cpy(&pentry->sha1, &oid);
-		pentry->offset = entry_start;
-		git_oid_fmt(fmt, &oid);
-		printf("adding %s to cache\n", fmt);
-		error = git_vector_insert(&idx->pack->cache, pentry);
-		if (error < 0)
-			goto cleanup;
-
-		git_oid_cpy(&entry->oid, &oid);
-		entry->crc = crc32(0L, Z_NULL, 0);
-
-		entry_size = (size_t)(off - entry_start);
-		packed = git_mwindow_open(mwf, &w, entry_start, entry_size, &left);
-		if (packed == NULL) {
-			error = -1;
-			goto cleanup;
-		}
-		entry->crc = htonl(crc32(entry->crc, packed, (uInt)entry_size));
-		git_mwindow_close(&w);
-
-		/* Add the object to the list */
-		error = git_vector_insert(&idx->objects, entry);
-		if (error < 0)
-			goto cleanup;
-
-		for (i = oid.id[0]; i < 256; ++i) {
-			idx->fanout[i]++;
-		}
-
-		git__free(obj.data);
-
-		stats->processed = ++processed;
+		git_oidmap_free(idx->pack->idx_cache);
 	}
 
-cleanup:
-	git_mwindow_free_all(mwf);
-
-	return error;
-
-}
-
-void git_indexer_free(git_indexer *idx)
-{
-	unsigned int i;
-	struct entry *e;
-	struct git_pack_entry *pe;
-
-	if (idx == NULL)
-		return;
-
-	p_close(idx->pack->mwf.fd);
-	git_vector_foreach(&idx->objects, i, e)
-		git__free(e);
-	git_vector_free(&idx->objects);
-	git_vector_foreach(&idx->pack->cache, i, pe)
-		git__free(pe);
-	git_vector_free(&idx->pack->cache);
-	git__free(idx->pack);
+	git_vector_foreach(&idx->deltas, i, delta)
+		git__free(delta);
+	git_vector_free(&idx->deltas);
+	git_packfile_free(idx->pack);
+	git_filebuf_cleanup(&idx->pack_file);
 	git__free(idx);
 }
-