1 files changed, 243 insertions, 49 deletions

diff --git a/src/odb.c b/src/odb.c
index a6a18f831..c98df247c 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2009-2012 the libgit2 contributors
+ * Copyright (C) the libgit2 contributors. All rights reserved.
  *
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
@@ -12,8 +12,10 @@
 #include "hash.h"
 #include "odb.h"
 #include "delta-apply.h"
+#include "filter.h"
 
 #include "git2/odb_backend.h"
+#include "git2/oid.h"
 
 #define GIT_ALTERNATES_FILE "info/alternates"
 
@@ -21,6 +23,8 @@
 #define GIT_LOOSE_PRIORITY 2
 #define GIT_PACKED_PRIORITY 1
 
+#define GIT_ALTERNATES_MAX_DEPTH 5
+
 typedef struct
 {
 	git_odb_backend *backend;
@@ -28,7 +32,11 @@ typedef struct
 	int is_alternate;
 } backend_internal;
 
-static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
+size_t git_odb__cache_size = GIT_DEFAULT_CACHE_SIZE;
+
+static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth);
+
+int git_odb__format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
 {
 	const char *type_str = git_object_type2string(obj_type);
 	int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
@@ -49,7 +57,7 @@ int git_odb__hashobj(git_oid *id, git_rawobj *obj)
 	if (!obj->data && obj->len != 0)
 		return -1;
 
-	hdrlen = format_object_header(header, sizeof(header), obj->len, obj->type);
+	hdrlen = git_odb__format_object_header(header, sizeof(header), obj->len, obj->type);
 
 	vec[0].data = header;
 	vec[0].len = hdrlen;
@@ -105,6 +113,9 @@ git_otype git_odb_object_type(git_odb_object *object)
 
 void git_odb_object_free(git_odb_object *object)
 {
+	if (object == NULL)
+		return;
+
 	git_cached_obj_decref((git_cached_obj *)object, &free_odb_object);
 }
 
@@ -112,31 +123,73 @@ int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
 {
 	int hdr_len;
 	char hdr[64], buffer[2048];
-	git_hash_ctx *ctx;
+	git_hash_ctx ctx;
+	ssize_t read_len = 0;
+	int error = 0;
 
-	hdr_len = format_object_header(hdr, sizeof(hdr), size, type);
+	if (!git_object_typeisloose(type)) {
+		giterr_set(GITERR_INVALID, "Invalid object type for hash");
+		return -1;
+	}
 
-	ctx = git_hash_new_ctx();
+	if ((error = git_hash_ctx_init(&ctx)) < 0)
+		return -1;
 
-	git_hash_update(ctx, hdr, hdr_len);
+	hdr_len = git_odb__format_object_header(hdr, sizeof(hdr), size, type);
 
-	while (size > 0) {
-		ssize_t read_len = read(fd, buffer, sizeof(buffer));
+	if ((error = git_hash_update(&ctx, hdr, hdr_len)) < 0)
+		goto done;
 
-		if (read_len < 0) {
-			git_hash_free_ctx(ctx);
-			giterr_set(GITERR_OS, "Error reading file");
-			return -1;
-		}
+	while (size > 0 && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
+		if ((error = git_hash_update(&ctx, buffer, read_len)) < 0)
+			goto done;
 
-		git_hash_update(ctx, buffer, read_len);
 		size -= read_len;
 	}
 
-	git_hash_final(out, ctx);
-	git_hash_free_ctx(ctx);
+	/* If p_read returned an error code, the read obviously failed.
+	 * If size is not zero, the file was truncated after we originally
+	 * stat'd it, so we consider this a read failure too */
+	if (read_len < 0 || size > 0) {
+		giterr_set(GITERR_OS, "Error reading file for hashing");
+		error = -1;
 
-	return 0;
+		goto done;
+		return -1;
+	}
+
+	error = git_hash_final(out, &ctx);
+
+done:
+	git_hash_ctx_cleanup(&ctx);
+	return error;
+}
+
+int git_odb__hashfd_filtered(
+	git_oid *out, git_file fd, size_t size, git_otype type, git_vector *filters)
+{
+	int error;
+	git_buf raw = GIT_BUF_INIT;
+	git_buf filtered = GIT_BUF_INIT;
+
+	if (!filters || !filters->length)
+		return git_odb__hashfd(out, fd, size, type);
+
+	/* size of data is used in header, so we have to read the whole file
+	 * into memory to apply filters before beginning to calculate the hash
+	 */
+
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, size)))
+		error = git_filters_apply(&filtered, &raw, filters);
+
+	git_buf_free(&raw);
+
+	if (!error)
+		error = git_odb_hash(out, filtered.ptr, filtered.size, type);
+
+	git_buf_free(&filtered);
+
+	return error;
 }
 
 int git_odb__hashlink(git_oid *out, const char *path)
@@ -159,10 +212,11 @@ int git_odb__hashlink(git_oid *out, const char *path)
 		char *link_data;
 		ssize_t read_len;
 
-		link_data = git__malloc((size_t)size);
+		link_data = git__malloc((size_t)(size + 1));
 		GITERR_CHECK_ALLOC(link_data);
 
-		read_len = p_readlink(path, link_data, (size_t)(size + 1));
+		read_len = p_readlink(path, link_data, (size_t)size);
+		link_data[size] = '\0';
 		if (read_len != (ssize_t)size) {
 			giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path);
 			return -1;
@@ -170,7 +224,7 @@ int git_odb__hashlink(git_oid *out, const char *path)
 
 		result = git_odb_hash(out, link_data, (size_t)size, GIT_OBJ_BLOB);
 		git__free(link_data);
-	} else { 
+	} else {
 		int fd = git_futils_open_ro(path);
 		if (fd < 0)
 			return -1;
@@ -299,7 +353,7 @@ int git_odb_new(git_odb **out)
 	git_odb *db = git__calloc(1, sizeof(*db));
 	GITERR_CHECK_ALLOC(db);
 
-	if (git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object) < 0 ||
+	if (git_cache_init(&db->cache, git_odb__cache_size, &free_odb_object) < 0 ||
 		git_vector_init(&db->backends, 4, backend_sort_cmp) < 0)
 	{
 		git__free(db);
@@ -317,6 +371,8 @@ static int add_backend_internal(git_odb *odb, git_odb_backend *backend, int prio
 
 	assert(odb && backend);
 
+	GITERR_CHECK_VERSION(backend, GIT_ODB_BACKEND_VERSION, "git_odb_backend");
+
 	/* Check if the backend is already owned by another ODB */
 	assert(!backend->odb || backend->odb == odb);
 
@@ -347,7 +403,7 @@ int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
 	return add_backend_internal(odb, backend, priority, 1);
 }
 
-static int add_default_backends(git_odb *db, const char *objects_dir, int as_alternates)
+static int add_default_backends(git_odb *db, const char *objects_dir, int as_alternates, int alternate_depth)
 {
 	git_odb_backend *loose, *packed;
 
@@ -361,10 +417,10 @@ static int add_default_backends(git_odb *db, const char *objects_dir, int as_alt
 		add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates) < 0)
 		return -1;
 
-	return 0;
+	return load_alternates(db, objects_dir, alternate_depth);
 }
 
-static int load_alternates(git_odb *odb, const char *objects_dir)
+static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth)
 {
 	git_buf alternates_path = GIT_BUF_INIT;
 	git_buf alternates_buf = GIT_BUF_INIT;
@@ -372,6 +428,11 @@ static int load_alternates(git_odb *odb, const char *objects_dir)
 	const char *alternate;
 	int result = 0;
 
+	/* Git reports an error, we just ignore anything deeper */
+	if (alternate_depth > GIT_ALTERNATES_MAX_DEPTH) {
+		return 0;
+	}
+
 	if (git_buf_joinpath(&alternates_path, objects_dir, GIT_ALTERNATES_FILE) < 0)
 		return -1;
 
@@ -392,14 +453,18 @@ static int load_alternates(git_odb *odb, const char *objects_dir)
 		if (*alternate == '\0' || *alternate == '#')
 			continue;
 
-		/* relative path: build based on the current `objects` folder */
-		if (*alternate == '.') {
+		/*
+		 * Relative path: build based on the current `objects`
+		 * folder. However, relative paths are only allowed in
+		 * the current repository.
+		 */
+		if (*alternate == '.' && !alternate_depth) {
 			if ((result = git_buf_joinpath(&alternates_path, objects_dir, alternate)) < 0)
 				break;
 			alternate = git_buf_cstr(&alternates_path);
 		}
 
-		if ((result = add_default_backends(odb, alternate, 1)) < 0)
+		if ((result = add_default_backends(odb, alternate, 1, alternate_depth + 1)) < 0)
 			break;
 	}
 
@@ -409,6 +474,11 @@ static int load_alternates(git_odb *odb, const char *objects_dir)
 	return result;
 }
 
+int git_odb_add_disk_alternate(git_odb *odb, const char *path)
+{
+	return add_default_backends(odb, path, 1, 0);
+}
+
 int git_odb_open(git_odb **out, const char *objects_dir)
 {
 	git_odb *db;
@@ -420,9 +490,7 @@ int git_odb_open(git_odb **out, const char *objects_dir)
 	if (git_odb_new(&db) < 0)
 		return -1;
 
-	if (add_default_backends(db, objects_dir, 0) < 0 ||
-		load_alternates(db, objects_dir) < 0)
-	{
+	if (add_default_backends(db, objects_dir, 0, 0) < 0) {
 		git_odb_free(db);
 		return -1;
 	}
@@ -433,7 +501,7 @@ int git_odb_open(git_odb **out, const char *objects_dir)
 
 static void odb_free(git_odb *db)
 {
-	unsigned int i;
+	size_t i;
 
 	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
@@ -461,8 +529,9 @@ void git_odb_free(git_odb *db)
 int git_odb_exists(git_odb *db, const git_oid *id)
 {
 	git_odb_object *object;
-	unsigned int i;
+	size_t i;
 	bool found = false;
+	bool refreshed = false;
 
 	assert(db && id);
 
@@ -471,6 +540,7 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 		return (int)true;
 	}
 
+attempt_lookup:
 	for (i = 0; i < db->backends.length && !found; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
@@ -479,24 +549,51 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 			found = b->exists(b, id);
 	}
 
+	if (!found && !refreshed) {
+		if (git_odb_refresh(db) < 0) {
+			giterr_clear();
+			return (int)false;
+		}
+
+		refreshed = true;
+		goto attempt_lookup;
+	}
+
 	return (int)found;
 }
 
 int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
 {
-	unsigned int i;
+	int error;
+	git_odb_object *object;
+
+	error = git_odb__read_header_or_object(&object, len_p, type_p, db, id);
+
+	if (object)
+		git_odb_object_free(object);
+
+	return error;
+}
+
+int git_odb__read_header_or_object(
+	git_odb_object **out, size_t *len_p, git_otype *type_p,
+	git_odb *db, const git_oid *id)
+{
+	size_t i;
 	int error = GIT_ENOTFOUND;
 	git_odb_object *object;
 
-	assert(db && id);
+	assert(db && id && out && len_p && type_p);
 
 	if ((object = git_cache_get(&db->cache, id)) != NULL) {
 		*len_p = object->raw.len;
 		*type_p = object->raw.type;
-		git_odb_object_free(object);
+		*out = object;
 		return 0;
 	}
 
+	*out = NULL;
+
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
@@ -517,22 +614,32 @@ int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git
 
 	*len_p = object->raw.len;
 	*type_p = object->raw.type;
-	git_odb_object_free(object);
+	*out = object;
+
 	return 0;
 }
 
 int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 {
-	unsigned int i;
-	int error = GIT_ENOTFOUND;
+	size_t i;
+	int error;
+	bool refreshed = false;
 	git_rawobj raw;
 
 	assert(out && db && id);
 
+	if (db->backends.length == 0) {
+		giterr_set(GITERR_ODB, "Failed to lookup object: no backends loaded");
+		return GIT_ENOTFOUND;
+	}
+
 	*out = git_cache_get(&db->cache, id);
 	if (*out != NULL)
 		return 0;
 
+attempt_lookup:
+	error = GIT_ENOTFOUND;
+
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
@@ -541,9 +648,13 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 			error = b->read(&raw.data, &raw.len, &raw.type, b, id);
 	}
 
-	/* TODO: If no backends are configured, this returns GIT_ENOTFOUND but
-	 * will never have called giterr_set().
-	 */
+	if (error == GIT_ENOTFOUND && !refreshed) {
+		if ((error = git_odb_refresh(db)) < 0)
+			return error;
+
+		refreshed = true;
+		goto attempt_lookup;
+	}
 
 	if (error && error != GIT_PASSTHROUGH)
 		return error;
@@ -553,13 +664,14 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 }
 
 int git_odb_read_prefix(
-	git_odb_object **out, git_odb *db, const git_oid *short_id, unsigned int len)
+	git_odb_object **out, git_odb *db, const git_oid *short_id, size_t len)
 {
-	unsigned int i;
+	size_t i;
 	int error = GIT_ENOTFOUND;
 	git_oid found_full_oid = {{0}};
 	git_rawobj raw;
-	bool found = false;
+	void *data = NULL;
+	bool found = false, refreshed = false;
 
 	assert(out && db);
 
@@ -575,11 +687,12 @@ int git_odb_read_prefix(
 			return 0;
 	}
 
+attempt_lookup:
 	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
 
-		if (b->read != NULL) {
+		if (b->read_prefix != NULL) {
 			git_oid full_oid;
 			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
 			if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
@@ -588,13 +701,25 @@ int git_odb_read_prefix(
 			if (error)
 				return error;
 
+			git__free(data);
+			data = raw.data;
+
 			if (found && git_oid_cmp(&full_oid, &found_full_oid))
 				return git_odb__error_ambiguous("multiple matches for prefix");
+
 			found_full_oid = full_oid;
 			found = true;
 		}
 	}
 
+	if (!found && !refreshed) {
+		if ((error = git_odb_refresh(db)) < 0)
+			return error;
+
+		refreshed = true;
+		goto attempt_lookup;
+	}
+
 	if (!found)
 		return git_odb__error_notfound("no match for prefix", short_id);
 
@@ -602,15 +727,34 @@ int git_odb_read_prefix(
 	return 0;
 }
 
+int git_odb_foreach(git_odb *db, git_odb_foreach_cb cb, void *payload)
+{
+	unsigned int i;
+	backend_internal *internal;
+
+	git_vector_foreach(&db->backends, i, internal) {
+		git_odb_backend *b = internal->backend;
+		int error = b->foreach(b, cb, payload);
+		if (error < 0)
+			return error;
+	}
+
+	return 0;
+}
+
 int git_odb_write(
 	git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
 {
-	unsigned int i;
+	size_t i;
 	int error = GIT_ERROR;
 	git_odb_stream *stream;
 
 	assert(oid && db);
 
+	git_odb_hash(oid, data, len, type);
+	if (git_odb_exists(db, oid))
+		return 0;
+
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
@@ -643,7 +787,7 @@ int git_odb_write(
 int git_odb_open_wstream(
 	git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
 {
-	unsigned int i;
+	size_t i;
 	int error = GIT_ERROR;
 
 	assert(stream && db);
@@ -670,7 +814,7 @@ int git_odb_open_wstream(
 
 int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
 {
-	unsigned int i;
+	size_t i;
 	int error = GIT_ERROR;
 
 	assert(stream && db);
@@ -689,6 +833,56 @@ int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oi
 	return error;
 }
 
+int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer_progress_callback progress_cb, void *progress_payload)
+{
+	size_t i;
+	int error = GIT_ERROR;
+
+	assert(out && db);
+
+	for (i = 0; i < db->backends.length && error < 0; ++i) {
+		backend_internal *internal = git_vector_get(&db->backends, i);
+		git_odb_backend *b = internal->backend;
+
+		/* we don't write in alternates! */
+		if (internal->is_alternate)
+			continue;
+
+		if (b->writepack != NULL)
+			error = b->writepack(out, b, progress_cb, progress_payload);
+	}
+
+	if (error == GIT_PASSTHROUGH)
+		error = 0;
+
+	return error;
+}
+
+void *git_odb_backend_malloc(git_odb_backend *backend, size_t len)
+{
+	GIT_UNUSED(backend);
+	return git__malloc(len);
+}
+
+int git_odb_refresh(struct git_odb *db)
+{
+	size_t i;
+	assert(db);
+
+	for (i = 0; i < db->backends.length; ++i) {
+		backend_internal *internal = git_vector_get(&db->backends, i);
+		git_odb_backend *b = internal->backend;
+
+		if (b->refresh != NULL) {
+			int error = b->refresh(b);
+			if (error < 0)
+				return error;
+		}
+	}
+
+	return 0;
+}
+
 int git_odb__error_notfound(const char *message, const git_oid *oid)
 {
 	if (oid != NULL) {