filter: Apply filters before writing a file to the ODB

Initial implementation. The relevant code is in `blob.c`: the blob write
function has been split into smaller functions.

	- Directly write a file to the ODB in streaming mode

	- Directly write a symlink to the ODB in direct mode

	- Apply a filter, and write a file to the ODB in direct mode

When trying to write a file, we first call `git_filter__load_for_file`,
which populates a filters array with the required filters based on the
filename.

If no filters are resolved to the filename, we can write to the ODB in
streaming mode straight from disk. Otherwise, we load the whole file in
memory and use double-buffering to apply the filter chain. We finish
by writing the file as a whole to the ODB.

1 files changed, 108 insertions, 48 deletions

diff --git a/src/blob.c b/src/blob.c
index 4065ffa12..57a31041e 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -11,6 +11,7 @@
 
 #include "common.h"
 #include "blob.h"
+#include "filter.h"
 
 const void *git_blob_rawcontent(git_blob *blob)
 {
@@ -65,15 +66,101 @@ int git_blob_create_frombuffer(git_oid *oid, git_repository *repo, const void *b
 	return GIT_SUCCESS;
 }
 
+static int write_file_stream(git_oid *oid, git_odb *odb, const char *path, git_off_t file_size)
+{
+	int fd, error;
+	char buffer[4096];
+	git_odb_stream *stream = NULL;
+
+	if ((error = git_odb_open_wstream(&stream, odb, file_size, GIT_OBJ_BLOB)) < GIT_SUCCESS)
+		return error;
+
+	if ((fd = p_open(path, O_RDONLY)) < 0) {
+		error = git__throw(GIT_ENOTFOUND, "Failed to create blob. Could not open '%s'", path);
+		goto cleanup;
+	}
+
+	while (file_size > 0) {
+		ssize_t read_len = p_read(fd, buffer, sizeof(buffer));
+
+		if (read_len < 0) {
+			error = git__throw(GIT_EOSERR, "Failed to create blob. Can't read full file");
+			p_close(fd);
+			goto cleanup;
+		}
+
+		stream->write(stream, buffer, read_len);
+		file_size -= read_len;
+	}
+
+	p_close(fd);
+	error = stream->finalize_write(oid, stream);
+
+cleanup:
+	stream->free(stream);
+	return error;
+}
+
+static int write_file_filtered(
+	git_oid *oid,
+	git_odb *odb,
+	const char *path,
+	git_vector *filters)
+{
+	int error;
+	git_buf file_in = GIT_BUF_INIT;
+	git_buf filter_result = GIT_BUF_INIT;
+
+	error = git_futils_readbuffer(&file_in, path);
+	if (error < GIT_SUCCESS)
+		return error;
+
+	error = git_filter__apply(&filter_result, &file_in, filters, path);
+
+	if (error < GIT_SUCCESS) {
+		git_buf_free(&file_in);
+		git_buf_free(&filter_result);
+		return error;
+	}
+
+	error = git_odb_write(oid, odb, filter_result.ptr, filter_result.size, GIT_OBJ_BLOB);
+
+	git_buf_free(&file_in);
+	git_buf_free(&filter_result);
+
+	return GIT_SUCCESS;
+}
+
+static int write_symlink(git_oid *oid, git_odb *odb, const char *path, size_t link_size)
+{
+	char *link_data;
+	ssize_t read_len;
+	int error;
+
+	link_data = git__malloc(link_size);
+	if (!link_data)
+		return GIT_ENOMEM;
+
+	read_len = p_readlink(path, link_data, link_size);
+
+	if (read_len != (ssize_t)link_size) {
+		free(link_data);
+		return git__throw(GIT_EOSERR, "Failed to create blob. Can't read symlink");
+	}
+
+	error = git_odb_write(oid, odb, (void *)link_data, link_size, GIT_OBJ_BLOB);
+	free(link_data);
+	return error;
+}
+
 int git_blob_create_fromfile(git_oid *oid, git_repository *repo, const char *path)
 {
 	int error = GIT_SUCCESS;
 	git_buf full_path = GIT_BUF_INIT;
 	git_off_t size;
-	git_odb_stream *stream = NULL;
 	struct stat st;
 	const char *workdir;
-	git_odb *odb;
+	git_odb *odb = NULL;
 
 	workdir = git_repository_workdir(repo);
 	if (workdir == NULL)
@@ -95,63 +182,36 @@ int git_blob_create_fromfile(git_oid *oid, git_repository *repo, const char *pat
 	if (error < GIT_SUCCESS)
 		goto cleanup;
 
-	if ((error = git_odb_open_wstream(&stream, odb, (size_t)size, GIT_OBJ_BLOB)) < GIT_SUCCESS)
-		goto cleanup;
-
 	if (S_ISLNK(st.st_mode)) {
-		char *link_data;
-		ssize_t read_len;
-
-		link_data = git__malloc((size_t)size);
-		if (!link_data) {
-			error = GIT_ENOMEM;
-			goto cleanup;
-		}
-
-		read_len = p_readlink(full_path.ptr, link_data, (size_t)size);
-
-		if (read_len != (ssize_t)size) {
-			error = git__throw(GIT_EOSERR, "Failed to create blob. Can't read symlink");
-			free(link_data);
-			goto cleanup;
-		}
-
-		stream->write(stream, link_data, (size_t)size);
-		free(link_data);
-
+		error = write_symlink(oid, odb, full_path.ptr, (size_t)size);
 	} else {
-		int fd;
-		char buffer[2048];
+		git_vector write_filters = GIT_VECTOR_INIT;
 
-		if ((fd = p_open(full_path.ptr, O_RDONLY)) < 0) {
-			error = git__throw(GIT_ENOTFOUND, "Failed to create blob. Could not open '%s'", full_path.ptr);
+		if ((error = git_filter__load_for_file(
+			&write_filters, repo, full_path.ptr, GIT_FILTER_TO_ODB)) < GIT_SUCCESS)
 			goto cleanup;
-		}
-
-		while (size > 0) {
-			ssize_t read_len = p_read(fd, buffer, sizeof(buffer));
 
-			if (read_len < 0) {
-				error = git__throw(GIT_EOSERR, "Failed to create blob. Can't read full file");
-				p_close(fd);
-				goto cleanup;
-			}
-
-			stream->write(stream, buffer, read_len);
-			size -= read_len;
+		if (write_filters.length == 0) {
+			error = write_file_stream(oid, odb, full_path.ptr, size);
+		} else {
+			error = write_file_filtered(oid, odb, full_path.ptr, &write_filters);
 		}
 
-		p_close(fd);
+		/*
+		 * TODO: eventually support streaming filtered files, for files which are bigger
+		 * than a given threshold. This is not a priority because applying a filter in
+		 * streaming mode changes the final size of the blob, and without knowing its
+		 * final size, the blob cannot be written in stream mode to the ODB.
+		 *
+		 * The plan is to do streaming writes to a tempfile on disk and then opening
+		 * streaming that file to the ODB, using `write_file_stream`.
+		 *
+		 * CAREFULLY DESIGNED APIS YO
+		 */
 	}
 
-	error = stream->finalize_write(oid, stream);
-
 cleanup:
-	if (stream)
-		stream->free(stream);
-
 	git_buf_free(&full_path);
-
 	return error;
 }