diff options
author | Junio C Hamano <junkio@cox.net> | 2005-06-26 15:29:18 +0400 |
---|---|---|
committer | Linus Torvalds <torvalds@ppc970.osdl.org> | 2005-06-26 18:33:23 +0400 |
commit | 8ee378a0f00a56cbffedee21fdbba30870d84436 (patch) | |
tree | 78600eeaac28aa694b72a70d524b5d0f9839f15c | |
parent | d116a45a9a505d9a58cdd1c79193f759316c440d (diff) |
[PATCH] Finish initial cut of git-pack-object/git-unpack-object pair.
This finishes the initial round of git-pack-object /
git-unpack-object pair. They are now good enough to be used as
a transport medium:
- Fix delta direction in pack-objects; the original was
computing delta to create the base object from the object to
be squashed, which was quite unfriendly for unpacker ;-).
- Add a script to test the very basics.
- Implement unpacker for both regular and deltified objects.
Signed-off-by: Junio C Hamano <junkio@cox.net>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
-rw-r--r-- | pack-objects.c | 6 | ||||
-rwxr-xr-x | t/t5300-pack-object.sh | 85 | ||||
-rw-r--r-- | unpack-objects.c | 188 |
3 files changed, 261 insertions, 18 deletions
diff --git a/pack-objects.c b/pack-objects.c index 9d87a64cf6..3900df3a29 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -82,7 +82,8 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e if (!otherbuf) die("unable to read %s", sha1_to_hex(entry->delta->sha1)); - delta_buf = diff_delta(buf, size, otherbuf, othersize, &delta_size, ~0UL); + delta_buf = diff_delta(otherbuf, othersize, + buf, size, &delta_size, ~0UL); if (!delta_buf || delta_size != entry->delta_size) die("delta size changed"); free(buf); @@ -318,7 +319,8 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de max_size = size / 2 - 20; if (cur_entry->delta) max_size = cur_entry->delta_size-1; - delta_buf = diff_delta(cur->data, size, old->data, oldsize, &delta_size, max_size); + delta_buf = diff_delta(old->data, oldsize, + cur->data, size, &delta_size, max_size); if (!delta_buf) return 0; cur_entry->delta = old_entry; diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh new file mode 100755 index 0000000000..171af643e5 --- /dev/null +++ b/t/t5300-pack-object.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# +# Copyright (c) 2005 Junio C Hamano +# + +test_description='git-pack-object + +' +. ./test-lib.sh + +TRASH=`pwd` + +test_expect_success \ + 'setup' \ + 'rm -f .git/index* + for i in a b c + do + dd if=/dev/zero bs=4k count=1 | tr "\\0" $i >$i && + git-update-cache --add $i || exit + done && + cat c >d && echo foo >>d && git-update-cache --add d && + tree=`git-write-tree` && { + echo $tree && + git-ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\) .*/\\1/" + } >obj-list' + +test_expect_success \ + 'pack without delta' \ + 'git-pack-objects --window=0 test-1 <obj-list' + +rm -fr .git2 +mkdir .git2 + +test_expect_success \ + 'unpack without delta' \ + 'GIT_OBJECT_DIRECTORY=.git2/objects && + export GIT_OBJECT_DIRECTORY && + git-init-db && + git-unpack-objects test-1' + +unset GIT_OBJECT_DIRECTORY +cd $TRASH/.git2 + +test_expect_success \ + 'check unpack without delta' \ + '(cd ../.git && find objects -type f -print) | + while read path + do + cmp $path ../.git/$path || { + echo $path differs. + exit 1 + } + done' +cd $TRASH + +test_expect_success \ + 'pack with delta' \ + 'pwd && + git-pack-objects test-2 <obj-list' + +rm -fr .git2 +mkdir .git2 + +test_expect_success \ + 'unpack with delta' \ + 'GIT_OBJECT_DIRECTORY=.git2/objects && + export GIT_OBJECT_DIRECTORY && + git-init-db && + git-unpack-objects test-2' + +unset GIT_OBJECT_DIRECTORY +cd $TRASH/.git2 +test_expect_success \ + 'check unpack with delta' \ + '(cd ../.git && find objects -type f -print) | + while read path + do + cmp $path ../.git/$path || { + echo $path differs. + exit 1 + } + done' +cd $TRASH + +test_done diff --git a/unpack-objects.c b/unpack-objects.c index 7b175c9a49..a7b8df0792 100644 --- a/unpack-objects.c +++ b/unpack-objects.c @@ -1,5 +1,6 @@ #include "cache.h" #include "object.h" +#include "delta.h" static int dry_run; static int nr_entries; @@ -7,12 +8,14 @@ static const char *base_name; static const char unpack_usage[] = "git-unpack-objects basename"; struct pack_entry { - unsigned int offset; + unsigned int offset; /* network byte order */ unsigned char sha1[20]; }; static void *pack_base; static unsigned long pack_size; +static void *index_base; +static unsigned long index_size; static struct pack_entry **pack_list; @@ -52,13 +55,13 @@ static int sort_by_offset(const void *_a, const void *_b) return o1 < o2 ? -1 : 1; } -static int check_index(void *index, unsigned long idx_size) +static int check_index(void) { - unsigned int *array = index; + unsigned int *array = index_base; unsigned int nr; int i; - if (idx_size < 4*256) + if (index_size < 4*256) return error("index file too small"); nr = 0; for (i = 0; i < 256; i++) { @@ -67,15 +70,16 @@ static int check_index(void *index, unsigned long idx_size) return error("non-monotonic index"); nr = n; } - if (idx_size != 4*256 + nr * 24) { - printf("idx_size=%lu, expected %u (%u)\n", idx_size, 4*256 + nr * 24, nr); + if (index_size != 4*256 + nr * 24) { + printf("index_size=%lu, expected %u (%u)\n", + index_size, 4*256 + nr * 24, nr); return error("wrong index file size"); } nr_entries = nr; pack_list = xmalloc(nr * sizeof(struct pack_entry *)); for (i = 0; i < nr; i++) - pack_list[i] = index + 4*256 + i*24; + pack_list[i] = index_base + 4*256 + i*24; qsort(pack_list, nr, sizeof(*pack_list), sort_by_offset); @@ -83,9 +87,165 @@ static int check_index(void *index, unsigned long idx_size) return 0; } -static void unpack_entry(struct pack_entry *entry) +static int unpack_non_delta_entry(struct pack_entry *entry, + unsigned char *pack) { + int st, kind; unsigned long size; + z_stream stream; + char *buffer; + unsigned char sha1[20]; + char *type_s; + unsigned long offset = ntohl(entry->offset); + + kind = pack[0]; + size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4]; + printf("%s %c %lu\n", sha1_to_hex(entry->sha1), kind, size); + pack += 5; + + buffer = xmalloc(size + 1); + buffer[size] = 0; + memset(&stream, 0, sizeof(stream)); + stream.next_in = pack; + stream.avail_in = pack_size - offset; /* sheesh. */ + stream.next_out = buffer; + stream.avail_out = size; + + inflateInit(&stream); + st = inflate(&stream, Z_FINISH); + inflateEnd(&stream); + if ((st != Z_STREAM_END) || stream.total_out != size) + goto err_finish; + switch (kind) { + case 'C': type_s = "commit"; break; + case 'T': type_s = "tree"; break; + case 'B': type_s = "blob"; break; + default: goto err_finish; + } + if (write_sha1_file(buffer, size, type_s, sha1) < 0) + die("failed to write %s (%s)", + sha1_to_hex(entry->sha1), type_s); + printf("%s %s\n", sha1_to_hex(sha1), type_s); + if (memcmp(sha1, entry->sha1, 20)) + die("resulting %s have wrong SHA1", type_s); + + finish: + st = 0; + free(buffer); + return st; + err_finish: + st = -1; + goto finish; +} + +static int find_pack_entry(unsigned char *sha1, struct pack_entry **ent) +{ + int *level1_ofs = index_base; + int hi = ntohl(level1_ofs[*sha1]); + int lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1])); + void *index = index_base + 4*256; + + do { + int mi = (lo + hi) / 2; + int cmp = memcmp(index + 24 * mi + 4, sha1, 20); + if (!cmp) { + *ent = index + 24 * mi; + return 1; + } + if (cmp < 0) + hi = mi; + else + lo = mi; + } while (lo < hi); + return 0; +} + +/* forward declaration for a mutually recursive function */ +static void unpack_entry(struct pack_entry *); + +static int unpack_delta_entry(struct pack_entry *entry, unsigned char *pack) +{ + void *delta_data, *result, *base; + unsigned long delta_alloc, delta_size, result_size, base_size; + z_stream stream; + int st; + char type[20]; + unsigned char sha1[20]; + + printf("%s D", sha1_to_hex(entry->sha1)); + printf(" %s\n", sha1_to_hex(pack+1)); + + /* pack+1 is the base sha1, unless we have it, we need to + * unpack it first. + */ + if (!has_sha1_file(pack+1)) { + struct pack_entry *base; + if (!find_pack_entry(pack+1, &base)) + die("cannot find delta-pack base object"); + unpack_entry(base); + } + + /* pack+1 thru pack+20 is the base sha1 and + * pack+21 thru unknown number is the delta data. + * we do not even have size of the delta data uncompressed. + * sheesh! + */ + delta_alloc = 1024; + delta_data = xmalloc(delta_alloc); + + memset(&stream, 0, sizeof(stream)); + + stream.next_in = pack + 21; + stream.avail_in = pack_size - ntohl(entry->offset); /* sheesh. */ + stream.next_out = delta_data; + stream.avail_out = delta_alloc; + delta_size = 0; + + inflateInit(&stream); + while (1) { + st = inflate(&stream, Z_FINISH); + if (st == Z_STREAM_END) { + delta_size = stream.total_out; + break; + } + if (st < 0) + break; + + if (delta_alloc <= stream.total_out) { + delta_alloc = (delta_alloc +1024) * 3 / 2; + delta_data = xrealloc(delta_data, delta_alloc); + stream.next_out = delta_data + stream.total_out; + stream.avail_out = delta_alloc - stream.total_out; + } + } + inflateEnd(&stream); + if (st != Z_STREAM_END) { + free(delta_data); + return -1; + } + + base = read_sha1_file(pack+1, type, &base_size); + if (!base) + die("failed to read delta-pack base object"); + result = patch_delta(base, base_size, + delta_data, delta_size, + &result_size); + if (!result) + die("failed to apply delta"); + free(delta_data); + + if (write_sha1_file(result, result_size, type, sha1) < 0) + die("failed to write %s (%s)", + sha1_to_hex(entry->sha1), type); + free(result); + printf("%s %s\n", sha1_to_hex(sha1), type); + if (memcmp(sha1, entry->sha1, 20)) + die("resulting %s have wrong SHA1", type); + return 0; +} + +static void unpack_entry(struct pack_entry *entry) +{ unsigned long offset; unsigned char *pack; @@ -100,12 +260,10 @@ static void unpack_entry(struct pack_entry *entry) offset = pack_size - offset; switch (*pack) { case 'C': case 'T': case 'B': - size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4]; - printf("%s %c %lu\n", sha1_to_hex(entry->sha1), *pack, size); + unpack_non_delta_entry(entry, pack); break; case 'D': - printf("%s D", sha1_to_hex(entry->sha1)); - printf(" %s\n", sha1_to_hex(pack+1)); + unpack_delta_entry(entry, pack); break; default: die("corrupted pack file"); @@ -130,8 +288,6 @@ static void unpack_all(void) int main(int argc, char **argv) { int i; - unsigned long idx_size; - void *index; for (i = 1 ; i < argc; i++) { const char *arg = argv[i]; @@ -149,9 +305,9 @@ int main(int argc, char **argv) } if (!base_name) usage(unpack_usage); - index = map_file("idx", &idx_size); + index_base = map_file("idx", &index_size); pack_base = map_file("pack", &pack_size); - if (check_index(index, idx_size) < 0) + if (check_index() < 0) die("bad index file"); unpack_all(); return 0; |