diff options
author | Shawn O. Pearce <spearce@spearce.org> | 2010-02-01 20:27:35 +0300 |
---|---|---|
committer | Junio C Hamano <gitster@pobox.com> | 2010-02-01 23:09:47 +0300 |
commit | 5eef828bc03649ba11acc05c983d72823c27f0bb (patch) | |
tree | 2554847a2c6f7853bf2b9b8226de1ccca67ce828 /t | |
parent | 63d04a78043c61c52ae437fa6106e7029b466bbf (diff) |
fast-import: Stream very large blobs directly to pack
If a blob is larger than the configured big-file-threshold, instead
of reading it into a single buffer obtained from malloc, stream it
onto the end of the current pack file. Streaming the larger objects
into the pack avoids the 4+ GiB memory footprint that occurs when
fast-import is processing 2+ GiB blobs.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
Diffstat (limited to 't')
-rwxr-xr-x | t/t5705-clone-2gb.sh | 2 | ||||
-rwxr-xr-x | t/t9300-fast-import.sh | 46 |
2 files changed, 47 insertions, 1 deletions
diff --git a/t/t5705-clone-2gb.sh b/t/t5705-clone-2gb.sh index 9f52154cac..adfaae8c5b 100755 --- a/t/t5705-clone-2gb.sh +++ b/t/t5705-clone-2gb.sh @@ -31,7 +31,7 @@ test_expect_success 'setup' ' echo "data 5" && echo ">2gb" && cat commit) | - git fast-import && + git fast-import --big-file-threshold=2 && test ! -f exit-status ' diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh index b49815d108..513db86ad2 100755 --- a/t/t9300-fast-import.sh +++ b/t/t9300-fast-import.sh @@ -1254,4 +1254,50 @@ test_expect_success \ 'Q: verify note for third commit' \ 'git cat-file blob refs/notes/foobar:$commit3 >actual && test_cmp expect actual' +## +## R: very large blobs +## +blobsize=$((2*1024*1024 + 53)) +test-genrandom bar $blobsize >expect +cat >input <<INPUT_END +commit refs/heads/big-file +committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE +data <<COMMIT +R - big file +COMMIT + +M 644 inline big1 +data $blobsize +INPUT_END +cat expect >>input +cat >>input <<INPUT_END +M 644 inline big2 +data $blobsize +INPUT_END +cat expect >>input +echo >>input + +test_expect_success \ + 'R: blob bigger than threshold' \ + 'test_create_repo R && + git --git-dir=R/.git fast-import --big-file-threshold=1 <input' +test_expect_success \ + 'R: verify created pack' \ + ': >verify && + for p in R/.git/objects/pack/*.pack; + do + git verify-pack -v $p >>verify || exit; + done' +test_expect_success \ + 'R: verify written objects' \ + 'git --git-dir=R/.git cat-file blob big-file:big1 >actual && + test_cmp expect actual && + a=$(git --git-dir=R/.git rev-parse big-file:big1) && + b=$(git --git-dir=R/.git rev-parse big-file:big2) && + test $a = $b' +test_expect_success \ + 'R: blob appears only once' \ + 'n=$(grep $a verify | wc -l) && + test 1 = $n' + test_done |