From 358e07f447e9ed7f87f419e8d027e73c97c00077 Mon Sep 17 00:00:00 2001 From: Campbell Barton Date: Fri, 22 Feb 2019 14:42:22 +1100 Subject: readfile: reduce memory usage at load time Delay loading all DATA sections of the blend file until they're needed. Loading all data-blocks caused high peak memory usage especially with libraries - since a lot of data may exist which isn't used directly. In one test (spring project: 10_010_A.anim.blend), peaked at ~12.5gig, dropping back to ~2.5gig once loaded. With this change peaks memory usage reaches ~2.7gig while loading. Besides this there are some minor gains from not having to read data from the file-system and we can skip an alloc + memcpy reading data written with the same version of Blender. --- source/blender/blenloader/intern/readfile.c | 147 +++++++++++++++++++++++++++- source/blender/blenloader/intern/readfile.h | 1 + 2 files changed, 145 insertions(+), 3 deletions(-) (limited to 'source/blender/blenloader/intern') diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c index faff680cea5..58865f0cd8d 100644 --- a/source/blender/blenloader/intern/readfile.c +++ b/source/blender/blenloader/intern/readfile.c @@ -223,6 +223,11 @@ * (added remark: oh, i thought that was solved? will look at that... (ton). */ +/** + * Delay reading blocks we might not use (especially applies to library linking). + * which keeps large arrays in memory from data-blocks we may not even use. */ +#define USE_BHEAD_READ_ON_DEMAND + /* use GHash for BHead name-based lookups (speeds up linking) */ #define USE_GHASH_BHEAD @@ -253,11 +258,22 @@ static void lib_link_animdata(FileData *fd, ID *id, AnimData *adt); typedef struct BHeadN { struct BHeadN *next, *prev; +#ifdef USE_BHEAD_READ_ON_DEMAND + /** Use to read the data from the file directly into memory as needed. */ + int file_offset; + /** When set, the remainder of this allocation is the data, otherwise it needs to be read. */ + bool has_data; +#endif struct BHead bhead; } BHeadN; #define BHEADN_FROM_BHEAD(bh) ((BHeadN *)POINTER_OFFSET(bh, -offsetof(BHeadN, bhead))) +/* We could change this in the future, for now it's simplest if only data is delayed + * because ID names are used in lookup tables. */ +#define BHEAD_USE_READ_ON_DEMAND(bhead) \ + ((bhead)->code == DATA) + /* this function ensures that reports are printed, * in the case of libraray linking errors this is important! * @@ -802,10 +818,39 @@ static BHeadN *get_bhead(FileData *fd) /* bhead now contains the (converted) bhead structure. Now read * the associated data and put everything in a BHeadN (creative naming !) */ - if (!fd->is_eof) { + if (fd->is_eof) { + /* pass */ + } +#ifdef USE_BHEAD_READ_ON_DEMAND + else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) { + /* Delay reading bhead content. */ + new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead"); + if (new_bhead) { + new_bhead->next = new_bhead->prev = NULL; + new_bhead->file_offset = fd->file_offset; + new_bhead->has_data = false; + new_bhead->bhead = bhead; + int seek_new = fd->seek(fd, bhead.len, SEEK_CUR); + if (seek_new == -1) { + fd->is_eof = true; + MEM_freeN(new_bhead); + new_bhead = NULL; + } + BLI_assert(fd->file_offset == seek_new); + } + else { + fd->is_eof = true; + } + } +#endif + else { new_bhead = MEM_mallocN(sizeof(BHeadN) + bhead.len, "new_bhead"); if (new_bhead) { new_bhead->next = new_bhead->prev = NULL; +#ifdef USE_BHEAD_READ_ON_DEMAND + new_bhead->file_offset = 0; /* don't seek. */ + new_bhead->has_data = true; +#endif new_bhead->bhead = bhead; readsize = fd->read(fd, new_bhead + 1, bhead.len); @@ -887,6 +932,42 @@ BHead *blo_bhead_next(FileData *fd, BHead *thisblock) return bhead; } +#ifdef USE_BHEAD_READ_ON_DEMAND +static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf) +{ + bool success = true; + BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock); + BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0); + int offset_backup = fd->file_offset; + if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) { + success = false; + } + else { + if (fd->read(fd, buf, new_bhead->bhead.len) != new_bhead->bhead.len) { + success = false; + } + } + if (fd->seek(fd, offset_backup, SEEK_SET) == -1) { + success = false; + } + return success; +} + +static BHead *blo_bhead_read_full(FileData *fd, BHead *thisblock) +{ + BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock); + BHeadN *new_bhead_data = MEM_mallocN(sizeof(BHeadN) + new_bhead->bhead.len, "new_bhead"); + new_bhead_data->bhead = new_bhead->bhead; + new_bhead_data->file_offset = new_bhead->file_offset; + new_bhead_data->has_data = true; + if (!blo_bhead_read_data(fd, thisblock, new_bhead_data + 1)) { + MEM_freeN(new_bhead_data); + return NULL; + } + return &new_bhead_data->bhead; +} +#endif /* USE_BHEAD_READ_ON_DEMAND */ + /* Warning! Caller's responsibility to ensure given bhead **is** and ID one! */ const char *blo_bhead_id_name(const FileData *fd, const BHead *bhead) { @@ -1039,6 +1120,12 @@ static int fd_read_gzip_from_file(FileData *filedata, void *buffer, uint size) return (readsize); } +static int fd_seek_gzip_from_file(FileData *filedata, int offset, int whence) +{ + filedata->file_offset = gzseek(filedata->gzfiledes, offset, whence); + return filedata->file_offset; +} + static int fd_read_from_memory(FileData *filedata, void *buffer, uint size) { /* don't read more bytes then there are available in the buffer */ @@ -1166,6 +1253,7 @@ FileData *blo_filedata_from_file(const char *filepath, ReportList *reports) FileData *fd = filedata_new(); fd->gzfiledes = gzfile; fd->read = fd_read_gzip_from_file; + fd->seek = fd_seek_gzip_from_file; /* needed for library_append and read_libraries */ BLI_strncpy(fd->relabase, filepath, sizeof(fd->relabase)); @@ -1306,8 +1394,18 @@ void blo_filedata_free(FileData *fd) fd->buffer = NULL; } - // Free all BHeadN data blocks + /* Free all BHeadN data blocks */ +#ifndef NDEBUG BLI_freelistN(&fd->listbase); +#else + /* Sanity check we're not keeping memory we don't need. */ + LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->listbase) { + if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) { + BLI_assert(new_bhead->has_data == 0); + } + MEM_freeN(new_bhead); + } +#endif if (fd->filesdna) DNA_sdna_free(fd->filesdna); @@ -1926,20 +2024,63 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname) void *temp = NULL; if (bh->len) { +#ifdef USE_BHEAD_READ_ON_DEMAND + BHead *bh_orig = bh; + #endif + /* switch is based on file dna */ - if (bh->SDNAnr && (fd->flags & FD_FLAGS_SWITCH_ENDIAN)) + if (bh->SDNAnr && (fd->flags & FD_FLAGS_SWITCH_ENDIAN)) { +#ifdef USE_BHEAD_READ_ON_DEMAND + if (BHEADN_FROM_BHEAD(bh)->has_data == false) { + bh = blo_bhead_read_full(fd, bh); + if (UNLIKELY(bh == NULL)) { + fd->flags &= ~FD_FLAGS_FILE_OK; + return NULL; + } + } + #endif switch_endian_structs(fd->filesdna, bh); + } if (fd->compflags[bh->SDNAnr] != SDNA_CMP_REMOVED) { if (fd->compflags[bh->SDNAnr] == SDNA_CMP_NOT_EQUAL) { +#ifdef USE_BHEAD_READ_ON_DEMAND + if (BHEADN_FROM_BHEAD(bh)->has_data == false) { + bh = blo_bhead_read_full(fd, bh); + if (UNLIKELY(bh == NULL)) { + fd->flags &= ~FD_FLAGS_FILE_OK; + return NULL; + } + } +#endif temp = DNA_struct_reconstruct(fd->memsdna, fd->filesdna, fd->compflags, bh->SDNAnr, bh->nr, (bh + 1)); } else { /* SDNA_CMP_EQUAL */ temp = MEM_mallocN(bh->len, blockname); +#ifdef USE_BHEAD_READ_ON_DEMAND + if (BHEADN_FROM_BHEAD(bh)->has_data) { + memcpy(temp, (bh + 1), bh->len); + } + else { + /* Instead of allocating the bhead, then copying it, + * read the data from the file directly into the memory. */ + if (UNLIKELY(!blo_bhead_read_data(fd, bh, temp))) { + fd->flags &= ~FD_FLAGS_FILE_OK; + MEM_freeN(temp); + temp = NULL; + } + } +#else memcpy(temp, (bh + 1), bh->len); +#endif } } +#ifdef USE_BHEAD_READ_ON_DEMAND + if (bh_orig != bh) { + MEM_freeN(BHEADN_FROM_BHEAD(bh)); + } +#endif } return temp; diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h index 91dfae4a71d..167d5511a0c 100644 --- a/source/blender/blenloader/intern/readfile.h +++ b/source/blender/blenloader/intern/readfile.h @@ -56,6 +56,7 @@ typedef struct FileData { int buffersize; int file_offset; int (*read)(struct FileData *filedata, void *buffer, unsigned int size); + int (*seek)(struct FileData *filedata, int offset, int whence); /** Variables needed for reading from memory / stream. */ const char *buffer; -- cgit v1.2.3