diff options
Diffstat (limited to 'source/blender/blenloader')
18 files changed, 685 insertions, 575 deletions
diff --git a/source/blender/blenloader/BLO_readfile.h b/source/blender/blenloader/BLO_readfile.h index 04e13fbd1d6..dbdb181281a 100644 --- a/source/blender/blenloader/BLO_readfile.h +++ b/source/blender/blenloader/BLO_readfile.h @@ -108,6 +108,9 @@ typedef struct BlendFileReadReport { * during this file read. */ int missing_libraries; int missing_linked_id; + /* Some sub-categories of the above `missing_linked_id` counter. */ + int missing_obdata; + int missing_obproxies; /* Number of root override IDs that were resynced. */ int resynced_lib_overrides; } count; @@ -115,6 +118,7 @@ typedef struct BlendFileReadReport { /* Number of libraries which had overrides that needed to be resynced, and a single linked list * of those. */ int resynced_lib_overrides_libraries_count; + bool do_resynced_lib_overrides_libraries_list; struct LinkNode *resynced_lib_overrides_libraries; } BlendFileReadReport; diff --git a/source/blender/blenloader/BLO_undofile.h b/source/blender/blenloader/BLO_undofile.h index fc41a6e832f..4e240e2462b 100644 --- a/source/blender/blenloader/BLO_undofile.h +++ b/source/blender/blenloader/BLO_undofile.h @@ -24,6 +24,8 @@ * \ingroup blenloader */ +#include "BLI_filereader.h" + struct GHash; struct Scene; @@ -65,6 +67,16 @@ typedef struct MemFileUndoData { size_t undo_size; } MemFileUndoData; +/* FileReader-compatible wrapper for reading MemFiles */ +typedef struct { + FileReader reader; + + MemFile *memfile; + int undo_direction; + + bool memchunk_identical; +} UndoReader; + /* actually only used writefile.c */ void BLO_memfile_write_init(MemFileWriteData *mem_data, @@ -84,3 +96,5 @@ extern struct Main *BLO_memfile_main_get(struct MemFile *memfile, struct Main *bmain, struct Scene **r_scene); extern bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename); + +FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction); diff --git a/source/blender/blenloader/CMakeLists.txt b/source/blender/blenloader/CMakeLists.txt index f5baf0dcb83..89631588ed0 100644 --- a/source/blender/blenloader/CMakeLists.txt +++ b/source/blender/blenloader/CMakeLists.txt @@ -42,7 +42,7 @@ set(INC ) set(INC_SYS - ${ZLIB_INCLUDE_DIRS} + ${ZSTD_INCLUDE_DIRS} ) set(SRC diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c index e48c305fc4b..49c3497f996 100644 --- a/source/blender/blenloader/intern/readfile.c +++ b/source/blender/blenloader/intern/readfile.c @@ -21,8 +21,6 @@ * \ingroup blenloader */ -#include "zlib.h" - #include <ctype.h> /* for isdigit. */ #include <fcntl.h> /* for open flags (O_BINARY, O_RDONLY). */ #include <limits.h> @@ -71,7 +69,6 @@ #include "BLI_math.h" #include "BLI_memarena.h" #include "BLI_mempool.h" -#include "BLI_mmap.h" #include "BLI_threads.h" #include "PIL_time.h" @@ -788,7 +785,7 @@ static BHeadN *get_bhead(FileData *fd) */ if (fd->flags & FD_FLAGS_FILE_POINTSIZE_IS_4) { bhead4.code = DATA; - readsize = fd->read(fd, &bhead4, sizeof(bhead4), NULL); + readsize = fd->file->read(fd->file, &bhead4, sizeof(bhead4)); if (readsize == sizeof(bhead4) || bhead4.code == ENDB) { if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) { @@ -811,7 +808,7 @@ static BHeadN *get_bhead(FileData *fd) } else { bhead8.code = DATA; - readsize = fd->read(fd, &bhead8, sizeof(bhead8), NULL); + readsize = fd->file->read(fd->file, &bhead8, sizeof(bhead8)); if (readsize == sizeof(bhead8) || bhead8.code == ENDB) { if (fd->flags & FD_FLAGS_SWITCH_ENDIAN) { @@ -845,22 +842,22 @@ static BHeadN *get_bhead(FileData *fd) /* pass */ } #ifdef USE_BHEAD_READ_ON_DEMAND - else if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) { + else if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&bhead)) { /* Delay reading bhead content. */ new_bhead = MEM_mallocN(sizeof(BHeadN), "new_bhead"); if (new_bhead) { new_bhead->next = new_bhead->prev = NULL; - new_bhead->file_offset = fd->file_offset; + new_bhead->file_offset = fd->file->offset; new_bhead->has_data = false; new_bhead->is_memchunk_identical = false; new_bhead->bhead = bhead; - off64_t seek_new = fd->seek(fd, bhead.len, SEEK_CUR); + off64_t seek_new = fd->file->seek(fd->file, bhead.len, SEEK_CUR); if (seek_new == -1) { fd->is_eof = true; MEM_freeN(new_bhead); new_bhead = NULL; } - BLI_assert(fd->file_offset == seek_new); + BLI_assert(fd->file->offset == seek_new); } else { fd->is_eof = true; @@ -878,14 +875,17 @@ static BHeadN *get_bhead(FileData *fd) new_bhead->is_memchunk_identical = false; new_bhead->bhead = bhead; - readsize = fd->read( - fd, new_bhead + 1, (size_t)bhead.len, &new_bhead->is_memchunk_identical); + readsize = fd->file->read(fd->file, new_bhead + 1, (size_t)bhead.len); - if (readsize != (ssize_t)bhead.len) { + if (readsize != bhead.len) { fd->is_eof = true; MEM_freeN(new_bhead); new_bhead = NULL; } + + if (fd->flags & FD_FLAGS_IS_MEMFILE) { + new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical; + } } else { fd->is_eof = true; @@ -964,17 +964,19 @@ static bool blo_bhead_read_data(FileData *fd, BHead *thisblock, void *buf) bool success = true; BHeadN *new_bhead = BHEADN_FROM_BHEAD(thisblock); BLI_assert(new_bhead->has_data == false && new_bhead->file_offset != 0); - off64_t offset_backup = fd->file_offset; - if (UNLIKELY(fd->seek(fd, new_bhead->file_offset, SEEK_SET) == -1)) { + off64_t offset_backup = fd->file->offset; + if (UNLIKELY(fd->file->seek(fd->file, new_bhead->file_offset, SEEK_SET) == -1)) { success = false; } else { - if (fd->read(fd, buf, (size_t)new_bhead->bhead.len, &new_bhead->is_memchunk_identical) != - (ssize_t)new_bhead->bhead.len) { + if (fd->file->read(fd->file, buf, (size_t)new_bhead->bhead.len) != new_bhead->bhead.len) { success = false; } + if (fd->flags & FD_FLAGS_IS_MEMFILE) { + new_bhead->is_memchunk_identical = ((UndoReader *)fd->file)->memchunk_identical; + } } - if (fd->seek(fd, offset_backup, SEEK_SET) == -1) { + if (fd->file->seek(fd->file, offset_backup, SEEK_SET) == -1) { success = false; } return success; @@ -1017,7 +1019,7 @@ static void decode_blender_header(FileData *fd) ssize_t readsize; /* read in the header data */ - readsize = fd->read(fd, header, sizeof(header), NULL); + readsize = fd->file->read(fd->file, header, sizeof(header)); if (readsize == sizeof(header) && STREQLEN(header, "BLENDER", 7) && ELEM(header[7], '_', '-') && ELEM(header[8], 'v', 'V') && @@ -1147,210 +1149,12 @@ static int *read_file_thumbnail(FileData *fd) /** \} */ -/* -------------------------------------------------------------------- */ -/** \name File Data API - * \{ */ - -/* Regular file reading. */ - -static ssize_t fd_read_data_from_file(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - ssize_t readsize = read(filedata->filedes, buffer, size); - - if (readsize < 0) { - readsize = EOF; - } - else { - filedata->file_offset += readsize; - } - - return readsize; -} - -static off64_t fd_seek_data_from_file(FileData *filedata, off64_t offset, int whence) -{ - filedata->file_offset = BLI_lseek(filedata->filedes, offset, whence); - return filedata->file_offset; -} - -/* GZip file reading. */ - -static ssize_t fd_read_gzip_from_file(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - BLI_assert(size <= INT_MAX); - - ssize_t readsize = gzread(filedata->gzfiledes, buffer, (uint)size); - - if (readsize < 0) { - readsize = EOF; - } - else { - filedata->file_offset += readsize; - } - - return readsize; -} - -/* Memory reading. */ - -static ssize_t fd_read_from_memory(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - /* don't read more bytes than there are available in the buffer */ - ssize_t readsize = (ssize_t)MIN2(size, filedata->buffersize - (size_t)filedata->file_offset); - - memcpy(buffer, filedata->buffer + filedata->file_offset, (size_t)readsize); - filedata->file_offset += readsize; - - return readsize; -} - -/* Memory-mapped file reading. - * By using mmap(), we can map a file so that it can be treated like normal memory, - * meaning that we can just read from it with memcpy() etc. - * This avoids system call overhead and can significantly speed up file loading. - */ - -static ssize_t fd_read_from_mmap(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - /* don't read more bytes than there are available in the buffer */ - size_t readsize = MIN2(size, (size_t)(filedata->buffersize - filedata->file_offset)); - - if (!BLI_mmap_read(filedata->mmap_file, buffer, filedata->file_offset, readsize)) { - return 0; - } - - filedata->file_offset += readsize; - - return readsize; -} - -static off64_t fd_seek_from_mmap(FileData *filedata, off64_t offset, int whence) -{ - off64_t new_pos; - if (whence == SEEK_CUR) { - new_pos = filedata->file_offset + offset; - } - else if (whence == SEEK_SET) { - new_pos = offset; - } - else if (whence == SEEK_END) { - new_pos = filedata->buffersize + offset; - } - else { - return -1; - } - - if (new_pos < 0 || new_pos > filedata->buffersize) { - return -1; - } - - filedata->file_offset = new_pos; - return filedata->file_offset; -} - -/* MemFile reading. */ - -static ssize_t fd_read_from_memfile(FileData *filedata, - void *buffer, - size_t size, - bool *r_is_memchunck_identical) -{ - static size_t seek = SIZE_MAX; /* the current position */ - static size_t offset = 0; /* size of previous chunks */ - static MemFileChunk *chunk = NULL; - size_t chunkoffset, readsize, totread; - - if (r_is_memchunck_identical != NULL) { - *r_is_memchunck_identical = true; - } - - if (size == 0) { - return 0; - } - - if (seek != (size_t)filedata->file_offset) { - chunk = filedata->memfile->chunks.first; - seek = 0; - - while (chunk) { - if (seek + chunk->size > (size_t)filedata->file_offset) { - break; - } - seek += chunk->size; - chunk = chunk->next; - } - offset = seek; - seek = (size_t)filedata->file_offset; - } - - if (chunk) { - totread = 0; - - do { - /* first check if it's on the end if current chunk */ - if (seek - offset == chunk->size) { - offset += chunk->size; - chunk = chunk->next; - } - - /* debug, should never happen */ - if (chunk == NULL) { - CLOG_ERROR(&LOG, "Illegal read, got a NULL chunk"); - return 0; - } - - chunkoffset = seek - offset; - readsize = size - totread; - - /* data can be spread over multiple chunks, so clamp size - * to within this chunk, and then it will read further in - * the next chunk */ - if (chunkoffset + readsize > chunk->size) { - readsize = chunk->size - chunkoffset; - } - - memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize); - totread += readsize; - filedata->file_offset += readsize; - seek += readsize; - if (r_is_memchunck_identical != NULL) { - /* `is_identical` of current chunk represents whether it changed compared to previous undo - * step. this is fine in redo case, but not in undo case, where we need an extra flag - * defined when saving the next (future) step after the one we want to restore, as we are - * supposed to 'come from' that future undo step, and not the one before current one. */ - *r_is_memchunck_identical &= filedata->undo_direction == STEP_REDO ? - chunk->is_identical : - chunk->is_identical_future; - } - } while (totread < size); - - return (ssize_t)totread; - } - - return 0; -} - static FileData *filedata_new(BlendFileReadReport *reports) { BLI_assert(reports != NULL); FileData *fd = MEM_callocN(sizeof(FileData), "FileData"); - fd->filedes = -1; - fd->gzfiledes = NULL; - fd->memsdna = DNA_sdna_current_get(); fd->datamap = oldnewmap_new(); @@ -1387,78 +1191,66 @@ static FileData *blo_decode_and_check(FileData *fd, ReportList *reports) static FileData *blo_filedata_from_file_descriptor(const char *filepath, BlendFileReadReport *reports, - int file) + int filedes) { - FileDataReadFn *read_fn = NULL; - FileDataSeekFn *seek_fn = NULL; /* Optional. */ - size_t buffersize = 0; - BLI_mmap_file *mmap_file = NULL; - - gzFile gzfile = (gzFile)Z_NULL; - char header[7]; + FileReader *rawfile = BLI_filereader_new_file(filedes); + FileReader *file = NULL; - /* Regular file. */ errno = 0; - if (read(file, header, sizeof(header)) != sizeof(header)) { + /* If opening the file failed or we can't read the header, give up. */ + if (rawfile == NULL || rawfile->read(rawfile, header, sizeof(header)) != sizeof(header)) { BKE_reportf(reports->reports, RPT_WARNING, "Unable to read '%s': %s", filepath, errno ? strerror(errno) : TIP_("insufficient content")); + if (rawfile) { + rawfile->close(rawfile); + } + else { + close(filedes); + } return NULL; } - /* Regular file. */ - if (memcmp(header, "BLENDER", sizeof(header)) == 0) { - read_fn = fd_read_data_from_file; - seek_fn = fd_seek_data_from_file; + /* Rewind the file after reading the header. */ + rawfile->seek(rawfile, 0, SEEK_SET); - mmap_file = BLI_mmap_open(file); - if (mmap_file != NULL) { - read_fn = fd_read_from_mmap; - seek_fn = fd_seek_from_mmap; - buffersize = BLI_lseek(file, 0, SEEK_END); + /* Check if we have a regular file. */ + if (memcmp(header, "BLENDER", sizeof(header)) == 0) { + /* Try opening the file with memory-mapped IO. */ + file = BLI_filereader_new_mmap(filedes); + if (file == NULL) { + /* mmap failed, so just keep using rawfile. */ + file = rawfile; + rawfile = NULL; } } - - BLI_lseek(file, 0, SEEK_SET); - - /* Gzip file. */ - errno = 0; - if ((read_fn == NULL) && - /* Check header magic. */ - (header[0] == 0x1f && header[1] == 0x8b)) { - gzfile = BLI_gzopen(filepath, "rb"); - if (gzfile == (gzFile)Z_NULL) { - BKE_reportf(reports->reports, - RPT_WARNING, - "Unable to open '%s': %s", - filepath, - errno ? strerror(errno) : TIP_("unknown error reading file")); - return NULL; + else if (BLI_file_magic_is_gzip(header)) { + file = BLI_filereader_new_gzip(rawfile); + if (file != NULL) { + rawfile = NULL; /* The Gzip FileReader takes ownership of `rawfile`. */ + } + } + else if (BLI_file_magic_is_zstd(header)) { + file = BLI_filereader_new_zstd(rawfile); + if (file != NULL) { + rawfile = NULL; /* The Zstd FileReader takes ownership of `rawfile`. */ } - - /* 'seek_fn' is too slow for gzip, don't set it. */ - read_fn = fd_read_gzip_from_file; - /* Caller must close. */ - file = -1; } - if (read_fn == NULL) { + /* Clean up `rawfile` if it wasn't taken over. */ + if (rawfile != NULL) { + rawfile->close(rawfile); + } + if (file == NULL) { BKE_reportf(reports->reports, RPT_WARNING, "Unrecognized file format '%s'", filepath); return NULL; } FileData *fd = filedata_new(reports); - - fd->filedes = file; - fd->gzfiledes = gzfile; - - fd->read = read_fn; - fd->seek = seek_fn; - fd->mmap_file = mmap_file; - fd->buffersize = buffersize; + fd->file = file; return fd; } @@ -1475,11 +1267,7 @@ static FileData *blo_filedata_from_file_open(const char *filepath, BlendFileRead errno ? strerror(errno) : TIP_("unknown error reading file")); return NULL; } - FileData *fd = blo_filedata_from_file_descriptor(filepath, reports, file); - if ((fd == NULL) || (fd->filedes == -1)) { - close(file); - } - return fd; + return blo_filedata_from_file_descriptor(filepath, reports, file); } /* cannot be called with relative paths anymore! */ @@ -1513,50 +1301,6 @@ static FileData *blo_filedata_from_file_minimal(const char *filepath) return NULL; } -static ssize_t fd_read_gzip_from_memory(FileData *filedata, - void *buffer, - size_t size, - bool *UNUSED(r_is_memchunck_identical)) -{ - int err; - - filedata->strm.next_out = (Bytef *)buffer; - filedata->strm.avail_out = (uint)size; - - /* Inflate another chunk. */ - err = inflate(&filedata->strm, Z_SYNC_FLUSH); - - if (err == Z_STREAM_END) { - return 0; - } - if (err != Z_OK) { - CLOG_ERROR(&LOG, "ZLib error (code %d)", err); - return 0; - } - - filedata->file_offset += size; - - return (ssize_t)size; -} - -static int fd_read_gzip_from_memory_init(FileData *fd) -{ - - fd->strm.next_in = (Bytef *)fd->buffer; - fd->strm.avail_in = fd->buffersize; - fd->strm.total_out = 0; - fd->strm.zalloc = Z_NULL; - fd->strm.zfree = Z_NULL; - - if (inflateInit2(&fd->strm, (16 + MAX_WBITS)) != Z_OK) { - return 0; - } - - fd->read = fd_read_gzip_from_memory; - - return 1; -} - FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadReport *reports) { if (!mem || memsize < SIZEOFBLENDERHEADER) { @@ -1565,24 +1309,24 @@ FileData *blo_filedata_from_memory(const void *mem, int memsize, BlendFileReadRe return NULL; } - FileData *fd = filedata_new(reports); - const char *cp = mem; - - fd->buffer = mem; - fd->buffersize = memsize; + FileReader *mem_file = BLI_filereader_new_memory(mem, memsize); + FileReader *file = mem_file; - /* test if gzip */ - if (cp[0] == 0x1f && cp[1] == 0x8b) { - if (0 == fd_read_gzip_from_memory_init(fd)) { - blo_filedata_free(fd); - return NULL; - } + if (BLI_file_magic_is_gzip(mem)) { + file = BLI_filereader_new_gzip(mem_file); } - else { - fd->read = fd_read_from_memory; + else if (BLI_file_magic_is_zstd(mem)) { + file = BLI_filereader_new_zstd(mem_file); } - fd->flags |= FD_FLAGS_NOT_MY_BUFFER; + if (file == NULL) { + /* Compression initialization failed. */ + mem_file->close(mem_file); + return NULL; + } + + FileData *fd = filedata_new(reports); + fd->file = file; return blo_decode_and_check(fd, reports->reports); } @@ -1597,11 +1341,9 @@ FileData *blo_filedata_from_memfile(MemFile *memfile, } FileData *fd = filedata_new(reports); - fd->memfile = memfile; + fd->file = BLO_memfile_new_filereader(memfile, params->undo_direction); fd->undo_direction = params->undo_direction; - - fd->read = fd_read_from_memfile; - fd->flags |= FD_FLAGS_NOT_MY_BUFFER; + fd->flags |= FD_FLAGS_IS_MEMFILE; return blo_decode_and_check(fd, reports->reports); } @@ -1609,30 +1351,7 @@ FileData *blo_filedata_from_memfile(MemFile *memfile, void blo_filedata_free(FileData *fd) { if (fd) { - if (fd->filedes != -1) { - close(fd->filedes); - } - - if (fd->gzfiledes != NULL) { - gzclose(fd->gzfiledes); - } - - if (fd->strm.next_in) { - int err = inflateEnd(&fd->strm); - if (err != Z_OK) { - CLOG_ERROR(&LOG, "Close gzip stream error (code %d)", err); - } - } - - if (fd->buffer && !(fd->flags & FD_FLAGS_NOT_MY_BUFFER)) { - MEM_freeN((void *)fd->buffer); - fd->buffer = NULL; - } - - if (fd->mmap_file) { - BLI_mmap_free(fd->mmap_file); - fd->mmap_file = NULL; - } + fd->file->close(fd->file); /* Free all BHeadN data blocks */ #ifndef NDEBUG @@ -1640,7 +1359,7 @@ void blo_filedata_free(FileData *fd) #else /* Sanity check we're not keeping memory we don't need. */ LISTBASE_FOREACH_MUTABLE (BHeadN *, new_bhead, &fd->bhead_list) { - if (fd->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) { + if (fd->file->seek != NULL && BHEAD_USE_READ_ON_DEMAND(&new_bhead->bhead)) { BLI_assert(new_bhead->has_data == 0); } MEM_freeN(new_bhead); @@ -2096,7 +1815,7 @@ static void blo_cache_storage_entry_clear_in_old(ID *UNUSED(id), void blo_cache_storage_init(FileData *fd, Main *bmain) { - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { BLI_assert(fd->cache_storage == NULL); fd->cache_storage = MEM_mallocN(sizeof(*fd->cache_storage), __func__); fd->cache_storage->memarena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__); @@ -2261,7 +1980,7 @@ static void *read_struct(FileData *fd, BHead *bh, const char *blockname) * undo since DNA must match. */ static const void *peek_struct_undo(FileData *fd, BHead *bhead) { - BLI_assert(fd->memfile != NULL); + BLI_assert(fd->flags & FD_FLAGS_IS_MEMFILE); UNUSED_VARS_NDEBUG(fd); return (bhead->len) ? (const void *)(bhead + 1) : NULL; } @@ -3679,7 +3398,7 @@ static BHead *read_libblock(FileData *fd, * When datablocks are changed but still exist, we restore them at the old * address and inherit recalc flags for the dependency graph. */ ID *id_old = NULL; - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { if (read_libblock_undo_restore(fd, main, bhead, tag, &id_old)) { if (r_id) { *r_id = id_old; @@ -3980,13 +3699,14 @@ static void lib_link_all(FileData *fd, Main *bmain) continue; } - if (fd->memfile != NULL && GS(id->name) == ID_WM) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) && GS(id->name) == ID_WM) { /* No load UI for undo memfiles. * Only WM currently, SCR needs it still (see below), and so does WS? */ continue; } - if (fd->memfile != NULL && do_partial_undo && (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) && do_partial_undo && + (id->tag & LIB_TAG_UNDO_OLD_ID_REUSED) != 0) { /* This ID has been re-used from 'old' bmain. Since it was therefore unchanged across * current undo step, and old IDs re-use their old memory address, we do not need to liblink * it at all. */ @@ -4165,7 +3885,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) BlendFileData *bfd; ListBase mainlist = {NULL, NULL}; - if (fd->memfile != NULL) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { CLOG_INFO(&LOG_UNDO, 2, "UNDO: read step"); } @@ -4256,7 +3976,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) } /* do before read_libraries, but skip undo case */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { if ((fd->skip_flags & BLO_READ_SKIP_DATA) == 0) { do_versions(fd, NULL, bfd->main); } @@ -4278,7 +3998,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) fd->reports->duration.libraries = PIL_check_seconds_timer() - fd->reports->duration.libraries; /* Skip in undo case. */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { /* Note that we can't recompute user-counts at this point in undo case, we play too much with * IDs from different memory realms, and Main database is not in a fully valid state yet. */ @@ -4311,7 +4031,7 @@ BlendFileData *blo_read_file_internal(FileData *fd, const char *filepath) /* Now that all our data-blocks are loaded, * we can re-generate overrides from their references. */ - if (fd->memfile == NULL) { + if ((fd->flags & FD_FLAGS_IS_MEMFILE) == 0) { /* Do not apply in undo case! */ fd->reports->duration.lib_overrides = PIL_check_seconds_timer(); @@ -4391,7 +4111,7 @@ static void sort_bhead_old_map(FileData *fd) static BHead *find_previous_lib(FileData *fd, BHead *bhead) { /* Skip library data-blocks in undo, see comment in read_libblock. */ - if (fd->memfile) { + if (fd->flags & FD_FLAGS_IS_MEMFILE) { return NULL; } @@ -5850,7 +5570,7 @@ void BLO_read_pointer_array(BlendDataReader *reader, void **ptr_p) bool BLO_read_data_is_undo(BlendDataReader *reader) { - return reader->fd->memfile != NULL; + return (reader->fd->flags & FD_FLAGS_IS_MEMFILE); } void BLO_read_data_globmap_add(BlendDataReader *reader, void *oldaddr, void *newaddr) @@ -5870,7 +5590,7 @@ BlendFileReadReport *BLO_read_data_reports(BlendDataReader *reader) bool BLO_read_lib_is_undo(BlendLibReader *reader) { - return reader->fd->memfile != NULL; + return (reader->fd->flags & FD_FLAGS_IS_MEMFILE); } Main *BLO_read_lib_get_main(BlendLibReader *reader) diff --git a/source/blender/blenloader/intern/readfile.h b/source/blender/blenloader/intern/readfile.h index b04043f9641..beeed8e45ae 100644 --- a/source/blender/blenloader/intern/readfile.h +++ b/source/blender/blenloader/intern/readfile.h @@ -28,10 +28,10 @@ # include "BLI_winstuff.h" #endif +#include "BLI_filereader.h" #include "DNA_sdna_types.h" #include "DNA_space_types.h" #include "DNA_windowmanager_types.h" /* for ReportType */ -#include "zlib.h" struct BLI_mmap_file; struct BLOCacheStorage; @@ -50,7 +50,7 @@ enum eFileDataFlag { FD_FLAGS_FILE_POINTSIZE_IS_4 = 1 << 1, FD_FLAGS_POINTSIZE_DIFFERS = 1 << 2, FD_FLAGS_FILE_OK = 1 << 3, - FD_FLAGS_NOT_MY_BUFFER = 1 << 4, + FD_FLAGS_IS_MEMFILE = 1 << 4, /* XXX Unused in practice (checked once but never set). */ FD_FLAGS_NOT_MY_LIBMAP = 1 << 5, }; @@ -60,44 +60,18 @@ enum eFileDataFlag { # pragma GCC poison off_t #endif -#if defined(_MSC_VER) || defined(__APPLE__) || defined(__HAIKU__) || defined(__NetBSD__) -typedef int64_t off64_t; -#endif - -typedef ssize_t(FileDataReadFn)(struct FileData *filedata, - void *buffer, - size_t size, - bool *r_is_memchunk_identical); -typedef off64_t(FileDataSeekFn)(struct FileData *filedata, off64_t offset, int whence); - typedef struct FileData { /** Linked list of BHeadN's. */ ListBase bhead_list; enum eFileDataFlag flags; bool is_eof; - size_t buffersize; - off64_t file_offset; - FileDataReadFn *read; - FileDataSeekFn *seek; + FileReader *file; - /** Regular file reading. */ - int filedes; - - /** Variables needed for reading from memory / stream / memory-mapped files. */ - const char *buffer; - struct BLI_mmap_file *mmap_file; - /** Variables needed for reading from memfile (undo). */ - struct MemFile *memfile; /** Whether we are undoing (< 0) or redoing (> 0), used to choose which 'unchanged' flag to use * to detect unchanged data from memfile. */ int undo_direction; /* eUndoStepDir */ - /** Variables needed for reading from file. */ - gzFile gzfiledes; - /** Gzip stream for memory decompression. */ - z_stream strm; - /** Now only in use for library appending. */ char relabase[FILE_MAX]; diff --git a/source/blender/blenloader/intern/undofile.c b/source/blender/blenloader/intern/undofile.c index 2eeeac2e8d7..62072cf7df5 100644 --- a/source/blender/blenloader/intern/undofile.c +++ b/source/blender/blenloader/intern/undofile.c @@ -48,6 +48,7 @@ #include "BKE_lib_id.h" #include "BKE_main.h" +#include "BKE_undo_system.h" /* keep last */ #include "BLI_strict_flags.h" @@ -273,3 +274,97 @@ bool BLO_memfile_write_file(struct MemFile *memfile, const char *filename) } return true; } + +static ssize_t undo_read(FileReader *reader, void *buffer, size_t size) +{ + UndoReader *undo = (UndoReader *)reader; + + static size_t seek = SIZE_MAX; /* The current position. */ + static size_t offset = 0; /* Size of previous chunks. */ + static MemFileChunk *chunk = NULL; + size_t chunkoffset, readsize, totread; + + undo->memchunk_identical = true; + + if (size == 0) { + return 0; + } + + if (seek != (size_t)undo->reader.offset) { + chunk = undo->memfile->chunks.first; + seek = 0; + + while (chunk) { + if (seek + chunk->size > (size_t)undo->reader.offset) { + break; + } + seek += chunk->size; + chunk = chunk->next; + } + offset = seek; + seek = (size_t)undo->reader.offset; + } + + if (chunk) { + totread = 0; + + do { + /* First check if it's on the end if current chunk. */ + if (seek - offset == chunk->size) { + offset += chunk->size; + chunk = chunk->next; + } + + /* Debug, should never happen. */ + if (chunk == NULL) { + printf("illegal read, chunk zero\n"); + return 0; + } + + chunkoffset = seek - offset; + readsize = size - totread; + + /* Data can be spread over multiple chunks, so clamp size + * to within this chunk, and then it will read further in + * the next chunk. */ + if (chunkoffset + readsize > chunk->size) { + readsize = chunk->size - chunkoffset; + } + + memcpy(POINTER_OFFSET(buffer, totread), chunk->buf + chunkoffset, readsize); + totread += readsize; + undo->reader.offset += (off64_t)readsize; + seek += readsize; + + /* `is_identical` of current chunk represents whether it changed compared to previous undo + * step. this is fine in redo case, but not in undo case, where we need an extra flag + * defined when saving the next (future) step after the one we want to restore, as we are + * supposed to 'come from' that future undo step, and not the one before current one. */ + undo->memchunk_identical &= undo->undo_direction == STEP_REDO ? chunk->is_identical : + chunk->is_identical_future; + } while (totread < size); + + return (ssize_t)totread; + } + + return 0; +} + +static void undo_close(FileReader *reader) +{ + MEM_freeN(reader); +} + +FileReader *BLO_memfile_new_filereader(MemFile *memfile, int undo_direction) +{ + UndoReader *undo = MEM_callocN(sizeof(UndoReader), __func__); + + undo->memfile = memfile; + undo->undo_direction = undo_direction; + + undo->reader.read = undo_read; + undo->reader.seek = NULL; + undo->reader.close = undo_close; + + return (FileReader *)undo; +} diff --git a/source/blender/blenloader/intern/versioning_250.c b/source/blender/blenloader/intern/versioning_250.c index e56c1995363..436645c2241 100644 --- a/source/blender/blenloader/intern/versioning_250.c +++ b/source/blender/blenloader/intern/versioning_250.c @@ -23,8 +23,7 @@ #else # include "BLI_winstuff.h" # include "winsock2.h" -# include <io.h> /* for open close read */ -# include <zlib.h> /* odd include order-issue */ +# include <io.h> /* for open close read */ #endif /* allow readfile to use deprecated functionality */ diff --git a/source/blender/blenloader/intern/versioning_260.c b/source/blender/blenloader/intern/versioning_260.c index 858f5d85a90..7c644fa3b55 100644 --- a/source/blender/blenloader/intern/versioning_260.c +++ b/source/blender/blenloader/intern/versioning_260.c @@ -2390,7 +2390,7 @@ void blo_do_versions_260(FileData *fd, Library *UNUSED(lib), Main *bmain) for (md = ob->modifiers.first; md; md = md->next) { if (md->type == eModifierType_Triangulate) { TriangulateModifierData *tmd = (TriangulateModifierData *)md; - if ((tmd->flag & MOD_TRIANGULATE_BEAUTY)) { + if (tmd->flag & MOD_TRIANGULATE_BEAUTY) { tmd->quad_method = MOD_TRIANGULATE_QUAD_BEAUTY; tmd->ngon_method = MOD_TRIANGULATE_NGON_BEAUTY; } diff --git a/source/blender/blenloader/intern/versioning_280.c b/source/blender/blenloader/intern/versioning_280.c index af05c4b902f..9d65488e8d4 100644 --- a/source/blender/blenloader/intern/versioning_280.c +++ b/source/blender/blenloader/intern/versioning_280.c @@ -111,40 +111,13 @@ #include "BLO_readfile.h" #include "readfile.h" +#include "versioning_common.h" + #include "MEM_guardedalloc.h" /* Make preferences read-only, use versioning_userdef.c. */ #define U (*((const UserDef *)&U)) -/** - * Rename if the ID doesn't exist. - */ -static ID *rename_id_for_versioning(Main *bmain, - const short id_type, - const char *name_src, - const char *name_dst) -{ - /* We can ignore libraries */ - ListBase *lb = which_libbase(bmain, id_type); - ID *id = NULL; - LISTBASE_FOREACH (ID *, idtest, lb) { - if (idtest->lib == NULL) { - if (STREQ(idtest->name + 2, name_src)) { - id = idtest; - } - if (STREQ(idtest->name + 2, name_dst)) { - return NULL; - } - } - } - if (id != NULL) { - BLI_strncpy(id->name + 2, name_dst, sizeof(id->name) - 2); - /* We know it's unique, this just sorts. */ - BLI_libblock_ensure_unique_name(bmain, id->name); - } - return id; -} - static bScreen *screen_parent_find(const bScreen *screen) { /* Can avoid lookup if screen state isn't maximized/full @@ -348,7 +321,7 @@ static void do_version_layer_collection_post(ViewLayer *view_layer, lc->flag |= LAYER_COLLECTION_EXCLUDE; } if (enabled && !selectable) { - lc->collection->flag |= COLLECTION_RESTRICT_SELECT; + lc->collection->flag |= COLLECTION_HIDE_SELECT; } } @@ -477,7 +450,7 @@ static void do_version_layers_to_collections(Main *bmain, Scene *scene) collections[layer] = collection; if (!(scene->lay & (1 << layer))) { - collection->flag |= COLLECTION_RESTRICT_VIEWPORT | COLLECTION_RESTRICT_RENDER; + collection->flag |= COLLECTION_HIDE_VIEWPORT | COLLECTION_HIDE_RENDER; } } @@ -1225,7 +1198,7 @@ void do_versions_after_linking_280(Main *bmain, ReportList *UNUSED(reports)) /* Add fake user for all existing groups. */ id_fake_user_set(&collection->id); - if (collection->flag & (COLLECTION_RESTRICT_VIEWPORT | COLLECTION_RESTRICT_RENDER)) { + if (collection->flag & (COLLECTION_HIDE_VIEWPORT | COLLECTION_HIDE_RENDER)) { continue; } @@ -1256,8 +1229,7 @@ void do_versions_after_linking_280(Main *bmain, ReportList *UNUSED(reports)) char name[MAX_ID_NAME]; BLI_snprintf(name, sizeof(name), DATA_("Hidden %d"), coll_idx + 1); *collection_hidden = BKE_collection_add(bmain, collection, name); - (*collection_hidden)->flag |= COLLECTION_RESTRICT_VIEWPORT | - COLLECTION_RESTRICT_RENDER; + (*collection_hidden)->flag |= COLLECTION_HIDE_VIEWPORT | COLLECTION_HIDE_RENDER; } BKE_collection_object_add(bmain, *collection_hidden, ob); @@ -1679,32 +1651,32 @@ void do_versions_after_linking_280(Main *bmain, ReportList *UNUSED(reports)) Brush *brush; Material *ma; /* Pen Soft brush. */ - brush = (Brush *)rename_id_for_versioning(bmain, ID_BR, "Draw Soft", "Pencil Soft"); + brush = (Brush *)do_versions_rename_id(bmain, ID_BR, "Draw Soft", "Pencil Soft"); if (brush) { brush->gpencil_settings->icon_id = GP_BRUSH_ICON_PEN; } - rename_id_for_versioning(bmain, ID_BR, "Draw Pencil", "Pencil"); - rename_id_for_versioning(bmain, ID_BR, "Draw Pen", "Pen"); - rename_id_for_versioning(bmain, ID_BR, "Draw Ink", "Ink Pen"); - rename_id_for_versioning(bmain, ID_BR, "Draw Noise", "Ink Pen Rough"); - rename_id_for_versioning(bmain, ID_BR, "Draw Marker", "Marker Bold"); - rename_id_for_versioning(bmain, ID_BR, "Draw Block", "Marker Chisel"); + do_versions_rename_id(bmain, ID_BR, "Draw Pencil", "Pencil"); + do_versions_rename_id(bmain, ID_BR, "Draw Pen", "Pen"); + do_versions_rename_id(bmain, ID_BR, "Draw Ink", "Ink Pen"); + do_versions_rename_id(bmain, ID_BR, "Draw Noise", "Ink Pen Rough"); + do_versions_rename_id(bmain, ID_BR, "Draw Marker", "Marker Bold"); + do_versions_rename_id(bmain, ID_BR, "Draw Block", "Marker Chisel"); ma = BLI_findstring(&bmain->materials, "Black", offsetof(ID, name) + 2); if (ma && ma->gp_style) { - rename_id_for_versioning(bmain, ID_MA, "Black", "Solid Stroke"); + do_versions_rename_id(bmain, ID_MA, "Black", "Solid Stroke"); } ma = BLI_findstring(&bmain->materials, "Red", offsetof(ID, name) + 2); if (ma && ma->gp_style) { - rename_id_for_versioning(bmain, ID_MA, "Red", "Squares Stroke"); + do_versions_rename_id(bmain, ID_MA, "Red", "Squares Stroke"); } ma = BLI_findstring(&bmain->materials, "Grey", offsetof(ID, name) + 2); if (ma && ma->gp_style) { - rename_id_for_versioning(bmain, ID_MA, "Grey", "Solid Fill"); + do_versions_rename_id(bmain, ID_MA, "Grey", "Solid Fill"); } ma = BLI_findstring(&bmain->materials, "Black Dots", offsetof(ID, name) + 2); if (ma && ma->gp_style) { - rename_id_for_versioning(bmain, ID_MA, "Black Dots", "Dots Stroke"); + do_versions_rename_id(bmain, ID_MA, "Black Dots", "Dots Stroke"); } brush = BLI_findstring(&bmain->brushes, "Pencil", offsetof(ID, name) + 2); @@ -4110,9 +4082,8 @@ void blo_do_versions_280(FileData *fd, Library *UNUSED(lib), Main *bmain) if (!MAIN_VERSION_ATLEAST(bmain, 280, 75)) { for (Scene *scene = bmain->scenes.first; scene; scene = scene->id.next) { if (scene->master_collection != NULL) { - scene->master_collection->flag &= ~(COLLECTION_RESTRICT_VIEWPORT | - COLLECTION_RESTRICT_SELECT | - COLLECTION_RESTRICT_RENDER); + scene->master_collection->flag &= ~(COLLECTION_HIDE_VIEWPORT | COLLECTION_HIDE_SELECT | + COLLECTION_HIDE_RENDER); } UnitSettings *unit = &scene->unit; diff --git a/source/blender/blenloader/intern/versioning_290.c b/source/blender/blenloader/intern/versioning_290.c index 09d43676b8f..7f7a2d97cbb 100644 --- a/source/blender/blenloader/intern/versioning_290.c +++ b/source/blender/blenloader/intern/versioning_290.c @@ -170,7 +170,7 @@ static void seq_convert_transform_crop(const Scene *scene, int image_size_x = scene->r.xsch; int image_size_y = scene->r.ysch; - /* Hardcoded legacy bit-flags which has been removed. */ + /* Hard-coded legacy bit-flags which has been removed. */ const uint32_t use_transform_flag = (1 << 16); const uint32_t use_crop_flag = (1 << 17); diff --git a/source/blender/blenloader/intern/versioning_300.c b/source/blender/blenloader/intern/versioning_300.c index 9aec18ea279..eb01bfbfb9c 100644 --- a/source/blender/blenloader/intern/versioning_300.c +++ b/source/blender/blenloader/intern/versioning_300.c @@ -22,6 +22,7 @@ #include "BLI_listbase.h" #include "BLI_math_vector.h" +#include "BLI_path_util.h" #include "BLI_string.h" #include "BLI_utildefines.h" @@ -29,6 +30,7 @@ #include "DNA_armature_types.h" #include "DNA_brush_types.h" #include "DNA_collection_types.h" +#include "DNA_constraint_types.h" #include "DNA_curve_types.h" #include "DNA_genfile.h" #include "DNA_listBase.h" @@ -383,6 +385,19 @@ static void do_version_bones_bbone_len_scale(ListBase *lb) } } +static void do_version_constraints_spline_ik_joint_bindings(ListBase *lb) +{ + /* Binding array data could be freed without properly resetting its size data. */ + LISTBASE_FOREACH (bConstraint *, con, lb) { + if (con->type == CONSTRAINT_TYPE_SPLINEIK) { + bSplineIKConstraint *data = (bSplineIKConstraint *)con->data; + if (data->points == NULL) { + data->numpoints = 0; + } + } + } +} + /* NOLINTNEXTLINE: readability-function-size */ void blo_do_versions_300(FileData *fd, Library *UNUSED(lib), Main *bmain) { @@ -643,12 +658,12 @@ void blo_do_versions_300(FileData *fd, Library *UNUSED(lib), Main *bmain) if (!DNA_struct_elem_find( fd->filesdna, "WorkSpace", "AssetLibraryReference", "asset_library")) { LISTBASE_FOREACH (WorkSpace *, workspace, &bmain->workspaces) { - BKE_asset_library_reference_init_default(&workspace->asset_library); + BKE_asset_library_reference_init_default(&workspace->asset_library_ref); } } if (!DNA_struct_elem_find( - fd->filesdna, "FileAssetSelectParams", "AssetLibraryReference", "asset_library")) { + fd->filesdna, "FileAssetSelectParams", "AssetLibraryReference", "asset_library_ref")) { LISTBASE_FOREACH (bScreen *, screen, &bmain->screens) { LISTBASE_FOREACH (ScrArea *, area, &screen->areabase) { LISTBASE_FOREACH (SpaceLink *, space, &area->spacedata) { @@ -657,7 +672,7 @@ void blo_do_versions_300(FileData *fd, Library *UNUSED(lib), Main *bmain) if (sfile->browse_mode != FILE_BROWSE_MODE_ASSETS) { continue; } - BKE_asset_library_reference_init_default(&sfile->asset_params->asset_library); + BKE_asset_library_reference_init_default(&sfile->asset_params->asset_library_ref); } } } @@ -691,6 +706,76 @@ void blo_do_versions_300(FileData *fd, Library *UNUSED(lib), Main *bmain) } } + /* Font names were copied directly into ID names, see: T90417. */ + if (!MAIN_VERSION_ATLEAST(bmain, 300, 16)) { + ListBase *lb = which_libbase(bmain, ID_VF); + BKE_main_id_repair_duplicate_names_listbase(lb); + } + + if (!MAIN_VERSION_ATLEAST(bmain, 300, 17)) { + if (!DNA_struct_elem_find( + fd->filesdna, "View3DOverlay", "float", "normals_constant_screen_size")) { + LISTBASE_FOREACH (bScreen *, screen, &bmain->screens) { + LISTBASE_FOREACH (ScrArea *, area, &screen->areabase) { + LISTBASE_FOREACH (SpaceLink *, sl, &area->spacedata) { + if (sl->spacetype == SPACE_VIEW3D) { + View3D *v3d = (View3D *)sl; + v3d->overlay.normals_constant_screen_size = 7.0f; + } + } + } + } + } + + /* Fix SplineIK constraint's inconsistency between binding points array and its stored size. */ + LISTBASE_FOREACH (Object *, ob, &bmain->objects) { + /* NOTE: Objects should never have SplineIK constraint, so no need to apply this fix on + * their constraints. */ + if (ob->pose) { + LISTBASE_FOREACH (bPoseChannel *, pchan, &ob->pose->chanbase) { + do_version_constraints_spline_ik_joint_bindings(&pchan->constraints); + } + } + } + } + + if (!MAIN_VERSION_ATLEAST(bmain, 300, 18)) { + if (!DNA_struct_elem_find( + fd->filesdna, "WorkSpace", "AssetLibraryReference", "asset_library_ref")) { + LISTBASE_FOREACH (WorkSpace *, workspace, &bmain->workspaces) { + BKE_asset_library_reference_init_default(&workspace->asset_library_ref); + } + } + + if (!DNA_struct_elem_find( + fd->filesdna, "FileAssetSelectParams", "AssetLibraryReference", "asset_library_ref")) { + LISTBASE_FOREACH (bScreen *, screen, &bmain->screens) { + LISTBASE_FOREACH (ScrArea *, area, &screen->areabase) { + LISTBASE_FOREACH (SpaceLink *, space, &area->spacedata) { + if (space->spacetype != SPACE_FILE) { + continue; + } + + SpaceFile *sfile = (SpaceFile *)space; + if (sfile->browse_mode != FILE_BROWSE_MODE_ASSETS) { + continue; + } + BKE_asset_library_reference_init_default(&sfile->asset_params->asset_library_ref); + } + } + } + } + + /* Previously, only text ending with `.py` would run, apply this logic + * to existing files so text that happens to have the "Register" enabled + * doesn't suddenly start running code on startup that was previously ignored. */ + LISTBASE_FOREACH (Text *, text, &bmain->texts) { + if ((text->flags & TXT_ISSCRIPT) && !BLI_path_extension_check(text->id.name + 2, ".py")) { + text->flags &= ~TXT_ISSCRIPT; + } + } + } + /** * Versioning code until next subversion bump goes here. * @@ -702,5 +787,23 @@ void blo_do_versions_300(FileData *fd, Library *UNUSED(lib), Main *bmain) */ { /* Keep this block, even when empty. */ + + /* Add node storage for subdivision surface node. */ + FOREACH_NODETREE_BEGIN (bmain, ntree, id) { + if (ntree->type == NTREE_GEOMETRY) { + LISTBASE_FOREACH (bNode *, node, &ntree->nodes) { + if (node->type == GEO_NODE_SUBDIVISION_SURFACE) { + if (node->storage == NULL) { + NodeGeometrySubdivisionSurface *data = MEM_callocN( + sizeof(NodeGeometrySubdivisionSurface), __func__); + data->uv_smooth = SUBSURF_UV_SMOOTH_PRESERVE_BOUNDARIES; + data->boundary_smooth = SUBSURF_BOUNDARY_SMOOTH_ALL; + node->storage = data; + } + } + } + } + } + FOREACH_NODETREE_END; } } diff --git a/source/blender/blenloader/intern/versioning_common.cc b/source/blender/blenloader/intern/versioning_common.cc index f5083b8e259..208c02b60d1 100644 --- a/source/blender/blenloader/intern/versioning_common.cc +++ b/source/blender/blenloader/intern/versioning_common.cc @@ -20,9 +20,15 @@ /* allow readfile to use deprecated functionality */ #define DNA_DEPRECATED_ALLOW +#include <cstring> + #include "DNA_screen_types.h" #include "BLI_listbase.h" +#include "BLI_string.h" + +#include "BKE_lib_id.h" +#include "BKE_main.h" #include "MEM_guardedalloc.h" @@ -48,3 +54,34 @@ ARegion *do_versions_add_region_if_not_found(ListBase *regionbase, BLI_insertlinkafter(regionbase, link_after_region, new_region); return new_region; } + +/** + * Rename if the ID doesn't exist. + * + * \return the ID (if found). + */ +ID *do_versions_rename_id(Main *bmain, + const short id_type, + const char *name_src, + const char *name_dst) +{ + /* We can ignore libraries */ + ListBase *lb = which_libbase(bmain, id_type); + ID *id = nullptr; + LISTBASE_FOREACH (ID *, idtest, lb) { + if (idtest->lib == nullptr) { + if (STREQ(idtest->name + 2, name_src)) { + id = idtest; + } + if (STREQ(idtest->name + 2, name_dst)) { + return nullptr; + } + } + } + if (id != nullptr) { + BLI_strncpy(id->name + 2, name_dst, sizeof(id->name) - 2); + /* We know it's unique, this just sorts. */ + BLI_libblock_ensure_unique_name(bmain, id->name); + } + return id; +} diff --git a/source/blender/blenloader/intern/versioning_common.h b/source/blender/blenloader/intern/versioning_common.h index a1769d4639e..47e0b74a3e4 100644 --- a/source/blender/blenloader/intern/versioning_common.h +++ b/source/blender/blenloader/intern/versioning_common.h @@ -22,6 +22,7 @@ struct ARegion; struct ListBase; +struct Main; #ifdef __cplusplus extern "C" { @@ -32,6 +33,11 @@ struct ARegion *do_versions_add_region_if_not_found(struct ListBase *regionbase, const char *name, int link_after_region_type); +ID *do_versions_rename_id(Main *bmain, + const short id_type, + const char *name_src, + const char *name_dst); + #ifdef __cplusplus } #endif diff --git a/source/blender/blenloader/intern/versioning_cycles.c b/source/blender/blenloader/intern/versioning_cycles.c index 94ee89c5120..90e6b43f02e 100644 --- a/source/blender/blenloader/intern/versioning_cycles.c +++ b/source/blender/blenloader/intern/versioning_cycles.c @@ -78,6 +78,12 @@ static IDProperty *cycles_properties_from_ID(ID *id) return (idprop) ? IDP_GetPropertyTypeFromGroup(idprop, "cycles", IDP_GROUP) : NULL; } +static IDProperty *cycles_visibility_properties_from_ID(ID *id) +{ + IDProperty *idprop = IDP_GetProperties(id, false); + return (idprop) ? IDP_GetPropertyTypeFromGroup(idprop, "cycles_visibility", IDP_GROUP) : NULL; +} + static IDProperty *cycles_properties_from_view_layer(ViewLayer *view_layer) { IDProperty *idprop = view_layer->id_properties; @@ -1600,4 +1606,35 @@ void do_versions_after_linking_cycles(Main *bmain) } } } + + /* Move visibility from Cycles to Blender. */ + if (!MAIN_VERSION_ATLEAST(bmain, 300, 17)) { + LISTBASE_FOREACH (Object *, object, &bmain->objects) { + IDProperty *cvisibility = cycles_visibility_properties_from_ID(&object->id); + int flag = 0; + + if (cvisibility) { + flag |= cycles_property_boolean(cvisibility, "camera", true) ? 0 : OB_HIDE_CAMERA; + flag |= cycles_property_boolean(cvisibility, "diffuse", true) ? 0 : OB_HIDE_DIFFUSE; + flag |= cycles_property_boolean(cvisibility, "glossy", true) ? 0 : OB_HIDE_GLOSSY; + flag |= cycles_property_boolean(cvisibility, "transmission", true) ? 0 : + OB_HIDE_TRANSMISSION; + flag |= cycles_property_boolean(cvisibility, "scatter", true) ? 0 : OB_HIDE_VOLUME_SCATTER; + flag |= cycles_property_boolean(cvisibility, "shadow", true) ? 0 : OB_HIDE_SHADOW; + } + + IDProperty *cobject = cycles_properties_from_ID(&object->id); + if (cobject) { + flag |= cycles_property_boolean(cobject, "is_holdout", false) ? OB_HOLDOUT : 0; + flag |= cycles_property_boolean(cobject, "is_shadow_catcher", false) ? OB_SHADOW_CATCHER : + 0; + } + + if (object->type == OB_LAMP) { + flag |= OB_HIDE_CAMERA | OB_SHADOW_CATCHER; + } + + object->visibility_flag |= flag; + } + } } diff --git a/source/blender/blenloader/intern/versioning_defaults.c b/source/blender/blenloader/intern/versioning_defaults.c index 8362e001ea6..82c577d11a0 100644 --- a/source/blender/blenloader/intern/versioning_defaults.c +++ b/source/blender/blenloader/intern/versioning_defaults.c @@ -65,38 +65,11 @@ #include "BLO_readfile.h" +#include "versioning_common.h" + /* Make preferences read-only, use versioning_userdef.c. */ #define U (*((const UserDef *)&U)) -/** - * Rename if the ID doesn't exist. - */ -static ID *rename_id_for_versioning(Main *bmain, - const short id_type, - const char *name_src, - const char *name_dst) -{ - /* We can ignore libraries */ - ListBase *lb = which_libbase(bmain, id_type); - ID *id = NULL; - LISTBASE_FOREACH (ID *, idtest, lb) { - if (idtest->lib == NULL) { - if (STREQ(idtest->name + 2, name_src)) { - id = idtest; - } - if (STREQ(idtest->name + 2, name_dst)) { - return NULL; - } - } - } - if (id != NULL) { - BLI_strncpy(id->name + 2, name_dst, sizeof(id->name) - 2); - /* We know it's unique, this just sorts. */ - BLI_libblock_ensure_unique_name(bmain, id->name); - } - return id; -} - static bool blo_is_builtin_template(const char *app_template) { /* For all builtin templates shipped with Blender. */ @@ -217,6 +190,7 @@ static void blo_update_defaults_screen(bScreen *screen, } /* Disable Curve Normals. */ v3d->overlay.edit_flag &= ~V3D_OVERLAY_EDIT_CU_NORMALS; + v3d->overlay.normals_constant_screen_size = 7.0f; } else if (area->spacetype == SPACE_CLIP) { SpaceClip *sclip = area->spacedata.first; @@ -406,28 +380,28 @@ void BLO_update_defaults_startup_blend(Main *bmain, const char *app_template) Brush *brush; /* Pencil brush. */ - rename_id_for_versioning(bmain, ID_BR, "Draw Pencil", "Pencil"); + do_versions_rename_id(bmain, ID_BR, "Draw Pencil", "Pencil"); /* Pen brush. */ - rename_id_for_versioning(bmain, ID_BR, "Draw Pen", "Pen"); + do_versions_rename_id(bmain, ID_BR, "Draw Pen", "Pen"); /* Pen Soft brush. */ - brush = (Brush *)rename_id_for_versioning(bmain, ID_BR, "Draw Soft", "Pencil Soft"); + brush = (Brush *)do_versions_rename_id(bmain, ID_BR, "Draw Soft", "Pencil Soft"); if (brush) { brush->gpencil_settings->icon_id = GP_BRUSH_ICON_PEN; } /* Ink Pen brush. */ - rename_id_for_versioning(bmain, ID_BR, "Draw Ink", "Ink Pen"); + do_versions_rename_id(bmain, ID_BR, "Draw Ink", "Ink Pen"); /* Ink Pen Rough brush. */ - rename_id_for_versioning(bmain, ID_BR, "Draw Noise", "Ink Pen Rough"); + do_versions_rename_id(bmain, ID_BR, "Draw Noise", "Ink Pen Rough"); /* Marker Bold brush. */ - rename_id_for_versioning(bmain, ID_BR, "Draw Marker", "Marker Bold"); + do_versions_rename_id(bmain, ID_BR, "Draw Marker", "Marker Bold"); /* Marker Chisel brush. */ - rename_id_for_versioning(bmain, ID_BR, "Draw Block", "Marker Chisel"); + do_versions_rename_id(bmain, ID_BR, "Draw Block", "Marker Chisel"); /* Remove useless Fill Area.001 brush. */ brush = BLI_findstring(&bmain->brushes, "Fill Area.001", offsetof(ID, name) + 2); @@ -438,10 +412,10 @@ void BLO_update_defaults_startup_blend(Main *bmain, const char *app_template) /* Rename and fix materials and enable default object lights on. */ if (app_template && STREQ(app_template, "2D_Animation")) { Material *ma = NULL; - rename_id_for_versioning(bmain, ID_MA, "Black", "Solid Stroke"); - rename_id_for_versioning(bmain, ID_MA, "Red", "Squares Stroke"); - rename_id_for_versioning(bmain, ID_MA, "Grey", "Solid Fill"); - rename_id_for_versioning(bmain, ID_MA, "Black Dots", "Dots Stroke"); + do_versions_rename_id(bmain, ID_MA, "Black", "Solid Stroke"); + do_versions_rename_id(bmain, ID_MA, "Red", "Squares Stroke"); + do_versions_rename_id(bmain, ID_MA, "Grey", "Solid Fill"); + do_versions_rename_id(bmain, ID_MA, "Black Dots", "Dots Stroke"); /* Dots Stroke. */ ma = BLI_findstring(&bmain->materials, "Dots Stroke", offsetof(ID, name) + 2); @@ -553,8 +527,8 @@ void BLO_update_defaults_startup_blend(Main *bmain, const char *app_template) } /* Objects */ - rename_id_for_versioning(bmain, ID_OB, "Lamp", "Light"); - rename_id_for_versioning(bmain, ID_LA, "Lamp", "Light"); + do_versions_rename_id(bmain, ID_OB, "Lamp", "Light"); + do_versions_rename_id(bmain, ID_LA, "Lamp", "Light"); if (app_template && STREQ(app_template, "2D_Animation")) { for (Object *object = bmain->objects.first; object; object = object->id.next) { diff --git a/source/blender/blenloader/intern/versioning_legacy.c b/source/blender/blenloader/intern/versioning_legacy.c index 95cfc9975d7..6ba27b6ee9e 100644 --- a/source/blender/blenloader/intern/versioning_legacy.c +++ b/source/blender/blenloader/intern/versioning_legacy.c @@ -28,8 +28,7 @@ #else # include "BLI_winstuff.h" # include "winsock2.h" -# include <io.h> /* for open close read */ -# include <zlib.h> /* odd include order-issue */ +# include <io.h> /* for open close read */ #endif /* allow readfile to use deprecated functionality */ @@ -1335,7 +1334,7 @@ void blo_do_versions_pre250(FileData *fd, Library *lib, Main *bmain) ME_OPT_EDGES = (1 << 8), }; - if ((me->flag & ME_SUBSURF)) { + if (me->flag & ME_SUBSURF) { SubsurfModifierData *smd = (SubsurfModifierData *)BKE_modifier_new( eModifierType_Subsurf); diff --git a/source/blender/blenloader/intern/versioning_userdef.c b/source/blender/blenloader/intern/versioning_userdef.c index c409f0a71fc..0042ff29dc2 100644 --- a/source/blender/blenloader/intern/versioning_userdef.c +++ b/source/blender/blenloader/intern/versioning_userdef.c @@ -873,13 +873,6 @@ void blo_do_versions_userdef(UserDef *userdef) } } - if (!USER_VERSION_ATLEAST(293, 2)) { - /* Enable asset browser features by default for alpha testing. - * BLO_sanitize_experimental_features_userpref_blend() will disable it again for non-alpha - * builds. */ - userdef->experimental.use_asset_browser = true; - } - if (!USER_VERSION_ATLEAST(293, 12)) { if (userdef->gizmo_size_navigate_v3d == 0) { userdef->gizmo_size_navigate_v3d = 80; diff --git a/source/blender/blenloader/intern/writefile.c b/source/blender/blenloader/intern/writefile.c index 12839a155e4..6f43fbf1fa0 100644 --- a/source/blender/blenloader/intern/writefile.c +++ b/source/blender/blenloader/intern/writefile.c @@ -78,12 +78,12 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <zlib.h> #ifdef WIN32 # include "BLI_winstuff.h" # include "winsock2.h" # include <io.h> -# include <zlib.h> /* odd include order-issue */ #else # include <unistd.h> /* FreeBSD, for write() and close(). */ #endif @@ -101,7 +101,12 @@ #include "BLI_bitmap.h" #include "BLI_blenlib.h" #include "BLI_endian_defines.h" +#include "BLI_endian_switch.h" +#include "BLI_link_utils.h" +#include "BLI_linklist.h" +#include "BLI_math_base.h" #include "BLI_mempool.h" +#include "BLI_threads.h" #include "MEM_guardedalloc.h" /* MEM_freeN */ #include "BKE_blender_version.h" @@ -129,14 +134,21 @@ #include <errno.h> +#include <zstd.h> + /* Make preferences read-only. */ #define U (*((const UserDef *)&U)) /* ********* my write, buffered writing with minimum size chunks ************ */ /* Use optimal allocation since blocks of this size are kept in memory for undo. */ -#define MYWRITE_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */ -#define MYWRITE_MAX_CHUNK (MEM_SIZE_OPTIMAL(1 << 15)) /* ~32kb */ +#define MEM_BUFFER_SIZE (MEM_SIZE_OPTIMAL(1 << 17)) /* 128kb */ +#define MEM_CHUNK_SIZE (MEM_SIZE_OPTIMAL(1 << 15)) /* ~32kb */ + +#define ZSTD_BUFFER_SIZE (1 << 21) /* 2mb */ +#define ZSTD_CHUNK_SIZE (1 << 20) /* 1mb */ + +#define ZSTD_COMPRESSION_LEVEL 3 /** Use if we want to store how many bytes have been written to the file. */ // #define USE_WRITE_DATA_LEN @@ -147,9 +159,16 @@ typedef enum { WW_WRAP_NONE = 1, - WW_WRAP_ZLIB, + WW_WRAP_ZSTD, } eWriteWrapType; +typedef struct ZstdFrame { + struct ZstdFrame *next, *prev; + + uint32_t compressed_size; + uint32_t uncompressed_size; +} ZstdFrame; + typedef struct WriteWrap WriteWrap; struct WriteWrap { /* callbacks */ @@ -161,15 +180,23 @@ struct WriteWrap { bool use_buf; /* internal */ - union { - int file_handle; - gzFile gz_handle; - } _user_data; + int file_handle; + struct { + ListBase threadpool; + ListBase tasks; + ThreadMutex mutex; + ThreadCondition condition; + int next_frame; + int num_frames; + + int level; + ListBase frames; + + bool write_error; + } zstd; }; /* none */ -#define FILE_HANDLE(ww) (ww)->_user_data.file_handle - static bool ww_open_none(WriteWrap *ww, const char *filepath) { int file; @@ -177,7 +204,7 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath) file = BLI_open(filepath, O_BINARY + O_WRONLY + O_CREAT + O_TRUNC, 0666); if (file != -1) { - FILE_HANDLE(ww) = file; + ww->file_handle = file; return true; } @@ -185,39 +212,170 @@ static bool ww_open_none(WriteWrap *ww, const char *filepath) } static bool ww_close_none(WriteWrap *ww) { - return (close(FILE_HANDLE(ww)) != -1); + return (close(ww->file_handle) != -1); } static size_t ww_write_none(WriteWrap *ww, const char *buf, size_t buf_len) { - return write(FILE_HANDLE(ww), buf, buf_len); + return write(ww->file_handle, buf, buf_len); } -#undef FILE_HANDLE -/* zlib */ -#define FILE_HANDLE(ww) (ww)->_user_data.gz_handle +/* zstd */ -static bool ww_open_zlib(WriteWrap *ww, const char *filepath) +typedef struct { + struct ZstdWriteBlockTask *next, *prev; + void *data; + size_t size; + int frame_number; + WriteWrap *ww; +} ZstdWriteBlockTask; + +static void *zstd_write_task(void *userdata) { - gzFile file; + ZstdWriteBlockTask *task = userdata; + WriteWrap *ww = task->ww; - file = BLI_gzopen(filepath, "wb1"); + size_t out_buf_len = ZSTD_compressBound(task->size); + void *out_buf = MEM_mallocN(out_buf_len, "Zstd out buffer"); + size_t out_size = ZSTD_compress( + out_buf, out_buf_len, task->data, task->size, ZSTD_COMPRESSION_LEVEL); - if (file != Z_NULL) { - FILE_HANDLE(ww) = file; - return true; + MEM_freeN(task->data); + + BLI_mutex_lock(&ww->zstd.mutex); + + while (ww->zstd.next_frame != task->frame_number) { + BLI_condition_wait(&ww->zstd.condition, &ww->zstd.mutex); } - return false; + if (ZSTD_isError(out_size)) { + ww->zstd.write_error = true; + } + else { + if (ww_write_none(ww, out_buf, out_size) == out_size) { + ZstdFrame *frameinfo = MEM_mallocN(sizeof(ZstdFrame), "zstd frameinfo"); + frameinfo->uncompressed_size = task->size; + frameinfo->compressed_size = out_size; + BLI_addtail(&ww->zstd.frames, frameinfo); + } + else { + ww->zstd.write_error = true; + } + } + + ww->zstd.next_frame++; + + BLI_mutex_unlock(&ww->zstd.mutex); + BLI_condition_notify_all(&ww->zstd.condition); + + MEM_freeN(out_buf); + return NULL; +} + +static bool ww_open_zstd(WriteWrap *ww, const char *filepath) +{ + if (!ww_open_none(ww, filepath)) { + return false; + } + + /* Leave one thread open for the main writing logic, unless we only have one HW thread. */ + int num_threads = max_ii(1, BLI_system_thread_count() - 1); + BLI_threadpool_init(&ww->zstd.threadpool, zstd_write_task, num_threads); + BLI_mutex_init(&ww->zstd.mutex); + BLI_condition_init(&ww->zstd.condition); + + return true; +} + +static void zstd_write_u32_le(WriteWrap *ww, uint32_t val) +{ +#ifdef __BIG_ENDIAN__ + BLI_endian_switch_uint32(&val); +#endif + ww_write_none(ww, (char *)&val, sizeof(uint32_t)); } -static bool ww_close_zlib(WriteWrap *ww) + +/* In order to implement efficient seeking when reading the .blend, we append + * a skippable frame that encodes information about the other frames present + * in the file. + * The format here follows the upstream spec for seekable files: + * https://github.com/facebook/zstd/blob/master/contrib/seekable_format/zstd_seekable_compression_format.md + * If this information is not present in a file (e.g. if it was compressed + * with external tools), it can still be opened in Blender, but seeking will + * not be supported, so more memory might be needed. */ +static void zstd_write_seekable_frames(WriteWrap *ww) { - return (gzclose(FILE_HANDLE(ww)) == Z_OK); + /* Write seek table header (magic number and frame size). */ + zstd_write_u32_le(ww, 0x184D2A5E); + + /* The actual frame number might not match ww->zstd.num_frames if there was a write error. */ + const uint32_t num_frames = BLI_listbase_count(&ww->zstd.frames); + /* Each frame consists of two u32, so 8 bytes each. + * After the frames, a footer containing two u32 and one byte (9 bytes total) is written. */ + const uint32_t frame_size = num_frames * 8 + 9; + zstd_write_u32_le(ww, frame_size); + + /* Write seek table entries. */ + LISTBASE_FOREACH (ZstdFrame *, frame, &ww->zstd.frames) { + zstd_write_u32_le(ww, frame->compressed_size); + zstd_write_u32_le(ww, frame->uncompressed_size); + } + + /* Write seek table footer (number of frames, option flags and second magic number). */ + zstd_write_u32_le(ww, num_frames); + const char flags = 0; /* We don't store checksums for each frame. */ + ww_write_none(ww, &flags, 1); + zstd_write_u32_le(ww, 0x8F92EAB1); } -static size_t ww_write_zlib(WriteWrap *ww, const char *buf, size_t buf_len) + +static bool ww_close_zstd(WriteWrap *ww) { - return gzwrite(FILE_HANDLE(ww), buf, buf_len); + BLI_threadpool_end(&ww->zstd.threadpool); + BLI_freelistN(&ww->zstd.tasks); + + BLI_mutex_end(&ww->zstd.mutex); + BLI_condition_end(&ww->zstd.condition); + + zstd_write_seekable_frames(ww); + BLI_freelistN(&ww->zstd.frames); + + return ww_close_none(ww) && !ww->zstd.write_error; +} + +static size_t ww_write_zstd(WriteWrap *ww, const char *buf, size_t buf_len) +{ + if (ww->zstd.write_error) { + return 0; + } + + ZstdWriteBlockTask *task = MEM_mallocN(sizeof(ZstdWriteBlockTask), __func__); + task->data = MEM_mallocN(buf_len, __func__); + memcpy(task->data, buf, buf_len); + task->size = buf_len; + task->frame_number = ww->zstd.num_frames++; + task->ww = ww; + + BLI_mutex_lock(&ww->zstd.mutex); + BLI_addtail(&ww->zstd.tasks, task); + + /* If there's a free worker thread, just push the block into that thread. + * Otherwise, we wait for the earliest thread to finish. + * We look up the earliest thread while holding the mutex, but release it + * before joining the thread to prevent a deadlock. */ + ZstdWriteBlockTask *first_task = ww->zstd.tasks.first; + BLI_mutex_unlock(&ww->zstd.mutex); + if (!BLI_available_threads(&ww->zstd.threadpool)) { + BLI_threadpool_remove(&ww->zstd.threadpool, first_task); + + /* If the task list was empty before we pushed our task, there should + * always be a free thread. */ + BLI_assert(first_task != task); + BLI_remlink(&ww->zstd.tasks, first_task); + MEM_freeN(first_task); + } + BLI_threadpool_insert(&ww->zstd.threadpool, task); + + return buf_len; } -#undef FILE_HANDLE /* --- end compression types --- */ @@ -226,11 +384,11 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww) memset(r_ww, 0, sizeof(*r_ww)); switch (ww_type) { - case WW_WRAP_ZLIB: { - r_ww->open = ww_open_zlib; - r_ww->close = ww_close_zlib; - r_ww->write = ww_write_zlib; - r_ww->use_buf = false; + case WW_WRAP_ZSTD: { + r_ww->open = ww_open_zstd; + r_ww->close = ww_close_zstd; + r_ww->write = ww_write_zstd; + r_ww->use_buf = true; break; } default: { @@ -252,10 +410,17 @@ static void ww_handle_init(eWriteWrapType ww_type, WriteWrap *r_ww) typedef struct { const struct SDNA *sdna; - /** Use for file and memory writing (fixed size of #MYWRITE_BUFFER_SIZE). */ - uchar *buf; - /** Number of bytes used in #WriteData.buf (flushed when exceeded). */ - size_t buf_used_len; + struct { + /** Use for file and memory writing (size stored in max_size). */ + uchar *buf; + /** Number of bytes used in #WriteData.buf (flushed when exceeded). */ + size_t used_len; + + /** Maximum size of the buffer. */ + size_t max_size; + /** Threshold above which writes get their own chunk. */ + size_t chunk_size; + } buffer; #ifdef USE_WRITE_DATA_LEN /** Total number of bytes written. */ @@ -271,7 +436,7 @@ typedef struct { bool use_memfile; /** - * Wrap writing, so we can use zlib or + * Wrap writing, so we can use zstd or * other compression types later, see: G_FILE_COMPRESS * Will be NULL for UNDO. */ @@ -291,7 +456,15 @@ static WriteData *writedata_new(WriteWrap *ww) wd->ww = ww; if ((ww == NULL) || (ww->use_buf)) { - wd->buf = MEM_mallocN(MYWRITE_BUFFER_SIZE, "wd->buf"); + if (ww == NULL) { + wd->buffer.max_size = MEM_BUFFER_SIZE; + wd->buffer.chunk_size = MEM_CHUNK_SIZE; + } + else { + wd->buffer.max_size = ZSTD_BUFFER_SIZE; + wd->buffer.chunk_size = ZSTD_CHUNK_SIZE; + } + wd->buffer.buf = MEM_mallocN(wd->buffer.max_size, "wd->buffer.buf"); } return wd; @@ -325,8 +498,8 @@ static void writedata_do_write(WriteData *wd, const void *mem, size_t memlen) static void writedata_free(WriteData *wd) { - if (wd->buf) { - MEM_freeN(wd->buf); + if (wd->buffer.buf) { + MEM_freeN(wd->buffer.buf); } MEM_freeN(wd); } @@ -343,9 +516,9 @@ static void writedata_free(WriteData *wd) */ static void mywrite_flush(WriteData *wd) { - if (wd->buf_used_len != 0) { - writedata_do_write(wd, wd->buf, wd->buf_used_len); - wd->buf_used_len = 0; + if (wd->buffer.used_len != 0) { + writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); + wd->buffer.used_len = 0; } } @@ -369,20 +542,20 @@ static void mywrite(WriteData *wd, const void *adr, size_t len) wd->write_len += len; #endif - if (wd->buf == NULL) { + if (wd->buffer.buf == NULL) { writedata_do_write(wd, adr, len); } else { /* if we have a single big chunk, write existing data in * buffer and write out big chunk in smaller pieces */ - if (len > MYWRITE_MAX_CHUNK) { - if (wd->buf_used_len != 0) { - writedata_do_write(wd, wd->buf, wd->buf_used_len); - wd->buf_used_len = 0; + if (len > wd->buffer.chunk_size) { + if (wd->buffer.used_len != 0) { + writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); + wd->buffer.used_len = 0; } do { - size_t writelen = MIN2(len, MYWRITE_MAX_CHUNK); + size_t writelen = MIN2(len, wd->buffer.chunk_size); writedata_do_write(wd, adr, writelen); adr = (const char *)adr + writelen; len -= writelen; @@ -392,14 +565,14 @@ static void mywrite(WriteData *wd, const void *adr, size_t len) } /* if data would overflow buffer, write out the buffer */ - if (len + wd->buf_used_len > MYWRITE_BUFFER_SIZE - 1) { - writedata_do_write(wd, wd->buf, wd->buf_used_len); - wd->buf_used_len = 0; + if (len + wd->buffer.used_len > wd->buffer.max_size - 1) { + writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); + wd->buffer.used_len = 0; } /* append data at end of buffer */ - memcpy(&wd->buf[wd->buf_used_len], adr, len); - wd->buf_used_len += len; + memcpy(&wd->buffer.buf[wd->buffer.used_len], adr, len); + wd->buffer.used_len += len; } } @@ -430,9 +603,9 @@ static WriteData *mywrite_begin(WriteWrap *ww, MemFile *compare, MemFile *curren */ static bool mywrite_end(WriteData *wd) { - if (wd->buf_used_len != 0) { - writedata_do_write(wd, wd->buf, wd->buf_used_len); - wd->buf_used_len = 0; + if (wd->buffer.used_len != 0) { + writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); + wd->buffer.used_len = 0; } if (wd->use_memfile) { @@ -757,8 +930,8 @@ static void write_userdef(BlendWriter *writer, const UserDef *userdef) BLO_write_struct(writer, bPathCompare, path_cmp); } - LISTBASE_FOREACH (const bUserAssetLibrary *, asset_library, &userdef->asset_libraries) { - BLO_write_struct(writer, bUserAssetLibrary, asset_library); + LISTBASE_FOREACH (const bUserAssetLibrary *, asset_library_ref, &userdef->asset_libraries) { + BLO_write_struct(writer, bUserAssetLibrary, asset_library_ref); } LISTBASE_FOREACH (const uiStyle *, style, &userdef->uistyles) { @@ -982,6 +1155,14 @@ static bool write_file_handle(Main *mainvar, BLI_assert( (id->tag & (LIB_TAG_NO_MAIN | LIB_TAG_NO_USER_REFCOUNT | LIB_TAG_NOT_ALLOCATED)) == 0); + /* We only write unused IDs in undo case. + * NOTE: All Scenes, WindowManagers and WorkSpaces should always be written to disk, so + * their usercount should never be NULL currently. */ + if (id->us == 0 && !wd->use_memfile) { + BLI_assert(!ELEM(GS(id->name), ID_SCE, ID_WM, ID_WS)); + continue; + } + const bool do_override = !ELEM(override_storage, NULL, bmain) && ID_IS_OVERRIDE_LIBRARY_REAL(id); @@ -1015,12 +1196,23 @@ static bool write_file_handle(Main *mainvar, memcpy(id_buffer, id, idtype_struct_size); + /* Clear runtime data to reduce false detection of changed data in undo/redo context. */ ((ID *)id_buffer)->tag = 0; + ((ID *)id_buffer)->us = 0; + ((ID *)id_buffer)->icon_id = 0; /* Those listbase data change every time we add/remove an ID, and also often when * renaming one (due to re-sorting). This avoids generating a lot of false 'is changed' * detections between undo steps. */ ((ID *)id_buffer)->prev = NULL; ((ID *)id_buffer)->next = NULL; + /* Those runtime pointers should never be set during writing stage, but just in case clear + * them too. */ + ((ID *)id_buffer)->orig_id = NULL; + ((ID *)id_buffer)->newid = NULL; + /* Even though in theory we could be able to preserve this python instance across undo even + * when we need to re-read the ID into its original address, this is currently cleared in + * #direct_link_id_common in `readfile.c` anyway, */ + ((ID *)id_buffer)->py_instance = NULL; const IDTypeInfo *id_type = BKE_idtype_get_info_from_id(id); if (id_type->blend_write != NULL) { @@ -1131,7 +1323,6 @@ bool BLO_write_file(Main *mainvar, ReportList *reports) { char tempname[FILE_MAX + 1]; - eWriteWrapType ww_type; WriteWrap ww; eBLO_WritePathRemap remap_mode = params->remap_mode; @@ -1153,14 +1344,7 @@ bool BLO_write_file(Main *mainvar, /* open temporary file, so we preserve the original in case we crash */ BLI_snprintf(tempname, sizeof(tempname), "%s@", filepath); - if (write_flags & G_FILE_COMPRESS) { - ww_type = WW_WRAP_ZLIB; - } - else { - ww_type = WW_WRAP_NONE; - } - - ww_handle_init(ww_type, &ww); + ww_handle_init((write_flags & G_FILE_COMPRESS) ? WW_WRAP_ZSTD : WW_WRAP_NONE, &ww); if (ww.open(&ww, tempname) == false) { BKE_reportf( |