From 0a6186dcdf3f3e42104ffb39f4ed8bc052eb9fe5 Mon Sep 17 00:00:00 2001 From: Lukas Stockner Date: Sat, 21 Aug 2021 03:26:57 +0200 Subject: D5799 Part 3: Sequencer cache --- source/blender/blenlib/BLI_fileops.h | 19 +-- source/blender/blenlib/intern/fileops.c | 217 +++++++------------------- source/blender/sequencer/intern/image_cache.c | 32 ++-- 3 files changed, 82 insertions(+), 186 deletions(-) diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h index 377b7bc3bc2..906a56ce909 100644 --- a/source/blender/blenlib/BLI_fileops.h +++ b/source/blender/blenlib/BLI_fileops.h @@ -154,20 +154,15 @@ bool BLI_file_is_writable(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL bool BLI_file_touch(const char *file) ATTR_NONNULL(); bool BLI_file_alias_target(const char *filepath, char *r_targetpath) ATTR_WARN_UNUSED_RESULT; -#if 0 /* UNUSED */ -int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); -#endif -char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT - ATTR_NONNULL(); -size_t BLI_gzip_mem_to_file_at_pos(void *buf, - size_t len, - FILE *file, - size_t gz_stream_offset, - int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); -size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset) - ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); bool BLI_file_magic_is_gzip(const char header[4]); +size_t BLI_file_zstd_from_mem_at_pos(void *buf, + size_t len, + FILE *file, + size_t file_offset, + int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); +size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset) + ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); bool BLI_file_magic_is_zstd(const char header[4]); size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT; diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c index 31825c69737..532a29b8147 100644 --- a/source/blender/blenlib/intern/fileops.c +++ b/source/blender/blenlib/intern/fileops.c @@ -31,6 +31,7 @@ #include #include "zlib.h" +#include "zstd.h" #ifdef WIN32 # include "BLI_fileops_types.h" @@ -61,200 +62,90 @@ #include "BLI_sys_types.h" /* for intptr_t support */ #include "BLI_utildefines.h" -#if 0 /* UNUSED */ -/* gzip the file in from and write it to "to". - * return -1 if zlib fails, -2 if the originating file does not exist - * NOTE: will remove the "from" file - */ -int BLI_file_gzip(const char *from, const char *to) +size_t BLI_file_zstd_from_mem_at_pos( + void *buf, size_t len, FILE *file, size_t file_offset, int compression_level) { - char buffer[10240]; - int file; - int readsize = 0; - int rval = 0, err; - gzFile gzfile; + fseek(file, file_offset, SEEK_SET); - /* level 1 is very close to 3 (the default) in terms of file size, - * but about twice as fast, best use for speedy saving - campbell */ - gzfile = BLI_gzopen(to, "wb1"); - if (gzfile == NULL) { - return -1; - } - file = BLI_open(from, O_BINARY | O_RDONLY, 0); - if (file == -1) { - return -2; - } + ZSTD_CCtx *ctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level); + + ZSTD_inBuffer input = {buf, len, 0}; - while (1) { - readsize = read(file, buffer, sizeof(buffer)); + size_t out_len = ZSTD_CStreamOutSize(); + void *out_buf = MEM_mallocN(out_len, __func__); + size_t total_written = 0; - if (readsize < 0) { - rval = -2; /* error happened in reading */ - fprintf(stderr, "Error reading file %s: %s.\n", from, strerror(errno)); + /* Compress block and write it out until the input has been consumed. */ + while (input.pos < input.size) { + ZSTD_outBuffer output = {out_buf, out_len, 0}; + size_t ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_continue); + if (ZSTD_isError(ret)) { break; } - else if (readsize == 0) { - break; /* done reading */ - } - - if (gzwrite(gzfile, buffer, readsize) <= 0) { - rval = -1; /* error happened in writing */ - fprintf(stderr, "Error writing gz file %s: %s.\n", to, gzerror(gzfile, &err)); + if (fwrite(out_buf, 1, output.pos, file) != output.pos) { break; } + total_written += output.pos; } - gzclose(gzfile); - close(file); - - return rval; -} -#endif - -/* gzip the file in from_file and write it to memory to_mem, at most size bytes. - * return the unzipped size - */ -char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) -{ - gzFile gzfile; - int readsize, size, alloc_size = 0; - char *mem = NULL; - const int chunk_size = 512 * 1024; - - size = 0; - - gzfile = BLI_gzopen(from_file, "rb"); - for (;;) { - if (mem == NULL) { - mem = MEM_callocN(chunk_size, "BLI_ungzip_to_mem"); - alloc_size = chunk_size; - } - else { - mem = MEM_reallocN(mem, size + chunk_size); - alloc_size += chunk_size; - } - - readsize = gzread(gzfile, mem + size, chunk_size); - if (readsize > 0) { - size += readsize; + /* Finalize the Zstd frame. */ + size_t ret = 1; + while (ret != 0) { + ZSTD_outBuffer output = {out_buf, out_len, 0}; + ret = ZSTD_compressStream2(ctx, &output, &input, ZSTD_e_end); + if (ZSTD_isError(ret)) { + break; } - else { + if (fwrite(out_buf, 1, output.pos, file) != output.pos) { break; } + total_written += output.pos; } - gzclose(gzfile); - - if (size == 0) { - MEM_freeN(mem); - mem = NULL; - } - else if (alloc_size != size) { - mem = MEM_reallocN(mem, size); - } - - *r_size = size; + MEM_freeN(out_buf); + ZSTD_freeCCtx(ctx); - return mem; + return ZSTD_isError(ret) ? 0 : total_written; } -#define CHUNK (256 * 1024) - -/* gzip byte array from memory and write it to file at certain position. - * return size of gzip stream. - */ -size_t BLI_gzip_mem_to_file_at_pos( - void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level) +size_t BLI_file_unzstd_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t file_offset) { - int ret, flush; - unsigned have; - z_stream strm; - unsigned char out[CHUNK]; - - BLI_fseek(file, gz_stream_offset, 0); - - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - ret = deflateInit(&strm, compression_level); - if (ret != Z_OK) { - return 0; - } - - strm.avail_in = len; - strm.next_in = (Bytef *)buf; - flush = Z_FINISH; - - do { - strm.avail_out = CHUNK; - strm.next_out = out; - ret = deflate(&strm, flush); - if (ret == Z_STREAM_ERROR) { - return 0; - } - have = CHUNK - strm.avail_out; - if (fwrite(out, 1, have, file) != have || ferror(file)) { - deflateEnd(&strm); - return 0; - } - } while (strm.avail_out == 0); + fseek(file, file_offset, SEEK_SET); - if (strm.avail_in != 0 || ret != Z_STREAM_END) { - return 0; - } + ZSTD_DCtx *ctx = ZSTD_createDCtx(); - deflateEnd(&strm); - return (size_t)strm.total_out; -} + size_t in_len = ZSTD_DStreamInSize(); + void *in_buf = MEM_mallocN(in_len, __func__); + ZSTD_inBuffer input = {in_buf, in_len, 0}; -/* read and decompress gzip stream from file at certain position to buffer. - * return size of decompressed data. - */ -size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset) -{ - int ret; - z_stream strm; - size_t chunk = 256 * 1024; - unsigned char in[CHUNK]; - - BLI_fseek(file, gz_stream_offset, 0); - - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = 0; - strm.next_in = Z_NULL; - ret = inflateInit(&strm); - if (ret != Z_OK) { - return 0; - } + ZSTD_outBuffer output = {buf, len, 0}; - do { - strm.avail_in = fread(in, 1, chunk, file); - strm.next_in = in; - if (ferror(file)) { - inflateEnd(&strm); - return 0; + size_t ret = 0; + /* Read and decompress chunks of input data until we have enough output. */ + while (output.pos < output.size && !ZSTD_isError(ret)) { + input.size = fread(in_buf, 1, in_len, file); + if (input.size == 0) { + break; } - do { - strm.avail_out = len; - strm.next_out = (Bytef *)buf + strm.total_out; + /* Consume input data until we run out or have enough output. */ + input.pos = 0; + while (input.pos < input.size && output.pos < output.size) { + ret = ZSTD_decompressStream(ctx, &output, &input); - ret = inflate(&strm, Z_NO_FLUSH); - if (ret == Z_STREAM_ERROR) { - return 0; + if (ZSTD_isError(ret)) { + break; } - } while (strm.avail_out == 0); + } + } - } while (ret != Z_STREAM_END); + MEM_freeN(in_buf); + ZSTD_freeDCtx(ctx); - inflateEnd(&strm); - return (size_t)strm.total_out; + return ZSTD_isError(ret) ? 0 : output.pos; } -#undef CHUNK - bool BLI_file_magic_is_gzip(const char header[4]) { /* GZIP itself starts with the magic bytes 0x1f 0x8b. diff --git a/source/blender/sequencer/intern/image_cache.c b/source/blender/sequencer/intern/image_cache.c index 604c9900355..86bd840ce31 100644 --- a/source/blender/sequencer/intern/image_cache.c +++ b/source/blender/sequencer/intern/image_cache.c @@ -102,7 +102,7 @@ /* -x-%()-.dcf */ #define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf" #define DCACHE_IMAGES_PER_FILE 100 -#define DCACHE_CURRENT_VERSION 1 +#define DCACHE_CURRENT_VERSION 2 #define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */ typedef struct DiskCacheHeaderEntry { @@ -496,24 +496,34 @@ static size_t deflate_imbuf_to_file(ImBuf *ibuf, int level, DiskCacheHeaderEntry *header_entry) { - if (ibuf->rect) { - return BLI_gzip_mem_to_file_at_pos( - ibuf->rect, header_entry->size_raw, file, header_entry->offset, level); + void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float; + + /* Apply compression if wanted, otherwise just write directly to the file. */ + if (level > 0) { + return BLI_file_zstd_from_mem_at_pos( + data, header_entry->size_raw, file, header_entry->offset, level); } - return BLI_gzip_mem_to_file_at_pos( - ibuf->rect_float, header_entry->size_raw, file, header_entry->offset, level); + fseek(file, header_entry->offset, SEEK_SET); + return fwrite(data, 1, header_entry->size_raw, file); } static size_t inflate_file_to_imbuf(ImBuf *ibuf, FILE *file, DiskCacheHeaderEntry *header_entry) { - if (ibuf->rect) { - return BLI_ungzip_file_to_mem_at_pos( - ibuf->rect, header_entry->size_raw, file, header_entry->offset); + void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float; + char header[4]; + fseek(file, header_entry->offset, SEEK_SET); + if (fread(header, 1, sizeof(header), file) != sizeof(header)) { + return 0; + } + + /* Check if the data is compressed or raw. */ + if (BLI_file_magic_is_zstd(header)) { + return BLI_file_unzstd_to_mem_at_pos(data, header_entry->size_raw, file, header_entry->offset); } - return BLI_ungzip_file_to_mem_at_pos( - ibuf->rect_float, header_entry->size_raw, file, header_entry->offset); + fseek(file, header_entry->offset, SEEK_SET); + return fread(data, 1, header_entry->size_raw, file); } static bool seq_disk_cache_read_header(FILE *file, DiskCacheHeader *header) -- cgit v1.2.3