diff options
| -rw-r--r-- | release/datafiles/userdef/userdef_default.c | 5 | ||||
| -rw-r--r-- | release/scripts/startup/bl_ui/space_userpref.py | 9 | ||||
| -rw-r--r-- | source/blender/blenkernel/BKE_sequencer.h | 9 | ||||
| -rw-r--r-- | source/blender/blenkernel/intern/seqcache.c | 772 | ||||
| -rw-r--r-- | source/blender/blenkernel/intern/sequencer.c | 32 | ||||
| -rw-r--r-- | source/blender/blenlib/BLI_fileops.h | 8 | ||||
| -rw-r--r-- | source/blender/blenlib/intern/fileops.c | 95 | ||||
| -rw-r--r-- | source/blender/makesdna/DNA_sequence_types.h | 4 | ||||
| -rw-r--r-- | source/blender/makesdna/DNA_userdef_types.h | 14 | ||||
| -rw-r--r-- | source/blender/makesrna/intern/rna_sequencer.c | 2 | ||||
| -rw-r--r-- | source/blender/makesrna/intern/rna_userdef.c | 59 |
11 files changed, 948 insertions, 61 deletions
diff --git a/release/datafiles/userdef/userdef_default.c b/release/datafiles/userdef/userdef_default.c index 18750c12d55..ec4b214c5f5 100644 --- a/release/datafiles/userdef/userdef_default.c +++ b/release/datafiles/userdef/userdef_default.c @@ -221,6 +221,11 @@ const UserDef U_default = { .temp_win_sizey = 600, }, + .sequencer_disk_cache_dir = "", + .sequencer_disk_cache_compression = 0, + .sequencer_disk_cache_size_limit = 100, + .sequencer_disk_cache_flag = 0, + .runtime = { .is_dirty = 0, diff --git a/release/scripts/startup/bl_ui/space_userpref.py b/release/scripts/startup/bl_ui/space_userpref.py index 4cccf429179..03828632d64 100644 --- a/release/scripts/startup/bl_ui/space_userpref.py +++ b/release/scripts/startup/bl_ui/space_userpref.py @@ -628,6 +628,15 @@ class USERPREF_PT_system_memory(SystemPanel, CenterAlignMixIn, Panel): flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=False) + flow.prop(system, "use_sequencer_disk_cache") + flow.prop(system, "sequencer_disk_cache_dir") + flow.prop(system, "sequencer_disk_cache_size_limit") + flow.prop(system, "sequencer_disk_cache_compression") + + layout.separator() + + flow = layout.grid_flow(row_major=False, columns=0, even_columns=True, even_rows=False, align=False) + flow.prop(system, "texture_time_out", text="Texture Time Out") flow.prop(system, "texture_collection_rate", text="Garbage Collection Rate") diff --git a/source/blender/blenkernel/BKE_sequencer.h b/source/blender/blenkernel/BKE_sequencer.h index 31951cc101a..df0f0e730a5 100644 --- a/source/blender/blenkernel/BKE_sequencer.h +++ b/source/blender/blenkernel/BKE_sequencer.h @@ -312,19 +312,22 @@ void BKE_sequencer_proxy_set(struct Sequence *seq, bool value); struct ImBuf *BKE_sequencer_cache_get(const SeqRenderData *context, struct Sequence *seq, float cfra, - int type); + int type, + bool skip_disk_cache); void BKE_sequencer_cache_put(const SeqRenderData *context, struct Sequence *seq, float cfra, int type, struct ImBuf *nval, - float cost); + float cost, + bool skip_disk_cache); bool BKE_sequencer_cache_put_if_possible(const SeqRenderData *context, struct Sequence *seq, float cfra, int type, struct ImBuf *nval, - float cost); + float cost, + bool skip_disk_cache); bool BKE_sequencer_cache_recycle_item(struct Scene *scene); void BKE_sequencer_cache_free_temp_cache(struct Scene *scene, short id, int cfra); void BKE_sequencer_cache_destruct(struct Scene *scene); diff --git a/source/blender/blenkernel/intern/seqcache.c b/source/blender/blenkernel/intern/seqcache.c index aa4066b643f..f98af4a9406 100644 --- a/source/blender/blenkernel/intern/seqcache.c +++ b/source/blender/blenkernel/intern/seqcache.c @@ -22,20 +22,29 @@ #include <stddef.h> #include <memory.h> +#include <time.h> #include "MEM_guardedalloc.h" #include "DNA_sequence_types.h" #include "DNA_scene_types.h" +#include "DNA_space_types.h" /* for FILE_MAX. */ #include "IMB_imbuf.h" #include "IMB_imbuf_types.h" +#include "IMB_colormanagement.h" #include "BLI_mempool.h" #include "BLI_threads.h" #include "BLI_listbase.h" #include "BLI_ghash.h" +#include "BLI_blenlib.h" +#include "BLI_endian_switch.h" +#include "BLI_fileops.h" +#include "BLI_fileops_types.h" +#include "BLI_path_util.h" +#include "BKE_global.h" #include "BKE_sequencer.h" #include "BKE_scene.h" #include "BKE_main.h" @@ -63,15 +72,75 @@ * entries one by one in reverse order to their creation. * * User can exclude caching of some images. Such entries will have is_temp_cache set. + * + * + * Disk Cache Design Notes + * ======================= + * + * Disk cache uses directory specified in user preferences + * For each cached non-temp image, image data and supplementary info are written to HDD. + * Multiple(DCACHE_IMAGES_PER_FILE) images share the same file. + * Each of these files contains header DiskCacheHeader followed by image data. + * Zlib compression with user definable level can be used to compress image data(per image) + * Images are written in oreder in which they are rendered. + * Overwriting of individual entry is not possible. + * Stored images are deleted by invalidation, or when size of all files exceeds maximum + * size specified in user preferences. + * To distinguish 2 blend files with same name, scene->ed->disk_cache_timestamp + * is used as UID. Blend file can still be copied manually which may cause conflict. + * */ +/* <cache type>-<resolution X>x<resolution Y>-<rendersize>%(<view_id>)-<frame no>.dcf */ +#define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf" +#define DCACHE_IMAGES_PER_FILE 100 +#define DCACHE_CURRENT_VERSION 1 +#define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */ + +typedef struct DiskCacheHeaderEntry { + unsigned char encoding; + uint64_t frameno; + uint64_t size_compressed; + uint64_t size_raw; + uint64_t offset; + char colorspace_name[COLORSPACE_NAME_MAX]; +} DiskCacheHeaderEntry; + +typedef struct DiskCacheHeader { + DiskCacheHeaderEntry entry[DCACHE_IMAGES_PER_FILE]; +} DiskCacheHeader; + +typedef struct SeqDiskCache { + Main *bmain; + int64_t timestamp; + ListBase files; + ThreadMutex read_write_mutex; + size_t size_total; +} SeqDiskCache; + +typedef struct DiskCacheFile { + struct DiskCacheFile *next, *prev; + char path[FILE_MAX]; + char dir[FILE_MAXDIR]; + char file[FILE_MAX]; + BLI_stat_t fstat; + int cache_type; + int rectx; + int recty; + int render_size; + int view_id; + int start_frame; +} DiskCacheFile; + typedef struct SeqCache { + Main *bmain; struct GHash *hash; ThreadMutex iterator_mutex; struct BLI_mempool *keys_pool; struct BLI_mempool *items_pool; struct SeqCacheKey *last_key; size_t memory_used; + SeqDiskCache *disk_cache; } SeqCache; typedef struct SeqCacheItem { @@ -82,8 +151,8 @@ typedef struct SeqCacheItem { typedef struct SeqCacheKey { struct SeqCache *cache_owner; void *userkey; - struct SeqCacheKey *link_prev; /* Used for linking intermediate items to final frame */ - struct SeqCacheKey *link_next; /* Used for linking intermediate items to final frame */ + struct SeqCacheKey *link_prev; /* Used for linking intermediate items to final frame. */ + struct SeqCacheKey *link_next; /* Used for linking intermediate items to final frame. */ struct Sequence *seq; SeqRenderData context; float nfra; @@ -95,6 +164,530 @@ typedef struct SeqCacheKey { } SeqCacheKey; static ThreadMutex cache_create_lock = BLI_MUTEX_INITIALIZER; +static float seq_cache_cfra_to_frame_index(Sequence *seq, float cfra); +static float seq_cache_frame_index_to_cfra(Sequence *seq, float nfra); + +static char *seq_disk_cache_base_dir(void) +{ + return U.sequencer_disk_cache_dir; +} + +static int seq_disk_cache_compression_level(void) +{ + switch (U.sequencer_disk_cache_compression) { + case USER_SEQ_DISK_CACHE_COMPRESSION_NONE: + return 0; + case USER_SEQ_DISK_CACHE_COMPRESSION_LOW: + return 1; + case USER_SEQ_DISK_CACHE_COMPRESSION_HIGH: + return 9; + } + + return U.sequencer_disk_cache_compression; +} + +static size_t seq_disk_cache_size_limit(void) +{ + return (size_t)U.sequencer_disk_cache_size_limit * (1024 * 1024 * 1024); +} + +static bool seq_disk_cache_is_enabled(Main *bmain) +{ + return (U.sequencer_disk_cache_dir[0] != '\0' && U.sequencer_disk_cache_size_limit != 0 && + (U.sequencer_disk_cache_flag & SEQ_CACHE_DISK_CACHE_ENABLE) != 0 && + bmain->name[0] != '\0'); +} + +static DiskCacheFile *seq_disk_cache_add_file_to_list(SeqDiskCache *disk_cache, const char *path) +{ + + DiskCacheFile *cache_file = MEM_callocN(sizeof(DiskCacheFile), "SeqDiskCacheFile"); + char dir[FILE_MAXDIR], file[FILE_MAX]; + BLI_split_dirfile(path, dir, file, sizeof(dir), sizeof(file)); + BLI_strncpy(cache_file->path, path, sizeof(cache_file->path)); + BLI_strncpy(cache_file->dir, dir, sizeof(cache_file->dir)); + BLI_strncpy(cache_file->file, file, sizeof(cache_file->file)); + sscanf(file, + DCACHE_FNAME_FORMAT, + &cache_file->cache_type, + &cache_file->rectx, + &cache_file->recty, + &cache_file->render_size, + &cache_file->view_id, + &cache_file->start_frame); + cache_file->start_frame *= DCACHE_IMAGES_PER_FILE; + BLI_addtail(&disk_cache->files, cache_file); + return cache_file; +} + +static void seq_disk_cache_get_files(SeqDiskCache *disk_cache, char *path) +{ + struct direntry *filelist, *fl; + uint nbr, i; + disk_cache->size_total = 0; + + i = nbr = BLI_filelist_dir_contents(path, &filelist); + fl = filelist; + while (i--) { + /* Don't follow links. */ + const eFileAttributes file_attrs = BLI_file_attributes(fl->path); + if (file_attrs & FILE_ATTR_ANY_LINK) { + fl++; + continue; + } + + char file[FILE_MAX]; + BLI_split_dirfile(fl->path, NULL, file, 0, sizeof(file)); + + bool is_dir = BLI_is_dir(fl->path); + if (is_dir && !FILENAME_IS_CURRPAR(file)) { + char subpath[FILE_MAX]; + BLI_strncpy(subpath, fl->path, sizeof(subpath)); + BLI_add_slash(subpath); + seq_disk_cache_get_files(disk_cache, subpath); + } + + if (!is_dir) { + const char *ext = BLI_path_extension(fl->path); + if (ext && ext[1] == 'd' && ext[2] == 'c' && ext[3] == 'f') { + DiskCacheFile *cache_file = seq_disk_cache_add_file_to_list(disk_cache, fl->path); + cache_file->fstat = fl->s; + disk_cache->size_total += cache_file->fstat.st_size; + } + } + fl++; + } + BLI_filelist_free(filelist, nbr); +} + +static DiskCacheFile *seq_disk_cache_get_oldest_file(SeqDiskCache *disk_cache) +{ + DiskCacheFile *cache_file = disk_cache->files.first; + if (!cache_file) { + return NULL; + } + + DiskCacheFile *oldest_file = cache_file; + __time64_t oldest_timestamp = cache_file->fstat.st_mtime; + + for (; cache_file; cache_file = cache_file->next) { + if (cache_file->fstat.st_mtime < oldest_timestamp) { + oldest_timestamp = cache_file->fstat.st_mtime; + oldest_file = cache_file; + } + } + + return oldest_file; +} + +static void seq_disk_cache_delete_file(SeqDiskCache *disk_cache, DiskCacheFile *file) +{ + disk_cache->size_total -= file->fstat.st_size; + BLI_delete(file->path, false, false); + BLI_remlink(&disk_cache->files, file); + MEM_freeN(file); +} + +static bool seq_disk_cache_enforce_limits(SeqDiskCache *disk_cache) +{ + BLI_mutex_lock(&disk_cache->read_write_mutex); + while (disk_cache->size_total > seq_disk_cache_size_limit()) { + DiskCacheFile *oldest_file = seq_disk_cache_get_oldest_file(disk_cache); + + if (!oldest_file) { + /* We shouldn't enforce limits with no files, do re-scan. */ + seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir()); + continue; + } + + if (BLI_exists(oldest_file->path) == 0) { + /* File may have been manually deleted during runtime, do re-scan. */ + BLI_freelistN(&disk_cache->files); + seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir()); + continue; + } + + seq_disk_cache_delete_file(disk_cache, oldest_file); + } + BLI_mutex_unlock(&disk_cache->read_write_mutex); + + return true; +} + +static DiskCacheFile *seq_disk_cache_get_file_entry_by_path(SeqDiskCache *disk_cache, char *path) +{ + DiskCacheFile *cache_file = disk_cache->files.first; + + for (; cache_file; cache_file = cache_file->next) { + if (BLI_strcasecmp(cache_file->path, path) == 0) { + return cache_file; + } + } + + return NULL; +} + +/* Update file size and timestamp. */ +static void seq_disk_cache_update_file(SeqDiskCache *disk_cache, char *path) +{ + DiskCacheFile *cache_file; + int64_t size_before; + int64_t size_after; + + cache_file = seq_disk_cache_get_file_entry_by_path(disk_cache, path); + size_before = cache_file->fstat.st_size; + + if (BLI_stat(path, &cache_file->fstat) == -1) { + BLI_assert(false); + memset(&cache_file->fstat, 0, sizeof(BLI_stat_t)); + } + + size_after = cache_file->fstat.st_size; + disk_cache->size_total += size_after - size_before; +} + +/* Path format: + * <cache dir>/<project name>/<scene name>-<timestamp>/<seq name>/DCACHE_FNAME_FORMAT + */ + +static void seq_disk_cache_get_project_dir(SeqDiskCache *disk_cache, char *path, size_t path_len) +{ + char main_name[FILE_MAX]; + BLI_split_file_part(BKE_main_blendfile_path(disk_cache->bmain), main_name, sizeof(main_name)); + BLI_strncpy(path, seq_disk_cache_base_dir(), path_len); + BLI_path_append(path, path_len, main_name); +} + +static void seq_disk_cache_get_dir( + SeqDiskCache *disk_cache, Scene *scene, Sequence *seq, char *path, size_t path_len) +{ + char scene_name[MAX_ID_NAME + 22]; /* + -%PRId64 */ + char seq_name[SEQ_NAME_MAXSTR]; + char project_dir[FILE_MAX]; + + seq_disk_cache_get_project_dir(disk_cache, project_dir, sizeof(project_dir)); + sprintf(scene_name, "%s-%" PRId64, scene->id.name, disk_cache->timestamp); + BLI_strncpy(seq_name, seq->name, sizeof(seq_name)); + BLI_filename_make_safe(scene_name); + BLI_filename_make_safe(seq_name); + BLI_strncpy(path, project_dir, path_len); + BLI_path_append(path, path_len, scene_name); + BLI_path_append(path, path_len, seq_name); +} + +static void seq_disk_cache_get_file_path(SeqDiskCache *disk_cache, + SeqCacheKey *key, + char *path, + size_t path_len) +{ + seq_disk_cache_get_dir(disk_cache, key->context.scene, key->seq, path, path_len); + int frameno = (int)key->nfra / DCACHE_IMAGES_PER_FILE; + char cache_filename[FILE_MAXFILE]; + sprintf(cache_filename, + DCACHE_FNAME_FORMAT, + key->type, + key->context.rectx, + key->context.recty, + key->context.preview_render_size, + key->context.view_id, + frameno); + + BLI_path_append(path, path_len, cache_filename); +} + +static void seq_disk_cache_create_version_file(char *path) +{ + BLI_make_existing_file(path); + + FILE *file = BLI_fopen(path, "w"); + if (file) { + fprintf(file, "%d", DCACHE_CURRENT_VERSION); + fclose(file); + } +} + +static void seq_disk_cache_handle_versioning(SeqDiskCache *disk_cache) +{ + char path[FILE_MAX]; + char path_version_file[FILE_MAX]; + int version = 0; + + seq_disk_cache_get_project_dir(disk_cache, path, sizeof(path)); + BLI_strncpy(path_version_file, path, sizeof(path_version_file)); + BLI_path_append(path_version_file, sizeof(path_version_file), "cache_version"); + + if (BLI_exists(path)) { + FILE *file = BLI_fopen(path_version_file, "r"); + + if (file) { + fscanf(file, "%d", &version); + fclose(file); + } + + if (version != DCACHE_CURRENT_VERSION) { + BLI_delete(path, false, true); + seq_disk_cache_create_version_file(path_version_file); + } + } + else { + seq_disk_cache_create_version_file(path_version_file); + } +} + +static void seq_disk_cache_delete_invalid_files(SeqDiskCache *disk_cache, + Scene *scene, + Sequence *seq, + int invalidate_types, + int range_start, + int range_end) +{ + DiskCacheFile *next_file, *cache_file = disk_cache->files.first; + char cache_dir[FILE_MAX]; + seq_disk_cache_get_dir(disk_cache, scene, seq, cache_dir, sizeof(cache_dir)); + BLI_add_slash(cache_dir); + + while (cache_file) { + next_file = cache_file->next; + if (cache_file->cache_type & invalidate_types) { + if (strcmp(cache_dir, cache_file->dir) == 0) { + int cfra_start = seq_cache_frame_index_to_cfra(seq, cache_file->start_frame); + if (cfra_start > range_start && cfra_start <= range_end) { + seq_disk_cache_delete_file(disk_cache, cache_file); + } + } + } + cache_file = next_file; + } +} + +static void seq_disk_cache_invalidate(Scene *scene, + Sequence *seq, + Sequence *seq_changed, + int invalidate_types) +{ + int start; + int end; + SeqDiskCache *disk_cache = scene->ed->cache->disk_cache; + + BLI_mutex_lock(&disk_cache->read_write_mutex); + + start = seq_changed->startdisp - DCACHE_IMAGES_PER_FILE; + end = seq_changed->enddisp; + + seq_disk_cache_delete_invalid_files(disk_cache, scene, seq, invalidate_types, start, end); + + BLI_mutex_unlock(&disk_cache->read_write_mutex); +} + +static size_t deflate_imbuf_to_file(ImBuf *ibuf, + FILE *file, + int level, + DiskCacheHeaderEntry *header_entry) +{ + if (ibuf->rect) { + return BLI_gzip_mem_to_file_at_pos( + ibuf->rect, header_entry->size_raw, file, header_entry->offset, level); + } + else { + return BLI_gzip_mem_to_file_at_pos( + ibuf->rect_float, header_entry->size_raw, file, header_entry->offset, level); + } +} + +static size_t inflate_file_to_imbuf(ImBuf *ibuf, FILE *file, DiskCacheHeaderEntry *header_entry) +{ + if (ibuf->rect) { + return BLI_ungzip_file_to_mem_at_pos( + ibuf->rect, header_entry->size_raw, file, header_entry->offset); + } + else { + return BLI_ungzip_file_to_mem_at_pos( + ibuf->rect_float, header_entry->size_raw, file, header_entry->offset); + } +} + +static void seq_disk_cache_read_header(FILE *file, DiskCacheHeader *header) +{ + fseek(file, 0, 0); + fread(header, sizeof(*header), 1, file); + + for (int i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { + if ((ENDIAN_ORDER == B_ENDIAN) && header->entry[i].encoding == 0) { + BLI_endian_switch_uint64(&header->entry[i].frameno); + BLI_endian_switch_uint64(&header->entry[i].offset); + BLI_endian_switch_uint64(&header->entry[i].size_compressed); + BLI_endian_switch_uint64(&header->entry[i].size_raw); + } + } +} + +static size_t seq_disk_cache_write_header(FILE *file, DiskCacheHeader *header) +{ + fseek(file, 0, 0); + return fwrite(header, sizeof(*header), 1, file); +} + +static int seq_disk_cache_add_header_entry(SeqCacheKey *key, ImBuf *ibuf, DiskCacheHeader *header) +{ + int i; + uint64_t offset = sizeof(*header); + + /* Lookup free entry, get offset for new data. */ + for (i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { + if (header->entry[i].size_compressed == 0) { + break; + } + } + + /* Attempt to write beyond set entry limit. + * Reset file header and start writing from beginning. + */ + if (i == DCACHE_IMAGES_PER_FILE) { + i = 0; + memset(header, 0, sizeof(*header)); + } + + /* Calculate offset for image data. */ + if (i > 0) { + offset = header->entry[i - 1].offset + header->entry[i - 1].size_compressed; + } + + if (ENDIAN_ORDER == B_ENDIAN) { + header->entry[i].encoding = 255; + } + else { + header->entry[i].encoding = 0; + } + + header->entry[i].offset = offset; + header->entry[i].frameno = key->nfra; + + /* Store colorspace name of ibuf. */ + const char *colorspace_name; + if (ibuf->rect) { + header->entry[i].size_raw = ibuf->x * ibuf->y * ibuf->channels; + colorspace_name = IMB_colormanagement_get_rect_colorspace(ibuf); + } + else { + header->entry[i].size_raw = ibuf->x * ibuf->y * ibuf->channels * 4; + colorspace_name = IMB_colormanagement_get_float_colorspace(ibuf); + } + BLI_strncpy( + header->entry[i].colorspace_name, colorspace_name, sizeof(header->entry[i].colorspace_name)); + + return i; +} + +static int seq_disk_cache_get_header_entry(SeqCacheKey *key, DiskCacheHeader *header) +{ + for (int i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { + if (header->entry[i].frameno == key->nfra) { + return i; + } + } + + return -1; +} + +static bool seq_disk_cache_write_file(SeqDiskCache *disk_cache, SeqCacheKey *key, ImBuf *ibuf) +{ + char path[FILE_MAX]; + + seq_disk_cache_get_file_path(disk_cache, key, path, sizeof(path)); + BLI_make_existing_file(path); + + FILE *file = BLI_fopen(path, "rb+"); + if (!file) { + file = BLI_fopen(path, "wb+"); + if (!file) { + return false; + } + seq_disk_cache_add_file_to_list(disk_cache, path); + } + + DiskCacheHeader header; + memset(&header, 0, sizeof(header)); + seq_disk_cache_read_header(file, &header); + int entry_index = seq_disk_cache_add_header_entry(key, ibuf, &header); + size_t bytes_written = deflate_imbuf_to_file( + ibuf, file, seq_disk_cache_compression_level(), &header.entry[entry_index]); + + if (bytes_written != 0) { + /* Last step is writing header, as image data can be overwritten, + * but missing data would cause problems. + */ + header.entry[entry_index].size_compressed = bytes_written; + seq_disk_cache_write_header(file, &header); + seq_disk_cache_update_file(disk_cache, path); + fclose(file); + + return true; + } + + return false; +} + +static ImBuf *seq_disk_cache_read_file(SeqDiskCache *disk_cache, SeqCacheKey *key) +{ + char path[FILE_MAX]; + DiskCacheHeader header; + + seq_disk_cache_get_file_path(disk_cache, key, path, sizeof(path)); + BLI_make_existing_file(path); + + FILE *file = BLI_fopen(path, "rb"); + if (!file) { + return NULL; + } + + seq_disk_cache_read_header(file, &header); + int entry_index = seq_disk_cache_get_header_entry(key, &header); + + /* Item not found. */ + if (entry_index < 0) { + fclose(file); + return NULL; + } + + ImBuf *ibuf; + uint64_t size_char = (uint64_t)key->context.rectx * key->context.recty * 4; + uint64_t size_float = (uint64_t)key->context.rectx * key->context.recty * 16; + size_t expected_size; + + if (header.entry[entry_index].size_raw == size_char) { + expected_size = size_char; + ibuf = IMB_allocImBuf(key->context.rectx, key->context.recty, 32, IB_rect); + IMB_colormanagement_assign_rect_colorspace(ibuf, header.entry[entry_index].colorspace_name); + } + else if (header.entry[entry_index].size_raw == size_float) { + expected_size = size_float; + ibuf = IMB_allocImBuf(key->context.rectx, key->context.recty, 32, IB_rectfloat); + IMB_colormanagement_assign_float_colorspace(ibuf, header.entry[entry_index].colorspace_name); + } + else { + fclose(file); + return NULL; + } + + size_t bytes_read = inflate_file_to_imbuf(ibuf, file, &header.entry[entry_index]); + + /* Sanity check. */ + if (bytes_read != expected_size) { + fclose(file); + IMB_freeImBuf(ibuf); + return NULL; + } + BLI_file_touch(path); + seq_disk_cache_update_file(disk_cache, path); + fclose(file); + + return ibuf; +} + +#undef DCACHE_FNAME_FORMAT +#undef DCACHE_IMAGES_PER_FILE +#undef COLORSPACE_NAME_MAX +#undef DCACHE_CURRENT_VERSION static bool seq_cmp_render_data(const SeqRenderData *a, const SeqRenderData *b) { @@ -140,6 +733,16 @@ static bool seq_cache_hashcmp(const void *a_, const void *b_) seq_cmp_render_data(&a->context, &b->context)); } +static float seq_cache_cfra_to_frame_index(Sequence *seq, float cfra) +{ + return cfra - seq->start; +} + +static float seq_cache_frame_index_to_cfra(Sequence *seq, float nfra) +{ + return nfra + seq->start; +} + static SeqCache *seq_cache_get_from_scene(Scene *scene) { if (scene && scene->ed && scene->ed->cache) { @@ -205,7 +808,7 @@ static void seq_cache_put(SeqCache *cache, SeqCacheKey *key, ImBuf *ibuf) } } -static ImBuf *seq_cache_get(SeqCache *cache, void *key) +static ImBuf *seq_cache_get(SeqCache *cache, SeqCacheKey *key) { SeqCacheItem *item = BLI_ghash_lookup(cache->hash, key); @@ -250,14 +853,14 @@ static SeqCacheKey *seq_cache_choose_key(Scene *scene, SeqCacheKey *lkey, SeqCac BKE_sequencer_prefetch_get_time_range(scene, &pfjob_start, &pfjob_end); if (lkey) { - int lkey_cfra = lkey->seq->start + lkey->nfra; + int lkey_cfra = seq_cache_frame_index_to_cfra(lkey->seq, lkey->nfra); if (lkey_cfra < pfjob_start || lkey_cfra > pfjob_end) { return lkey; } } if (rkey) { - int rkey_cfra = rkey->seq->start + rkey->nfra; + int rkey_cfra = seq_cache_frame_index_to_cfra(rkey->seq, rkey->nfra); if (rkey_cfra < pfjob_start || rkey_cfra > pfjob_end) { return rkey; } @@ -267,8 +870,8 @@ static SeqCacheKey *seq_cache_choose_key(Scene *scene, SeqCacheKey *lkey, SeqCac } if (rkey && lkey) { - int lkey_cfra = lkey->seq->start + lkey->nfra; - int rkey_cfra = rkey->seq->start + rkey->nfra; + int lkey_cfra = seq_cache_frame_index_to_cfra(lkey->seq, lkey->nfra); + int rkey_cfra = seq_cache_frame_index_to_cfra(rkey->seq, rkey->nfra); if (lkey_cfra > rkey_cfra) { SeqCacheKey *swapkey = lkey; @@ -324,9 +927,9 @@ static SeqCacheKey *seq_cache_get_item_for_removal(Scene *scene) { SeqCache *cache = seq_cache_get_from_scene(scene); SeqCacheKey *finalkey = NULL; - /*leftmost key*/ + /* Leftmost key. */ SeqCacheKey *lkey = NULL; - /*rightmost key*/ + /* Rightmost key. */ SeqCacheKey *rkey = NULL; SeqCacheKey *key = NULL; @@ -340,10 +943,10 @@ static SeqCacheKey *seq_cache_get_item_for_removal(Scene *scene) SeqCacheItem *item = BLI_ghashIterator_getValue(&gh_iter); BLI_ghashIterator_step(&gh_iter); - /* this shouldn't happen, but better be safe than sorry */ + /* This shouldn't happen, but better be safe than sorry. */ if (!item->ibuf) { seq_cache_recycle_linked(scene, key); - /* can not continue iterating after linked remove */ + /* Can not continue iterating after linked remove. */ BLI_ghashIterator_init(&gh_iter, cache->hash); continue; } @@ -357,7 +960,8 @@ static SeqCacheKey *seq_cache_get_item_for_removal(Scene *scene) if (key->cost <= scene->ed->recycle_max_cost) { cheap_count++; if (lkey) { - if (key->seq->start + key->nfra < lkey->seq->start + lkey->nfra) { + if (seq_cache_frame_index_to_cfra(key->seq, key->nfra) < + seq_cache_frame_index_to_cfra(lkey->seq, lkey->nfra)) { lkey = key; } } @@ -365,7 +969,8 @@ static SeqCacheKey *seq_cache_get_item_for_removal(Scene *scene) lkey = key; } if (rkey) { - if (key->seq->start + key->nfra > rkey->seq->start + rkey->nfra) { + if (seq_cache_frame_index_to_cfra(key->seq, key->nfra) > + seq_cache_frame_index_to_cfra(rkey->seq, rkey->nfra)) { rkey = key; } } @@ -380,8 +985,8 @@ static SeqCacheKey *seq_cache_get_item_for_removal(Scene *scene) return finalkey; } -/* Find only "base" keys - * Sources(other types) for a frame must be freed all at once +/* Find only "base" keys. + * Sources(other types) for a frame must be freed all at once. */ bool BKE_sequencer_cache_recycle_item(Scene *scene) { @@ -432,7 +1037,29 @@ static void seq_cache_set_temp_cache_linked(Scene *scene, SeqCacheKey *base) } } -static void BKE_sequencer_cache_create(Scene *scene) +static void seq_disk_cache_create(Main *bmain, Scene *scene) +{ + BLI_mutex_lock(&cache_create_lock); + SeqCache *cache = seq_cache_get_from_scene(scene); + + if (cache == NULL) { + return; + } + + if (cache->disk_cache != NULL) { + return; + } + + cache->disk_cache = MEM_callocN(sizeof(SeqDiskCache), "SeqDiskCache"); + cache->disk_cache->bmain = bmain; + BLI_mutex_init(&cache->disk_cache->read_write_mutex); + seq_disk_cache_handle_versioning(cache->disk_cache); + seq_disk_cache_get_files(cache->disk_cache, seq_disk_cache_base_dir()); + cache->disk_cache->timestamp = scene->ed->disk_cache_timestamp; + BLI_mutex_unlock(&cache_create_lock); +} + +static void seq_cache_create(Main *bmain, Scene *scene) { BLI_mutex_lock(&cache_create_lock); if (scene->ed->cache == NULL) { @@ -441,8 +1068,13 @@ static void BKE_sequencer_cache_create(Scene *scene) cache->items_pool = BLI_mempool_create(sizeof(SeqCacheItem), 0, 64, BLI_MEMPOOL_NOP); cache->hash = BLI_ghash_new(seq_cache_hashhash, seq_cache_hashcmp, "SeqCache hash"); cache->last_key = NULL; + cache->bmain = bmain; BLI_mutex_init(&cache->iterator_mutex); scene->ed->cache = cache; + + if (scene->ed->disk_cache_timestamp == 0) { + scene->ed->disk_cache_timestamp = time(NULL); + } } BLI_mutex_unlock(&cache_create_lock); } @@ -464,7 +1096,8 @@ void BKE_sequencer_cache_free_temp_cache(Scene *scene, short id, int cfra) SeqCacheKey *key = BLI_ghashIterator_getKey(&gh_iter); BLI_ghashIterator_step(&gh_iter); - if (key->is_temp_cache && key->task_id == id && key->seq->start + key->nfra != cfra) { + if (key->is_temp_cache && key->task_id == id && + seq_cache_frame_index_to_cfra(key->seq, key->nfra) != cfra) { BLI_ghash_remove(cache->hash, key, seq_cache_keyfree, seq_cache_valfree); } } @@ -482,6 +1115,13 @@ void BKE_sequencer_cache_destruct(Scene *scene) BLI_mempool_destroy(cache->keys_pool); BLI_mempool_destroy(cache->items_pool); BLI_mutex_end(&cache->iterator_mutex); + + if (cache->disk_cache != NULL) { + BLI_freelistN(&cache->disk_cache->files); + BLI_mutex_end(&cache->disk_cache->read_write_mutex); + MEM_freeN(cache->disk_cache); + } + MEM_freeN(cache); scene->ed->cache = NULL; } @@ -525,6 +1165,10 @@ void BKE_sequencer_cache_cleanup_sequence(Scene *scene, return; } + if (seq_disk_cache_is_enabled(cache->bmain) && cache->disk_cache != NULL) { + seq_disk_cache_invalidate(scene, seq, seq_changed, invalidate_types); + } + seq_cache_lock(scene); int range_start = seq_changed->startdisp; @@ -548,9 +1192,9 @@ void BKE_sequencer_cache_cleanup_sequence(Scene *scene, SeqCacheKey *key = BLI_ghashIterator_getKey(&gh_iter); BLI_ghashIterator_step(&gh_iter); - int key_cfra = key->seq->start + key->nfra; + int key_cfra = seq_cache_frame_index_to_cfra(key->seq, key->nfra); - /* clean all final and composite in intersection of seq and seq_changed */ + /* Clean all final and composite in intersection of seq and seq_changed. */ if (key->type & invalidate_composite && key_cfra >= range_start && key_cfra <= range_end) { if (key->link_next || key->link_prev) { seq_cache_relink_keys(key->link_next, key->link_prev); @@ -572,10 +1216,8 @@ void BKE_sequencer_cache_cleanup_sequence(Scene *scene, seq_cache_unlock(scene); } -struct ImBuf *BKE_sequencer_cache_get(const SeqRenderData *context, - Sequence *seq, - float cfra, - int type) +struct ImBuf *BKE_sequencer_cache_get( + const SeqRenderData *context, Sequence *seq, float cfra, int type, bool skip_disk_cache) { Scene *scene = context->scene; @@ -586,31 +1228,58 @@ struct ImBuf *BKE_sequencer_cache_get(const SeqRenderData *context, } if (!scene->ed->cache) { - BKE_sequencer_cache_create(scene); - return NULL; + seq_cache_create(context->bmain, scene); } seq_cache_lock(scene); SeqCache *cache = seq_cache_get_from_scene(scene); ImBuf *ibuf = NULL; + SeqCacheKey key; + /* Try RAM cache: */ if (cache && seq) { - SeqCacheKey key; - key.seq = seq; key.context = *context; - key.nfra = cfra - seq->start; + key.nfra = seq_cache_cfra_to_frame_index(seq, cfra); key.type = type; ibuf = seq_cache_get(cache, &key); } seq_cache_unlock(scene); + if (ibuf) { + return ibuf; + } + + /* Try disk cache: */ + if (!skip_disk_cache && seq_disk_cache_is_enabled(context->bmain)) { + if (cache->disk_cache == NULL) { + seq_disk_cache_create(context->bmain, context->scene); + } + + BLI_mutex_lock(&cache->disk_cache->read_write_mutex); + ibuf = seq_disk_cache_read_file(cache->disk_cache, &key); + BLI_mutex_unlock(&cache->disk_cache->read_write_mutex); + if (ibuf) { + if (key.type == SEQ_CACHE_STORE_FINAL_OUT) { + BKE_sequencer_cache_put_if_possible(context, seq, cfra, type, ibuf, 0.0f, true); + } + else { + BKE_sequencer_cache_put(context, seq, cfra, type, ibuf, 0.0f, true); + } + } + } + return ibuf; } -bool BKE_sequencer_cache_put_if_possible( - const SeqRenderData *context, Sequence *seq, float cfra, int type, ImBuf *ibuf, float cost) +bool BKE_sequencer_cache_put_if_possible(const SeqRenderData *context, + Sequence *seq, + float cfra, + int type, + ImBuf *ibuf, + float cost, + bool skip_disk_cache) { Scene *scene = context->scene; @@ -621,7 +1290,7 @@ bool BKE_sequencer_cache_put_if_possible( } if (BKE_sequencer_cache_recycle_item(scene)) { - BKE_sequencer_cache_put(context, seq, cfra, type, ibuf, cost); + BKE_sequencer_cache_put(context, seq, cfra, type, ibuf, cost, skip_disk_cache); return true; } else { @@ -631,8 +1300,13 @@ bool BKE_sequencer_cache_put_if_possible( } } -void BKE_sequencer_cache_put( - const SeqRenderData *context, Sequence *seq, float cfra, int type, ImBuf *i, float cost) +void BKE_sequencer_cache_put(const SeqRenderData *context, + Sequence *seq, + float cfra, + int type, + ImBuf *i, + float cost, + bool skip_disk_cache) { Scene *scene = context->scene; @@ -646,15 +1320,15 @@ void BKE_sequencer_cache_put( return; } - /* Prevent reinserting, it breaks cache key linking */ - ImBuf *test = BKE_sequencer_cache_get(context, seq, cfra, type); + /* Prevent reinserting, it breaks cache key linking. */ + ImBuf *test = BKE_sequencer_cache_get(context, seq, cfra, type, true); if (test) { IMB_freeImBuf(test); return; } if (!scene->ed->cache) { - BKE_sequencer_cache_create(scene); + seq_cache_create(context->bmain, scene); } seq_cache_lock(scene); @@ -664,6 +1338,9 @@ void BKE_sequencer_cache_put( if (seq->cache_flag & SEQ_CACHE_OVERRIDE) { flag = seq->cache_flag; + /* Final_out is invalid in context of sequence override. */ + flag -= seq->cache_flag & SEQ_CACHE_STORE_FINAL_OUT; + /* If global setting is enabled however, use it. */ flag |= scene->ed->cache_flag & SEQ_CACHE_STORE_FINAL_OUT; } else { @@ -679,7 +1356,7 @@ void BKE_sequencer_cache_put( key->cache_owner = cache; key->seq = seq; key->context = *context; - key->nfra = cfra - seq->start; + key->nfra = seq_cache_cfra_to_frame_index(seq, cfra); key->type = type; key->cost = cost; key->link_prev = NULL; @@ -696,24 +1373,37 @@ void BKE_sequencer_cache_put( SeqCacheKey *temp_last_key = cache->last_key; seq_cache_put(cache, key, i); - /* Restore pointer to previous item as this one will be freed when stack is rendered */ + /* Restore pointer to previous item as this one will be freed when stack is rendered. */ if (key->is_temp_cache) { cache->last_key = temp_last_key; } - /* Set last_key's reference to this key so we can look up chain backwards - * Item is already put in cache, so cache->last_key points to current key; + /* Set last_key's reference to this key so we can look up chain backwards. + * Item is already put in cache, so cache->last_key points to current key. */ if (flag & type && temp_last_key) { temp_last_key->link_next = cache->last_key; } - /* Reset linking */ + /* Reset linking. */ if (key->type == SEQ_CACHE_STORE_FINAL_OUT) { cache->last_key = NULL; } seq_cache_unlock(scene); + + if (!key->is_temp_cache && !skip_disk_cache) { + if (seq_disk_cache_is_enabled(context->bmain)) { + if (cache->disk_cache == NULL) { + seq_disk_cache_create(context->bmain, context->scene); + } + + BLI_mutex_lock(&cache->disk_cache->read_write_mutex); + seq_disk_cache_write_file(cache->disk_cache, key, i); + BLI_mutex_unlock(&cache->disk_cache->read_write_mutex); + seq_disk_cache_enforce_limits(cache->disk_cache); + } + } } size_t BKE_sequencer_cache_get_num_items(struct Scene *scene) diff --git a/source/blender/blenkernel/intern/sequencer.c b/source/blender/blenkernel/intern/sequencer.c index 827cccc6bd6..a11326ef099 100644 --- a/source/blender/blenkernel/intern/sequencer.c +++ b/source/blender/blenkernel/intern/sequencer.c @@ -3081,7 +3081,7 @@ static ImBuf *seq_render_image_strip(const SeqRenderData *context, if (i != context->view_id) { BKE_sequencer_cache_put( - &localcontext, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibufs_arr[i], 0); + &localcontext, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibufs_arr[i], 0, false); } } } @@ -3201,7 +3201,7 @@ static ImBuf *seq_render_movie_strip(const SeqRenderData *context, } if (i != context->view_id) { BKE_sequencer_cache_put( - &localcontext, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibuf_arr[i], 0); + &localcontext, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibuf_arr[i], 0, false); } } @@ -3600,7 +3600,8 @@ static ImBuf *seq_render_scene_strip(const SeqRenderData *context, } if (i != context->view_id) { - BKE_sequencer_cache_put(&localcontext, seq, cfra, SEQ_CACHE_STORE_RAW, ibufs_arr[i], 0); + BKE_sequencer_cache_put( + &localcontext, seq, cfra, SEQ_CACHE_STORE_RAW, ibufs_arr[i], 0, false); } RE_ReleaseResultImage(re); @@ -3813,10 +3814,10 @@ static ImBuf *seq_render_strip(const SeqRenderData *context, clock_t begin = seq_estimate_render_cost_begin(); - ibuf = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED); + ibuf = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, false); if (ibuf == NULL) { - ibuf = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_RAW); + ibuf = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_RAW, false); if (ibuf == NULL) { /* MOVIECLIPs have their own proxy management */ if (seq->type != SEQ_TYPE_MOVIECLIP) { @@ -3852,7 +3853,7 @@ static ImBuf *seq_render_strip(const SeqRenderData *context, if (use_preprocess) { float cost = seq_estimate_render_cost_end(context->scene, begin); - BKE_sequencer_cache_put(context, seq, cfra, SEQ_CACHE_STORE_RAW, ibuf, cost); + BKE_sequencer_cache_put(context, seq, cfra, SEQ_CACHE_STORE_RAW, ibuf, cost, false); /* reset timer so we can get partial render time */ begin = seq_estimate_render_cost_begin(); @@ -3860,7 +3861,7 @@ static ImBuf *seq_render_strip(const SeqRenderData *context, } float cost = seq_estimate_render_cost_end(context->scene, begin); - BKE_sequencer_cache_put(context, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibuf, cost); + BKE_sequencer_cache_put(context, seq, cfra, SEQ_CACHE_STORE_PREPROCESSED, ibuf, cost, false); } return ibuf; } @@ -3954,7 +3955,7 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context, int early_out; Sequence *seq = seq_arr[i]; - out = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_COMPOSITE); + out = BKE_sequencer_cache_get(context, seq, cfra, SEQ_CACHE_STORE_COMPOSITE, false); if (out) { break; @@ -3986,7 +3987,8 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context, out = seq_render_strip_stack_apply_effect(context, seq, cfra, ibuf1, ibuf2); float cost = seq_estimate_render_cost_end(context->scene, begin); - BKE_sequencer_cache_put(context, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE, out, cost); + BKE_sequencer_cache_put( + context, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE, out, cost, false); IMB_freeImBuf(ibuf1); IMB_freeImBuf(ibuf2); @@ -4014,7 +4016,8 @@ static ImBuf *seq_render_strip_stack(const SeqRenderData *context, } float cost = seq_estimate_render_cost_end(context->scene, begin); - BKE_sequencer_cache_put(context, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE, out, cost); + BKE_sequencer_cache_put( + context, seq_arr[i], cfra, SEQ_CACHE_STORE_COMPOSITE, out, cost, false); } return out; @@ -4053,7 +4056,8 @@ ImBuf *BKE_sequencer_give_ibuf(const SeqRenderData *context, float cfra, int cha count = get_shown_sequences(seqbasep, cfra, chanshown, seq_arr); if (count) { - out = BKE_sequencer_cache_get(context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT); + out = BKE_sequencer_cache_get( + context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, false); } BKE_sequencer_cache_free_temp_cache(context->scene, context->task_id, cfra); @@ -4068,11 +4072,11 @@ ImBuf *BKE_sequencer_give_ibuf(const SeqRenderData *context, float cfra, int cha if (context->is_prefetch_render) { BKE_sequencer_cache_put( - context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, out, cost); + context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, out, cost, false); } else { BKE_sequencer_cache_put_if_possible( - context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, out, cost); + context, seq_arr[count - 1], cfra, SEQ_CACHE_STORE_FINAL_OUT, out, cost, false); } BLI_mutex_unlock(&seq_render_mutex); } @@ -5294,7 +5298,7 @@ Sequence *BKE_sequence_alloc(ListBase *lb, int cfra, int machine, int type) seq->strip = seq_strip_alloc(type); seq->stereo3d_format = MEM_callocN(sizeof(Stereo3dFormat), "Sequence Stereo Format"); - seq->cache_flag = SEQ_CACHE_ALL_TYPES; + seq->cache_flag = SEQ_CACHE_STORE_RAW | SEQ_CACHE_STORE_PREPROCESSED | SEQ_CACHE_STORE_COMPOSITE; return seq; } diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h index 8596a60bc6a..89f7d01ffd6 100644 --- a/source/blender/blenlib/BLI_fileops.h +++ b/source/blender/blenlib/BLI_fileops.h @@ -153,7 +153,13 @@ int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR #endif char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); - +size_t BLI_gzip_mem_to_file_at_pos(void *buf, + size_t len, + FILE *file, + size_t gz_stream_offset, + int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); +size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset) + ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT; size_t BLI_file_size(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(); diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c index 3a45989fb63..afb46dd8d82 100644 --- a/source/blender/blenlib/intern/fileops.c +++ b/source/blender/blenlib/intern/fileops.c @@ -158,6 +158,101 @@ char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) return mem; } +#define CHUNK 256 * 1024 + +/* gzip byte array from memory and write it to file at certain position. + * return size of gzip stream. + */ +size_t BLI_gzip_mem_to_file_at_pos( + void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level) +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char out[CHUNK]; + + fseek(file, gz_stream_offset, 0); + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, compression_level); + if (ret != Z_OK) + return 0; + + strm.avail_in = len; + strm.next_in = (Bytef *)buf; + flush = Z_FINISH; + + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); + if (ret == Z_STREAM_ERROR) { + return 0; + } + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, file) != have || ferror(file)) { + deflateEnd(&strm); + return 0; + } + } while (strm.avail_out == 0); + + if (strm.avail_in != 0 || ret != Z_STREAM_END) { + return 0; + } + + deflateEnd(&strm); + return (size_t)strm.total_out; +} + +/* read and decompress gzip stream from file at certain position to buffer. + * return size of decompressed data. + */ +size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset) +{ + int ret; + z_stream strm; + size_t chunk = 256 * 1024; + unsigned char in[CHUNK]; + + fseek(file, gz_stream_offset, 0); + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); + if (ret != Z_OK) + return 0; + + do { + strm.avail_in = fread(in, 1, chunk, file); + strm.next_in = in; + if (ferror(file)) { + inflateEnd(&strm); + return 0; + } + + do { + strm.avail_out = len; + strm.next_out = (Bytef *)buf + strm.total_out; + + ret = inflate(&strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR) { + return 0; + } + } while (strm.avail_out == 0); + + } while (ret != Z_STREAM_END); + + inflateEnd(&strm); + return (size_t)strm.total_out; +} + +#undef CHUNK + /** * Returns true if the file with the specified name can be written. * This implementation uses access(2), which makes the check according diff --git a/source/blender/makesdna/DNA_sequence_types.h b/source/blender/makesdna/DNA_sequence_types.h index fb941a61ae8..c275c8187e4 100644 --- a/source/blender/makesdna/DNA_sequence_types.h +++ b/source/blender/makesdna/DNA_sequence_types.h @@ -276,6 +276,9 @@ typedef struct Editing { int cache_flag; struct PrefetchJob *prefetch_job; + + /* Must be initialized only by BKE_sequencer_cache_create() */ + int64_t disk_cache_timestamp; } Editing; /* ************* Effect Variable Structs ********* */ @@ -682,6 +685,7 @@ enum { SEQ_CACHE_VIEW_FINAL_OUT = (1 << 9), SEQ_CACHE_PREFETCH_ENABLE = (1 << 10), + SEQ_CACHE_DISK_CACHE_ENABLE = (1 << 11), }; #ifdef __cplusplus diff --git a/source/blender/makesdna/DNA_userdef_types.h b/source/blender/makesdna/DNA_userdef_types.h index 4a3c5695827..d5ebe4f8778 100644 --- a/source/blender/makesdna/DNA_userdef_types.h +++ b/source/blender/makesdna/DNA_userdef_types.h @@ -862,7 +862,13 @@ typedef struct UserDef { char render_display_type; /* eUserpref_RenderDisplayType */ char filebrowser_display_type; /* eUserpref_TempSpaceDisplayType */ - char _pad5[4]; + + char sequencer_disk_cache_dir[1024]; + int sequencer_disk_cache_compression; /* eUserpref_DiskCacheCompression */ + int sequencer_disk_cache_size_limit; + short sequencer_disk_cache_flag; + + char _pad5[2]; struct WalkNavigation walk_navigation; @@ -1286,6 +1292,12 @@ typedef enum eUserpref_EmulateMMBMod { USER_EMU_MMB_MOD_OSKEY = 1, } eUserpref_EmulateMMBMod; +typedef enum eUserpref_DiskCacheCompression { + USER_SEQ_DISK_CACHE_COMPRESSION_NONE = 0, + USER_SEQ_DISK_CACHE_COMPRESSION_LOW = 1, + USER_SEQ_DISK_CACHE_COMPRESSION_HIGH = 2, +} eUserpref_DiskCacheCompression; + #ifdef __cplusplus } #endif diff --git a/source/blender/makesrna/intern/rna_sequencer.c b/source/blender/makesrna/intern/rna_sequencer.c index 661f1007395..d4dd3365935 100644 --- a/source/blender/makesrna/intern/rna_sequencer.c +++ b/source/blender/makesrna/intern/rna_sequencer.c @@ -1951,7 +1951,7 @@ static void rna_def_editor(BlenderRNA *brna) RNA_def_property_ui_range(prop, 0.0f, SEQ_CACHE_COST_MAX, 0.1f, 1); RNA_def_property_float_sdna(prop, NULL, "recycle_max_cost"); RNA_def_property_ui_text( - prop, "Recycle Up to Cost", "Only frames with cost lower than this value will be recycled"); + prop, "Recycle Up To Cost", "Only frames with cost lower than this value will be recycled"); } static void rna_def_filter_video(StructRNA *srna) diff --git a/source/blender/makesrna/intern/rna_userdef.c b/source/blender/makesrna/intern/rna_userdef.c index 1557cd3d4b1..cada2cb13a9 100644 --- a/source/blender/makesrna/intern/rna_userdef.c +++ b/source/blender/makesrna/intern/rna_userdef.c @@ -192,6 +192,8 @@ static const EnumPropertyItem rna_enum_userdef_viewport_aa_items[] = { # include "BLF_api.h" +# include "BLI_path_util.h" + # include "MEM_guardedalloc.h" # include "MEM_CacheLimiterC-Api.h" @@ -536,6 +538,19 @@ static void rna_Userdef_memcache_update(Main *UNUSED(bmain), USERDEF_TAG_DIRTY; } +static void rna_Userdef_disk_cache_dir_update(Main *UNUSED(bmain), + Scene *UNUSED(scene), + PointerRNA *UNUSED(ptr)) +{ + if (U.sequencer_disk_cache_dir[0] != '\0') { + BLI_path_abs(U.sequencer_disk_cache_dir, BKE_main_blendfile_path_from_global()); + BLI_add_slash(U.sequencer_disk_cache_dir); + BLI_path_make_safe(U.sequencer_disk_cache_dir); + } + + USERDEF_TAG_DIRTY; +} + static void rna_UserDef_weight_color_update(Main *bmain, Scene *scene, PointerRNA *ptr) { Object *ob; @@ -5126,6 +5141,25 @@ static void rna_def_userdef_system(BlenderRNA *brna) {0, NULL, 0, NULL, NULL}, }; + static const EnumPropertyItem seq_disk_cache_compression_levels[] = { + {USER_SEQ_DISK_CACHE_COMPRESSION_NONE, + "NONE", + 0, + "None", + "Requires fast storage, but uses minimum CPU resources"}, + {USER_SEQ_DISK_CACHE_COMPRESSION_LOW, + "LOW", + 0, + "Low", + "Doesn't require fast storage and uses less CPU resources"}, + {USER_SEQ_DISK_CACHE_COMPRESSION_HIGH, + "HIGH", + 0, + "High", + "Works on slower storage devices and uses most CPU resources"}, + {0, NULL, 0, NULL, NULL}, + }; + srna = RNA_def_struct(brna, "PreferencesSystem", NULL); RNA_def_struct_sdna(srna, "UserDef"); RNA_def_struct_nested(brna, srna, "Preferences"); @@ -5168,6 +5202,31 @@ static void rna_def_userdef_system(BlenderRNA *brna) RNA_def_property_ui_text(prop, "Memory Cache Limit", "Memory cache limit (in megabytes)"); RNA_def_property_update(prop, 0, "rna_Userdef_memcache_update"); + /* Sequencer disk cache */ + + prop = RNA_def_property(srna, "use_sequencer_disk_cache", PROP_BOOLEAN, PROP_NONE); + RNA_def_property_boolean_sdna( + prop, NULL, "sequencer_disk_cache_flag", SEQ_CACHE_DISK_CACHE_ENABLE); + RNA_def_property_ui_text(prop, "Use Disk Cache", "Store cached images to disk"); + + prop = RNA_def_property(srna, "sequencer_disk_cache_dir", PROP_STRING, PROP_DIRPATH); + RNA_def_property_string_sdna(prop, NULL, "sequencer_disk_cache_dir"); + RNA_def_property_update(prop, 0, "rna_Userdef_disk_cache_dir_update"); + RNA_def_property_ui_text(prop, "Disk Cache Directory", "Override default directory"); + + prop = RNA_def_property(srna, "sequencer_disk_cache_size_limit", PROP_INT, PROP_NONE); + RNA_def_property_int_sdna(prop, NULL, "sequencer_disk_cache_size_limit"); + RNA_def_property_range(prop, 0, INT_MAX); + RNA_def_property_ui_text(prop, "Disk Cache Limit", "Disk cache limit (in gigabytes)"); + + prop = RNA_def_property(srna, "sequencer_disk_cache_compression", PROP_ENUM, PROP_NONE); + RNA_def_property_enum_items(prop, seq_disk_cache_compression_levels); + RNA_def_property_enum_sdna(prop, NULL, "sequencer_disk_cache_compression"); + RNA_def_property_ui_text( + prop, + "Disk Cache Compression Level", + "Smaller compression will result in larger files, but less decoding overhead"); + prop = RNA_def_property(srna, "scrollback", PROP_INT, PROP_UNSIGNED); RNA_def_property_int_sdna(prop, NULL, "scrollback"); RNA_def_property_range(prop, 32, 32768); |