From 70cc80ea1c7f2678762c8ec31b924f9624172810 Mon Sep 17 00:00:00 2001 From: Richard Antalik Date: Thu, 7 Oct 2021 03:04:34 +0200 Subject: Cleanup: Move VSE disk cache code into own file No functional changes. --- source/blender/sequencer/CMakeLists.txt | 2 + source/blender/sequencer/intern/disk_cache.c | 698 ++++++++++++++++++++++++++ source/blender/sequencer/intern/disk_cache.h | 56 +++ source/blender/sequencer/intern/image_cache.c | 667 +----------------------- source/blender/sequencer/intern/image_cache.h | 21 +- 5 files changed, 780 insertions(+), 664 deletions(-) create mode 100644 source/blender/sequencer/intern/disk_cache.c create mode 100644 source/blender/sequencer/intern/disk_cache.h diff --git a/source/blender/sequencer/CMakeLists.txt b/source/blender/sequencer/CMakeLists.txt index f060e6ad69b..eccc336141a 100644 --- a/source/blender/sequencer/CMakeLists.txt +++ b/source/blender/sequencer/CMakeLists.txt @@ -63,6 +63,8 @@ set(SRC SEQ_utils.h intern/clipboard.c + intern/disk_cache.c + intern/disk_cache.h intern/effects.c intern/effects.h intern/image_cache.c diff --git a/source/blender/sequencer/intern/disk_cache.c b/source/blender/sequencer/intern/disk_cache.c new file mode 100644 index 00000000000..543c23b184b --- /dev/null +++ b/source/blender/sequencer/intern/disk_cache.c @@ -0,0 +1,698 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2021 Blender Foundation. + * All rights reserved. + */ + +/** \file + * \ingroup sequencer + */ + +#include +#include +#include + +#include "MEM_guardedalloc.h" + +#include "DNA_scene_types.h" +#include "DNA_sequence_types.h" +#include "DNA_space_types.h" /* for FILE_MAX. */ + +#include "IMB_colormanagement.h" +#include "IMB_imbuf.h" +#include "IMB_imbuf_types.h" + +#include "BLI_blenlib.h" +#include "BLI_endian_defines.h" +#include "BLI_endian_switch.h" +#include "BLI_fileops.h" +#include "BLI_fileops_types.h" +#include "BLI_ghash.h" +#include "BLI_listbase.h" +#include "BLI_mempool.h" +#include "BLI_path_util.h" +#include "BLI_threads.h" + +#include "BKE_main.h" +#include "BKE_scene.h" + +#include "SEQ_prefetch.h" +#include "SEQ_relations.h" +#include "SEQ_render.h" +#include "SEQ_sequencer.h" + +#include "disk_cache.h" +#include "image_cache.h" +#include "prefetch.h" +#include "strip_time.h" + +/** + * Disk Cache Design Notes + * ======================= + * + * Disk cache uses directory specified in user preferences + * For each cached non-temp image, image data and supplementary info are written to HDD. + * Multiple(DCACHE_IMAGES_PER_FILE) images share the same file. + * Each of these files contains header DiskCacheHeader followed by image data. + * Zlib compression with user definable level can be used to compress image data(per image) + * Images are written in order in which they are rendered. + * Overwriting of individual entry is not possible. + * Stored images are deleted by invalidation, or when size of all files exceeds maximum + * size specified in user preferences. + * To distinguish 2 blend files with same name, scene->ed->disk_cache_timestamp + * is used as UID. Blend file can still be copied manually which may cause conflict. + */ + +/* -x-%()-.dcf */ +#define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf" +#define DCACHE_IMAGES_PER_FILE 100 +#define DCACHE_CURRENT_VERSION 2 +#define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */ + +typedef struct DiskCacheHeaderEntry { + unsigned char encoding; + uint64_t frameno; + uint64_t size_compressed; + uint64_t size_raw; + uint64_t offset; + char colorspace_name[COLORSPACE_NAME_MAX]; +} DiskCacheHeaderEntry; + +typedef struct DiskCacheHeader { + DiskCacheHeaderEntry entry[DCACHE_IMAGES_PER_FILE]; +} DiskCacheHeader; + +typedef struct SeqDiskCache { + Main *bmain; + int64_t timestamp; + ListBase files; + ThreadMutex read_write_mutex; + size_t size_total; +} SeqDiskCache; + +typedef struct DiskCacheFile { + struct DiskCacheFile *next, *prev; + char path[FILE_MAX]; + char dir[FILE_MAXDIR]; + char file[FILE_MAX]; + BLI_stat_t fstat; + int cache_type; + int rectx; + int recty; + int render_size; + int view_id; + int start_frame; +} DiskCacheFile; + +static ThreadMutex cache_create_lock = BLI_MUTEX_INITIALIZER; + +static char *seq_disk_cache_base_dir(void) +{ + return U.sequencer_disk_cache_dir; +} + +static int seq_disk_cache_compression_level(void) +{ + switch (U.sequencer_disk_cache_compression) { + case USER_SEQ_DISK_CACHE_COMPRESSION_NONE: + return 0; + case USER_SEQ_DISK_CACHE_COMPRESSION_LOW: + return 1; + case USER_SEQ_DISK_CACHE_COMPRESSION_HIGH: + return 9; + } + + return U.sequencer_disk_cache_compression; +} + +static size_t seq_disk_cache_size_limit(void) +{ + return (size_t)U.sequencer_disk_cache_size_limit * (1024 * 1024 * 1024); +} + +bool seq_disk_cache_is_enabled(Main *bmain) +{ + return (U.sequencer_disk_cache_dir[0] != '\0' && U.sequencer_disk_cache_size_limit != 0 && + (U.sequencer_disk_cache_flag & SEQ_CACHE_DISK_CACHE_ENABLE) != 0 && + bmain->name[0] != '\0'); +} + +static DiskCacheFile *seq_disk_cache_add_file_to_list(SeqDiskCache *disk_cache, const char *path) +{ + + DiskCacheFile *cache_file = MEM_callocN(sizeof(DiskCacheFile), "SeqDiskCacheFile"); + char dir[FILE_MAXDIR], file[FILE_MAX]; + BLI_split_dirfile(path, dir, file, sizeof(dir), sizeof(file)); + BLI_strncpy(cache_file->path, path, sizeof(cache_file->path)); + BLI_strncpy(cache_file->dir, dir, sizeof(cache_file->dir)); + BLI_strncpy(cache_file->file, file, sizeof(cache_file->file)); + sscanf(file, + DCACHE_FNAME_FORMAT, + &cache_file->cache_type, + &cache_file->rectx, + &cache_file->recty, + &cache_file->render_size, + &cache_file->view_id, + &cache_file->start_frame); + cache_file->start_frame *= DCACHE_IMAGES_PER_FILE; + BLI_addtail(&disk_cache->files, cache_file); + return cache_file; +} + +static void seq_disk_cache_get_files(SeqDiskCache *disk_cache, char *path) +{ + struct direntry *filelist, *fl; + uint nbr, i; + disk_cache->size_total = 0; + + i = nbr = BLI_filelist_dir_contents(path, &filelist); + fl = filelist; + while (i--) { + /* Don't follow links. */ + const eFileAttributes file_attrs = BLI_file_attributes(fl->path); + if (file_attrs & FILE_ATTR_ANY_LINK) { + fl++; + continue; + } + + char file[FILE_MAX]; + BLI_split_dirfile(fl->path, NULL, file, 0, sizeof(file)); + + bool is_dir = BLI_is_dir(fl->path); + if (is_dir && !FILENAME_IS_CURRPAR(file)) { + char subpath[FILE_MAX]; + BLI_strncpy(subpath, fl->path, sizeof(subpath)); + BLI_path_slash_ensure(subpath); + seq_disk_cache_get_files(disk_cache, subpath); + } + + if (!is_dir) { + const char *ext = BLI_path_extension(fl->path); + if (ext && ext[1] == 'd' && ext[2] == 'c' && ext[3] == 'f') { + DiskCacheFile *cache_file = seq_disk_cache_add_file_to_list(disk_cache, fl->path); + cache_file->fstat = fl->s; + disk_cache->size_total += cache_file->fstat.st_size; + } + } + fl++; + } + BLI_filelist_free(filelist, nbr); +} + +static DiskCacheFile *seq_disk_cache_get_oldest_file(SeqDiskCache *disk_cache) +{ + DiskCacheFile *oldest_file = disk_cache->files.first; + if (oldest_file == NULL) { + return NULL; + } + for (DiskCacheFile *cache_file = oldest_file->next; cache_file; cache_file = cache_file->next) { + if (cache_file->fstat.st_mtime < oldest_file->fstat.st_mtime) { + oldest_file = cache_file; + } + } + + return oldest_file; +} + +static void seq_disk_cache_delete_file(SeqDiskCache *disk_cache, DiskCacheFile *file) +{ + disk_cache->size_total -= file->fstat.st_size; + BLI_delete(file->path, false, false); + BLI_remlink(&disk_cache->files, file); + MEM_freeN(file); +} + +bool seq_disk_cache_enforce_limits(SeqDiskCache *disk_cache) +{ + BLI_mutex_lock(&disk_cache->read_write_mutex); + while (disk_cache->size_total > seq_disk_cache_size_limit()) { + DiskCacheFile *oldest_file = seq_disk_cache_get_oldest_file(disk_cache); + + if (!oldest_file) { + /* We shouldn't enforce limits with no files, do re-scan. */ + seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir()); + continue; + } + + if (BLI_exists(oldest_file->path) == 0) { + /* File may have been manually deleted during runtime, do re-scan. */ + BLI_freelistN(&disk_cache->files); + seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir()); + continue; + } + + seq_disk_cache_delete_file(disk_cache, oldest_file); + } + BLI_mutex_unlock(&disk_cache->read_write_mutex); + + return true; +} + +static DiskCacheFile *seq_disk_cache_get_file_entry_by_path(SeqDiskCache *disk_cache, char *path) +{ + DiskCacheFile *cache_file = disk_cache->files.first; + + for (; cache_file; cache_file = cache_file->next) { + if (BLI_strcasecmp(cache_file->path, path) == 0) { + return cache_file; + } + } + + return NULL; +} + +/* Update file size and timestamp. */ +static void seq_disk_cache_update_file(SeqDiskCache *disk_cache, char *path) +{ + DiskCacheFile *cache_file; + int64_t size_before; + int64_t size_after; + + cache_file = seq_disk_cache_get_file_entry_by_path(disk_cache, path); + size_before = cache_file->fstat.st_size; + + if (BLI_stat(path, &cache_file->fstat) == -1) { + BLI_assert(false); + memset(&cache_file->fstat, 0, sizeof(BLI_stat_t)); + } + + size_after = cache_file->fstat.st_size; + disk_cache->size_total += size_after - size_before; +} + +/* Path format: + * /_seq_cache/-//DCACHE_FNAME_FORMAT + */ + +static void seq_disk_cache_get_project_dir(SeqDiskCache *disk_cache, char *path, size_t path_len) +{ + char cache_dir[FILE_MAX]; + BLI_split_file_part(BKE_main_blendfile_path(disk_cache->bmain), cache_dir, sizeof(cache_dir)); + /* Use suffix, so that the cache directory name does not conflict with the bmain's blend file. */ + const char *suffix = "_seq_cache"; + strncat(cache_dir, suffix, sizeof(cache_dir) - strlen(cache_dir) - 1); + BLI_strncpy(path, seq_disk_cache_base_dir(), path_len); + BLI_path_append(path, path_len, cache_dir); +} + +static void seq_disk_cache_get_dir( + SeqDiskCache *disk_cache, Scene *scene, Sequence *seq, char *path, size_t path_len) +{ + char scene_name[MAX_ID_NAME + 22]; /* + -%PRId64 */ + char seq_name[SEQ_NAME_MAXSTR]; + char project_dir[FILE_MAX]; + + seq_disk_cache_get_project_dir(disk_cache, project_dir, sizeof(project_dir)); + sprintf(scene_name, "%s-%" PRId64, scene->id.name, disk_cache->timestamp); + BLI_strncpy(seq_name, seq->name, sizeof(seq_name)); + BLI_filename_make_safe(scene_name); + BLI_filename_make_safe(seq_name); + BLI_strncpy(path, project_dir, path_len); + BLI_path_append(path, path_len, scene_name); + BLI_path_append(path, path_len, seq_name); +} + +static void seq_disk_cache_get_file_path(SeqDiskCache *disk_cache, + SeqCacheKey *key, + char *path, + size_t path_len) +{ + seq_disk_cache_get_dir(disk_cache, key->context.scene, key->seq, path, path_len); + int frameno = (int)key->frame_index / DCACHE_IMAGES_PER_FILE; + char cache_filename[FILE_MAXFILE]; + sprintf(cache_filename, + DCACHE_FNAME_FORMAT, + key->type, + key->context.rectx, + key->context.recty, + key->context.preview_render_size, + key->context.view_id, + frameno); + + BLI_path_append(path, path_len, cache_filename); +} + +static void seq_disk_cache_create_version_file(char *path) +{ + BLI_make_existing_file(path); + + FILE *file = BLI_fopen(path, "w"); + if (file) { + fprintf(file, "%d", DCACHE_CURRENT_VERSION); + fclose(file); + } +} + +static void seq_disk_cache_handle_versioning(SeqDiskCache *disk_cache) +{ + char path[FILE_MAX]; + char path_version_file[FILE_MAX]; + int version = 0; + + seq_disk_cache_get_project_dir(disk_cache, path, sizeof(path)); + BLI_strncpy(path_version_file, path, sizeof(path_version_file)); + BLI_path_append(path_version_file, sizeof(path_version_file), "cache_version"); + + if (BLI_exists(path) && BLI_is_dir(path)) { + FILE *file = BLI_fopen(path_version_file, "r"); + + if (file) { + const int num_items_read = fscanf(file, "%d", &version); + if (num_items_read == 0) { + version = -1; + } + fclose(file); + } + + if (version != DCACHE_CURRENT_VERSION) { + BLI_delete(path, false, true); + seq_disk_cache_create_version_file(path_version_file); + } + } + else { + seq_disk_cache_create_version_file(path_version_file); + } +} + +static void seq_disk_cache_delete_invalid_files(SeqDiskCache *disk_cache, + Scene *scene, + Sequence *seq, + int invalidate_types, + int range_start, + int range_end) +{ + DiskCacheFile *next_file, *cache_file = disk_cache->files.first; + char cache_dir[FILE_MAX]; + seq_disk_cache_get_dir(disk_cache, scene, seq, cache_dir, sizeof(cache_dir)); + BLI_path_slash_ensure(cache_dir); + + while (cache_file) { + next_file = cache_file->next; + if (cache_file->cache_type & invalidate_types) { + if (STREQ(cache_dir, cache_file->dir)) { + int timeline_frame_start = seq_cache_frame_index_to_timeline_frame( + seq, cache_file->start_frame); + if (timeline_frame_start > range_start && timeline_frame_start <= range_end) { + seq_disk_cache_delete_file(disk_cache, cache_file); + } + } + } + cache_file = next_file; + } +} + +void seq_disk_cache_invalidate(SeqDiskCache *disk_cache, + Scene *scene, + Sequence *seq, + Sequence *seq_changed, + int invalidate_types) +{ + int start; + int end; + + BLI_mutex_lock(&disk_cache->read_write_mutex); + + start = seq_changed->startdisp - DCACHE_IMAGES_PER_FILE; + end = seq_changed->enddisp; + + seq_disk_cache_delete_invalid_files(disk_cache, scene, seq, invalidate_types, start, end); + + BLI_mutex_unlock(&disk_cache->read_write_mutex); +} + +static size_t deflate_imbuf_to_file(ImBuf *ibuf, + FILE *file, + int level, + DiskCacheHeaderEntry *header_entry) +{ + void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float; + + /* Apply compression if wanted, otherwise just write directly to the file. */ + if (level > 0) { + return BLI_file_zstd_from_mem_at_pos( + data, header_entry->size_raw, file, header_entry->offset, level); + } + + fseek(file, header_entry->offset, SEEK_SET); + return fwrite(data, 1, header_entry->size_raw, file); +} + +static size_t inflate_file_to_imbuf(ImBuf *ibuf, FILE *file, DiskCacheHeaderEntry *header_entry) +{ + void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float; + char header[4]; + fseek(file, header_entry->offset, SEEK_SET); + if (fread(header, 1, sizeof(header), file) != sizeof(header)) { + return 0; + } + + /* Check if the data is compressed or raw. */ + if (BLI_file_magic_is_zstd(header)) { + return BLI_file_unzstd_to_mem_at_pos(data, header_entry->size_raw, file, header_entry->offset); + } + + fseek(file, header_entry->offset, SEEK_SET); + return fread(data, 1, header_entry->size_raw, file); +} + +static bool seq_disk_cache_read_header(FILE *file, DiskCacheHeader *header) +{ + BLI_fseek(file, 0LL, SEEK_SET); + const size_t num_items_read = fread(header, sizeof(*header), 1, file); + if (num_items_read < 1) { + BLI_assert_msg(0, "unable to read disk cache header"); + perror("unable to read disk cache header"); + return false; + } + + for (int i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { + if ((ENDIAN_ORDER == B_ENDIAN) && header->entry[i].encoding == 0) { + BLI_endian_switch_uint64(&header->entry[i].frameno); + BLI_endian_switch_uint64(&header->entry[i].offset); + BLI_endian_switch_uint64(&header->entry[i].size_compressed); + BLI_endian_switch_uint64(&header->entry[i].size_raw); + } + } + + return true; +} + +static size_t seq_disk_cache_write_header(FILE *file, DiskCacheHeader *header) +{ + BLI_fseek(file, 0LL, SEEK_SET); + return fwrite(header, sizeof(*header), 1, file); +} + +static int seq_disk_cache_add_header_entry(SeqCacheKey *key, ImBuf *ibuf, DiskCacheHeader *header) +{ + int i; + uint64_t offset = sizeof(*header); + + /* Lookup free entry, get offset for new data. */ + for (i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { + if (header->entry[i].size_compressed == 0) { + break; + } + } + + /* Attempt to write beyond set entry limit. + * Reset file header and start writing from beginning. + */ + if (i == DCACHE_IMAGES_PER_FILE) { + i = 0; + memset(header, 0, sizeof(*header)); + } + + /* Calculate offset for image data. */ + if (i > 0) { + offset = header->entry[i - 1].offset + header->entry[i - 1].size_compressed; + } + + if (ENDIAN_ORDER == B_ENDIAN) { + header->entry[i].encoding = 255; + } + else { + header->entry[i].encoding = 0; + } + + header->entry[i].offset = offset; + header->entry[i].frameno = key->frame_index; + + /* Store colorspace name of ibuf. */ + const char *colorspace_name; + if (ibuf->rect) { + header->entry[i].size_raw = ibuf->x * ibuf->y * ibuf->channels; + colorspace_name = IMB_colormanagement_get_rect_colorspace(ibuf); + } + else { + header->entry[i].size_raw = ibuf->x * ibuf->y * ibuf->channels * 4; + colorspace_name = IMB_colormanagement_get_float_colorspace(ibuf); + } + BLI_strncpy( + header->entry[i].colorspace_name, colorspace_name, sizeof(header->entry[i].colorspace_name)); + + return i; +} + +static int seq_disk_cache_get_header_entry(SeqCacheKey *key, DiskCacheHeader *header) +{ + for (int i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { + if (header->entry[i].frameno == key->frame_index) { + return i; + } + } + + return -1; +} + +bool seq_disk_cache_write_file(SeqDiskCache *disk_cache, SeqCacheKey *key, ImBuf *ibuf) +{ + BLI_mutex_lock(&disk_cache->read_write_mutex); + + char path[FILE_MAX]; + + seq_disk_cache_get_file_path(disk_cache, key, path, sizeof(path)); + BLI_make_existing_file(path); + + FILE *file = BLI_fopen(path, "rb+"); + if (!file) { + file = BLI_fopen(path, "wb+"); + if (!file) { + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return false; + } + seq_disk_cache_add_file_to_list(disk_cache, path); + } + + DiskCacheFile *cache_file = seq_disk_cache_get_file_entry_by_path(disk_cache, path); + DiskCacheHeader header; + memset(&header, 0, sizeof(header)); + /* #BLI_make_existing_file() above may create an empty file. This is fine, don't attempt reading + * the header in that case. */ + if (cache_file->fstat.st_size != 0 && !seq_disk_cache_read_header(file, &header)) { + fclose(file); + seq_disk_cache_delete_file(disk_cache, cache_file); + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return false; + } + int entry_index = seq_disk_cache_add_header_entry(key, ibuf, &header); + + size_t bytes_written = deflate_imbuf_to_file( + ibuf, file, seq_disk_cache_compression_level(), &header.entry[entry_index]); + + if (bytes_written != 0) { + /* Last step is writing header, as image data can be overwritten, + * but missing data would cause problems. + */ + header.entry[entry_index].size_compressed = bytes_written; + seq_disk_cache_write_header(file, &header); + seq_disk_cache_update_file(disk_cache, path); + fclose(file); + + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return true; + } + + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return false; +} + +ImBuf *seq_disk_cache_read_file(SeqDiskCache *disk_cache, SeqCacheKey *key) +{ + BLI_mutex_lock(&disk_cache->read_write_mutex); + + char path[FILE_MAX]; + DiskCacheHeader header; + + seq_disk_cache_get_file_path(disk_cache, key, path, sizeof(path)); + BLI_make_existing_file(path); + + FILE *file = BLI_fopen(path, "rb"); + if (!file) { + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return NULL; + } + + if (!seq_disk_cache_read_header(file, &header)) { + fclose(file); + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return NULL; + } + int entry_index = seq_disk_cache_get_header_entry(key, &header); + + /* Item not found. */ + if (entry_index < 0) { + fclose(file); + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return NULL; + } + + ImBuf *ibuf; + uint64_t size_char = (uint64_t)key->context.rectx * key->context.recty * 4; + uint64_t size_float = (uint64_t)key->context.rectx * key->context.recty * 16; + size_t expected_size; + + if (header.entry[entry_index].size_raw == size_char) { + expected_size = size_char; + ibuf = IMB_allocImBuf(key->context.rectx, key->context.recty, 32, IB_rect); + IMB_colormanagement_assign_rect_colorspace(ibuf, header.entry[entry_index].colorspace_name); + } + else if (header.entry[entry_index].size_raw == size_float) { + expected_size = size_float; + ibuf = IMB_allocImBuf(key->context.rectx, key->context.recty, 32, IB_rectfloat); + IMB_colormanagement_assign_float_colorspace(ibuf, header.entry[entry_index].colorspace_name); + } + else { + fclose(file); + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return NULL; + } + + size_t bytes_read = inflate_file_to_imbuf(ibuf, file, &header.entry[entry_index]); + + /* Sanity check. */ + if (bytes_read != expected_size) { + fclose(file); + IMB_freeImBuf(ibuf); + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return NULL; + } + BLI_file_touch(path); + seq_disk_cache_update_file(disk_cache, path); + fclose(file); + + BLI_mutex_unlock(&disk_cache->read_write_mutex); + return ibuf; +} + +SeqDiskCache *seq_disk_cache_create(Main *bmain, Scene *scene) +{ + SeqDiskCache *disk_cache = MEM_callocN(sizeof(SeqDiskCache), "SeqDiskCache"); + disk_cache->bmain = bmain; + BLI_mutex_init(&disk_cache->read_write_mutex); + seq_disk_cache_handle_versioning(disk_cache); + seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir()); + disk_cache->timestamp = scene->ed->disk_cache_timestamp; + BLI_mutex_unlock(&cache_create_lock); + return disk_cache; +} + +void seq_disk_cache_free(SeqDiskCache *disk_cache) +{ + BLI_freelistN(&disk_cache->files); + BLI_mutex_end(&disk_cache->read_write_mutex); + MEM_freeN(disk_cache); +} diff --git a/source/blender/sequencer/intern/disk_cache.h b/source/blender/sequencer/intern/disk_cache.h new file mode 100644 index 00000000000..a84bfaf5ea0 --- /dev/null +++ b/source/blender/sequencer/intern/disk_cache.h @@ -0,0 +1,56 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * The Original Code is Copyright (C) 2021 Blender Foundation. + * All rights reserved. + */ + +/** \file + * \ingroup sequencer + */ + +#pragma once + +/** \file + * \ingroup sequencer + */ + +#ifdef __cplusplus +extern "C" { +#endif + +struct ImBuf; +struct Main; +struct Scene; +struct SeqCacheKey; +struct SeqDiskCache; +struct Sequence; + +struct SeqDiskCache *seq_disk_cache_create(struct Main *bmain, struct Scene *scene); +void seq_disk_cache_free(struct SeqDiskCache *disk_cache); +bool seq_disk_cache_is_enabled(struct Main *bmain); +struct ImBuf *seq_disk_cache_read_file(struct SeqDiskCache *disk_cache, struct SeqCacheKey *key); +bool seq_disk_cache_write_file(struct SeqDiskCache *disk_cache, + struct SeqCacheKey *key, + struct ImBuf *ibuf); +bool seq_disk_cache_enforce_limits(struct SeqDiskCache *disk_cache); +void seq_disk_cache_invalidate(struct SeqDiskCache *disk_cache, + struct Scene *scene, + struct Sequence *seq, + struct Sequence *seq_changed, + int invalidate_types); +#ifdef __cplusplus +} +#endif diff --git a/source/blender/sequencer/intern/image_cache.c b/source/blender/sequencer/intern/image_cache.c index 86c198075e9..7b3d342b2e4 100644 --- a/source/blender/sequencer/intern/image_cache.c +++ b/source/blender/sequencer/intern/image_cache.c @@ -50,9 +50,9 @@ #include "SEQ_prefetch.h" #include "SEQ_relations.h" -#include "SEQ_render.h" #include "SEQ_sequencer.h" +#include "disk_cache.h" #include "image_cache.h" #include "prefetch.h" #include "strip_time.h" @@ -80,67 +80,10 @@ * entries one by one in reverse order to their creation. * * User can exclude caching of some images. Such entries will have is_temp_cache set. - * - * - * Disk Cache Design Notes - * ======================= - * - * Disk cache uses directory specified in user preferences - * For each cached non-temp image, image data and supplementary info are written to HDD. - * Multiple(DCACHE_IMAGES_PER_FILE) images share the same file. - * Each of these files contains header DiskCacheHeader followed by image data. - * Zlib compression with user definable level can be used to compress image data(per image) - * Images are written in order in which they are rendered. - * Overwriting of individual entry is not possible. - * Stored images are deleted by invalidation, or when size of all files exceeds maximum - * size specified in user preferences. - * To distinguish 2 blend files with same name, scene->ed->disk_cache_timestamp - * is used as UID. Blend file can still be copied manually which may cause conflict. - * */ -/* -x-%()-.dcf */ -#define DCACHE_FNAME_FORMAT "%d-%dx%d-%d%%(%d)-%d.dcf" -#define DCACHE_IMAGES_PER_FILE 100 -#define DCACHE_CURRENT_VERSION 2 -#define COLORSPACE_NAME_MAX 64 /* XXX: defined in imb intern */ #define THUMB_CACHE_LIMIT 5000 -typedef struct DiskCacheHeaderEntry { - unsigned char encoding; - uint64_t frameno; - uint64_t size_compressed; - uint64_t size_raw; - uint64_t offset; - char colorspace_name[COLORSPACE_NAME_MAX]; -} DiskCacheHeaderEntry; - -typedef struct DiskCacheHeader { - DiskCacheHeaderEntry entry[DCACHE_IMAGES_PER_FILE]; -} DiskCacheHeader; - -typedef struct SeqDiskCache { - Main *bmain; - int64_t timestamp; - ListBase files; - ThreadMutex read_write_mutex; - size_t size_total; -} SeqDiskCache; - -typedef struct DiskCacheFile { - struct DiskCacheFile *next, *prev; - char path[FILE_MAX]; - char dir[FILE_MAXDIR]; - char file[FILE_MAX]; - BLI_stat_t fstat; - int cache_type; - int rectx; - int recty; - int render_size; - int view_id; - int start_frame; -} DiskCacheFile; - typedef struct SeqCache { Main *bmain; struct GHash *hash; @@ -148,7 +91,7 @@ typedef struct SeqCache { struct BLI_mempool *keys_pool; struct BLI_mempool *items_pool; struct SeqCacheKey *last_key; - SeqDiskCache *disk_cache; + struct SeqDiskCache *disk_cache; int thumbnail_count; } SeqCache; @@ -157,577 +100,7 @@ typedef struct SeqCacheItem { struct ImBuf *ibuf; } SeqCacheItem; -typedef struct SeqCacheKey { - struct SeqCache *cache_owner; - void *userkey; - struct SeqCacheKey *link_prev; /* Used for linking intermediate items to final frame. */ - struct SeqCacheKey *link_next; /* Used for linking intermediate items to final frame. */ - struct Sequence *seq; - SeqRenderData context; - float frame_index; /* Usually same as timeline_frame. Mapped to media for RAW entries. */ - float timeline_frame; /* Only for reference - used for freeing when cache is full. */ - float cost; /* In short: render time(s) divided by playback frame duration(s) */ - bool is_temp_cache; /* this cache entry will be freed before rendering next frame */ - /* ID of task for assigning temp cache entries to particular task(thread, etc.) */ - eSeqTaskId task_id; - int type; -} SeqCacheKey; - static ThreadMutex cache_create_lock = BLI_MUTEX_INITIALIZER; -static float seq_cache_timeline_frame_to_frame_index(Sequence *seq, - float timeline_frame, - int type); -static float seq_cache_frame_index_to_timeline_frame(Sequence *seq, float frame_index); - -static char *seq_disk_cache_base_dir(void) -{ - return U.sequencer_disk_cache_dir; -} - -static int seq_disk_cache_compression_level(void) -{ - switch (U.sequencer_disk_cache_compression) { - case USER_SEQ_DISK_CACHE_COMPRESSION_NONE: - return 0; - case USER_SEQ_DISK_CACHE_COMPRESSION_LOW: - return 1; - case USER_SEQ_DISK_CACHE_COMPRESSION_HIGH: - return 9; - } - - return U.sequencer_disk_cache_compression; -} - -static size_t seq_disk_cache_size_limit(void) -{ - return (size_t)U.sequencer_disk_cache_size_limit * (1024 * 1024 * 1024); -} - -static bool seq_disk_cache_is_enabled(Main *bmain) -{ - return (U.sequencer_disk_cache_dir[0] != '\0' && U.sequencer_disk_cache_size_limit != 0 && - (U.sequencer_disk_cache_flag & SEQ_CACHE_DISK_CACHE_ENABLE) != 0 && - bmain->name[0] != '\0'); -} - -static DiskCacheFile *seq_disk_cache_add_file_to_list(SeqDiskCache *disk_cache, const char *path) -{ - - DiskCacheFile *cache_file = MEM_callocN(sizeof(DiskCacheFile), "SeqDiskCacheFile"); - char dir[FILE_MAXDIR], file[FILE_MAX]; - BLI_split_dirfile(path, dir, file, sizeof(dir), sizeof(file)); - BLI_strncpy(cache_file->path, path, sizeof(cache_file->path)); - BLI_strncpy(cache_file->dir, dir, sizeof(cache_file->dir)); - BLI_strncpy(cache_file->file, file, sizeof(cache_file->file)); - sscanf(file, - DCACHE_FNAME_FORMAT, - &cache_file->cache_type, - &cache_file->rectx, - &cache_file->recty, - &cache_file->render_size, - &cache_file->view_id, - &cache_file->start_frame); - cache_file->start_frame *= DCACHE_IMAGES_PER_FILE; - BLI_addtail(&disk_cache->files, cache_file); - return cache_file; -} - -static void seq_disk_cache_get_files(SeqDiskCache *disk_cache, char *path) -{ - struct direntry *filelist, *fl; - uint nbr, i; - disk_cache->size_total = 0; - - i = nbr = BLI_filelist_dir_contents(path, &filelist); - fl = filelist; - while (i--) { - /* Don't follow links. */ - const eFileAttributes file_attrs = BLI_file_attributes(fl->path); - if (file_attrs & FILE_ATTR_ANY_LINK) { - fl++; - continue; - } - - char file[FILE_MAX]; - BLI_split_dirfile(fl->path, NULL, file, 0, sizeof(file)); - - bool is_dir = BLI_is_dir(fl->path); - if (is_dir && !FILENAME_IS_CURRPAR(file)) { - char subpath[FILE_MAX]; - BLI_strncpy(subpath, fl->path, sizeof(subpath)); - BLI_path_slash_ensure(subpath); - seq_disk_cache_get_files(disk_cache, subpath); - } - - if (!is_dir) { - const char *ext = BLI_path_extension(fl->path); - if (ext && ext[1] == 'd' && ext[2] == 'c' && ext[3] == 'f') { - DiskCacheFile *cache_file = seq_disk_cache_add_file_to_list(disk_cache, fl->path); - cache_file->fstat = fl->s; - disk_cache->size_total += cache_file->fstat.st_size; - } - } - fl++; - } - BLI_filelist_free(filelist, nbr); -} - -static DiskCacheFile *seq_disk_cache_get_oldest_file(SeqDiskCache *disk_cache) -{ - DiskCacheFile *oldest_file = disk_cache->files.first; - if (oldest_file == NULL) { - return NULL; - } - for (DiskCacheFile *cache_file = oldest_file->next; cache_file; cache_file = cache_file->next) { - if (cache_file->fstat.st_mtime < oldest_file->fstat.st_mtime) { - oldest_file = cache_file; - } - } - - return oldest_file; -} - -static void seq_disk_cache_delete_file(SeqDiskCache *disk_cache, DiskCacheFile *file) -{ - disk_cache->size_total -= file->fstat.st_size; - BLI_delete(file->path, false, false); - BLI_remlink(&disk_cache->files, file); - MEM_freeN(file); -} - -static bool seq_disk_cache_enforce_limits(SeqDiskCache *disk_cache) -{ - BLI_mutex_lock(&disk_cache->read_write_mutex); - while (disk_cache->size_total > seq_disk_cache_size_limit()) { - DiskCacheFile *oldest_file = seq_disk_cache_get_oldest_file(disk_cache); - - if (!oldest_file) { - /* We shouldn't enforce limits with no files, do re-scan. */ - seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir()); - continue; - } - - if (BLI_exists(oldest_file->path) == 0) { - /* File may have been manually deleted during runtime, do re-scan. */ - BLI_freelistN(&disk_cache->files); - seq_disk_cache_get_files(disk_cache, seq_disk_cache_base_dir()); - continue; - } - - seq_disk_cache_delete_file(disk_cache, oldest_file); - } - BLI_mutex_unlock(&disk_cache->read_write_mutex); - - return true; -} - -static DiskCacheFile *seq_disk_cache_get_file_entry_by_path(SeqDiskCache *disk_cache, char *path) -{ - DiskCacheFile *cache_file = disk_cache->files.first; - - for (; cache_file; cache_file = cache_file->next) { - if (BLI_strcasecmp(cache_file->path, path) == 0) { - return cache_file; - } - } - - return NULL; -} - -/* Update file size and timestamp. */ -static void seq_disk_cache_update_file(SeqDiskCache *disk_cache, char *path) -{ - DiskCacheFile *cache_file; - int64_t size_before; - int64_t size_after; - - cache_file = seq_disk_cache_get_file_entry_by_path(disk_cache, path); - size_before = cache_file->fstat.st_size; - - if (BLI_stat(path, &cache_file->fstat) == -1) { - BLI_assert(false); - memset(&cache_file->fstat, 0, sizeof(BLI_stat_t)); - } - - size_after = cache_file->fstat.st_size; - disk_cache->size_total += size_after - size_before; -} - -/* Path format: - * /_seq_cache/-//DCACHE_FNAME_FORMAT - */ - -static void seq_disk_cache_get_project_dir(SeqDiskCache *disk_cache, char *path, size_t path_len) -{ - char cache_dir[FILE_MAX]; - BLI_split_file_part(BKE_main_blendfile_path(disk_cache->bmain), cache_dir, sizeof(cache_dir)); - /* Use suffix, so that the cache directory name does not conflict with the bmain's blend file. */ - const char *suffix = "_seq_cache"; - strncat(cache_dir, suffix, sizeof(cache_dir) - strlen(cache_dir) - 1); - BLI_strncpy(path, seq_disk_cache_base_dir(), path_len); - BLI_path_append(path, path_len, cache_dir); -} - -static void seq_disk_cache_get_dir( - SeqDiskCache *disk_cache, Scene *scene, Sequence *seq, char *path, size_t path_len) -{ - char scene_name[MAX_ID_NAME + 22]; /* + -%PRId64 */ - char seq_name[SEQ_NAME_MAXSTR]; - char project_dir[FILE_MAX]; - - seq_disk_cache_get_project_dir(disk_cache, project_dir, sizeof(project_dir)); - sprintf(scene_name, "%s-%" PRId64, scene->id.name, disk_cache->timestamp); - BLI_strncpy(seq_name, seq->name, sizeof(seq_name)); - BLI_filename_make_safe(scene_name); - BLI_filename_make_safe(seq_name); - BLI_strncpy(path, project_dir, path_len); - BLI_path_append(path, path_len, scene_name); - BLI_path_append(path, path_len, seq_name); -} - -static void seq_disk_cache_get_file_path(SeqDiskCache *disk_cache, - SeqCacheKey *key, - char *path, - size_t path_len) -{ - seq_disk_cache_get_dir(disk_cache, key->context.scene, key->seq, path, path_len); - int frameno = (int)key->frame_index / DCACHE_IMAGES_PER_FILE; - char cache_filename[FILE_MAXFILE]; - sprintf(cache_filename, - DCACHE_FNAME_FORMAT, - key->type, - key->context.rectx, - key->context.recty, - key->context.preview_render_size, - key->context.view_id, - frameno); - - BLI_path_append(path, path_len, cache_filename); -} - -static void seq_disk_cache_create_version_file(char *path) -{ - BLI_make_existing_file(path); - - FILE *file = BLI_fopen(path, "w"); - if (file) { - fprintf(file, "%d", DCACHE_CURRENT_VERSION); - fclose(file); - } -} - -static void seq_disk_cache_handle_versioning(SeqDiskCache *disk_cache) -{ - char path[FILE_MAX]; - char path_version_file[FILE_MAX]; - int version = 0; - - seq_disk_cache_get_project_dir(disk_cache, path, sizeof(path)); - BLI_strncpy(path_version_file, path, sizeof(path_version_file)); - BLI_path_append(path_version_file, sizeof(path_version_file), "cache_version"); - - if (BLI_exists(path) && BLI_is_dir(path)) { - FILE *file = BLI_fopen(path_version_file, "r"); - - if (file) { - const int num_items_read = fscanf(file, "%d", &version); - if (num_items_read == 0) { - version = -1; - } - fclose(file); - } - - if (version != DCACHE_CURRENT_VERSION) { - BLI_delete(path, false, true); - seq_disk_cache_create_version_file(path_version_file); - } - } - else { - seq_disk_cache_create_version_file(path_version_file); - } -} - -static void seq_disk_cache_delete_invalid_files(SeqDiskCache *disk_cache, - Scene *scene, - Sequence *seq, - int invalidate_types, - int range_start, - int range_end) -{ - DiskCacheFile *next_file, *cache_file = disk_cache->files.first; - char cache_dir[FILE_MAX]; - seq_disk_cache_get_dir(disk_cache, scene, seq, cache_dir, sizeof(cache_dir)); - BLI_path_slash_ensure(cache_dir); - - while (cache_file) { - next_file = cache_file->next; - if (cache_file->cache_type & invalidate_types) { - if (STREQ(cache_dir, cache_file->dir)) { - int timeline_frame_start = seq_cache_frame_index_to_timeline_frame( - seq, cache_file->start_frame); - if (timeline_frame_start > range_start && timeline_frame_start <= range_end) { - seq_disk_cache_delete_file(disk_cache, cache_file); - } - } - } - cache_file = next_file; - } -} - -static void seq_disk_cache_invalidate(Scene *scene, - Sequence *seq, - Sequence *seq_changed, - int invalidate_types) -{ - int start; - int end; - SeqDiskCache *disk_cache = scene->ed->cache->disk_cache; - - BLI_mutex_lock(&disk_cache->read_write_mutex); - - start = seq_changed->startdisp - DCACHE_IMAGES_PER_FILE; - end = seq_changed->enddisp; - - seq_disk_cache_delete_invalid_files(disk_cache, scene, seq, invalidate_types, start, end); - - BLI_mutex_unlock(&disk_cache->read_write_mutex); -} - -static size_t deflate_imbuf_to_file(ImBuf *ibuf, - FILE *file, - int level, - DiskCacheHeaderEntry *header_entry) -{ - void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float; - - /* Apply compression if wanted, otherwise just write directly to the file. */ - if (level > 0) { - return BLI_file_zstd_from_mem_at_pos( - data, header_entry->size_raw, file, header_entry->offset, level); - } - - fseek(file, header_entry->offset, SEEK_SET); - return fwrite(data, 1, header_entry->size_raw, file); -} - -static size_t inflate_file_to_imbuf(ImBuf *ibuf, FILE *file, DiskCacheHeaderEntry *header_entry) -{ - void *data = (ibuf->rect != NULL) ? (void *)ibuf->rect : (void *)ibuf->rect_float; - char header[4]; - fseek(file, header_entry->offset, SEEK_SET); - if (fread(header, 1, sizeof(header), file) != sizeof(header)) { - return 0; - } - - /* Check if the data is compressed or raw. */ - if (BLI_file_magic_is_zstd(header)) { - return BLI_file_unzstd_to_mem_at_pos(data, header_entry->size_raw, file, header_entry->offset); - } - - fseek(file, header_entry->offset, SEEK_SET); - return fread(data, 1, header_entry->size_raw, file); -} - -static bool seq_disk_cache_read_header(FILE *file, DiskCacheHeader *header) -{ - BLI_fseek(file, 0LL, SEEK_SET); - const size_t num_items_read = fread(header, sizeof(*header), 1, file); - if (num_items_read < 1) { - BLI_assert_msg(0, "unable to read disk cache header"); - perror("unable to read disk cache header"); - return false; - } - - for (int i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { - if ((ENDIAN_ORDER == B_ENDIAN) && header->entry[i].encoding == 0) { - BLI_endian_switch_uint64(&header->entry[i].frameno); - BLI_endian_switch_uint64(&header->entry[i].offset); - BLI_endian_switch_uint64(&header->entry[i].size_compressed); - BLI_endian_switch_uint64(&header->entry[i].size_raw); - } - } - - return true; -} - -static size_t seq_disk_cache_write_header(FILE *file, DiskCacheHeader *header) -{ - BLI_fseek(file, 0LL, SEEK_SET); - return fwrite(header, sizeof(*header), 1, file); -} - -static int seq_disk_cache_add_header_entry(SeqCacheKey *key, ImBuf *ibuf, DiskCacheHeader *header) -{ - int i; - uint64_t offset = sizeof(*header); - - /* Lookup free entry, get offset for new data. */ - for (i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { - if (header->entry[i].size_compressed == 0) { - break; - } - } - - /* Attempt to write beyond set entry limit. - * Reset file header and start writing from beginning. - */ - if (i == DCACHE_IMAGES_PER_FILE) { - i = 0; - memset(header, 0, sizeof(*header)); - } - - /* Calculate offset for image data. */ - if (i > 0) { - offset = header->entry[i - 1].offset + header->entry[i - 1].size_compressed; - } - - if (ENDIAN_ORDER == B_ENDIAN) { - header->entry[i].encoding = 255; - } - else { - header->entry[i].encoding = 0; - } - - header->entry[i].offset = offset; - header->entry[i].frameno = key->frame_index; - - /* Store colorspace name of ibuf. */ - const char *colorspace_name; - if (ibuf->rect) { - header->entry[i].size_raw = ibuf->x * ibuf->y * ibuf->channels; - colorspace_name = IMB_colormanagement_get_rect_colorspace(ibuf); - } - else { - header->entry[i].size_raw = ibuf->x * ibuf->y * ibuf->channels * 4; - colorspace_name = IMB_colormanagement_get_float_colorspace(ibuf); - } - BLI_strncpy( - header->entry[i].colorspace_name, colorspace_name, sizeof(header->entry[i].colorspace_name)); - - return i; -} - -static int seq_disk_cache_get_header_entry(SeqCacheKey *key, DiskCacheHeader *header) -{ - for (int i = 0; i < DCACHE_IMAGES_PER_FILE; i++) { - if (header->entry[i].frameno == key->frame_index) { - return i; - } - } - - return -1; -} - -static bool seq_disk_cache_write_file(SeqDiskCache *disk_cache, SeqCacheKey *key, ImBuf *ibuf) -{ - char path[FILE_MAX]; - - seq_disk_cache_get_file_path(disk_cache, key, path, sizeof(path)); - BLI_make_existing_file(path); - - FILE *file = BLI_fopen(path, "rb+"); - if (!file) { - file = BLI_fopen(path, "wb+"); - if (!file) { - return false; - } - seq_disk_cache_add_file_to_list(disk_cache, path); - } - - DiskCacheFile *cache_file = seq_disk_cache_get_file_entry_by_path(disk_cache, path); - DiskCacheHeader header; - memset(&header, 0, sizeof(header)); - /* #BLI_make_existing_file() above may create an empty file. This is fine, don't attempt reading - * the header in that case. */ - if (cache_file->fstat.st_size != 0 && !seq_disk_cache_read_header(file, &header)) { - fclose(file); - seq_disk_cache_delete_file(disk_cache, cache_file); - return false; - } - int entry_index = seq_disk_cache_add_header_entry(key, ibuf, &header); - - size_t bytes_written = deflate_imbuf_to_file( - ibuf, file, seq_disk_cache_compression_level(), &header.entry[entry_index]); - - if (bytes_written != 0) { - /* Last step is writing header, as image data can be overwritten, - * but missing data would cause problems. - */ - header.entry[entry_index].size_compressed = bytes_written; - seq_disk_cache_write_header(file, &header); - seq_disk_cache_update_file(disk_cache, path); - fclose(file); - - return true; - } - - return false; -} - -static ImBuf *seq_disk_cache_read_file(SeqDiskCache *disk_cache, SeqCacheKey *key) -{ - char path[FILE_MAX]; - DiskCacheHeader header; - - seq_disk_cache_get_file_path(disk_cache, key, path, sizeof(path)); - BLI_make_existing_file(path); - - FILE *file = BLI_fopen(path, "rb"); - if (!file) { - return NULL; - } - - if (!seq_disk_cache_read_header(file, &header)) { - fclose(file); - return NULL; - } - int entry_index = seq_disk_cache_get_header_entry(key, &header); - - /* Item not found. */ - if (entry_index < 0) { - fclose(file); - return NULL; - } - - ImBuf *ibuf; - uint64_t size_char = (uint64_t)key->context.rectx * key->context.recty * 4; - uint64_t size_float = (uint64_t)key->context.rectx * key->context.recty * 16; - size_t expected_size; - - if (header.entry[entry_index].size_raw == size_char) { - expected_size = size_char; - ibuf = IMB_allocImBuf(key->context.rectx, key->context.recty, 32, IB_rect); - IMB_colormanagement_assign_rect_colorspace(ibuf, header.entry[entry_index].colorspace_name); - } - else if (header.entry[entry_index].size_raw == size_float) { - expected_size = size_float; - ibuf = IMB_allocImBuf(key->context.rectx, key->context.recty, 32, IB_rectfloat); - IMB_colormanagement_assign_float_colorspace(ibuf, header.entry[entry_index].colorspace_name); - } - else { - fclose(file); - return NULL; - } - - size_t bytes_read = inflate_file_to_imbuf(ibuf, file, &header.entry[entry_index]); - - /* Sanity check. */ - if (bytes_read != expected_size) { - fclose(file); - IMB_freeImBuf(ibuf); - return NULL; - } - BLI_file_touch(path); - seq_disk_cache_update_file(disk_cache, path); - fclose(file); - - return ibuf; -} - -#undef DCACHE_FNAME_FORMAT -#undef DCACHE_IMAGES_PER_FILE -#undef COLORSPACE_NAME_MAX -#undef DCACHE_CURRENT_VERSION static bool seq_cmp_render_data(const SeqRenderData *a, const SeqRenderData *b) { @@ -785,7 +158,7 @@ static float seq_cache_timeline_frame_to_frame_index(Sequence *seq, float timeli return timeline_frame - seq->start; } -static float seq_cache_frame_index_to_timeline_frame(Sequence *seq, float frame_index) +float seq_cache_frame_index_to_timeline_frame(Sequence *seq, float frame_index) { return frame_index + seq->start; } @@ -1131,28 +504,6 @@ static void seq_cache_set_temp_cache_linked(Scene *scene, SeqCacheKey *base) } } -static void seq_disk_cache_create(Main *bmain, Scene *scene) -{ - BLI_mutex_lock(&cache_create_lock); - SeqCache *cache = seq_cache_get_from_scene(scene); - - if (cache == NULL) { - return; - } - - if (cache->disk_cache != NULL) { - return; - } - - cache->disk_cache = MEM_callocN(sizeof(SeqDiskCache), "SeqDiskCache"); - cache->disk_cache->bmain = bmain; - BLI_mutex_init(&cache->disk_cache->read_write_mutex); - seq_disk_cache_handle_versioning(cache->disk_cache); - seq_disk_cache_get_files(cache->disk_cache, seq_disk_cache_base_dir()); - cache->disk_cache->timestamp = scene->ed->disk_cache_timestamp; - BLI_mutex_unlock(&cache_create_lock); -} - static void seq_cache_create(Main *bmain, Scene *scene) { BLI_mutex_lock(&cache_create_lock); @@ -1246,9 +597,7 @@ void seq_cache_destruct(Scene *scene) BLI_mutex_end(&cache->iterator_mutex); if (cache->disk_cache != NULL) { - BLI_freelistN(&cache->disk_cache->files); - BLI_mutex_end(&cache->disk_cache->read_write_mutex); - MEM_freeN(cache->disk_cache); + seq_disk_cache_free(cache->disk_cache); } MEM_freeN(cache); @@ -1297,7 +646,7 @@ void seq_cache_cleanup_sequence(Scene *scene, } if (seq_disk_cache_is_enabled(cache->bmain) && cache->disk_cache != NULL) { - seq_disk_cache_invalidate(scene, seq, seq_changed, invalidate_types); + seq_disk_cache_invalidate(cache->disk_cache, scene, seq, seq_changed, invalidate_types); } seq_cache_lock(scene); @@ -1434,12 +783,10 @@ struct ImBuf *seq_cache_get(const SeqRenderData *context, /* Try disk cache: */ if (seq_disk_cache_is_enabled(context->bmain)) { if (cache->disk_cache == NULL) { - seq_disk_cache_create(context->bmain, context->scene); + cache->disk_cache = seq_disk_cache_create(context->bmain, context->scene); } - BLI_mutex_lock(&cache->disk_cache->read_write_mutex); ibuf = seq_disk_cache_read_file(cache->disk_cache, &key); - BLI_mutex_unlock(&cache->disk_cache->read_write_mutex); if (ibuf == NULL) { return NULL; @@ -1550,9 +897,7 @@ void seq_cache_put( seq_disk_cache_create(context->bmain, context->scene); } - BLI_mutex_lock(&cache->disk_cache->read_write_mutex); seq_disk_cache_write_file(cache->disk_cache, key, i); - BLI_mutex_unlock(&cache->disk_cache->read_write_mutex); seq_disk_cache_enforce_limits(cache->disk_cache); } } diff --git a/source/blender/sequencer/intern/image_cache.h b/source/blender/sequencer/intern/image_cache.h index 60031311985..e7827c15305 100644 --- a/source/blender/sequencer/intern/image_cache.h +++ b/source/blender/sequencer/intern/image_cache.h @@ -27,15 +27,29 @@ extern "C" { #endif +#include "SEQ_render.h" /* Needed for #eSeqTaskId. */ + struct ImBuf; struct Main; struct Scene; struct SeqRenderData; struct Sequence; -#ifdef __cplusplus -} -#endif +typedef struct SeqCacheKey { + struct SeqCache *cache_owner; + void *userkey; + struct SeqCacheKey *link_prev; /* Used for linking intermediate items to final frame. */ + struct SeqCacheKey *link_next; /* Used for linking intermediate items to final frame. */ + struct Sequence *seq; + struct SeqRenderData context; + float frame_index; /* Usually same as timeline_frame. Mapped to media for RAW entries. */ + float timeline_frame; /* Only for reference - used for freeing when cache is full. */ + float cost; /* In short: render time(s) divided by playback frame duration(s) */ + bool is_temp_cache; /* this cache entry will be freed before rendering next frame */ + /* ID of task for assigning temp cache entries to particular task(thread, etc.) */ + eSeqTaskId task_id; + int type; +} SeqCacheKey; struct ImBuf *seq_cache_get(const struct SeqRenderData *context, struct Sequence *seq, @@ -67,6 +81,7 @@ void seq_cache_cleanup_sequence(struct Scene *scene, bool force_seq_changed_range); void seq_cache_thumbnail_cleanup(Scene *scene, rctf *view_area); bool seq_cache_is_full(void); +float seq_cache_frame_index_to_timeline_frame(struct Sequence *seq, float frame_index); #ifdef __cplusplus } -- cgit v1.2.3