VSE: Disk cache

This patch implements dumping images from cache to HDD.
The main goal of this system is to provide a means to achieve consistent playback speed mainly for strips that are not possible to preview in real time.

How to use:
Disk cache has own settings in user preferences for path to storage, size limit and compression level.
To use disk cache, you need to check `Use Disk Cache` box, set `Disk Cache Directory`, `Disk Cache Limit` and save or open existing .blend file.
By default sequencer output will be cached only. Manual setting is possible in cache panel.

Uses:
 - Replacement or alternative for proxies. Disk cache will work with any strip type, supports float images as well.
 - Storage for strip thumbnails.
 - Less RAM needs to be allocated for preview cache

How it works:
Disk cache is extension of RAM cache. Every image, that is stored or deleted in RAM will be stored or deleted on HDD as well. Images can be compressed to save space and for use on slower drives. Compressed images are slower to write and read though.
Images are stored in bulk of 100 rendered frames per one file. This is to overcome slow file access time for large amount of files. Drawback is, that if one frame needs to be redrawn, all 100 frames are deleted.

Reviewed By: sergey

Differential Revision: https://developer.blender.org/D5524

2 files changed, 102 insertions, 1 deletions

diff --git a/source/blender/blenlib/BLI_fileops.h b/source/blender/blenlib/BLI_fileops.h
index 8596a60bc6a..89f7d01ffd6 100644
--- a/source/blender/blenlib/BLI_fileops.h
+++ b/source/blender/blenlib/BLI_fileops.h
@@ -153,7 +153,13 @@ int BLI_file_gzip(const char *from, const char *to) ATTR_WARN_UNUSED_RESULT ATTR
 #endif
 char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size) ATTR_WARN_UNUSED_RESULT
     ATTR_NONNULL();
-
+size_t BLI_gzip_mem_to_file_at_pos(void *buf,
+                                   size_t len,
+                                   FILE *file,
+                                   size_t gz_stream_offset,
+                                   int compression_level) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
+    ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 size_t BLI_file_descriptor_size(int file) ATTR_WARN_UNUSED_RESULT;
 size_t BLI_file_size(const char *file) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 
diff --git a/source/blender/blenlib/intern/fileops.c b/source/blender/blenlib/intern/fileops.c
index 3a45989fb63..afb46dd8d82 100644
--- a/source/blender/blenlib/intern/fileops.c
+++ b/source/blender/blenlib/intern/fileops.c
@@ -158,6 +158,101 @@ char *BLI_file_ungzip_to_mem(const char *from_file, int *r_size)
   return mem;
 }
 
+#define CHUNK 256 * 1024
+
+/* gzip byte array from memory and write it to file at certain position.
+ * return size of gzip stream.
+ */
+size_t BLI_gzip_mem_to_file_at_pos(
+    void *buf, size_t len, FILE *file, size_t gz_stream_offset, int compression_level)
+{
+  int ret, flush;
+  unsigned have;
+  z_stream strm;
+  unsigned char out[CHUNK];
+
+  fseek(file, gz_stream_offset, 0);
+
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+  ret = deflateInit(&strm, compression_level);
+  if (ret != Z_OK)
+    return 0;
+
+  strm.avail_in = len;
+  strm.next_in = (Bytef *)buf;
+  flush = Z_FINISH;
+
+  do {
+    strm.avail_out = CHUNK;
+    strm.next_out = out;
+    ret = deflate(&strm, flush);
+    if (ret == Z_STREAM_ERROR) {
+      return 0;
+    }
+    have = CHUNK - strm.avail_out;
+    if (fwrite(out, 1, have, file) != have || ferror(file)) {
+      deflateEnd(&strm);
+      return 0;
+    }
+  } while (strm.avail_out == 0);
+
+  if (strm.avail_in != 0 || ret != Z_STREAM_END) {
+    return 0;
+  }
+
+  deflateEnd(&strm);
+  return (size_t)strm.total_out;
+}
+
+/* read and decompress gzip stream from file at certain position to buffer.
+ * return size of decompressed data.
+ */
+size_t BLI_ungzip_file_to_mem_at_pos(void *buf, size_t len, FILE *file, size_t gz_stream_offset)
+{
+  int ret;
+  z_stream strm;
+  size_t chunk = 256 * 1024;
+  unsigned char in[CHUNK];
+
+  fseek(file, gz_stream_offset, 0);
+
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+  strm.avail_in = 0;
+  strm.next_in = Z_NULL;
+  ret = inflateInit(&strm);
+  if (ret != Z_OK)
+    return 0;
+
+  do {
+    strm.avail_in = fread(in, 1, chunk, file);
+    strm.next_in = in;
+    if (ferror(file)) {
+      inflateEnd(&strm);
+      return 0;
+    }
+
+    do {
+      strm.avail_out = len;
+      strm.next_out = (Bytef *)buf + strm.total_out;
+
+      ret = inflate(&strm, Z_NO_FLUSH);
+      if (ret == Z_STREAM_ERROR) {
+        return 0;
+      }
+    } while (strm.avail_out == 0);
+
+  } while (ret != Z_STREAM_END);
+
+  inflateEnd(&strm);
+  return (size_t)strm.total_out;
+}
+
+#undef CHUNK
+
 /**
  * Returns true if the file with the specified name can be written.
  * This implementation uses access(2), which makes the check according