Alembic Procedural: basic cache control settings

This adds a setting to enable data caching, and another one to set the
maximum cache size in megabytes.

When caching is enabled we load the data for the entire animation in
memory, as we already do, however, if the data exceeds the memory limit,
render is aborted.

When caching is disabled, we simply load the data for the current frame
in memory.

Ref D10197

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D11163

8 files changed, 159 insertions, 4 deletions

diff --git a/intern/cycles/blender/blender_object.cpp b/intern/cycles/blender/blender_object.cpp
index 2dbebac4cc3..a7eae421b55 100644
--- a/intern/cycles/blender/blender_object.cpp
+++ b/intern/cycles/blender/blender_object.cpp
@@ -523,6 +523,9 @@ void BlenderSync::sync_procedural(BL::Object &b_ob,
 
   procedural->set_scale(cache_file.scale());
 
+  procedural->set_use_prefetch(cache_file.use_prefetch());
+  procedural->set_prefetch_cache_size(cache_file.prefetch_cache_size());
+
   /* create or update existing AlembicObjects */
   ustring object_path = ustring(b_mesh_cache.object_path());
 
diff --git a/intern/cycles/render/alembic.cpp b/intern/cycles/render/alembic.cpp
index 81f47256739..69bc0712674 100644
--- a/intern/cycles/render/alembic.cpp
+++ b/intern/cycles/render/alembic.cpp
@@ -25,6 +25,7 @@
 #include "render/shader.h"
 
 #include "util/util_foreach.h"
+#include "util/util_logging.h"
 #include "util/util_progress.h"
 #include "util/util_transform.h"
 #include "util/util_vector.h"
@@ -211,6 +212,35 @@ void CachedData::set_time_sampling(TimeSampling time_sampling)
   }
 }
 
+size_t CachedData::memory_used() const
+{
+  size_t mem_used = 0;
+
+  mem_used += curve_first_key.memory_used();
+  mem_used += curve_keys.memory_used();
+  mem_used += curve_radius.memory_used();
+  mem_used += curve_shader.memory_used();
+  mem_used += num_ngons.memory_used();
+  mem_used += shader.memory_used();
+  mem_used += subd_creases_edge.memory_used();
+  mem_used += subd_creases_weight.memory_used();
+  mem_used += subd_face_corners.memory_used();
+  mem_used += subd_num_corners.memory_used();
+  mem_used += subd_ptex_offset.memory_used();
+  mem_used += subd_smooth.memory_used();
+  mem_used += subd_start_corner.memory_used();
+  mem_used += transforms.memory_used();
+  mem_used += triangles.memory_used();
+  mem_used += uv_loops.memory_used();
+  mem_used += vertices.memory_used();
+
+  for (const CachedAttribute &attr : attributes) {
+    mem_used += attr.data.memory_used();
+  }
+
+  return mem_used;
+}
+
 static M44d convert_yup_zup(const M44d &mtx, float scale_mult)
 {
   V3d scale, shear, rotation, translation;
@@ -706,6 +736,9 @@ NODE_DEFINE(AlembicProcedural)
 
   SOCKET_NODE_ARRAY(objects, "Objects", AlembicObject::get_node_type());
 
+  SOCKET_BOOLEAN(use_prefetch, "Use Prefetch", true);
+  SOCKET_INT(prefetch_cache_size, "Prefetch Cache Size", 4096);
+
   return type;
 }
 
@@ -823,6 +856,30 @@ void AlembicProcedural::generate(Scene *scene, Progress &progress)
     }
   }
 
+  if (use_prefetch_is_modified()) {
+    if (!use_prefetch) {
+      for (Node *node : objects) {
+        AlembicObject *object = static_cast<AlembicObject *>(node);
+        object->clear_cache();
+      }
+    }
+  }
+
+  if (prefetch_cache_size_is_modified()) {
+    /* Check whether the current memory usage fits in the new requested size,
+     * abort the render if it is any higher. */
+    size_t memory_used = 0ul;
+    for (Node *node : objects) {
+      AlembicObject *object = static_cast<AlembicObject *>(node);
+      memory_used += object->get_cached_data().memory_used();
+    }
+
+    if (memory_used > get_prefetch_cache_size_in_bytes()) {
+      progress.set_error("Error: Alembic Procedural memory limit reached");
+      return;
+    }
+  }
+
   build_caches(progress);
 
   foreach (Node *node, objects) {
@@ -1300,6 +1357,8 @@ void AlembicProcedural::walk_hierarchy(
 
 void AlembicProcedural::build_caches(Progress &progress)
 {
+  size_t memory_used = 0;
+
   for (Node *node : objects) {
     AlembicObject *object = static_cast<AlembicObject *>(node);
 
@@ -1353,7 +1412,18 @@ void AlembicProcedural::build_caches(Progress &progress)
     if (scale_is_modified() || object->get_cached_data().transforms.size() == 0) {
       object->setup_transform_cache(object->get_cached_data(), scale);
     }
+
+    memory_used += object->get_cached_data().memory_used();
+
+    if (use_prefetch) {
+      if (memory_used > get_prefetch_cache_size_in_bytes()) {
+        progress.set_error("Error: Alembic Procedural memory limit reached");
+        return;
+      }
+    }
   }
+
+  VLOG(1) << "AlembicProcedural memory usage : " << string_human_readable_size(memory_used);
 }
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/alembic.h b/intern/cycles/render/alembic.h
index 9c58af720f6..8e166a5ab04 100644
--- a/intern/cycles/render/alembic.h
+++ b/intern/cycles/render/alembic.h
@@ -272,6 +272,21 @@ template<typename T> class DataStore {
     node->set(*socket, value);
   }
 
+  size_t memory_used() const
+  {
+    if constexpr (is_array<T>::value) {
+      size_t mem_used = 0;
+
+      for (const T &array : data) {
+        mem_used += array.size() * sizeof(array[0]);
+      }
+
+      return mem_used;
+    }
+
+    return data.size() * sizeof(T);
+  }
+
  private:
   const TimeIndexPair &get_index_for_time(double time) const
   {
@@ -332,6 +347,8 @@ struct CachedData {
   void invalidate_last_loaded_time(bool attributes_only = false);
 
   void set_time_sampling(Alembic::AbcCoreAbstract::TimeSampling time_sampling);
+
+  size_t memory_used() const;
 };
 
 /* Representation of an Alembic object for the AlembicProcedural.
@@ -482,6 +499,13 @@ class AlembicProcedural : public Procedural {
    * software. */
   NODE_SOCKET_API(float, scale)
 
+  /* Cache controls */
+  NODE_SOCKET_API(bool, use_prefetch)
+
+  /* Memory limit for the cache, if the data does not fit within this limit, rendering is aborted.
+   */
+  NODE_SOCKET_API(int, prefetch_cache_size)
+
   AlembicProcedural();
   ~AlembicProcedural();
 
@@ -531,6 +555,12 @@ class AlembicProcedural : public Procedural {
   void read_subd(AlembicObject *abc_object, Alembic::AbcGeom::Abc::chrono_t frame_time);
 
   void build_caches(Progress &progress);
+
+  size_t get_prefetch_cache_size_in_bytes() const
+  {
+    /* prefetch_cache_size is in megabytes, so convert to bytes. */
+    return static_cast<size_t>(prefetch_cache_size) * 1024 * 1024;
+  }
 };
 
 CCL_NAMESPACE_END
diff --git a/intern/cycles/render/alembic_read.cpp b/intern/cycles/render/alembic_read.cpp
index c53ec668938..b105af63b44 100644
--- a/intern/cycles/render/alembic_read.cpp
+++ b/intern/cycles/render/alembic_read.cpp
@@ -44,9 +44,19 @@ static set<chrono_t> get_relevant_sample_times(AlembicProcedural *proc,
     return result;
   }
 
-  // load the data for the entire animation
-  const double start_frame = static_cast<double>(proc->get_start_frame());
-  const double end_frame = static_cast<double>(proc->get_end_frame());
+  double start_frame;
+  double end_frame;
+
+  if (proc->get_use_prefetch()) {
+    // load the data for the entire animation
+    start_frame = static_cast<double>(proc->get_start_frame());
+    end_frame = static_cast<double>(proc->get_end_frame());
+  }
+  else {
+    // load the data for the current frame
+    start_frame = static_cast<double>(proc->get_frame());
+    end_frame = start_frame;
+  }
 
   const double frame_rate = static_cast<double>(proc->get_frame_rate());
   const double start_time = start_frame / frame_rate;
diff --git a/source/blender/editors/interface/interface_templates.c b/source/blender/editors/interface/interface_templates.c
index 351b73c320b..cdb35d19855 100644
--- a/source/blender/editors/interface/interface_templates.c
+++ b/source/blender/editors/interface/interface_templates.c
@@ -6489,6 +6489,17 @@ void uiTemplateCacheFile(uiLayout *layout,
   uiLayoutSetActive(row, engine_supports_procedural);
   uiItemR(row, &fileptr, "use_render_procedural", 0, NULL, ICON_NONE);
 
+  const bool use_render_procedural = RNA_boolean_get(&fileptr, "use_render_procedural");
+  const bool use_prefetch = RNA_boolean_get(&fileptr, "use_prefetch");
+
+  row = uiLayoutRow(layout, false);
+  uiLayoutSetEnabled(row, use_render_procedural);
+  uiItemR(row, &fileptr, "use_prefetch", 0, NULL, ICON_NONE);
+
+  sub = uiLayoutRow(layout, false);
+  uiLayoutSetEnabled(sub, use_prefetch && use_render_procedural);
+  uiItemR(sub, &fileptr, "prefetch_cache_size", 0, NULL, ICON_NONE);
+
   row = uiLayoutRowWithHeading(layout, true, IFACE_("Override Frame"));
   sub = uiLayoutRow(row, true);
   uiLayoutSetPropDecorate(sub, false);
@@ -6510,6 +6521,10 @@ void uiTemplateCacheFile(uiLayout *layout,
   uiItemR(layout, &fileptr, "velocity_name", 0, NULL, ICON_NONE);
   uiItemR(layout, &fileptr, "velocity_unit", 0, NULL, ICON_NONE);
 
+  row = uiLayoutRow(layout, false);
+  uiLayoutSetActive(row, engine_supports_procedural && use_render_procedural);
+  uiItemR(row, &fileptr, "default_radius", 0, NULL, ICON_NONE);
+
   /* TODO: unused for now, so no need to expose. */
 #if 0
   row = uiLayoutRow(layout, false);
diff --git a/source/blender/makesdna/DNA_cachefile_defaults.h b/source/blender/makesdna/DNA_cachefile_defaults.h
index 521b72567d4..74fbe5012ab 100644
--- a/source/blender/makesdna/DNA_cachefile_defaults.h
+++ b/source/blender/makesdna/DNA_cachefile_defaults.h
@@ -40,6 +40,8 @@
     .handle = NULL, \
     .handle_filepath[0] = '\0', \
     .handle_readers = NULL, \
+    .use_prefetch = 1, \
+    .prefetch_cache_size = 4096, \
   }
 
 /** \} */
diff --git a/source/blender/makesdna/DNA_cachefile_types.h b/source/blender/makesdna/DNA_cachefile_types.h
index ae4ade49be1..0f4c53a6e7e 100644
--- a/source/blender/makesdna/DNA_cachefile_types.h
+++ b/source/blender/makesdna/DNA_cachefile_types.h
@@ -101,7 +101,15 @@ typedef struct CacheFile {
    */
   char use_render_procedural;
 
-  char _pad1[7];
+  char _pad1[3];
+
+  /** Enable data prefetching when using the Cycles Procedural. */
+  char use_prefetch;
+
+  /** Size in megabytes for the prefetch cache used by the Cycles Procedural. */
+  int prefetch_cache_size;
+
+  char _pad2[7];
 
   char velocity_unit;
   /* Name of the velocity property in the archive. */
diff --git a/source/blender/makesrna/intern/rna_cachefile.c b/source/blender/makesrna/intern/rna_cachefile.c
index cb0a490417d..74d924b8937 100644
--- a/source/blender/makesrna/intern/rna_cachefile.c
+++ b/source/blender/makesrna/intern/rna_cachefile.c
@@ -150,6 +150,23 @@ static void rna_def_cachefile(BlenderRNA *brna)
                            "determine which file to use in a file sequence");
   RNA_def_property_update(prop, 0, "rna_CacheFile_update");
 
+  /* ----------------- Cache controls ----------------- */
+
+  prop = RNA_def_property(srna, "use_prefetch", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_ui_text(
+      prop,
+      "Use Prefetch",
+      "When enabled, the Cycles Procedural will preload animation data for faster updates");
+  RNA_def_property_update(prop, 0, "rna_CacheFile_update");
+
+  prop = RNA_def_property(srna, "prefetch_cache_size", PROP_INT, PROP_UNSIGNED);
+  RNA_def_property_ui_text(
+      prop,
+      "Prefetch Cache Size",
+      "Memory usage limit in megabytes for the Cycles Procedural cache, if the data does not "
+      "fit within the limit, rendering is aborted");
+  RNA_def_property_update(prop, 0, "rna_CacheFile_update");
+
   /* ----------------- Axis Conversion ----------------- */
 
   prop = RNA_def_property(srna, "forward_axis", PROP_ENUM, PROP_NONE);