progress

5 files changed, 125 insertions, 5 deletions

diff --git a/source/blender/blenlib/BLI_cpp_type.hh b/source/blender/blenlib/BLI_cpp_type.hh
index 9453eb89a2e..c8824813a21 100644
--- a/source/blender/blenlib/BLI_cpp_type.hh
+++ b/source/blender/blenlib/BLI_cpp_type.hh
@@ -118,9 +118,11 @@ class CPPType : NonCopyable, NonMovable {
 
   void (*copy_assign_)(const void *src, void *dst) = nullptr;
   void (*copy_assign_indices_)(const void *src, void *dst, IndexMask mask) = nullptr;
+  void (*copy_assign_compressed_)(const void *src, void *dst, IndexMask mask) = nullptr;
 
   void (*copy_construct_)(const void *src, void *dst) = nullptr;
   void (*copy_construct_indices_)(const void *src, void *dst, IndexMask mask) = nullptr;
+  void (*copy_construct_compressed_)(const void *src, void *dst, IndexMask mask) = nullptr;
 
   void (*move_assign_)(void *src, void *dst) = nullptr;
   void (*move_assign_indices_)(void *src, void *dst, IndexMask mask) = nullptr;
@@ -408,6 +410,15 @@ class CPPType : NonCopyable, NonMovable {
     copy_assign_indices_(src, dst, mask);
   }
 
+  void copy_assign_compressed(const void *src, void *dst, IndexMask mask) const
+  {
+    BLI_assert(mask.size() == 0 || src != dst);
+    BLI_assert(mask.size() == 0 || this->pointer_can_point_to_instance(src));
+    BLI_assert(mask.size() == 0 || this->pointer_can_point_to_instance(dst));
+
+    copy_assign_compressed_(src, dst, mask);
+  }
+
   /**
    * Copy an instance of this type from src to dst.
    *
@@ -439,6 +450,15 @@ class CPPType : NonCopyable, NonMovable {
     copy_construct_indices_(src, dst, mask);
   }
 
+  void copy_construct_compressed(const void *src, void *dst, IndexMask mask) const
+  {
+    BLI_assert(mask.size() == 0 || src != dst);
+    BLI_assert(mask.size() == 0 || this->pointer_can_point_to_instance(src));
+    BLI_assert(mask.size() == 0 || this->pointer_can_point_to_instance(dst));
+
+    copy_construct_compressed_(src, dst, mask);
+  }
+
   /**
    * Move an instance of this type from src to dst.
    *
diff --git a/source/blender/blenlib/BLI_cpp_type_make.hh b/source/blender/blenlib/BLI_cpp_type_make.hh
index 2612348075b..b0dbbff7ca8 100644
--- a/source/blender/blenlib/BLI_cpp_type_make.hh
+++ b/source/blender/blenlib/BLI_cpp_type_make.hh
@@ -51,6 +51,17 @@ template<typename T> void copy_assign_indices_cb(const void *src, void *dst, Ind
 
   mask.foreach_index([&](int64_t i) { dst_[i] = src_[i]; });
 }
+template<typename T> void copy_assign_compressed_cb(const void *src, void *dst, IndexMask mask)
+{
+  const T *src_ = static_cast<const T *>(src);
+  T *dst_ = static_cast<T *>(dst);
+
+  mask.to_best_mask_type([&](auto best_mask) {
+    for (const int64_t i : IndexRange(best_mask.size())) {
+      dst_[i] = src_[best_mask[i]];
+    }
+  });
+}
 
 template<typename T> void copy_construct_cb(const void *src, void *dst)
 {
@@ -63,6 +74,17 @@ template<typename T> void copy_construct_indices_cb(const void *src, void *dst,
 
   mask.foreach_index([&](int64_t i) { new (dst_ + i) T(src_[i]); });
 }
+template<typename T> void copy_construct_compressed_cb(const void *src, void *dst, IndexMask mask)
+{
+  const T *src_ = static_cast<const T *>(src);
+  T *dst_ = static_cast<T *>(dst);
+
+  mask.to_best_mask_type([&](auto best_mask) {
+    for (const int64_t i : IndexRange(best_mask.size())) {
+      new (dst_ + i) T(src_[best_mask[i]]);
+    }
+  });
+}
 
 template<typename T> void move_assign_cb(void *src, void *dst)
 {
@@ -208,10 +230,12 @@ CPPType::CPPType(CPPTypeParam<T, Flags> /* unused */, StringRef debug_name)
   if constexpr (std::is_copy_assignable_v<T>) {
     copy_assign_ = copy_assign_cb<T>;
     copy_assign_indices_ = copy_assign_indices_cb<T>;
+    copy_assign_compressed_ = copy_assign_compressed_cb<T>;
   }
   if constexpr (std::is_copy_constructible_v<T>) {
     copy_construct_ = copy_construct_cb<T>;
     copy_construct_indices_ = copy_construct_indices_cb<T>;
+    copy_construct_compressed_ = copy_construct_compressed_cb<T>;
   }
   if constexpr (std::is_move_assignable_v<T>) {
     move_assign_ = move_assign_cb<T>;
diff --git a/source/blender/blenlib/BLI_generic_virtual_array.hh b/source/blender/blenlib/BLI_generic_virtual_array.hh
index 3b9ae33c9f1..cb1a984c8ce 100644
--- a/source/blender/blenlib/BLI_generic_virtual_array.hh
+++ b/source/blender/blenlib/BLI_generic_virtual_array.hh
@@ -51,6 +51,9 @@ class GVArrayImpl {
   virtual void materialize(const IndexMask mask, void *dst) const;
   virtual void materialize_to_uninitialized(const IndexMask mask, void *dst) const;
 
+  virtual void materialize_compressed(IndexMask mask, void *dst) const;
+  virtual void materialize_compressed_to_uninitialized(IndexMask mask, void *dst) const;
+
   virtual bool try_assign_VArray(void *varray) const;
   virtual bool may_have_ownership() const;
 };
@@ -593,6 +596,10 @@ class GVArrayImpl_For_GSpan : public GVMutableArrayImpl {
 
   virtual void materialize(const IndexMask mask, void *dst) const override;
   virtual void materialize_to_uninitialized(const IndexMask mask, void *dst) const override;
+
+  virtual void materialize_compressed(const IndexMask mask, void *dst) const override;
+  virtual void materialize_compressed_to_uninitialized(const IndexMask mask,
+                                                       void *dst) const override;
 };
 
 /** \} */
diff --git a/source/blender/blenlib/intern/generic_virtual_array.cc b/source/blender/blenlib/intern/generic_virtual_array.cc
index 7a2b45ff857..3764367b6b7 100644
--- a/source/blender/blenlib/intern/generic_virtual_array.cc
+++ b/source/blender/blenlib/intern/generic_virtual_array.cc
@@ -24,6 +24,22 @@ void GVArrayImpl::materialize_to_uninitialized(const IndexMask mask, void *dst)
   }
 }
 
+void GVArrayImpl::materialize_compressed(IndexMask mask, void *dst) const
+{
+  for (const int64_t i : mask.index_range()) {
+    void *elem_dst = POINTER_OFFSET(dst, type_->size() * i);
+    this->get(mask[i], elem_dst);
+  }
+}
+
+void GVArrayImpl::materialize_compressed_to_uninitialized(IndexMask mask, void *dst) const
+{
+  for (const int64_t i : mask.index_range()) {
+    void *elem_dst = POINTER_OFFSET(dst, type_->size() * i);
+    this->get_to_uninitialized(mask[i], elem_dst);
+  }
+}
+
 void GVArrayImpl::get(const int64_t index, void *r_value) const
 {
   type_->destruct(r_value);
@@ -182,6 +198,17 @@ void GVArrayImpl_For_GSpan::materialize_to_uninitialized(const IndexMask mask, v
   type_->copy_construct_indices(data_, dst, mask);
 }
 
+void GVArrayImpl_For_GSpan::materialize_compressed(const IndexMask mask, void *dst) const
+{
+  type_->copy_assign_compressed(data_, dst, mask);
+}
+
+void GVArrayImpl_For_GSpan::materialize_compressed_to_uninitialized(const IndexMask mask,
+                                                                    void *dst) const
+{
+  type_->copy_construct_compressed(data_, dst, mask);
+}
+
 class GVArrayImpl_For_GSpan_final final : public GVArrayImpl_For_GSpan {
  public:
   using GVArrayImpl_For_GSpan::GVArrayImpl_For_GSpan;
diff --git a/source/blender/functions/FN_multi_function_builder.hh b/source/blender/functions/FN_multi_function_builder.hh
index b041e67390c..cf6a57df8a0 100644
--- a/source/blender/functions/FN_multi_function_builder.hh
+++ b/source/blender/functions/FN_multi_function_builder.hh
@@ -47,11 +47,53 @@ template<typename In1, typename Out1> class CustomMF_SI_SO : public MultiFunctio
   template<typename ElementFuncT> static FunctionT create_function(ElementFuncT element_fn)
   {
     return [=](IndexMask mask, const VArray<In1> &in1, MutableSpan<Out1> out1) {
-      /* Devirtualization results in a 2-3x speedup for some simple functions. */
-      devirtualize_varray(in1, [&](const auto &in1) {
-        mask.to_best_mask_type(
-            [&](const auto &mask) { execute_SI_SO(element_fn, mask, in1, out1.data()); });
-      });
+      const int64_t mask_size = mask.size();
+      const bool in1_is_single = in1.is_single();
+      const bool in1_is_span = in1.is_span();
+
+      static constexpr int64_t MaxChunkSize = 32;
+      /* Properly handle initialization. */
+      TypedBuffer<In1, MaxChunkSize> in1_buffer_owner;
+      MutableSpan<In1> in1_buffer{in1_buffer_owner.ptr(), MaxChunkSize};
+
+      if (in1_is_single) {
+        const In1 in1_single = in1.get_internal_single();
+        in1_buffer.fill(in1_single);
+      }
+
+      Span<In1> in1_span;
+      if (in1_is_span) {
+        in1_span = in1.get_internal_span();
+      }
+
+      for (int64_t chunk_start = 0; chunk_start < mask_size; chunk_start += MaxChunkSize) {
+        const int64_t chunk_size = std::min(mask_size - chunk_start, MaxChunkSize);
+        const IndexMask sliced_mask = mask.slice(chunk_start, chunk_size);
+        if (sliced_mask.is_range()) {
+          const IndexRange sliced_mask_range = sliced_mask.as_range();
+          Span<In1> in1_chunk;
+          if (in1_is_single) {
+            in1_chunk = in1_buffer;
+          }
+          else if (in1_is_span) {
+            in1_chunk = in1_span.slice(sliced_mask_range);
+          }
+          else {
+            in1.materialize_compressed_to_uninitialized(sliced_mask,
+                                                        in1_buffer.take_front(chunk_size));
+            in1_chunk = in1_buffer;
+          }
+
+          execute_SI_SO(element_fn, IndexRange(chunk_size), in1_chunk, out1.data() + chunk_start);
+        }
+        else {
+          // const Span<int64_t> sliced_mask_indices = sliced_mask.indices();
+          /* TODO */
+          BLI_assert_unreachable();
+        }
+      }
+
+      return;
     };
   }