Functions: speedup preparing multi-function parameters

My benchmark which spend most time preparing function parameters
takes `250 ms` now, from `510 ms` before. This is mainly achieved by
doing less unnecessary work and by giving the compiler more inlined
code to optimize.

* Reserve correct vector sizes and use unchecked `append` function.
* Construct `GVArray` parameters directly in the vector, instead of
  moving/copying them in the vector afterwards.
* Inline some constructors, because that allows the compiler understand
  what is happening, resulting in less code.

This probably has negilible impact on the user experience currently,
because there are other bottlenecks.

Differential Revision: https://developer.blender.org/D15009

1 files changed, 58 insertions, 104 deletions

diff --git a/source/blender/blenlib/intern/generic_virtual_array.cc b/source/blender/blenlib/intern/generic_virtual_array.cc
index a3a17952a97..a6fbf4bff5b 100644
--- a/source/blender/blenlib/intern/generic_virtual_array.cc
+++ b/source/blender/blenlib/intern/generic_virtual_array.cc
@@ -85,11 +85,6 @@ bool GVArrayImpl::may_have_ownership() const
 /** \name #GVMutableArrayImpl
  * \{ */
 
-GVMutableArrayImpl::GVMutableArrayImpl(const CPPType &type, const int64_t size)
-    : GVArrayImpl(type, size)
-{
-}
-
 void GVMutableArrayImpl::set_by_copy(const int64_t index, const void *value)
 {
   BUFFER_FOR_CPP_TYPE_VALUE(*type_, buffer);
@@ -141,18 +136,6 @@ bool GVMutableArrayImpl::try_assign_VMutableArray(void *UNUSED(varray)) const
 /** \name #GVArrayImpl_For_GSpan
  * \{ */
 
-GVArrayImpl_For_GSpan::GVArrayImpl_For_GSpan(const GMutableSpan span)
-    : GVMutableArrayImpl(span.type(), span.size()),
-      data_(span.data()),
-      element_size_(span.type().size())
-{
-}
-
-GVArrayImpl_For_GSpan::GVArrayImpl_For_GSpan(const CPPType &type, const int64_t size)
-    : GVMutableArrayImpl(type, size), element_size_(type.size())
-{
-}
-
 void GVArrayImpl_For_GSpan::get(const int64_t index, void *r_value) const
 {
   type_->copy_assign(POINTER_OFFSET(data_, element_size_ * index), r_value);
@@ -209,17 +192,6 @@ void GVArrayImpl_For_GSpan::materialize_compressed_to_uninitialized(const IndexM
   type_->copy_construct_compressed(data_, dst, mask);
 }
 
-class GVArrayImpl_For_GSpan_final final : public GVArrayImpl_For_GSpan {
- public:
-  using GVArrayImpl_For_GSpan::GVArrayImpl_For_GSpan;
-
- private:
-  bool may_have_ownership() const override
-  {
-    return false;
-  }
-};
-
 /** \} */
 
 /* -------------------------------------------------------------------- */
@@ -227,79 +199,56 @@ class GVArrayImpl_For_GSpan_final final : public GVArrayImpl_For_GSpan {
  * \{ */
 
 /* Generic virtual array where each element has the same value. The value is not owned. */
-class GVArrayImpl_For_SingleValueRef : public GVArrayImpl {
- protected:
-  const void *value_ = nullptr;
 
- public:
-  GVArrayImpl_For_SingleValueRef(const CPPType &type, const int64_t size, const void *value)
-      : GVArrayImpl(type, size), value_(value)
-  {
-  }
-
- protected:
-  GVArrayImpl_For_SingleValueRef(const CPPType &type, const int64_t size) : GVArrayImpl(type, size)
-  {
-  }
-
-  void get(const int64_t UNUSED(index), void *r_value) const override
-  {
-    type_->copy_assign(value_, r_value);
-  }
-  void get_to_uninitialized(const int64_t UNUSED(index), void *r_value) const override
-  {
-    type_->copy_construct(value_, r_value);
-  }
-
-  bool is_span() const override
-  {
-    return size_ == 1;
-  }
-  GSpan get_internal_span() const override
-  {
-    return GSpan{*type_, value_, 1};
-  }
-
-  bool is_single() const override
-  {
-    return true;
-  }
-  void get_internal_single(void *r_value) const override
-  {
-    type_->copy_assign(value_, r_value);
-  }
+void GVArrayImpl_For_SingleValueRef::get(const int64_t UNUSED(index), void *r_value) const
+{
+  type_->copy_assign(value_, r_value);
+}
+void GVArrayImpl_For_SingleValueRef::get_to_uninitialized(const int64_t UNUSED(index),
+                                                          void *r_value) const
+{
+  type_->copy_construct(value_, r_value);
+}
 
-  void materialize(const IndexMask mask, void *dst) const override
-  {
-    type_->fill_assign_indices(value_, dst, mask);
-  }
+bool GVArrayImpl_For_SingleValueRef::is_span() const
+{
+  return size_ == 1;
+}
+GSpan GVArrayImpl_For_SingleValueRef::get_internal_span() const
+{
+  return GSpan{*type_, value_, 1};
+}
 
-  void materialize_to_uninitialized(const IndexMask mask, void *dst) const override
-  {
-    type_->fill_construct_indices(value_, dst, mask);
-  }
+bool GVArrayImpl_For_SingleValueRef::is_single() const
+{
+  return true;
+}
+void GVArrayImpl_For_SingleValueRef::get_internal_single(void *r_value) const
+{
+  type_->copy_assign(value_, r_value);
+}
 
-  void materialize_compressed(const IndexMask mask, void *dst) const override
-  {
-    type_->fill_assign_n(value_, dst, mask.size());
-  }
+void GVArrayImpl_For_SingleValueRef::materialize(const IndexMask mask, void *dst) const
+{
+  type_->fill_assign_indices(value_, dst, mask);
+}
 
-  void materialize_compressed_to_uninitialized(const IndexMask mask, void *dst) const override
-  {
-    type_->fill_construct_n(value_, dst, mask.size());
-  }
-};
+void GVArrayImpl_For_SingleValueRef::materialize_to_uninitialized(const IndexMask mask,
+                                                                  void *dst) const
+{
+  type_->fill_construct_indices(value_, dst, mask);
+}
 
-class GVArrayImpl_For_SingleValueRef_final final : public GVArrayImpl_For_SingleValueRef {
- public:
-  using GVArrayImpl_For_SingleValueRef::GVArrayImpl_For_SingleValueRef;
+void GVArrayImpl_For_SingleValueRef::materialize_compressed(const IndexMask mask, void *dst) const
+{
+  type_->fill_assign_n(value_, dst, mask.size());
+}
 
- private:
-  bool may_have_ownership() const override
-  {
-    return false;
-  }
-};
+void GVArrayImpl_For_SingleValueRef::materialize_compressed_to_uninitialized(const IndexMask mask,
+                                                                             void *dst) const
+{
+  type_->fill_construct_n(value_, dst, mask.size());
+}
 
 /** \} */
 
@@ -529,8 +478,6 @@ class GVArrayImpl_For_SlicedGVArray : public GVArrayImpl {
 /** \name #GVArrayCommon
  * \{ */
 
-GVArrayCommon::GVArrayCommon() = default;
-
 GVArrayCommon::GVArrayCommon(const GVArrayCommon &other) : storage_(other.storage_)
 {
   impl_ = this->impl_from_storage();
@@ -672,17 +619,27 @@ GVArray::GVArray(std::shared_ptr<const GVArrayImpl> impl) : GVArrayCommon(std::m
 {
 }
 
-GVArray GVArray::ForSingle(const CPPType &type, const int64_t size, const void *value)
+GVArray::GVArray(varray_tag::single /* tag */,
+                 const CPPType &type,
+                 int64_t size,
+                 const void *value)
 {
   if (type.is_trivial() && type.size() <= 16 && type.alignment() <= 8) {
-    return GVArray::For<GVArrayImpl_For_SmallTrivialSingleValue<16>>(type, size, value);
+    this->emplace<GVArrayImpl_For_SmallTrivialSingleValue<16>>(type, size, value);
   }
-  return GVArray::For<GVArrayImpl_For_SingleValue>(type, size, value);
+  else {
+    this->emplace<GVArrayImpl_For_SingleValue>(type, size, value);
+  }
+}
+
+GVArray GVArray::ForSingle(const CPPType &type, const int64_t size, const void *value)
+{
+  return GVArray(varray_tag::single{}, type, size, value);
 }
 
 GVArray GVArray::ForSingleRef(const CPPType &type, const int64_t size, const void *value)
 {
-  return GVArray::For<GVArrayImpl_For_SingleValueRef_final>(type, size, value);
+  return GVArray(varray_tag::single_ref{}, type, size, value);
 }
 
 GVArray GVArray::ForSingleDefault(const CPPType &type, const int64_t size)
@@ -692,10 +649,7 @@ GVArray GVArray::ForSingleDefault(const CPPType &type, const int64_t size)
 
 GVArray GVArray::ForSpan(GSpan span)
 {
-  /* Use const-cast because the underlying virtual array implementation is shared between const
-   * and non const data. */
-  GMutableSpan mutable_span{span.type(), const_cast<void *>(span.data()), span.size()};
-  return GVArray::For<GVArrayImpl_For_GSpan_final>(mutable_span);
+  return GVArray(varray_tag::span{}, span);
 }
 
 class GVArrayImpl_For_GArray : public GVArrayImpl_For_GSpan {