Make all draw command inside indirect buffers

This make sure the resource pipeline is the same for both multidraw and
non multidraw calls.

7 files changed, 276 insertions, 118 deletions

diff --git a/source/blender/draw/intern/draw_command.cc b/source/blender/draw/intern/draw_command.cc
index 94e15112719..edd6efe1f7b 100644
--- a/source/blender/draw/intern/draw_command.cc
+++ b/source/blender/draw/intern/draw_command.cc
@@ -66,17 +66,15 @@ void Draw::execute(RecordingState &state) const
 {
   state.front_facing_set(handle.has_inverted_handedness());
 
-  uint resource_id = handle.resource_index();
-
   GPU_batch_set_shader(batch, state.shader);
-  GPU_batch_draw_advanced(batch, vertex_first, vertex_len, resource_id, instance_len);
+  GPU_batch_draw_advanced(batch, vertex_first, vertex_len, 0, instance_len);
 }
 
 void DrawIndirect::execute(RecordingState &state) const
 {
   state.front_facing_set(handle.has_inverted_handedness());
 
-  GPU_batch_draw_indirect(batch, *indirect_buf);
+  GPU_batch_draw_indirect(batch, *indirect_buf, 0);
 }
 
 void Dispatch::execute(RecordingState &state) const
diff --git a/source/blender/draw/intern/draw_command.hh b/source/blender/draw/intern/draw_command.hh
index a2168201360..20a8a13d060 100644
--- a/source/blender/draw/intern/draw_command.hh
+++ b/source/blender/draw/intern/draw_command.hh
@@ -17,6 +17,8 @@
 
 namespace blender::draw::command {
 
+class DrawCommandBuf;
+
 /* -------------------------------------------------------------------- */
 /** \name Recording State
  * \{ */
@@ -222,9 +224,9 @@ struct PushConstant {
 
 struct Draw {
   GPUBatch *batch;
-  uint instance_len;
   uint vertex_len;
   uint vertex_first;
+  uint instance_len;
   ResourceHandle handle;
 
   void execute(RecordingState &state) const;
@@ -391,10 +393,73 @@ BLI_STATIC_ASSERT(sizeof(Undetermined) <= 24, "One of the command type is too la
 /** \} */
 
 /* -------------------------------------------------------------------- */
-/** \name Multi Draw Commands
+/** \name Draw Commands
+ *
+ * A draw command buffer used to issue single draw commands without instance merging or any
+ * other optimizations.
  * \{ */
 
-/**
+class DrawCommandBuf {
+ private:
+  using ResourceIdBuf = StorageArrayBuffer<uint, 128>;
+
+ public:
+  void clear(){};
+
+  void append_draw(Vector<command::Header> &headers,
+                   Vector<command::Undetermined> &commands,
+                   GPUBatch *batch,
+                   uint instance_len,
+                   uint vertex_len,
+                   uint vertex_first,
+                   ResourceHandle handle)
+  {
+    vertex_first = vertex_first != -1 ? vertex_first : 0;
+    instance_len = instance_len != -1 ? instance_len : 1;
+
+    int64_t index = commands.append_and_get_index({});
+    headers_.append({type, static_cast<uint>(index)});
+    commands[index].draw = {batch, instance_len, vertex_len, vertex_first, handle};
+  }
+
+  void bind(ResourceIdBuf &resource_id_buf)
+  {
+    uint total_instance = 0;
+    for (DrawCommand &cmd : command_buf_) {
+      int batch_vert_len, batch_inst_len;
+      /* Now that GPUBatches are guaranteed to be finished, extract their parameters. */
+      GPU_batch_draw_parameter_get(cmd.batch, &batch_vert_len, &batch_inst_len);
+      /* Instancing attributes are not supported using the new pipeline since we use the base
+       * instance to set the correct resource_id. Workaround is a storage_buf + gl_InstanceID. */
+      BLI_assert(batch_inst_len == 1);
+
+      cmd.v_count = max_ii(cmd.v_count, batch_vert_len);
+
+      if (cmd.resource_id > 0) {
+        /* Save correct offset to start of resource_id buffer region for this draw. */
+        cmd.i_first = total_instance;
+        total_instance += cmd.i_count;
+        /* Ensure the buffer is big enough. */
+        resource_id_buf.get_or_resize(total_instance - 1);
+
+        /* Copy the resource id for all instances. */
+        for (int i = cmd.i_first; i < (cmd.i_first + cmd.i_count); i++) {
+          resource_id_buf[i] = cmd.resource_id;
+        }
+      }
+    }
+
+    if (total_instance > 0) {
+      resource_id_buf.push_update();
+    }
+  }
+};
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Multi Draw Commands
+ *
  * For efficient rendering of large scene we strive to minimize the number of draw call and state
  * changes. This reduces the amount of work the CPU has to do. To this end, we group many rendering
  * commands and sort them per render state using Command::MultiDraw as a container.
@@ -419,19 +484,20 @@ BLI_STATIC_ASSERT(sizeof(Undetermined) <= 24, "One of the command type is too la
  * +---------------------------------------------------------------+------------------------------+
  * |   4   |   2       5   |   1   |   3       4   |   6       7   |  < Resource_id (sorted)      |
  * |   1   |   1       1   |   1   |   0   |   1   |   1       1   |  < Visibility test result    |
- * |   4   |   2   |   5   |   1   |   4   |   x   |  6+7  |   x   |  < DrawCommand (compacted)   |
+ * |   4   |     2 + 5     |         1 + 4         |      6+7      |  < DrawCommand (compacted)   |
  * +---------------------------------------------------------------+------------------------------+
  *
  * In the example above, we will issue 4 multi draw indirect calls.
  *
- * Note that commands with Non-consecutive resources ID cannot be compacted together.
- * Note that visibility test can split a resource id sequence.
- *
- * As you can see, even if commands are not executed in order, but are always in increasing order
- * inside a MDI command. This is because we want to keep drw_resource_id in increasing order to
- * be able to batch consecutive commands together.
- */
-struct MultiDrawBuffer {};
+ * \{ */
+
+struct MultiDrawBuf {
+  void clear(){};
+
+  void bind(){};
+
+  uint append(uint instance_len, uint vertex_len, uint vertex_first, ResourceHandle handle){};
+};
 
 /** \} */
 
diff --git a/source/blender/draw/intern/draw_pass.cc b/source/blender/draw/intern/draw_pass.cc
index 65ed901b9ec..e869f4095bc 100644
--- a/source/blender/draw/intern/draw_pass.cc
+++ b/source/blender/draw/intern/draw_pass.cc
@@ -17,6 +17,9 @@ namespace blender::draw {
 
 void PassSimple::submit(command::RecordingState &state) const
 {
+  draw_commands_buf_.finalize();
+  draw_commands_buf_.bind();
+
   GPU_debug_group_begin(debug_name);
 
   for (const command::Header &header : headers_) {
@@ -67,6 +70,9 @@ void PassMain::submit(command::RecordingState &state) const
 
 void PassMain::Sub::submit(command::RecordingState &state) const
 {
+  draw_commands_buf_.finalize();
+  draw_commands_buf_.bind();
+
   GPU_debug_group_begin(debug_name);
 
   for (const command::Header &header : headers_) {
diff --git a/source/blender/draw/intern/draw_pass.hh b/source/blender/draw/intern/draw_pass.hh
index 560073ab4f2..5381a005325 100644
--- a/source/blender/draw/intern/draw_pass.hh
+++ b/source/blender/draw/intern/draw_pass.hh
@@ -114,31 +114,40 @@ class PassBase {
    * Record a draw call.
    * NOTE: Setting the count or first to -1 will use the values from the batch.
    * NOTE: An instance or vertex count of 0 will discard the draw call. It will not be recorded.
+   * NOTE: Implemented in derived class. Not a virtual function to avoid indirection. Here for API
+   * listing.
    */
   void draw(GPUBatch *batch,
             uint instance_len = -1,
             uint vertex_len = -1,
             uint vertex_first = -1,
             ResourceHandle handle = {0});
-  void draw_indirect(GPUBatch *batch,
-                     StorageBuffer<DrawCommand> &indirect_buffer,
-                     ResourceHandle handle = {0});
 
-  /** Shorter version for the common case. */
-  void draw(GPUBatch *batch, ResourceHandle handle)
-  {
-    draw(batch, -1, -1, -1, handle);
-  }
+  /**
+   * Shorter version for the common case.
+   * NOTE: Implemented in derived class. Not a virtual function to avoid indirection.
+   */
+  void draw(GPUBatch *batch, ResourceHandle handle);
 
   /**
    * Record a procedural draw call. Geometry is **NOT** source from a GPUBatch.
    * NOTE: An instance or vertex count of 0 will discard the draw call. It will not be recorded.
+   * NOTE: Implemented in derived class. Not a virtual function to avoid indirection. Here for
+   * API listing.
    */
   void draw_procedural(GPUPrimType primitive,
                        uint instance_len,
                        uint vertex_len,
                        uint vertex_first = -1,
                        ResourceHandle handle = {0});
+
+  /**
+   * Indirect variants.
+   * NOTE: If needed, the resource id need to also be set accordingly in the DrawCommand.
+   */
+  void draw_indirect(GPUBatch *batch,
+                     StorageBuffer<DrawCommand> &indirect_buffer,
+                     ResourceHandle handle = {0});
   void draw_procedural_indirect(GPUPrimType primitive,
                                 StorageBuffer<DrawCommand> &indirect_buffer,
                                 ResourceHandle handle = {0});
@@ -241,6 +250,12 @@ class PassBase {
 
 }  // namespace detail
 
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Normal pass type
+ * \{ */
+
 /**
  * Normal pass type. No visibility or draw-call optimisation.
  */
@@ -251,17 +266,51 @@ class PassSimple : public detail::PassBase {
   class Sub : public detail::PassBase {
     friend PassSimple;
 
+   private:
+    /* Parent pass draw commands. */
+    command::DrawCommandBuf &draw_commands_buf_;
+
    public:
-    /**
-     * Turn the pass into a string for inspection.
-     */
-    std::string serialize() const;
+    void draw(GPUBatch *batch,
+              uint instance_len = -1,
+              uint vertex_len = -1,
+              uint vertex_first = -1,
+              ResourceHandle handle = {0})
+    {
+      if (instance_len == 0 || vertex_len == 0) {
+        return;
+      }
+      BLI_assert(shader_);
+      create_command(Type::Draw).draw = {
+          batch,
+          &draw_commands_buf_,
+          draw_commands_buf_.append(instance_len, vertex_len, vertex_first, handle),
+          handle};
+    }
+
+    void draw(GPUBatch *batch, ResourceHandle handle)
+    {
+      draw(batch, -1, -1, -1, handle);
+    }
+
+    void draw_procedural(GPUPrimType primitive,
+                         uint instance_len,
+                         uint vertex_len,
+                         uint vertex_first = -1,
+                         ResourceHandle handle = {0})
+    {
+      draw(procedural_batch_get(primitive), instance_len, vertex_len, vertex_first, handle);
+    }
+
+    std::string serialize() const override final;
 
    private:
-    Sub(const char *name, GPUShader *shader) : PassBase(name)
+    Sub(const char *name, GPUShader *shader, command::DrawCommandBuf &draw_commands)
+        : PassBase(name), draw_commands_buf_(draw_commands)
     {
-      this->init();
-      this->shader_ = shader;
+      shader_ = shader;
+      headers_.clear();
+      commands_.clear();
     }
 
     void submit(command::RecordingState &state) const;
@@ -270,36 +319,74 @@ class PassSimple : public detail::PassBase {
  private:
   /** Sub-passes referenced by headers. */
   Vector<PassSimple::Sub> sub_passes_;
+  /** Draws are recorded as indirect draws for compatibility with the multi-draw pipeline. */
+  command::DrawCommandBuf draw_commands_buf_;
 
  public:
   PassSimple() = delete;
   PassSimple(const char *name) : PassBase(name){};
 
-  /**
-   * Create a sub-pass inside this pass. The sub-pass memory is auto managed.
-   */
-  PassSimple::Sub &sub(const char *name);
+  void init()
+  {
+    headers_.clear();
+    commands_.clear();
+    sub_passes_.clear();
+    draw_commands_buf_.clear();
+  }
+
+  PassSimple::Sub &sub(const char *name)
+  {
+    int64_t index = sub_passes_.append_and_get_index(
+        PassSimple::Sub(name, shader_, draw_commands_buf_));
+    headers_.append({command::Type::SubPass, static_cast<uint>(index)});
+    return sub_passes_[index];
+  }
 
   void draw(GPUBatch *batch,
             uint instance_len = -1,
             uint vertex_len = -1,
             uint vertex_first = -1,
-            ResourceHandle handle = {0});
+            ResourceHandle handle = {0})
+  {
+    if (instance_len == 0 || vertex_len == 0) {
+      return;
+    }
+    BLI_assert(shader_);
+    create_command(Type::Draw).draw = {
+        batch,
+        &draw_commands_buf_,
+        draw_commands_buf_.append(instance_len, vertex_len, vertex_first, handle),
+        handle};
+  }
 
-  /** Shorter version for the common case. */
   void draw(GPUBatch *batch, ResourceHandle handle)
   {
     draw(batch, -1, -1, -1, handle);
   }
 
+  void draw_procedural(GPUPrimType primitive,
+                       uint instance_len,
+                       uint vertex_len,
+                       uint vertex_first = -1,
+                       ResourceHandle handle = {0})
+  {
+    draw(procedural_batch_get(primitive), instance_len, vertex_len, vertex_first, handle);
+  }
+
   /**
    * Turn the pass into a string for inspection.
    */
-  std::string serialize() const;
+  std::string serialize() const override final;
 
  private:
   void submit(command::RecordingState &state) const;
-};
+};  // namespace blender::draw
+
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Main pass type
+ * \{ */
 
 /**
  * Main pass type.
@@ -316,26 +403,42 @@ class PassMain : public detail::PassBase {
     friend PassMain;
 
    public:
-    /**
-     * Turn the pass into a string for inspection.
-     */
-    std::string serialize() const;
+    std::string serialize() const override final;
 
    private:
-    command::MultiDrawBuffer &multi_draws_;
+    command::MultiDrawBuf &multi_draws_;
 
-    Sub(const char *name, GPUShader *shader, command::MultiDrawBuffer &multi_draws)
+    Sub(const char *name, GPUShader *shader, command::MultiDrawBuf &multi_draws)
         : PassBase(name), multi_draws_(multi_draws)
     {
-      this->init();
-      this->shader_ = shader;
+      headers_.clear();
+      commands_.clear();
+      shader_ = shader;
     }
 
     void draw(GPUBatch *batch,
               uint instance_len = -1,
               uint vertex_len = -1,
               uint vertex_first = -1,
-              ResourceHandle handle = {0});
+              ResourceHandle handle = {0})
+    {
+      /* TODO */
+      UNUSED_VARS(batch, instance_len, vertex_len, vertex_first, handle);
+    }
+
+    void draw(GPUBatch *batch, ResourceHandle handle)
+    {
+      draw(batch, -1, -1, -1, handle);
+    }
+
+    void draw_procedural(GPUPrimType primitive,
+                         uint instance_len,
+                         uint vertex_len,
+                         uint vertex_first = -1,
+                         ResourceHandle handle = {0})
+    {
+      draw(procedural_batch_get(primitive), instance_len, vertex_len, vertex_first, handle);
+    }
 
     void submit(command::RecordingState &state) const;
   };
@@ -344,33 +447,46 @@ class PassMain : public detail::PassBase {
   /** Sub-passes referenced by headers. */
   Vector<PassMain::Sub> sub_passes_;
   /** Multi draw indirect rendering for many draw calls efficient rendering. */
-  command::MultiDrawBuffer multi_draws_;
+  command::MultiDrawBuf multi_draws_;
 
  public:
   PassMain() = delete;
   PassMain(const char *name) : PassBase(name){};
 
-  /**
-   * Create a sub-pass inside this pass. The sub-pass memory is auto managed.
-   */
-  PassMain::Sub &sub(const char *name);
+  void init();
+
+  PassMain::Sub &sub(const char *name)
+  {
+    int64_t index = sub_passes_.append_and_get_index(PassMain::Sub(name, shader_, multi_draws_));
+    headers_.append({command::Type::SubPass, static_cast<uint>(index)});
+    return sub_passes_[index];
+  }
 
   void draw(GPUBatch *batch,
             uint instance_len = -1,
             uint vertex_len = -1,
             uint vertex_first = -1,
-            ResourceHandle handle = {0});
+            ResourceHandle handle = {0})
+  {
+    /* TODO */
+    UNUSED_VARS(batch, instance_len, vertex_len, vertex_first, handle);
+  }
 
-  /** Shorter version for the common case. */
   void draw(GPUBatch *batch, ResourceHandle handle)
   {
     draw(batch, -1, -1, -1, handle);
   }
 
-  /**
-   * Turn the pass into a string for inspection.
-   */
-  std::string serialize() const;
+  void draw_procedural(GPUPrimType primitive,
+                       uint instance_len,
+                       uint vertex_len,
+                       uint vertex_first = -1,
+                       ResourceHandle handle = {0})
+  {
+    draw(procedural_batch_get(primitive), instance_len, vertex_len, vertex_first, handle);
+  }
+
+  std::string serialize() const override final;
 
  private:
   void submit(command::RecordingState &state) const;
@@ -378,44 +494,12 @@ class PassMain : public detail::PassBase {
 
 /** \} */
 
-/* -------------------------------------------------------------------- */
-/** \name PassSimple Implementation
- * \{ */
-
-inline PassSimple::Sub &PassSimple::sub(const char *name)
-{
-  int64_t index = sub_passes_.append_and_get_index(PassSimple::Sub(name, shader_));
-  headers_.append({command::Type::SubPass, static_cast<uint>(index)});
-  return sub_passes_[index];
-}
-
-/** \} */
-
-/* -------------------------------------------------------------------- */
-/** \name PassMain Implementation
- * \{ */
-
-inline PassMain::Sub &PassMain::sub(const char *name)
-{
-  int64_t index = sub_passes_.append_and_get_index(PassMain::Sub(name, shader_, multi_draws_));
-  headers_.append({command::Type::SubPass, static_cast<uint>(index)});
-  return sub_passes_[index];
-}
-
-/** \} */
-
 namespace detail {
 
 /* -------------------------------------------------------------------- */
 /** \name PassBase Implementation
  * \{ */
 
-inline void PassBase::init()
-{
-  headers_.clear();
-  commands_.clear();
-}
-
 inline command::Undetermined &PassBase::create_command(command::Type type)
 {
   int64_t index = commands_.append_and_get_index({});
@@ -423,12 +507,6 @@ inline command::Undetermined &PassBase::create_command(command::Type type)
   return commands_[index];
 }
 
-/** \} */
-
-/* -------------------------------------------------------------------- */
-/** \name Draw Calls Implementation
- * \{ */
-
 inline void PassBase::clear(eGPUFrameBufferBits planes, float4 color, float depth, uint8_t stencil)
 {
   create_command(command::Type::Clear).clear = {(uint8_t)planes, stencil, depth, color};
@@ -452,15 +530,11 @@ inline GPUBatch *PassBase::procedural_batch_get(GPUPrimType primitive)
   }
 }
 
-inline void PassBase::draw(
-    GPUBatch *batch, uint instance_len, uint vertex_len, uint vertex_first, ResourceHandle handle)
-{
-  if (instance_len == 0 || vertex_len == 0) {
-    return;
-  }
-  BLI_assert(shader_);
-  create_command(Type::Draw).draw = {batch, instance_len, vertex_len, vertex_first, handle};
-}
+/** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Indirect drawcalls
+ * \{ */
 
 inline void PassBase::draw_indirect(GPUBatch *batch,
                                     StorageBuffer<DrawCommand> &indirect_buffer,
@@ -470,15 +544,6 @@ inline void PassBase::draw_indirect(GPUBatch *batch,
   create_command(Type::DrawIndirect).draw_indirect = {batch, &indirect_buffer, handle};
 }
 
-inline void PassBase::draw_procedural(GPUPrimType primitive,
-                                      uint instance_len,
-                                      uint vertex_len,
-                                      uint vertex_first,
-                                      ResourceHandle handle)
-{
-  draw(procedural_batch_get(primitive), instance_len, vertex_len, vertex_first, handle);
-}
-
 inline void PassBase::draw_procedural_indirect(GPUPrimType primitive,
                                                StorageBuffer<DrawCommand> &indirect_buffer,
                                                ResourceHandle handle)
diff --git a/source/blender/draw/intern/draw_shader_shared.h b/source/blender/draw/intern/draw_shader_shared.h
index e343098aaf3..27ef8e6efe8 100644
--- a/source/blender/draw/intern/draw_shader_shared.h
+++ b/source/blender/draw/intern/draw_shader_shared.h
@@ -14,7 +14,6 @@ typedef struct ObjectBounds ObjectBounds;
 typedef struct VolumeInfos VolumeInfos;
 typedef struct CurvesInfos CurvesInfos;
 typedef struct DrawCommand DrawCommand;
-typedef struct DrawCommandIndexed DrawCommandIndexed;
 typedef struct DispatchCommand DispatchCommand;
 typedef struct DRWDebugPrintBuffer DRWDebugPrintBuffer;
 typedef struct DRWDebugVert DRWDebugVert;
@@ -199,11 +198,13 @@ struct DrawCommand {
   /* NOTE: base_index is i_first for non-indexed draw-calls. */
 #define _instance_first_array base_index
   uint i_first; /* TODO(fclem): Rename to instance_first_indexed */
-  /* TODO(fclem): Pass other draw parameters here for shaders to read. */
-  uint _pad0;
+
+  /** Number of instances requested by the engine for this draw. */
   uint engine_instance_count;
   /** Access to object / component resources (matrices, object infos, object attributes). */
   uint resource_id;
+
+  uint _pad0;
 };
 BLI_STATIC_ASSERT_ALIGN(DrawCommand, 16)
 
diff --git a/source/blender/gpu/GPU_batch.h b/source/blender/gpu/GPU_batch.h
index 867d2b5e1bd..ba3ab3265a1 100644
--- a/source/blender/gpu/GPU_batch.h
+++ b/source/blender/gpu/GPU_batch.h
@@ -164,6 +164,8 @@ void GPU_batch_program_set_builtin_with_config(GPUBatch *batch,
 #define GPU_batch_texture_bind(batch, name, tex) \
   GPU_texture_bind(tex, GPU_shader_get_texture_binding((batch)->shader, name));
 
+void GPU_batch_draw_parameter_get(GPUBatch *batch, int *r_v_count, int *r_i_count);
+
 void GPU_batch_draw(GPUBatch *batch);
 void GPU_batch_draw_range(GPUBatch *batch, int v_first, int v_count);
 /**
diff --git a/source/blender/gpu/intern/gpu_batch.cc b/source/blender/gpu/intern/gpu_batch.cc
index 897f21e8b54..10482398a50 100644
--- a/source/blender/gpu/intern/gpu_batch.cc
+++ b/source/blender/gpu/intern/gpu_batch.cc
@@ -220,6 +220,26 @@ void GPU_batch_set_shader(GPUBatch *batch, GPUShader *shader)
 /** \name Drawing / Drawcall functions
  * \{ */
 
+void GPU_batch_draw_parameter_get(GPUBatch *gpu_batch, int *r_v_count, int *r_i_count)
+{
+  BLI_assert(Context::get()->shader != nullptr);
+  Batch *batch = static_cast<Batch *>(gpu_batch);
+
+  if (batch->elem) {
+    *r_v_count = batch->elem_()->index_len_get();
+  }
+  else {
+    *r_v_count = batch->verts_(0)->vertex_len;
+  }
+
+  int i_count = (batch->inst[0]) ? batch->inst_(0)->vertex_len : 1;
+  /* Meh. This is to be able to use different numbers of verts in instance VBO's. */
+  if (batch->inst[1] != nullptr) {
+    i_count = min_ii(i_count, batch->inst_(1)->vertex_len);
+  }
+  *r_i_count = i_count;
+}
+
 void GPU_batch_draw(GPUBatch *batch)
 {
   GPU_shader_bind(batch->shader);