Image Editor: Fix background drawing of empty tiles.

Empty (UDIM) tiles where drawn with a transparency checkerboard. They
should be rendered with a border background. The cause is that the image
engine would select a single area that contained all tiles and draw them
as being part of an image.

The fix is to separate the color and depth part of the image engine
shader and only draw the depths of tiles that are enabled.

13 files changed, 140 insertions, 106 deletions

diff --git a/source/blender/draw/CMakeLists.txt b/source/blender/draw/CMakeLists.txt
index f4e57d9521b..3c25252efe8 100644
--- a/source/blender/draw/CMakeLists.txt
+++ b/source/blender/draw/CMakeLists.txt
@@ -518,8 +518,10 @@ set(GLSL_SRC
   engines/overlay/shaders/wireframe_frag.glsl
   engines/overlay/shaders/xray_fade_frag.glsl
 
-  engines/image/shaders/image_engine_frag.glsl
-  engines/image/shaders/image_engine_vert.glsl
+  engines/image/shaders/image_engine_color_frag.glsl
+  engines/image/shaders/image_engine_color_vert.glsl
+  engines/image/shaders/image_engine_depth_frag.glsl
+  engines/image/shaders/image_engine_depth_vert.glsl
 )
 
 set(GLSL_C)
diff --git a/source/blender/draw/engines/image/image_drawing_mode.hh b/source/blender/draw/engines/image/image_drawing_mode.hh
index f501bc78b11..d5a4a317a22 100644
--- a/source/blender/draw/engines/image/image_drawing_mode.hh
+++ b/source/blender/draw/engines/image/image_drawing_mode.hh
@@ -26,6 +26,8 @@
 
 #include "IMB_imbuf_types.h"
 
+#include "BLI_math_vec_types.hh"
+
 #include "image_batches.hh"
 #include "image_private.hh"
 #include "image_wrappers.hh"
@@ -80,13 +82,19 @@ template<typename TextureMethod> class ScreenSpaceDrawingMode : public AbstractD
  private:
   DRWPass *create_image_pass() const
   {
-    /* Write depth is needed for background overlay rendering. Near depth is used for
-     * transparency checker and Far depth is used for indicating the image size. */
-    DRWState state = static_cast<DRWState>(DRW_STATE_WRITE_COLOR | DRW_STATE_WRITE_DEPTH |
-                                           DRW_STATE_DEPTH_ALWAYS | DRW_STATE_BLEND_ALPHA_PREMUL);
+    DRWState state = static_cast<DRWState>(DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_ALWAYS |
+                                           DRW_STATE_BLEND_ALPHA_PREMUL);
     return DRW_pass_create("Image", state);
   }
 
+  DRWPass *create_depth_pass() const
+  {
+    /* Depth is needed for background overlay rendering. Near depth is used for
+     * transparency checker and Far depth is used for indicating the image size. */
+    DRWState state = static_cast<DRWState>(DRW_STATE_WRITE_DEPTH | DRW_STATE_DEPTH_LESS_EQUAL);
+    return DRW_pass_create("Depth", state);
+  }
+
   void add_shgroups(const IMAGE_InstanceData *instance_data) const
   {
     const ShaderParameters &sh_params = instance_data->sh_params;
@@ -97,7 +105,6 @@ template<typename TextureMethod> class ScreenSpaceDrawingMode : public AbstractD
     DRW_shgroup_uniform_vec4_copy(shgrp, "shuffle", sh_params.shuffle);
     DRW_shgroup_uniform_int_copy(shgrp, "drawFlags", sh_params.flags);
     DRW_shgroup_uniform_bool_copy(shgrp, "imgPremultiplied", sh_params.use_premul_alpha);
-    DRW_shgroup_uniform_vec2_copy(shgrp, "maxUv", instance_data->max_uv);
     float image_mat[4][4];
     unit_m4(image_mat);
     for (int i = 0; i < SCREEN_SPACE_DRAWING_MODE_TEXTURE_LEN; i++) {
@@ -113,6 +120,37 @@ template<typename TextureMethod> class ScreenSpaceDrawingMode : public AbstractD
   }
 
   /**
+   * \brief add depth drawing calls.
+   *
+   * The depth is used to identify if the tile exist or transparent.
+   */
+  void add_depth_shgroups(IMAGE_InstanceData &instance_data,
+                          Image *image,
+                          ImageUser *UNUSED(image_user)) const
+  {
+    GPUShader *shader = IMAGE_shader_depth_get();
+    DRWShadingGroup *shgrp = DRW_shgroup_create(shader, instance_data.passes.depth_pass);
+    float image_mat[4][4];
+    unit_m4(image_mat);
+    for (int i = 0; i < SCREEN_SPACE_DRAWING_MODE_TEXTURE_LEN; i++) {
+      const TextureInfo &info = instance_data.texture_infos[i];
+      if (!info.visible) {
+        continue;
+      }
+
+      LISTBASE_FOREACH (ImageTile *, image_tile_ptr, &image->tiles) {
+        DRWShadingGroup *shsub = DRW_shgroup_create_sub(shgrp);
+        const ImageTileWrapper image_tile(image_tile_ptr);
+        const int tile_x = image_tile.get_tile_x_offset();
+        const int tile_y = image_tile.get_tile_y_offset();
+        float4 min_max_uv(tile_x, tile_y, tile_x + 1, tile_y + 1);
+        DRW_shgroup_uniform_vec4_copy(shsub, "min_max_uv", min_max_uv);
+        DRW_shgroup_call_obmat(shsub, info.batch, image_mat);
+      }
+    }
+  }
+
+  /**
    * \brief Update GPUTextures for drawing the image.
    *
    * GPUTextures that are marked dirty are rebuild. GPUTextures that aren't marked dirty are
@@ -367,6 +405,7 @@ template<typename TextureMethod> class ScreenSpaceDrawingMode : public AbstractD
   {
     IMAGE_InstanceData *instance_data = vedata->instance_data;
     instance_data->passes.image_pass = create_image_pass();
+    instance_data->passes.depth_pass = create_depth_pass();
   }
 
   void cache_image(IMAGE_Data *vedata, Image *image, ImageUser *iuser) const override
@@ -376,7 +415,6 @@ template<typename TextureMethod> class ScreenSpaceDrawingMode : public AbstractD
     TextureMethod method(instance_data);
 
     instance_data->partial_update.ensure_image(image);
-    instance_data->max_uv_update();
     instance_data->clear_dirty_flag();
 
     // Step: Find out which screen space textures are needed to draw on the screen. Remove the
@@ -391,6 +429,7 @@ template<typename TextureMethod> class ScreenSpaceDrawingMode : public AbstractD
 
     // Step: Add the GPU textures to the shgroup.
     instance_data->update_batches();
+    add_depth_shgroups(*instance_data, image, iuser);
     add_shgroups(instance_data);
   }
 
@@ -408,6 +447,7 @@ template<typename TextureMethod> class ScreenSpaceDrawingMode : public AbstractD
     GPU_framebuffer_clear_color_depth(dfbl->default_fb, clear_col, 1.0);
 
     DRW_view_set_active(instance_data->view);
+    DRW_draw_pass(instance_data->passes.depth_pass);
     DRW_draw_pass(instance_data->passes.image_pass);
     DRW_view_set_active(nullptr);
   }
diff --git a/source/blender/draw/engines/image/image_engine.cc b/source/blender/draw/engines/image/image_engine.cc
index 840d4840f1b..201733f41a7 100644
--- a/source/blender/draw/engines/image/image_engine.cc
+++ b/source/blender/draw/engines/image/image_engine.cc
@@ -145,7 +145,6 @@ class ImageEngine {
 
 static void IMAGE_engine_init(void *ved)
 {
-  IMAGE_shader_library_ensure();
   IMAGE_Data *vedata = (IMAGE_Data *)ved;
   if (vedata->instance_data == nullptr) {
     vedata->instance_data = MEM_new<IMAGE_InstanceData>(__func__);
diff --git a/source/blender/draw/engines/image/image_instance_data.hh b/source/blender/draw/engines/image/image_instance_data.hh
index 7fb0a6ca3eb..1a7a20b8b9a 100644
--- a/source/blender/draw/engines/image/image_instance_data.hh
+++ b/source/blender/draw/engines/image/image_instance_data.hh
@@ -57,29 +57,14 @@ struct IMAGE_InstanceData {
 
   struct {
     DRWPass *image_pass;
+    DRWPass *depth_pass;
   } passes;
 
   /** \brief Transform matrix to convert a normalized screen space coordinates to texture space. */
   float ss_to_texture[4][4];
   TextureInfo texture_infos[SCREEN_SPACE_DRAWING_MODE_TEXTURE_LEN];
 
-  /**
-   * \brief Maximum uv's that are on the border of the image.
-   *
-   * Larger UV coordinates would be drawn as a border. */
-  float max_uv[2];
-
  public:
-  void max_uv_update()
-  {
-    copy_v2_fl2(max_uv, 1.0f, 1.0);
-    LISTBASE_FOREACH (ImageTile *, image_tile_ptr, &image->tiles) {
-      ImageTileWrapper image_tile(image_tile_ptr);
-      max_uv[0] = max_ii(max_uv[0], image_tile.get_tile_x_offset() + 1);
-      max_uv[1] = max_ii(max_uv[1], image_tile.get_tile_y_offset() + 1);
-    }
-  }
-
   void clear_dirty_flag()
   {
     reset_dirty_flag(false);
diff --git a/source/blender/draw/engines/image/image_private.hh b/source/blender/draw/engines/image/image_private.hh
index 6d665db4c41..d8f8adb7e84 100644
--- a/source/blender/draw/engines/image/image_private.hh
+++ b/source/blender/draw/engines/image/image_private.hh
@@ -72,7 +72,7 @@ class AbstractDrawingMode {
 
 /* image_shader.c */
 GPUShader *IMAGE_shader_image_get();
-void IMAGE_shader_library_ensure();
+GPUShader *IMAGE_shader_depth_get();
 void IMAGE_shader_free();
 
 }  // namespace blender::draw::image_engine
diff --git a/source/blender/draw/engines/image/image_shader.cc b/source/blender/draw/engines/image/image_shader.cc
index e3cb60c5511..952843d7dd7 100644
--- a/source/blender/draw/engines/image/image_shader.cc
+++ b/source/blender/draw/engines/image/image_shader.cc
@@ -29,44 +29,33 @@
 #include "image_engine.h"
 #include "image_private.hh"
 
-extern "C" {
-extern char datatoc_common_colormanagement_lib_glsl[];
-extern char datatoc_common_globals_lib_glsl[];
-extern char datatoc_common_view_lib_glsl[];
-
-extern char datatoc_engine_image_frag_glsl[];
-extern char datatoc_engine_image_vert_glsl[];
-}
-
 namespace blender::draw::image_engine {
 
 struct IMAGE_Shaders {
   GPUShader *image_sh;
+  GPUShader *depth_sh;
 };
 
 static struct {
   IMAGE_Shaders shaders;
-  DRWShaderLibrary *lib;
 } e_data = {{nullptr}}; /* Engine data */
 
-void IMAGE_shader_library_ensure()
+GPUShader *IMAGE_shader_image_get()
 {
-  if (e_data.lib == nullptr) {
-    e_data.lib = DRW_shader_library_create();
-    /* NOTE: These need to be ordered by dependencies. */
-    DRW_SHADER_LIB_ADD(e_data.lib, common_colormanagement_lib);
-    DRW_SHADER_LIB_ADD(e_data.lib, common_globals_lib);
-    DRW_SHADER_LIB_ADD(e_data.lib, common_view_lib);
+  IMAGE_Shaders *sh_data = &e_data.shaders;
+  if (sh_data->image_sh == nullptr) {
+    sh_data->image_sh = GPU_shader_create_from_info_name("image_engine_color_shader");
   }
+  return sh_data->image_sh;
 }
 
-GPUShader *IMAGE_shader_image_get()
+GPUShader *IMAGE_shader_depth_get()
 {
   IMAGE_Shaders *sh_data = &e_data.shaders;
-  if (sh_data->image_sh == nullptr) {
-    sh_data->image_sh = GPU_shader_create_from_info_name("image_engine_shader");
+  if (sh_data->depth_sh == nullptr) {
+    sh_data->depth_sh = GPU_shader_create_from_info_name("image_engine_depth_shader");
   }
-  return sh_data->image_sh;
+  return sh_data->depth_sh;
 }
 
 void IMAGE_shader_free()
@@ -75,8 +64,6 @@ void IMAGE_shader_free()
   for (int i = 0; i < (sizeof(IMAGE_Shaders) / sizeof(GPUShader *)); i++) {
     DRW_SHADER_FREE_SAFE(sh_data_as_array[i]);
   }
-
-  DRW_SHADER_LIB_FREE_SAFE(e_data.lib);
 }
 
 }  // namespace blender::draw::image_engine
diff --git a/source/blender/draw/engines/image/shaders/image_engine_color_frag.glsl b/source/blender/draw/engines/image/shaders/image_engine_color_frag.glsl
new file mode 100644
index 00000000000..fbb624e54ba
--- /dev/null
+++ b/source/blender/draw/engines/image/shaders/image_engine_color_frag.glsl
@@ -0,0 +1,33 @@
+#pragma BLENDER_REQUIRE(common_colormanagement_lib.glsl)
+
+/* Keep in sync with image_engine.c */
+#define IMAGE_DRAW_FLAG_SHOW_ALPHA (1 << 0)
+#define IMAGE_DRAW_FLAG_APPLY_ALPHA (1 << 1)
+#define IMAGE_DRAW_FLAG_SHUFFLING (1 << 2)
+#define IMAGE_DRAW_FLAG_DEPTH (1 << 3)
+
+#define FAR_DISTANCE farNearDistances.x
+#define NEAR_DISTANCE farNearDistances.y
+
+void main()
+{
+  ivec2 uvs_clamped = ivec2(uv_screen);
+  vec4 tex_color = texelFetch(imageTexture, uvs_clamped, 0);
+
+  if ((drawFlags & IMAGE_DRAW_FLAG_APPLY_ALPHA) != 0) {
+    if (!imgPremultiplied) {
+      tex_color.rgb *= tex_color.a;
+    }
+  }
+  if ((drawFlags & IMAGE_DRAW_FLAG_DEPTH) != 0) {
+    tex_color = smoothstep(FAR_DISTANCE, NEAR_DISTANCE, tex_color);
+  }
+
+  if ((drawFlags & IMAGE_DRAW_FLAG_SHUFFLING) != 0) {
+    tex_color = vec4(dot(tex_color, shuffle));
+  }
+  if ((drawFlags & IMAGE_DRAW_FLAG_SHOW_ALPHA) == 0) {
+    tex_color.a = 1.0;
+  }
+  fragColor = tex_color;
+}
diff --git a/source/blender/draw/engines/image/shaders/image_engine_vert.glsl b/source/blender/draw/engines/image/shaders/image_engine_color_vert.glsl
index deefab655d2..fb72a132613 100644
--- a/source/blender/draw/engines/image/shaders/image_engine_vert.glsl
+++ b/source/blender/draw/engines/image/shaders/image_engine_color_vert.glsl
@@ -4,7 +4,6 @@ void main()
 {
   vec3 image_pos = vec3(pos, 0.0);
   uv_screen = image_pos.xy;
-  uv_image = uv;
 
   vec3 world_pos = point_object_to_world(image_pos);
   vec4 position = point_world_to_ndc(world_pos);
diff --git a/source/blender/draw/engines/image/shaders/image_engine_depth_frag.glsl b/source/blender/draw/engines/image/shaders/image_engine_depth_frag.glsl
new file mode 100644
index 00000000000..88610fb97fd
--- /dev/null
+++ b/source/blender/draw/engines/image/shaders/image_engine_depth_frag.glsl
@@ -0,0 +1,16 @@
+#pragma BLENDER_REQUIRE(common_colormanagement_lib.glsl)
+
+#define Z_DEPTH_BORDER 1.0
+#define Z_DEPTH_IMAGE 0.75
+
+bool is_border(vec2 uv)
+{
+  return (uv.x < min_max_uv.x || uv.y < min_max_uv.y || uv.x >= min_max_uv.z ||
+          uv.y >= min_max_uv.w);
+}
+
+void main()
+{
+  bool border = is_border(uv_image);
+  gl_FragDepth = border ? Z_DEPTH_BORDER : Z_DEPTH_IMAGE;
+}
diff --git a/source/blender/draw/engines/image/shaders/image_engine_depth_vert.glsl b/source/blender/draw/engines/image/shaders/image_engine_depth_vert.glsl
new file mode 100644
index 00000000000..3181a85ff55
--- /dev/null
+++ b/source/blender/draw/engines/image/shaders/image_engine_depth_vert.glsl
@@ -0,0 +1,11 @@
+#pragma BLENDER_REQUIRE(common_view_lib.glsl)
+
+void main()
+{
+  vec3 image_pos = vec3(pos, 0.0);
+  uv_image = uv;
+
+  vec3 world_pos = point_object_to_world(image_pos);
+  vec4 position = point_world_to_ndc(world_pos);
+  gl_Position = position;
+}
diff --git a/source/blender/draw/engines/image/shaders/image_engine_frag.glsl b/source/blender/draw/engines/image/shaders/image_engine_frag.glsl
deleted file mode 100644
index b0ac7af457f..00000000000
--- a/source/blender/draw/engines/image/shaders/image_engine_frag.glsl
+++ /dev/null
@@ -1,45 +0,0 @@
-#pragma BLENDER_REQUIRE(common_colormanagement_lib.glsl)
-
-/* Keep in sync with image_engine.c */
-#define IMAGE_DRAW_FLAG_SHOW_ALPHA (1 << 0)
-#define IMAGE_DRAW_FLAG_APPLY_ALPHA (1 << 1)
-#define IMAGE_DRAW_FLAG_SHUFFLING (1 << 2)
-#define IMAGE_DRAW_FLAG_DEPTH (1 << 3)
-
-#define Z_DEPTH_BORDER 1.0
-#define Z_DEPTH_IMAGE 0.75
-
-#define FAR_DISTANCE farNearDistances.x
-#define NEAR_DISTANCE farNearDistances.y
-
-bool is_border(vec2 uv)
-{
-  return (uv.x < 0.0 || uv.y < 0.0 || uv.x > maxUv.x || uv.y > maxUv.y);
-}
-
-void main()
-{
-  ivec2 uvs_clamped = ivec2(uv_screen);
-  vec4 tex_color = texelFetch(imageTexture, uvs_clamped, 0);
-
-  bool border = is_border(uv_image);
-  if (!border) {
-    if ((drawFlags & IMAGE_DRAW_FLAG_APPLY_ALPHA) != 0) {
-      if (!imgPremultiplied) {
-        tex_color.rgb *= tex_color.a;
-      }
-    }
-    if ((drawFlags & IMAGE_DRAW_FLAG_DEPTH) != 0) {
-      tex_color = smoothstep(FAR_DISTANCE, NEAR_DISTANCE, tex_color);
-    }
-
-    if ((drawFlags & IMAGE_DRAW_FLAG_SHUFFLING) != 0) {
-      tex_color = vec4(dot(tex_color, shuffle));
-    }
-    if ((drawFlags & IMAGE_DRAW_FLAG_SHOW_ALPHA) == 0) {
-      tex_color.a = 1.0;
-    }
-  }
-  fragColor = tex_color;
-  gl_FragDepth = border ? Z_DEPTH_BORDER : Z_DEPTH_IMAGE;
-}
diff --git a/source/blender/draw/engines/image/shaders/infos/engine_image_info.hh b/source/blender/draw/engines/image/shaders/infos/engine_image_info.hh
index 1dd20a9a95b..86a79d13e40 100644
--- a/source/blender/draw/engines/image/shaders/infos/engine_image_info.hh
+++ b/source/blender/draw/engines/image/shaders/infos/engine_image_info.hh
@@ -1,21 +1,29 @@
 #include "gpu_shader_create_info.hh"
 
-GPU_SHADER_INTERFACE_INFO(image_engine_iface, "")
-    .smooth(Type::VEC2, "uv_screen")
-    .smooth(Type::VEC2, "uv_image");
+GPU_SHADER_INTERFACE_INFO(image_engine_color_iface, "").smooth(Type::VEC2, "uv_screen");
 
-GPU_SHADER_CREATE_INFO(image_engine_shader)
+GPU_SHADER_CREATE_INFO(image_engine_color_shader)
     .vertex_in(0, Type::VEC2, "pos")
-    .vertex_in(1, Type::VEC2, "uv")
-    .vertex_out(image_engine_iface)
+    .vertex_out(image_engine_color_iface)
     .fragment_out(0, Type::VEC4, "fragColor")
     .push_constant(Type::VEC4, "shuffle")
-    .push_constant(Type::VEC2, "maxUv")
     .push_constant(Type::VEC2, "farNearDistances")
     .push_constant(Type::INT, "drawFlags")
     .push_constant(Type::BOOL, "imgPremultiplied")
     .sampler(0, ImageType::FLOAT_2D, "imageTexture")
-    .vertex_source("image_engine_vert.glsl")
-    .fragment_source("image_engine_frag.glsl")
+    .vertex_source("image_engine_color_vert.glsl")
+    .fragment_source("image_engine_color_frag.glsl")
+    .additional_info("draw_modelmat")
+    .do_static_compilation(true);
+
+GPU_SHADER_INTERFACE_INFO(image_engine_depth_iface, "").smooth(Type::VEC2, "uv_image");
+
+GPU_SHADER_CREATE_INFO(image_engine_depth_shader)
+    .vertex_in(0, Type::VEC2, "pos")
+    .vertex_in(1, Type::VEC2, "uv")
+    .vertex_out(image_engine_depth_iface)
+    .push_constant(Type::VEC4, "min_max_uv")
+    .vertex_source("image_engine_depth_vert.glsl")
+    .fragment_source("image_engine_depth_frag.glsl")
     .additional_info("draw_modelmat")
     .do_static_compilation(true);
diff --git a/source/blender/draw/tests/shaders_test.cc b/source/blender/draw/tests/shaders_test.cc
index b97298cbe95..80f87eea7ca 100644
--- a/source/blender/draw/tests/shaders_test.cc
+++ b/source/blender/draw/tests/shaders_test.cc
@@ -183,9 +183,8 @@ DRAW_TEST(gpencil_glsl_shaders)
 
 static void test_image_glsl_shaders()
 {
-  IMAGE_shader_library_ensure();
-
   EXPECT_NE(IMAGE_shader_image_get(), nullptr);
+  EXPECT_NE(IMAGE_shader_depth_get(), nullptr);
 
   IMAGE_shader_free();
 }