Fix T62876: Camera Background Images

Migrate old legacy code to the draw mamager/object mode. The old legacy
version did not work with wireframe. By migrating the code
to modern draw manager code we have mode control on the drawing process.

Still background images do not work with OIT, the cause seems to be that the transparent pixels are treated as background pixels.
Also There are some artifacts when working with Holdouts and DoF, this
is because the draw engines do not pass the correct alpha values.

Reviewers: fclem, brecht

Differential Revision: https://developer.blender.org/D4638

1 files changed, 328 insertions, 2 deletions

diff --git a/source/blender/draw/modes/object_mode.c b/source/blender/draw/modes/object_mode.c
index e8c600ee545..1c60fc09057 100644
--- a/source/blender/draw/modes/object_mode.c
+++ b/source/blender/draw/modes/object_mode.c
@@ -38,6 +38,7 @@
 #include "DNA_rigidbody_types.h"
 #include "DNA_smoke_types.h"
 #include "DNA_view3d_types.h"
+#include "DNA_screen_types.h"
 #include "DNA_world_types.h"
 
 #include "BKE_anim.h"
@@ -45,6 +46,7 @@
 #include "BKE_constraint.h"
 #include "BKE_curve.h"
 #include "BKE_editmesh.h"
+#include "BKE_image.h"
 #include "BKE_mball.h"
 #include "BKE_mesh.h"
 #include "BKE_modifier.h"
@@ -55,6 +57,8 @@
 
 #include "BLI_ghash.h"
 
+#include "IMB_imbuf_types.h"
+
 #include "ED_view3d.h"
 
 #include "GPU_batch.h"
@@ -80,6 +84,8 @@ extern char datatoc_object_outline_detect_frag_glsl[];
 extern char datatoc_object_outline_expand_frag_glsl[];
 extern char datatoc_object_grid_frag_glsl[];
 extern char datatoc_object_grid_vert_glsl[];
+extern char datatoc_object_camera_image_frag_glsl[];
+extern char datatoc_object_camera_image_vert_glsl[];
 extern char datatoc_object_empty_image_frag_glsl[];
 extern char datatoc_object_empty_image_vert_glsl[];
 extern char datatoc_object_lightprobe_grid_vert_glsl[];
@@ -87,6 +93,7 @@ extern char datatoc_object_loose_points_frag_glsl[];
 extern char datatoc_object_particle_prim_vert_glsl[];
 extern char datatoc_object_particle_dot_vert_glsl[];
 extern char datatoc_object_particle_dot_frag_glsl[];
+extern char datatoc_common_colormanagement_lib_glsl[];
 extern char datatoc_common_globals_lib_glsl[];
 extern char datatoc_common_view_lib_glsl[];
 extern char datatoc_common_fxaa_lib_glsl[];
@@ -115,6 +122,8 @@ typedef struct OBJECT_PassList {
   struct DRWPass *bone_axes[2];
   struct DRWPass *particle;
   struct DRWPass *lightprobes;
+  struct DRWPass *camera_images_back;
+  struct DRWPass *camera_images_front;
 } OBJECT_PassList;
 
 typedef struct OBJECT_FramebufferList {
@@ -148,6 +157,8 @@ typedef struct OBJECT_Shaders {
   GPUShader *outline_fade_large;
 
   /* regular shaders */
+  GPUShader *object_camera_image;
+  GPUShader *object_camera_image_cm;
   GPUShader *object_empty_image;
   GPUShader *object_empty_image_wire;
   GPUShader *grid;
@@ -336,6 +347,7 @@ static struct {
   struct GPUTexture *outlines_blur_tx;
 
   ListBase smoke_domains;
+  ListBase movie_clips;
 } e_data = {NULL}; /* Engine data */
 
 enum {
@@ -460,7 +472,9 @@ static void OBJECT_engine_init(void *vedata)
                                    datatoc_common_view_lib_glsl,
                                    datatoc_object_empty_image_vert_glsl,
                                    NULL},
-          .frag = (const char *[]){datatoc_object_empty_image_frag_glsl, NULL},
+          .frag = (const char *[]){datatoc_common_colormanagement_lib_glsl,
+                                   datatoc_object_empty_image_frag_glsl,
+                                   NULL},
           .defs = (const char *[]){sh_cfg_data->def, empty_image_defs, NULL},
       });
       sh_data->object_empty_image_wire = GPU_shader_create_from_arrays({
@@ -471,6 +485,21 @@ static void OBJECT_engine_init(void *vedata)
           .frag = (const char *[]){datatoc_object_empty_image_frag_glsl, NULL},
           .defs = (const char *[]){sh_cfg_data->def, "#define USE_WIRE\n", empty_image_defs, NULL},
       });
+
+      sh_data->object_camera_image_cm = GPU_shader_create_from_arrays({
+          .vert = (const char *[]){sh_cfg_data->lib, datatoc_object_camera_image_vert_glsl, NULL},
+          .frag = (const char *[]){datatoc_common_colormanagement_lib_glsl,
+                                   datatoc_object_camera_image_frag_glsl,
+                                   NULL},
+          .defs =
+              (const char *[]){sh_cfg_data->def, "#define DRW_STATE_DO_COLOR_MANAGEMENT\n", NULL},
+      });
+      sh_data->object_camera_image = GPU_shader_create_from_arrays({
+          .vert = (const char *[]){sh_cfg_data->lib, datatoc_object_camera_image_vert_glsl, NULL},
+          .frag = (const char *[]){datatoc_common_colormanagement_lib_glsl,
+                                   datatoc_object_camera_image_frag_glsl,
+                                   NULL},
+      });
     }
 
     /* Grid */
@@ -1035,6 +1064,291 @@ static void DRW_shgroup_empty_image(OBJECT_Shaders *sh_data,
   }
 }
 
+/* Draw Camera Background Images */
+typedef struct CameraEngineData {
+  DrawData dd;
+  ListBase bg_data;
+} CameraEngineData;
+typedef struct CameraEngineBGData {
+  float transform_mat[4][4];
+} CameraEngineBGData;
+
+static void camera_engine_data_free(DrawData *dd)
+{
+  CameraEngineData *data = (CameraEngineData *)dd;
+  for (LinkData *link = data->bg_data.first; link; link = link->next) {
+    CameraEngineBGData *bg_data = (CameraEngineBGData *)link->data;
+    MEM_freeN(bg_data);
+  }
+  BLI_freelistN(&data->bg_data);
+}
+
+static void camera_background_images_stereo_setup(Scene *scene,
+                                                  View3D *v3d,
+                                                  Image *ima,
+                                                  ImageUser *iuser)
+{
+  if (BKE_image_is_stereo(ima)) {
+    iuser->flag |= IMA_SHOW_STEREO;
+
+    if ((scene->r.scemode & R_MULTIVIEW) == 0) {
+      iuser->multiview_eye = STEREO_LEFT_ID;
+    }
+    else if (v3d->stereo3d_camera != STEREO_3D_ID) {
+      /* show only left or right camera */
+      iuser->multiview_eye = v3d->stereo3d_camera;
+    }
+
+    BKE_image_multiview_index(ima, iuser);
+  }
+  else {
+    iuser->flag &= ~IMA_SHOW_STEREO;
+  }
+}
+
+static void DRW_shgroup_camera_background_images(OBJECT_Shaders *sh_data,
+                                                 OBJECT_PassList *psl,
+                                                 Object *ob,
+                                                 RegionView3D *rv3d)
+{
+  if (!BKE_object_empty_image_frame_is_visible_in_view3d(ob, rv3d)) {
+    return;
+  }
+
+  const DRWContextState *draw_ctx = DRW_context_state_get();
+  struct ARegion *ar = draw_ctx->ar;
+  View3D *v3d = draw_ctx->v3d;
+  Scene *scene = draw_ctx->scene;
+  Depsgraph *depsgraph = draw_ctx->depsgraph;
+  Camera *cam = ob->data;
+  const Object *camera_object = DEG_get_evaluated_object(depsgraph, v3d->camera);
+  const bool is_active = (ob == camera_object);
+  const bool look_through = (is_active && (rv3d->persp == RV3D_CAMOB));
+
+  if (look_through && (cam->flag & CAM_SHOW_BG_IMAGE)) {
+    GPUBatch *batch = DRW_cache_image_plane_get();
+
+    /* load camera engine data */
+    CameraEngineData *camera_engine_data = (CameraEngineData *)DRW_drawdata_ensure(
+        &ob->id,
+        &draw_engine_object_type,
+        sizeof(CameraEngineData),
+        NULL,
+        camera_engine_data_free);
+    LinkData *list_node = camera_engine_data->bg_data.first;
+
+    for (CameraBGImage *bgpic = cam->bg_images.first; bgpic; bgpic = bgpic->next) {
+      if ((bgpic->flag & CAM_BGIMG_FLAG_DISABLED)) {
+        continue;
+      }
+
+      /* retrieve the image we want to show, continue to next when no image could be found */
+      ImBuf *ibuf = NULL;
+      GPUTexture *tex = NULL;
+      float image_aspect_x, image_aspect_y;
+      float image_aspect = 1.0;
+      int image_width, image_height;
+      bool premultiplied = false;
+
+      if (bgpic->source == CAM_BGIMG_SOURCE_IMAGE) {
+        Image *image = bgpic->ima;
+        if (image == NULL) {
+          continue;
+        }
+        premultiplied = (image->alpha_mode == IMA_ALPHA_PREMUL);
+        ImageUser *iuser = &bgpic->iuser;
+        BKE_image_user_frame_calc(image, iuser, (int)DEG_get_ctime(depsgraph));
+        if (image->source == IMA_SRC_SEQUENCE && !(iuser->flag & IMA_USER_FRAME_IN_RANGE)) {
+          /* frame is out of range, dont show */
+          continue;
+        }
+        else {
+          camera_background_images_stereo_setup(scene, v3d, image, iuser);
+        }
+        tex = GPU_texture_from_blender(image, iuser, GL_TEXTURE_2D);
+        if (tex == NULL) {
+          continue;
+        }
+        ibuf = BKE_image_acquire_ibuf(image, iuser, NULL);
+        if (ibuf == NULL) {
+          continue;
+        }
+
+        image_aspect_x = bgpic->ima->aspx;
+        image_aspect_y = bgpic->ima->aspy;
+
+        image_width = ibuf->x;
+        image_height = ibuf->y;
+        BKE_image_release_ibuf(image, ibuf, NULL);
+        image_aspect = (image_width * image_aspect_x) / (image_height * image_aspect_y);
+      }
+      else if (bgpic->source == CAM_BGIMG_SOURCE_MOVIE) {
+        MovieClip *clip = NULL;
+        if (bgpic->flag & CAM_BGIMG_FLAG_CAMERACLIP) {
+          if (scene->camera) {
+            clip = BKE_object_movieclip_get(scene, scene->camera, true);
+          }
+        }
+        else {
+          clip = bgpic->clip;
+        }
+
+        if (clip == NULL) {
+          continue;
+        }
+
+        image_aspect_x = clip->aspx;
+        image_aspect_y = clip->aspy;
+
+        BKE_movieclip_user_set_frame(&bgpic->cuser, (int)DEG_get_ctime(depsgraph));
+        tex = GPU_texture_from_movieclip(clip, &bgpic->cuser, GL_TEXTURE_2D);
+        if (tex == NULL) {
+          continue;
+        }
+        BLI_addtail(&e_data.movie_clips, BLI_genericNodeN(clip));
+        BKE_movieclip_get_size(clip, &bgpic->cuser, &image_width, &image_height);
+        image_aspect = (image_width * image_aspect_x) / (image_height * image_aspect_y);
+      }
+
+      /* ensure link_data is allocated to store matrice */
+      CameraEngineBGData *bg_data;
+      if (list_node != NULL) {
+        bg_data = (CameraEngineBGData *)list_node->data;
+        list_node = list_node->next;
+      }
+      else {
+        bg_data = MEM_mallocN(sizeof(CameraEngineBGData), __func__);
+        BLI_addtail(&camera_engine_data->bg_data, BLI_genericNodeN(bg_data));
+      }
+
+      /* calculate the transformation matric for the current bg image */
+      float uv2img_space[4][4];
+      float img2cam_space[4][4];
+      float rot_m4[4][4];
+      float scale_m4[4][4];
+      float translate_m4[4][4];
+      float win_m4_scale[4][4];
+      float win_m4_translate[4][4];
+
+      unit_m4(uv2img_space);
+      unit_m4(img2cam_space);
+      unit_m4(win_m4_scale);
+      unit_m4(win_m4_translate);
+      unit_m4(scale_m4);
+      axis_angle_to_mat4_single(rot_m4, 'Z', bgpic->rotation);
+      unit_m4(translate_m4);
+
+      const float *size = DRW_viewport_size_get();
+      float camera_aspect_x = 1.0;
+      float camera_aspect_y = 1.0;
+      float camera_offset_x = 0.0;
+      float camera_offset_y = 0.0;
+      float camera_aspect = 1.0;
+      float camera_width = size[0];
+      float camera_height = size[1];
+
+      if (!DRW_state_is_image_render()) {
+        rctf render_border;
+        ED_view3d_calc_camera_border(scene, depsgraph, ar, v3d, rv3d, &render_border, true);
+        camera_width = render_border.xmax - render_border.xmin;
+        camera_height = render_border.ymax - render_border.ymin;
+        camera_aspect = camera_width / camera_height;
+        const float camera_aspect_center_x = (render_border.xmax + render_border.xmin) / 2.0;
+        const float camera_aspect_center_y = (render_border.ymax + render_border.ymin) / 2.0;
+
+        camera_aspect_x = camera_width / size[0];
+        camera_aspect_y = camera_height / size[1];
+        win_m4_scale[0][0] = camera_aspect_x;
+        win_m4_scale[1][1] = camera_aspect_y;
+
+        camera_offset_x = (camera_aspect_center_x - (ar->winx / 2.0)) /
+                          (0.5 * camera_width / camera_aspect_x);
+        camera_offset_y = (camera_aspect_center_y - (ar->winy / 2.0)) /
+                          (0.5 * camera_height / camera_aspect_y);
+        win_m4_translate[3][0] = camera_offset_x;
+        win_m4_translate[3][1] = camera_offset_y;
+      }
+
+      /* Convert from uv space to image space -0.5..-.5 */
+      uv2img_space[0][0] = image_width;
+      uv2img_space[1][1] = image_height;
+
+      img2cam_space[0][0] = (1.0 / image_width);
+      img2cam_space[1][1] = (1.0 / image_height);
+
+      /* Update scaling based on image and camera framing */
+      float scale_x = bgpic->scale;
+      float scale_y = bgpic->scale;
+
+      if (bgpic->flag & CAM_BGIMG_FLAG_CAMERA_ASPECT) {
+        float fit_scale = image_aspect / camera_aspect;
+        if (bgpic->flag & CAM_BGIMG_FLAG_CAMERA_CROP) {
+          if (image_aspect > camera_aspect) {
+            scale_x *= fit_scale;
+          }
+          else {
+            scale_y /= fit_scale;
+          }
+        }
+        else {
+          if (image_aspect > camera_aspect) {
+            scale_y /= fit_scale;
+          }
+          else {
+            scale_x *= fit_scale;
+          }
+        }
+      }
+
+      // scale image to match the desired aspect ratio
+      scale_m4[0][0] = scale_x;
+      scale_m4[1][1] = scale_y;
+
+      // translate
+      translate_m4[3][0] = bgpic->offset[0];
+      translate_m4[3][1] = bgpic->offset[1];
+
+      mul_m4_series(bg_data->transform_mat,
+                    win_m4_translate,
+                    win_m4_scale,
+                    translate_m4,
+                    img2cam_space,
+                    scale_m4,
+                    rot_m4,
+                    uv2img_space);
+
+      DRWPass *pass = (bgpic->flag & CAM_BGIMG_FLAG_FOREGROUND) ? psl->camera_images_front :
+                                                                  psl->camera_images_back;
+      GPUShader *shader = DRW_state_do_color_management() ? sh_data->object_camera_image_cm :
+                                                            sh_data->object_camera_image;
+      DRWShadingGroup *grp = DRW_shgroup_create(shader, pass);
+
+      DRW_shgroup_uniform_float_copy(
+          grp, "depth", (bgpic->flag & CAM_BGIMG_FLAG_FOREGROUND) ? 0.000001 : 0.999999);
+      DRW_shgroup_uniform_float_copy(grp, "alpha", bgpic->alpha);
+      DRW_shgroup_uniform_texture(grp, "image", tex);
+      DRW_shgroup_uniform_bool_copy(grp, "imagePremultiplied", premultiplied);
+
+      DRW_shgroup_uniform_float_copy(
+          grp, "flipX", (bgpic->flag & CAM_BGIMG_FLAG_FLIP_X) ? -1.0 : 1.0);
+      DRW_shgroup_uniform_float_copy(
+          grp, "flipY", (bgpic->flag & CAM_BGIMG_FLAG_FLIP_Y) ? -1.0 : 1.0);
+      DRW_shgroup_uniform_mat4(grp, "TransformMat", bg_data->transform_mat);
+
+      DRW_shgroup_call(grp, batch, NULL);
+    }
+  }
+}
+
+static void camera_background_images_free_textures(void)
+{
+  for (LinkData *link = e_data.movie_clips.first; link; link = link->next) {
+    MovieClip *clip = (MovieClip *)link->data;
+    GPU_free_texture_movieclip(clip);
+  }
+  BLI_freelistN(&e_data.movie_clips);
+}
+
 static void OBJECT_cache_init(void *vedata)
 {
   const GlobalsUboStorage *gb = &G_draw.block;
@@ -1206,6 +1520,13 @@ static void OBJECT_cache_init(void *vedata)
     DRW_shgroup_call(grp, geom, NULL);
   }
 
+  /* Camera background images */
+  {
+    DRWState state = DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_LESS_EQUAL | DRW_STATE_BLEND_ALPHA;
+    psl->camera_images_back = DRW_pass_create("Camera Images Back", state);
+    psl->camera_images_front = DRW_pass_create("Camera Images Front", state);
+  }
+
   for (int i = 0; i < 2; ++i) {
     OBJECT_ShadingGroupList *sgl = (i == 1) ? &stl->g_data->sgl_ghost : &stl->g_data->sgl;
 
@@ -3266,6 +3587,7 @@ static void OBJECT_cache_populate(void *vedata, Object *ob)
           break;
         }
         DRW_shgroup_camera(sgl, ob, view_layer);
+        DRW_shgroup_camera_background_images(sh_data, psl, ob, rv3d);
         break;
       case OB_EMPTY:
         if (hide_object_extra) {
@@ -3449,6 +3771,8 @@ static void OBJECT_draw_scene(void *vedata)
 
   float clearcol[4] = {0.0f, 0.0f, 0.0f, 0.0f};
 
+  DRW_draw_pass(psl->camera_images_back);
+
   /* Don't draw Transparent passes in MSAA buffer. */
   //  DRW_draw_pass(psl->bone_envelope);  /* Never drawn in Object mode currently. */
   DRW_draw_pass(stl->g_data->sgl.transp_shapes);
@@ -3518,7 +3842,6 @@ static void OBJECT_draw_scene(void *vedata)
       DRW_draw_pass(psl->outlines_resolve);
     }
   }
-
   volumes_free_smoke_textures();
   batch_camera_path_free(&stl->g_data->sgl.camera_path);
 
@@ -3567,6 +3890,9 @@ static void OBJECT_draw_scene(void *vedata)
 
   batch_camera_path_free(&stl->g_data->sgl_ghost.camera_path);
 
+  DRW_draw_pass(psl->camera_images_front);
+  camera_background_images_free_textures();
+
   DRW_draw_pass(psl->ob_center);
 }