Tracking: Image from Plane Marker operators

There are two operators added, which are available via a special
content menu next to the plane track image selector:

- New Image from Plane Marker
- Update Image from Plane Marker

The former one creates an image from pixels which the active plane
track marker "sees" at the current frame and sets it as the plane
track's image.

The latter one instead of creating the new image data-block updates
the image in-place.

This allows to create unwarped texture from a billboard from footage.
The intent is to allow this image to be touched up and re-projected
back to the footage with an updated content.

Available from a plane track image context menu, as well as from the
Track menu.

{F13243219}

The demo of the feature from Sebastian Koenig: https://www.youtube.com/watch?v=PDphO-w2SsA

Differential Revision: https://developer.blender.org/D15312

2 files changed, 95 insertions, 0 deletions

diff --git a/source/blender/blenkernel/BKE_tracking.h b/source/blender/blenkernel/BKE_tracking.h
index 3f6d32e2f70..23b1f7c09bb 100644
--- a/source/blender/blenkernel/BKE_tracking.h
+++ b/source/blender/blenkernel/BKE_tracking.h
@@ -560,6 +560,11 @@ struct ImBuf *BKE_tracking_get_search_imbuf(struct ImBuf *ibuf,
                                             bool anchored,
                                             bool disable_channels);
 
+/* Create a new image buffer which consists of pixels which the plane marker "sees".
+ * The function will choose best image resolution based on the plane marker size. */
+struct ImBuf *BKE_tracking_get_plane_imbuf(const struct ImBuf *frame_ibuf,
+                                           const struct MovieTrackingPlaneMarker *plane_marker);
+
 /**
  * Zap channels from the imbuf that are disabled by the user. this can lead to
  * better tracks sometimes. however, instead of simply zeroing the channels
diff --git a/source/blender/blenkernel/intern/tracking.c b/source/blender/blenkernel/intern/tracking.c
index 8b462cba7ed..6abc40e89bc 100644
--- a/source/blender/blenkernel/intern/tracking.c
+++ b/source/blender/blenkernel/intern/tracking.c
@@ -2798,6 +2798,96 @@ ImBuf *BKE_tracking_get_search_imbuf(ImBuf *ibuf,
   return searchibuf;
 }
 
+BLI_INLINE int plane_marker_size_len_in_pixels(const float a[2],
+                                               const float b[2],
+                                               const int frame_width,
+                                               const int frame_height)
+{
+  const float a_px[2] = {a[0] * frame_width, a[1] * frame_height};
+  const float b_px[2] = {b[0] * frame_width, b[1] * frame_height};
+
+  return ceilf(len_v2v2(a_px, b_px));
+}
+
+ImBuf *BKE_tracking_get_plane_imbuf(const ImBuf *frame_ibuf,
+                                    const MovieTrackingPlaneMarker *plane_marker)
+{
+  /* Alias for corners, allowing shorter access to coordinates. */
+  const float(*corners)[2] = plane_marker->corners;
+
+  /* Dimensions of the frame image in pixels. */
+  const int frame_width = frame_ibuf->x;
+  const int frame_height = frame_ibuf->y;
+
+  /* Lengths of left and right edges of the plane marker, in pixels. */
+  const int left_side_len_px = plane_marker_size_len_in_pixels(
+      corners[0], corners[3], frame_width, frame_height);
+  const int right_side_len_px = plane_marker_size_len_in_pixels(
+      corners[1], corners[2], frame_width, frame_height);
+
+  /* Lengths of top and bottom edges of the plane marker, in pixels. */
+  const int top_side_len_px = plane_marker_size_len_in_pixels(
+      corners[3], corners[2], frame_width, frame_height);
+  const int bottom_side_len_px = plane_marker_size_len_in_pixels(
+      corners[0], corners[1], frame_width, frame_height);
+
+  /* Choose the number of samples as a maximum of the corresponding sides in pixels. */
+  const int num_samples_x = max_ii(top_side_len_px, bottom_side_len_px);
+  const int num_samples_y = max_ii(left_side_len_px, right_side_len_px);
+
+  /* Create new result image with the same type of content as the original. */
+  ImBuf *plane_ibuf = IMB_allocImBuf(
+      num_samples_x, num_samples_y, 32, frame_ibuf->rect_float ? IB_rectfloat : IB_rect);
+
+  /* Calculate corner coordinates in pixel space, as spearate X/Y arrays. */
+  const double src_pixel_x[4] = {corners[0][0] * frame_width,
+                                 corners[1][0] * frame_width,
+                                 corners[2][0] * frame_width,
+                                 corners[3][0] * frame_width};
+  const double src_pixel_y[4] = {corners[0][1] * frame_height,
+                                 corners[1][1] * frame_height,
+                                 corners[2][1] * frame_height,
+                                 corners[3][1] * frame_height};
+
+  /* Warped Position is unused but is expected to be provided by the API. */
+  double warped_position_x, warped_position_y;
+
+  /* Actual sampling. */
+  if (frame_ibuf->rect_float != NULL) {
+    libmv_samplePlanarPatchFloat(frame_ibuf->rect_float,
+                                 frame_ibuf->x,
+                                 frame_ibuf->y,
+                                 4,
+                                 src_pixel_x,
+                                 src_pixel_y,
+                                 num_samples_x,
+                                 num_samples_y,
+                                 NULL,
+                                 plane_ibuf->rect_float,
+                                 &warped_position_x,
+                                 &warped_position_y);
+  }
+  else {
+    libmv_samplePlanarPatchByte((unsigned char *)frame_ibuf->rect,
+                                frame_ibuf->x,
+                                frame_ibuf->y,
+                                4,
+                                src_pixel_x,
+                                src_pixel_y,
+                                num_samples_x,
+                                num_samples_y,
+                                NULL,
+                                (unsigned char *)plane_ibuf->rect,
+                                &warped_position_x,
+                                &warped_position_y);
+  }
+
+  plane_ibuf->rect_colorspace = frame_ibuf->rect_colorspace;
+  plane_ibuf->float_colorspace = frame_ibuf->float_colorspace;
+
+  return plane_ibuf;
+}
+
 void BKE_tracking_disable_channels(
     ImBuf *ibuf, bool disable_red, bool disable_green, bool disable_blue, bool grayscale)
 {