Image wrapping for plane track in clip editor

Summary:
Now it's possible to assign an image to plane tracks
in clip editor. This image is only used for display
in clip editor and this image is being warped into
the plane track rectangle.

Main purpose of this is to get early feedback about
how good image warping matches the footage, before
clip goes to the compositor.

Pretty much straightforward change: just compute
homography from undeformed normalized frame corner
coordinates (unity square) to plane marker corners
and apply this matrix to opengl stack.

Still could improve behavior when perspective
plane transform is degenerate, but that's not so
much critical for now i'd say.

Reviewers: brecht, campbellbarton

Reviewed By: brecht

CC: sebastian_k

Differential Revision: http://developer.blender.org/D57

6 files changed, 212 insertions, 44 deletions

diff --git a/source/blender/blenkernel/intern/tracking.c b/source/blender/blenkernel/intern/tracking.c
index d519b93f963..355922d157a 100644
--- a/source/blender/blenkernel/intern/tracking.c
+++ b/source/blender/blenkernel/intern/tracking.c
@@ -1344,6 +1344,8 @@ MovieTrackingPlaneTrack *BKE_tracking_plane_track_add(MovieTracking *tracking, L
 	/* Use some default name. */
 	strcpy(plane_track->name, "Plane Track");
 
+	plane_track->image_opacity = 1.0f;
+
 	/* Use selected tracks from given list as a plane. */
 	plane_track->point_tracks = MEM_mallocN(sizeof(MovieTrackingTrack *) * num_selected_tracks, "new plane tracks array");
 	for (track = tracks->first, track_index = 0; track; track = track->next) {
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index d0eb0ed0be0..cb1d2a0b98f 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -6777,6 +6777,15 @@ static void lib_link_movieTracks(FileData *fd, MovieClip *clip, ListBase *tracks
 	}
 }
 
+static void lib_link_moviePlaneTracks(FileData *fd, MovieClip *clip, ListBase *tracksbase)
+{
+	MovieTrackingPlaneTrack *plane_track;
+
+	for (plane_track = tracksbase->first; plane_track; plane_track = plane_track->next) {
+		plane_track->image = newlibadr_us(fd, clip->id.lib, plane_track->image);
+	}
+}
+
 static void lib_link_movieclip(FileData *fd, Main *main)
 {
 	MovieClip *clip;
@@ -6792,6 +6801,7 @@ static void lib_link_movieclip(FileData *fd, Main *main)
 			clip->gpd = newlibadr_us(fd, clip->id.lib, clip->gpd);
 			
 			lib_link_movieTracks(fd, clip, &tracking->tracks);
+			lib_link_moviePlaneTracks(fd, clip, &tracking->plane_tracks);
 
 			for (object = tracking->objects.first; object; object = object->next) {
 				lib_link_movieTracks(fd, clip, &object->tracks);
diff --git a/source/blender/blenloader/intern/versioning_260.c b/source/blender/blenloader/intern/versioning_260.c
index 45e764c1e73..14eb8ff4ea6 100644
--- a/source/blender/blenloader/intern/versioning_260.c
+++ b/source/blender/blenloader/intern/versioning_260.c
@@ -2529,4 +2529,19 @@ void blo_do_versions_260(FileData *fd, Library *UNUSED(lib), Main *main)
 
 		}
 	}
+
+	{
+		if (!DNA_struct_elem_find(fd->filesdna, "MovieTrackingPlaneTrack", "float", "image_opacity")) {
+			MovieClip *clip;
+			for (clip = main->movieclip.first; clip; clip = clip->id.next) {
+				MovieTrackingPlaneTrack *plane_track;
+				for (plane_track = clip->tracking.plane_tracks.first;
+				     plane_track;
+				     plane_track = plane_track->next)
+				{
+					plane_track->image_opacity = 1.0f;
+				}
+			}
+		}
+	}
 }
diff --git a/source/blender/editors/space_clip/clip_draw.c b/source/blender/editors/space_clip/clip_draw.c
index 3c9eb1fe79e..38e7f87f478 100644
--- a/source/blender/editors/space_clip/clip_draw.c
+++ b/source/blender/editors/space_clip/clip_draw.c
@@ -48,6 +48,7 @@
 #include "BLI_math_base.h"
 
 #include "BKE_context.h"
+#include "BKE_image.h"
 #include "BKE_movieclip.h"
 #include "BKE_tracking.h"
 #include "BKE_mask.h"
@@ -1038,12 +1039,127 @@ static void getArrowEndPoint(const int width, const int height, const float zoom
 	add_v2_v2v2(end_point, start_corner, direction);
 }
 
-static void draw_plane_marker_ex(SpaceClip *sc, MovieTrackingPlaneTrack *plane_track,
+static void homogeneous_2d_to_gl_matrix(/*const*/ float matrix[3][3],
+                                        float gl_matrix[4][4])
+{
+	gl_matrix[0][0] = matrix[0][0];
+	gl_matrix[0][1] = matrix[0][1];
+	gl_matrix[0][2] = 0.0f;
+	gl_matrix[0][3] = matrix[0][2];
+
+	gl_matrix[1][0] = matrix[1][0];
+	gl_matrix[1][1] = matrix[1][1];
+	gl_matrix[1][2] = 0.0f;
+	gl_matrix[1][3] = matrix[1][2];
+
+	gl_matrix[2][0] = 0.0f;
+	gl_matrix[2][1] = 0.0f;
+	gl_matrix[2][2] = 1.0f;
+	gl_matrix[2][3] = 0.0f;
+
+	gl_matrix[3][0] = matrix[2][0];
+	gl_matrix[3][1] = matrix[2][1];
+	gl_matrix[3][2] = 0.0f;
+	gl_matrix[3][3] = matrix[2][2];
+}
+
+static void draw_plane_marker_image(Scene *scene,
+                                    MovieTrackingPlaneTrack *plane_track,
+                                    MovieTrackingPlaneMarker *plane_marker)
+{
+	Image *image = plane_track->image;
+	ImBuf *ibuf;
+	void *lock;
+
+	if (image == NULL) {
+		return;
+	}
+
+	ibuf = BKE_image_acquire_ibuf(image, NULL, &lock);
+
+	if (ibuf) {
+		unsigned char *display_buffer;
+		void *cache_handle;
+
+		if (image->flag & IMA_VIEW_AS_RENDER) {
+			display_buffer = IMB_display_buffer_acquire(ibuf,
+			                                            &scene->view_settings,
+			                                            &scene->display_settings,
+			                                            &cache_handle);
+		}
+		else {
+			display_buffer = IMB_display_buffer_acquire(ibuf, NULL,
+			                                            &scene->display_settings,
+			                                            &cache_handle);
+		}
+
+		if (display_buffer) {
+			GLuint texid, last_texid;
+			float frame_corners[4][2] = {{0.0f, 0.0f},
+			                             {1.0f, 0.0f},
+			                             {1.0f, 1.0f},
+			                             {0.0f, 1.0f}};
+			float perspective_matrix[3][3];
+			float gl_matrix[4][4];
+			BKE_tracking_homography_between_two_quads(frame_corners,
+			                                          plane_marker->corners,
+			                                          perspective_matrix);
+
+			homogeneous_2d_to_gl_matrix(perspective_matrix, gl_matrix);
+
+			if (plane_track->image_opacity != 1.0f) {
+				glEnable(GL_BLEND);
+				glBlendFunc(GL_SRC_ALPHA,  GL_ONE_MINUS_SRC_ALPHA);
+			}
+
+			glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE);
+			glColor4f(1.0, 1.0, 1.0, plane_track->image_opacity);
+
+			last_texid = glaGetOneInteger(GL_TEXTURE_2D);
+			glEnable(GL_TEXTURE_2D);
+			glGenTextures(1, (GLuint *)&texid);
+
+			glBindTexture(GL_TEXTURE_2D, texid);
+
+			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+
+			glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, ibuf->x, ibuf->y, 0, GL_RGBA,
+			             GL_UNSIGNED_BYTE, display_buffer);
+
+			glPushMatrix();
+			glMultMatrixf(gl_matrix);
+
+			glBegin(GL_QUADS);
+			glTexCoord2f(0.0f, 0.0f); glVertex2f(0.0f, 0.0f);
+			glTexCoord2f(1.0f, 0.0f); glVertex2f(1.0f, 0.0f);
+			glTexCoord2f(1.0f, 1.0f); glVertex2f(1.0f, 1.0f);
+			glTexCoord2f(0.0f, 1.0f); glVertex2f(0.0f, 1.0f);
+			glEnd();
+
+			glPopMatrix();
+
+			glBindTexture(GL_TEXTURE_2D, last_texid);
+			glDisable(GL_TEXTURE_2D);
+
+			if (plane_track->image_opacity != 1.0f) {
+				glDisable(GL_BLEND);
+			}
+		}
+
+		IMB_display_buffer_release(cache_handle);
+	}
+
+	BKE_image_release_ibuf(image, ibuf, lock);
+}
+
+static void draw_plane_marker_ex(SpaceClip *sc, Scene *scene, MovieTrackingPlaneTrack *plane_track,
                                  MovieTrackingPlaneMarker *plane_marker, bool is_active_track,
                                  bool draw_outline, int width, int height)
 {
 	bool tiny = (sc->flag & SC_SHOW_TINY_MARKER) != 0;
 	bool is_selected_track = plane_track->flag & SELECT;
+	bool draw_plane_quad = plane_track->image == NULL || plane_track->image_opacity == 0.0f;
 	float px[2];
 
 	if (draw_outline) {
@@ -1063,6 +1179,11 @@ static void draw_plane_marker_ex(SpaceClip *sc, MovieTrackingPlaneTrack *plane_t
 	px[0] = 1.0f / width / sc->zoom;
 	px[1] = 1.0f / height / sc->zoom;
 
+	/* Draw image */
+	if (draw_outline == false) {
+		draw_plane_marker_image(scene, plane_track, plane_marker);
+	}
+
 	if (draw_outline) {
 		if (!tiny) {
 			glLineWidth(3.0f);
@@ -1073,34 +1194,36 @@ static void draw_plane_marker_ex(SpaceClip *sc, MovieTrackingPlaneTrack *plane_t
 		glEnable(GL_LINE_STIPPLE);
 	}
 
-	/* Draw rectangle itself. */
-	glBegin(GL_LINE_LOOP);
-	glVertex2fv(plane_marker->corners[0]);
-	glVertex2fv(plane_marker->corners[1]);
-	glVertex2fv(plane_marker->corners[2]);
-	glVertex2fv(plane_marker->corners[3]);
-	glEnd();
-
-	/* Draw axis. */
-	if (!draw_outline) {
-		float end_point[2];
-		glPushAttrib(GL_COLOR_BUFFER_BIT | GL_CURRENT_BIT);
-
-		getArrowEndPoint(width, height, sc->zoom, plane_marker->corners[0], plane_marker->corners[1], end_point);
-		glColor3f(1.0, 0.0, 0.0f);
-		glBegin(GL_LINES);
+	if (draw_plane_quad) {
+		/* Draw rectangle itself. */
+		glBegin(GL_LINE_LOOP);
 		glVertex2fv(plane_marker->corners[0]);
-		glVertex2fv(end_point);
+		glVertex2fv(plane_marker->corners[1]);
+		glVertex2fv(plane_marker->corners[2]);
+		glVertex2fv(plane_marker->corners[3]);
 		glEnd();
 
-		getArrowEndPoint(width, height, sc->zoom, plane_marker->corners[0], plane_marker->corners[3], end_point);
-		glColor3f(0.0, 1.0, 0.0f);
-		glBegin(GL_LINES);
-		glVertex2fv(plane_marker->corners[0]);
-		glVertex2fv(end_point);
-		glEnd();
+		/* Draw axis. */
+		if (!draw_outline) {
+			float end_point[2];
+			glPushAttrib(GL_COLOR_BUFFER_BIT | GL_CURRENT_BIT);
+
+			getArrowEndPoint(width, height, sc->zoom, plane_marker->corners[0], plane_marker->corners[1], end_point);
+			glColor3f(1.0, 0.0, 0.0f);
+			glBegin(GL_LINES);
+			glVertex2fv(plane_marker->corners[0]);
+			glVertex2fv(end_point);
+			glEnd();
+
+			getArrowEndPoint(width, height, sc->zoom, plane_marker->corners[0], plane_marker->corners[3], end_point);
+			glColor3f(0.0, 1.0, 0.0f);
+			glBegin(GL_LINES);
+			glVertex2fv(plane_marker->corners[0]);
+			glVertex2fv(end_point);
+			glEnd();
 
-		glPopAttrib();
+			glPopAttrib();
+		}
 	}
 
 	/* Draw sliders. */
@@ -1122,32 +1245,32 @@ static void draw_plane_marker_ex(SpaceClip *sc, MovieTrackingPlaneTrack *plane_t
 	}
 }
 
-static void draw_plane_marker_outline(SpaceClip *sc, MovieTrackingPlaneTrack *plane_track,
+static void draw_plane_marker_outline(SpaceClip *sc, Scene *scene, MovieTrackingPlaneTrack *plane_track,
                                       MovieTrackingPlaneMarker *plane_marker, int width, int height)
 {
-	draw_plane_marker_ex(sc, plane_track, plane_marker, false, true, width, height);
+	draw_plane_marker_ex(sc, scene, plane_track, plane_marker, false, true, width, height);
 }
 
-static void draw_plane_marker(SpaceClip *sc, MovieTrackingPlaneTrack *plane_track,
+static void draw_plane_marker(SpaceClip *sc, Scene *scene, MovieTrackingPlaneTrack *plane_track,
                               MovieTrackingPlaneMarker *plane_marker, bool is_active_track,
                               int width, int height)
 {
-	draw_plane_marker_ex(sc, plane_track, plane_marker, is_active_track, false, width, height);
+	draw_plane_marker_ex(sc, scene, plane_track, plane_marker, is_active_track, false, width, height);
 }
 
-static void draw_plane_track(SpaceClip *sc, MovieTrackingPlaneTrack *plane_track,
+static void draw_plane_track(SpaceClip *sc, Scene *scene, MovieTrackingPlaneTrack *plane_track,
                              int framenr, bool is_active_track, int width, int height)
 {
 	MovieTrackingPlaneMarker *plane_marker;
 
 	plane_marker = BKE_tracking_plane_marker_get(plane_track, framenr);
 
-	draw_plane_marker_outline(sc, plane_track, plane_marker, width, height);
-	draw_plane_marker(sc, plane_track, plane_marker, is_active_track, width, height);
+	draw_plane_marker_outline(sc, scene, plane_track, plane_marker, width, height);
+	draw_plane_marker(sc, scene, plane_track, plane_marker, is_active_track, width, height);
 }
 
 /* Draw all kind of tracks. */
-static void draw_tracking_tracks(SpaceClip *sc, ARegion *ar, MovieClip *clip,
+static void draw_tracking_tracks(SpaceClip *sc, Scene *scene, ARegion *ar, MovieClip *clip,
                                  int width, int height, float zoomx, float zoomy)
 {
 	float x, y;
@@ -1180,6 +1303,15 @@ static void draw_tracking_tracks(SpaceClip *sc, ARegion *ar, MovieClip *clip,
 
 	act_track = BKE_tracking_track_get_active(tracking);
 
+	/* Draw plane tracks */
+	active_plane_track = BKE_tracking_plane_track_get_active(tracking);
+	for (plane_track = plane_tracks_base->first;
+	     plane_track;
+	     plane_track = plane_track->next)
+	{
+		draw_plane_track(sc, scene, plane_track, framenr, plane_track == active_plane_track, width, height);
+	}
+
 	if (sc->user.render_flag & MCLIP_PROXY_RENDER_UNDISTORT) {
 		int count = 0;
 
@@ -1348,15 +1480,6 @@ static void draw_tracking_tracks(SpaceClip *sc, ARegion *ar, MovieClip *clip,
 		glDisable(GL_POINT_SMOOTH);
 	}
 
-	/* Draw plane tracks */
-	active_plane_track = BKE_tracking_plane_track_get_active(tracking);
-	for (plane_track = plane_tracks_base->first;
-	     plane_track;
-	     plane_track = plane_track->next)
-	{
-		draw_plane_track(sc, plane_track, framenr, plane_track == active_plane_track, width, height);
-	}
-
 	glPopMatrix();
 
 	if (sc->flag & SC_SHOW_NAMES) {
@@ -1666,7 +1789,7 @@ void clip_draw_main(const bContext *C, SpaceClip *sc, ARegion *ar)
 
 	if (width && height) {
 		draw_stabilization_border(sc, ar, width, height, zoomx, zoomy);
-		draw_tracking_tracks(sc, ar, clip, width, height, zoomx, zoomy);
+		draw_tracking_tracks(sc, scene, ar, clip, width, height, zoomx, zoomy);
 		draw_distortion(sc, ar, clip, width, height, zoomx, zoomy);
 	}
 
diff --git a/source/blender/makesdna/DNA_tracking_types.h b/source/blender/makesdna/DNA_tracking_types.h
index ad9a2ea169c..4e6a762c36a 100644
--- a/source/blender/makesdna/DNA_tracking_types.h
+++ b/source/blender/makesdna/DNA_tracking_types.h
@@ -44,6 +44,7 @@
 
 struct bGPdata;
 struct ImBuf;
+struct Image;
 struct MovieReconstructedCamera;
 struct MovieTrackingCamera;
 struct MovieTrackingBundle;
@@ -189,8 +190,11 @@ typedef struct MovieTrackingPlaneTrack {
 
 	int flag;    /* flags (selection, ...) */
 
+	struct Image *image;                 /* Image displaying during editing */
+	float image_opacity;                 /* Opacity of the image */
+
 	/* Runtime data */
-	int last_marker, pad2;               /* Most recently used marker */
+	int last_marker;                     /* Most recently used marker */
 } MovieTrackingPlaneTrack;
 
 typedef struct MovieTrackingSettings {
diff --git a/source/blender/makesrna/intern/rna_tracking.c b/source/blender/makesrna/intern/rna_tracking.c
index 6107d4e4a51..871bc159f28 100644
--- a/source/blender/makesrna/intern/rna_tracking.c
+++ b/source/blender/makesrna/intern/rna_tracking.c
@@ -1474,6 +1474,20 @@ static void rna_def_trackingPlaneTrack(BlenderRNA *brna)
 	RNA_def_property_boolean_sdna(prop, NULL, "flag", PLANE_TRACK_AUTOKEY);
 	RNA_def_property_ui_text(prop, "Auto Keyframe", "Automatic keyframe insertion when moving plane corners");
 	RNA_def_property_ui_icon(prop, ICON_REC, 0);
+
+	/* image */
+	prop = RNA_def_property(srna, "image", PROP_POINTER, PROP_NONE);
+	RNA_def_property_struct_type(prop, "Image");
+	RNA_def_property_flag(prop, PROP_EDITABLE);
+	RNA_def_property_ui_text(prop, "Image", "Image displayed in the track during editing in clip editor");
+	RNA_def_property_update(prop, NC_MOVIECLIP | ND_DISPLAY, NULL);
+
+	/* image opacity */
+	prop = RNA_def_property(srna, "image_opacity", PROP_FLOAT, PROP_NONE);
+	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
+	RNA_def_property_range(prop, 0.0, 1.0);
+	RNA_def_property_ui_text(prop, "Image Opacity", "Opacity of the image");
+	RNA_def_property_update(prop, NC_MOVIECLIP | ND_DISPLAY, NULL);
 }
 
 static void rna_def_trackingStabilization(BlenderRNA *brna)