Camera tracking integration

===========================

- Distortion/undistortion were accidentally swapped in both of grease
  pencil calibration and grid.

  This was fixed but distortion coefficients should be re-adjusted.

  There are some issues with high distortion coefficients but it
  came from libmv and until it'll be fixed there nothing really
  can be done here.

- Fixed displayed aspect ratio for muted footage when pixel/display
  aspect ratio is differ from 1.0 or proxy size is not Full Render,

- Added c-api for image distortion/undistortion, Not used yet.

8 files changed, 205 insertions, 100 deletions

diff --git a/extern/libmv/ChangeLog b/extern/libmv/ChangeLog
index 8bdbe6c2154..228ae54c06a 100644
--- a/extern/libmv/ChangeLog
+++ b/extern/libmv/ChangeLog
@@ -1,3 +1,33 @@
+commit 10bdad9ad2cea2603896263cde5a5339169a9af0
+Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
+Date:   Fri Aug 12 21:05:32 2011 +0200
+
+    Fix out of bound access in warp bilinear sampling.
+
+commit dd9a418db021a28af2c1198d5e5b9e68fe048a03
+Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
+Date:   Fri Aug 12 19:14:36 2011 +0200
+
+    Fix compilation with -funsigned-char.
+
+commit bd1a268ede39b67f2ba4b360f6fc693419e7cd7f
+Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
+Date:   Fri Aug 12 18:39:27 2011 +0200
+
+    CameraIntrinsics fixes.
+
+commit ae513b39fb779632f96ceff7c1e014fb8e68702a
+Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
+Date:   Tue Aug 9 19:38:58 2011 +0200
+
+    Remove stray QDebug include.
+
+commit 1e58f55078ce6009a885be30ae0316aec6ed8239
+Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
+Date:   Tue Aug 9 14:16:31 2011 +0200
+
+    Make API future-proof (for an eventual affine or planar tracker).
+
 commit c2af303e7bf0dddcb02937323ac5846b1801e6cc
 Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
 Date:   Tue Aug 9 11:13:29 2011 +0200
@@ -388,58 +418,3 @@ Date:   Wed Jul 13 10:21:52 2011 +0200
     Add feature detection.
     
     Remove unused keyframe selection code.
-
-commit 1088517da70ae9ec2369752fbc17dde4bb869c47
-Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
-Date:   Mon Jul 11 22:42:14 2011 +0200
-
-    Replace system file dialog with custom one which supports multiple file/folder selection.
-    Add Stop button when adding frame from video.
-
-commit 323af7a3a1344e7ef419c72fe22b674c17c578e3
-Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
-Date:   Mon Jul 11 16:30:59 2011 +0200
-
-    Add stub for feature detection.
-
-commit 739ec729fa8bc6e0ba6a40cc7b89d85e5332e900
-Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
-Date:   Mon Jul 11 16:30:42 2011 +0200
-
-    Fix compilation of Qt Calibration tool.
-
-commit 113ccc413db6bdfb75839418bddfba5ce3ba60f7
-Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
-Date:   Mon Jul 11 12:27:47 2011 +0200
-
-    Add raw on-disk video cache.
-    
-    This is useful for faster testing iterations.
-    It avoids waiting for the footage to decode on startup.
-    Using file memory mapping, we let the OS handle caching.
-    The OS disk cache also has the advantage of being kept in memory between runs.
-    A cache is only created for shot < 1GiB (e.g 17s of 1080@30).
-
-commit efbe8b6a3e2a8cff497726e11afffbce0ec6aa53
-Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
-Date:   Mon Jul 11 11:11:47 2011 +0200
-
-    Miscellaneous UI fixes, faster grayscale conversion, compute window size from pattern size and pyramid level count.
-
-commit 0c6911831b0a391da14bf1c5833b1e5c0d5056b7
-Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
-Date:   Sun Jul 10 22:48:15 2011 +0200
-
-    Add Video support using FFmpeg to Qt Calibration.
-
-commit c9f2bd5a17585f816bf0ef89097ac8f222ad058e
-Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
-Date:   Sun Jul 10 19:44:35 2011 +0200
-
-    Add Video support using FFmpeg to Qt Tracker.
-
-commit af8f9c412b0592868a0e68a14686651b45de9737
-Author: Matthias Fauconneau <matthias.fauconneau@gmail.com>
-Date:   Sat Jul 9 12:08:28 2011 +0200
-
-    Update README with instructions to build Qt OpenCV Calibration tool.
diff --git a/extern/libmv/libmv-capi.cpp b/extern/libmv/libmv-capi.cpp
index 8390a197cf5..0d1e381a103 100644
--- a/extern/libmv/libmv-capi.cpp
+++ b/extern/libmv/libmv-capi.cpp
@@ -501,6 +501,56 @@ void libmv_destroyCorners(struct libmv_Corners *libmv_corners)
 	delete (std::vector<libmv::Corner> *)libmv_corners;
 }
 
+/* ************ distortion ************ */
+
+void libmv_undistortByte(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+			unsigned char *src, unsigned char *dst, int width, int height, int channels)
+{
+	libmv::CameraIntrinsics intrinsics;
+
+	intrinsics.SetFocalLength(focal_length, focal_length);
+	intrinsics.SetPrincipalPoint(principal_x, principal_y);
+	intrinsics.SetRadialDistortion(k1, k2, k3);
+
+	intrinsics.Undistort(src, dst, width, height, channels);
+}
+
+void libmv_undistortFloat(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+			float *src, float *dst, int width, int height, int channels)
+{
+	libmv::CameraIntrinsics intrinsics;
+
+	intrinsics.SetFocalLength(focal_length, focal_length);
+	intrinsics.SetPrincipalPoint(principal_x, principal_y);
+	intrinsics.SetRadialDistortion(k1, k2, k3);
+
+	intrinsics.Undistort(src, dst, width, height, channels);
+}
+
+void libmv_distortByte(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+			unsigned char *src, unsigned char *dst, int width, int height, int channels)
+{
+	libmv::CameraIntrinsics intrinsics;
+
+	intrinsics.SetFocalLength(focal_length, focal_length);
+	intrinsics.SetPrincipalPoint(principal_x, principal_y);
+	intrinsics.SetRadialDistortion(k1, k2, k3);
+
+	intrinsics.Distort(src, dst, width, height, channels);
+}
+
+void libmv_distortFloat(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+			float *src, float *dst, int width, int height, int channels)
+{
+	libmv::CameraIntrinsics intrinsics;
+
+	intrinsics.SetFocalLength(focal_length, focal_length);
+	intrinsics.SetPrincipalPoint(principal_x, principal_y);
+	intrinsics.SetRadialDistortion(k1, k2, k3);
+
+	intrinsics.Distort(src, dst, width, height, channels);
+}
+
 /* ************ utils ************ */
 
 void libmv_applyCameraIntrinsics(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
diff --git a/extern/libmv/libmv-capi.h b/extern/libmv/libmv-capi.h
index cf5b5d09e30..a63cbd02083 100644
--- a/extern/libmv/libmv-capi.h
+++ b/extern/libmv/libmv-capi.h
@@ -71,6 +71,17 @@ int libmv_countCorners(struct libmv_Corners *libmv_corners);
 void libmv_getCorner(struct libmv_Corners *libmv_corners, int number, double *x, double *y, double *score, double *size);
 void libmv_destroyCorners(struct libmv_Corners *libmv_corners);
 
+/* dsitortion */
+void libmv_undistortByte(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+			unsigned char *src, unsigned char *dst, int width, int height, int channels);
+void libmv_undistortFloat(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+			float *src, float *dst, int width, int height, int channels);
+
+void libmv_distortByte(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+			unsigned char *src, unsigned char *dst, int width, int height, int channels);
+void libmv_distortFloat(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
+			float *src, float *dst, int width, int height, int channels);
+
 /* utils */
 void libmv_applyCameraIntrinsics(double focal_length, double principal_x, double principal_y, double k1, double k2, double k3,
 			double x, double y, double *x1, double *y1);
diff --git a/extern/libmv/libmv/simple_pipeline/camera_intrinsics.cc b/extern/libmv/libmv/simple_pipeline/camera_intrinsics.cc
index 40a9459e03f..4e88e1f6a5d 100644
--- a/extern/libmv/libmv/simple_pipeline/camera_intrinsics.cc
+++ b/extern/libmv/libmv/simple_pipeline/camera_intrinsics.cc
@@ -23,7 +23,7 @@
 
 namespace libmv {
 
-struct Offset { char ix,iy; unsigned char fx,fy; };
+struct Offset { signed char ix,iy; unsigned char fx,fy; };
 
 CameraIntrinsics::CameraIntrinsics()
     : K_(Mat3::Identity()),
@@ -42,6 +42,44 @@ CameraIntrinsics::~CameraIntrinsics() {
   if(undistort_) delete[] undistort_;
 }
 
+/// Set the entire calibration matrix at once.
+void CameraIntrinsics::SetK(const Mat3 new_k) {
+  K_ = new_k;
+  FreeLookupGrid();
+}
+
+/// Set both x and y focal length in pixels.
+void CameraIntrinsics::SetFocalLength(double focal_x, double focal_y) {
+  K_(0, 0) = focal_x;
+  K_(1, 1) = focal_y;
+  FreeLookupGrid();
+}
+
+void CameraIntrinsics::SetPrincipalPoint(double cx, double cy) {
+  K_(0, 2) = cx;
+  K_(1, 2) = cy;
+  FreeLookupGrid();
+}
+
+void CameraIntrinsics::SetImageSize(int width, int height) {
+  image_width_ = width;
+  image_height_ = height;
+  FreeLookupGrid();
+}
+
+void CameraIntrinsics::SetRadialDistortion(double k1, double k2, double k3) {
+  k1_ = k1;
+  k2_ = k2;
+  k3_ = k3;
+  FreeLookupGrid();
+}
+
+void CameraIntrinsics::SetTangentialDistortion(double p1, double p2) {
+  p1_ = p1;
+  p2_ = p2;
+  FreeLookupGrid();
+}
+
 void CameraIntrinsics::ApplyIntrinsics(double normalized_x,
                                        double normalized_y,
                                        double *image_x,
@@ -117,18 +155,11 @@ void CameraIntrinsics::ComputeLookupGrid(Offset* grid, int width, int height) {
       int fx = round((warp_x-ix)*256), fy = round((warp_y-iy)*256);
       if(fx == 256) { fx=0; ix++; }
       if(fy == 256) { fy=0; iy++; }
-#ifdef CLIP
       // Use nearest border pixel
       if( ix < 0 ) { ix = 0, fx = 0; }
       if( iy < 0 ) { iy = 0, fy = 0; }
-      if( ix >= width-1 ) { ix = width-1, fx = 0; }
-      if( iy >= height-1 ) { iy = height-1, fy = 0; }
-#else
-      // No offset: Avoid adding out of bounds to error.
-      if( ix < 0 || iy < 0 || ix >= width-1 || iy >= height-1 ) {
-        ix = x; iy = y; fx = fy = 0;
-      }
-#endif
+      if( ix >= width-2 ) ix = width-2;
+      if( iy >= height-2 ) iy = height-2;
       //assert( ix-x > -128 && ix-x < 128 && iy-y > -128 && iy-y < 128 );
       Offset offset = { ix-x, iy-y, fx, fy };
       grid[y*width+x] = offset;
@@ -152,6 +183,11 @@ static void Warp(const Offset* grid, const T* src, T* dst,
   }
 }
 
+void CameraIntrinsics::FreeLookupGrid() {
+  if(distort_) delete distort_, distort_=0;
+  if(undistort_) delete undistort_, undistort_=0;
+}
+
 // FIXME: C++ templates limitations makes thing complicated, but maybe there is a simpler method.
 struct ApplyIntrinsicsFunction {
   ApplyIntrinsicsFunction(CameraIntrinsics* intrinsics, double x, double y,
diff --git a/extern/libmv/libmv/simple_pipeline/camera_intrinsics.h b/extern/libmv/libmv/simple_pipeline/camera_intrinsics.h
index 79d10b93d85..6ff40fb7232 100644
--- a/extern/libmv/libmv/simple_pipeline/camera_intrinsics.h
+++ b/extern/libmv/libmv/simple_pipeline/camera_intrinsics.h
@@ -49,36 +49,18 @@ class CameraIntrinsics {
   double      p2()                const { return p2_; }
 
   /// Set the entire calibration matrix at once.
-  void SetK(const Mat3 new_k) {
-    K_ = new_k;
-  }
+  void SetK(const Mat3 new_k);
 
   /// Set both x and y focal length in pixels.
-  void SetFocalLength(double focal_x, double focal_y) {
-    K_(0, 0) = focal_x;
-    K_(1, 1) = focal_y;
-  }
-
-  void SetPrincipalPoint(double cx, double cy) {
-    K_(0, 2) = cx;
-    K_(1, 2) = cy;
-  }
-
-  void SetImageSize(int width, int height) {
-    image_width_ = width;
-    image_height_ = height;
-  }
-
-  void SetRadialDistortion(double k1, double k2, double k3 = 0) {
-    k1_ = k1;
-    k2_ = k2;
-    k3_ = k3;
-  }
-
-  void SetTangentialDistortion(double p1, double p2) {
-    p1_ = p1;
-    p2_ = p2;
-  }
+  void SetFocalLength(double focal_x, double focal_y);
+
+  void SetPrincipalPoint(double cx, double cy);
+
+  void SetImageSize(int width, int height);
+
+  void SetRadialDistortion(double k1, double k2, double k3 = 0);
+
+  void SetTangentialDistortion(double p1, double p2);
 
   /*!
       Apply camera intrinsics to the normalized point to get image coordinates.
@@ -142,6 +124,7 @@ class CameraIntrinsics {
 
  private:
   template<typename WarpFunction> void ComputeLookupGrid(Offset* grid, int width, int height);
+  void FreeLookupGrid();
 
   // The traditional intrinsics matrix from x = K[R|t]X.
   Mat3 K_;
diff --git a/source/blender/blenkernel/BKE_tracking.h b/source/blender/blenkernel/BKE_tracking.h
index c88b6688da8..3681cb5f0e4 100644
--- a/source/blender/blenkernel/BKE_tracking.h
+++ b/source/blender/blenkernel/BKE_tracking.h
@@ -92,6 +92,8 @@ void BKE_tracking_stabilization_data(struct MovieTracking *tracking, int framenr
 struct ImBuf *BKE_tracking_stabilize_shot(struct MovieTracking *tracking, int framenr, struct ImBuf *ibuf, float loc[2], float *scale);
 void BKE_tracking_stabdata_to_mat4(float loc[2], float scale, float mat[4][4]);
 
+struct ImBuf *BKE_tracking_undistort(struct MovieTracking *tracking, struct ImBuf *ibuf);
+
 #define TRACK_SELECTED(track)				((((track)->flag&TRACK_HIDDEN)==0) && ((track)->flag&SELECT || (track)->pat_flag&SELECT || (track)->search_flag&SELECT))
 #define TRACK_AREA_SELECTED(track, area)	((((track)->flag&TRACK_HIDDEN)==0) && ((area)==TRACK_AREA_POINT?(track)->flag&SELECT : ((area)==TRACK_AREA_PAT?(track)->pat_flag&SELECT:(track)->search_flag&SELECT)))
 #define TRACK_VIEW_SELECTED(sc, track)		((TRACK_AREA_SELECTED(track, TRACK_AREA_POINT) || (((sc)->flag&SC_SHOW_MARKER_PATTERN && TRACK_AREA_SELECTED(track, TRACK_AREA_PAT))) || (((sc)->flag&SC_SHOW_MARKER_SEARCH && TRACK_AREA_SELECTED(track, TRACK_AREA_SEARCH)))))
diff --git a/source/blender/blenkernel/intern/tracking.c b/source/blender/blenkernel/intern/tracking.c
index 0037db4bafb..827df3aecaa 100644
--- a/source/blender/blenkernel/intern/tracking.c
+++ b/source/blender/blenkernel/intern/tracking.c
@@ -1516,3 +1516,51 @@ void BKE_tracking_stabdata_to_mat4(float loc[2], float scale, float mat[4][4])
 
 	copy_v2_v2(mat[3], loc);
 }
+
+ImBuf *BKE_tracking_undistort(MovieTracking *tracking, ImBuf *ibuf)
+{
+	ImBuf *resibuf;
+	MovieTrackingCamera *camera= &tracking->camera;
+
+	resibuf= IMB_dupImBuf(ibuf);
+
+	if(ibuf->rect_float) {
+		libmv_undistortFloat(camera->focal,
+		                     camera->principal[0], camera->principal[1],
+		                     camera->k1, camera->k2, camera->k3,
+		                     ibuf->rect_float, resibuf->rect_float, ibuf->x, ibuf->y, ibuf->channels);
+
+		ibuf->userflags|= IB_RECT_INVALID;
+	} else {
+		libmv_undistortByte(camera->focal,
+		                    camera->principal[0], camera->principal[1],
+		                    camera->k1, camera->k2, camera->k3,
+		                    (unsigned char*)ibuf->rect, (unsigned char*)resibuf->rect, ibuf->x, ibuf->y, ibuf->channels);
+	}
+
+	return resibuf;
+}
+
+ImBuf *BKE_tracking_distort(MovieTracking *tracking, ImBuf *ibuf)
+{
+	ImBuf *resibuf;
+	MovieTrackingCamera *camera= &tracking->camera;
+
+	resibuf= IMB_dupImBuf(ibuf);
+
+	if(ibuf->rect_float) {
+		libmv_distortFloat(camera->focal,
+		                   camera->principal[0], camera->principal[1],
+		                   camera->k1, camera->k2, camera->k3,
+		                   ibuf->rect_float, resibuf->rect_float, ibuf->x, ibuf->y, ibuf->channels);
+
+		ibuf->userflags|= IB_RECT_INVALID;
+	} else {
+		libmv_distortByte(camera->focal,
+		                  camera->principal[0], camera->principal[1],
+		                  camera->k1, camera->k2, camera->k3,
+		                  (unsigned char*)ibuf->rect, (unsigned char*)resibuf->rect, ibuf->x, ibuf->y, ibuf->channels);
+	}
+
+	return resibuf;
+}
diff --git a/source/blender/editors/space_clip/clip_draw.c b/source/blender/editors/space_clip/clip_draw.c
index f858b6b8430..87b73ad2c12 100644
--- a/source/blender/editors/space_clip/clip_draw.c
+++ b/source/blender/editors/space_clip/clip_draw.c
@@ -206,7 +206,7 @@ static void draw_movieclip_buffer(SpaceClip *sc, ARegion *ar, ImBuf *ibuf,
 
 	if(sc->flag&SC_MUTE_FOOTAGE) {
 		glColor3f(0.0f, 0.0f, 0.0f);
-		glRectf(x, y, x+ibuf->x*sc->zoom, y+ibuf->y*sc->zoom);
+		glRectf(x, y, x+zoomx*width, y+zoomy*height);
 	} else {
 		if(ibuf->rect_float && !ibuf->rect) {
 			IMB_rect_from_float(ibuf);
@@ -971,7 +971,7 @@ static void draw_distortion(SpaceClip *sc, ARegion *ar, MovieClip *clip, int wid
 		for(i= 0; i<=n; i++) {
 			for(j= 0; j<=n; j++) {
 				if(i==0 || j==0 || i==n || j==n) {
-					BKE_tracking_invert_intrinsics(tracking, pos, tpos);
+					BKE_tracking_apply_intrinsics(tracking, pos, tpos);
 
 					for(a=0; a<4; a++) {
 						int ok;
@@ -1000,7 +1000,7 @@ static void draw_distortion(SpaceClip *sc, ARegion *ar, MovieClip *clip, int wid
 			pos[0]= idx[a][0]*dx;
 			pos[1]= idx[a][1]*dy;
 
-			BKE_tracking_apply_intrinsics(tracking, pos, tpos);
+			BKE_tracking_invert_intrinsics(tracking, pos, tpos);
 
 			DO_MINMAX2(tpos, min, max);
 		}
@@ -1011,7 +1011,7 @@ static void draw_distortion(SpaceClip *sc, ARegion *ar, MovieClip *clip, int wid
 
 		for(i= 0; i<=n; i++) {
 			for(j= 0; j<=n; j++) {
-				BKE_tracking_invert_intrinsics(tracking, pos, grid[i][j]);
+				BKE_tracking_apply_intrinsics(tracking, pos, grid[i][j]);
 
 				grid[i][j][0]/= width;
 				grid[i][j][1]/= height;
@@ -1074,15 +1074,15 @@ static void draw_distortion(SpaceClip *sc, ARegion *ar, MovieClip *clip, int wid
 
 									/* we want to distort only long straight lines */
 									if(stroke->totpoints==2) {
-										BKE_tracking_apply_intrinsics(tracking, pos, pos);
-										BKE_tracking_apply_intrinsics(tracking, npos, npos);
+										BKE_tracking_invert_intrinsics(tracking, pos, pos);
+										BKE_tracking_invert_intrinsics(tracking, npos, npos);
 									}
 
 									sub_v2_v2v2(dpos, npos, pos);
 									mul_v2_fl(dpos, 1.f/steps);
 
 									for(j= 0; j<steps; j++) {
-										BKE_tracking_invert_intrinsics(tracking, pos, tpos);
+										BKE_tracking_apply_intrinsics(tracking, pos, tpos);
 										glVertex2f(tpos[0]/width, tpos[1]/height);
 
 										add_v2_v2(pos, dpos);