Weighted tracks

Added a weight slider to track which defines
how much particular track affects in a final
reconstruction. This weight is for sure
animateable.

Currently it affects on BA step only which in
most cases will work just fine.

The usecase of this slider is to have it set
to 1.0 most of the time where the track is
good, but blend it's weight down to 0 when
tracker looses the track. This will prevent
camera from jump.

Tutorial is to be done by Sebastian.

15 files changed, 141 insertions, 49 deletions

diff --git a/extern/libmv/libmv-capi.cc b/extern/libmv/libmv-capi.cc
index 91a3b845815..0585bd3e8ac 100644
--- a/extern/libmv/libmv-capi.cc
+++ b/extern/libmv/libmv-capi.cc
@@ -393,9 +393,9 @@ void libmv_tracksDestroy(struct libmv_Tracks *libmv_tracks)
 	LIBMV_OBJECT_DELETE(libmv_tracks, Tracks);
 }
 
-void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y)
+void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y, double weight)
 {
-	((libmv::Tracks*) libmv_tracks)->Insert(image, track, x, y);
+	((libmv::Tracks*) libmv_tracks)->Insert(image, track, x, y, weight);
 }
 
 /* ************ Reconstruction ************ */
diff --git a/extern/libmv/libmv-capi.h b/extern/libmv/libmv-capi.h
index 9541f411ba0..13cc3ae8499 100644
--- a/extern/libmv/libmv-capi.h
+++ b/extern/libmv/libmv-capi.h
@@ -73,7 +73,7 @@ void libmv_samplePlanarPatch(const float *image, int width, int height,
 /* Tracks */
 struct libmv_Tracks *libmv_tracksNew(void);
 void libmv_tracksDestroy(struct libmv_Tracks *libmv_tracks);
-void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y);
+void libmv_tracksInsert(struct libmv_Tracks *libmv_tracks, int image, int track, double x, double y, double weight);
 
 /* Reconstruction */
 #define LIBMV_REFINE_FOCAL_LENGTH          (1 << 0)
diff --git a/extern/libmv/libmv-capi_stub.cc b/extern/libmv/libmv-capi_stub.cc
index e6d3753961b..bda9605b422 100644
--- a/extern/libmv/libmv-capi_stub.cc
+++ b/extern/libmv/libmv-capi_stub.cc
@@ -85,7 +85,7 @@ struct libmv_Tracks *libmv_tracksNew(void)
 }
 
 void libmv_tracksInsert(struct libmv_Tracks * /*libmv_tracks*/, int /*image*/,
-                        int /*track*/, double /*x*/, double /*y*/)
+                        int /*track*/, double /*x*/, double /*y*/, double /*weight*/)
 {
 }
 
diff --git a/extern/libmv/libmv/simple_pipeline/bundle.cc b/extern/libmv/libmv/simple_pipeline/bundle.cc
index c778c11b3f6..e7887256892 100644
--- a/extern/libmv/libmv/simple_pipeline/bundle.cc
+++ b/extern/libmv/libmv/simple_pipeline/bundle.cc
@@ -60,8 +60,11 @@ namespace {
 //
 // This functor uses a radial distortion model.
 struct OpenCVReprojectionError {
-  OpenCVReprojectionError(const double observed_x, const double observed_y)
-      : observed_x(observed_x), observed_y(observed_y) {}
+  OpenCVReprojectionError(const double observed_x,
+                          const double observed_y,
+                          const double weight)
+      : observed_x_(observed_x), observed_y_(observed_y),
+        weight_(weight) {}
 
   template <typename T>
   bool operator()(const T* const intrinsics,
@@ -112,13 +115,14 @@ struct OpenCVReprojectionError {
                                           &predicted_y);
 
     // The error is the difference between the predicted and observed position.
-    residuals[0] = predicted_x - T(observed_x);
-    residuals[1] = predicted_y - T(observed_y);
+    residuals[0] = (predicted_x - T(observed_x_)) * weight_;
+    residuals[1] = (predicted_y - T(observed_y_)) * weight_;
     return true;
   }
 
-  const double observed_x;
-  const double observed_y;
+  const double observed_x_;
+  const double observed_y_;
+  const double weight_;
 };
 
 // Print a message to the log which camera intrinsics are gonna to be optimixed.
@@ -378,25 +382,31 @@ void EuclideanBundleCommonIntrinsics(const Tracks &tracks,
     // camera translaiton.
     double *current_camera_R_t = &all_cameras_R_t[camera->image](0);
 
-    problem.AddResidualBlock(new ceres::AutoDiffCostFunction<
-        OpenCVReprojectionError, 2, 8, 6, 3>(
-            new OpenCVReprojectionError(
-                marker.x,
-                marker.y)),
-        NULL,
-        ceres_intrinsics,
-        current_camera_R_t,
-        &point->X(0));
-
-    // We lock the first camera to better deal with scene orientation ambiguity.
-    if (!have_locked_camera) {
-      problem.SetParameterBlockConstant(current_camera_R_t);
-      have_locked_camera = true;
-    }
+    // Skip residual block for markers which does have absolutely
+    // no affect on the final solution.
+    // This way ceres is not gonna to go crazy.
+    if (marker.weight != 0.0) {
+      problem.AddResidualBlock(new ceres::AutoDiffCostFunction<
+          OpenCVReprojectionError, 2, 8, 6, 3>(
+              new OpenCVReprojectionError(
+                  marker.x,
+                  marker.y,
+                  marker.weight)),
+          NULL,
+          ceres_intrinsics,
+          current_camera_R_t,
+          &point->X(0));
+
+      // We lock the first camera to better deal with scene orientation ambiguity.
+      if (!have_locked_camera) {
+        problem.SetParameterBlockConstant(current_camera_R_t);
+        have_locked_camera = true;
+      }
 
-    if (bundle_constraints & BUNDLE_NO_TRANSLATION) {
-      problem.SetParameterization(current_camera_R_t,
+      if (bundle_constraints & BUNDLE_NO_TRANSLATION) {
+        problem.SetParameterization(current_camera_R_t,
                                   constant_translation_parameterization);
+      }
     }
 
     num_residuals++;
diff --git a/extern/libmv/libmv/simple_pipeline/modal_solver.cc b/extern/libmv/libmv/simple_pipeline/modal_solver.cc
index 90dfde15660..caccce68cbe 100644
--- a/extern/libmv/libmv/simple_pipeline/modal_solver.cc
+++ b/extern/libmv/libmv/simple_pipeline/modal_solver.cc
@@ -57,8 +57,10 @@ void ModalSolverLogProress(ProgressUpdateCallback *update_callback,
 struct ModalReprojectionError {
   ModalReprojectionError(double observed_x,
                          double observed_y,
+                         const double weight,
                          const Vec3 &bundle)
-    : observed_x(observed_x), observed_y(observed_y), bundle(bundle) { }
+    : observed_x_(observed_x), observed_y_(observed_y),
+      weight_(weight), bundle_(bundle) { }
 
   template <typename T>
   bool operator()(const T* quaternion,   // Rotation quaternion
@@ -68,9 +70,9 @@ struct ModalReprojectionError {
 
     // Convert bundle position from double to T.
     T X[3];
-    X[0] = T(bundle(0));
-    X[1] = T(bundle(1));
-    X[2] = T(bundle(2));
+    X[0] = T(bundle_(0));
+    X[1] = T(bundle_(1));
+    X[2] = T(bundle_(2));
 
     // Compute projective coordinates: x = RX.
     T x[3];
@@ -84,15 +86,16 @@ struct ModalReprojectionError {
 
     // The error is the difference between reprojected
     // and observed marker position.
-    residuals[0] = xn - T(observed_x);
-    residuals[1] = yn - T(observed_y);
+    residuals[0] = xn - T(observed_x_);
+    residuals[1] = yn - T(observed_y_);
 
     return true;
   }
 
-  double observed_x;
-  double observed_y;
-  Vec3 bundle;
+  double observed_x_;
+  double observed_y_;
+  double weight_;
+  Vec3 bundle_;
 };
 }  // namespace
 
@@ -180,11 +183,13 @@ void ModalSolver(const Tracks &tracks,
       Marker &marker = all_markers[i];
       EuclideanPoint *point = reconstruction->PointForTrack(marker.track);
 
-      if (point) {
+      if (point && marker.weight != 0.0) {
         problem.AddResidualBlock(new ceres::AutoDiffCostFunction<
             ModalReprojectionError,
             2, /* num_residuals */
-            4>(new ModalReprojectionError(marker.x, marker.y,
+            4>(new ModalReprojectionError(marker.x,
+                                          marker.y,
+                                          marker.weight,
                                           point->X)),
             NULL,
             &quaternion(0));
diff --git a/extern/libmv/libmv/simple_pipeline/tracks.cc b/extern/libmv/libmv/simple_pipeline/tracks.cc
index f9e50d20af9..d5d009708ba 100644
--- a/extern/libmv/libmv/simple_pipeline/tracks.cc
+++ b/extern/libmv/libmv/simple_pipeline/tracks.cc
@@ -34,7 +34,7 @@ Tracks::Tracks(const Tracks &other) {
 
 Tracks::Tracks(const vector<Marker> &markers) : markers_(markers) {}
 
-void Tracks::Insert(int image, int track, double x, double y) {
+void Tracks::Insert(int image, int track, double x, double y, double weight) {
   // TODO(keir): Wow, this is quadratic for repeated insertions. Fix this by
   // adding a smarter data structure like a set<>.
   for (int i = 0; i < markers_.size(); ++i) {
@@ -45,7 +45,7 @@ void Tracks::Insert(int image, int track, double x, double y) {
       return;
     }
   }
-  Marker marker = { image, track, x, y };
+  Marker marker = { image, track, x, y, weight };
   markers_.push_back(marker);
 }
 
@@ -122,7 +122,7 @@ Marker Tracks::MarkerInImageForTrack(int image, int track) const {
       return markers_[i];
     }
   }
-  Marker null = { -1, -1, -1, -1 };
+  Marker null = { -1, -1, -1, -1, 0.0 };
   return null;
 }
 
diff --git a/extern/libmv/libmv/simple_pipeline/tracks.h b/extern/libmv/libmv/simple_pipeline/tracks.h
index f9af3ada45b..e2f8cf6b459 100644
--- a/extern/libmv/libmv/simple_pipeline/tracks.h
+++ b/extern/libmv/libmv/simple_pipeline/tracks.h
@@ -33,14 +33,20 @@ namespace libmv {
     in the image identified by \a image. All markers for to the same target
     form a track identified by a common \a track number.
 
+    \a weight is used by bundle adjustment and weight means how much the
+    track affects on a final solution.
+
     \note Markers are typically aggregated with the help of the \l Tracks class.
 
     \sa Tracks
 */
+// TODO(sergey): Consider using comment for every member separately
+//               instead of having one giantic comment block.
 struct Marker {
   int image;
   int track;
   double x, y;
+  double weight;
 };
 
 /*!
@@ -72,9 +78,14 @@ class Tracks {
       \a image and \a track are the keys used to retrieve the markers with the
       other methods in this class.
 
+      \a weight is used by bundle adjustment and weight means how much the
+      track affects on a final solution.
+
       \note To get an identifier for a new track, use \l MaxTrack() + 1.
   */
-  void Insert(int image, int track, double x, double y);
+  // TODO(sergey): Consider using InsetWeightedMarker istead of using
+  //               stupid default value?
+  void Insert(int image, int track, double x, double y, double weight = 1.0);
 
   /// Returns all the markers.
   vector<Marker> AllMarkers() const;
diff --git a/release/scripts/startup/bl_ui/space_clip.py b/release/scripts/startup/bl_ui/space_clip.py
index 3247d2f5e4c..3db8697a457 100644
--- a/release/scripts/startup/bl_ui/space_clip.py
+++ b/release/scripts/startup/bl_ui/space_clip.py
@@ -569,6 +569,8 @@ class CLIP_PT_track(CLIP_PT_tracking_panel, Panel):
         if act_track.use_custom_color:
             row.prop(act_track, "color", text="")
 
+        layout.prop(act_track, "weight")
+
         if act_track.has_bundle:
             label_text = "Average Error: %.4f" % (act_track.average_error)
             layout.label(text=label_text)
diff --git a/source/blender/blenkernel/BKE_tracking.h b/source/blender/blenkernel/BKE_tracking.h
index 51f97180ddb..94e530529ec 100644
--- a/source/blender/blenkernel/BKE_tracking.h
+++ b/source/blender/blenkernel/BKE_tracking.h
@@ -225,7 +225,7 @@ void BKE_tracking_homography_between_two_quads(/*const*/ float reference_corners
 bool BKE_tracking_reconstruction_check(struct MovieTracking *tracking, struct MovieTrackingObject *object,
                                        char *error_msg, int error_size);
 
-struct MovieReconstructContext *BKE_tracking_reconstruction_context_new(struct MovieTracking *tracking,
+struct MovieReconstructContext *BKE_tracking_reconstruction_context_new(struct MovieClip *clip,
                                                                         struct MovieTrackingObject *object,
                                                                         int keyframe1, int keyframe2,
                                                                         int width, int height);
diff --git a/source/blender/blenkernel/intern/tracking.c b/source/blender/blenkernel/intern/tracking.c
index 0ce4d54a74c..d519b93f963 100644
--- a/source/blender/blenkernel/intern/tracking.c
+++ b/source/blender/blenkernel/intern/tracking.c
@@ -36,6 +36,7 @@
 
 #include "MEM_guardedalloc.h"
 
+#include "DNA_anim_types.h"
 #include "DNA_gpencil_types.h"
 #include "DNA_camera_types.h"
 #include "DNA_movieclip_types.h"
@@ -53,6 +54,7 @@
 
 #include "BLF_translation.h"
 
+#include "BKE_fcurve.h"
 #include "BKE_global.h"
 #include "BKE_tracking.h"
 #include "BKE_movieclip.h"
@@ -62,6 +64,8 @@
 #include "IMB_imbuf_types.h"
 #include "IMB_imbuf.h"
 
+#include "RNA_access.h"
+
 #include "raskter.h"
 
 #include "libmv-capi.h"
@@ -590,6 +594,7 @@ MovieTrackingTrack *BKE_tracking_track_add(MovieTracking *tracking, ListBase *tr
 	track->frames_limit = settings->default_frames_limit;
 	track->flag = settings->default_flag;
 	track->algorithm_flag = settings->default_algorithm_flag;
+	track->weight = 1.0f;
 
 	memset(&marker, 0, sizeof(marker));
 	marker.pos[0] = x;
@@ -3467,7 +3472,7 @@ typedef struct ReconstructProgressData {
 } ReconstructProgressData;
 
 /* Create new libmv Tracks structure from blender's tracks list. */
-static struct libmv_Tracks *libmv_tracks_new(ListBase *tracksbase, int width, int height)
+static struct libmv_Tracks *libmv_tracks_new(MovieClip *clip, ListBase *tracksbase, int width, int height)
 {
 	int tracknr = 0;
 	MovieTrackingTrack *track;
@@ -3475,15 +3480,28 @@ static struct libmv_Tracks *libmv_tracks_new(ListBase *tracksbase, int width, in
 
 	track = tracksbase->first;
 	while (track) {
+		FCurve *weight_fcurve;
 		int a = 0;
 
+		weight_fcurve = id_data_find_fcurve(&clip->id, track, &RNA_MovieTrackingTrack,
+		                                    "weight", 0, NULL);
+
 		for (a = 0; a < track->markersnr; a++) {
 			MovieTrackingMarker *marker = &track->markers[a];
 
 			if ((marker->flag & MARKER_DISABLED) == 0) {
+				float weight = track->weight;
+
+				if (weight_fcurve) {
+					int scene_framenr =
+						BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
+					weight = evaluate_fcurve(weight_fcurve, scene_framenr);
+				}
+
 				libmv_tracksInsert(tracks, marker->framenr, tracknr,
 				                   (marker->pos[0] + track->offset[0]) * width,
-				                   (marker->pos[1] + track->offset[1]) * height);
+				                   (marker->pos[1] + track->offset[1]) * height,
+				                   weight);
 			}
 		}
 
@@ -3730,9 +3748,10 @@ bool BKE_tracking_reconstruction_check(MovieTracking *tracking, MovieTrackingObj
  * clip datablock, so editing this clip is safe during
  * reconstruction job is in progress.
  */
-MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieTracking *tracking, MovieTrackingObject *object,
+MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieClip *clip, MovieTrackingObject *object,
                                                                  int keyframe1, int keyframe2, int width, int height)
 {
+	MovieTracking *tracking = &clip->tracking;
 	MovieReconstructContext *context = MEM_callocN(sizeof(MovieReconstructContext), "MovieReconstructContext data");
 	MovieTrackingCamera *camera = &tracking->camera;
 	ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, object);
@@ -3793,7 +3812,7 @@ MovieReconstructContext *BKE_tracking_reconstruction_context_new(MovieTracking *
 	context->sfra = sfra;
 	context->efra = efra;
 
-	context->tracks = libmv_tracks_new(tracksbase, width, height * aspy);
+	context->tracks = libmv_tracks_new(clip, tracksbase, width, height * aspy);
 	context->keyframe1 = keyframe1;
 	context->keyframe2 = keyframe2;
 	context->refine_flags = reconstruct_refine_intrinsics_get_flags(tracking, object);
diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index e6804725587..51158fc5321 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -9735,6 +9735,27 @@ static void do_versions(FileData *fd, Library *lib, Main *main)
 				}
 			}
 		}
+
+		if (!DNA_struct_elem_find(fd->filesdna, "MovieTrackingTrack", "float", "weight")) {
+			MovieClip *clip;
+			for (clip = main->movieclip.first; clip; clip = clip->id.next) {
+				MovieTracking *tracking = &clip->tracking;
+				MovieTrackingObject *tracking_object;
+				for (tracking_object = tracking->objects.first;
+				     tracking_object;
+				     tracking_object = tracking_object->next)
+				{
+					ListBase *tracksbase = BKE_tracking_object_get_tracks(tracking, tracking_object);
+					MovieTrackingTrack *track;
+					for (track = tracksbase->first;
+					     track;
+					     track = track->next)
+					{
+						track->weight = 1.0f;
+					}
+				}
+			}
+		}
 	}
 
 	/* WATCH IT!!!: pointers from libdata have not been converted yet here! */
diff --git a/source/blender/editors/space_clip/clip_utils.c b/source/blender/editors/space_clip/clip_utils.c
index dfd2b6e259d..060531ae82c 100644
--- a/source/blender/editors/space_clip/clip_utils.c
+++ b/source/blender/editors/space_clip/clip_utils.c
@@ -37,7 +37,9 @@
 #include "BLI_utildefines.h"
 #include "BLI_math.h"
 #include "BLI_listbase.h"
+#include "BLI_string.h"
 
+#include "BKE_animsys.h"
 #include "BKE_context.h"
 #include "BKE_movieclip.h"
 #include "BKE_tracking.h"
@@ -184,8 +186,8 @@ void clip_delete_track(bContext *C, MovieClip *clip, MovieTrackingTrack *track)
 	MovieTrackingPlaneTrack *plane_track, *next_plane_track;
 	ListBase *tracksbase = BKE_tracking_get_active_tracks(tracking);
 	ListBase *plane_tracks_base = BKE_tracking_get_active_plane_tracks(tracking);
-
 	bool has_bundle = false, update_stab = false;
+	char track_name_escaped[MAX_NAME], prefix[MAX_NAME * 2];
 
 	if (track == act_track)
 		tracking->act_track = NULL;
@@ -245,6 +247,11 @@ void clip_delete_track(bContext *C, MovieClip *clip, MovieTrackingTrack *track)
 		}
 	}
 
+	/* Delete f-curves associated with the track (such as weight, i.e.) */
+	BLI_strescape(track_name_escaped, track->name, sizeof(track_name_escaped));
+	BLI_snprintf(prefix, sizeof(prefix), "tracks[\"%s\"]", track_name_escaped);
+	BKE_animdata_fix_paths_remove(&clip->id, prefix);
+
 	BKE_tracking_track_free(track);
 	BLI_freelinkN(tracksbase, track);
 
diff --git a/source/blender/editors/space_clip/tracking_ops.c b/source/blender/editors/space_clip/tracking_ops.c
index 246ea7fe140..51d7bc3139a 100644
--- a/source/blender/editors/space_clip/tracking_ops.c
+++ b/source/blender/editors/space_clip/tracking_ops.c
@@ -1550,7 +1550,7 @@ static int solve_camera_initjob(bContext *C, SolveCameraJob *scj, wmOperator *op
 	scj->reports = op->reports;
 	scj->user = sc->user;
 
-	scj->context = BKE_tracking_reconstruction_context_new(tracking, object,
+	scj->context = BKE_tracking_reconstruction_context_new(clip, object,
 	                                                       object->keyframe1, object->keyframe2, width, height);
 
 	tracking->stats = MEM_callocN(sizeof(MovieTrackingStats), "solve camera stats");
diff --git a/source/blender/makesdna/DNA_tracking_types.h b/source/blender/makesdna/DNA_tracking_types.h
index 65d36adde18..ad9a2ea169c 100644
--- a/source/blender/makesdna/DNA_tracking_types.h
+++ b/source/blender/makesdna/DNA_tracking_types.h
@@ -142,6 +142,17 @@ typedef struct MovieTrackingTrack {
 	float minimum_correlation;          /* minimal correlation which is still treated as successful tracking */
 
 	struct bGPdata *gpd;        /* grease-pencil data */
+
+	/* Weight of this track.
+	 *
+	 * Weight defines how much the track affects on the final reconstruction,
+	 * usually gets animated in a way so when track has just appeared it's
+	 * weight is zero and then it gets faded up.
+	 *
+	 * Used to prevent jumps of the camera when tracks are appearing or
+	 * disappearing.
+	 */
+	float weight, pad;
 } MovieTrackingTrack;
 
 typedef struct MovieTrackingPlaneMarker {
diff --git a/source/blender/makesrna/intern/rna_tracking.c b/source/blender/makesrna/intern/rna_tracking.c
index 8e26bb96a1f..f62778a5c1e 100644
--- a/source/blender/makesrna/intern/rna_tracking.c
+++ b/source/blender/makesrna/intern/rna_tracking.c
@@ -1339,6 +1339,12 @@ static void rna_def_trackingTrack(BlenderRNA *brna)
 	RNA_def_property_struct_type(prop, "GreasePencil");
 	RNA_def_property_ui_text(prop, "Grease Pencil", "Grease pencil data for this track");
 	RNA_def_property_update(prop, NC_MOVIECLIP | ND_DISPLAY, NULL);
+
+	/* weight */
+	prop = RNA_def_property(srna, "weight", PROP_FLOAT, PROP_FACTOR);
+	RNA_def_property_float_sdna(prop, NULL, "weight");
+	RNA_def_property_range(prop, 0.0f, 1.0f);
+	RNA_def_property_ui_text(prop, "Weight", "How much this track affects on a final solution");
 }
 
 static void rna_def_trackingPlaneMarker(BlenderRNA *brna)