Add support for detecting tracking failure in the ESM tracker component of

libmv. Since both KLT and Hybrid rely on ESM underneath, KLT and Hybrid now
have a minimum correlation setting to match. With this fix, track failures
should get detected quicker, with the issue that sometimes the tracker will
give up too easily. That is fixable by reducing the required correlation (in
the track properties).

4 files changed, 33 insertions, 9 deletions

diff --git a/extern/libmv/libmv-capi.cpp b/extern/libmv/libmv-capi.cpp
index ed89f8cbfb3..e83a826e640 100644
--- a/extern/libmv/libmv-capi.cpp
+++ b/extern/libmv/libmv-capi.cpp
@@ -111,12 +111,13 @@ void libmv_setLoggingVerbosity(int verbosity)
 
 /* ************ RegionTracker ************ */
 
-libmv_RegionTracker *libmv_pyramidRegionTrackerNew(int max_iterations, int pyramid_level, int half_window_size)
+libmv_RegionTracker *libmv_pyramidRegionTrackerNew(int max_iterations, int pyramid_level, int half_window_size, double minimum_correlation)
 {
 	libmv::EsmRegionTracker *esm_region_tracker = new libmv::EsmRegionTracker;
 	esm_region_tracker->half_window_size = half_window_size;
 	esm_region_tracker->max_iterations = max_iterations;
 	esm_region_tracker->min_determinant = 1e-4;
+	esm_region_tracker->minimum_correlation = minimum_correlation;
 
 	libmv::PyramidRegionTracker *pyramid_region_tracker =
 		new libmv::PyramidRegionTracker(esm_region_tracker, pyramid_level);
@@ -124,12 +125,13 @@ libmv_RegionTracker *libmv_pyramidRegionTrackerNew(int max_iterations, int pyram
 	return (libmv_RegionTracker *)pyramid_region_tracker;
 }
 
-libmv_RegionTracker *libmv_hybridRegionTrackerNew(int max_iterations, int half_window_size)
+libmv_RegionTracker *libmv_hybridRegionTrackerNew(int max_iterations, int half_window_size, double minimum_correlation)
 {
 	libmv::EsmRegionTracker *esm_region_tracker = new libmv::EsmRegionTracker;
 	esm_region_tracker->half_window_size = half_window_size;
 	esm_region_tracker->max_iterations = max_iterations;
 	esm_region_tracker->min_determinant = 1e-4;
+	esm_region_tracker->minimum_correlation = minimum_correlation;
 
 	libmv::BruteRegionTracker *brute_region_tracker = new libmv::BruteRegionTracker;
   brute_region_tracker->half_window_size = half_window_size;
diff --git a/extern/libmv/libmv-capi.h b/extern/libmv/libmv-capi.h
index c5e61d69c56..e10d4ef842a 100644
--- a/extern/libmv/libmv-capi.h
+++ b/extern/libmv/libmv-capi.h
@@ -43,8 +43,8 @@ void libmv_startDebugLogging(void);
 void libmv_setLoggingVerbosity(int verbosity);
 
 /* RegionTracker */
-struct libmv_RegionTracker *libmv_pyramidRegionTrackerNew(int max_iterations, int pyramid_level, int half_window_size);
-struct libmv_RegionTracker *libmv_hybridRegionTrackerNew(int max_iterations, int half_window_size);
+struct libmv_RegionTracker *libmv_pyramidRegionTrackerNew(int max_iterations, int pyramid_level, int half_window_size, double minimum_correlation);
+struct libmv_RegionTracker *libmv_hybridRegionTrackerNew(int max_iterations, int half_window_size, double minimum_correlation);
 int libmv_regionTrackerTrack(struct libmv_RegionTracker *libmv_tracker, const float *ima1, const float *ima2,
 			int width, int height, double  x1, double  y1, double *x2, double *y2);
 void libmv_regionTrackerDestroy(struct libmv_RegionTracker *libmv_tracker);
diff --git a/extern/libmv/libmv/tracking/esm_region_tracker.cc b/extern/libmv/libmv/tracking/esm_region_tracker.cc
index 844b5ff3cb9..9784b61c068 100644
--- a/extern/libmv/libmv/tracking/esm_region_tracker.cc
+++ b/extern/libmv/libmv/tracking/esm_region_tracker.cc
@@ -273,10 +273,30 @@ bool EsmRegionTracker::Track(const FloatImage &image1,
 
     // If the step was accepted, then check for termination.
     if (d.squaredNorm() < min_update_squared_distance) {
-      if (new_error > reasonable_error) {
-        LG << "Update size shrank but reasonable error ("
-           << reasonable_error << ") not achieved; failing.";
-        return true; // XXX
+      // Compute the Pearson product-moment correlation coefficient to check
+      // for sanity.
+      // TODO(keir): Put this somewhere smarter.
+      double sX=0,sY=0,sXX=0,sYY=0,sXY=0;
+      for (int r = 0; r < width; ++r) {
+        for (int c = 0; c < width; ++c) {
+          double x = image_and_gradient1_sampled(r, c, 0);
+          double y = image_and_gradient2_sampled[new_image](r, c, 0);
+          sX += x;
+          sY += y;
+          sXX += x*x;
+          sYY += y*y;
+          sXY += x*y;
+        }
+      }
+      double N = width*width;
+      sX /= N, sY /= N, sXX /= N, sYY /= N, sXY /= N;
+      double correlation = (sXY-sX*sY)/sqrt(double((sXX-sX*sX)*(sYY-sY*sY)));
+      LG << "Final correlation: " << correlation;
+
+      if (correlation < minimum_correlation) {
+        LG << "Correlation " << correlation << " greater than "
+           << minimum_correlation << "; bailing.";
+        return false;
       }
       LG << "Successful track in " << (i + 1) << " iterations.";
       return true;
diff --git a/extern/libmv/libmv/tracking/esm_region_tracker.h b/extern/libmv/libmv/tracking/esm_region_tracker.h
index c63417201ad..602b32f1675 100644
--- a/extern/libmv/libmv/tracking/esm_region_tracker.h
+++ b/extern/libmv/libmv/tracking/esm_region_tracker.h
@@ -38,7 +38,8 @@ struct EsmRegionTracker : public RegionTracker {
         max_iterations(16),
         min_determinant(1e-6),
         min_update_squared_distance(1e-4),
-        sigma(0.9) {}
+        sigma(0.9),
+        minimum_correlation(0.78) {}
   
   virtual ~EsmRegionTracker() {}
 
@@ -54,6 +55,7 @@ struct EsmRegionTracker : public RegionTracker {
   double min_determinant;
   double min_update_squared_distance;
   double sigma;
+  double minimum_correlation;
 };
 
 }  // namespace libmv