Tracking: Simplify configuration of intrinsics to refine

Previously, only predefined and limited set of intrinsics combinations
could have been refined. This was caused by a bundle adjustment library
used in the early days of the solver.

Now it is possible to fully customize which intrinsics are to be refined
during camera solving. Internally solver supports per-parameter settings
but in the interface they are grouped as following:

* Focal length
* Optical center
* Radial distortion coefficients (which includes k1, k2, k3, k4)
* Tangential distortion coefficients (which includes p1, p2)

Differential Revision: https://developer.blender.org/D9294

12 files changed, 126 insertions, 49 deletions

diff --git a/intern/libmv/intern/reconstruction.cc b/intern/libmv/intern/reconstruction.cc
index 64f00b19df0..0f4e890d4ca 100644
--- a/intern/libmv/intern/reconstruction.cc
+++ b/intern/libmv/intern/reconstruction.cc
@@ -95,12 +95,23 @@ void libmv_solveRefineIntrinsics(
   if (refine_intrinsics & LIBMV_REFINE_PRINCIPAL_POINT) {
     bundle_intrinsics |= libmv::BUNDLE_PRINCIPAL_POINT;
   }
-  if (refine_intrinsics & LIBMV_REFINE_RADIAL_DISTORTION_K1) {
-    bundle_intrinsics |= libmv::BUNDLE_RADIAL_K1;
-  }
-  if (refine_intrinsics & LIBMV_REFINE_RADIAL_DISTORTION_K2) {
-    bundle_intrinsics |= libmv::BUNDLE_RADIAL_K2;
-  }
+
+#define SET_DISTORTION_FLAG_CHECKED(type, coefficient) \
+  do { \
+    if (refine_intrinsics & LIBMV_REFINE_ ##  type ##_DISTORTION_ ## coefficient) { \
+      bundle_intrinsics |= libmv::BUNDLE_ ## type ## _ ## coefficient; \
+    } \
+  } while (0)
+
+  SET_DISTORTION_FLAG_CHECKED(RADIAL, K1);
+  SET_DISTORTION_FLAG_CHECKED(RADIAL, K2);
+  SET_DISTORTION_FLAG_CHECKED(RADIAL, K3);
+  SET_DISTORTION_FLAG_CHECKED(RADIAL, K4);
+
+  SET_DISTORTION_FLAG_CHECKED(TANGENTIAL, P1);
+  SET_DISTORTION_FLAG_CHECKED(TANGENTIAL, P2);
+
+#undef SET_DISTORTION_FLAG_CHECKED
 
   progress_update_callback(callback_customdata, 1.0, "Refining solution");
 
diff --git a/intern/libmv/intern/reconstruction.h b/intern/libmv/intern/reconstruction.h
index 727a4943add..408ac884684 100644
--- a/intern/libmv/intern/reconstruction.h
+++ b/intern/libmv/intern/reconstruction.h
@@ -31,10 +31,22 @@ struct libmv_CameraIntrinsicsOptions;
 typedef struct libmv_Reconstruction libmv_Reconstruction;
 
 enum {
-  LIBMV_REFINE_FOCAL_LENGTH         = (1 << 0),
-  LIBMV_REFINE_PRINCIPAL_POINT      = (1 << 1),
-  LIBMV_REFINE_RADIAL_DISTORTION_K1 = (1 << 2),
-  LIBMV_REFINE_RADIAL_DISTORTION_K2 = (1 << 4),
+  LIBMV_REFINE_FOCAL_LENGTH             = (1 << 0),
+  LIBMV_REFINE_PRINCIPAL_POINT          = (1 << 1),
+
+  LIBMV_REFINE_RADIAL_DISTORTION_K1     = (1 << 2),
+  LIBMV_REFINE_RADIAL_DISTORTION_K2     = (1 << 3),
+  LIBMV_REFINE_RADIAL_DISTORTION_K3     = (1 << 4),
+  LIBMV_REFINE_RADIAL_DISTORTION_K4     = (1 << 5),
+  LIBMV_REFINE_RADIAL_DISTORTION        = (LIBMV_REFINE_RADIAL_DISTORTION_K1 |
+                                           LIBMV_REFINE_RADIAL_DISTORTION_K2 |
+                                           LIBMV_REFINE_RADIAL_DISTORTION_K3 |
+                                           LIBMV_REFINE_RADIAL_DISTORTION_K4),
+
+  LIBMV_REFINE_TANGENTIAL_DISTORTION_P1 = (1 << 6),
+  LIBMV_REFINE_TANGENTIAL_DISTORTION_P2 = (1 << 7),
+  LIBMV_REFINE_TANGENTIAL_DISTORTION = (LIBMV_REFINE_TANGENTIAL_DISTORTION_P1 |
+                                        LIBMV_REFINE_TANGENTIAL_DISTORTION_P2),
 };
 
 typedef struct libmv_ReconstructionOptions {
diff --git a/intern/libmv/libmv/simple_pipeline/bundle.cc b/intern/libmv/libmv/simple_pipeline/bundle.cc
index 3df39ab8936..d62f0ebe94f 100644
--- a/intern/libmv/libmv/simple_pipeline/bundle.cc
+++ b/intern/libmv/libmv/simple_pipeline/bundle.cc
@@ -368,6 +368,8 @@ void BundleIntrinsicsLogMessage(const int bundle_intrinsics) {
     APPEND_BUNDLING_INTRINSICS("px, py", BUNDLE_PRINCIPAL_POINT);
     APPEND_BUNDLING_INTRINSICS("k1",     BUNDLE_RADIAL_K1);
     APPEND_BUNDLING_INTRINSICS("k2",     BUNDLE_RADIAL_K2);
+    APPEND_BUNDLING_INTRINSICS("k3",     BUNDLE_RADIAL_K3);
+    APPEND_BUNDLING_INTRINSICS("k4",     BUNDLE_RADIAL_K4);
     APPEND_BUNDLING_INTRINSICS("p1",     BUNDLE_TANGENTIAL_P1);
     APPEND_BUNDLING_INTRINSICS("p2",     BUNDLE_TANGENTIAL_P2);
 
@@ -744,7 +746,8 @@ void EuclideanBundleCommonIntrinsics(
 
     std::vector<int> constant_intrinsics;
 #define MAYBE_SET_CONSTANT(bundle_enum, offset) \
-    if (!(bundle_intrinsics & bundle_enum)) { \
+    if (!(bundle_intrinsics & bundle_enum) || \
+        !packed_intrinsics.IsParameterDefined(offset)) { \
       constant_intrinsics.push_back(offset); \
     }
     MAYBE_SET_CONSTANT(BUNDLE_FOCAL_LENGTH,
@@ -755,22 +758,19 @@ void EuclideanBundleCommonIntrinsics(
                        PackedIntrinsics::OFFSET_PRINCIPAL_POINT_Y);
     MAYBE_SET_CONSTANT(BUNDLE_RADIAL_K1,       PackedIntrinsics::OFFSET_K1);
     MAYBE_SET_CONSTANT(BUNDLE_RADIAL_K2,       PackedIntrinsics::OFFSET_K2);
+    MAYBE_SET_CONSTANT(BUNDLE_RADIAL_K3,       PackedIntrinsics::OFFSET_K3);
+    MAYBE_SET_CONSTANT(BUNDLE_RADIAL_K4,       PackedIntrinsics::OFFSET_K4);
     MAYBE_SET_CONSTANT(BUNDLE_TANGENTIAL_P1,   PackedIntrinsics::OFFSET_P1);
     MAYBE_SET_CONSTANT(BUNDLE_TANGENTIAL_P2,   PackedIntrinsics::OFFSET_P2);
 #undef MAYBE_SET_CONSTANT
 
-    // Always set K3 and K4 constant, it's not used at the moment.
-    constant_intrinsics.push_back(PackedIntrinsics::OFFSET_K3);
-    constant_intrinsics.push_back(PackedIntrinsics::OFFSET_K4);
+    if (!constant_intrinsics.empty()) {
+      ceres::SubsetParameterization *subset_parameterization =
+        new ceres::SubsetParameterization(PackedIntrinsics::NUM_PARAMETERS,
+                                          constant_intrinsics);
 
-   // TODO(sergey): Mark all parameters which are not used by the distortion
-   // model as constant.
-
-    ceres::SubsetParameterization *subset_parameterization =
-      new ceres::SubsetParameterization(PackedIntrinsics::NUM_PARAMETERS,
-                                        constant_intrinsics);
-
-    problem.SetParameterization(intrinsics_block, subset_parameterization);
+      problem.SetParameterization(intrinsics_block, subset_parameterization);
+    }
   }
 
   // Configure the solver.
diff --git a/intern/libmv/libmv/simple_pipeline/bundle.h b/intern/libmv/libmv/simple_pipeline/bundle.h
index 781bd8476fe..5f420da0045 100644
--- a/intern/libmv/libmv/simple_pipeline/bundle.h
+++ b/intern/libmv/libmv/simple_pipeline/bundle.h
@@ -101,14 +101,22 @@ void EuclideanBundle(const Tracks &tracks,
 */
 enum BundleIntrinsics {
   BUNDLE_NO_INTRINSICS = 0,
-  BUNDLE_FOCAL_LENGTH = 1,
-  BUNDLE_PRINCIPAL_POINT = 2,
-  BUNDLE_RADIAL_K1 = 4,
-  BUNDLE_RADIAL_K2 = 8,
-  BUNDLE_RADIAL = 12,
-  BUNDLE_TANGENTIAL_P1 = 16,
-  BUNDLE_TANGENTIAL_P2 = 32,
-  BUNDLE_TANGENTIAL = 48,
+
+  BUNDLE_FOCAL_LENGTH = (1 << 0),
+  BUNDLE_PRINCIPAL_POINT = (1 << 1),
+
+  BUNDLE_RADIAL_K1 = (1 << 2),
+  BUNDLE_RADIAL_K2 = (1 << 3),
+  BUNDLE_RADIAL_K3 = (1 << 4),
+  BUNDLE_RADIAL_K4 = (1 << 5),
+  BUNDLE_RADIAL = (BUNDLE_RADIAL_K1 |
+                   BUNDLE_RADIAL_K2 |
+                   BUNDLE_RADIAL_K3 |
+                   BUNDLE_RADIAL_K4),
+
+  BUNDLE_TANGENTIAL_P1 = (1 << 6),
+  BUNDLE_TANGENTIAL_P2 = (1 << 7),
+  BUNDLE_TANGENTIAL = (BUNDLE_TANGENTIAL_P1 | BUNDLE_TANGENTIAL_P2),
 };
 enum BundleConstraints {
   BUNDLE_NO_CONSTRAINTS = 0,
diff --git a/intern/libmv/libmv/simple_pipeline/packed_intrinsics.cc b/intern/libmv/libmv/simple_pipeline/packed_intrinsics.cc
index 1888e18867e..73ad4faed9c 100644
--- a/intern/libmv/libmv/simple_pipeline/packed_intrinsics.cc
+++ b/intern/libmv/libmv/simple_pipeline/packed_intrinsics.cc
@@ -55,4 +55,8 @@ double PackedIntrinsics::GetParameter(int index) const {
   return parameters_.at(index);
 }
 
+bool PackedIntrinsics::IsParameterDefined(int offset) {
+  return known_parameters_.at(offset);
+}
+
 }  // namespace libmv
diff --git a/intern/libmv/libmv/simple_pipeline/packed_intrinsics.h b/intern/libmv/libmv/simple_pipeline/packed_intrinsics.h
index e551fe74800..cbea599fccd 100644
--- a/intern/libmv/libmv/simple_pipeline/packed_intrinsics.h
+++ b/intern/libmv/libmv/simple_pipeline/packed_intrinsics.h
@@ -86,6 +86,8 @@ class PackedIntrinsics {
   double* GetParametersBlock() { return parameters_.data(); }
   const double* GetParametersBlock() const { return parameters_.data(); }
 
+  bool IsParameterDefined(int offset);
+
  private:
   void SetParameter(int index, double value);
   double GetParameter(int index) const;
diff --git a/release/scripts/startup/bl_ui/space_clip.py b/release/scripts/startup/bl_ui/space_clip.py
index 67d3db7d871..7cf45e73a27 100644
--- a/release/scripts/startup/bl_ui/space_clip.py
+++ b/release/scripts/startup/bl_ui/space_clip.py
@@ -532,6 +532,7 @@ class CLIP_PT_tools_solve(CLIP_PT_tracking_panel, Panel):
         tracking = clip.tracking
         settings = tracking.settings
         tracking_object = tracking.objects.active
+        camera = clip.tracking.camera
 
         col = layout.column()
         col.prop(settings, "use_tripod_solver", text="Tripod")
@@ -548,9 +549,13 @@ class CLIP_PT_tools_solve(CLIP_PT_tracking_panel, Panel):
         col = layout.column(heading="Refine", align=True)
         col.active = tracking_object.is_camera
         col.prop(settings, "refine_intrinsics_focal_length", text="Focal Length")
-        col.prop(settings, "refine_intrinsics_principal_point", text="Principal Point")
-        col.prop(settings, "refine_intrinsics_k1", text="K1")
-        col.prop(settings, "refine_intrinsics_k2", text="K2")
+        col.prop(settings, "refine_intrinsics_principal_point", text="Optical Center")
+
+        col.prop(settings, "refine_intrinsics_radial_distortion", text="Radial Distortion")
+
+        row = col.row()
+        row.active = (camera.distortion_model == 'BROWN')
+        row.prop(settings, "refine_intrinsics_tangential_distortion", text="Tangential Distortion")
 
         col = layout.column(align=True)
         col.scale_y = 2.0
diff --git a/source/blender/blenkernel/intern/tracking.c b/source/blender/blenkernel/intern/tracking.c
index cb33610a93f..7f927a8838e 100644
--- a/source/blender/blenkernel/intern/tracking.c
+++ b/source/blender/blenkernel/intern/tracking.c
@@ -338,6 +338,7 @@ void BKE_tracking_settings_init(MovieTracking *tracking)
   tracking->settings.default_weight = 1.0f;
   tracking->settings.dist = 1;
   tracking->settings.object_distance = 1;
+  tracking->settings.refine_camera_intrinsics = REFINE_NO_INTRINSICS;
 
   tracking->stabilization.scaleinf = 1.0f;
   tracking->stabilization.anchor_frame = 1;
diff --git a/source/blender/blenkernel/intern/tracking_solver.c b/source/blender/blenkernel/intern/tracking_solver.c
index 7df8bf62b16..d89d36f85ea 100644
--- a/source/blender/blenkernel/intern/tracking_solver.c
+++ b/source/blender/blenkernel/intern/tracking_solver.c
@@ -291,12 +291,12 @@ static int reconstruct_refine_intrinsics_get_flags(MovieTracking *tracking,
     flags |= LIBMV_REFINE_PRINCIPAL_POINT;
   }
 
-  if (refine & REFINE_RADIAL_DISTORTION_K1) {
-    flags |= LIBMV_REFINE_RADIAL_DISTORTION_K1;
+  if (refine & REFINE_RADIAL_DISTORTION) {
+    flags |= LIBMV_REFINE_RADIAL_DISTORTION;
   }
 
-  if (refine & REFINE_RADIAL_DISTORTION_K2) {
-    flags |= LIBMV_REFINE_RADIAL_DISTORTION_K2;
+  if (refine & REFINE_TANGENTIAL_DISTORTION) {
+    flags |= LIBMV_REFINE_TANGENTIAL_DISTORTION;
   }
 
   return flags;
diff --git a/source/blender/blenloader/intern/versioning_290.c b/source/blender/blenloader/intern/versioning_290.c
index f6bd9a3d00a..c3b85a011db 100644
--- a/source/blender/blenloader/intern/versioning_290.c
+++ b/source/blender/blenloader/intern/versioning_290.c
@@ -44,6 +44,7 @@
 #include "DNA_rigidbody_types.h"
 #include "DNA_screen_types.h"
 #include "DNA_shader_fx_types.h"
+#include "DNA_tracking_types.h"
 #include "DNA_workspace_types.h"
 
 #include "BKE_animsys.h"
@@ -929,6 +930,35 @@ void blo_do_versions_290(FileData *fd, Library *UNUSED(lib), Main *bmain)
     }
   }
 
+  if (!MAIN_VERSION_ATLEAST(bmain, 292, 1)) {
+    {
+      const int LEGACY_REFINE_RADIAL_DISTORTION_K1 = (1 << 2);
+
+      LISTBASE_FOREACH (MovieClip *, clip, &bmain->movieclips) {
+        MovieTracking *tracking = &clip->tracking;
+        MovieTrackingSettings *settings = &tracking->settings;
+        int new_refine_camera_intrinsics = 0;
+
+        if (settings->refine_camera_intrinsics & REFINE_FOCAL_LENGTH) {
+          new_refine_camera_intrinsics |= REFINE_FOCAL_LENGTH;
+        }
+
+        if (settings->refine_camera_intrinsics & REFINE_PRINCIPAL_POINT) {
+          new_refine_camera_intrinsics |= REFINE_PRINCIPAL_POINT;
+        }
+
+        /* The end goal is to enable radial distorion refinement if either K1 or K2 were set for
+         * refinement. It is enough to only check for L1 it was not possible to refine K2 without
+         * K1. */
+        if (settings->refine_camera_intrinsics & LEGACY_REFINE_RADIAL_DISTORTION_K1) {
+          new_refine_camera_intrinsics |= REFINE_RADIAL_DISTORTION;
+        }
+
+        settings->refine_camera_intrinsics = new_refine_camera_intrinsics;
+      }
+    }
+  }
+
   /**
    * Versioning code until next subversion bump goes here.
    *
diff --git a/source/blender/makesdna/DNA_tracking_types.h b/source/blender/makesdna/DNA_tracking_types.h
index d0105cda1ea..0bbeabf130f 100644
--- a/source/blender/makesdna/DNA_tracking_types.h
+++ b/source/blender/makesdna/DNA_tracking_types.h
@@ -288,8 +288,7 @@ typedef struct MovieTrackingSettings {
   int reconstruction_flag;
 
   /* which camera intrinsics to refine. uses on the REFINE_* flags */
-  short refine_camera_intrinsics;
-  char _pad2[2];
+  int refine_camera_intrinsics;
 
   /* ** tool settings ** */
 
@@ -551,10 +550,12 @@ enum {
 
 /* MovieTrackingSettings->refine_camera_intrinsics */
 enum {
+  REFINE_NO_INTRINSICS = (0),
+
   REFINE_FOCAL_LENGTH = (1 << 0),
   REFINE_PRINCIPAL_POINT = (1 << 1),
-  REFINE_RADIAL_DISTORTION_K1 = (1 << 2),
-  REFINE_RADIAL_DISTORTION_K2 = (1 << 4),
+  REFINE_RADIAL_DISTORTION = (1 << 2),
+  REFINE_TANGENTIAL_DISTORTION = (1 << 3),
 };
 
 /* MovieTrackingStrabilization->flag */
diff --git a/source/blender/makesrna/intern/rna_tracking.c b/source/blender/makesrna/intern/rna_tracking.c
index 17a7ab63077..effea4d5c8f 100644
--- a/source/blender/makesrna/intern/rna_tracking.c
+++ b/source/blender/makesrna/intern/rna_tracking.c
@@ -924,19 +924,22 @@ static void rna_def_trackingSettings(BlenderRNA *brna)
   RNA_def_property_ui_text(
       prop, "Refine Principal Point", "Refine principal point during camera solving");
 
-  prop = RNA_def_property(srna, "refine_intrinsics_k1", PROP_BOOLEAN, PROP_NONE);
-  RNA_def_property_boolean_sdna(
-      prop, NULL, "refine_camera_intrinsics", REFINE_RADIAL_DISTORTION_K1);
+  prop = RNA_def_property(srna, "refine_intrinsics_radial_distortion", PROP_BOOLEAN, PROP_NONE);
+  RNA_def_property_boolean_sdna(prop, NULL, "refine_camera_intrinsics", REFINE_RADIAL_DISTORTION);
   RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
-  RNA_def_property_ui_text(
-      prop, "Refine K1", "Refine K1 coefficient of distortion model during camera solving");
+  RNA_def_property_ui_text(prop,
+                           "Refine Radial",
+                           "Refine radial coefficients of distortion model during camera solving");
 
-  prop = RNA_def_property(srna, "refine_intrinsics_k2", PROP_BOOLEAN, PROP_NONE);
+  prop = RNA_def_property(
+      srna, "refine_intrinsics_tangential_distortion", PROP_BOOLEAN, PROP_NONE);
   RNA_def_property_boolean_sdna(
-      prop, NULL, "refine_camera_intrinsics", REFINE_RADIAL_DISTORTION_K2);
+      prop, NULL, "refine_camera_intrinsics", REFINE_TANGENTIAL_DISTORTION);
   RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
   RNA_def_property_ui_text(
-      prop, "Refine K2", "Refine K2 coefficient of distortion model during camera solving");
+      prop,
+      "Refine Tangential",
+      "Refine tangential coefficients of distortion model during camera solving");
 
   /* tool settings */