Tracking: Support selecting tracks by their reprojection error curve

Quite straightforward after previous refactor: now speed and error curves
are handled in the same exact manner.

4 files changed, 206 insertions, 164 deletions

diff --git a/source/blender/editors/space_clip/clip_graph_draw.c b/source/blender/editors/space_clip/clip_graph_draw.c
index 55012462527..f9c72741979 100644
--- a/source/blender/editors/space_clip/clip_graph_draw.c
+++ b/source/blender/editors/space_clip/clip_graph_draw.c
@@ -48,6 +48,7 @@
 #include "clip_intern.h"  // own include
 
 typedef struct TrackMotionCurveUserData {
+  SpaceClip *sc;
   MovieTrackingTrack *act_track;
   bool sel;
   float xscale, yscale, hsize;
@@ -72,8 +73,13 @@ static void tracking_segment_start_cb(void *userdata,
                                       bool is_point)
 {
   TrackMotionCurveUserData *data = (TrackMotionCurveUserData *)userdata;
+  SpaceClip *sc = data->sc;
   float col[4] = {0.0f, 0.0f, 0.0f, 0.0f};
 
+  if (!clip_graph_value_visible(sc, value_source)) {
+    return;
+  }
+
   switch (value_source) {
     case CLIP_VALUE_SOURCE_SPEED_X:
       col[0] = 1.0f;
@@ -81,8 +87,9 @@ static void tracking_segment_start_cb(void *userdata,
     case CLIP_VALUE_SOURCE_SPEED_Y:
       col[1] = 1.0f;
       break;
-    default:
-      return;
+    case CLIP_VALUE_SOURCE_REPROJECTION_ERROR:
+      col[2] = 1.0f;
+      break;
   }
 
   if (track == data->act_track) {
@@ -105,9 +112,13 @@ static void tracking_segment_start_cb(void *userdata,
   }
 }
 
-static void tracking_segment_end_cb(void *UNUSED(userdata),
-                                    eClipCurveValueSource UNUSED(value_source))
+static void tracking_segment_end_cb(void *userdata, eClipCurveValueSource value_source)
 {
+  TrackMotionCurveUserData *data = (TrackMotionCurveUserData *)userdata;
+  SpaceClip *sc = data->sc;
+  if (!clip_graph_value_visible(sc, value_source)) {
+    return;
+  }
   immEnd();
 }
 
@@ -144,34 +155,39 @@ static void tracking_segment_knot_cb(void *userdata,
   }
 }
 
-static void draw_tracks_motion_curves(View2D *v2d, SpaceClip *sc, unsigned int pos)
+static void draw_tracks_motion_and_error_curves(View2D *v2d, SpaceClip *sc, unsigned int pos)
 {
   MovieClip *clip = ED_space_clip_get_clip(sc);
   MovieTracking *tracking = &clip->tracking;
   MovieTrackingTrack *act_track = BKE_tracking_track_get_active(tracking);
-  int width, height;
-  TrackMotionCurveUserData userdata;
+  const bool draw_knots = (sc->flag & SC_SHOW_GRAPH_TRACKS_MOTION) != 0;
 
+  int width, height;
   BKE_movieclip_get_size(clip, &sc->user, &width, &height);
-
   if (!width || !height) {
     return;
   }
 
-  /* non-selected knot handles */
+  TrackMotionCurveUserData userdata;
+  userdata.sc = sc;
   userdata.hsize = UI_GetThemeValuef(TH_HANDLE_VERTEX_SIZE);
   userdata.sel = false;
   userdata.act_track = act_track;
   userdata.pos = pos;
-  UI_view2d_scale_get(v2d, &userdata.xscale, &userdata.yscale);
-  clip_graph_tracking_values_iterate(sc,
-                                     (sc->flag & SC_SHOW_GRAPH_SEL_ONLY) != 0,
-                                     (sc->flag & SC_SHOW_GRAPH_HIDDEN) != 0,
-                                     &userdata,
-                                     tracking_segment_knot_cb,
-                                     NULL,
-                                     NULL);
-  /* draw graph lines */
+
+  /* Non-selected knot handles. */
+  if (draw_knots) {
+    UI_view2d_scale_get(v2d, &userdata.xscale, &userdata.yscale);
+    clip_graph_tracking_values_iterate(sc,
+                                       (sc->flag & SC_SHOW_GRAPH_SEL_ONLY) != 0,
+                                       (sc->flag & SC_SHOW_GRAPH_HIDDEN) != 0,
+                                       &userdata,
+                                       tracking_segment_knot_cb,
+                                       NULL,
+                                       NULL);
+  }
+
+  /* Draw graph lines. */
   GPU_blend(true);
   clip_graph_tracking_values_iterate(sc,
                                      (sc->flag & SC_SHOW_GRAPH_SEL_ONLY) != 0,
@@ -182,137 +198,17 @@ static void draw_tracks_motion_curves(View2D *v2d, SpaceClip *sc, unsigned int p
                                      tracking_segment_end_cb);
   GPU_blend(false);
 
-  /* selected knot handles on top of curves */
-  userdata.sel = true;
-  clip_graph_tracking_values_iterate(sc,
-                                     (sc->flag & SC_SHOW_GRAPH_SEL_ONLY) != 0,
-                                     (sc->flag & SC_SHOW_GRAPH_HIDDEN) != 0,
-                                     &userdata,
-                                     tracking_segment_knot_cb,
-                                     NULL,
-                                     NULL);
-}
-
-typedef struct TrackErrorCurveUserData {
-  MovieClip *clip;
-  MovieTracking *tracking;
-  MovieTrackingObject *tracking_object;
-  MovieTrackingTrack *active_track;
-  bool matrix_initialized;
-  int matrix_frame;
-  float projection_matrix[4][4];
-  int width, height;
-  float aspy;
-  unsigned int pos;
-} TrackErrorCurveUserData;
-
-static void tracking_error_segment_point_cb(void *userdata,
-                                            MovieTrackingTrack *track,
-                                            MovieTrackingMarker *marker,
-                                            eClipCurveValueSource value_source,
-                                            int scene_framenr,
-                                            float UNUSED(value))
-{
-  if (value_source == CLIP_VALUE_SOURCE_SPEED_Y) {
-    TrackErrorCurveUserData *data = (TrackErrorCurveUserData *)userdata;
-    float reprojected_position[4], bundle_position[4], marker_position[2], delta[2];
-    float reprojection_error;
-    float weight = BKE_tracking_track_get_weight_for_marker(data->clip, track, marker);
-
-    if (!data->matrix_initialized || data->matrix_frame != scene_framenr) {
-      BKE_tracking_get_projection_matrix(data->tracking,
-                                         data->tracking_object,
-                                         scene_framenr,
-                                         data->width,
-                                         data->height,
-                                         data->projection_matrix);
-    }
-
-    copy_v3_v3(bundle_position, track->bundle_pos);
-    bundle_position[3] = 1;
-
-    mul_v4_m4v4(reprojected_position, data->projection_matrix, bundle_position);
-    reprojected_position[0] = (reprojected_position[0] / (reprojected_position[3] * 2.0f) + 0.5f) *
-                              data->width;
-    reprojected_position[1] = (reprojected_position[1] / (reprojected_position[3] * 2.0f) + 0.5f) *
-                              data->height * data->aspy;
-
-    BKE_tracking_distort_v2(data->tracking, reprojected_position, reprojected_position);
-
-    marker_position[0] = (marker->pos[0] + track->offset[0]) * data->width;
-    marker_position[1] = (marker->pos[1] + track->offset[1]) * data->height * data->aspy;
-
-    sub_v2_v2v2(delta, reprojected_position, marker_position);
-    reprojection_error = len_v2(delta) * weight;
-
-    immVertex2f(data->pos, scene_framenr, reprojection_error);
-  }
-}
-
-static void tracking_error_segment_start_cb(void *userdata,
-                                            MovieTrackingTrack *track,
-                                            eClipCurveValueSource value_source,
-                                            bool is_point)
-{
-  if (value_source == CLIP_VALUE_SOURCE_SPEED_Y) {
-    TrackErrorCurveUserData *data = (TrackErrorCurveUserData *)userdata;
-    float col[4] = {0.0f, 0.0f, 1.0f, 1.0f};
-
-    if (track == data->active_track) {
-      col[3] = 1.0f;
-      GPU_line_width(2.0f);
-    }
-    else {
-      col[3] = 0.5f;
-      GPU_line_width(1.0f);
-    }
-
-    immUniformColor4fv(col);
-
-    if (is_point) { /* This probably never happens here, but just in case... */
-      immBeginAtMost(GPU_PRIM_POINTS, 1);
-    }
-    else {
-      /* Graph can be composed of smaller segments, if any marker is disabled */
-      immBeginAtMost(GPU_PRIM_LINE_STRIP, track->markersnr);
-    }
-  }
-}
-
-static void tracking_error_segment_end_cb(void *UNUSED(userdata),
-                                          eClipCurveValueSource value_source)
-{
-  if (value_source == CLIP_VALUE_SOURCE_SPEED_Y) {
-    immEnd();
-  }
-}
-
-static void draw_tracks_error_curves(SpaceClip *sc, unsigned int pos)
-{
-  MovieClip *clip = ED_space_clip_get_clip(sc);
-  MovieTracking *tracking = &clip->tracking;
-  TrackErrorCurveUserData data;
-
-  data.clip = clip;
-  data.tracking = tracking;
-  data.tracking_object = BKE_tracking_object_get_active(tracking);
-  data.active_track = BKE_tracking_track_get_active(tracking);
-  data.matrix_initialized = false;
-  data.pos = pos;
-  BKE_movieclip_get_size(clip, &sc->user, &data.width, &data.height);
-  data.aspy = 1.0f / tracking->camera.pixel_aspect;
-
-  if (!data.width || !data.height) {
-    return;
+  /* Selected knot handles on top of curves. */
+  if (draw_knots) {
+    userdata.sel = true;
+    clip_graph_tracking_values_iterate(sc,
+                                       (sc->flag & SC_SHOW_GRAPH_SEL_ONLY) != 0,
+                                       (sc->flag & SC_SHOW_GRAPH_HIDDEN) != 0,
+                                       &userdata,
+                                       tracking_segment_knot_cb,
+                                       NULL,
+                                       NULL);
   }
-
-  clip_graph_tracking_values_iterate(sc,
-                                     (sc->flag & SC_SHOW_GRAPH_SEL_ONLY) != 0,
-                                     (sc->flag & SC_SHOW_GRAPH_HIDDEN) != 0,
-                                     &data,
-                                     tracking_error_segment_point_cb,
-                                     tracking_error_segment_start_cb,
-                                     tracking_error_segment_end_cb);
 }
 
 static void draw_frame_curves(SpaceClip *sc, unsigned int pos)
@@ -364,12 +260,8 @@ void clip_draw_graph(SpaceClip *sc, ARegion *ar, Scene *scene)
 
     GPU_point_size(3.0f);
 
-    if (sc->flag & SC_SHOW_GRAPH_TRACKS_MOTION) {
-      draw_tracks_motion_curves(v2d, sc, pos);
-    }
-
-    if (sc->flag & SC_SHOW_GRAPH_TRACKS_ERROR) {
-      draw_tracks_error_curves(sc, pos);
+    if (sc->flag & (SC_SHOW_GRAPH_TRACKS_MOTION | SC_SHOW_GRAPH_TRACKS_ERROR)) {
+      draw_tracks_motion_and_error_curves(v2d, sc, pos);
     }
 
     if (sc->flag & SC_SHOW_GRAPH_FRAMES) {
diff --git a/source/blender/editors/space_clip/clip_graph_ops.c b/source/blender/editors/space_clip/clip_graph_ops.c
index 1ebe56fccb6..58a00d2e6b9 100644
--- a/source/blender/editors/space_clip/clip_graph_ops.c
+++ b/source/blender/editors/space_clip/clip_graph_ops.c
@@ -64,7 +64,7 @@ static bool clip_graph_knots_poll(bContext *C)
   if (ED_space_clip_graph_poll(C)) {
     SpaceClip *sc = CTX_wm_space_clip(C);
 
-    return (sc->flag & SC_SHOW_GRAPH_TRACKS_MOTION) != 0;
+    return (sc->flag & (SC_SHOW_GRAPH_TRACKS_MOTION | SC_SHOW_GRAPH_TRACKS_ERROR)) != 0;
   }
   return false;
 }
@@ -93,6 +93,7 @@ static void toggle_selection_cb(void *userdata, MovieTrackingMarker *marker)
 /******************** mouse select operator ********************/
 
 typedef struct {
+  SpaceClip *sc;
   eClipCurveValueSource value_source;
   bool has_prev; /* if there's valid coordinate of previous point of curve segment */
 
@@ -115,6 +116,10 @@ static void find_nearest_tracking_segment_cb(void *userdata,
   MouseSelectUserData *data = userdata;
   float co[2] = {scene_framenr, val};
 
+  if (!clip_graph_value_visible(data->sc, value_source)) {
+    return;
+  }
+
   if (data->has_prev) {
     float dist_sq = dist_squared_to_line_segment_v2(data->mouse_co, data->prev_co, co);
 
@@ -149,6 +154,10 @@ static void find_nearest_tracking_knot_cb(void *userdata,
   float mdiff[2] = {scene_framenr - data->mouse_co[0], val - data->mouse_co[1]};
   float dist_sq = len_squared_v2(mdiff);
 
+  if (!clip_graph_value_visible(data->sc, value_source)) {
+    return;
+  }
+
   if (data->marker == NULL || dist_sq < data->min_dist_sq) {
     float co[2] = {scene_framenr, val};
 
@@ -160,9 +169,11 @@ static void find_nearest_tracking_knot_cb(void *userdata,
   }
 }
 
-static void mouse_select_init_data(MouseSelectUserData *userdata, const float co[2])
+static void mouse_select_init_data(bContext *C, MouseSelectUserData *userdata, const float co[2])
 {
+  SpaceClip *sc = CTX_wm_space_clip(C);
   memset(userdata, 0, sizeof(MouseSelectUserData));
+  userdata->sc = sc;
   userdata->min_dist_sq = FLT_MAX;
   copy_v2_v2(userdata->mouse_co, co);
 }
@@ -180,7 +191,7 @@ static bool mouse_select_knot(bContext *C, float co[2], bool extend)
   if (act_track) {
     MouseSelectUserData userdata;
 
-    mouse_select_init_data(&userdata, co);
+    mouse_select_init_data(C, &userdata, co);
     clip_graph_tracking_values_iterate_track(
         sc, act_track, &userdata, find_nearest_tracking_knot_cb, NULL, NULL);
 
@@ -233,7 +244,7 @@ static bool mouse_select_curve(bContext *C, float co[2], bool extend)
   MovieTrackingTrack *act_track = BKE_tracking_track_get_active(tracking);
   MouseSelectUserData userdata;
 
-  mouse_select_init_data(&userdata, co);
+  mouse_select_init_data(C, &userdata, co);
   clip_graph_tracking_values_iterate(sc,
                                      (sc->flag & SC_SHOW_GRAPH_SEL_ONLY) != 0,
                                      (sc->flag & SC_SHOW_GRAPH_HIDDEN) != 0,
diff --git a/source/blender/editors/space_clip/clip_intern.h b/source/blender/editors/space_clip/clip_intern.h
index 3f8ac2bb7c2..8599de9f16f 100644
--- a/source/blender/editors/space_clip/clip_intern.h
+++ b/source/blender/editors/space_clip/clip_intern.h
@@ -134,6 +134,8 @@ typedef void (*ClipTrackValueSegmentStartCallback)(void *userdata,
 typedef void (*ClipTrackValueSegmentEndCallback)(void *userdata,
                                                  eClipCurveValueSource value_source);
 
+bool clip_graph_value_visible(struct SpaceClip *sc, eClipCurveValueSource value_source);
+
 void clip_graph_tracking_values_iterate_track(struct SpaceClip *sc,
                                               struct MovieTrackingTrack *track,
                                               void *userdata,
diff --git a/source/blender/editors/space_clip/clip_utils.c b/source/blender/editors/space_clip/clip_utils.c
index 94badf7fdd6..48f788e2e3a 100644
--- a/source/blender/editors/space_clip/clip_utils.c
+++ b/source/blender/editors/space_clip/clip_utils.c
@@ -27,6 +27,7 @@
 
 #include "BLI_utildefines.h"
 #include "BLI_listbase.h"
+#include "BLI_math.h"
 
 #include "BKE_animsys.h"
 #include "BKE_context.h"
@@ -52,12 +53,28 @@
 
 #include "clip_intern.h"  // own include
 
-void clip_graph_tracking_values_iterate_track(SpaceClip *sc,
-                                              MovieTrackingTrack *track,
-                                              void *userdata,
-                                              ClipTrackValueCallback func,
-                                              ClipTrackValueSegmentStartCallback segment_start,
-                                              ClipTrackValueSegmentEndCallback segment_end)
+bool clip_graph_value_visible(SpaceClip *sc, eClipCurveValueSource value_source)
+{
+  if (ELEM(value_source, CLIP_VALUE_SOURCE_SPEED_X, CLIP_VALUE_SOURCE_SPEED_Y)) {
+    if ((sc->flag & SC_SHOW_GRAPH_TRACKS_MOTION) == 0) {
+      return false;
+    }
+  }
+  else if (value_source == CLIP_VALUE_SOURCE_REPROJECTION_ERROR) {
+    if ((sc->flag & SC_SHOW_GRAPH_TRACKS_ERROR) == 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+static void clip_graph_tracking_values_iterate_track_speed_values(
+    SpaceClip *sc,
+    MovieTrackingTrack *track,
+    void *userdata,
+    ClipTrackValueCallback func,
+    ClipTrackValueSegmentStartCallback segment_start,
+    ClipTrackValueSegmentEndCallback segment_end)
 {
   MovieClip *clip = ED_space_clip_get_clip(sc);
   int width, height, coord;
@@ -124,6 +141,126 @@ void clip_graph_tracking_values_iterate_track(SpaceClip *sc,
   }
 }
 
+static float calculate_reprojection_error_at_marker(MovieClip *clip,
+                                                    MovieTracking *tracking,
+                                                    MovieTrackingObject *tracking_object,
+                                                    MovieTrackingTrack *track,
+                                                    MovieTrackingMarker *marker,
+                                                    const int clip_width,
+                                                    const int clip_height,
+                                                    const int scene_framenr)
+{
+  float reprojected_position[4], bundle_position[4], marker_position[2], delta[2];
+  float weight = BKE_tracking_track_get_weight_for_marker(clip, track, marker);
+  const float aspy = 1.0f / tracking->camera.pixel_aspect;
+
+  float projection_matrix[4][4];
+  BKE_tracking_get_projection_matrix(
+      tracking, tracking_object, scene_framenr, clip_width, clip_height, projection_matrix);
+
+  copy_v3_v3(bundle_position, track->bundle_pos);
+  bundle_position[3] = 1;
+
+  mul_v4_m4v4(reprojected_position, projection_matrix, bundle_position);
+  reprojected_position[0] = (reprojected_position[0] / (reprojected_position[3] * 2.0f) + 0.5f) *
+                            clip_width;
+  reprojected_position[1] = (reprojected_position[1] / (reprojected_position[3] * 2.0f) + 0.5f) *
+                            clip_height * aspy;
+
+  BKE_tracking_distort_v2(tracking, reprojected_position, reprojected_position);
+
+  marker_position[0] = (marker->pos[0] + track->offset[0]) * clip_width;
+  marker_position[1] = (marker->pos[1] + track->offset[1]) * clip_height * aspy;
+
+  sub_v2_v2v2(delta, reprojected_position, marker_position);
+  return len_v2(delta) * weight;
+}
+
+static void clip_graph_tracking_values_iterate_track_reprojection_error_values(
+    SpaceClip *sc,
+    MovieTrackingTrack *track,
+    void *userdata,
+    ClipTrackValueCallback func,
+    ClipTrackValueSegmentStartCallback segment_start,
+    ClipTrackValueSegmentEndCallback segment_end)
+{
+  /* Tracks without bundle can not have any reprojection error curve. */
+  if ((track->flag & TRACK_HAS_BUNDLE) == 0) {
+    return;
+  }
+
+  MovieClip *clip = ED_space_clip_get_clip(sc);
+  MovieTracking *tracking = &clip->tracking;
+  MovieTrackingObject *tracking_object = BKE_tracking_object_get_active(tracking);
+
+  int clip_width, clip_height;
+  BKE_movieclip_get_size(clip, &sc->user, &clip_width, &clip_height);
+
+  /* Iterate over segments. */
+  bool is_segment_open = false;
+  for (int marker_index = 0; marker_index < track->markersnr; marker_index++) {
+    MovieTrackingMarker *marker = &track->markers[marker_index];
+
+    /* End of tracked segment, no reprojection error can be calculated here since the ground truth
+     * 2D position is not known. */
+    if (marker->flag & MARKER_DISABLED) {
+      if (is_segment_open) {
+        if (segment_end != NULL) {
+          segment_end(userdata, CLIP_VALUE_SOURCE_REPROJECTION_ERROR);
+        }
+        is_segment_open = false;
+      }
+      continue;
+    }
+
+    /* Begin new segment if it is not open yet. */
+    if (!is_segment_open) {
+      if (segment_start != NULL) {
+        if ((marker_index + 1) == track->markersnr) {
+          segment_start(userdata, track, CLIP_VALUE_SOURCE_REPROJECTION_ERROR, true);
+        }
+        else {
+          segment_start(userdata,
+                        track,
+                        CLIP_VALUE_SOURCE_REPROJECTION_ERROR,
+                        (track->markers[marker_index + 1].flag & MARKER_DISABLED));
+        }
+      }
+      is_segment_open = true;
+    }
+
+    if (func != NULL) {
+      const int scene_framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
+      const float reprojection_error = calculate_reprojection_error_at_marker(
+          clip, tracking, tracking_object, track, marker, clip_width, clip_height, scene_framenr);
+      func(userdata,
+           track,
+           marker,
+           CLIP_VALUE_SOURCE_REPROJECTION_ERROR,
+           scene_framenr,
+           reprojection_error);
+    }
+  }
+
+  if (is_segment_open && segment_end != NULL) {
+    segment_end(userdata, CLIP_VALUE_SOURCE_REPROJECTION_ERROR);
+  }
+}
+
+void clip_graph_tracking_values_iterate_track(SpaceClip *sc,
+                                              MovieTrackingTrack *track,
+                                              void *userdata,
+                                              ClipTrackValueCallback func,
+                                              ClipTrackValueSegmentStartCallback segment_start,
+                                              ClipTrackValueSegmentEndCallback segment_end)
+{
+  clip_graph_tracking_values_iterate_track_speed_values(
+      sc, track, userdata, func, segment_start, segment_end);
+
+  clip_graph_tracking_values_iterate_track_reprojection_error_values(
+      sc, track, userdata, func, segment_start, segment_end);
+}
+
 void clip_graph_tracking_values_iterate(SpaceClip *sc,
                                         bool selected_only,
                                         bool include_hidden,