Cleanup: Rearrange Vert and Edge slide functions

And make `trans_mesh_customdata_correction` a more generic utility.

6 files changed, 128 insertions, 173 deletions

diff --git a/source/blender/editors/transform/transform.c b/source/blender/editors/transform/transform.c
index d8503a28774..d7da0cfba5c 100644
--- a/source/blender/editors/transform/transform.c
+++ b/source/blender/editors/transform/transform.c
@@ -2088,14 +2088,6 @@ int transformEnd(bContext *C, TransInfo *t)
   if (t->state != TRANS_STARTING && t->state != TRANS_RUNNING) {
     /* handle restoring objects */
     if (t->state == TRANS_CANCEL) {
-      /* exception, edge slide transformed UVs too */
-      if (t->mode == TFM_EDGE_SLIDE) {
-        doEdgeSlide(t, 0.0f);
-      }
-      else if (t->mode == TFM_VERT_SLIDE) {
-        doVertSlide(t, 0.0f);
-      }
-
       exit_code = OPERATOR_CANCELLED;
       restoreTransObjects(t);  // calls recalcData()
     }
diff --git a/source/blender/editors/transform/transform_convert.h b/source/blender/editors/transform/transform_convert.h
index d130da199b8..a94bd609d94 100644
--- a/source/blender/editors/transform/transform_convert.h
+++ b/source/blender/editors/transform/transform_convert.h
@@ -47,8 +47,7 @@ bool clipUVTransform(TransInfo *t, float vec[2], const bool resize);
 void clipUVData(TransInfo *t);
 
 /* transform_convert_mesh.c */
-void trans_mesh_customdata_correction_init(TransInfo *t);
-void trans_mesh_customdata_correction_apply(struct TransDataContainer *tc, bool is_final);
+void trans_mesh_customdata_correction_init(TransInfo *t, struct TransDataContainer *tc);
 
 /* transform_convert_sequencer.c */
 int transform_convert_sequencer_get_snap_bound(TransInfo *t);
diff --git a/source/blender/editors/transform/transform_convert_mesh.c b/source/blender/editors/transform/transform_convert_mesh.c
index 598604eac0d..e273c23e675 100644
--- a/source/blender/editors/transform/transform_convert_mesh.c
+++ b/source/blender/editors/transform/transform_convert_mesh.c
@@ -1070,105 +1070,99 @@ static void create_trans_vert_customdata_layer(BMVert *v,
   BLI_ghash_insert(tcld->origverts, v, r_tcld_vert);
 }
 
-void trans_mesh_customdata_correction_init(TransInfo *t)
+void trans_mesh_customdata_correction_init(TransInfo *t, TransDataContainer *tc)
 {
-  FOREACH_TRANS_DATA_CONTAINER (t, tc) {
-    if (tc->custom.type.data) {
-      if (tc->custom.type.free_cb == trans_mesh_customdata_free_cb) {
-        /* Custom data correction has initiated before. */
-        continue;
-      }
-      else {
-        BLI_assert(false);
-      }
-    }
-    int i;
+  if (tc->custom.type.data) {
+    /* Custom data correction has initiated before. */
+    BLI_assert(tc->custom.type.free_cb == trans_mesh_customdata_free_cb);
+    return;
+  }
+  int i;
 
-    BMEditMesh *em = BKE_editmesh_from_object(tc->obedit);
-    BMesh *bm = em->bm;
+  BMEditMesh *em = BKE_editmesh_from_object(tc->obedit);
+  BMesh *bm = em->bm;
 
-    bool use_origfaces;
-    int cd_loop_mdisp_offset;
-    {
-      const bool has_layer_math = CustomData_has_math(&bm->ldata);
-      cd_loop_mdisp_offset = CustomData_get_offset(&bm->ldata, CD_MDISPS);
-      if ((t->settings->uvcalc_flag & UVCALC_TRANSFORM_CORRECT) &&
-          /* don't do this at all for non-basis shape keys, too easy to
-           * accidentally break uv maps or vertex colors then */
-          (bm->shapenr <= 1) && (has_layer_math || (cd_loop_mdisp_offset != -1))) {
-        use_origfaces = true;
-      }
-      else {
-        use_origfaces = false;
-        cd_loop_mdisp_offset = -1;
-      }
+  bool use_origfaces;
+  int cd_loop_mdisp_offset;
+  {
+    const bool has_layer_math = CustomData_has_math(&bm->ldata);
+    cd_loop_mdisp_offset = CustomData_get_offset(&bm->ldata, CD_MDISPS);
+    if ((t->settings->uvcalc_flag & UVCALC_TRANSFORM_CORRECT) &&
+        /* don't do this at all for non-basis shape keys, too easy to
+         * accidentally break uv maps or vertex colors then */
+        (bm->shapenr <= 1) && (has_layer_math || (cd_loop_mdisp_offset != -1))) {
+      use_origfaces = true;
     }
+    else {
+      use_origfaces = false;
+      cd_loop_mdisp_offset = -1;
+    }
+  }
 
-    if (use_origfaces) {
-      /* create copies of faces for customdata projection */
-      bmesh_edit_begin(bm, BMO_OPTYPE_FLAG_UNTAN_MULTIRES);
+  if (use_origfaces) {
+    /* create copies of faces for customdata projection */
+    bmesh_edit_begin(bm, BMO_OPTYPE_FLAG_UNTAN_MULTIRES);
 
-      struct GHash *origfaces = BLI_ghash_ptr_new(__func__);
-      struct BMesh *bm_origfaces = BM_mesh_create(&bm_mesh_allocsize_default,
-                                                  &((struct BMeshCreateParams){
-                                                      .use_toolflags = false,
-                                                  }));
+    struct GHash *origfaces = BLI_ghash_ptr_new(__func__);
+    struct BMesh *bm_origfaces = BM_mesh_create(&bm_mesh_allocsize_default,
+                                                &((struct BMeshCreateParams){
+                                                    .use_toolflags = false,
+                                                }));
 
-      /* we need to have matching customdata */
-      BM_mesh_copy_init_customdata(bm_origfaces, bm, NULL);
+    /* we need to have matching customdata */
+    BM_mesh_copy_init_customdata(bm_origfaces, bm, NULL);
 
-      int *layer_math_map = NULL;
-      int layer_index_dst = 0;
-      {
-        /* TODO: We don't need `sod->layer_math_map` when there are no loops linked
-         * to one of the sliding vertices. */
-        if (CustomData_has_math(&bm->ldata)) {
-          /* over alloc, only 'math' layers are indexed */
-          layer_math_map = MEM_mallocN(bm->ldata.totlayer * sizeof(int), __func__);
-          for (i = 0; i < bm->ldata.totlayer; i++) {
-            if (CustomData_layer_has_math(&bm->ldata, i)) {
-              layer_math_map[layer_index_dst++] = i;
-            }
+    int *layer_math_map = NULL;
+    int layer_index_dst = 0;
+    {
+      /* TODO: We don't need `sod->layer_math_map` when there are no loops linked
+       * to one of the sliding vertices. */
+      if (CustomData_has_math(&bm->ldata)) {
+        /* over alloc, only 'math' layers are indexed */
+        layer_math_map = MEM_mallocN(bm->ldata.totlayer * sizeof(int), __func__);
+        for (i = 0; i < bm->ldata.totlayer; i++) {
+          if (CustomData_layer_has_math(&bm->ldata, i)) {
+            layer_math_map[layer_index_dst++] = i;
           }
-          BLI_assert(layer_index_dst != 0);
         }
+        BLI_assert(layer_index_dst != 0);
       }
+    }
 
-      struct TransCustomDataLayer *tcld;
-      tc->custom.type.data = tcld = MEM_mallocN(sizeof(*tcld), __func__);
-      tc->custom.type.free_cb = trans_mesh_customdata_free_cb;
-
-      tcld->bm = bm;
-      tcld->origfaces = origfaces;
-      tcld->bm_origfaces = bm_origfaces;
-      tcld->cd_loop_mdisp_offset = cd_loop_mdisp_offset;
-      tcld->layer_math_map = layer_math_map;
-      tcld->layer_math_map_num = layer_index_dst;
-      tcld->arena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
-
-      int data_len = tc->data_len + tc->data_mirror_len;
-      struct GHash *origverts = BLI_ghash_ptr_new_ex(__func__, data_len);
-      tcld->origverts = origverts;
-
-      struct TransCustomDataLayerVert *tcld_vert, *tcld_vert_iter;
-      tcld_vert = BLI_memarena_alloc(tcld->arena, data_len * sizeof(*tcld_vert));
-      tcld_vert_iter = &tcld_vert[0];
-
-      TransData *tob;
-      for (i = tc->data_len, tob = tc->data; i--; tob++, tcld_vert_iter++) {
-        BMVert *v = tob->extra;
-        create_trans_vert_customdata_layer(v, tcld, tcld_vert_iter);
-      }
-
-      TransDataMirror *td_mirror = tc->data_mirror;
-      for (i = tc->data_mirror_len; i--; td_mirror++, tcld_vert_iter++) {
-        BMVert *v = td_mirror->extra;
-        create_trans_vert_customdata_layer(v, tcld, tcld_vert_iter);
-      }
+    struct TransCustomDataLayer *tcld;
+    tc->custom.type.data = tcld = MEM_mallocN(sizeof(*tcld), __func__);
+    tc->custom.type.free_cb = trans_mesh_customdata_free_cb;
+
+    tcld->bm = bm;
+    tcld->origfaces = origfaces;
+    tcld->bm_origfaces = bm_origfaces;
+    tcld->cd_loop_mdisp_offset = cd_loop_mdisp_offset;
+    tcld->layer_math_map = layer_math_map;
+    tcld->layer_math_map_num = layer_index_dst;
+    tcld->arena = BLI_memarena_new(BLI_MEMARENA_STD_BUFSIZE, __func__);
+
+    int data_len = tc->data_len + tc->data_mirror_len;
+    struct GHash *origverts = BLI_ghash_ptr_new_ex(__func__, data_len);
+    tcld->origverts = origverts;
+
+    struct TransCustomDataLayerVert *tcld_vert, *tcld_vert_iter;
+    tcld_vert = BLI_memarena_alloc(tcld->arena, data_len * sizeof(*tcld_vert));
+    tcld_vert_iter = &tcld_vert[0];
+
+    TransData *tob;
+    for (i = tc->data_len, tob = tc->data; i--; tob++, tcld_vert_iter++) {
+      BMVert *v = tob->extra;
+      create_trans_vert_customdata_layer(v, tcld, tcld_vert_iter);
+    }
 
-      tcld->data = tcld_vert;
-      tcld->data_len = data_len;
+    TransDataMirror *td_mirror = tc->data_mirror;
+    for (i = tc->data_mirror_len; i--; td_mirror++, tcld_vert_iter++) {
+      BMVert *v = td_mirror->extra;
+      create_trans_vert_customdata_layer(v, tcld, tcld_vert_iter);
     }
+
+    tcld->data = tcld_vert;
+    tcld->data_len = data_len;
   }
 }
 
@@ -1313,7 +1307,7 @@ static void trans_mesh_customdata_correction_apply_vert(struct TransCustomDataLa
   }
 }
 
-void trans_mesh_customdata_correction_apply(struct TransDataContainer *tc, bool is_final)
+static void trans_mesh_customdata_correction_apply(struct TransDataContainer *tc, bool is_final)
 {
   struct TransCustomDataLayer *tcld = tc->custom.type.data;
   if (!tcld) {
@@ -1387,11 +1381,10 @@ void recalcData_mesh(TransInfo *t)
     }
   }
 
-  if (t->mode == TFM_EDGE_SLIDE) {
-    projectEdgeSlideData(t, false);
-  }
-  else if (t->mode == TFM_VERT_SLIDE) {
-    projectVertSlideData(t, false);
+  if (ELEM(t->mode, TFM_EDGE_SLIDE, TFM_VERT_SLIDE)) {
+    FOREACH_TRANS_DATA_CONTAINER (t, tc) {
+      trans_mesh_customdata_correction_apply(tc, false);
+    }
   }
 
   FOREACH_TRANS_DATA_CONTAINER (t, tc) {
@@ -1410,15 +1403,24 @@ void recalcData_mesh(TransInfo *t)
 void special_aftertrans_update__mesh(bContext *UNUSED(C), TransInfo *t)
 {
   const bool canceled = (t->state == TRANS_CANCEL);
-  if (t->mode == TFM_EDGE_SLIDE) {
-    /* handle multires re-projection, done
-     * on transform completion since it's
-     * really slow -joeedh */
-    projectEdgeSlideData(t, !canceled);
+
+  if (canceled) {
+    /* Exception, edge slide transformed UVs too. */
+    if (t->mode == TFM_EDGE_SLIDE) {
+      doEdgeSlide(t, 0.0f);
+    }
+    else if (t->mode == TFM_VERT_SLIDE) {
+      doVertSlide(t, 0.0f);
+    }
   }
-  else if (t->mode == TFM_VERT_SLIDE) {
-    /* as above */
-    projectVertSlideData(t, !canceled);
+
+  if (ELEM(t->mode, TFM_EDGE_SLIDE, TFM_VERT_SLIDE)) {
+    /* Handle multires re-projection, done
+     * on transform completion since it's
+     * really slow -joeedh. */
+    FOREACH_TRANS_DATA_CONTAINER (t, tc) {
+      trans_mesh_customdata_correction_apply(tc, !canceled);
+    }
   }
 
   bool use_automerge = !canceled && (t->flag & (T_AUTOMERGE | T_AUTOSPLIT)) != 0;
diff --git a/source/blender/editors/transform/transform_mode.h b/source/blender/editors/transform/transform_mode.h
index 074e89390c2..464deff983b 100644
--- a/source/blender/editors/transform/transform_mode.h
+++ b/source/blender/editors/transform/transform_mode.h
@@ -94,7 +94,6 @@ void initNormalRotation(TransInfo *t);
 void initSeqSlide(TransInfo *t);
 
 /* transform_mode_edge_slide.c */
-void projectEdgeSlideData(TransInfo *t, bool is_final);
 void drawEdgeSlide(TransInfo *t);
 void doEdgeSlide(TransInfo *t, float perc);
 void initEdgeSlide_ex(
@@ -153,7 +152,6 @@ void initTrackball(TransInfo *t);
 void initTranslation(TransInfo *t);
 
 /* transform_mode_vert_slide.c */
-void projectVertSlideData(TransInfo *t, bool is_final);
 void drawVertSlide(TransInfo *t);
 void doVertSlide(TransInfo *t, float perc);
 void initVertSlide_ex(TransInfo *t, bool use_even, bool flipped, bool use_clamp);
diff --git a/source/blender/editors/transform/transform_mode_edge_slide.c b/source/blender/editors/transform/transform_mode_edge_slide.c
index 9f9e5742ffe..42f75f041b9 100644
--- a/source/blender/editors/transform/transform_mode_edge_slide.c
+++ b/source/blender/editors/transform/transform_mode_edge_slide.c
@@ -519,7 +519,7 @@ static void calcEdgeSlide_even(TransInfo *t,
   }
 }
 
-static bool createEdgeSlideVerts_double_side(TransInfo *t, TransDataContainer *tc)
+static EdgeSlideData *createEdgeSlideVerts_double_side(TransInfo *t, TransDataContainer *tc)
 {
   BMEditMesh *em = BKE_editmesh_from_object(tc->obedit);
   BMesh *bm = em->bm;
@@ -554,8 +554,9 @@ static bool createEdgeSlideVerts_double_side(TransInfo *t, TransDataContainer *t
       }
 
       if (numsel == 0 || numsel > 2) {
+        /* Invalid edge selection. */
         MEM_freeN(sld);
-        return false; /* invalid edge selection */
+        return NULL;
       }
     }
   }
@@ -566,7 +567,7 @@ static bool createEdgeSlideVerts_double_side(TransInfo *t, TransDataContainer *t
       if (!BM_edge_is_manifold(e) && !BM_edge_is_boundary(e)) {
         /* can edges with at least once face user */
         MEM_freeN(sld);
-        return false;
+        return NULL;
       }
     }
   }
@@ -595,7 +596,7 @@ static bool createEdgeSlideVerts_double_side(TransInfo *t, TransDataContainer *t
     if (!j) {
       MEM_freeN(sld);
       MEM_freeN(sv_table);
-      return false;
+      return NULL;
     }
     sv_tot = j;
   }
@@ -870,18 +871,16 @@ static bool createEdgeSlideVerts_double_side(TransInfo *t, TransDataContainer *t
     calcEdgeSlide_even(t, tc, sld, mval);
   }
 
-  tc->custom.mode.data = sld;
-
   MEM_freeN(sv_table);
 
-  return true;
+  return sld;
 }
 
 /**
  * A simple version of #createEdgeSlideVerts_double_side
  * Which assumes the longest unselected.
  */
-static bool createEdgeSlideVerts_single_side(TransInfo *t, TransDataContainer *tc)
+static EdgeSlideData *createEdgeSlideVerts_single_side(TransInfo *t, TransDataContainer *tc)
 {
   BMEditMesh *em = BKE_editmesh_from_object(tc->obedit);
   BMesh *bm = em->bm;
@@ -933,7 +932,7 @@ static bool createEdgeSlideVerts_single_side(TransInfo *t, TransDataContainer *t
 
     if (!j) {
       MEM_freeN(sld);
-      return false;
+      return NULL;
     }
 
     sv_tot = j;
@@ -1055,24 +1054,9 @@ static bool createEdgeSlideVerts_single_side(TransInfo *t, TransDataContainer *t
     calcEdgeSlide_even(t, tc, sld, mval);
   }
 
-  tc->custom.mode.data = sld;
-
   MEM_freeN(sv_table);
 
-  return true;
-}
-
-void projectEdgeSlideData(TransInfo *t, bool is_final)
-{
-  FOREACH_TRANS_DATA_CONTAINER (t, tc) {
-    EdgeSlideData *sld = tc->custom.mode.data;
-
-    if (sld == NULL) {
-      continue;
-    }
-
-    trans_mesh_customdata_correction_apply(tc, is_final);
-  }
+  return sld;
 }
 
 static void freeEdgeSlideVerts(TransInfo *UNUSED(t),
@@ -1450,12 +1434,14 @@ void initEdgeSlide_ex(
 
   if (use_double_side) {
     FOREACH_TRANS_DATA_CONTAINER (t, tc) {
-      ok |= createEdgeSlideVerts_double_side(t, tc);
-    }
-  }
-  else {
-    FOREACH_TRANS_DATA_CONTAINER (t, tc) {
-      ok |= createEdgeSlideVerts_single_side(t, tc);
+      sld = use_double_side ? createEdgeSlideVerts_double_side(t, tc) :
+                              createEdgeSlideVerts_single_side(t, tc);
+      if (sld) {
+        tc->custom.mode.data = sld;
+        tc->custom.mode.free_cb = freeEdgeSlideVerts;
+        trans_mesh_customdata_correction_init(t, tc);
+        ok = true;
+      }
     }
   }
 
@@ -1464,16 +1450,6 @@ void initEdgeSlide_ex(
     return;
   }
 
-  FOREACH_TRANS_DATA_CONTAINER (t, tc) {
-    sld = tc->custom.mode.data;
-    if (!sld) {
-      continue;
-    }
-    tc->custom.mode.free_cb = freeEdgeSlideVerts;
-  }
-
-  trans_mesh_customdata_correction_init(t);
-
   /* set custom point first if you want value to be initialized by init */
   calcEdgeSlideCustomPoints(t);
   initMouseInputMode(t, &t->mouse, INPUT_CUSTOM_RATIO_FLIP);
diff --git a/source/blender/editors/transform/transform_mode_vert_slide.c b/source/blender/editors/transform/transform_mode_vert_slide.c
index 7bee48337f9..62415b1ddc6 100644
--- a/source/blender/editors/transform/transform_mode_vert_slide.c
+++ b/source/blender/editors/transform/transform_mode_vert_slide.c
@@ -197,7 +197,7 @@ static void calcVertSlideMouseActiveEdges(struct TransInfo *t, const int mval[2]
   }
 }
 
-static bool createVertSlideVerts(TransInfo *t, TransDataContainer *tc)
+static VertSlideData *createVertSlideVerts(TransInfo *t, const TransDataContainer *tc)
 {
   BMEditMesh *em = BKE_editmesh_from_object(tc->obedit);
   BMesh *bm = em->bm;
@@ -234,7 +234,7 @@ static bool createVertSlideVerts(TransInfo *t, TransDataContainer *tc)
 
   if (!j) {
     MEM_freeN(sld);
-    return false;
+    return NULL;
   }
 
   sv_array = MEM_callocN(sizeof(TransDataVertSlideVert) * j, "sv_array");
@@ -272,8 +272,6 @@ static bool createVertSlideVerts(TransInfo *t, TransDataContainer *tc)
   sld->sv = sv_array;
   sld->totsv = j;
 
-  tc->custom.mode.data = sld;
-
   /* most likely will be set below */
   unit_m4(sld->proj_mat);
 
@@ -288,13 +286,7 @@ static bool createVertSlideVerts(TransInfo *t, TransDataContainer *tc)
     }
   }
 
-  /* XXX, calc vert slide across all objects */
-  if (tc == t->data_container) {
-    calcVertSlideMouseActiveVert(t, t->mval);
-    calcVertSlideMouseActiveEdges(t, t->mval);
-  }
-
-  return true;
+  return sld;
 }
 
 static void freeVertSlideVerts(TransInfo *UNUSED(t),
@@ -381,13 +373,6 @@ static eRedrawFlag handleEventVertSlide(struct TransInfo *t, const struct wmEven
   return TREDRAW_NOTHING;
 }
 
-void projectVertSlideData(TransInfo *t, bool is_final)
-{
-  FOREACH_TRANS_DATA_CONTAINER (t, tc) {
-    trans_mesh_customdata_correction_apply(tc, is_final);
-  }
-}
-
 void drawVertSlide(TransInfo *t)
 {
   if ((t->mode == TFM_VERT_SLIDE) && TRANS_DATA_CONTAINER_FIRST_OK(t)->custom.mode.data) {
@@ -628,10 +613,12 @@ void initVertSlide_ex(TransInfo *t, bool use_even, bool flipped, bool use_clamp)
 
   bool ok = false;
   FOREACH_TRANS_DATA_CONTAINER (t, tc) {
-    ok |= createVertSlideVerts(t, tc);
-    VertSlideData *sld = tc->custom.mode.data;
+    VertSlideData *sld = createVertSlideVerts(t, tc);
     if (sld) {
+      tc->custom.mode.data = sld;
       tc->custom.mode.free_cb = freeVertSlideVerts;
+      trans_mesh_customdata_correction_init(t, tc);
+      ok = true;
     }
   }
 
@@ -640,7 +627,8 @@ void initVertSlide_ex(TransInfo *t, bool use_even, bool flipped, bool use_clamp)
     return;
   }
 
-  trans_mesh_customdata_correction_init(t);
+  calcVertSlideMouseActiveVert(t, t->mval);
+  calcVertSlideMouseActiveEdges(t, t->mval);
 
   /* set custom point first if you want value to be initialized by init */
   calcVertSlideCustomPoints(t);