Mesh Edit: preserve Custom Normal vectors in topology operators.

Custom Loop Normals are normally encoded relative to the default
normals, similar to normal maps, allowing them to naturally follow
mesh deformations. Changes to mesh topology however often result
in nonsensical effects that are not desired.

The Remove Doubles operation especially (now known as Merge By
Distance) is intended as a purely topological operation, and
definitely should not change the vector of the custom normals.

This patch implements that behavior by converting the relative
encoding into an absolute vector layer for the duration of the
operation. It also modifies other Merge types in this way for
consistency, the Rip operator as their inverse counterpart;
and also Delete, Dissolve, Connect Path and Knife operators
as other examples more related to topology than shape.

On the technical side, this ports mesh_normals_loop_custom_set
to BMesh, and then uses a temporary Custom Data layer to store
the normals as vectors for the duration of the above mentioned
operations. When the normals are converted back to custom data,
the caller can choose whether to mark edges as sharp to preserve
distinct normals, or just average them instead. All but Remove
Doubles choose to average for now.

Differential Revision: https://developer.blender.org/D4994

2 files changed, 46 insertions, 0 deletions

diff --git a/source/blender/editors/mesh/editmesh_rip.c b/source/blender/editors/mesh/editmesh_rip.c
index 5d9923c6a7d..e805507351c 100644
--- a/source/blender/editors/mesh/editmesh_rip.c
+++ b/source/blender/editors/mesh/editmesh_rip.c
@@ -1053,6 +1053,8 @@ static int edbm_rip_invoke(bContext *C, wmOperator *op, const wmEvent *event)
      * useful selection for grabbing.
      */
 
+    BM_custom_loop_normals_to_vector_layer(bm);
+
     /* BM_ELEM_SELECT --> BM_ELEM_TAG */
     BM_ITER_MESH (e, &iter, bm, BM_EDGES_OF_MESH) {
       BM_elem_flag_set(e, BM_ELEM_TAG, BM_elem_flag_test(e, BM_ELEM_SELECT));
@@ -1070,6 +1072,8 @@ static int edbm_rip_invoke(bContext *C, wmOperator *op, const wmEvent *event)
       continue;
     }
 
+    BM_custom_loop_normals_from_vector_layer(bm, false);
+
     BLI_assert(singlesel ? (bm->totvertsel > 0) : (bm->totedgesel > 0));
 
     if (bm->totvertsel == 0) {
diff --git a/source/blender/editors/mesh/editmesh_tools.c b/source/blender/editors/mesh/editmesh_tools.c
index 142bc119958..69497d019b8 100644
--- a/source/blender/editors/mesh/editmesh_tools.c
+++ b/source/blender/editors/mesh/editmesh_tools.c
@@ -451,6 +451,8 @@ static int edbm_delete_exec(bContext *C, wmOperator *op)
     BMEditMesh *em = BKE_editmesh_from_object(obedit);
     const int type = RNA_enum_get(op->ptr, "type");
 
+    BM_custom_loop_normals_to_vector_layer(em->bm);
+
     switch (type) {
       case MESH_DELETE_VERT: /* Erase Vertices */
         if (!(em->bm->totvertsel &&
@@ -495,6 +497,8 @@ static int edbm_delete_exec(bContext *C, wmOperator *op)
 
     EDBM_flag_disable_all(em, BM_ELEM_SELECT);
 
+    BM_custom_loop_normals_from_vector_layer(em->bm, false);
+
     EDBM_update_generic(obedit->data, true, true);
 
     DEG_id_tag_update(obedit->data, ID_RECALC_SELECT);
@@ -1216,6 +1220,8 @@ static bool edbm_connect_vert_pair(BMEditMesh *em, struct Mesh *me, wmOperator *
     }
   }
   if (checks_succeded) {
+    BM_custom_loop_normals_to_vector_layer(bm);
+
     BMO_op_exec(bm, &bmop);
     len = BMO_slot_get(bmop.slots_out, "edges.out")->len;
 
@@ -1232,6 +1238,8 @@ static bool edbm_connect_vert_pair(BMEditMesh *em, struct Mesh *me, wmOperator *
       /* so newly created edges get the selection state from the vertex */
       EDBM_selectmode_flush(em);
 
+      BM_custom_loop_normals_from_vector_layer(bm, false);
+
       EDBM_update_generic(me, true, true);
     }
   }
@@ -1524,8 +1532,13 @@ static int edbm_vert_connect_path_exec(bContext *C, wmOperator *op)
       }
     }
 
+    BM_custom_loop_normals_to_vector_layer(bm);
+
     if (bm_vert_connect_select_history(bm)) {
       EDBM_selectmode_flush(em);
+
+      BM_custom_loop_normals_from_vector_layer(bm, false);
+
       EDBM_update_generic(obedit->data, true, true);
     }
     else {
@@ -3183,6 +3196,8 @@ static int edbm_merge_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_to_vector_layer(em->bm);
+
     bool ok = false;
     switch (type) {
       case MESH_MERGE_CENTER:
@@ -3209,6 +3224,8 @@ static int edbm_merge_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_from_vector_layer(em->bm, false);
+
     EDBM_update_generic(obedit->data, true, true);
 
     /* once collapsed, we can't have edge/face selection */
@@ -3353,6 +3370,8 @@ static int edbm_remove_doubles_exec(bContext *C, wmOperator *op)
       htype_select = BM_FACE;
     }
 
+    BM_custom_loop_normals_to_vector_layer(em->bm);
+
     /* store selection as tags */
     BM_mesh_elem_hflag_enable_test(em->bm, htype_select, BM_ELEM_TAG, true, true, BM_ELEM_SELECT);
 
@@ -3380,6 +3399,8 @@ static int edbm_remove_doubles_exec(bContext *C, wmOperator *op)
     BM_mesh_elem_hflag_enable_test(em->bm, htype_select, BM_ELEM_SELECT, true, true, BM_ELEM_TAG);
     EDBM_selectmode_flush(em);
 
+    BM_custom_loop_normals_from_vector_layer(em->bm, true);
+
     if (count) {
       count_multi += count;
       EDBM_update_generic(obedit->data, true, true);
@@ -4047,6 +4068,8 @@ static int edbm_knife_cut_exec(bContext *C, wmOperator *op)
   MEM_freeN(screen_vert_coords);
   MEM_freeN(mouse_path);
 
+  BM_custom_loop_normals_to_vector_layer(bm);
+
   BMO_slot_buffer_from_enabled_flag(bm, &bmop, bmop.slots_in, "edges", BM_EDGE, ELE_EDGE_CUT);
 
   if (mode == KNIFE_MIDPOINT) {
@@ -4065,6 +4088,8 @@ static int edbm_knife_cut_exec(bContext *C, wmOperator *op)
     return OPERATOR_CANCELLED;
   }
 
+  BM_custom_loop_normals_from_vector_layer(bm, false);
+
   EDBM_update_generic(obedit->data, true, true);
 
   return OPERATOR_FINISHED;
@@ -5344,6 +5369,8 @@ static int edbm_tris_convert_to_quads_exec(bContext *C, wmOperator *op)
     do_vcols = RNA_boolean_get(op->ptr, "vcols");
     do_materials = RNA_boolean_get(op->ptr, "materials");
 
+    BM_custom_loop_normals_to_vector_layer(em->bm);
+
     if (!EDBM_op_call_and_selectf(
             em,
             op,
@@ -5362,6 +5389,8 @@ static int edbm_tris_convert_to_quads_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_from_vector_layer(em->bm, false);
+
     EDBM_update_generic(obedit->data, true, true);
   }
   MEM_freeN(objects);
@@ -5674,6 +5703,8 @@ static int edbm_dissolve_verts_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_to_vector_layer(em->bm);
+
     if (!EDBM_op_callf(em,
                        op,
                        "dissolve_verts verts=%hv use_face_split=%b use_boundary_tear=%b",
@@ -5682,6 +5713,9 @@ static int edbm_dissolve_verts_exec(bContext *C, wmOperator *op)
                        use_boundary_tear)) {
       continue;
     }
+
+    BM_custom_loop_normals_from_vector_layer(em->bm, false);
+
     EDBM_update_generic(obedit->data, true, true);
   }
 
@@ -5730,6 +5764,8 @@ static int edbm_dissolve_edges_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_to_vector_layer(em->bm);
+
     if (!EDBM_op_callf(em,
                        op,
                        "dissolve_edges edges=%he use_verts=%b use_face_split=%b",
@@ -5739,6 +5775,8 @@ static int edbm_dissolve_edges_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_from_vector_layer(em->bm, false);
+
     EDBM_update_generic(obedit->data, true, true);
   }
 
@@ -5786,6 +5824,8 @@ static int edbm_dissolve_faces_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_to_vector_layer(em->bm);
+
     if (!EDBM_op_call_and_selectf(em,
                                   op,
                                   "region.out",
@@ -5796,6 +5836,8 @@ static int edbm_dissolve_faces_exec(bContext *C, wmOperator *op)
       continue;
     }
 
+    BM_custom_loop_normals_from_vector_layer(em->bm, false);
+
     EDBM_update_generic(obedit->data, true, true);
   }
   MEM_freeN(objects);