Fixed crash when adding/removing custom normals from pinned mesh

When a mesh is pinned in the properties panel, Blender crashes when you
click the "Add Custom Split Normals Data".

The code calls `ob = ED_object_context(C)` which returns NULL when the
mesh is pinned in the properties panel, causing a segfault when trying
to get the mesh via `ob->data`.

A new function `ED_mesh_context(C)` avoids this by first checking
whether a mesh was pinned in the context. If not, it checks the pinned
object's data. If that's not there, or it's not a mesh, it returns the
active object's mesh. Finally it returns NULL if there is no active
object, or if the active object is not a mesh object.

Reviewed By: brecht, mont29

Differential Revision: https://developer.blender.org/D5223

4 files changed, 37 insertions, 8 deletions

diff --git a/source/blender/editors/include/ED_mesh.h b/source/blender/editors/include/ED_mesh.h
index 5d8038d0b28..d2613facd83 100644
--- a/source/blender/editors/include/ED_mesh.h
+++ b/source/blender/editors/include/ED_mesh.h
@@ -393,6 +393,9 @@ bool ED_mesh_color_remove_named(struct Mesh *me, const char *name);
 void ED_mesh_report_mirror(struct wmOperator *op, int totmirr, int totfail);
 void ED_mesh_report_mirror_ex(struct wmOperator *op, int totmirr, int totfail, char selectmode);
 
+/* Returns the pinned mesh, the mesh from the pinned object, or the mesh from the active object. */
+struct Mesh *ED_mesh_context(struct bContext *C);
+
 /* mesh backup */
 typedef struct BMBackup {
   struct BMesh *bmcopy;
diff --git a/source/blender/editors/include/ED_screen.h b/source/blender/editors/include/ED_screen.h
index fc43144417a..e67a3b003fc 100644
--- a/source/blender/editors/include/ED_screen.h
+++ b/source/blender/editors/include/ED_screen.h
@@ -337,6 +337,7 @@ bool ED_operator_object_active(struct bContext *C);
 bool ED_operator_object_active_editable(struct bContext *C);
 bool ED_operator_object_active_editable_mesh(struct bContext *C);
 bool ED_operator_object_active_editable_font(struct bContext *C);
+bool ED_operator_editable_mesh(struct bContext *C);
 bool ED_operator_editmesh(struct bContext *C);
 bool ED_operator_editmesh_view3d(struct bContext *C);
 bool ED_operator_editmesh_region_view3d(struct bContext *C);
diff --git a/source/blender/editors/mesh/mesh_data.c b/source/blender/editors/mesh/mesh_data.c
index a6934326d68..ee8de9d8ea9 100644
--- a/source/blender/editors/mesh/mesh_data.c
+++ b/source/blender/editors/mesh/mesh_data.c
@@ -41,6 +41,7 @@
 
 #include "DEG_depsgraph.h"
 
+#include "RNA_access.h"
 #include "RNA_define.h"
 
 #include "WM_api.h"
@@ -629,8 +630,7 @@ void MESH_OT_vertex_color_remove(wmOperatorType *ot)
 
 static int mesh_customdata_clear_exec__internal(bContext *C, char htype, int type)
 {
-  Object *obedit = ED_object_context(C);
-  Mesh *me = obedit->data;
+  Mesh *me = ED_mesh_context(C);
 
   int tot;
   CustomData *data = mesh_customdata_get_type(me, htype, &tot);
@@ -788,8 +788,7 @@ void MESH_OT_customdata_skin_clear(wmOperatorType *ot)
 /* Clear custom loop normals */
 static int mesh_customdata_custom_splitnormals_add_exec(bContext *C, wmOperator *UNUSED(op))
 {
-  Object *ob = ED_object_context(C);
-  Mesh *me = ob->data;
+  Mesh *me = ED_mesh_context(C);
 
   if (!BKE_mesh_has_custom_loop_normals(me)) {
     CustomData *data = GET_CD_DATA(me, ldata);
@@ -853,7 +852,7 @@ void MESH_OT_customdata_custom_splitnormals_add(wmOperatorType *ot)
 
   /* api callbacks */
   ot->exec = mesh_customdata_custom_splitnormals_add_exec;
-  ot->poll = ED_operator_object_active_editable_mesh;
+  ot->poll = ED_operator_editable_mesh;
 
   /* flags */
   ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
@@ -861,8 +860,7 @@ void MESH_OT_customdata_custom_splitnormals_add(wmOperatorType *ot)
 
 static int mesh_customdata_custom_splitnormals_clear_exec(bContext *C, wmOperator *UNUSED(op))
 {
-  Object *ob = ED_object_context(C);
-  Mesh *me = ob->data;
+  Mesh *me = ED_mesh_context(C);
 
   if (BKE_mesh_has_custom_loop_normals(me)) {
     return mesh_customdata_clear_exec__internal(C, BM_LOOP, CD_CUSTOMLOOPNORMAL);
@@ -879,7 +877,7 @@ void MESH_OT_customdata_custom_splitnormals_clear(wmOperatorType *ot)
 
   /* api callbacks */
   ot->exec = mesh_customdata_custom_splitnormals_clear_exec;
-  ot->poll = ED_operator_object_active_editable_mesh;
+  ot->poll = ED_operator_editable_mesh;
 
   /* flags */
   ot->flag = OPTYPE_REGISTER | OPTYPE_UNDO;
@@ -1289,3 +1287,23 @@ void ED_mesh_report_mirror(wmOperator *op, int totmirr, int totfail)
 {
   ED_mesh_report_mirror_ex(op, totmirr, totfail, SCE_SELECT_VERTEX);
 }
+
+Mesh *ED_mesh_context(struct bContext *C)
+{
+  Mesh *mesh = CTX_data_pointer_get_type(C, "mesh", &RNA_Mesh).data;
+  if (mesh != NULL) {
+    return mesh;
+  }
+
+  Object *ob = ED_object_active_context(C);
+  if (ob == NULL) {
+    return NULL;
+  }
+
+  ID *data = (ID *)ob->data;
+  if (data == NULL || GS(data->name) != ID_ME) {
+    return NULL;
+  }
+
+  return (Mesh *)data;
+}
diff --git a/source/blender/editors/screen/screen_ops.c b/source/blender/editors/screen/screen_ops.c
index 5fc7979a631..c9a45728bc2 100644
--- a/source/blender/editors/screen/screen_ops.c
+++ b/source/blender/editors/screen/screen_ops.c
@@ -74,6 +74,7 @@
 #include "ED_clip.h"
 #include "ED_image.h"
 #include "ED_keyframes_draw.h"
+#include "ED_mesh.h"
 #include "ED_object.h"
 #include "ED_screen.h"
 #include "ED_screen_types.h"
@@ -371,6 +372,12 @@ bool ED_operator_object_active_editable_font(bContext *C)
   return ((ob != NULL) && !ID_IS_LINKED(ob) && !ed_object_hidden(ob) && (ob->type == OB_FONT));
 }
 
+bool ED_operator_editable_mesh(bContext *C)
+{
+  Mesh *mesh = ED_mesh_context(C);
+  return (mesh != NULL) && !ID_IS_LINKED(mesh);
+}
+
 bool ED_operator_editmesh(bContext *C)
 {
   Object *obedit = CTX_data_edit_object(C);