BMesh: ensure iterator macros assign to valid types

note, this is for C++ code which expects a cast, (will be added later)
also add a macro for nop-expressions (EXPR_NOP),
when we never want an expression to be evaluated, but it should still be valid.

6 files changed, 53 insertions, 27 deletions

diff --git a/source/blender/blenlib/BLI_compiler_typecheck.h b/source/blender/blenlib/BLI_compiler_typecheck.h
index 551569b066d..46c57772f64 100644
--- a/source/blender/blenlib/BLI_compiler_typecheck.h
+++ b/source/blender/blenlib/BLI_compiler_typecheck.h
@@ -51,9 +51,9 @@
 }))
 
 #else
-#  define CHECK_TYPE(var, type)
-#  define CHECK_TYPE_PAIR(var_a, var_b)
-#  define CHECK_TYPE_PAIR_INLINE(var_a, var_b) (void)0
+#  define CHECK_TYPE(var, type)  { EXPR_NOP(var); }(void)0
+#  define CHECK_TYPE_PAIR(var_a, var_b)  { (EXPR_NOP(var_a), EXPR_NOP(var_b)); }(void)0
+#  define CHECK_TYPE_PAIR_INLINE(var_a, var_b)  (EXPR_NOP(var_a), EXPR_NOP(var_b))
 #endif
 
 /* can be used in simple macros */
@@ -66,10 +66,15 @@
 	((void)(((type)0) != (0 ? (val) : ((type)0))))
 #endif
 
-#define CHECK_TYPE_NONCONST(var)  {      \
-	void *non_const = 0 ? (var) : NULL;  \
-	(void)non_const;                     \
-} (void)0
+#if defined(__GNUC__) || defined(__clang__)
+#  define CHECK_TYPE_NONCONST(var) __extension__ ({ \
+	void *non_const = 0 ? (var) : NULL; \
+	(void)non_const; \
+})
+#else
+#  define CHECK_TYPE_NONCONST(var) EXPR_NOP(var)
+#endif
+
 
 /**
  * CHECK_TYPE_ANY: handy macro, eg:
diff --git a/source/blender/blenlib/BLI_utildefines.h b/source/blender/blenlib/BLI_utildefines.h
index ef1312a1d94..53896bb31b6 100644
--- a/source/blender/blenlib/BLI_utildefines.h
+++ b/source/blender/blenlib/BLI_utildefines.h
@@ -219,6 +219,8 @@ extern "C" {
 /* reusable ELEM macro */
 #define ELEM(...) VA_NARGS_CALL_OVERLOAD(_VA_ELEM, __VA_ARGS__)
 
+/* no-op for expressions we don't want to instansiate, but must remian valid */
+#define EXPR_NOP(expr) (void)(0 ? ((void)(expr), 1) : 0)
 
 /* shift around elements */
 #define SHIFT3(type, a, b, c)  {                                              \
diff --git a/source/blender/bmesh/bmesh_class.h b/source/blender/bmesh/bmesh_class.h
index 39359b97a4e..120ff4997dc 100644
--- a/source/blender/bmesh/bmesh_class.h
+++ b/source/blender/bmesh/bmesh_class.h
@@ -256,6 +256,12 @@ enum {
 #define BM_ALL (BM_VERT | BM_EDGE | BM_LOOP | BM_FACE)
 #define BM_ALL_NOLOOP (BM_VERT | BM_EDGE | BM_FACE)
 
+#define BM_CHECK_TYPE_ELEM(ele) \
+	CHECK_TYPE_ANY(ele, void *, BMFace *, BMEdge *, BMVert *, BMLoop *, BMElem *, BMElemF *, BMHeader *)
+
+#define BM_CHECK_TYPE_ELEM_ASSIGN(ele) \
+	(BM_CHECK_TYPE_ELEM(ele), CHECK_TYPE_NONCONST(ele)), ele
+
 /* BMHeader->hflag (char) */
 enum {
 	BM_ELEM_SELECT  = (1 << 0),
diff --git a/source/blender/bmesh/intern/bmesh_iterators.h b/source/blender/bmesh/intern/bmesh_iterators.h
index 792a9cdfe0d..49e511bdcb5 100644
--- a/source/blender/bmesh/intern/bmesh_iterators.h
+++ b/source/blender/bmesh/intern/bmesh_iterators.h
@@ -84,30 +84,40 @@ typedef enum BMIterType {
 extern const char bm_iter_itype_htype_map[BM_ITYPE_MAX];
 
 #define BM_ITER_MESH(ele, iter, bm, itype) \
-	for (ele = BM_iter_new(iter, bm, itype, NULL); ele; ele = BM_iter_step(iter))
+	for (BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_new(iter, bm, itype, NULL); \
+	     ele; \
+	     BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_step(iter))
 
 #define BM_ITER_MESH_INDEX(ele, iter, bm, itype, indexvar) \
-	for (ele = BM_iter_new(iter, bm, itype, NULL), indexvar = 0; ele; ele = BM_iter_step(iter), (indexvar)++)
+	for (BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_new(iter, bm, itype, NULL), indexvar = 0; \
+	     ele; \
+	     BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_step(iter), (indexvar)++)
 
 /* a version of BM_ITER_MESH which keeps the next item in storage
  * so we can delete the current item, see bug [#36923] */
 #ifdef DEBUG
 #  define BM_ITER_MESH_MUTABLE(ele, ele_next, iter, bm, itype) \
-	for (ele = BM_iter_new(iter, bm, itype, NULL); \
-	ele ? ((void)((iter)->count = BM_iter_mesh_count(itype, bm)), \
-	       (void)(ele_next = BM_iter_step(iter)), 1) : 0; \
-	ele = ele_next)
+	for (BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_new(iter, bm, itype, NULL); \
+	     ele ? ((void)((iter)->count = BM_iter_mesh_count(itype, bm)), \
+	            (void)(ele_next = BM_iter_step(iter)), 1) : 0; \
+	     BM_CHECK_TYPE_ELEM_ASSIGN(ele) = ele_next)
 #else
 #  define BM_ITER_MESH_MUTABLE(ele, ele_next, iter, bm, itype) \
-	for (ele = BM_iter_new(iter, bm, itype, NULL); ele ? ((ele_next = BM_iter_step(iter)), 1) : 0; ele = ele_next)
+	for (BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_new(iter, bm, itype, NULL); \
+	     ele ? ((BM_CHECK_TYPE_ELEM_ASSIGN(ele_next) = BM_iter_step(iter)), 1) : 0; \
+	     ele = ele_next)
 #endif
 
 
 #define BM_ITER_ELEM(ele, iter, data, itype) \
-	for (ele = BM_iter_new(iter, NULL, itype, data); ele; ele = BM_iter_step(iter))
+	for (BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_new(iter, NULL, itype, data); \
+	     ele; \
+	     BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_step(iter))
 
 #define BM_ITER_ELEM_INDEX(ele, iter, data, itype, indexvar) \
-	for (ele = BM_iter_new(iter, NULL, itype, data), indexvar = 0; ele; ele = BM_iter_step(iter), (indexvar)++)
+	for (BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_new(iter, NULL, itype, data), indexvar = 0; \
+	     ele; \
+	     BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_step(iter), (indexvar)++)
 
 /* iterator type structs */
 struct BMIter__elem_of_mesh {
diff --git a/source/blender/bmesh/intern/bmesh_operator_api.h b/source/blender/bmesh/intern/bmesh_operator_api.h
index 825bbb136b1..d966d882c67 100644
--- a/source/blender/bmesh/intern/bmesh_operator_api.h
+++ b/source/blender/bmesh/intern/bmesh_operator_api.h
@@ -479,7 +479,9 @@ int   BMO_iter_map_value_int(BMOIter *iter);
 bool  BMO_iter_map_value_bool(BMOIter *iter);
 
 #define BMO_ITER(ele, iter, slot_args, slot_name, restrict_flag)   \
-	for (ele = BMO_iter_new(iter, slot_args, slot_name, restrict_flag); ele; ele = BMO_iter_step(iter))
+	for (BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BMO_iter_new(iter, slot_args, slot_name, restrict_flag); \
+	     ele; \
+	     BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BMO_iter_step(iter))
 
 /******************* Inlined Functions********************/
 typedef void (*opexec)(BMesh *bm, BMOperator *op);
diff --git a/source/blender/python/bmesh/bmesh_py_types.h b/source/blender/python/bmesh/bmesh_py_types.h
index a2c2c312e71..66059a642d1 100644
--- a/source/blender/python/bmesh/bmesh_py_types.h
+++ b/source/blender/python/bmesh/bmesh_py_types.h
@@ -198,16 +198,17 @@ int  bpy_bm_generic_valid_check_source(BMesh *bm_source, const char *error_prefi
 
 #define BPY_BM_IS_VALID(obj) (LIKELY((obj)->bm != NULL))
 
-#define BM_ITER_BPY_BM_SEQ(ele, iter, bpy_bmelemseq)                          \
-	for (ele = BM_iter_new(iter,                                              \
-	                       (bpy_bmelemseq)->bm,                               \
-	                       (bpy_bmelemseq)->itype,                            \
-	                       (bpy_bmelemseq)->py_ele ?                          \
-	                           ((BPy_BMElem *)(bpy_bmelemseq)->py_ele)->ele : \
-	                           NULL                                           \
-	                       );                                                 \
-	     ele;                                                                 \
-	     ele = BM_iter_step(iter))
+#define BM_ITER_BPY_BM_SEQ(ele, iter, bpy_bmelemseq)                \
+	for (BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_new(              \
+	             iter,                                              \
+	             (bpy_bmelemseq)->bm,                               \
+	             (bpy_bmelemseq)->itype,                            \
+	             (bpy_bmelemseq)->py_ele ?                          \
+	                 ((BPy_BMElem *)(bpy_bmelemseq)->py_ele)->ele : \
+	                 NULL                                           \
+	             );                                                 \
+	     ele;                                                       \
+	     BM_CHECK_TYPE_ELEM_ASSIGN(ele) = BM_iter_step(iter))
 
 
 #ifdef __PY_CAPI_UTILS_H__