fix [#30772] No more than two subdivions give correct result when adding an icosphere

bug was introduced in r45297, which inadvertently broke testing for multiple flags at once.

added BM_elem_flag_test_bool() and BMO_elem_flag_test_bool() to get TRUE/FALSE results rather then the flag value.

5 files changed, 68 insertions, 53 deletions

diff --git a/source/blender/bmesh/intern/bmesh_inline.h b/source/blender/bmesh/intern/bmesh_inline.h
index 2cfaf49d51e..400f4a55b0e 100644
--- a/source/blender/bmesh/intern/bmesh_inline.h
+++ b/source/blender/bmesh/intern/bmesh_inline.h
@@ -30,18 +30,24 @@
 #define __BMESH_INLINE_H__
 
 /* stuff for dealing with header flags */
-#define BM_elem_flag_test(   ele, hflag)      _bm_elem_flag_test    (&(ele)->head, hflag)
-#define BM_elem_flag_enable( ele, hflag)      _bm_elem_flag_enable  (&(ele)->head, hflag)
-#define BM_elem_flag_disable(ele, hflag)      _bm_elem_flag_disable (&(ele)->head, hflag)
-#define BM_elem_flag_set(    ele, hflag, val) _bm_elem_flag_set     (&(ele)->head, hflag, val)
-#define BM_elem_flag_toggle( ele, hflag)      _bm_elem_flag_toggle  (&(ele)->head, hflag)
-#define BM_elem_flag_merge(  ele_a, ele_b)    _bm_elem_flag_merge   (&(ele_a)->head, &(ele_b)->head)
+#define BM_elem_flag_test(     ele, hflag)      _bm_elem_flag_test     (&(ele)->head, hflag)
+#define BM_elem_flag_test_bool(ele, hflag)      _bm_elem_flag_test_bool(&(ele)->head, hflag)
+#define BM_elem_flag_enable(   ele, hflag)      _bm_elem_flag_enable   (&(ele)->head, hflag)
+#define BM_elem_flag_disable(  ele, hflag)      _bm_elem_flag_disable  (&(ele)->head, hflag)
+#define BM_elem_flag_set(      ele, hflag, val) _bm_elem_flag_set      (&(ele)->head, hflag, val)
+#define BM_elem_flag_toggle(   ele, hflag)      _bm_elem_flag_toggle   (&(ele)->head, hflag)
+#define BM_elem_flag_merge(    ele_a, ele_b)    _bm_elem_flag_merge    (&(ele_a)->head, &(ele_b)->head)
 
 BLI_INLINE char _bm_elem_flag_test(const BMHeader *head, const char hflag)
 {
 	return head->hflag & hflag;
 }
 
+BLI_INLINE short _bm_elem_flag_test_bool(const BMHeader *head, const char hflag)
+{
+	return (head->hflag & hflag) != 0;
+}
+
 BLI_INLINE void _bm_elem_flag_enable(BMHeader *head, const char hflag)
 {
 	head->hflag |= hflag;
diff --git a/source/blender/bmesh/intern/bmesh_marking.c b/source/blender/bmesh/intern/bmesh_marking.c
index ed826cbf75a..03c4d3c9cbb 100644
--- a/source/blender/bmesh/intern/bmesh_marking.c
+++ b/source/blender/bmesh/intern/bmesh_marking.c
@@ -449,29 +449,30 @@ void BM_select_mode_set(BMesh *bm, int selectmode)
  * counts number of elements with flag enabled/disabled
  */
 static int bm_mesh_flag_count(BMesh *bm, const char htype, const char hflag,
-                              int respecthide, int test_for_enabled)
+                              const short respecthide, const short test_for_enabled)
 {
 	BMElem *ele;
 	BMIter iter;
-	const char hflag_test = (test_for_enabled ? hflag : 0);
 	int tot = 0;
 
+	BLI_assert(ELEM(TRUE, FALSE, test_for_enabled));
+
 	if (htype & BM_VERT) {
 		for (ele = BM_iter_new(&iter, bm, BM_VERTS_OF_MESH, NULL); ele; ele = BM_iter_step(&iter)) {
 			if (respecthide && BM_elem_flag_test(ele, BM_ELEM_HIDDEN)) continue;
-			if (BM_elem_flag_test(ele, hflag) == hflag_test) tot++;
+			if (BM_elem_flag_test_bool(ele, hflag) == test_for_enabled) tot++;
 		}
 	}
 	if (htype & BM_EDGE) {
 		for (ele = BM_iter_new(&iter, bm, BM_EDGES_OF_MESH, NULL); ele; ele = BM_iter_step(&iter)) {
 			if (respecthide && BM_elem_flag_test(ele, BM_ELEM_HIDDEN)) continue;
-			if (BM_elem_flag_test(ele, hflag) == hflag_test) tot++;
+			if (BM_elem_flag_test_bool(ele, hflag) == test_for_enabled) tot++;
 		}
 	}
 	if (htype & BM_FACE) {
 		for (ele = BM_iter_new(&iter, bm, BM_FACES_OF_MESH, NULL); ele; ele = BM_iter_step(&iter)) {
 			if (respecthide && BM_elem_flag_test(ele, BM_ELEM_HIDDEN)) continue;
-			if (BM_elem_flag_test(ele, hflag) == hflag_test) tot++;
+			if (BM_elem_flag_test_bool(ele, hflag) == test_for_enabled) tot++;
 		}
 	}
 
diff --git a/source/blender/bmesh/intern/bmesh_operator_api.h b/source/blender/bmesh/intern/bmesh_operator_api.h
index bb980a63861..7cba66995c9 100644
--- a/source/blender/bmesh/intern/bmesh_operator_api.h
+++ b/source/blender/bmesh/intern/bmesh_operator_api.h
@@ -75,17 +75,19 @@ extern "C" {
 
 struct GHashIterator;
 
-#define BMO_elem_flag_test(   bm, ele, oflag)      _bmo_elem_flag_test    (bm, (ele)->oflags, oflag)
-#define BMO_elem_flag_enable( bm, ele, oflag)      _bmo_elem_flag_enable  (bm, (ele)->oflags, oflag)
-#define BMO_elem_flag_disable(bm, ele, oflag)      _bmo_elem_flag_disable (bm, (ele)->oflags, oflag)
-#define BMO_elem_flag_set(    bm, ele, oflag, val) _bmo_elem_flag_set     (bm, (ele)->oflags, oflag, val)
-#define BMO_elem_flag_toggle( bm, ele, oflag)      _bmo_elem_flag_toggle  (bm, (ele)->oflags, oflag)
-
-BLI_INLINE short _bmo_elem_flag_test(BMesh *bm, BMFlagLayer *oflags, const short oflag);
-BLI_INLINE void  _bmo_elem_flag_enable(BMesh *bm, BMFlagLayer *oflags, const short oflag);
-BLI_INLINE void  _bmo_elem_flag_disable(BMesh *bm, BMFlagLayer *oflags, const short oflag);
-BLI_INLINE void  _bmo_elem_flag_set(BMesh *bm, BMFlagLayer *oflags, const short oflag, int val);
-BLI_INLINE void  _bmo_elem_flag_toggle(BMesh *bm, BMFlagLayer *oflags, const short oflag);
+#define BMO_elem_flag_test(     bm, ele, oflag)      _bmo_elem_flag_test     (bm, (ele)->oflags, oflag)
+#define BMO_elem_flag_test_bool(bm, ele, oflag)      _bmo_elem_flag_test_bool(bm, (ele)->oflags, oflag)
+#define BMO_elem_flag_enable(   bm, ele, oflag)      _bmo_elem_flag_enable   (bm, (ele)->oflags, oflag)
+#define BMO_elem_flag_disable(  bm, ele, oflag)      _bmo_elem_flag_disable  (bm, (ele)->oflags, oflag)
+#define BMO_elem_flag_set(      bm, ele, oflag, val) _bmo_elem_flag_set      (bm, (ele)->oflags, oflag, val)
+#define BMO_elem_flag_toggle(   bm, ele, oflag)      _bmo_elem_flag_toggle   (bm, (ele)->oflags, oflag)
+
+BLI_INLINE short _bmo_elem_flag_test(     BMesh *bm, BMFlagLayer *oflags, const short oflag);
+BLI_INLINE short _bmo_elem_flag_test_bool(BMesh *bm, BMFlagLayer *oflags, const short oflag);
+BLI_INLINE void  _bmo_elem_flag_enable(   BMesh *bm, BMFlagLayer *oflags, const short oflag);
+BLI_INLINE void  _bmo_elem_flag_disable(  BMesh *bm, BMFlagLayer *oflags, const short oflag);
+BLI_INLINE void  _bmo_elem_flag_set(      BMesh *bm, BMFlagLayer *oflags, const short oflag, int val);
+BLI_INLINE void  _bmo_elem_flag_toggle(   BMesh *bm, BMFlagLayer *oflags, const short oflag);
 
 /* slot type arrays are terminated by the last member
  * having a slot type of 0.*/
@@ -297,17 +299,17 @@ void BMO_mesh_flag_disable_all(BMesh *bm, BMOperator *op, const char htype, cons
 
 /* copies the values from another slot to the end of the output slot */
 void BMO_slot_buffer_append(BMOperator *output_op, const char *output_op_slot,
-							BMOperator *other_op, const char *other_op_slot);
+                            BMOperator *other_op, const char *other_op_slot);
 
 /* puts every element of type 'type' (which is a bitmask) with tool
  * flag 'flag', into a slot. */
 void BMO_slot_buffer_from_enabled_flag(BMesh *bm, BMOperator *op, const char *slotname,
-									   const char htype, const short oflag);
+                                       const char htype, const short oflag);
 
 /* puts every element of type 'type' (which is a bitmask) without tool
  * flag 'flag', into a slot. */
 void BMO_slot_buffer_from_disabled_flag(BMesh *bm, BMOperator *op, const char *slotname,
-										const char htype, const short oflag);
+                                        const char htype, const short oflag);
 
 /* tool-flags all elements inside an element slot array with flag flag. */
 void BMO_slot_buffer_flag_enable(BMesh *bm, BMOperator *op, const char *slotname,
@@ -327,15 +329,15 @@ void BMO_slot_buffer_hflag_disable(BMesh *bm, BMOperator *op, const char *slotna
  * flag 'flag', into a slot.  note: ignores hidden elements
  * (e.g. elements with header flag BM_ELEM_HIDDEN set).*/
 void BMO_slot_buffer_from_enabled_hflag(BMesh *bm, BMOperator *op,
-										const char *slotname,
-										const char htype, const char hflag);
+                                        const char *slotname,
+                                        const char htype, const char hflag);
 
 /* puts every element of type 'type' (which is a bitmask) without
  * header flag 'flag', into a slot.  note: ignores hidden elements
  * (e.g. elements with header flag BM_ELEM_HIDDEN set).*/
 void BMO_slot_buffer_from_disabled_hflag(BMesh *bm, BMOperator *op,
-										 const char *slotname,
-										 const char htype, const char hflag);
+                                         const char *slotname,
+                                         const char htype, const char hflag);
 
 /* counts number of elements inside a slot array. */
 int BMO_slot_buffer_count(BMesh *bm, BMOperator *op, const char *slotname);
diff --git a/source/blender/bmesh/intern/bmesh_operator_api_inline.h b/source/blender/bmesh/intern/bmesh_operator_api_inline.h
index 268736cfe0e..e04079f42c9 100644
--- a/source/blender/bmesh/intern/bmesh_operator_api_inline.h
+++ b/source/blender/bmesh/intern/bmesh_operator_api_inline.h
@@ -40,7 +40,12 @@
 /* flags 15 and 16 (1 << 14 and 1 << 15) are reserved for bmesh api use */
 BLI_INLINE short _bmo_elem_flag_test(BMesh *bm, BMFlagLayer *oflags, const short oflag)
 {
-    return oflags[bm->stackdepth - 1].f & oflag;
+	return oflags[bm->stackdepth - 1].f & oflag;
+}
+
+BLI_INLINE short _bmo_elem_flag_test_bool(BMesh *bm, BMFlagLayer *oflags, const short oflag)
+{
+	return (oflags[bm->stackdepth - 1].f & oflag) != 0;
 }
 
 BLI_INLINE void _bmo_elem_flag_enable(BMesh *bm, BMFlagLayer *oflags, const short oflag)
@@ -65,13 +70,13 @@ BLI_INLINE void _bmo_elem_flag_toggle(BMesh *bm, BMFlagLayer *oflags, const shor
 }
 
 BLI_INLINE void BMO_slot_map_int_insert(BMesh *bm, BMOperator *op, const char *slotname,
-                                       void *element, int val)
+                                        void *element, int val)
 {
 	BMO_slot_map_insert(bm, op, slotname, element, &val, sizeof(int));
 }
 
 BLI_INLINE void BMO_slot_map_float_insert(BMesh *bm, BMOperator *op, const char *slotname,
-                                         void *element, float val)
+                                          void *element, float val)
 {
 	BMO_slot_map_insert(bm, op, slotname, element, &val, sizeof(float));
 }
@@ -83,7 +88,7 @@ BLI_INLINE void BMO_slot_map_float_insert(BMesh *bm, BMOperator *op, const char
  * use BMO_slot_map_data_get and BMO_slot_map_insert, which copies the data. */
 
 BLI_INLINE void BMO_slot_map_ptr_insert(BMesh *bm, BMOperator *op, const char *slotname,
-                                       void *element, void *val)
+                                        void *element, void *val)
 {
 	BMO_slot_map_insert(bm, op, slotname, element, &val, sizeof(void *));
 }
@@ -100,7 +105,7 @@ BLI_INLINE int BMO_slot_map_contains(BMesh *UNUSED(bm), BMOperator *op, const ch
 }
 
 BLI_INLINE void *BMO_slot_map_data_get(BMesh *UNUSED(bm), BMOperator *op, const char *slotname,
-                                      void *element)
+                                       void *element)
 {
 	BMOElemMapping *mapping;
 	BMOpSlot *slot = BMO_slot_get(op, slotname);
@@ -117,7 +122,7 @@ BLI_INLINE void *BMO_slot_map_data_get(BMesh *UNUSED(bm), BMOperator *op, const
 }
 
 BLI_INLINE float BMO_slot_map_float_get(BMesh *bm, BMOperator *op, const char *slotname,
-                                       void *element)
+                                        void *element)
 {
 	float *val = (float *) BMO_slot_map_data_get(bm, op, slotname, element);
 	if (val) return *val;
@@ -126,7 +131,7 @@ BLI_INLINE float BMO_slot_map_float_get(BMesh *bm, BMOperator *op, const char *s
 }
 
 BLI_INLINE int BMO_slot_map_int_get(BMesh *bm, BMOperator *op, const char *slotname,
-                                   void *element)
+                                    void *element)
 {
 	int *val = (int *) BMO_slot_map_data_get(bm, op, slotname, element);
 	if (val) return *val;
@@ -135,7 +140,7 @@ BLI_INLINE int BMO_slot_map_int_get(BMesh *bm, BMOperator *op, const char *slotn
 }
 
 BLI_INLINE void *BMO_slot_map_ptr_get(BMesh *bm, BMOperator *op, const char *slotname,
-                                     void *element)
+                                      void *element)
 {
 	void **val = (void **) BMO_slot_map_data_get(bm, op, slotname, element);
 	if (val) return *val;
diff --git a/source/blender/bmesh/intern/bmesh_operators.c b/source/blender/bmesh/intern/bmesh_operators.c
index 44eed679cbe..00292e481b2 100644
--- a/source/blender/bmesh/intern/bmesh_operators.c
+++ b/source/blender/bmesh/intern/bmesh_operators.c
@@ -456,7 +456,7 @@ void BMO_slot_vec_get(BMOperator *op, const char *slotname, float r_vec[3])
  */
 
 static int bmo_mesh_flag_count(BMesh *bm, const char htype, const short oflag,
-                               int test_for_enabled)
+                               const short test_for_enabled)
 {
 	const char iter_types[3] = {BM_VERTS_OF_MESH,
 	                            BM_EDGES_OF_MESH,
@@ -467,13 +467,14 @@ static int bmo_mesh_flag_count(BMesh *bm, const char htype, const short oflag,
 	BMIter iter;
 	int count = 0;
 	BMElemF *ele_f;
-	const char hflag_test = (test_for_enabled ? oflag : 0);
 	int i;
 
+	BLI_assert(ELEM(TRUE, FALSE, test_for_enabled));
+
 	for (i = 0; i < 3; i++) {
 		if (htype & flag_types[i]) {
 			BM_ITER(ele_f, &iter, bm, iter_types[i], NULL) {
-				if (BMO_elem_flag_test(bm, ele_f, oflag) == hflag_test)
+				if (BMO_elem_flag_test_bool(bm, ele_f, oflag) == test_for_enabled)
 					count++;
 			}
 		}
@@ -692,11 +693,13 @@ static void BMO_slot_buffer_from_all(BMesh *bm, BMOperator *op, const char *slot
  */
 static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, const char *slotname,
                                        const char htype, const char hflag,
-                                       int test_for_enabled)
+                                       const short test_for_enabled)
 {
 	BMOpSlot *output = BMO_slot_get(op, slotname);
 	int totelement = 0, i = 0;
 
+	BLI_assert(ELEM(TRUE, FALSE, test_for_enabled));
+
 	if (test_for_enabled)
 		totelement = BM_mesh_enabled_flag_count(bm, htype, hflag, TRUE);
 	else
@@ -706,8 +709,6 @@ static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, const char *sl
 		BMIter iter;
 		BMElem *ele;
 
-		const char hflag_test = (test_for_enabled ? hflag : 0);
-
 		bmo_slot_buffer_alloc(op, slotname, totelement);
 
 		/* TODO - collapse these loops into one */
@@ -715,7 +716,7 @@ static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, const char *sl
 		if (htype & BM_VERT) {
 			BM_ITER(ele, &iter, bm, BM_VERTS_OF_MESH, NULL) {
 				if (!BM_elem_flag_test(ele, BM_ELEM_HIDDEN) &&
-				    BM_elem_flag_test(ele, hflag) == hflag_test)
+				    BM_elem_flag_test_bool(ele, hflag) == test_for_enabled)
 				{
 					((BMElem **)output->data.p)[i] = ele;
 					i++;
@@ -726,7 +727,7 @@ static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, const char *sl
 		if (htype & BM_EDGE) {
 			BM_ITER(ele, &iter, bm, BM_EDGES_OF_MESH, NULL) {
 				if (!BM_elem_flag_test(ele, BM_ELEM_HIDDEN) &&
-				    BM_elem_flag_test(ele, hflag) == hflag_test)
+				    BM_elem_flag_test_bool(ele, hflag) == test_for_enabled)
 				{
 					((BMElem **)output->data.p)[i] = ele;
 					i++;
@@ -737,7 +738,7 @@ static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, const char *sl
 		if (htype & BM_FACE) {
 			BM_ITER(ele, &iter, bm, BM_FACES_OF_MESH, NULL) {
 				if (!BM_elem_flag_test(ele, BM_ELEM_HIDDEN) &&
-				    BM_elem_flag_test(ele, hflag) == hflag_test)
+				    BM_elem_flag_test_bool(ele, hflag) == test_for_enabled)
 				{
 					((BMElem **)output->data.p)[i] = ele;
 					i++;
@@ -786,8 +787,7 @@ void BMO_slot_buffer_append(BMOperator *output_op, const char *output_slot_name,
 
 		/* copy slot data */
 		memcpy(buf, output_slot->data.buf, elem_size * output_slot->len);
-		memcpy(((char*)buf) + elem_size * output_slot->len,
-			   other_slot->data.buf, elem_size * other_slot->len);
+		memcpy(((char *)buf) + elem_size * output_slot->len, other_slot->data.buf, elem_size * other_slot->len);
 
 		output_slot->data.buf = buf;
 		output_slot->len += other_slot->len;
@@ -802,11 +802,13 @@ void BMO_slot_buffer_append(BMOperator *output_op, const char *output_slot_name,
  */
 static void bmo_slot_buffer_from_flag(BMesh *bm, BMOperator *op, const char *slotname,
                                       const char htype, const short oflag,
-                                      int test_for_enabled)
+                                      const short test_for_enabled)
 {
 	BMOpSlot *slot = BMO_slot_get(op, slotname);
 	int totelement, i = 0;
 
+	BLI_assert(ELEM(TRUE, FALSE, test_for_enabled));
+
 	if (test_for_enabled)
 		totelement = BMO_mesh_enabled_flag_count(bm, htype, oflag);
 	else
@@ -818,7 +820,6 @@ static void bmo_slot_buffer_from_flag(BMesh *bm, BMOperator *op, const char *slo
 		BMIter iter;
 		BMHeader *ele;
 		BMHeader **ele_array;
-		const char hflag_test = (test_for_enabled ? oflag : 0);
 
 		bmo_slot_buffer_alloc(op, slotname, totelement);
 
@@ -828,7 +829,7 @@ static void bmo_slot_buffer_from_flag(BMesh *bm, BMOperator *op, const char *slo
 
 		if (htype & BM_VERT) {
 			BM_ITER(ele, &iter, bm, BM_VERTS_OF_MESH, NULL) {
-				if (BMO_elem_flag_test(bm, (BMElemF *)ele, oflag) == hflag_test) {
+				if (BMO_elem_flag_test_bool(bm, (BMElemF *)ele, oflag) == test_for_enabled) {
 					ele_array[i] = ele;
 					i++;
 				}
@@ -837,7 +838,7 @@ static void bmo_slot_buffer_from_flag(BMesh *bm, BMOperator *op, const char *slo
 
 		if (htype & BM_EDGE) {
 			BM_ITER(ele, &iter, bm, BM_EDGES_OF_MESH, NULL) {
-				if (BMO_elem_flag_test(bm, (BMElemF *)ele, oflag) == hflag_test) {
+				if (BMO_elem_flag_test_bool(bm, (BMElemF *)ele, oflag) == test_for_enabled) {
 					ele_array[i] = ele;
 					i++;
 				}
@@ -846,7 +847,7 @@ static void bmo_slot_buffer_from_flag(BMesh *bm, BMOperator *op, const char *slo
 
 		if (htype & BM_FACE) {
 			BM_ITER(ele, &iter, bm, BM_FACES_OF_MESH, NULL) {
-				if (BMO_elem_flag_test(bm, (BMElemF *)ele, oflag) == hflag_test) {
+				if (BMO_elem_flag_test_bool(bm, (BMElemF *)ele, oflag) == test_for_enabled) {
 					ele_array[i] = ele;
 					i++;
 				}