BMesh: make toolflags optional

Saves 8 bytes per vert/edge/face.
Gives overall ~20-25% memory saving for dyntopo sculpting
and modifiers that use BMesh.

10 files changed, 274 insertions, 118 deletions

diff --git a/source/blender/bmesh/intern/bmesh_construct.c b/source/blender/bmesh/intern/bmesh_construct.c
index fdad93ee90d..4d92baab6eb 100644
--- a/source/blender/bmesh/intern/bmesh_construct.c
+++ b/source/blender/bmesh/intern/bmesh_construct.c
@@ -714,7 +714,9 @@ BMesh *BM_mesh_copy(BMesh *bm_old)
 	const BMAllocTemplate allocsize = BMALLOC_TEMPLATE_FROM_BM(bm_old);
 
 	/* allocate a bmesh */
-	bm_new = BM_mesh_create(&allocsize);
+	bm_new = BM_mesh_create(
+	        &allocsize,
+	        &((struct BMeshCreateParams){.use_toolflags = bm_old->use_toolflags,}));
 
 	BM_mesh_copy_init_customdata(bm_new, bm_old, &allocsize);
 
diff --git a/source/blender/bmesh/intern/bmesh_core.c b/source/blender/bmesh/intern/bmesh_core.c
index 1d68cdcf28a..e83b752947c 100644
--- a/source/blender/bmesh/intern/bmesh_core.c
+++ b/source/blender/bmesh/intern/bmesh_core.c
@@ -81,7 +81,9 @@ BMVert *BM_vert_create(
 	v->head.api_flag = 0;
 
 	/* allocate flags */
-	v->oflags = bm->vtoolflagpool ? BLI_mempool_calloc(bm->vtoolflagpool) : NULL;
+	if (bm->use_toolflags) {
+		((BMVert_OFlag *)v)->oflags = bm->vtoolflagpool ? BLI_mempool_calloc(bm->vtoolflagpool) : NULL;
+	}
 
 	/* 'v->no' is handled by BM_elem_attrs_copy */
 	if (co) {
@@ -174,7 +176,9 @@ BMEdge *BM_edge_create(
 	e->head.api_flag = 0;
 
 	/* allocate flags */
-	e->oflags = bm->etoolflagpool ? BLI_mempool_calloc(bm->etoolflagpool) : NULL;
+	if (bm->use_toolflags) {
+		((BMEdge_OFlag *)e)->oflags = bm->etoolflagpool ? BLI_mempool_calloc(bm->etoolflagpool) : NULL;
+	}
 
 	e->v1 = v1;
 	e->v2 = v2;
@@ -386,7 +390,9 @@ BLI_INLINE BMFace *bm_face_create__internal(BMesh *bm)
 	f->head.api_flag = 0;
 
 	/* allocate flags */
-	f->oflags = bm->ftoolflagpool ? BLI_mempool_calloc(bm->ftoolflagpool) : NULL;
+	if (bm->use_toolflags) {
+		((BMFace_OFlag *)f)->oflags = bm->ftoolflagpool ? BLI_mempool_calloc(bm->ftoolflagpool) : NULL;
+	}
 
 #ifdef USE_BMESH_HOLES
 	BLI_listbase_clear(&f->loops);
@@ -758,7 +764,7 @@ static void bm_kill_only_vert(BMesh *bm, BMVert *v)
 		CustomData_bmesh_free_block(&bm->vdata, &v->head.data);
 
 	if (bm->vtoolflagpool) {
-		BLI_mempool_free(bm->vtoolflagpool, v->oflags);
+		BLI_mempool_free(bm->vtoolflagpool, ((BMVert_OFlag *)v)->oflags);
 	}
 	BLI_mempool_free(bm->vpool, v);
 }
@@ -779,7 +785,7 @@ static void bm_kill_only_edge(BMesh *bm, BMEdge *e)
 		CustomData_bmesh_free_block(&bm->edata, &e->head.data);
 
 	if (bm->etoolflagpool) {
-		BLI_mempool_free(bm->etoolflagpool, e->oflags);
+		BLI_mempool_free(bm->etoolflagpool, ((BMEdge_OFlag *)e)->oflags);
 	}
 	BLI_mempool_free(bm->epool, e);
 }
@@ -803,7 +809,7 @@ static void bm_kill_only_face(BMesh *bm, BMFace *f)
 		CustomData_bmesh_free_block(&bm->pdata, &f->head.data);
 
 	if (bm->ftoolflagpool) {
-		BLI_mempool_free(bm->ftoolflagpool, f->oflags);
+		BLI_mempool_free(bm->ftoolflagpool, ((BMFace_OFlag *)f)->oflags);
 	}
 	BLI_mempool_free(bm->fpool, f);
 }
@@ -2196,7 +2202,7 @@ BMFace *bmesh_jfke(BMesh *bm, BMFace *f1, BMFace *f2, BMEdge *e)
 	
 	/* deallocate edge and its two loops as well as f2 */
 	if (bm->etoolflagpool) {
-		BLI_mempool_free(bm->etoolflagpool, l_f1->e->oflags);
+		BLI_mempool_free(bm->etoolflagpool, ((BMEdge_OFlag *)l_f1->e)->oflags);
 	}
 	BLI_mempool_free(bm->epool, l_f1->e);
 	bm->totedge--;
@@ -2205,7 +2211,7 @@ BMFace *bmesh_jfke(BMesh *bm, BMFace *f1, BMFace *f2, BMEdge *e)
 	BLI_mempool_free(bm->lpool, l_f2);
 	bm->totloop--;
 	if (bm->ftoolflagpool) {
-		BLI_mempool_free(bm->ftoolflagpool, f2->oflags);
+		BLI_mempool_free(bm->ftoolflagpool, ((BMFace_OFlag *)f2)->oflags);
 	}
 	BLI_mempool_free(bm->fpool, f2);
 	bm->totface--;
diff --git a/source/blender/bmesh/intern/bmesh_delete.c b/source/blender/bmesh/intern/bmesh_delete.c
index 882d78ce6b3..1449a6ef9d7 100644
--- a/source/blender/bmesh/intern/bmesh_delete.c
+++ b/source/blender/bmesh/intern/bmesh_delete.c
@@ -54,7 +54,7 @@ static void bmo_remove_tagged_faces(BMesh *bm, const short oflag)
 	BMIter iter;
 
 	BM_ITER_MESH_MUTABLE (f, f_next, &iter, bm, BM_FACES_OF_MESH) {
-		if (BMO_elem_flag_test(bm, f, oflag)) {
+		if (BMO_face_flag_test(bm, f, oflag)) {
 			BM_face_kill(bm, f);
 		}
 	}
@@ -66,7 +66,7 @@ static void bmo_remove_tagged_edges(BMesh *bm, const short oflag)
 	BMIter iter;
 
 	BM_ITER_MESH_MUTABLE (e, e_next, &iter, bm, BM_EDGES_OF_MESH) {
-		if (BMO_elem_flag_test(bm, e, oflag)) {
+		if (BMO_edge_flag_test(bm, e, oflag)) {
 			BM_edge_kill(bm, e);
 		}
 	}
@@ -78,7 +78,7 @@ static void bmo_remove_tagged_verts(BMesh *bm, const short oflag)
 	BMIter iter;
 
 	BM_ITER_MESH_MUTABLE (v, v_next, &iter, bm, BM_VERTS_OF_MESH) {
-		if (BMO_elem_flag_test(bm, v, oflag)) {
+		if (BMO_vert_flag_test(bm, v, oflag)) {
 			BM_vert_kill(bm, v);
 		}
 	}
@@ -90,7 +90,7 @@ static void bmo_remove_tagged_verts_loose(BMesh *bm, const short oflag)
 	BMIter iter;
 
 	BM_ITER_MESH_MUTABLE (v, v_next, &iter, bm, BM_VERTS_OF_MESH) {
-		if (BMO_elem_flag_test(bm, v, oflag) && (v->e == NULL)) {
+		if (BMO_vert_flag_test(bm, v, oflag) && (v->e == NULL)) {
 			BM_vert_kill(bm, v);
 		}
 	}
@@ -132,9 +132,9 @@ void BMO_mesh_delete_oflag_context(BMesh *bm, const short oflag, const int type)
 		{
 			/* flush down to vert */
 			BM_ITER_MESH (e, &eiter, bm, BM_EDGES_OF_MESH) {
-				if (BMO_elem_flag_test(bm, e, oflag)) {
-					BMO_elem_flag_enable(bm, e->v1, oflag);
-					BMO_elem_flag_enable(bm, e->v2, oflag);
+				if (BMO_edge_flag_test(bm, e, oflag)) {
+					BMO_vert_flag_enable(bm, e->v1, oflag);
+					BMO_vert_flag_enable(bm, e->v2, oflag);
 				}
 			}
 			bmo_remove_tagged_edges(bm, oflag);
@@ -165,27 +165,27 @@ void BMO_mesh_delete_oflag_context(BMesh *bm, const short oflag, const int type)
 		{
 			/* go through and mark all edges and all verts of all faces for delete */
 			BM_ITER_MESH (f, &fiter, bm, BM_FACES_OF_MESH) {
-				if (BMO_elem_flag_test(bm, f, oflag)) {
+				if (BMO_face_flag_test(bm, f, oflag)) {
 					BMLoop *l_first = BM_FACE_FIRST_LOOP(f);
 					BMLoop *l_iter;
 
 					l_iter = l_first;
 					do {
-						BMO_elem_flag_enable(bm, l_iter->v, oflag);
-						BMO_elem_flag_enable(bm, l_iter->e, oflag);
+						BMO_vert_flag_enable(bm, l_iter->v, oflag);
+						BMO_edge_flag_enable(bm, l_iter->e, oflag);
 					} while ((l_iter = l_iter->next) != l_first);
 				}
 			}
 			/* now go through and mark all remaining faces all edges for keeping */
 			BM_ITER_MESH (f, &fiter, bm, BM_FACES_OF_MESH) {
-				if (!BMO_elem_flag_test(bm, f, oflag)) {
+				if (!BMO_face_flag_test(bm, f, oflag)) {
 					BMLoop *l_first = BM_FACE_FIRST_LOOP(f);
 					BMLoop *l_iter;
 
 					l_iter = l_first;
 					do {
-						BMO_elem_flag_disable(bm, l_iter->v, oflag);
-						BMO_elem_flag_disable(bm, l_iter->e, oflag);
+						BMO_vert_flag_disable(bm, l_iter->v, oflag);
+						BMO_edge_flag_disable(bm, l_iter->e, oflag);
 					} while ((l_iter = l_iter->next) != l_first);
 				}
 			}
@@ -195,13 +195,13 @@ void BMO_mesh_delete_oflag_context(BMesh *bm, const short oflag, const int type)
 				/* Only exception to normal 'DEL_FACES' logic. */
 				if (type == DEL_FACES_KEEP_BOUNDARY) {
 					if (BM_edge_is_boundary(e)) {
-						BMO_elem_flag_disable(bm, e, oflag);
+						BMO_edge_flag_disable(bm, e, oflag);
 					}
 				}
 
-				if (!BMO_elem_flag_test(bm, e, oflag)) {
-					BMO_elem_flag_disable(bm, e->v1, oflag);
-					BMO_elem_flag_disable(bm, e->v2, oflag);
+				if (!BMO_edge_flag_test(bm, e, oflag)) {
+					BMO_vert_flag_disable(bm, e->v1, oflag);
+					BMO_vert_flag_disable(bm, e->v2, oflag);
 				}
 			}
 
diff --git a/source/blender/bmesh/intern/bmesh_iterators.c b/source/blender/bmesh/intern/bmesh_iterators.c
index cc79e28a361..961b10d848a 100644
--- a/source/blender/bmesh/intern/bmesh_iterators.c
+++ b/source/blender/bmesh/intern/bmesh_iterators.c
@@ -339,16 +339,43 @@ int BMO_iter_elem_count_flag(
         const short oflag, const bool value)
 {
 	BMIter iter;
-	BMElemF *ele;
 	int count = 0;
 
 	/* loops have no header flags */
 	BLI_assert(bm_iter_itype_htype_map[itype] != BM_LOOP);
 
-	BM_ITER_ELEM (ele, &iter, data, itype) {
-		if (BMO_elem_flag_test_bool(bm, ele, oflag) == value) {
-			count++;
+	switch (bm_iter_itype_htype_map[itype]) {
+		case BM_VERT:
+		{
+			BMVert *ele;
+			BM_ITER_ELEM (ele, &iter, data, itype) {
+				if (BMO_vert_flag_test_bool(bm, ele, oflag) == value) {
+					count++;
+				}
+			}
+			break;
 		}
+		case BM_EDGE:
+		{
+			BMEdge *ele;
+			BM_ITER_ELEM (ele, &iter, data, itype) {
+				if (BMO_edge_flag_test_bool(bm, ele, oflag) == value) {
+					count++;
+				}
+			}
+			break;
+		}
+		case BM_FACE:
+		{
+			BMFace *ele;
+			BM_ITER_ELEM (ele, &iter, data, itype) {
+				if (BMO_face_flag_test_bool(bm, ele, oflag) == value) {
+					count++;
+				}
+			}
+			break;
+		}
+
 	}
 	return count;
 }
diff --git a/source/blender/bmesh/intern/bmesh_mesh.c b/source/blender/bmesh/intern/bmesh_mesh.c
index ed1bd16b2e4..71b57756e98 100644
--- a/source/blender/bmesh/intern/bmesh_mesh.c
+++ b/source/blender/bmesh/intern/bmesh_mesh.c
@@ -48,16 +48,35 @@
 const BMAllocTemplate bm_mesh_allocsize_default = {512, 1024, 2048, 512};
 const BMAllocTemplate bm_mesh_chunksize_default = {512, 1024, 2048, 512};
 
-static void bm_mempool_init(BMesh *bm, const BMAllocTemplate *allocsize)
+static void bm_mempool_init(BMesh *bm, const BMAllocTemplate *allocsize, const bool use_toolflags)
 {
-	bm->vpool = BLI_mempool_create(sizeof(BMVert), allocsize->totvert,
-	                               bm_mesh_chunksize_default.totvert, BLI_MEMPOOL_ALLOW_ITER);
-	bm->epool = BLI_mempool_create(sizeof(BMEdge), allocsize->totedge,
-	                               bm_mesh_chunksize_default.totedge, BLI_MEMPOOL_ALLOW_ITER);
-	bm->lpool = BLI_mempool_create(sizeof(BMLoop), allocsize->totloop,
-	                               bm_mesh_chunksize_default.totloop, BLI_MEMPOOL_NOP);
-	bm->fpool = BLI_mempool_create(sizeof(BMFace), allocsize->totface,
-	                               bm_mesh_chunksize_default.totface, BLI_MEMPOOL_ALLOW_ITER);
+	size_t vert_size, edge_size, loop_size, face_size;
+
+	if (use_toolflags == true) {
+		vert_size = sizeof(BMVert_OFlag);
+		edge_size = sizeof(BMEdge_OFlag);
+		loop_size = sizeof(BMLoop);
+		face_size = sizeof(BMFace_OFlag);
+	}
+	else {
+		vert_size = sizeof(BMVert);
+		edge_size = sizeof(BMEdge);
+		loop_size = sizeof(BMLoop);
+		face_size = sizeof(BMFace);
+	}
+
+	bm->vpool = BLI_mempool_create(
+	        vert_size, allocsize->totvert,
+	        bm_mesh_chunksize_default.totvert, BLI_MEMPOOL_ALLOW_ITER);
+	bm->epool = BLI_mempool_create(
+	        edge_size, allocsize->totedge,
+	        bm_mesh_chunksize_default.totedge, BLI_MEMPOOL_ALLOW_ITER);
+	bm->lpool = BLI_mempool_create(
+	        loop_size, allocsize->totloop,
+	        bm_mesh_chunksize_default.totloop, BLI_MEMPOOL_NOP);
+	bm->fpool = BLI_mempool_create(
+	        face_size, allocsize->totface,
+	        bm_mesh_chunksize_default.totface, BLI_MEMPOOL_ALLOW_ITER);
 
 #ifdef USE_BMESH_HOLES
 	bm->looplistpool = BLI_mempool_create(sizeof(BMLoopList), 512, 512, BLI_MEMPOOL_NOP);
@@ -66,6 +85,8 @@ static void bm_mempool_init(BMesh *bm, const BMAllocTemplate *allocsize)
 
 void BM_mesh_elem_toolflags_ensure(BMesh *bm)
 {
+	BLI_assert(bm->use_toolflags);
+
 	if (bm->vtoolflagpool && bm->etoolflagpool && bm->ftoolflagpool) {
 		return;
 	}
@@ -80,7 +101,7 @@ void BM_mesh_elem_toolflags_ensure(BMesh *bm)
 		{
 			BLI_mempool *toolflagpool = bm->vtoolflagpool;
 			BMIter iter;
-			BMElemF *ele;
+			BMVert_OFlag *ele;
 			BM_ITER_MESH (ele, &iter, bm, BM_VERTS_OF_MESH) {
 				ele->oflags = BLI_mempool_calloc(toolflagpool);
 			}
@@ -89,7 +110,7 @@ void BM_mesh_elem_toolflags_ensure(BMesh *bm)
 		{
 			BLI_mempool *toolflagpool = bm->etoolflagpool;
 			BMIter iter;
-			BMElemF *ele;
+			BMEdge_OFlag *ele;
 			BM_ITER_MESH (ele, &iter, bm, BM_EDGES_OF_MESH) {
 				ele->oflags = BLI_mempool_calloc(toolflagpool);
 			}
@@ -98,7 +119,7 @@ void BM_mesh_elem_toolflags_ensure(BMesh *bm)
 		{
 			BLI_mempool *toolflagpool = bm->ftoolflagpool;
 			BMIter iter;
-			BMElemF *ele;
+			BMFace_OFlag *ele;
 			BM_ITER_MESH (ele, &iter, bm, BM_FACES_OF_MESH) {
 				ele->oflags = BLI_mempool_calloc(toolflagpool);
 			}
@@ -134,15 +155,18 @@ void BM_mesh_elem_toolflags_clear(BMesh *bm)
  *
  * \note ob is needed by multires
  */
-BMesh *BM_mesh_create(const BMAllocTemplate *allocsize)
+BMesh *BM_mesh_create(
+        const BMAllocTemplate *allocsize,
+        const struct BMeshCreateParams *params)
 {
 	/* allocate the structure */
 	BMesh *bm = MEM_callocN(sizeof(BMesh), __func__);
 	
 	/* allocate the memory pools for the mesh elements */
-	bm_mempool_init(bm, allocsize);
+	bm_mempool_init(bm, allocsize, params->use_toolflags);
 
 	/* allocate one flag pool that we don't get rid of. */
+	bm->use_toolflags = params->use_toolflags;
 	bm->toolflag_index = 0;
 	bm->totflags = 0;
 
@@ -239,13 +263,16 @@ void BM_mesh_data_free(BMesh *bm)
  */
 void BM_mesh_clear(BMesh *bm)
 {
+	const bool use_toolflags = bm->use_toolflags;
+
 	/* free old mesh */
 	BM_mesh_data_free(bm);
 	memset(bm, 0, sizeof(BMesh));
 
 	/* allocate the memory pools for the mesh elements */
-	bm_mempool_init(bm, &bm_mesh_allocsize_default);
+	bm_mempool_init(bm, &bm_mesh_allocsize_default, use_toolflags);
 
+	bm->use_toolflags = use_toolflags;
 	bm->toolflag_index = 0;
 	bm->totflags = 0;
 
diff --git a/source/blender/bmesh/intern/bmesh_mesh.h b/source/blender/bmesh/intern/bmesh_mesh.h
index b9cdc4ccf66..c071205e9f5 100644
--- a/source/blender/bmesh/intern/bmesh_mesh.h
+++ b/source/blender/bmesh/intern/bmesh_mesh.h
@@ -32,7 +32,14 @@ struct MLoopNorSpaceArray;
 
 void   BM_mesh_elem_toolflags_ensure(BMesh *bm);
 void   BM_mesh_elem_toolflags_clear(BMesh *bm);
-BMesh *BM_mesh_create(const struct BMAllocTemplate *allocsize);
+
+struct BMeshCreateParams {
+	unsigned int use_toolflags : 1;
+};
+
+BMesh *BM_mesh_create(
+        const struct BMAllocTemplate *allocsize,
+        const struct BMeshCreateParams *params);
 
 void   BM_mesh_free(BMesh *bm);
 void   BM_mesh_data_free(BMesh *bm);
@@ -113,10 +120,4 @@ extern const BMAllocTemplate bm_mesh_chunksize_default;
 
 #define BMALLOC_TEMPLATE_FROM_DM(...) VA_NARGS_CALL_OVERLOAD(_VA_BMALLOC_TEMPLATE_FROM_DM_, __VA_ARGS__)
 
-
-
-enum {
-	BM_MESH_CREATE_USE_TOOLFLAGS = (1 << 0)
-};
-
 #endif /* __BMESH_MESH_H__ */
diff --git a/source/blender/bmesh/intern/bmesh_operator_api.h b/source/blender/bmesh/intern/bmesh_operator_api.h
index cf93fe0935e..30cd1df9c4e 100644
--- a/source/blender/bmesh/intern/bmesh_operator_api.h
+++ b/source/blender/bmesh/intern/bmesh_operator_api.h
@@ -75,15 +75,74 @@ extern "C" {
 
 struct GHashIterator;
 
-#define BMO_elem_flag_test(     bm, ele, oflag)      _bmo_elem_flag_test     (bm, (ele)->oflags, oflag)
-#define BMO_elem_flag_test_bool(bm, ele, oflag)      _bmo_elem_flag_test_bool(bm, (ele)->oflags, oflag)
-#define BMO_elem_flag_enable(   bm, ele, oflag)      _bmo_elem_flag_enable   (bm, (ele)->oflags, oflag)
-#define BMO_elem_flag_disable(  bm, ele, oflag)      _bmo_elem_flag_disable  (bm, (ele)->oflags, oflag)
-#define BMO_elem_flag_set(      bm, ele, oflag, val) _bmo_elem_flag_set      (bm, (ele)->oflags, oflag, val)
-#define BMO_elem_flag_toggle(   bm, ele, oflag)      _bmo_elem_flag_toggle   (bm, (ele)->oflags, oflag)
-
-BLI_INLINE short _bmo_elem_flag_test(     BMesh *bm, BMFlagLayer *oflags, const short oflag);
-BLI_INLINE bool  _bmo_elem_flag_test_bool(BMesh *bm, BMFlagLayer *oflags, const short oflag);
+BLI_INLINE BMFlagLayer *BMO_elem_flag_from_header(BMHeader *ele_head)
+{
+	switch (ele_head->htype) {
+		case BM_VERT: return ((BMVert_OFlag *)ele_head)->oflags;
+		case BM_EDGE: return ((BMEdge_OFlag *)ele_head)->oflags;
+		default:      return ((BMFace_OFlag *)ele_head)->oflags;
+	}
+}
+
+#define BMO_elem_flag_test(bm, ele, oflag) \
+	_bmo_elem_flag_test(bm, BMO_elem_flag_from_header(&(ele)->head), oflag)
+#define BMO_elem_flag_test_bool(bm, ele, oflag) \
+	_bmo_elem_flag_test_bool(bm, BMO_elem_flag_from_header(&(ele)->head), oflag)
+#define BMO_elem_flag_enable(bm, ele, oflag) \
+	_bmo_elem_flag_enable(bm, (BM_CHECK_TYPE_ELEM_NONCONST(ele), BMO_elem_flag_from_header(&(ele)->head)), oflag)
+#define BMO_elem_flag_disable(bm, ele, oflag) \
+	_bmo_elem_flag_disable(bm, (BM_CHECK_TYPE_ELEM_NONCONST(ele), BMO_elem_flag_from_header(&(ele)->head)), oflag)
+#define BMO_elem_flag_set(bm, ele, oflag, val) \
+	_bmo_elem_flag_set(bm, (BM_CHECK_TYPE_ELEM_NONCONST(ele), BMO_elem_flag_from_header(&(ele)->head)), oflag, val)
+#define BMO_elem_flag_toggle(bm, ele, oflag) \
+	_bmo_elem_flag_toggle(bm, (BM_CHECK_TYPE_ELEM_NONCONST(ele), BMO_elem_flag_from_header(&(ele)->head)), oflag)
+
+/* take care not to instansiate args multiple times */
+#ifdef __GNUC___
+#define _BMO_CAST_V_CONST(e) ({ typeof(e) _e = e; \
+	(BM_CHECK_TYPE_VERT(_e), BLI_assert(((const BMHeader *)_e)->htype == BM_VERT), (const BMVert_OFlag *)_e); })
+#define _BMO_CAST_E_CONST(e) ({ typeof(e) _e = e; \
+	(BM_CHECK_TYPE_EDGE(_e), BLI_assert(((const BMHeader *)_e)->htype == BM_EDGE), (const BMEdge_OFlag *)_e); })
+#define _BMO_CAST_F_CONST(e) ({ typeof(e) _e = e; \
+	(BM_CHECK_TYPE_FACE(_e), BLI_assert(((const BMHeader *)_e)->htype == BM_FACE), (const BMFace_OFlag *)_e); })
+#define _BMO_CAST_V(e) ({ typeof(e) _e = e; \
+	(BM_CHECK_TYPE_VERT_NONCONST(_e), BLI_assert(((BMHeader *)_e)->htype == BM_VERT), (BMVert_OFlag *)_e); })
+#define _BMO_CAST_E(e) ({ typeof(e) _e = e; \
+	(BM_CHECK_TYPE_EDGE_NONCONST(_e), BLI_assert(((BMHeader *)_e)->htype == BM_EDGE), (BMEdge_OFlag *)_e); })
+#define _BMO_CAST_F(e) ({ typeof(e) _e = e; \
+	(BM_CHECK_TYPE_FACE_NONCONST(_e), BLI_assert(((BMHeader *)_e)->htype == BM_FACE), (BMFace_OFlag *)_e); })
+#else
+#define _BMO_CAST_V_CONST(e) (BM_CHECK_TYPE_VERT(e), (const BMVert_OFlag *)e)
+#define _BMO_CAST_E_CONST(e) (BM_CHECK_TYPE_EDGE(e), (const BMEdge_OFlag *)e)
+#define _BMO_CAST_F_CONST(e) (BM_CHECK_TYPE_FACE(e), (const BMFace_OFlag *)e)
+#define _BMO_CAST_V(e) (BM_CHECK_TYPE_VERT_NONCONST(e), (BMVert_OFlag *)e)
+#define _BMO_CAST_E(e) (BM_CHECK_TYPE_EDGE_NONCONST(e), (BMEdge_OFlag *)e)
+#define _BMO_CAST_F(e) (BM_CHECK_TYPE_FACE_NONCONST(e), (BMFace_OFlag *)e)
+#endif
+
+#define BMO_vert_flag_test(     bm, e, oflag)      _bmo_elem_flag_test     (bm, _BMO_CAST_V_CONST(e)->oflags, oflag)
+#define BMO_vert_flag_test_bool(bm, e, oflag)      _bmo_elem_flag_test_bool(bm, _BMO_CAST_V_CONST(e)->oflags, oflag)
+#define BMO_vert_flag_enable(   bm, e, oflag)      _bmo_elem_flag_enable   (bm, _BMO_CAST_V(e)->oflags, oflag)
+#define BMO_vert_flag_disable(  bm, e, oflag)      _bmo_elem_flag_disable  (bm, _BMO_CAST_V(e)->oflags, oflag)
+#define BMO_vert_flag_set(      bm, e, oflag, val) _bmo_elem_flag_set      (bm, _BMO_CAST_V(e)->oflags, oflag, val)
+#define BMO_vert_flag_toggle(   bm, e, oflag)      _bmo_elem_flag_toggle   (bm, _BMO_CAST_V(e)->oflags, oflag)
+
+#define BMO_edge_flag_test(     bm, e, oflag)      _bmo_elem_flag_test     (bm, _BMO_CAST_E_CONST(e)->oflags, oflag)
+#define BMO_edge_flag_test_bool(bm, e, oflag)      _bmo_elem_flag_test_bool(bm, _BMO_CAST_E_CONST(e)->oflags, oflag)
+#define BMO_edge_flag_enable(   bm, e, oflag)      _bmo_elem_flag_enable   (bm, _BMO_CAST_E(e)->oflags, oflag)
+#define BMO_edge_flag_disable(  bm, e, oflag)      _bmo_elem_flag_disable  (bm, _BMO_CAST_E(e)->oflags, oflag)
+#define BMO_edge_flag_set(      bm, e, oflag, val) _bmo_elem_flag_set      (bm, _BMO_CAST_E(e)->oflags, oflag, val)
+#define BMO_edge_flag_toggle(   bm, e, oflag)      _bmo_elem_flag_toggle   (bm, _BMO_CAST_E(e)->oflags, oflag)
+
+#define BMO_face_flag_test(     bm, e, oflag)      _bmo_elem_flag_test     (bm, _BMO_CAST_F_CONST(e)->oflags, oflag)
+#define BMO_face_flag_test_bool(bm, e, oflag)      _bmo_elem_flag_test_bool(bm, _BMO_CAST_F_CONST(e)->oflags, oflag)
+#define BMO_face_flag_enable(   bm, e, oflag)      _bmo_elem_flag_enable   (bm, _BMO_CAST_F(e)->oflags, oflag)
+#define BMO_face_flag_disable(  bm, e, oflag)      _bmo_elem_flag_disable  (bm, _BMO_CAST_F(e)->oflags, oflag)
+#define BMO_face_flag_set(      bm, e, oflag, val) _bmo_elem_flag_set      (bm, _BMO_CAST_F(e)->oflags, oflag, val)
+#define BMO_face_flag_toggle(   bm, e, oflag)      _bmo_elem_flag_toggle   (bm, _BMO_CAST_F(e)->oflags, oflag)
+
+BLI_INLINE short _bmo_elem_flag_test(     BMesh *bm, const BMFlagLayer *oflags, const short oflag);
+BLI_INLINE bool  _bmo_elem_flag_test_bool(BMesh *bm, const BMFlagLayer *oflags, const short oflag);
 BLI_INLINE void  _bmo_elem_flag_enable(   BMesh *bm, BMFlagLayer *oflags, const short oflag);
 BLI_INLINE void  _bmo_elem_flag_disable(  BMesh *bm, BMFlagLayer *oflags, const short oflag);
 BLI_INLINE void  _bmo_elem_flag_set(      BMesh *bm, BMFlagLayer *oflags, const short oflag, int val);
diff --git a/source/blender/bmesh/intern/bmesh_operator_api_inline.h b/source/blender/bmesh/intern/bmesh_operator_api_inline.h
index 00fcd9e7a9b..eb1c161f19d 100644
--- a/source/blender/bmesh/intern/bmesh_operator_api_inline.h
+++ b/source/blender/bmesh/intern/bmesh_operator_api_inline.h
@@ -39,32 +39,37 @@
 
 /* flags 15 and 16 (1 << 14 and 1 << 15) are reserved for bmesh api use */
 ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1, 2)
-BLI_INLINE short _bmo_elem_flag_test(BMesh *bm, BMFlagLayer *oflags, const short oflag)
+BLI_INLINE short _bmo_elem_flag_test(BMesh *bm, const BMFlagLayer *oflags, const short oflag)
 {
+	BLI_assert(bm->use_toolflags);
 	return oflags[bm->toolflag_index].f & oflag;
 }
 
 ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1, 2)
-BLI_INLINE bool _bmo_elem_flag_test_bool(BMesh *bm, BMFlagLayer *oflags, const short oflag)
+BLI_INLINE bool _bmo_elem_flag_test_bool(BMesh *bm, const BMFlagLayer *oflags, const short oflag)
 {
+	BLI_assert(bm->use_toolflags);
 	return (oflags[bm->toolflag_index].f & oflag) != 0;
 }
 
 ATTR_NONNULL(1, 2)
 BLI_INLINE void _bmo_elem_flag_enable(BMesh *bm, BMFlagLayer *oflags, const short oflag)
 {
+	BLI_assert(bm->use_toolflags);
 	oflags[bm->toolflag_index].f |= oflag;
 }
 
 ATTR_NONNULL(1, 2)
 BLI_INLINE void _bmo_elem_flag_disable(BMesh *bm, BMFlagLayer *oflags, const short oflag)
 {
+	BLI_assert(bm->use_toolflags);
 	oflags[bm->toolflag_index].f &= (short)~oflag;
 }
 
 ATTR_NONNULL(1, 2)
 BLI_INLINE void _bmo_elem_flag_set(BMesh *bm, BMFlagLayer *oflags, const short oflag, int val)
 {
+	BLI_assert(bm->use_toolflags);
 	if (val) oflags[bm->toolflag_index].f |= oflag;
 	else     oflags[bm->toolflag_index].f &= (short)~oflag;
 }
@@ -72,6 +77,7 @@ BLI_INLINE void _bmo_elem_flag_set(BMesh *bm, BMFlagLayer *oflags, const short o
 ATTR_NONNULL(1, 2)
 BLI_INLINE void _bmo_elem_flag_toggle(BMesh *bm, BMFlagLayer *oflags, const short oflag)
 {
+	BLI_assert(bm->use_toolflags);
 	oflags[bm->toolflag_index].f ^= oflag;
 }
 
diff --git a/source/blender/bmesh/intern/bmesh_operators.c b/source/blender/bmesh/intern/bmesh_operators.c
index 960ff568e93..706a7f74ed2 100644
--- a/source/blender/bmesh/intern/bmesh_operators.c
+++ b/source/blender/bmesh/intern/bmesh_operators.c
@@ -548,27 +548,44 @@ static int bmo_mesh_flag_count(
         BMesh *bm, const char htype, const short oflag,
         const bool test_for_enabled)
 {
-	const char iter_types[3] = {BM_VERTS_OF_MESH,
-	                            BM_EDGES_OF_MESH,
-	                            BM_FACES_OF_MESH};
+	int count_vert = 0, count_edge = 0, count_face = 0;
 
-	const char flag_types[3] = {BM_VERT, BM_EDGE, BM_FACE};
-
-	BMIter iter;
-	int count = 0;
-	BMElemF *ele_f;
-	int i;
-
-	for (i = 0; i < 3; i++) {
-		if (htype & flag_types[i]) {
-			BM_ITER_MESH (ele_f, &iter, bm, iter_types[i]) {
-				if (BMO_elem_flag_test_bool(bm, ele_f, oflag) == test_for_enabled)
-					count++;
+#pragma omp parallel sections if ((bm->totvert + bm->totedge + bm->totface >= BM_OMP_LIMIT) && \
+	                              (ELEM(htype, BM_VERT, BM_EDGE, BM_FACE) == 0))
+	{
+#pragma omp section
+		if (htype & BM_VERT) {
+			BMIter iter;
+			BMVert *ele;
+			BM_ITER_MESH (ele, &iter, bm, BM_VERTS_OF_MESH) {
+				if (BMO_vert_flag_test_bool(bm, ele, oflag) == test_for_enabled) {
+					count_vert++;
+				}
+			}
+		}
+#pragma omp section
+		if (htype & BM_EDGE) {
+			BMIter iter;
+			BMEdge *ele;
+			BM_ITER_MESH (ele, &iter, bm, BM_EDGES_OF_MESH) {
+				if (BMO_edge_flag_test_bool(bm, ele, oflag) == test_for_enabled) {
+					count_edge++;
+				}
+			}
+		}
+#pragma omp section
+		if (htype & BM_FACE) {
+			BMIter iter;
+			BMFace *ele;
+			BM_ITER_MESH (ele, &iter, bm, BM_FACES_OF_MESH) {
+				if (BMO_face_flag_test_bool(bm, ele, oflag) == test_for_enabled) {
+					count_face++;
+				}
 			}
 		}
 	}
 
-	return count;
+	return (count_vert + count_edge + count_face);
 }
 
 
@@ -584,21 +601,32 @@ int BMO_mesh_disabled_flag_count(BMesh *bm, const char htype, const short oflag)
 
 void BMO_mesh_flag_disable_all(BMesh *bm, BMOperator *UNUSED(op), const char htype, const short oflag)
 {
-	const char iter_types[3] = {BM_VERTS_OF_MESH,
-	                            BM_EDGES_OF_MESH,
-	                            BM_FACES_OF_MESH};
-
-	const char flag_types[3] = {BM_VERT, BM_EDGE, BM_FACE};
 
-	BMElemF *ele;
-	int i;
-
-#pragma omp parallel for schedule(static) if (bm->totvert + bm->totedge + bm->totface >= BM_OMP_LIMIT)
-	for (i = 0; i < 3; i++) {
-		if (htype & flag_types[i]) {
+#pragma omp parallel sections if ((bm->totvert + bm->totedge + bm->totface >= BM_OMP_LIMIT) && \
+	                              (ELEM(htype, BM_VERT, BM_EDGE, BM_FACE) == 0))
+	{
+#pragma omp section
+		if (htype & BM_VERT) {
 			BMIter iter;
-			BM_ITER_MESH (ele, &iter, bm, iter_types[i]) {
-				BMO_elem_flag_disable(bm, ele, oflag);
+			BMVert *ele;
+			BM_ITER_MESH (ele, &iter, bm, BM_VERTS_OF_MESH) {
+				BMO_vert_flag_disable(bm, ele, oflag);
+			}
+		}
+#pragma omp section
+		if (htype & BM_EDGE) {
+			BMIter iter;
+			BMEdge *ele;
+			BM_ITER_MESH (ele, &iter, bm, BM_EDGES_OF_MESH) {
+				BMO_edge_flag_disable(bm, ele, oflag);
+			}
+		}
+#pragma omp section
+		if (htype & BM_FACE) {
+			BMIter iter;
+			BMFace *ele;
+			BM_ITER_MESH (ele, &iter, bm, BM_FACES_OF_MESH) {
+				BMO_face_flag_disable(bm, ele, oflag);
 			}
 		}
 	}
@@ -1007,7 +1035,7 @@ static void bmo_slot_buffer_from_flag(
 
 		if (htype & BM_VERT) {
 			BM_ITER_MESH (ele, &iter, bm, BM_VERTS_OF_MESH) {
-				if (BMO_elem_flag_test_bool(bm, (BMElemF *)ele, oflag) == test_for_enabled) {
+				if (BMO_vert_flag_test_bool(bm, (BMVert *)ele, oflag) == test_for_enabled) {
 					ele_array[i] = ele;
 					i++;
 				}
@@ -1016,7 +1044,7 @@ static void bmo_slot_buffer_from_flag(
 
 		if (htype & BM_EDGE) {
 			BM_ITER_MESH (ele, &iter, bm, BM_EDGES_OF_MESH) {
-				if (BMO_elem_flag_test_bool(bm, (BMElemF *)ele, oflag) == test_for_enabled) {
+				if (BMO_edge_flag_test_bool(bm, (BMEdge *)ele, oflag) == test_for_enabled) {
 					ele_array[i] = ele;
 					i++;
 				}
@@ -1025,7 +1053,7 @@ static void bmo_slot_buffer_from_flag(
 
 		if (htype & BM_FACE) {
 			BM_ITER_MESH (ele, &iter, bm, BM_FACES_OF_MESH) {
-				if (BMO_elem_flag_test_bool(bm, (BMElemF *)ele, oflag) == test_for_enabled) {
+				if (BMO_face_flag_test_bool(bm, (BMFace *)ele, oflag) == test_for_enabled) {
 					ele_array[i] = ele;
 					i++;
 				}
@@ -1213,7 +1241,7 @@ static void bmo_flag_layer_alloc(BMesh *bm)
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMVert_OFlag *ele;
 			int i;
 
 			BLI_mempool *newpool = bm->vtoolflagpool;
@@ -1223,14 +1251,14 @@ static void bmo_flag_layer_alloc(BMesh *bm)
 				void *oldflags = ele->oflags;
 				ele->oflags = BLI_mempool_calloc(newpool);
 				memcpy(ele->oflags, oldflags, old_totflags_size);
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 				BM_ELEM_API_FLAG_CLEAR((BMElemF *)ele);
 			}
 		}
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMEdge_OFlag *ele;
 			int i;
 
 			BLI_mempool *newpool = bm->etoolflagpool;
@@ -1239,14 +1267,14 @@ static void bmo_flag_layer_alloc(BMesh *bm)
 				void *oldflags = ele->oflags;
 				ele->oflags = BLI_mempool_calloc(newpool);
 				memcpy(ele->oflags, oldflags, old_totflags_size);
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 				BM_ELEM_API_FLAG_CLEAR((BMElemF *)ele);
 			}
 		}
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMFace_OFlag *ele;
 			int i;
 
 			BLI_mempool *newpool = bm->ftoolflagpool;
@@ -1255,7 +1283,7 @@ static void bmo_flag_layer_alloc(BMesh *bm)
 				void *oldflags = ele->oflags;
 				ele->oflags = BLI_mempool_calloc(newpool);
 				memcpy(ele->oflags, oldflags, old_totflags_size);
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 				BM_ELEM_API_FLAG_CLEAR((BMElemF *)ele);
 			}
 		}
@@ -1292,7 +1320,7 @@ static void bmo_flag_layer_free(BMesh *bm)
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMVert_OFlag *ele;
 			int i;
 
 			BLI_mempool *newpool = bm->vtoolflagpool;
@@ -1302,14 +1330,14 @@ static void bmo_flag_layer_free(BMesh *bm)
 				void *oldflags = ele->oflags;
 				ele->oflags = BLI_mempool_alloc(newpool);
 				memcpy(ele->oflags, oldflags, new_totflags_size);
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 				BM_ELEM_API_FLAG_CLEAR((BMElemF *)ele);
 			}
 		}
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMEdge_OFlag *ele;
 			int i;
 
 			BLI_mempool *newpool = bm->etoolflagpool;
@@ -1318,14 +1346,14 @@ static void bmo_flag_layer_free(BMesh *bm)
 				void *oldflags = ele->oflags;
 				ele->oflags = BLI_mempool_alloc(newpool);
 				memcpy(ele->oflags, oldflags, new_totflags_size);
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 				BM_ELEM_API_FLAG_CLEAR((BMElemF *)ele);
 			}
 		}
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMFace_OFlag *ele;
 			int i;
 
 			BLI_mempool *newpool = bm->ftoolflagpool;
@@ -1334,7 +1362,7 @@ static void bmo_flag_layer_free(BMesh *bm)
 				void *oldflags = ele->oflags;
 				ele->oflags = BLI_mempool_alloc(newpool);
 				memcpy(ele->oflags, oldflags, new_totflags_size);
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 				BM_ELEM_API_FLAG_CLEAR((BMElemF *)ele);
 			}
 		}
@@ -1361,31 +1389,31 @@ static void bmo_flag_layer_clear(BMesh *bm)
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMVert_OFlag *ele;
 			int i;
 			BM_ITER_MESH_INDEX (ele, &iter, bm, BM_VERTS_OF_MESH, i) {
 				ele->oflags[totflags_offset] = zero_flag;
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 			}
 		}
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMEdge_OFlag *ele;
 			int i;
 			BM_ITER_MESH_INDEX (ele, &iter, bm, BM_EDGES_OF_MESH, i) {
 				ele->oflags[totflags_offset] = zero_flag;
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 			}
 		}
 #pragma omp section
 		{
 			BMIter iter;
-			BMElemF *ele;
+			BMFace_OFlag *ele;
 			int i;
 			BM_ITER_MESH_INDEX (ele, &iter, bm, BM_FACES_OF_MESH, i) {
 				ele->oflags[totflags_offset] = zero_flag;
-				BM_elem_index_set(ele, i); /* set_inline */
+				BM_elem_index_set(&ele->base, i); /* set_inline */
 			}
 		}
 	}
diff --git a/source/blender/bmesh/intern/bmesh_walkers_impl.c b/source/blender/bmesh/intern/bmesh_walkers_impl.c
index 018700c0efa..279440984bb 100644
--- a/source/blender/bmesh/intern/bmesh_walkers_impl.c
+++ b/source/blender/bmesh/intern/bmesh_walkers_impl.c
@@ -49,7 +49,7 @@ static bool bmw_mask_check_vert(BMWalker *walker, BMVert *v)
 	if ((walker->flag & BMW_FLAG_TEST_HIDDEN) && BM_elem_flag_test(v, BM_ELEM_HIDDEN)) {
 		return false;
 	}
-	else if (walker->mask_vert && !BMO_elem_flag_test(walker->bm, v, walker->mask_vert)) {
+	else if (walker->mask_vert && !BMO_vert_flag_test(walker->bm, v, walker->mask_vert)) {
 		return false;
 	}
 	else {
@@ -62,7 +62,7 @@ static bool bmw_mask_check_edge(BMWalker *walker, BMEdge *e)
 	if ((walker->flag & BMW_FLAG_TEST_HIDDEN) && BM_elem_flag_test(e, BM_ELEM_HIDDEN)) {
 		return false;
 	}
-	else if (walker->mask_edge && !BMO_elem_flag_test(walker->bm, e, walker->mask_edge)) {
+	else if (walker->mask_edge && !BMO_edge_flag_test(walker->bm, e, walker->mask_edge)) {
 		return false;
 	}
 	else {
@@ -75,7 +75,7 @@ static bool bmw_mask_check_face(BMWalker *walker, BMFace *f)
 	if ((walker->flag & BMW_FLAG_TEST_HIDDEN) && BM_elem_flag_test(f, BM_ELEM_HIDDEN)) {
 		return false;
 	}
-	else if (walker->mask_face && !BMO_elem_flag_test(walker->bm, f, walker->mask_face)) {
+	else if (walker->mask_face && !BMO_face_flag_test(walker->bm, f, walker->mask_face)) {
 		return false;
 	}
 	else {
@@ -223,7 +223,7 @@ static void *bmw_VertShellWalker_step(BMWalker *walker)
 	do {
 		if (!BLI_gset_haskey(walker->visit_set, curedge)) {
 			if (!walker->restrictflag ||
-			    (walker->restrictflag && BMO_elem_flag_test(walker->bm, curedge, walker->restrictflag)))
+			    (walker->restrictflag && BMO_edge_flag_test(walker->bm, curedge, walker->restrictflag)))
 			{
 				BMwShellWalker *newstate;
 
@@ -748,7 +748,7 @@ static void *bmw_IslandboundWalker_step(BMWalker *walker)
 	iwalk = BMW_state_add(walker);
 	iwalk->base = owalk.base;
 
-	//if (!BMO_elem_flag_test(walker->bm, l->f, walker->restrictflag))
+	//if (!BMO_face_flag_test(walker->bm, l->f, walker->restrictflag))
 	//	iwalk->curloop = l->radial_next;
 	iwalk->curloop = l; //else iwalk->curloop = l;
 	iwalk->lastv = v;