bmesh operator api edits, add macros and NULL the buffer if BMO_slot_buffer_alloc()'s len is zero.

8 files changed, 54 insertions, 42 deletions

diff --git a/source/blender/bmesh/intern/bmesh_operator_api.h b/source/blender/bmesh/intern/bmesh_operator_api.h
index fbce08ed896..a44900cd4f5 100644
--- a/source/blender/bmesh/intern/bmesh_operator_api.h
+++ b/source/blender/bmesh/intern/bmesh_operator_api.h
@@ -126,23 +126,31 @@ typedef struct BMOpSlot {
 		float f;
 		void *p;
 		float vec[3];
-		void *buf;
+		void **buf;
 		GHash *ghash;
 	} data;
 } BMOpSlot;
 
+/* mainly for use outside bmesh internal code */
+#define BMO_SLOT_AS_BOOL(slot)         ((slot)->data.i)
+#define BMO_SLOT_AS_INT(slot)          ((slot)->data.i)
+#define BMO_SLOT_AS_FLOAT(slot)        ((slot)->data.f)
+#define BMO_SLOT_AS_VECTOR(slot)       ((slot)->data.vec)
+#define BMO_SLOT_AS_MATRIX(slot )      ((float (*)[4])((slot)->data.p))
+#define BMO_SLOT_AS_BUFFER(slot )      ((slot)->data.buf)
+#define BMO_SLOT_AS_GHASH(slot )       ((slot)->data.ghash)
+
 /* way more than probably needed, compiler complains if limit hit */
 #define BMO_OP_MAX_SLOTS 16
 
 typedef struct BMOperator {
-	int type;
-	int slot_type;
-	int type_flag;
-	int flag;  /* runtime options */
 	struct BMOpSlot slots_in[BMO_OP_MAX_SLOTS];
 	struct BMOpSlot slots_out[BMO_OP_MAX_SLOTS];
 	void (*exec)(BMesh *bm, struct BMOperator *op);
 	struct MemArena *arena;
+	int type;
+	int type_flag;
+	int flag;  /* runtime options */
 } BMOperator;
 
 enum {
diff --git a/source/blender/bmesh/intern/bmesh_operators.c b/source/blender/bmesh/intern/bmesh_operators.c
index 967574464cf..740f8eb6059 100644
--- a/source/blender/bmesh/intern/bmesh_operators.c
+++ b/source/blender/bmesh/intern/bmesh_operators.c
@@ -389,7 +389,7 @@ void BMO_slot_mat4_get(BMOpSlot slot_args[BMO_OP_MAX_SLOTS], const char *slot_na
 		return;
 
 	if (slot->data.p) {
-		copy_m4_m4(r_mat, (float (*)[4])slot->data.p);
+		copy_m4_m4(r_mat, BMO_SLOT_AS_MATRIX(slot));
 	}
 	else {
 		unit_m4(r_mat);
@@ -404,7 +404,7 @@ void BMO_slot_mat3_get(BMOpSlot slot_args[BMO_OP_MAX_SLOTS], const char *slot_na
 		return;
 
 	if (slot->data.p) {
-		copy_m3_m4(r_mat, slot->data.p);
+		copy_m3_m4(r_mat, BMO_SLOT_AS_MATRIX(slot));
 	}
 	else {
 		unit_m3(r_mat);
@@ -471,8 +471,8 @@ void *BMO_slot_as_arrayN(BMOpSlot slot_args[BMO_OP_MAX_SLOTS], const char *slot_
 	/* could add support for mapping type */
 	BLI_assert(slot->slot_type == BMO_OP_SLOT_ELEMENT_BUF);
 
-	ret = MEM_mallocN(sizeof(void *) * slot->len, __func__);
-	memcpy(ret, slot->data.buf, sizeof(void *) * slot->len);
+	ret = MEM_mallocN(sizeof(void **) * slot->len, __func__);
+	memcpy(ret, slot->data.buf, sizeof(void **) * slot->len);
 	*len = slot->len;
 	return ret;
 }
@@ -685,8 +685,13 @@ void *BMO_slot_buffer_alloc(BMOperator *op, BMOpSlot slot_args[BMO_OP_MAX_SLOTS]
 		return NULL;
 	
 	slot->len = len;
-	if (len)
+	if (len) {
 		slot->data.buf = BLI_memarena_alloc(op->arena, BMO_OPSLOT_TYPEINFO[slot->slot_type] * len);
+	}
+	else {
+		slot->data.buf = NULL;
+	}
+
 	return slot->data.buf;
 }
 
@@ -715,21 +720,21 @@ void BMO_slot_buffer_from_all(BMesh *bm, BMOperator *op, BMOpSlot slot_args[BMO_
 
 		if (htype & BM_VERT) {
 			BM_ITER_MESH (ele, &iter, bm, BM_VERTS_OF_MESH) {
-				((BMHeader **)output->data.p)[i] = ele;
+				output->data.buf[i] = ele;
 				i++;
 			}
 		}
 
 		if (htype & BM_EDGE) {
 			BM_ITER_MESH (ele, &iter, bm, BM_EDGES_OF_MESH) {
-				((BMHeader **)output->data.p)[i] = ele;
+				output->data.buf[i] = ele;
 				i++;
 			}
 		}
 
 		if (htype & BM_FACE) {
 			BM_ITER_MESH (ele, &iter, bm, BM_FACES_OF_MESH) {
-				((BMHeader **)output->data.p)[i] = ele;
+				output->data.buf[i] = ele;
 				i++;
 			}
 		}
@@ -770,7 +775,7 @@ static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, BMOpSlot slot_
 				if ((!respecthide || !BM_elem_flag_test(ele, BM_ELEM_HIDDEN)) &&
 				    BM_elem_flag_test_bool(ele, hflag) == test_for_enabled)
 				{
-					((BMElem **)output->data.p)[i] = ele;
+					output->data.buf[i] = ele;
 					i++;
 				}
 			}
@@ -781,7 +786,7 @@ static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, BMOpSlot slot_
 				if ((!respecthide || !BM_elem_flag_test(ele, BM_ELEM_HIDDEN)) &&
 				    BM_elem_flag_test_bool(ele, hflag) == test_for_enabled)
 				{
-					((BMElem **)output->data.p)[i] = ele;
+					output->data.buf[i] = ele;
 					i++;
 				}
 			}
@@ -792,7 +797,7 @@ static void bmo_slot_buffer_from_hflag(BMesh *bm, BMOperator *op, BMOpSlot slot_
 				if ((!respecthide || !BM_elem_flag_test(ele, BM_ELEM_HIDDEN)) &&
 				    BM_elem_flag_test_bool(ele, hflag) == test_for_enabled)
 				{
-					((BMElem **)output->data.p)[i] = ele;
+					output->data.buf[i] = ele;
 					i++;
 				}
 			}
@@ -882,7 +887,7 @@ static void bmo_slot_buffer_from_flag(BMesh *bm, BMOperator *op,
 
 		BMO_slot_buffer_alloc(op, slot_args, slot_name, totelement);
 
-		ele_array = (BMHeader **)slot->data.p;
+		ele_array = (BMHeader **)slot->data.buf;
 
 		/* TODO - collapse these loops into one */
 
@@ -943,7 +948,7 @@ void BMO_slot_buffer_hflag_enable(BMesh *bm,
                                   const char htype, const char hflag, const char do_flush)
 {
 	BMOpSlot *slot = BMO_slot_get(slot_args, slot_name);
-	BMElem **data =  slot->data.p;
+	BMElem **data =  (BMElem **)slot->data.buf;
 	int i;
 	const char do_flush_select = (do_flush && (hflag & BM_ELEM_SELECT));
 	const char do_flush_hide = (do_flush && (hflag & BM_ELEM_HIDDEN));
@@ -977,7 +982,7 @@ void BMO_slot_buffer_hflag_disable(BMesh *bm,
                                    const char htype, const char hflag, const char do_flush)
 {
 	BMOpSlot *slot = BMO_slot_get(slot_args, slot_name);
-	BMElem **data =  slot->data.p;
+	BMElem **data =  (BMElem **)slot->data.buf;
 	int i;
 	const char do_flush_select = (do_flush && (hflag & BM_ELEM_SELECT));
 	const char do_flush_hide = (do_flush && (hflag & BM_ELEM_HIDDEN));
@@ -1052,7 +1057,7 @@ void BMO_slot_buffer_flag_disable(BMesh *bm,
                                   const char htype, const short oflag)
 {
 	BMOpSlot *slot = BMO_slot_get(slot_args, slot_name);
-	BMHeader **data =  slot->data.p;
+	BMHeader **data = (BMHeader **)slot->data.buf;
 	int i;
 
 	BLI_assert(slot->slot_type == BMO_OP_SLOT_ELEMENT_BUF);
@@ -1211,7 +1216,7 @@ void *BMO_slot_buffer_elem_first(BMOpSlot slot_args[BMO_OP_MAX_SLOTS], const cha
 	if (slot->slot_type != BMO_OP_SLOT_ELEMENT_BUF)
 		return NULL;
 
-	return slot->data.buf ? *(void **)slot->data.buf : NULL;
+	return slot->data.buf ? *slot->data.buf : NULL;
 }
 
 /**
@@ -1253,13 +1258,13 @@ void *BMO_iter_step(BMOIter *iter)
 			return NULL;
 		}
 
-		h = ((void **)iter->slot->data.buf)[iter->cur++];
+		h = iter->slot->data.buf[iter->cur++];
 		while (!(iter->restrictmask & h->htype)) {
 			if (iter->cur >= iter->slot->len) {
 				return NULL;
 			}
 
-			h = ((void **)iter->slot->data.buf)[iter->cur++];
+			h = iter->slot->data.buf[iter->cur++];
 		}
 
 		return h;
@@ -1523,7 +1528,7 @@ int BMO_op_vinitf(BMesh *bm, BMOperator *op, const int flag, const char *_fmt, v
 
 					slot->data.buf = BLI_memarena_alloc(op->arena, sizeof(void *) * 4);
 					slot->len = 1;
-					*((void **)slot->data.buf) = ele;
+					*slot->data.buf = ele;
 
 					state = 1;
 					break;
diff --git a/source/blender/bmesh/operators/bmo_dissolve.c b/source/blender/bmesh/operators/bmo_dissolve.c
index 8518928cf7b..649aa8b214b 100644
--- a/source/blender/bmesh/operators/bmo_dissolve.c
+++ b/source/blender/bmesh/operators/bmo_dissolve.c
@@ -486,6 +486,6 @@ void bmo_dissolve_limit_exec(BMesh *bm, BMOperator *op)
 	const int do_dissolve_boundaries = BMO_slot_bool_get(op->slots_in, "use_dissolve_boundaries");
 
 	BM_mesh_decimate_dissolve_ex(bm, angle_limit, do_dissolve_boundaries,
-	                             vinput->data.p, vinput->len,
-	                             einput->data.p, einput->len);
+	                             (BMVert **)BMO_SLOT_AS_BUFFER(vinput), vinput->len,
+	                             (BMEdge **)BMO_SLOT_AS_BUFFER(einput), einput->len);
 }
diff --git a/source/blender/bmesh/operators/bmo_subdivide.c b/source/blender/bmesh/operators/bmo_subdivide.c
index 623109a7eb0..c693851336a 100644
--- a/source/blender/bmesh/operators/bmo_subdivide.c
+++ b/source/blender/bmesh/operators/bmo_subdivide.c
@@ -913,7 +913,7 @@ void bmo_subdivide_edges_exec(BMesh *bm, BMOperator *op)
 
 	/* go through and split edges */
 	for (i = 0; i < einput->len; i++) {
-		edge = ((BMEdge **)einput->data.p)[i];
+		edge = einput->data.buf[i];
 		bm_subdivide_multicut(bm, edge, &params, edge->v1, edge->v2);
 	}
 
diff --git a/source/blender/bmesh/operators/bmo_unsubdivide.c b/source/blender/bmesh/operators/bmo_unsubdivide.c
index aa0d3f4831b..fae7db3d175 100644
--- a/source/blender/bmesh/operators/bmo_unsubdivide.c
+++ b/source/blender/bmesh/operators/bmo_unsubdivide.c
@@ -42,7 +42,7 @@ void bmo_unsubdivide_exec(BMesh *bm, BMOperator *op)
 	const int iterations = max_ii(1, BMO_slot_int_get(op->slots_in, "iterations"));
 
 	BMOpSlot *vinput = BMO_slot_get(op->slots_in, "verts");
-	BMVert **vinput_arr = (BMVert **)vinput->data.p;
+	BMVert **vinput_arr = (BMVert **)vinput->data.buf;
 	int v_index;
 
 	/* tag verts */
diff --git a/source/blender/modifiers/intern/MOD_triangulate.c b/source/blender/modifiers/intern/MOD_triangulate.c
index 7dfbadac142..1e5b10e3d62 100644
--- a/source/blender/modifiers/intern/MOD_triangulate.c
+++ b/source/blender/modifiers/intern/MOD_triangulate.c
@@ -53,9 +53,6 @@ static DerivedMesh *triangulate_dm(DerivedMesh *dm, const int flag)
 	result = CDDM_from_bmesh(bm, FALSE);
 	BM_mesh_free(bm);
 
-	/* we don't really need to calc edges,
-	 * this is called to update the origindex values
-	 * This could be made into a different function? - Campbell */
 	CDDM_calc_edges(result);
 
 	CDDM_calc_normals(result);
diff --git a/source/blender/python/bmesh/bmesh_py_ops.c b/source/blender/python/bmesh/bmesh_py_ops.c
index a00e09b00ad..3446209e660 100644
--- a/source/blender/python/bmesh/bmesh_py_ops.c
+++ b/source/blender/python/bmesh/bmesh_py_ops.c
@@ -147,7 +147,7 @@ static PyObject *pyrna_op_call(BPy_BMeshOpFunc *self, PyObject *args, PyObject *
 						return NULL;
 					}
 					else {
-						slot->data.i = param;
+						BMO_SLOT_AS_BOOL(slot) = param;
 					}
 
 					break;
@@ -170,7 +170,7 @@ static PyObject *pyrna_op_call(BPy_BMeshOpFunc *self, PyObject *args, PyObject *
 						return NULL;
 					}
 					else {
-						slot->data.i = (int)param;
+						BMO_SLOT_AS_INT(slot) = (int)param;
 					}
 					break;
 				}
@@ -184,7 +184,7 @@ static PyObject *pyrna_op_call(BPy_BMeshOpFunc *self, PyObject *args, PyObject *
 						return NULL;
 					}
 					else {
-						slot->data.f = param;
+						BMO_SLOT_AS_FLOAT(slot) = param;
 					}
 					break;
 				}
@@ -217,7 +217,7 @@ static PyObject *pyrna_op_call(BPy_BMeshOpFunc *self, PyObject *args, PyObject *
 				case BMO_OP_SLOT_VEC:
 				{
 					/* passing slot name here is a bit non-descriptive */
-					if (mathutils_array_parse(slot->data.vec, 3, 3, value, slot_name) == -1) {
+					if (mathutils_array_parse(BMO_SLOT_AS_VECTOR(slot), 3, 3, value, slot_name) == -1) {
 						return NULL;
 					}
 					break;
@@ -271,7 +271,7 @@ static PyObject *pyrna_op_call(BPy_BMeshOpFunc *self, PyObject *args, PyObject *
 
 						i = 0;
 						BM_ITER_BPY_BM_SEQ (ele, &iter, ((BPy_BMElemSeq *)value)) {
-							((void **)slot->data.buf)[i] = (void *)ele;
+							slot->data.buf[i] = ele;
 							i++;
 						}
 					}
@@ -339,28 +339,29 @@ static PyObject *pyrna_op_call(BPy_BMeshOpFunc *self, PyObject *args, PyObject *
 			/* keep switch in same order as above */
 			switch (slot->slot_type) {
 				case BMO_OP_SLOT_BOOL:
-					item = PyBool_FromLong(slot->data.i);
+					item = PyBool_FromLong((BMO_SLOT_AS_BOOL(slot)));
 					break;
 				case BMO_OP_SLOT_INT:
-					item = PyLong_FromSsize_t((Py_ssize_t)slot->data.i);
+					item = PyLong_FromSsize_t(BMO_SLOT_AS_INT(slot));
 					break;
 				case BMO_OP_SLOT_FLT:
-					item = PyFloat_FromDouble((double)slot->data.f);
+					item = PyFloat_FromDouble((double)BMO_SLOT_AS_FLOAT(slot));
 					break;
 				case BMO_OP_SLOT_MAT:
-					item = Matrix_CreatePyObject(slot->data.p, 4, 4, Py_NEW, NULL);
+					item = Matrix_CreatePyObject((float *)BMO_SLOT_AS_MATRIX(slot), 4, 4, Py_NEW, NULL);
 					break;
 				case BMO_OP_SLOT_VEC:
-					item = Vector_CreatePyObject(slot->data.vec, slot->len, Py_NEW, NULL);
+					item = Vector_CreatePyObject(BMO_SLOT_AS_VECTOR(slot), slot->len, Py_NEW, NULL);
 					break;
 				case BMO_OP_SLOT_ELEMENT_BUF:
 				{
 					const int size = slot->len;
+					void **buffer = BMO_SLOT_AS_BUFFER(slot);
 					int j;
 
 					item = PyList_New(size);
 					for (j = 0; j < size; j++) {
-						BMHeader *ele = ((BMHeader **)slot->data.buf)[i];
+						BMHeader *ele = buffer[i];
 						PyList_SET_ITEM(item, j, ele ? BPy_BMElem_CreatePyObject(bm, ele) : (Py_INCREF(Py_None), Py_None));
 					}
 					break;
diff --git a/source/blender/python/bmesh/bmesh_py_types.c b/source/blender/python/bmesh/bmesh_py_types.c
index bc62e63cf2f..f98e7c71ec0 100644
--- a/source/blender/python/bmesh/bmesh_py_types.c
+++ b/source/blender/python/bmesh/bmesh_py_types.c
@@ -3306,6 +3306,7 @@ PyObject *BPy_BMElem_CreatePyObject(BMesh *bm, BMHeader *ele)
 		case BM_LOOP:
 			return BPy_BMLoop_CreatePyObject(bm, (BMLoop *)ele);
 		default:
+			BLI_assert(0);
 			PyErr_SetString(PyExc_SystemError, "internal error");
 			return NULL;
 	}