fix for a bug in mathutils when a vector was accessing a matrix and the matrix size increased, future access too the vector would write past the allocated bounds. now raise an exception.

3 files changed, 156 insertions, 25 deletions

diff --git a/source/blender/python/mathutils/mathutils.c b/source/blender/python/mathutils/mathutils.c
index 739206feefd..c9afb7edf4b 100644
--- a/source/blender/python/mathutils/mathutils.c
+++ b/source/blender/python/mathutils/mathutils.c
@@ -284,7 +284,7 @@ PyObject *mathutils_dynstr_to_py(struct DynStr *ds)
 /* Mathutils Callbacks */
 
 /* for mathutils internal use only, eventually should re-alloc but to start with we only have a few users */
-static Mathutils_Callback *mathutils_callbacks[8] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL};
+static Mathutils_Callback *mathutils_callbacks[8] = {NULL};
 
 int Mathutils_RegisterCallback(Mathutils_Callback *cb)
 {
@@ -459,6 +459,7 @@ PyMODINIT_FUNC PyInit_mathutils(void)
 
 	mathutils_matrix_row_cb_index = Mathutils_RegisterCallback(&mathutils_matrix_row_cb);
 	mathutils_matrix_col_cb_index = Mathutils_RegisterCallback(&mathutils_matrix_col_cb);
+	mathutils_matrix_translation_cb_index = Mathutils_RegisterCallback(&mathutils_matrix_translation_cb);
 
 	return submodule;
 }
diff --git a/source/blender/python/mathutils/mathutils_Matrix.c b/source/blender/python/mathutils/mathutils_Matrix.c
index 7e881fd5dda..039a62c130a 100644
--- a/source/blender/python/mathutils/mathutils_Matrix.c
+++ b/source/blender/python/mathutils/mathutils_Matrix.c
@@ -48,22 +48,53 @@ static int Matrix_ass_slice(MatrixObject *self, int begin, int end, PyObject *va
 static PyObject *matrix__apply_to_copy(PyNoArgsFunction matrix_func, MatrixObject *self);
 static PyObject *MatrixAccess_CreatePyObject(MatrixObject *matrix, const eMatrixAccess_t type);
 
-/* matrix row callbacks */
+static int matrix_row_vector_check(MatrixObject *mat, VectorObject *vec, int row)
+{
+	if ((vec->size != mat->num_col) || (row >= mat->num_row)) {
+		PyErr_SetString(PyExc_AttributeError,
+		                "Matrix(): "
+		                "owner matrix has been resized since this row vector was created");
+		return 0;
+	}
+	else {
+		return 1;
+	}
+}
+
+static int matrix_col_vector_check(MatrixObject *mat, VectorObject *vec, int col)
+{
+	if ((vec->size != mat->num_row) || (col >= mat->num_col)) {
+		PyErr_SetString(PyExc_AttributeError,
+		                "Matrix(): "
+		                "owner matrix has been resized since this column vector was created");
+		return 0;
+	}
+	else {
+		return 1;
+	}
+}
+
+/* ----------------------------------------------------------------------------
+ * matrix row callbacks
+ * this is so you can do matrix[i][j] = val OR matrix.row[i][j] = val */
+
 int mathutils_matrix_row_cb_index = -1;
 
-static int mathutils_matrix_vector_check(BaseMathObject *bmo)
+static int mathutils_matrix_row_check(BaseMathObject *bmo)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 	return BaseMath_ReadCallback(self);
 }
 
-static int mathutils_matrix_vector_get(BaseMathObject *bmo, int row)
+static int mathutils_matrix_row_get(BaseMathObject *bmo, int row)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 	int col;
 
 	if (BaseMath_ReadCallback(self) == -1)
 		return -1;
+	if (!matrix_row_vector_check(self, (VectorObject *)bmo, row))
+		return -1;
 
 	for (col = 0; col < self->num_col; col++) {
 		bmo->data[col] = MATRIX_ITEM(self, row, col);
@@ -72,13 +103,15 @@ static int mathutils_matrix_vector_get(BaseMathObject *bmo, int row)
 	return 0;
 }
 
-static int mathutils_matrix_vector_set(BaseMathObject *bmo, int row)
+static int mathutils_matrix_row_set(BaseMathObject *bmo, int row)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 	int col;
 
 	if (BaseMath_ReadCallback(self) == -1)
 		return -1;
+	if (!matrix_row_vector_check(self, (VectorObject *)bmo, row))
+		return -1;
 
 	for (col = 0; col < self->num_col; col++) {
 		MATRIX_ITEM(self, row, col) = bmo->data[col];
@@ -88,23 +121,27 @@ static int mathutils_matrix_vector_set(BaseMathObject *bmo, int row)
 	return 0;
 }
 
-static int mathutils_matrix_vector_get_index(BaseMathObject *bmo, int row, int col)
+static int mathutils_matrix_row_get_index(BaseMathObject *bmo, int row, int col)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 
 	if (BaseMath_ReadCallback(self) == -1)
 		return -1;
+	if (!matrix_row_vector_check(self, (VectorObject *)bmo, row))
+		return -1;
 
 	bmo->data[col] = MATRIX_ITEM(self, row, col);
 	return 0;
 }
 
-static int mathutils_matrix_vector_set_index(BaseMathObject *bmo, int row, int col)
+static int mathutils_matrix_row_set_index(BaseMathObject *bmo, int row, int col)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 
 	if (BaseMath_ReadCallback(self) == -1)
 		return -1;
+	if (!matrix_row_vector_check(self, (VectorObject *)bmo, row))
+		return -1;
 
 	MATRIX_ITEM(self, row, col) = bmo->data[col];
 
@@ -113,24 +150,27 @@ static int mathutils_matrix_vector_set_index(BaseMathObject *bmo, int row, int c
 }
 
 Mathutils_Callback mathutils_matrix_row_cb = {
-	mathutils_matrix_vector_check,
-	mathutils_matrix_vector_get,
-	mathutils_matrix_vector_set,
-	mathutils_matrix_vector_get_index,
-	mathutils_matrix_vector_set_index
+	mathutils_matrix_row_check,
+	mathutils_matrix_row_get,
+	mathutils_matrix_row_set,
+	mathutils_matrix_row_get_index,
+	mathutils_matrix_row_set_index
 };
-/* matrix vector callbacks, this is so you can do matrix[i][j] = val  */
 
-/* matrix row callbacks */
+
+/* ----------------------------------------------------------------------------
+ * matrix row callbacks
+ * this is so you can do matrix.col[i][j] = val */
+
 int mathutils_matrix_col_cb_index = -1;
 
-static int mathutils_matrix_column_check(BaseMathObject *bmo)
+static int mathutils_matrix_col_check(BaseMathObject *bmo)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 	return BaseMath_ReadCallback(self);
 }
 
-static int mathutils_matrix_column_get(BaseMathObject *bmo, int col)
+static int mathutils_matrix_col_get(BaseMathObject *bmo, int col)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 	int num_row;
@@ -138,6 +178,8 @@ static int mathutils_matrix_column_get(BaseMathObject *bmo, int col)
 
 	if (BaseMath_ReadCallback(self) == -1)
 		return -1;
+	if (!matrix_col_vector_check(self, (VectorObject *)bmo, col))
+		return -1;
 
 	/* for 'translation' size will always be '3' even on 4x4 vec */
 	num_row = MIN2(self->num_row, ((VectorObject *)bmo)->size);
@@ -149,7 +191,7 @@ static int mathutils_matrix_column_get(BaseMathObject *bmo, int col)
 	return 0;
 }
 
-static int mathutils_matrix_column_set(BaseMathObject *bmo, int col)
+static int mathutils_matrix_col_set(BaseMathObject *bmo, int col)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 	int num_row;
@@ -157,6 +199,8 @@ static int mathutils_matrix_column_set(BaseMathObject *bmo, int col)
 
 	if (BaseMath_ReadCallback(self) == -1)
 		return -1;
+	if (!matrix_col_vector_check(self, (VectorObject *)bmo, col))
+		return -1;
 
 	/* for 'translation' size will always be '3' even on 4x4 vec */
 	num_row = MIN2(self->num_row, ((VectorObject *)bmo)->size);
@@ -169,23 +213,27 @@ static int mathutils_matrix_column_set(BaseMathObject *bmo, int col)
 	return 0;
 }
 
-static int mathutils_matrix_column_get_index(BaseMathObject *bmo, int col, int row)
+static int mathutils_matrix_col_get_index(BaseMathObject *bmo, int col, int row)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 
 	if (BaseMath_ReadCallback(self) == -1)
 		return -1;
+	if (!matrix_col_vector_check(self, (VectorObject *)bmo, col))
+		return -1;
 
 	bmo->data[row] = MATRIX_ITEM(self, row, col);
 	return 0;
 }
 
-static int mathutils_matrix_column_set_index(BaseMathObject *bmo, int col, int row)
+static int mathutils_matrix_col_set_index(BaseMathObject *bmo, int col, int row)
 {
 	MatrixObject *self = (MatrixObject *)bmo->cb_user;
 
 	if (BaseMath_ReadCallback(self) == -1)
 		return -1;
+	if (!matrix_col_vector_check(self, (VectorObject *)bmo, col))
+		return -1;
 
 	MATRIX_ITEM(self, row, col) = bmo->data[row];
 
@@ -194,12 +242,91 @@ static int mathutils_matrix_column_set_index(BaseMathObject *bmo, int col, int r
 }
 
 Mathutils_Callback mathutils_matrix_col_cb = {
-	mathutils_matrix_column_check,
-	mathutils_matrix_column_get,
-	mathutils_matrix_column_set,
-	mathutils_matrix_column_get_index,
-	mathutils_matrix_column_set_index
+	mathutils_matrix_col_check,
+	mathutils_matrix_col_get,
+	mathutils_matrix_col_set,
+	mathutils_matrix_col_get_index,
+	mathutils_matrix_col_set_index
 };
+
+
+/* ----------------------------------------------------------------------------
+ * matrix row callbacks
+ * this is so you can do matrix.translation = val
+ * note, this is _exactly like matrix.col except the 4th component is always omitted */
+
+int mathutils_matrix_translation_cb_index = -1;
+
+static int mathutils_matrix_translation_check(BaseMathObject *bmo)
+{
+	MatrixObject *self = (MatrixObject *)bmo->cb_user;
+	return BaseMath_ReadCallback(self);
+}
+
+static int mathutils_matrix_translation_get(BaseMathObject *bmo, int col)
+{
+	MatrixObject *self = (MatrixObject *)bmo->cb_user;
+	int row;
+
+	if (BaseMath_ReadCallback(self) == -1)
+		return -1;
+
+	for (row = 0; row < 3; row++) {
+		bmo->data[row] = MATRIX_ITEM(self, row, col);
+	}
+
+	return 0;
+}
+
+static int mathutils_matrix_translation_set(BaseMathObject *bmo, int col)
+{
+	MatrixObject *self = (MatrixObject *)bmo->cb_user;
+	int row;
+
+	if (BaseMath_ReadCallback(self) == -1)
+		return -1;
+
+	for (row = 0; row < 3; row++) {
+		MATRIX_ITEM(self, row, col) = bmo->data[row];
+	}
+
+	(void)BaseMath_WriteCallback(self);
+	return 0;
+}
+
+static int mathutils_matrix_translation_get_index(BaseMathObject *bmo, int col, int row)
+{
+	MatrixObject *self = (MatrixObject *)bmo->cb_user;
+
+	if (BaseMath_ReadCallback(self) == -1)
+		return -1;
+
+	bmo->data[row] = MATRIX_ITEM(self, row, col);
+	return 0;
+}
+
+static int mathutils_matrix_translation_set_index(BaseMathObject *bmo, int col, int row)
+{
+	MatrixObject *self = (MatrixObject *)bmo->cb_user;
+
+	if (BaseMath_ReadCallback(self) == -1)
+		return -1;
+
+	MATRIX_ITEM(self, row, col) = bmo->data[row];
+
+	(void)BaseMath_WriteCallback(self);
+	return 0;
+}
+
+Mathutils_Callback mathutils_matrix_translation_cb = {
+	mathutils_matrix_translation_check,
+	mathutils_matrix_translation_get,
+	mathutils_matrix_translation_set,
+	mathutils_matrix_translation_get_index,
+	mathutils_matrix_translation_set_index
+};
+
+
 /* matrix column callbacks, this is so you can do matrix.translation = Vector()  */
 
 //----------------------------------mathutils.Matrix() -----------------
@@ -1972,7 +2099,7 @@ static PyObject *Matrix_translation_get(MatrixObject *self, void *UNUSED(closure
 		return NULL;
 	}
 
-	ret = (PyObject *)Vector_CreatePyObject_cb((PyObject *)self, 3, mathutils_matrix_col_cb_index, 3);
+	ret = (PyObject *)Vector_CreatePyObject_cb((PyObject *)self, 3, mathutils_matrix_translation_cb_index, 3);
 
 	return ret;
 }
diff --git a/source/blender/python/mathutils/mathutils_Matrix.h b/source/blender/python/mathutils/mathutils_Matrix.h
index 7602f98dea4..2ecbc55da35 100644
--- a/source/blender/python/mathutils/mathutils_Matrix.h
+++ b/source/blender/python/mathutils/mathutils_Matrix.h
@@ -75,8 +75,11 @@ PyObject *Matrix_CreatePyObject_cb(PyObject *user,
 
 extern int mathutils_matrix_row_cb_index; /* default */
 extern int mathutils_matrix_col_cb_index;
+extern int mathutils_matrix_translation_cb_index;
+
 extern struct Mathutils_Callback mathutils_matrix_row_cb; /* default */
 extern struct Mathutils_Callback mathutils_matrix_col_cb;
+extern struct Mathutils_Callback mathutils_matrix_translation_cb;
 
 void matrix_as_3x3(float mat[3][3], MatrixObject *self);