Fix transform crash in rare cases

In some cases transform modes would use the custom-data pointer,
other times the transform conversion functions would.

However with some combinations (bone mirror + bend for eg),
both conversion & transform mode would use this pointer causing a crash.

Fix this by having 2 custom-data pointers:
one for the mode, another for the data-type.

This also simplifies time-slide which was conditionally mixing mode/type data in the one array.

1 files changed, 44 insertions, 87 deletions

diff --git a/source/blender/editors/transform/transform_conversions.c b/source/blender/editors/transform/transform_conversions.c
index 4514171c991..df0edae9a56 100644
--- a/source/blender/editors/transform/transform_conversions.c
+++ b/source/blender/editors/transform/transform_conversions.c
@@ -1088,7 +1088,7 @@ static void createTransPose(TransInfo *t, Object *ob)
 void restoreBones(TransInfo *t)
 {
 	bArmature *arm = t->obedit->data;
-	BoneInitData *bid = t->customData;
+	BoneInitData *bid = t->custom.type.data;
 	EditBone *ebo;
 
 	while (bid->bone) {
@@ -1177,8 +1177,8 @@ static void createTransArmatureVerts(TransInfo *t)
 	td = t->data = MEM_callocN(t->total * sizeof(TransData), "TransEditBone");
 
 	if (mirror) {
-		t->customData = bid = MEM_mallocN((total_mirrored + 1) * sizeof(BoneInitData), "BoneInitData");
-		t->flag |= T_FREE_CUSTOMDATA;
+		t->custom.type.data = bid = MEM_mallocN((total_mirrored + 1) * sizeof(BoneInitData), "BoneInitData");
+		t->custom.type.use_free = true;
 	}
 
 	i = 0;
@@ -3087,9 +3087,8 @@ static void createTransNlaData(bContext *C, TransInfo *t)
 	
 	t->data = MEM_callocN(t->total * sizeof(TransData), "TransData(NLA Editor)");
 	td = t->data;
-	t->customData = MEM_callocN(t->total * sizeof(TransDataNla), "TransDataNla (NLA Editor)");
-	tdn = t->customData;
-	t->flag |= T_FREE_CUSTOMDATA;
+	t->custom.type.data = tdn = MEM_callocN(t->total * sizeof(TransDataNla), "TransDataNla (NLA Editor)");
+	t->custom.type.use_free = true;
 	
 	/* loop 2: build transdata array */
 	for (ale = anim_data.first; ale; ale = ale->next) {
@@ -3550,15 +3549,9 @@ typedef struct tGPFtransdata {
 /* This function helps flush transdata written to tempdata into the gp-frames  */
 void flushTransIntFrameActionData(TransInfo *t)
 {
-	tGPFtransdata *tfd;
+	tGPFtransdata *tfd = t->custom.type.data;
 	int i;
 
-	/* find the first one to start from */
-	if (t->mode == TFM_TIME_SLIDE)
-		tfd = (tGPFtransdata *)((float *)(t->customData) + 2);
-	else
-		tfd = (tGPFtransdata *)(t->customData);
-
 	/* flush data! */
 	for (i = 0; i < t->total; i++, tfd++) {
 		*(tfd->sdata) = iroundf(tfd->val);
@@ -3726,20 +3719,8 @@ static void createTransActionData(bContext *C, TransInfo *t)
 	td2d = t->data2d;
 	
 	if (ELEM(ac.datatype, ANIMCONT_GPENCIL, ANIMCONT_MASK)) {
-		if (t->mode == TFM_TIME_SLIDE) {
-			t->customData = MEM_callocN((sizeof(float) * 2) + (sizeof(tGPFtransdata) * count), "TimeSlide + tGPFtransdata");
-			t->flag |= T_FREE_CUSTOMDATA;
-			tfd = (tGPFtransdata *)((float *)(t->customData) + 2);
-		}
-		else {
-			t->customData = MEM_callocN(sizeof(tGPFtransdata) * count, "tGPFtransdata");
-			t->flag |= T_FREE_CUSTOMDATA;
-			tfd = (tGPFtransdata *)(t->customData);
-		}
-	}
-	else if (t->mode == TFM_TIME_SLIDE) {
-		t->customData = MEM_callocN(sizeof(float) * 2, "TimeSlide Min/Max");
-		t->flag |= T_FREE_CUSTOMDATA;
+		t->custom.type.data = tfd = MEM_callocN(sizeof(tGPFtransdata) * count, "tGPFtransdata");
+		t->custom.type.use_free = true;
 	}
 	
 	/* loop 2: build transdata array */
@@ -3781,31 +3762,6 @@ static void createTransActionData(bContext *C, TransInfo *t)
 			td = ActionFCurveToTransData(td, &td2d, fcu, adt, t->frame_side, cfra, is_prop_edit, ypos);
 		}
 	}
-	
-	/* check if we're supposed to be setting minx/maxx for TimeSlide */
-	if (t->mode == TFM_TIME_SLIDE) {
-		float min = 999999999.0f, max = -999999999.0f;
-		int i;
-		
-		td = t->data;
-		for (i = 0; i < count; i++, td++) {
-			if (min > *(td->val)) min = *(td->val);
-			if (max < *(td->val)) max = *(td->val);
-		}
-		
-		if (min == max) {
-			/* just use the current frame ranges */
-			min = (float)PSFRA;
-			max = (float)PEFRA;
-		}
-		
-		/* minx/maxx values used by TimeSlide are stored as a
-		 * calloced 2-float array in t->customData. This gets freed
-		 * in postTrans (T_FREE_CUSTOMDATA).
-		 */
-		*((float *)(t->customData)) = min;
-		*((float *)(t->customData) + 1) = max;
-	}
 
 	/* calculate distances for proportional editing */
 	if (is_prop_edit) {
@@ -4158,12 +4114,12 @@ static void createTransGraphEditData(bContext *C, TransInfo *t)
 	t->data = MEM_callocN(t->total * sizeof(TransData), "TransData (Graph Editor)");
 	/* for each 2d vert a 3d vector is allocated, so that they can be treated just as if they were 3d verts */
 	t->data2d = MEM_callocN(t->total * sizeof(TransData2D), "TransData2D (Graph Editor)");
-	t->customData = MEM_callocN(t->total * sizeof(TransDataGraph), "TransDataGraph");
-	t->flag |= T_FREE_CUSTOMDATA;
+	t->custom.type.data = MEM_callocN(t->total * sizeof(TransDataGraph), "TransDataGraph");
+	t->custom.type.use_free = true;
 	
 	td = t->data;
 	td2d = t->data2d;
-	tdg = t->customData;
+	tdg = t->custom.type.data;
 	
 	/* precompute space-conversion matrices for dealing with non-uniform scaling of Graph Editor */
 	unit_m3(mtx);
@@ -4565,7 +4521,7 @@ void flushTransGraphData(TransInfo *t)
 	int a;
 
 	/* flush to 2d vector from internally used 3d vector */
-	for (a = 0, td = t->data, td2d = t->data2d, tdg = t->customData;
+	for (a = 0, td = t->data, td2d = t->data2d, tdg = t->custom.type.data;
 	     a < t->total;
 	     a++, td++, td2d++, tdg++)
 	{
@@ -4928,7 +4884,7 @@ static void SeqTransDataBounds(TransInfo *t, ListBase *seqbase, TransSeq *ts)
 }
 
 
-static void freeSeqData(TransInfo *t)
+static void freeSeqData(TransInfo *t, TransCustomData *custom_data)
 {
 	Editing *ed = BKE_sequencer_editing_get(t->scene, false);
 
@@ -5089,11 +5045,11 @@ static void freeSeqData(TransInfo *t)
 		}
 	}
 
-	if ((t->customData != NULL) && (t->flag & T_FREE_CUSTOMDATA)) {
-		TransSeq *ts = t->customData;
+	if ((custom_data->data != NULL) && custom_data->use_free) {
+		TransSeq *ts = custom_data->data;
 		MEM_freeN(ts->tdseq);
-		MEM_freeN(t->customData);
-		t->customData = NULL;
+		MEM_freeN(custom_data->data);
+		custom_data->data = NULL;
 	}
 	if (t->data) {
 		MEM_freeN(t->data); // XXX postTrans usually does this
@@ -5121,7 +5077,7 @@ static void createTransSeqData(bContext *C, TransInfo *t)
 		return;
 	}
 
-	t->customFree = freeSeqData;
+	t->custom.type.free_cb = freeSeqData;
 
 	xmouse = (int)UI_view2d_region_to_view_x(v2d, t->mouse.imval[0]);
 
@@ -5167,11 +5123,11 @@ static void createTransSeqData(bContext *C, TransInfo *t)
 		return;
 	}
 
-	t->customData = ts = MEM_mallocN(sizeof(TransSeq), "transseq");
+	t->custom.type.data = ts = MEM_mallocN(sizeof(TransSeq), "transseq");
+	t->custom.type.use_free = true;
 	td = t->data = MEM_callocN(t->total * sizeof(TransData), "TransSeq TransData");
 	td2d = t->data2d = MEM_callocN(t->total * sizeof(TransData2D), "TransSeq TransData2D");
 	ts->tdseq = tdsq = MEM_callocN(t->total * sizeof(TransDataSeq), "TransSeq TransDataSeq");
-	t->flag |= T_FREE_CUSTOMDATA;
 
 	/* loop 2: build transdata array */
 	SeqToTransData_Recursive(t, ed->seqbasep, td, td2d, tdsq);
@@ -5956,10 +5912,13 @@ void special_aftertrans_update(bContext *C, TransInfo *t)
 	
 	if (t->spacetype == SPACE_VIEW3D) {
 		if (t->obedit) {
+			/* Special Exception:
+			 * We don't normally access 't->custom.mode' here, but its needed in this case. */
+
 			if (canceled == 0) {
 				/* we need to delete the temporary faces before automerging */
 				if (t->mode == TFM_EDGE_SLIDE) {
-					EdgeSlideData *sld = t->customData;
+					EdgeSlideData *sld = t->custom.mode.data;
 
 					/* handle multires re-projection, done
 					 * on transform completion since it's
@@ -5972,7 +5931,7 @@ void special_aftertrans_update(bContext *C, TransInfo *t)
 				}
 				else if (t->mode == TFM_VERT_SLIDE) {
 					/* as above */
-					VertSlideData *sld = t->customData;
+					VertSlideData *sld = t->custom.mode.data;
 					projectVertSlideData(t, true);
 					freeVertSlideTempFaces(sld);
 				}
@@ -5983,13 +5942,13 @@ void special_aftertrans_update(bContext *C, TransInfo *t)
 			}
 			else {
 				if (t->mode == TFM_EDGE_SLIDE) {
-					EdgeSlideData *sld = t->customData;
+					EdgeSlideData *sld = t->custom.mode.data;
 
 					sld->perc = 0.0;
 					projectEdgeSlideData(t, false);
 				}
 				else if (t->mode == TFM_VERT_SLIDE) {
-					VertSlideData *sld = t->customData;
+					VertSlideData *sld = t->custom.mode.data;
 
 					sld->perc = 0.0;
 					projectVertSlideData(t, false);
@@ -6749,16 +6708,15 @@ static void planeTrackToTransData(const int framenr, TransData *td, TransData2D
 	}
 }
 
-static void transDataTrackingFree(TransInfo *t)
+static void transDataTrackingFree(TransInfo *UNUSED(t), TransCustomData *custom_data)
 {
-	TransDataTracking *tdt = t->customData;
-
-	if (tdt) {
+	if (custom_data->data) {
+		TransDataTracking *tdt = custom_data->data;
 		if (tdt->smarkers)
 			MEM_freeN(tdt->smarkers);
 
 		MEM_freeN(tdt);
-		t->customData = NULL;
+		custom_data->data = NULL;
 	}
 }
 
@@ -6810,9 +6768,9 @@ static void createTransTrackingTracksData(bContext *C, TransInfo *t)
 
 	td = t->data = MEM_callocN(t->total * sizeof(TransData), "TransTracking TransData");
 	td2d = t->data2d = MEM_callocN(t->total * sizeof(TransData2D), "TransTracking TransData2D");
-	tdt = t->customData = MEM_callocN(t->total * sizeof(TransDataTracking), "TransTracking TransDataTracking");
+	tdt = t->custom.type.data = MEM_callocN(t->total * sizeof(TransDataTracking), "TransTracking TransDataTracking");
 
-	t->customFree = transDataTrackingFree;
+	t->custom.type.free_cb = transDataTrackingFree;
 
 	/* create actual data */
 	track = tracksbase->first;
@@ -6947,9 +6905,8 @@ static void createTransTrackingCurvesData(bContext *C, TransInfo *t)
 
 	td = t->data = MEM_callocN(t->total * sizeof(TransData), "TransTracking TransData");
 	td2d = t->data2d = MEM_callocN(t->total * sizeof(TransData2D), "TransTracking TransData2D");
-	tdt = t->customData = MEM_callocN(t->total * sizeof(TransDataTracking), "TransTracking TransDataTracking");
-
-	t->customFree = transDataTrackingFree;
+	t->custom.type.data = tdt = MEM_callocN(t->total * sizeof(TransDataTracking), "TransTracking TransDataTracking");
+	t->custom.type.free_cb = transDataTrackingFree;
 
 	/* create actual data */
 	track = tracksbase->first;
@@ -7013,10 +6970,11 @@ static void cancelTransTracking(TransInfo *t)
 {
 	SpaceClip *sc = t->sa->spacedata.first;
 	int i, framenr = ED_space_clip_get_clip_frame_number(sc);
+	TransDataTracking *tdt_array = t->custom.type.data;
 
 	i = 0;
 	while (i < t->total) {
-		TransDataTracking *tdt = (TransDataTracking *) t->customData + i;
+		TransDataTracking *tdt = &tdt_array[i];
 
 		if (tdt->mode == transDataTracking_ModeTracks) {
 			MovieTrackingTrack *track = tdt->track;
@@ -7073,7 +7031,7 @@ void flushTransTracking(TransInfo *t)
 		cancelTransTracking(t);
 
 	/* flush to 2d vector from internally used 3d vector */
-	for (a = 0, td = t->data, td2d = t->data2d, tdt = t->customData; a < t->total; a++, td2d++, td++, tdt++) {
+	for (a = 0, td = t->data, td2d = t->data2d, tdt = t->custom.type.data; a < t->total; a++, td2d++, td++, tdt++) {
 		if (tdt->mode == transDataTracking_ModeTracks) {
 			float loc2d[2];
 
@@ -7404,9 +7362,8 @@ static void createTransMaskingData(bContext *C, TransInfo *t)
 	/* for each 2d uv coord a 3d vector is allocated, so that they can be
 	 * treated just as if they were 3d verts */
 	td2d = t->data2d = MEM_callocN(t->total * sizeof(TransData2D), "TransObData2D(Mask Editing)");
-	tdm = t->customData = MEM_callocN(t->total * sizeof(TransDataMasking), "TransDataMasking(Mask Editing)");
-
-	t->flag |= T_FREE_CUSTOMDATA;
+	t->custom.type.data = tdm = MEM_callocN(t->total * sizeof(TransDataMasking), "TransDataMasking(Mask Editing)");
+	t->custom.type.use_free = true;
 
 	/* create data */
 	for (masklay = mask->masklayers.first; masklay; masklay = masklay->next) {
@@ -7468,7 +7425,7 @@ void flushTransMasking(TransInfo *t)
 	inv[1] = 1.0f / asp[1];
 
 	/* flush to 2d vector from internally used 3d vector */
-	for (a = 0, td = t->data2d, tdm = t->customData; a < t->total; a++, td++, tdm++) {
+	for (a = 0, td = t->data2d, tdm = t->custom.type.data; a < t->total; a++, td++, tdm++) {
 		td->loc2d[0] = td->loc[0] * inv[0];
 		td->loc2d[1] = td->loc[1] * inv[1];
 		mul_m3_v2(tdm->parent_inverse_matrix, td->loc2d);
@@ -7616,8 +7573,8 @@ static void createTransPaintCurveVerts(bContext *C, TransInfo *t)
 	t->total = total;
 	td2d = t->data2d = MEM_callocN(t->total * sizeof(TransData2D), "TransData2D");
 	td = t->data = MEM_callocN(t->total * sizeof(TransData), "TransData");
-	tdpc = t->customData = MEM_callocN(t->total * sizeof(TransDataPaintCurve), "TransDataPaintCurve");
-	t->flag |= T_FREE_CUSTOMDATA;
+	t->custom.type.data = tdpc = MEM_callocN(t->total * sizeof(TransDataPaintCurve), "TransDataPaintCurve");
+	t->custom.type.use_free = true;
 
 	for (pcp = pc->points, i = 0; i < pc->tot_points; i++, pcp++) {
 		if (PC_IS_ANY_SEL(pcp)) {
@@ -7649,7 +7606,7 @@ void flushTransPaintCurve(TransInfo *t)
 {
 	int i;
 	TransData2D *td2d = t->data2d;
-	TransDataPaintCurve *tdpc = (TransDataPaintCurve *)t->customData;
+	TransDataPaintCurve *tdpc = t->custom.type.data;
 
 	for (i = 0; i < t->total; i++, tdpc++, td2d++) {
 		PaintCurvePoint *pcp = tdpc->pcp;