New: CrazySpace [tm] correction

When Modifiers are used in Edit Mode to show the deformed result for
editing, all actual coordinates Blender works with are still the ones from
the original Cage. You can notice that with the Transform Widget or
helper lines while transforming.

Even worse, the actual transformations still happened on the original Cage
as well, making it very hard to edit. That caused the feature to be named
"CrazySpace" (baptized by Andy, afaik?).

This commit calculates the deformation transformation per vertex, and
inverse corrects it, so it's more intuitive editing this way.

Unfortunately all the deformation features of Blender don't use matrices
for defining deform, so the existing code cannot be re-used to retrieve
the correct deformation matrix per vertex. The solution I found is based
on calculating per face the transformation based on its first 3 vertices,
and store this transformation averaged in the face's vertices.
The solution can also only work on entire faces, because the full deform
can only be retrieved using 3 vertices. (using 2 vertices will miss edge-
aligned rotation, using 1 vertex can only retrieve translation).

By deriving the deformations per face, small errors will still happen,
especially on very low-poly Meshes with extreme deformations.

The only alternative I know now, is providing each vertex in
a mesh with 2 extreme small tangent vectors, which get deformed using the
existing code as well. That will mess up the existing deformation code too
much though, this solution has the benefit it works with each deform we can
up with later too.

Last note about CrazySpace: it can only be used to tweak Meshes. Do not
even try to add vertices, extrude, or duplicate. Probably we should disable
this... but preventing user errors isn't always power-user-friendly, eh. :)

3 files changed, 133 insertions, 10 deletions

diff --git a/source/blender/blenkernel/BKE_modifier.h b/source/blender/blenkernel/BKE_modifier.h
index 0ceb39db401..b90758a52fb 100644
--- a/source/blender/blenkernel/BKE_modifier.h
+++ b/source/blender/blenkernel/BKE_modifier.h
@@ -202,6 +202,7 @@ int				modifiers_getCageIndex		(struct Object *ob, int *lastPossibleCageIndex_r)
 
 int				modifiers_isSoftbodyEnabled	(struct Object *ob);
 struct Object*	modifiers_isDeformedByArmature(struct Object *ob);
+int				modifiers_isDeformed		(struct Object *ob);
 
 ModifierData*	modifiers_getVirtualModifierList	(struct Object *ob);
 
diff --git a/source/blender/blenkernel/intern/modifier.c b/source/blender/blenkernel/intern/modifier.c
index d8d037d31a9..81319cfd555 100644
--- a/source/blender/blenkernel/intern/modifier.c
+++ b/source/blender/blenkernel/intern/modifier.c
@@ -1461,6 +1461,8 @@ void modifier_setError(ModifierData *md, char *format, ...)
 	allqueue(REDRAWBUTSEDIT, 0);
 }
 
+/* used for buttons, to find out if the 'draw deformed in editmode' option is there */
+/* also used in transform_conversion.c, to detect CrazySpace [tm] (2nd arg then is NULL) */
 int modifiers_getCageIndex(Object *ob, int *lastPossibleCageIndex_r)
 {
 	ModifierData *md = ob->modifiers.first;
@@ -1564,10 +1566,21 @@ Object *modifiers_isDeformedByArmature(Object *ob)
 			return amd->object;
 		}
 	}
-	
-	if(ob->parent && ob->parent->type==OB_ARMATURE)
-		if(ob->partype==PARSKEL)
-			return ob->parent;
 
 	return NULL;
-}
\ No newline at end of file
+}
+
+int modifiers_isDeformed(Object *ob)
+{
+	ModifierData *md = modifiers_getVirtualModifierList(ob);
+	
+	for (; md; md=md->next) {
+		if (md->type==eModifierType_Armature)
+			return 1;
+		if (md->type==eModifierType_Curve)
+			return 1;
+		if (md->type==eModifierType_Lattice)
+			return 1;
+	}
+	return 0;
+}
diff --git a/source/blender/src/transform_conversions.c b/source/blender/src/transform_conversions.c
index 3abc2a45cff..2b67b4dcaa3 100755
--- a/source/blender/src/transform_conversions.c
+++ b/source/blender/src/transform_conversions.c
@@ -78,6 +78,7 @@
 #include "BKE_constraint.h"
 #include "BKE_depsgraph.h"
 #include "BKE_displist.h"
+#include "BKE_DerivedMesh.h"
 #include "BKE_effect.h"
 #include "BKE_font.h"
 #include "BKE_global.h"
@@ -1104,13 +1105,95 @@ static void VertsToTransData(TransData *td, EditVert *eve)
 	td->val = NULL;
 }
 
+static void make_vertexcos__mapFunc(void *userData, int index, float *co, float *no_f, short *no_s)
+{
+	float *vec = userData;
+	
+	vec+= 3*index;
+	VECCOPY(vec, co);
+}
+
+static float *get_mapped_editverts(void)
+{
+	int needsFree;
+	DerivedMesh *dm= editmesh_get_derived_cage(&needsFree);
+	float *vertexcos;
+	
+	vertexcos= MEM_mallocN(3*sizeof(float)*G.totvert, "vertexcos map");
+	
+	dm->foreachMappedVert(dm, make_vertexcos__mapFunc, vertexcos);
+	
+	if (needsFree) dm->release(dm);
+	return vertexcos;
+}
+
+/* helper for below, interpolates or assigns and increments */
+static float *crazy_quat_blend(EditVert *eve, float *quat)
+{
+	if(eve->vn==NULL) {
+		eve->vn= (EditVert *)quat;
+		QUATCOPY(quat+4, quat);
+		return quat+4;
+	}
+	else {
+		float *q1= (float *)eve->vn;
+		QuatInterpol(q1, q1, quat, 0.5f);
+		return quat;
+	}
+}
+
+static void set_crazyspace_quats(float *mappedcos, float *quats)
+{
+	EditMesh *em = G.editMesh;
+	EditVert *eve, *prev;
+	EditFace *efa;
+	float q1[4], q2[4];
+	float *v1, *v2, *v3, *quatp;
+	int index= 0;
+	
+	/* 2 abused locations in vertices */
+	for(eve= em->verts.first; eve; eve= eve->next, index++) {
+		eve->vn= NULL;
+		eve->prev= (EditVert *)index;
+	}
+	
+	quatp= quats;
+	for(efa= em->faces.first; efa; efa= efa->next) {
+		/* vertex f1 flags were set for transform */
+		
+		if( (efa->v1->f1 && efa->v1->vn==NULL) || (efa->v2->f1 && efa->v2->vn==NULL)
+			|| (efa->v3->f1 && efa->v3->vn==NULL) || (efa->v4 && efa->v4->f1 && efa->v4->vn==NULL) ) {
+		
+			triatoquat(efa->v1->co, efa->v2->co, efa->v3->co, q1);
+			
+			/* retrieve mapped coordinates */
+			v1= mappedcos + 3*( (int)(efa->v1->prev) );
+			v2= mappedcos + 3*( (int)(efa->v2->prev) );
+			v3= mappedcos + 3*( (int)(efa->v3->prev) );
+			triatoquat(v1, v2, v3, q2);
+			
+			QuatSub(quatp, q2, q1);
+			
+			if(efa->v1->f1) quatp= crazy_quat_blend(efa->v1, quatp);
+			if(efa->v2->f1) quatp= crazy_quat_blend(efa->v2, quatp);
+			if(efa->v3->f1) quatp= crazy_quat_blend(efa->v3, quatp);
+			if(efa->v4 && efa->v4->f1) quatp= crazy_quat_blend(efa->v4, quatp);
+		}
+	}
+
+	/* restore abused prev pointer */
+	for(prev= NULL, eve= em->verts.first; eve; prev= eve, eve= eve->next)
+		eve->prev= prev;
+
+}
+
 static void createTransEditVerts(TransInfo *t)
 {
 	TransData *tob = NULL;
 	EditMesh *em = G.editMesh;
 	EditVert *eve;
 	EditVert **nears = NULL;
-	float *vectors = NULL;
+	float *vectors = NULL, *mappedcos = NULL, *quats= NULL;
 	float mtx[3][3], smtx[3][3];
 	int count=0, countsel=0;
 	int propmode = t->flag & T_PROP_EDIT;
@@ -1169,6 +1252,16 @@ static void createTransEditVerts(TransInfo *t)
 
 	if(propmode) editmesh_set_connectivity_distance(t->total, vectors, nears);
 	
+	/* detect CrazySpace [tm] */
+	if(modifiers_getCageIndex(G.obedit, NULL)>=0) {
+		if(modifiers_isDeformed(G.obedit)) {
+			mappedcos= get_mapped_editverts();
+			/* add one more quaternion, because of crazy_quat_blend */
+			quats= MEM_mallocN( (t->total+1)*sizeof(float)*4, "crazy quats");
+			set_crazyspace_quats(mappedcos, quats);
+		}
+	}
+	
 	for (eve=em->verts.first; eve; eve=eve->next) {
 		if(eve->h==0) {
 			if(propmode || eve->f1) {
@@ -1188,9 +1281,21 @@ static void createTransEditVerts(TransInfo *t)
 					}
 				}
 				
-				Mat3CpyMat3(tob->smtx, smtx);
-				Mat3CpyMat3(tob->mtx, mtx);
-
+				/* CrazySpace */
+				if(quats && eve->vn) {
+					float mat[3][3], imat[3][3], qmat[3][3];
+					
+					QuatToMat3((float *)eve->vn, qmat);
+					Mat3MulMat3(mat, mtx, qmat);
+					Mat3Inv(imat, mat);
+					
+					Mat3CpyMat3(tob->smtx, imat);
+					Mat3CpyMat3(tob->mtx, mat);
+				}
+				else {
+					Mat3CpyMat3(tob->smtx, smtx);
+					Mat3CpyMat3(tob->mtx, mtx);
+				}
 				tob++;
 			}
 		}	
@@ -1199,7 +1304,11 @@ static void createTransEditVerts(TransInfo *t)
 		MEM_freeN(vectors);
 		MEM_freeN(nears);
 	}
-
+	/* crazy space free */
+	if(mappedcos)
+		MEM_freeN(mappedcos);
+	if(quats)
+		MEM_freeN(quats);
 }
 
 /* ********************* UV ****************** */