- modifier UI update (aka, find the modifier buttons!!)

 - moved back to editing buttons, where life is now cramped... switched
   to constraint style foldout panes, still a WIP. In particular not
   sure what buttons should be in header (and if current toggles stay
   in header if they should also be in an expanded pane). Also need
   new icons for move up/move down (and drag and drop would of course
   be nice). Finally current plane is to make it so modifiers will
   expand out in modifier pane for horizontal orientations instead of
   just going down down down to goblin town.
 - added error field to modifiers that is displayed in UI, need to have
   some way for modifiers to return errors back to interface (esp. important
   for python)
 - tweaked cage determination and handling, currently the editmode cage
   is determined by last modifier with OnCage set that is preceeded completely
   by modifiers that support mapping or are disabled in editmode. it is
   kinda confusing, but the interface only lets you toggle OnCage for modifiers
   that support it - it just might not be clear all the time why you can't
   toggle a certain modifier OnCage.
 - update displistmesh_copy to only copy edges if non-NULL

There is a display bug that already existed but is more obvious with
new modifiers where parts of the pane get drawn in a different area
after toggling editmode. It has to do with drawing parts of the interface
using GL instead of 100% buttons. I try to keep my grubby little toes
out of the interface code so this can wait for Ton to return.

5 files changed, 89 insertions, 26 deletions

diff --git a/source/blender/blenkernel/BKE_bad_level_calls.h b/source/blender/blenkernel/BKE_bad_level_calls.h
index bee768a5214..591e1913566 100644
--- a/source/blender/blenkernel/BKE_bad_level_calls.h
+++ b/source/blender/blenkernel/BKE_bad_level_calls.h
@@ -80,6 +80,7 @@ float Phong_Spec(float *, float *, float *, int);
 void waitcursor(int);
 void allqueue(unsigned short event, short val);
 #define REDRAWVIEW3D	0x4010
+#define REDRAWBUTSEDIT	0x4019
 struct Material;
 extern struct Material defmaterial;
 
diff --git a/source/blender/blenkernel/BKE_modifier.h b/source/blender/blenkernel/BKE_modifier.h
index b584fca68e1..19fc11424ce 100644
--- a/source/blender/blenkernel/BKE_modifier.h
+++ b/source/blender/blenkernel/BKE_modifier.h
@@ -180,8 +180,12 @@ void					modifier_free			(struct ModifierData *md);
 void					modifier_copyData		(struct ModifierData *md, struct ModifierData *target);
 int						modifier_dependsOnTime	(struct ModifierData *md);
 int						modifier_supportsMapping(struct ModifierData *md);
+int						modifier_couldBeCage	(struct ModifierData *md);
+void					modifier_setError		(struct ModifierData *md, char *format, ...);
 
 struct ModifierData*	modifiers_findByType	(struct ListBase *lb, ModifierType type);
+void					modifiers_clearErrors	(struct ListBase *lb);
+int						modifiers_getCageIndex	(struct ListBase *lb, int *lastPossibleCageIndex_r);
 
 #endif
 
diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index 33108dcd235..8bf5df95257 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -1310,6 +1310,8 @@ static void mesh_calc_modifiers(Object *ob, float (*inputVertexCos)[3], DerivedM
 	DerivedMesh *dm;
 	int numVerts = me->totvert;
 
+	modifiers_clearErrors(&ob->modifiers);
+
 	if (deform_r) *deform_r = NULL;
 	*final_r = NULL;
 
@@ -1447,33 +1449,19 @@ static void editmesh_calc_modifiers(DerivedMesh **cage_r, DerivedMesh **final_r)
 {
 	Object *ob = G.obedit;
 	EditMesh *em = G.editMesh;
-	ModifierData *md, *cageModifier;
+	ModifierData *md;
 	float (*deformedVerts)[3] = NULL;
 	DerivedMesh *dm;
-	int numVerts;
-
-		/* Find the last modifier acting on the cage. */
-	cageModifier = NULL;
-	for (md= ob->modifiers.first; md; md=md->next) {
-		ModifierTypeInfo *mti = modifierType_getInfo(md->type);
-
-		if (!(md->mode&eModifierMode_Realtime)) continue;
-		if (!(md->mode&eModifierMode_Editmode)) continue;
-		if (mti->isDisabled && mti->isDisabled(md)) continue;
-		if (!(mti->flags&eModifierTypeFlag_SupportsEditmode)) continue;
+	int i, numVerts, cageIndex = modifiers_getCageIndex(&ob->modifiers, NULL);
 
-		if (!modifier_supportsMapping(md) || !(md->mode&eModifierMode_OnCage))
-			break;
-
-		cageModifier = md;
-	}
+	modifiers_clearErrors(&ob->modifiers);
 
-	if (cage_r && !cageModifier) {
+	if (cage_r && cageIndex==-1) {
 		*cage_r = getEditMeshDerivedMesh(em, NULL);
 	}
 
 	dm = NULL;
-	for (md= ob->modifiers.first; md; md=md->next) {
+	for (i=0,md= ob->modifiers.first; md; i++,md=md->next) {
 		ModifierTypeInfo *mti = modifierType_getInfo(md->type);
 
 		if (!(md->mode&eModifierMode_Realtime)) continue;
@@ -1521,7 +1509,7 @@ static void editmesh_calc_modifiers(DerivedMesh **cage_r, DerivedMesh **final_r)
 			}
 		}
 
-		if (cage_r && md==cageModifier) {
+		if (cage_r && i==cageIndex) {
 			if (dm && deformedVerts) {
 					// XXX  this is not right, need to convert the dm
 				*cage_r = dm;
diff --git a/source/blender/blenkernel/intern/displist.c b/source/blender/blenkernel/intern/displist.c
index 45d20bddaa3..fa1d61bbf02 100644
--- a/source/blender/blenkernel/intern/displist.c
+++ b/source/blender/blenkernel/intern/displist.c
@@ -128,7 +128,7 @@ DispListMesh *displistmesh_copy(DispListMesh *odlm)
 {
 	DispListMesh *ndlm= MEM_dupallocN(odlm);
 	ndlm->mvert= MEM_dupallocN(odlm->mvert);
-	ndlm->medge= MEM_dupallocN(odlm->medge);
+	if (odlm->medge) ndlm->medge= MEM_dupallocN(odlm->medge);
 	ndlm->mface= MEM_dupallocN(odlm->mface);
 	if (odlm->nors) ndlm->nors = MEM_dupallocN(odlm->nors);
 	if (odlm->mcol) ndlm->mcol= MEM_dupallocN(odlm->mcol);
diff --git a/source/blender/blenkernel/intern/modifier.c b/source/blender/blenkernel/intern/modifier.c
index b5453ed6001..7aa26393443 100644
--- a/source/blender/blenkernel/intern/modifier.c
+++ b/source/blender/blenkernel/intern/modifier.c
@@ -1,4 +1,5 @@
 #include "string.h"
+#include "stdarg.h"
 #include "math.h"
 
 #include "BLI_blenlib.h"
@@ -14,6 +15,7 @@
 #include "DNA_scene_types.h"
 #include "BLI_editVert.h"
 
+#include "BKE_bad_level_calls.h"
 #include "BKE_global.h"
 #include "BKE_utildefines.h"
 #include "BKE_DerivedMesh.h"
@@ -752,8 +754,8 @@ static void *decimateModifier_applyModifier(ModifierData *md, Object *ob, void *
 	}
 
 	if(numTris<3) {
-		// ("You must have more than 3 input faces selected.");
-		return NULL;
+		modifier_setError(md, "There must be more than 3 input faces (triangles).");
+		goto exit;
 	}
 
 	lod.vertex_buffer= MEM_mallocN(3*sizeof(float)*totvert, "vertices");
@@ -830,19 +832,20 @@ static void *decimateModifier_applyModifier(ModifierData *md, Object *ob, void *
 			}
 		}
 		else {
-			// No memory
+			modifier_setError(md, "Out of memory.");
 		}
 
 		LOD_FreeDecimationData(&lod);
 	}
 	else {
-		// Non-manifold mesh
+		modifier_setError(md, "Non-manifold mesh as input.");
 	}
 
 	MEM_freeN(lod.vertex_buffer);
 	MEM_freeN(lod.vertex_normal_buffer);
 	MEM_freeN(lod.triangle_index_buffer);
 
+exit:
 	if (dlm) displistmesh_free(dlm);
 
 	if (ndlm) {
@@ -1057,7 +1060,7 @@ ModifierData *modifier_new(int type)
 	ModifierData *md = MEM_callocN(mti->structSize, mti->structName);
 
 	md->type = type;
-	md->mode = eModifierMode_Realtime|eModifierMode_Render;
+	md->mode = eModifierMode_Realtime|eModifierMode_Render|eModifierMode_Expanded;
 
 	if (mti->flags&eModifierTypeFlag_EnableInEditmode)
 		md->mode |= eModifierMode_Editmode;
@@ -1072,6 +1075,7 @@ void modifier_free(ModifierData *md)
 	ModifierTypeInfo *mti = modifierType_getInfo(md->type);
 
 	if (mti->freeData) mti->freeData(md);
+	if (md->error) MEM_freeN(md->error);
 
 	MEM_freeN(md);
 }
@@ -1103,6 +1107,20 @@ ModifierData *modifiers_findByType(struct ListBase *lb, ModifierType type)
 	return md;
 }
 
+void modifiers_clearErrors(struct ListBase *lb)
+{
+	ModifierData *md = lb->first;
+
+	for (; md; md=md->next) {
+		if (md->error) {
+			MEM_freeN(md->error);
+			md->error = NULL;
+
+			allqueue(REDRAWBUTSEDIT, 0);
+		}
+	}
+}
+
 void modifier_copyData(ModifierData *md, ModifierData *target)
 {
 	ModifierTypeInfo *mti = modifierType_getInfo(md->type);
@@ -1112,3 +1130,55 @@ void modifier_copyData(ModifierData *md, ModifierData *target)
 	if (mti->copyData)
 		mti->copyData(md, target);
 }
+
+int modifier_couldBeCage(ModifierData *md)
+{
+	ModifierTypeInfo *mti = modifierType_getInfo(md->type);
+
+	return (	(md->mode&eModifierMode_Realtime) &&
+				(md->mode&eModifierMode_Editmode) &&
+				(!mti->isDisabled || !mti->isDisabled(md)) &&
+				modifier_supportsMapping(md));	
+}
+
+void modifier_setError(ModifierData *md, char *format, ...)
+{
+	char buffer[2048];
+	va_list ap;
+
+	va_start(ap, format);
+	vsprintf(buffer, format, ap);
+	va_end(ap);
+
+	if (md->error)
+		MEM_freeN(md->error);
+
+	md->error = BLI_strdup(buffer);
+
+	allqueue(REDRAWBUTSEDIT, 0);
+}
+
+int modifiers_getCageIndex(ListBase *lb, int *lastPossibleCageIndex_r)
+{
+	ModifierData *md = lb->first;
+	int i, cageIndex = -1;
+
+		/* Find the last modifier acting on the cage. */
+	for (i=0; md; i++,md=md->next) {
+		ModifierTypeInfo *mti = modifierType_getInfo(md->type);
+
+		if (!(md->mode&eModifierMode_Realtime)) continue;
+		if (!(md->mode&eModifierMode_Editmode)) continue;
+		if (mti->isDisabled && mti->isDisabled(md)) continue;
+		if (!(mti->flags&eModifierTypeFlag_SupportsEditmode)) continue;
+
+		if (!modifier_supportsMapping(md))
+			break;
+
+		if (lastPossibleCageIndex_r) *lastPossibleCageIndex_r = i;
+		if (md->mode&eModifierMode_OnCage)
+			cageIndex = i;
+	}
+
+	return cageIndex;
+}