Added vertex masking support for the weight paint vgroup functions including:

Normalize, Normalize All, Invert, Clean, and Levels.

Face mask and vertex select/mask are now mutually exclusive options.

Went ahead and allowed vertex selection support and masking for vertex paint.

8 files changed, 156 insertions, 33 deletions

diff --git a/source/blender/blenkernel/intern/DerivedMesh.c b/source/blender/blenkernel/intern/DerivedMesh.c
index a35ad48e599..ba67e580aca 100644
--- a/source/blender/blenkernel/intern/DerivedMesh.c
+++ b/source/blender/blenkernel/intern/DerivedMesh.c
@@ -2333,7 +2333,7 @@ static void clear_mesh_caches(Object *ob)
 static void mesh_build_data(Scene *scene, Object *ob, CustomDataMask dataMask)
 {
 	Object *obact = scene->basact?scene->basact->object:NULL;
-	int editing = paint_facesel_test(ob);
+	int editing = paint_facesel_test(ob) || paint_vertsel_test(ob);// Jason: paint_vertsel_test
 	/* weight paint and face select need original indices because of selection buffer drawing */
 	int needMapping = (ob==obact) && (editing || (ob->mode & (OB_MODE_WEIGHT_PAINT|OB_MODE_VERTEX_PAINT)));
 
diff --git a/source/blender/blenkernel/intern/paint.c b/source/blender/blenkernel/intern/paint.c
index 9a0fc541bce..5dc58ba4aa8 100644
--- a/source/blender/blenkernel/intern/paint.c
+++ b/source/blender/blenkernel/intern/paint.c
@@ -99,7 +99,7 @@ int paint_facesel_test(Object *ob)
 /* Jason */
 int paint_vertsel_test(Object *ob)
 {
-	return (ob && ob->type==OB_MESH && ob->data && (((Mesh *)ob->data)->editflag & ME_EDIT_VERT_SEL) && (ob->mode & (OB_MODE_WEIGHT_PAINT)));
+	return (ob && ob->type==OB_MESH && ob->data && (((Mesh *)ob->data)->editflag & ME_EDIT_VERT_SEL) && (ob->mode & (OB_MODE_VERTEX_PAINT|OB_MODE_WEIGHT_PAINT)));
 }
 void paint_init(Paint *p, const char col[3])
 {
diff --git a/source/blender/editors/object/object_vgroup.c b/source/blender/editors/object/object_vgroup.c
index 4250e7e76fb..e70b36cfa13 100644
--- a/source/blender/editors/object/object_vgroup.c
+++ b/source/blender/editors/object/object_vgroup.c
@@ -701,6 +701,10 @@ static void vgroup_normalize(Object *ob)
 	MDeformWeight *dw;
 	MDeformVert *dvert, **dvert_array=NULL;
 	int i, def_nr, dvert_tot=0;
+	// Jason
+	Mesh *me = ob->data;
+	MVert *mv = me->mvert;
+	int selectedVerts = me->editflag & ME_EDIT_VERT_SEL;
 
 	ED_vgroup_give_parray(ob->data, &dvert_array, &dvert_tot);
 
@@ -712,6 +716,11 @@ static void vgroup_normalize(Object *ob)
 		def_nr= ob->actdef-1;
 
 		for(i = 0; i < dvert_tot; i++) {
+			// Jason
+			if(selectedVerts && !((mv+i)->flag & SELECT)) {
+				continue;
+			}
+
 			dvert = dvert_array[i];
 			dw = defvert_find_index(dvert, def_nr);
 			if(dw) {
@@ -721,6 +730,11 @@ static void vgroup_normalize(Object *ob)
 
 		if(weight_max > 0.0f) {
 			for(i = 0; i < dvert_tot; i++) {
+				// Jason
+				if(selectedVerts && !((mv+i)->flag & SELECT)) {
+					continue;
+				}
+
 				dvert = dvert_array[i];
 				dw = defvert_find_index(dvert, def_nr);
 				if(dw) {
@@ -742,7 +756,11 @@ static void vgroup_levels(Object *ob, float offset, float gain)
 	MDeformWeight *dw;
 	MDeformVert *dvert, **dvert_array=NULL;
 	int i, def_nr, dvert_tot=0;
-	
+	// Jason
+	Mesh *me = ob->data;
+	MVert *mv = me->mvert;
+	int selectedVerts = me->editflag & ME_EDIT_VERT_SEL;
+
 	ED_vgroup_give_parray(ob->data, &dvert_array, &dvert_tot);
 	
 	dg = BLI_findlink(&ob->defbase, (ob->actdef-1));
@@ -751,6 +769,11 @@ static void vgroup_levels(Object *ob, float offset, float gain)
 		def_nr= ob->actdef-1;
 		
 		for(i = 0; i < dvert_tot; i++) {
+			// Jason
+			if(selectedVerts && !((mv+i)->flag & SELECT)) {
+				continue;
+			}
+
 			dvert = dvert_array[i];
 			dw = defvert_find_index(dvert, def_nr);
 			if(dw) {
@@ -772,6 +795,11 @@ static void vgroup_normalize_all(Object *ob, int lock_active)
 	int i, dvert_tot=0;
 	float tot_weight;
 
+	// Jason
+	Mesh *me = ob->data;
+	MVert *mv = me->mvert;
+	int selectedVerts = me->editflag & ME_EDIT_VERT_SEL;
+
 	ED_vgroup_give_parray(ob->data, &dvert_array, &dvert_tot);
 
 	if(dvert_array) {
@@ -781,6 +809,10 @@ static void vgroup_normalize_all(Object *ob, int lock_active)
 			for(i = 0; i < dvert_tot; i++) {
 				float lock_iweight= 1.0f;
 				int j;
+				// Jason
+				if(selectedVerts && !((mv+i)->flag & SELECT)) {
+					continue;
+				}
 
 				tot_weight= 0.0f;
 				dw_act= NULL;
@@ -821,6 +853,11 @@ static void vgroup_normalize_all(Object *ob, int lock_active)
 		else {
 			for(i = 0; i < dvert_tot; i++) {
 				int j;
+				// Jason
+				if(selectedVerts && !((mv+i)->flag & SELECT)) {
+					continue;
+				}
+
 				tot_weight= 0.0f;
 				dvert = dvert_array[i];
 
@@ -880,6 +917,10 @@ static void vgroup_invert(Object *ob, int auto_assign, int auto_remove)
 	MDeformWeight *dw;
 	MDeformVert *dvert, **dvert_array=NULL;
 	int i, def_nr, dvert_tot=0;
+	// Jason
+	Mesh *me = ob->data;
+	MVert *mv = me->mvert;
+	int selectedVerts = me->editflag & ME_EDIT_VERT_SEL;
 
 	ED_vgroup_give_parray(ob->data, &dvert_array, &dvert_tot);
 
@@ -890,6 +931,10 @@ static void vgroup_invert(Object *ob, int auto_assign, int auto_remove)
 
 
 		for(i = 0; i < dvert_tot; i++) {
+			// Jason
+			if(selectedVerts && !((mv+i)->flag & SELECT)) {
+				continue;
+			}
 			dvert = dvert_array[i];
 
 			if(auto_assign) {
@@ -1002,6 +1047,10 @@ static void vgroup_clean(Object *ob, float eul, int keep_single)
 	MDeformWeight *dw;
 	MDeformVert *dvert, **dvert_array=NULL;
 	int i, def_nr, dvert_tot=0;
+	// Jason
+	Mesh *me = ob->data;
+	MVert *mv = me->mvert;
+	int selectedVerts = me->editflag & ME_EDIT_VERT_SEL;
 
 	ED_vgroup_give_parray(ob->data, &dvert_array, &dvert_tot);
 
@@ -1011,6 +1060,10 @@ static void vgroup_clean(Object *ob, float eul, int keep_single)
 		def_nr= ob->actdef-1;
 
 		for(i = 0; i < dvert_tot; i++) {
+			// Jason
+			if(selectedVerts && !((mv+i)->flag & SELECT)) {
+				continue;
+			}
 			dvert = dvert_array[i];
 
 			dw= defvert_find_index(dvert, def_nr);
@@ -1032,12 +1085,21 @@ static void vgroup_clean_all(Object *ob, float eul, int keep_single)
 	MDeformWeight *dw;
 	MDeformVert *dvert, **dvert_array=NULL;
 	int i, dvert_tot=0;
+	// Jason
+	Mesh *me = ob->data;
+	MVert *mv = me->mvert;
+	int selectedVerts = me->editflag & ME_EDIT_VERT_SEL;
 
 	ED_vgroup_give_parray(ob->data, &dvert_array, &dvert_tot);
 
 	if(dvert_array) {
 		for(i = 0; i < dvert_tot; i++) {
 			int j;
+			// Jason
+			if(selectedVerts && !((mv+i)->flag & SELECT)) {
+				continue;
+			}
+
 			dvert = dvert_array[i];
 			j= dvert->totweight;
 
diff --git a/source/blender/editors/sculpt_paint/paint_vertex.c b/source/blender/editors/sculpt_paint/paint_vertex.c
index 0f2e8b84e9f..c0822ca134a 100644
--- a/source/blender/editors/sculpt_paint/paint_vertex.c
+++ b/source/blender/editors/sculpt_paint/paint_vertex.c
@@ -361,8 +361,10 @@ void vpaint_fill(Object *ob, unsigned int paintcol)
 	Mesh *me;
 	MFace *mf;
 	unsigned int *mcol;
-	int i, selected;
-
+	int i, selected, selectedVerts;//Jason
+	// Jason
+	MVert *mv;
+	
 	me= get_mesh(ob);
 	if(me==NULL || me->totface==0) return;
 
@@ -370,15 +372,29 @@ void vpaint_fill(Object *ob, unsigned int paintcol)
 		make_vertexcol(ob);
 
 	selected= (me->editflag & ME_EDIT_PAINT_MASK);
+	// Jason
+	selectedVerts = (me->editflag & ME_EDIT_VERT_SEL);
+	mv = me->mvert;
 
 	mf = me->mface;
 	mcol = (unsigned int*)me->mcol;
 	for (i = 0; i < me->totface; i++, mf++, mcol+=4) {
-		if (!selected || mf->flag & ME_FACE_SEL) {
-			mcol[0] = paintcol;
-			mcol[1] = paintcol;
-			mcol[2] = paintcol;
-			mcol[3] = paintcol;
+		if (!selected || mf->flag & ME_FACE_SEL || selectedVerts) {
+			if(selectedVerts) {
+				if(((mv+mf->v1)->flag & SELECT))
+					mcol[0] = paintcol;
+				if(((mv+mf->v2)->flag & SELECT))
+					mcol[1] = paintcol;
+				if(((mv+mf->v3)->flag & SELECT))
+					mcol[2] = paintcol;
+				if(mf->v4 && ((mv+mf->v4)->flag & SELECT))
+					mcol[3] = paintcol;
+			} else {
+				mcol[0] = paintcol;
+				mcol[1] = paintcol;
+				mcol[2] = paintcol;
+				mcol[3] = paintcol;
+			}
 		}
 	}
 	
@@ -443,7 +459,7 @@ void wpaint_fill(VPaint *wp, Object *ob, float paintweight)
 			for (i=0; i<3 || faceverts[i]; i++) {
 				if(!((me->dvert+faceverts[i])->flag)) {
 					// Jason
-					if(selectedVerts && !((me->mvert+faceverts[i])->flag & 1)) {
+					if(selectedVerts && !((me->mvert+faceverts[i])->flag & SELECT)) {
 						continue;
 					}
 
@@ -1972,10 +1988,10 @@ static void wpaint_stroke_update_step(bContext *C, struct PaintStroke *stroke, P
 		if(indexar[index] && indexar[index]<=me->totface) {
 			MFace *mface= me->mface + (indexar[index]-1);
 					
-			(me->dvert+mface->v1)->flag= selectedVerts ? ((me->mvert+mface->v1)->flag & 1): 1;
-			(me->dvert+mface->v2)->flag= selectedVerts ? ((me->mvert+mface->v2)->flag & 1): 1;
-			(me->dvert+mface->v3)->flag= selectedVerts ? ((me->mvert+mface->v3)->flag & 1): 1;
-			if(mface->v4) (me->dvert+mface->v4)->flag= selectedVerts ? ((me->mvert+mface->v4)->flag & 1): 1;
+			(me->dvert+mface->v1)->flag= selectedVerts ? ((me->mvert+mface->v1)->flag & SELECT): 1;
+			(me->dvert+mface->v2)->flag= selectedVerts ? ((me->mvert+mface->v2)->flag & SELECT): 1;
+			(me->dvert+mface->v3)->flag= selectedVerts ? ((me->mvert+mface->v3)->flag & SELECT): 1;
+			if(mface->v4) (me->dvert+mface->v4)->flag= selectedVerts ? ((me->mvert+mface->v4)->flag & SELECT): 1;
 					
 			if(brush->vertexpaint_tool==VP_BLUR) {
 				MDeformWeight *dw, *(*dw_func)(MDeformVert *, const int);
@@ -2298,14 +2314,33 @@ static void vpaint_paint_face(VPaint *vp, VPaintData *vpd, Object *ob, int index
 	Brush *brush = paint_brush(&vp->paint);
 	Mesh *me = get_mesh(ob);
 	MFace *mface= ((MFace*)me->mface) + index;
+	// Jason
+	MVert *mv = me->mvert;
+	int selectedVerts = (me->editflag & ME_EDIT_VERT_SEL);
+	char *m;
+
 	unsigned int *mcol= ((unsigned int*)me->mcol) + 4*index;
 	unsigned int *mcolorig= ((unsigned int*)vp->vpaint_prev) + 4*index;
 	float alpha;
 	int i;
 	
+	// Jason
+	if(selectedVerts) {
+		m = MEM_mallocN(sizeof(char)*4, "selectedVerts");
+		m[0] = ((mv+mface->v1)->flag & SELECT);
+		m[1] = ((mv+mface->v2)->flag & SELECT);
+		m[2] = ((mv+mface->v3)->flag & SELECT);
+		m[3] = (mface->v4 && ((mv+mface->v4)->flag & SELECT));
+	}
+
 	if((vp->flag & VP_COLINDEX && mface->mat_nr!=ob->actcol-1) ||
-	   ((me->editflag & ME_EDIT_PAINT_MASK) && !(mface->flag & ME_FACE_SEL)))
-		return;
+	   ((me->editflag & ME_EDIT_PAINT_MASK) && !(mface->flag & ME_FACE_SEL))
+	   || (selectedVerts && !(m[0] || m[1] || m[2] || m[3]))) {// Jason
+		if(selectedVerts) {
+			MEM_freeN(m);
+		}
+		   return;
+	}
 
 	if(brush->vertexpaint_tool==VP_BLUR) {
 		unsigned int fcol1= mcol_blend( mcol[0], mcol[1], 128);
@@ -2320,9 +2355,15 @@ static void vpaint_paint_face(VPaint *vp, VPaintData *vpd, Object *ob, int index
 	}
 
 	for(i = 0; i < (mface->v4 ? 4 : 3); ++i) {
-		alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*(&mface->v1)[i], mval, pressure);
-		if(alpha)
-			vpaint_blend(vp, mcol+i, mcolorig+i, vpd->paintcol, (int)(alpha*255.0f));
+		if(!selectedVerts || m[i]) {
+			alpha= calc_vp_alpha_dl(vp, vc, vpd->vpimat, vpd->vertexcosnos+6*(&mface->v1)[i], mval, pressure);
+			if(alpha)
+				vpaint_blend(vp, mcol+i, mcolorig+i, vpd->paintcol, (int)(alpha*255.0f));
+		}
+	}
+	// Jason
+	if(selectedVerts) {
+		MEM_freeN(m);
 	}
 }
 
diff --git a/source/blender/editors/space_view3d/drawmesh.c b/source/blender/editors/space_view3d/drawmesh.c
index f070bae4e54..81de12320e4 100644
--- a/source/blender/editors/space_view3d/drawmesh.c
+++ b/source/blender/editors/space_view3d/drawmesh.c
@@ -587,7 +587,7 @@ static void draw_mesh_text(Scene *scene, Object *ob, int glsl)
 	if(ob->mode & OB_MODE_EDIT)
 		return;
 	else if(ob==OBACT)
-		if(paint_facesel_test(ob))
+		if(paint_facesel_test(ob) || paint_vertsel_test(ob))
 			return;
 
 	ddm = mesh_get_derived_deform(scene, ob, CD_MASK_BAREMESH);
diff --git a/source/blender/editors/space_view3d/drawobject.c b/source/blender/editors/space_view3d/drawobject.c
index 66539b2b25d..457d882845e 100644
--- a/source/blender/editors/space_view3d/drawobject.c
+++ b/source/blender/editors/space_view3d/drawobject.c
@@ -2619,7 +2619,7 @@ static void draw_mesh_fancy(Scene *scene, ARegion *ar, View3D *v3d, RegionView3D
 	Mesh *me = ob->data;
 	Material *ma= give_current_material(ob, 1);
 	const short hasHaloMat = (ma && (ma->material_type == MA_TYPE_HALO));
-	const short is_paint_sel= (ob==OBACT && (paint_facesel_test(ob)));
+	const short is_paint_sel= (ob==OBACT && paint_facesel_test(ob));
 	int draw_wire = 0;
 	int /* totvert,*/ totedge, totface;
 	DerivedMesh *dm= mesh_get_derived_final(scene, ob, scene->customdata_mask);
@@ -2847,11 +2847,11 @@ static void draw_mesh_fancy(Scene *scene, ARegion *ar, View3D *v3d, RegionView3D
 	// Jason
 	if(paint_vertsel_test(ob)) {
 		glColor3f(0.0f, 0.0f, 0.0f);
-		glPointSize(2.0f);
+		 glPointSize(2.0f);
 		// TODO clarify:
 		// there is clearly something I don't understand, when it was 
-		// dt != OB_WIRE instead, it still drew in wire mode!
-		if(dt != OB_SOLID) {
+		// dt != OB_WIRE instead, it still drew in wire mode! (in weight paint mode)
+		if(dt != OB_SOLID || ob->mode & OB_MODE_VERTEX_PAINT) {
 			dm->drawEdges(dm, (totface==0), TRUE);
 		}
 		glPointSize(3.0f);
diff --git a/source/blender/editors/space_view3d/view3d_header.c b/source/blender/editors/space_view3d/view3d_header.c
index a1a73b27b8d..680bd6b62e4 100644
--- a/source/blender/editors/space_view3d/view3d_header.c
+++ b/source/blender/editors/space_view3d/view3d_header.c
@@ -496,12 +496,14 @@ void uiTemplateHeader3D(uiLayout *layout, struct bContext *C)
 		
 		PointerRNA meshptr;
 		RNA_pointer_create(&ob->id, &RNA_Mesh, ob->data, &meshptr);
-		//uiItemR(layout, &meshptr, "use_paint_mask", UI_ITEM_R_ICON_ONLY, "", ICON_NONE);
-		// Jason
-		row= uiLayoutRow(layout, 1);
-		// TODO: make it so at most one can be active.
-		uiItemR(row, &meshptr, "use_paint_mask", UI_ITEM_R_ICON_ONLY, "", ICON_NONE);
-		uiItemR(row, &meshptr, "wp_vert_sel", UI_ITEM_R_ICON_ONLY, "", ICON_NONE);
+		if(ob->mode & OB_MODE_TEXTURE_PAINT) {
+			uiItemR(layout, &meshptr, "use_paint_mask", UI_ITEM_R_ICON_ONLY, "", ICON_NONE);
+		} else {
+			// Jason
+			row= uiLayoutRow(layout, 1);
+			uiItemR(row, &meshptr, "use_paint_mask", UI_ITEM_R_ICON_ONLY, "", ICON_NONE);
+			uiItemR(row, &meshptr, "wp_vert_sel", UI_ITEM_R_ICON_ONLY, "", ICON_NONE);
+		}
 	} else {
 		const char *str_menu;
 
diff --git a/source/blender/makesrna/intern/rna_mesh.c b/source/blender/makesrna/intern/rna_mesh.c
index c1b92f8fc56..adec9883ca8 100644
--- a/source/blender/makesrna/intern/rna_mesh.c
+++ b/source/blender/makesrna/intern/rna_mesh.c
@@ -90,6 +90,24 @@ void rna_Mesh_update_draw(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA
 	}
 }
 
+/* Jason */
+void rna_Mesh_update_vertmask(Main *bmain, Scene *scene, PointerRNA *ptr)
+{
+	Mesh* me = ptr->data;
+	if ((me->editflag & ME_EDIT_VERT_SEL) && (me->editflag & ME_EDIT_PAINT_MASK)) {
+		me->editflag ^= ME_EDIT_PAINT_MASK;
+	}
+	rna_Mesh_update_draw(bmain, scene, ptr);
+}
+/* Jason */
+void rna_Mesh_update_facemask(Main *bmain, Scene *scene, PointerRNA *ptr)
+{
+	Mesh* me = ptr->data;
+	if ((me->editflag & ME_EDIT_VERT_SEL) && (me->editflag & ME_EDIT_PAINT_MASK)) {
+		me->editflag ^= ME_EDIT_VERT_SEL;
+	}
+	rna_Mesh_update_draw(bmain, scene, ptr);
+}
 static void rna_MeshVertex_normal_get(PointerRNA *ptr, float *value)
 {
 	MVert *mvert= (MVert*)ptr->data;
@@ -2075,14 +2093,14 @@ static void rna_def_mesh(BlenderRNA *brna)
 	RNA_def_property_boolean_sdna(prop, NULL, "editflag", ME_EDIT_PAINT_MASK);
 	RNA_def_property_ui_text(prop, "Paint Mask", "Face selection masking for painting");
 	RNA_def_property_ui_icon(prop, ICON_FACESEL_HLT, 0);
-	RNA_def_property_update(prop, 0, "rna_Mesh_update_draw");
+	RNA_def_property_update(prop, 0, "rna_Mesh_update_facemask");
 
 	/* Jason */
 	prop= RNA_def_property(srna, "wp_vert_sel", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_sdna(prop, NULL, "editflag", ME_EDIT_VERT_SEL);
 	RNA_def_property_ui_text(prop, "Vertex Selection", "Vertex Selection for auto brushes");
 	RNA_def_property_ui_icon(prop, ICON_VERTEXSEL, 0);
-	RNA_def_property_update(prop, 0, "rna_Mesh_update_draw");
+	RNA_def_property_update(prop, 0, "rna_Mesh_update_vertmask");
 
 	/* readonly editmesh info - use for extrude menu */
 	prop= RNA_def_property(srna, "total_vert_sel", PROP_INT, PROP_UNSIGNED);