fixes for weight paint mode:

- sample weight didnt work when the object was transformed.
- sample weight didnt work when vertex selection was enabled.
- 'All faces' option is used by weight paint mode, but there was no UI access.

add ED_mesh_pick_face_vert(). which uses the face selection buffer but returns the closest vertex.

4 files changed, 73 insertions, 38 deletions

diff --git a/release/scripts/startup/bl_ui/space_view3d_toolbar.py b/release/scripts/startup/bl_ui/space_view3d_toolbar.py
index 2c1e963dcfe..1b67b30d17a 100644
--- a/release/scripts/startup/bl_ui/space_view3d_toolbar.py
+++ b/release/scripts/startup/bl_ui/space_view3d_toolbar.py
@@ -980,6 +980,7 @@ class VIEW3D_PT_tools_weightpaint_options(Panel, View3DPaintPanel):
 
         col = layout.column()
 
+        col.prop(wpaint, "use_all_faces")
         col.prop(wpaint, "use_normal")
         col.prop(wpaint, "use_spray")
         col.prop(wpaint, "use_group_restrict")
diff --git a/source/blender/editors/include/ED_mesh.h b/source/blender/editors/include/ED_mesh.h
index 528eeab0473..1d4fbe81e02 100644
--- a/source/blender/editors/include/ED_mesh.h
+++ b/source/blender/editors/include/ED_mesh.h
@@ -281,6 +281,7 @@ int           *mesh_get_x_mirror_faces(struct Object *ob, struct BMEditMesh *em)
 
 int ED_mesh_pick_vert(struct bContext *C, struct Mesh *me, const int mval[2], unsigned int *index, int size);
 int ED_mesh_pick_face(struct bContext *C, struct Mesh *me, const int mval[2], unsigned int *index, int size);
+int ED_mesh_pick_face_vert(struct bContext *C, struct Mesh *me, struct Object *ob, const int mval[2], unsigned int *index, int size);
 
 #define ED_MESH_PICK_DEFAULT_VERT_SIZE 50
 #define ED_MESH_PICK_DEFAULT_FACE_SIZE 3
diff --git a/source/blender/editors/mesh/meshtools.c b/source/blender/editors/mesh/meshtools.c
index 3d6face167b..06dcac0505a 100644
--- a/source/blender/editors/mesh/meshtools.c
+++ b/source/blender/editors/mesh/meshtools.c
@@ -1185,6 +1185,55 @@ int ED_mesh_pick_face(bContext *C, Mesh *me, const int mval[2], unsigned int *in
 
 	return 1;
 }
+/**
+ * Use when the back buffer stores face index values. but we want a vert.
+ * This gets the face then finds the closest vertex to mval.
+ */
+int ED_mesh_pick_face_vert(bContext *C, Mesh *me, Object *ob, const int mval[2], unsigned int *index, int size)
+{
+	unsigned int poly_index;
+
+	if (ED_mesh_pick_face(C, me, mval, &poly_index, size)) {
+		Scene *scene = CTX_data_scene(C);
+		struct ARegion *ar = CTX_wm_region(C);
+
+		/* derived mesh to find deformed locations */
+		DerivedMesh *dm = mesh_get_derived_final(scene, ob, CD_MASK_BAREMESH);
+		int v_idx_best = -1;
+
+		if (dm->getVertCo) {
+			/* find the vert closest to 'mval' */
+			const float mval_f[2] = {(float)mval[0],
+			                         (float)mval[1]};
+			MPoly *mp = &me->mpoly[poly_index];
+			int fidx;
+			float len_best = FLT_MAX;
+
+			fidx = mp->totloop - 1;
+			do {
+				float co[3], sco[2], len;
+				const int v_idx = me->mloop[mp->loopstart + fidx].v;
+				dm->getVertCo(dm, v_idx, co);
+				mul_m4_v3(ob->obmat, co);
+				project_float_noclip(ar, co, sco);
+				len = len_squared_v2v2(mval_f, sco);
+				if (len < len_best) {
+					len_best = len;
+					v_idx_best = v_idx;
+				}
+			} while (fidx--);
+		}
+
+		dm->release(dm);
+
+		if (v_idx_best != -1) {
+			*index = v_idx_best;
+			return 1;
+		}
+	}
+
+	return 0;
+}
 
 /**
  * Vertex selection in object mode,
diff --git a/source/blender/editors/sculpt_paint/paint_vertex.c b/source/blender/editors/sculpt_paint/paint_vertex.c
index cb2826491f5..c52bae215ec 100644
--- a/source/blender/editors/sculpt_paint/paint_vertex.c
+++ b/source/blender/editors/sculpt_paint/paint_vertex.c
@@ -1009,50 +1009,34 @@ static int weight_sample_invoke(bContext *C, wmOperator *op, wmEvent *event)
 	me = BKE_mesh_from_object(vc.obact);
 
 	if (me && me->dvert && vc.v3d && vc.rv3d) {
-		int index;
+		const int use_vert_sel = (me->editflag & ME_EDIT_VERT_SEL) != 0;
+		int v_idx_best = -1;
+		unsigned int index;
 
 		view3d_operator_needs_opengl(C);
 
-		index = view3d_sample_backbuf(&vc, event->mval[0], event->mval[1]);
-
-		if (index && index <= me->totpoly) {
-			DerivedMesh *dm = mesh_get_derived_final(vc.scene, vc.obact, CD_MASK_BAREMESH);
-
-			if (dm->getVertCo == NULL) {
+		if (use_vert_sel) {
+			if (ED_mesh_pick_vert(C, me, event->mval, &index, ED_MESH_PICK_DEFAULT_VERT_SIZE)) {
+				v_idx_best = index;
+			}
+		}
+		else {
+			if (ED_mesh_pick_face_vert(C, me, vc.obact, event->mval, &index, ED_MESH_PICK_DEFAULT_FACE_SIZE)) {
+				v_idx_best = index;
+			}
+			else if (ED_mesh_pick_face(C, me, event->mval, &index, ED_MESH_PICK_DEFAULT_FACE_SIZE)) {
+				/* this relies on knowning the internal worksings of ED_mesh_pick_face_vert() */
 				BKE_report(op->reports, RPT_WARNING, "The modifier used does not support deformed locations");
 			}
-			else {
-				MPoly *mp = ((MPoly *)me->mpoly) + (index - 1);
-				const int vgroup_active = vc.obact->actdef - 1;
-				Scene *scene = vc.scene;
-				ToolSettings *ts = vc.scene->toolsettings;
-				Brush *brush = paint_brush(&ts->wpaint->paint);
-				const float mval_f[2] = {(float)event->mval[0],
-				                         (float)event->mval[1]};
-				int v_idx_best = -1;
-				int fidx;
-				float len_best = FLT_MAX;
-
-				fidx = mp->totloop - 1;
-				do {
-					float co[3], sco[3], len;
-					const int v_idx = me->mloop[mp->loopstart + fidx].v;
-					dm->getVertCo(dm, v_idx, co);
-					project_float_noclip(vc.ar, co, sco);
-					len = len_squared_v2v2(mval_f, sco);
-					if (len < len_best) {
-						len_best = len;
-						v_idx_best = v_idx;
-					}
-				} while (fidx--);
+		}
 
-				if (v_idx_best != -1) { /* should always be valid */
-					float vgroup_weight = defvert_find_weight(&me->dvert[v_idx_best], vgroup_active);
-					BKE_brush_weight_set(scene, brush, vgroup_weight);
-					change = TRUE;
-				}
-			}
-			dm->release(dm);
+		if (v_idx_best != -1) { /* should always be valid */
+			ToolSettings *ts = vc.scene->toolsettings;
+			Brush *brush = paint_brush(&ts->wpaint->paint);
+			const int vgroup_active = vc.obact->actdef - 1;
+			float vgroup_weight = defvert_find_weight(&me->dvert[v_idx_best], vgroup_active);
+			BKE_brush_weight_set(vc.scene, brush, vgroup_weight);
+			change = TRUE;
 		}
 	}