Part-Bugfix, Part-Feature Completion: 'Armature' Option for Mask Modifier

finally works

This commit finally hooks up the Mask Modifier's "Armature" option with the
relevant depsgraph updates on bone selection. Hence, this feature finally works
as it was originally intended - that is, bone selections can be used to control
which parts of the mesh that the mask modifier is applied to are displayed,
giving riggers more freedom to experiment with rigs that don't necessarily
feature overbearing/cluttering widgets.

Regarding the implementation ("has_viz_deps" flag):
This feature is just the "tip of the iceberg" of a number of related set of
rigging/visual animation tools I've had in mind for a while now (dating back to
the introduction of this modifier). Key considerations
- Not all rigs will use this, so we don't want an extra (depsgraph-flush +
search) recalc cost for those that don't use this.
- There are some planned features which will also use this

6 files changed, 120 insertions, 36 deletions

diff --git a/source/blender/editors/armature/editarmature.c b/source/blender/editors/armature/editarmature.c
index 856743d9997..fdebddbf41d 100644
--- a/source/blender/editors/armature/editarmature.c
+++ b/source/blender/editors/armature/editarmature.c
@@ -1418,7 +1418,8 @@ static void selectconnected_posebonechildren(Object *ob, Bone *bone, int extend)
 /* previously known as "selectconnected_posearmature" */
 static int pose_select_connected_invoke(bContext *C, wmOperator *op, wmEvent *event)
 {
-	Object *ob = CTX_data_edit_object(C);
+	Object *ob = BKE_object_pose_armature_get(CTX_data_active_object(C));
+	bArmature *arm = (bArmature *)ob->data;
 	Bone *bone, *curBone, *next = NULL;
 	int extend = RNA_boolean_get(op->ptr, "extend");
 
@@ -1457,14 +1458,20 @@ static int pose_select_connected_invoke(bContext *C, wmOperator *op, wmEvent *ev
 	for (curBone = bone->childbase.first; curBone; curBone = next)
 		selectconnected_posebonechildren(ob, curBone, extend);
 	
+	/* updates */
 	WM_event_add_notifier(C, NC_OBJECT | ND_BONE_SELECT, ob);
+	
+	if (arm->flag & ARM_HAS_VIZ_DEPS) {
+		/* mask modifier ('armature' mode), etc. */
+		DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
+	}
 
 	return OPERATOR_FINISHED;
 }
 
 static int pose_select_linked_poll(bContext *C)
 {
-	return (ED_operator_view3d_active(C) && ED_operator_posemode(C) );
+	return (ED_operator_view3d_active(C) && ED_operator_posemode(C));
 }
 
 void POSE_OT_select_linked(wmOperatorType *ot)
@@ -4556,14 +4563,21 @@ int ED_do_pose_selectbuffer(Scene *scene, Base *base, unsigned int *buffer, shor
 			}	
 		}
 		
-		/* in weightpaint we select the associated vertex group too */
-		if (ob_act && ob_act->mode & OB_MODE_WEIGHT_PAINT) {
-			if (nearBone == arm->act_bone) {
-				ED_vgroup_select_by_name(OBACT, nearBone->name);
-				DAG_id_tag_update(&OBACT->id, OB_RECALC_DATA);
+		if (ob_act) {
+			/* in weightpaint we select the associated vertex group too */
+			if (ob_act->mode & OB_MODE_WEIGHT_PAINT) {
+				if (nearBone == arm->act_bone) {
+					ED_vgroup_select_by_name(ob_act, nearBone->name);
+					DAG_id_tag_update(&ob_act->id, OB_RECALC_DATA);
+				}
+			}
+			/* if there are some dependencies for visualising armature state 
+			 * (e.g. Mask Modifier in 'Armature' mode), force update 
+			 */
+			else if (arm->flag & ARM_HAS_VIZ_DEPS) {
+				DAG_id_tag_update(&ob_act->id, OB_RECALC_DATA);
 			}
 		}
-		
 	}
 	
 	return nearBone != NULL;
@@ -5265,6 +5279,8 @@ static int pose_de_select_all_exec(bContext *C, wmOperator *op)
 	int action = RNA_enum_get(op->ptr, "action");
 	
 	Scene *scene = CTX_data_scene(C);
+	Object *ob = ED_object_context(C);
+	bArmature *arm = ob->data;
 	int multipaint = scene->toolsettings->multipaint;
 
 	if (action == SEL_TOGGLE) {
@@ -5297,8 +5313,8 @@ static int pose_de_select_all_exec(bContext *C, wmOperator *op)
 
 	WM_event_add_notifier(C, NC_OBJECT | ND_BONE_SELECT, NULL);
 	
-	if (multipaint) {
-		Object *ob = ED_object_context(C);
+	/* weightpaint or mask modifiers need depsgraph updates */
+	if (multipaint || (arm->flag & ARM_HAS_VIZ_DEPS)) {
 		DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
 	}
 
@@ -5325,12 +5341,12 @@ void POSE_OT_select_all(wmOperatorType *ot)
 static int pose_select_parent_exec(bContext *C, wmOperator *UNUSED(op))
 {
 	Object *ob = BKE_object_pose_armature_get(CTX_data_active_object(C));
+	bArmature *arm = (bArmature *)ob->data;
 	bPoseChannel *pchan, *parent;
 
-	/*	Determine if there is an active bone */
+	/* Determine if there is an active bone */
 	pchan = CTX_data_active_pose_bone(C);
 	if (pchan) {
-		bArmature *arm = ob->data;
 		parent = pchan->parent;
 		if ((parent) && !(parent->bone->flag & (BONE_HIDDEN_P | BONE_UNSELECTABLE))) {
 			parent->bone->flag |= BONE_SELECTED;
@@ -5343,9 +5359,15 @@ static int pose_select_parent_exec(bContext *C, wmOperator *UNUSED(op))
 	else {
 		return OPERATOR_CANCELLED;
 	}
-
+	
+	/* updates */
 	WM_event_add_notifier(C, NC_OBJECT | ND_BONE_SELECT, ob);
 	
+	if (arm->flag & ARM_HAS_VIZ_DEPS) {
+		/* mask modifier ('armature' mode), etc. */
+		DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
+	}
+	
 	return OPERATOR_FINISHED;
 }
 
diff --git a/source/blender/editors/armature/poseobject.c b/source/blender/editors/armature/poseobject.c
index a8e84116699..9bcbf313f13 100644
--- a/source/blender/editors/armature/poseobject.c
+++ b/source/blender/editors/armature/poseobject.c
@@ -391,6 +391,7 @@ void POSE_OT_paths_clear(wmOperatorType *ot)
 static int pose_select_constraint_target_exec(bContext *C, wmOperator *UNUSED(op))
 {
 	Object *ob = BKE_object_pose_armature_get(CTX_data_active_object(C));
+	bArmature *arm = (bArmature *)ob->data;
 	bConstraint *con;
 	int found = 0;
 	
@@ -422,12 +423,18 @@ static int pose_select_constraint_target_exec(bContext *C, wmOperator *UNUSED(op
 		}
 	}
 	CTX_DATA_END;
-
+	
 	if (!found)
 		return OPERATOR_CANCELLED;
-
+	
+	/* updates */
 	WM_event_add_notifier(C, NC_OBJECT | ND_BONE_SELECT, ob);
-
+	
+	if (arm->flag & ARM_HAS_VIZ_DEPS) {
+		/* mask modifier ('armature' mode), etc. */
+		DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
+	}
+	
 	return OPERATOR_FINISHED;
 }
 
@@ -477,7 +484,6 @@ static int pose_select_hierarchy_exec(bContext *C, wmOperator *op)
 					}
 				} 
 				else { /* direction == BONE_SELECT_CHILD */
-
 					/* the child member is only assigned to connected bones, see [#30340] */
 #if 0
 					if (pchan->child == NULL) continue;
@@ -518,9 +524,15 @@ static int pose_select_hierarchy_exec(bContext *C, wmOperator *op)
 
 	if (found == 0)
 		return OPERATOR_CANCELLED;
-
+	
+	/* updates */
 	WM_event_add_notifier(C, NC_OBJECT | ND_BONE_SELECT, ob);
-
+	
+	if (arm->flag & ARM_HAS_VIZ_DEPS) {
+		/* mask modifier ('armature' mode), etc. */
+		DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
+	}
+	
 	return OPERATOR_FINISHED;
 }
 
@@ -547,7 +559,6 @@ void POSE_OT_select_hierarchy(wmOperatorType *ot)
 	/* props */
 	ot->prop = RNA_def_enum(ot->srna, "direction", direction_items, BONE_SELECT_PARENT, "Direction", "");
 	RNA_def_boolean(ot->srna, "extend", 0, "Add to Selection", "");
-	
 }
 
 /* ******************* select grouped operator ************* */
@@ -711,6 +722,7 @@ static int pose_select_same_keyingset(bContext *C, Object *ob, short extend)
 static int pose_select_grouped_exec(bContext *C, wmOperator *op)
 {
 	Object *ob = BKE_object_pose_armature_get(CTX_data_active_object(C));
+	bArmature *arm = (bArmature *)ob->data;
 	short extend = RNA_boolean_get(op->ptr, "extend");
 	short changed = 0;
 	
@@ -736,6 +748,11 @@ static int pose_select_grouped_exec(bContext *C, wmOperator *op)
 	/* notifiers for updates */
 	WM_event_add_notifier(C, NC_OBJECT | ND_POSE, ob);
 	
+	if (arm->flag & ARM_HAS_VIZ_DEPS) {
+		/* mask modifier ('armature' mode), etc. */
+		DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
+	}
+	
 	/* report done status */
 	if (changed)
 		return OPERATOR_FINISHED;
diff --git a/source/blender/editors/space_view3d/view3d_select.c b/source/blender/editors/space_view3d/view3d_select.c
index bef9219f44c..602f790c8df 100644
--- a/source/blender/editors/space_view3d/view3d_select.c
+++ b/source/blender/editors/space_view3d/view3d_select.c
@@ -64,6 +64,7 @@
 #include "BKE_context.h"
 #include "BKE_paint.h"
 #include "BKE_armature.h"
+#include "BKE_depsgraph.h"
 #include "BKE_tessmesh.h"
 #include "BKE_movieclip.h"
 #include "BKE_object.h"
@@ -333,7 +334,7 @@ static void do_lasso_select_pose(ViewContext *vc, Object *ob, int mcords[][2], s
 	int sco1[2], sco2[2];
 	bArmature *arm = ob->data;
 	
-	if (ob->type != OB_ARMATURE || ob->pose == NULL) return;
+	if ((ob->type != OB_ARMATURE) || (ob->pose == NULL)) return;
 
 	for (pchan = ob->pose->chanbase.first; pchan; pchan = pchan->next) {
 		if (PBONE_VISIBLE(arm, pchan->bone) && (pchan->bone->flag & BONE_UNSELECTABLE) == 0) {
@@ -348,6 +349,11 @@ static void do_lasso_select_pose(ViewContext *vc, Object *ob, int mcords[][2], s
 			}
 		}
 	}
+	
+	if (arm->flag & ARM_HAS_VIZ_DEPS) {
+		/* mask modifier ('armature' mode), etc. */
+		DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
+	}
 }
 
 static void object_deselect_all_visible(Scene *scene, View3D *v3d)
@@ -1899,12 +1905,19 @@ static int do_object_pose_box_select(bContext *C, ViewContext *vc, rcti *rect, i
 			}
 			
 			if (bone_selected) {
-				WM_event_add_notifier(C, NC_OBJECT | ND_BONE_SELECT, base->object);
+				Object *ob = base->object;
+				bArmature *arm = ob->data;
+				
+				WM_event_add_notifier(C, NC_OBJECT | ND_BONE_SELECT, ob);
+				
+				if (arm->flag & ARM_HAS_VIZ_DEPS) {
+					/* mask modifier ('armature' mode), etc. */
+					DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
+				}
 			}
 		}
-
+		
 		WM_event_add_notifier(C, NC_SCENE | ND_OB_SELECT, vc->scene);
-
 	}
 	MEM_freeN(vbuffer);
 
diff --git a/source/blender/makesdna/DNA_armature_types.h b/source/blender/makesdna/DNA_armature_types.h
index ea564e8c499..7faeccd2a32 100644
--- a/source/blender/makesdna/DNA_armature_types.h
+++ b/source/blender/makesdna/DNA_armature_types.h
@@ -128,7 +128,8 @@ typedef enum eArmature_Flag {
 	ARM_NO_CUSTOM		= (1<<10), 	/* made option negative, for backwards compat */
 	ARM_COL_CUSTOM		= (1<<11),	/* draw custom colors  */
 	ARM_GHOST_ONLYSEL 	= (1<<12),	/* when ghosting, only show selected bones (this should belong to ghostflag instead) */ // XXX depreceated
-	ARM_DS_EXPAND 		= (1<<13)
+	ARM_DS_EXPAND 		= (1<<13),	/* dopesheet channel is expanded */
+	ARM_HAS_VIZ_DEPS	= (1<<14),	/* other objects are used for visualising various states (hack for efficient updates) */
 } eArmature_Flag;
 
 /* armature->drawtype */
diff --git a/source/blender/makesrna/intern/rna_armature.c b/source/blender/makesrna/intern/rna_armature.c
index f3742687fc5..b86077ff1bd 100644
--- a/source/blender/makesrna/intern/rna_armature.c
+++ b/source/blender/makesrna/intern/rna_armature.c
@@ -131,9 +131,10 @@ static void rna_Armature_update_layers(Main *bmain, Scene *UNUSED(scene), Pointe
 
 	/* proxy lib exception, store it here so we can restore layers on file
 	 * load, since it would otherwise get lost due to being linked data */
-	for (ob = bmain->object.first; ob; ob = ob->id.next)
+	for (ob = bmain->object.first; ob; ob = ob->id.next) {
 		if (ob->data == arm && ob->pose)
 			ob->pose->proxy_layer = arm->layer;
+	}
 
 	WM_main_add_notifier(NC_GEOM | ND_DATA, arm);
 }
@@ -141,7 +142,35 @@ static void rna_Armature_update_layers(Main *bmain, Scene *UNUSED(scene), Pointe
 static void rna_Armature_redraw_data(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr)
 {
 	ID *id = ptr->id.data;
+	
+	WM_main_add_notifier(NC_GEOM | ND_DATA, id);
+}
 
+static void rna_Bone_select_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr)
+{
+	ID *id = ptr->id.data;
+	
+	/* special updates for cases where rigs try to hook into armature drawing stuff 
+	 * e.g. Mask Modifier - 'Armature' option
+	 */
+	if (id) {
+		if (GS(id->name) == ID_AR) {
+			bArmature *arm = (bArmature *)id;
+			
+			if (arm->flag & ARM_HAS_VIZ_DEPS) {
+				DAG_id_tag_update(id, OB_RECALC_DATA);
+			}
+		}
+		else if (GS(id->name) == ID_OB) {
+			Object *ob = (Object *)id;
+			bArmature *arm = (bArmature *)ob->data;
+			
+			if (arm->flag & ARM_HAS_VIZ_DEPS) {
+				DAG_id_tag_update(id, OB_RECALC_DATA);
+			}
+		}
+	}
+	
 	WM_main_add_notifier(NC_GEOM | ND_DATA, id);
 }
 
@@ -608,8 +637,8 @@ static void rna_def_bone(BlenderRNA *brna)
 	prop = RNA_def_property(srna, "select", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_sdna(prop, NULL, "flag", BONE_SELECTED);
 	RNA_def_property_ui_text(prop, "Select", "");
-	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE);
-	RNA_def_property_update(prop, 0, "rna_Armature_redraw_data");
+	RNA_def_property_clear_flag(prop, PROP_ANIMATABLE); /* XXX: review whether this could be used for interesting effects... */
+	RNA_def_property_update(prop, 0, "rna_Bone_select_update");
 	
 	prop = RNA_def_property(srna, "select_head", PROP_BOOLEAN, PROP_NONE);
 	RNA_def_property_boolean_sdna(prop, NULL, "flag", BONE_ROOTSEL);
diff --git a/source/blender/modifiers/intern/MOD_mask.c b/source/blender/modifiers/intern/MOD_mask.c
index fe17c3ee988..5039ffef382 100644
--- a/source/blender/modifiers/intern/MOD_mask.c
+++ b/source/blender/modifiers/intern/MOD_mask.c
@@ -86,9 +86,12 @@ static void updateDepgraph(ModifierData *md, DagForest *forest,
 	MaskModifierData *mmd = (MaskModifierData *)md;
 
 	if (mmd->ob_arm) {
+		bArmature *arm = (bArmature *)mmd->ob_arm->data;
 		DagNode *armNode = dag_get_node(forest, mmd->ob_arm);
 		
+		/* tag relationship in depsgraph, but also on the armature */
 		dag_add_relation(forest, armNode, obNode, DAG_RL_DATA_DATA | DAG_RL_OB_DATA, "Mask Modifier");
+		arm->flag |= ARM_HAS_VIZ_DEPS;
 	}
 }
 
@@ -149,7 +152,10 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 		if (ELEM3(NULL, oba, oba->pose, ob->defbase.first))
 			return derivedData;
 		
-		/* determine whether each vertexgroup is associated with a selected bone or not */
+		/* determine whether each vertexgroup is associated with a selected bone or not 
+		 * - each cell is a boolean saying whether bone corresponding to the ith group is selected
+		 * - groups that don't match a bone are treated as not existing (along with the corresponding ungrouped verts)
+		 */
 		bone_select_array = MEM_mallocN(defbase_tot * sizeof(char), "mask array");
 		
 		for (i = 0, def = ob->defbase.first; def; def = def->next, i++) {
@@ -163,13 +169,9 @@ static DerivedMesh *applyModifier(ModifierData *md, Object *ob,
 			}
 		}
 		
-		/* if no bones selected, free hashes and return original mesh */
-		if (bone_select_tot == 0) {
-			MEM_freeN(bone_select_array);
-			return derivedData;
-		}
-		
-		/* repeat the previous check, but for dverts */
+		/* if no dverts (i.e. no data for vertex groups exists), we've got an
+		 * inconsistent situation, so free hashes and return oirginal mesh
+		 */
 		dvert = dm->getVertDataArray(dm, CD_MDEFORMVERT);
 		if (dvert == NULL) {
 			MEM_freeN(bone_select_array);