Fix T38619: Confusing logic for Keying Set keyframing Settings

The logic used for determining whether certain keyframing settings (i.e. visual,
only needed, xyz -> rgb) got applied was wonky. The original intention here was
that the Keying Set settings would override the global settings, and the path
settings would override what was used for the Keying Set. However, that was not
happening in all cases previously, as it was only possible to add flags and not
to turn them off.

This commit fixes that by introducing separate toggles to control whether the
Keying Set/Path's settings override the settings inherited from its parent
(i.e. the Keying Set for the Path, and the User Prefs for the Keying Set).
The icons used for these toggles could get revised a bit (we need something
which communicates "override this"; the current one is the closest I could find)

WARNING: If you have old keying sets, this may cause some breakage!

1 files changed, 37 insertions, 8 deletions

diff --git a/source/blender/editors/animation/keyingsets.c b/source/blender/editors/animation/keyingsets.c
index a5c0eee8d3b..6c2cc10f2a3 100644
--- a/source/blender/editors/animation/keyingsets.c
+++ b/source/blender/editors/animation/keyingsets.c
@@ -914,6 +914,37 @@ short ANIM_validate_keyingset(bContext *C, ListBase *dsources, KeyingSet *ks)
 	return 0;
 } 
 
+/* Determine which keying flags apply based on the override flags */
+static short keyingset_apply_keying_flags(const short base_flags, const short overrides, const short own_flags)
+{
+	short result = 0;
+	
+	/* The logic for whether a keying flag applies is as follows:
+	 *  - If the flag in question is set in "overrides", that means that the
+	 *    status of that flag in "own_flags" is used
+	 *  - If however the flag isn't set, then its value in "base_flags" is used
+	 *    instead (i.e. no override)
+	 */
+#define APPLY_KEYINGFLAG_OVERRIDE(kflag) \
+	if (overrides & kflag) {             \
+		result |= (own_flags & kflag);   \
+	}                                    \
+	else {                               \
+		result |= (base_flags & kflag);  \
+	}
+	
+	/* Apply the flags one by one... 
+	 * (See rna_def_common_keying_flags() for the supported flags)
+	 */
+	APPLY_KEYINGFLAG_OVERRIDE(INSERTKEY_NEEDED)
+	APPLY_KEYINGFLAG_OVERRIDE(INSERTKEY_MATRIX)
+	APPLY_KEYINGFLAG_OVERRIDE(INSERTKEY_XYZ2RGB)
+	
+#undef APPLY_KEYINGFLAG_OVERRIDE
+
+	return result;
+}
+
 /* Given a KeyingSet and context info (if required), modify keyframes for the channels specified
  * by the KeyingSet. This takes into account many of the different combinations of using KeyingSets.
  * Returns the number of channels that keyframes were added to
@@ -923,7 +954,8 @@ int ANIM_apply_keyingset(bContext *C, ListBase *dsources, bAction *act, KeyingSe
 	Scene *scene = CTX_data_scene(C);
 	ReportList *reports = CTX_wm_reports(C);
 	KS_Path *ksp;
-	int kflag = 0, success = 0;
+	const short base_kflags = ANIM_get_keyframing_flags(scene, 1);
+	short kflag = 0, success = 0;
 	const char *groupname = NULL;
 	
 	/* sanity checks */
@@ -932,11 +964,8 @@ int ANIM_apply_keyingset(bContext *C, ListBase *dsources, bAction *act, KeyingSe
 	
 	/* get flags to use */
 	if (mode == MODIFYKEY_MODE_INSERT) {
-		/* use KeyingSet's flags as base */
-		kflag = ks->keyingflag;
-		
-		/* supplement with info from the context */
-		kflag |= ANIM_get_keyframing_flags(scene, 1);
+		/* use context settings as base */
+		kflag = keyingset_apply_keying_flags(base_kflags, ks->keyingoverride, ks->keyingflag);
 	}
 	else if (mode == MODIFYKEY_MODE_DELETE)
 		kflag = 0;
@@ -962,8 +991,8 @@ int ANIM_apply_keyingset(bContext *C, ListBase *dsources, bAction *act, KeyingSe
 			continue;
 		}
 		
-		/* since keying settings can be defined on the paths too, extend the path before using it */
-		kflag2 = (kflag | ksp->keyingflag);
+		/* since keying settings can be defined on the paths too, apply the settings for this path first */
+		kflag2 = keyingset_apply_keying_flags(kflag, ksp->keyingoverride, ksp->keyingflag);
 		
 		/* get pointer to name of group to add channels to */
 		if (ksp->groupmode == KSP_GROUP_NONE)