Fix #36226, Select Linked works not in touch with Prefs.

When setting keymap properties to values equalling the RNA default, they will
get "unset" and automatic operator behavior is used. There is no way to
explicitly set the default value as a user.

1) To allow distinguishing uninitialized (not set) properties in the keymap
items, a few changes to the RNA struct comparison function are needed: Instead
of allowing only strict/non-strict comparison of 2 properties A and B in a
struct, this now has 3 modes:
* STRICT: compare only the actual property values (same as 'strict' before)
* UNSET_MATCH_ANY: if either A or B is unset, consider them a match (same as
non-strict before)
* UNSET_MATCH_NONE: if one property is set and the other not, consider them a
mismatch.

The new UNSET_MATCH_NONE mode is useful for keymaps, because it allows keeping
user-defined property values in the keymap even if they match the default
property value (see wm_keymap_diff function in wm_keymap.c)

2) A new operator is added for unsetting ID properties in the RMB context menu
and in user preferences next to keymap properties. This only works on ID
properties and deletes the ID property storage, so that the default value is used.

In the user preferences for keymaps the properties are shown in an inactive
layout to indicate that the default value is used (which some operators such as
the "select linked" op from the report use to trigger automatic behavior). When
the user sets a property it gets set and stays that way until explicitly "unset"
using the new operator.

6 files changed, 114 insertions, 39 deletions

diff --git a/source/blender/editors/interface/interface_handlers.c b/source/blender/editors/interface/interface_handlers.c
index 5d982b60c91..4a6d14d7004 100644
--- a/source/blender/editors/interface/interface_handlers.c
+++ b/source/blender/editors/interface/interface_handlers.c
@@ -5092,7 +5092,12 @@ static bool ui_but_menu(bContext *C, uiBut *but)
 	uiLayoutSetOperatorContext(layout, WM_OP_INVOKE_DEFAULT);
 
 	if (but->rnapoin.data && but->rnaprop) {
-		bool is_anim = RNA_property_animateable(&but->rnapoin, but->rnaprop);
+		PointerRNA *ptr = &but->rnapoin;
+		PropertyRNA *prop = but->rnaprop;
+		bool is_anim = RNA_property_animateable(ptr, prop);
+		bool is_editable = RNA_property_editable(ptr, prop);
+		/*bool is_idprop = RNA_property_is_idprop(prop);*/ /* XXX does not work as expected, not strictly needed */
+		bool is_set = RNA_property_is_set(ptr, prop);
 
 		/* second slower test, saved people finding keyframe items in menus when its not possible */
 		if (is_anim)
@@ -5239,6 +5244,10 @@ static bool ui_but_menu(bContext *C, uiBut *but)
 			uiItemBooleanO(layout, CTX_IFACE_(BLF_I18NCONTEXT_OPERATOR_DEFAULT, "Reset to Default Value"),
 			        ICON_NONE, "UI_OT_reset_default_button", "all", 1);
 		}
+		if (is_editable /*&& is_idprop*/ && is_set) {
+			uiItemO(layout, CTX_IFACE_(BLF_I18NCONTEXT_OPERATOR_DEFAULT, "Unset"),
+			        ICON_NONE, "UI_OT_unset_property_button");
+		}
 		
 		uiItemO(layout, CTX_IFACE_(BLF_I18NCONTEXT_OPERATOR_DEFAULT, "Copy Data Path"),
 		        ICON_NONE, "UI_OT_copy_data_path_button");
diff --git a/source/blender/editors/interface/interface_ops.c b/source/blender/editors/interface/interface_ops.c
index d714402de9f..d52702c2d51 100644
--- a/source/blender/editors/interface/interface_ops.c
+++ b/source/blender/editors/interface/interface_ops.c
@@ -434,6 +434,28 @@ static void UI_OT_copy_data_path_button(wmOperatorType *ot)
 
 /* Reset to Default Values Button Operator ------------------------ */
 
+static int operator_button_property_finish(bContext *C, PointerRNA *ptr, PropertyRNA *prop)
+{
+	ID *id = ptr->id.data;
+
+	/* perform updates required for this property */
+	RNA_property_update(C, ptr, prop);
+
+	/* as if we pressed the button */
+	uiContextActivePropertyHandle(C);
+
+	/* Since we don't want to undo _all_ edits to settings, eg window
+	 * edits on the screen or on operator settings.
+	 * it might be better to move undo's inline - campbell */
+	if (id && ID_CHECK_UNDO(id)) {
+		/* do nothing, go ahead with undo */
+		return OPERATOR_FINISHED;
+	}
+	else {
+		return OPERATOR_CANCELLED;
+	}
+}
+
 static int reset_default_button_poll(bContext *C)
 {
 	PointerRNA ptr;
@@ -449,7 +471,6 @@ static int reset_default_button_exec(bContext *C, wmOperator *op)
 {
 	PointerRNA ptr;
 	PropertyRNA *prop;
-	int success = 0;
 	int index, all = RNA_boolean_get(op->ptr, "all");
 
 	/* try to reset the nominated setting to its default value */
@@ -457,32 +478,11 @@ static int reset_default_button_exec(bContext *C, wmOperator *op)
 	
 	/* if there is a valid property that is editable... */
 	if (ptr.data && prop && RNA_property_editable(&ptr, prop)) {
-		if (RNA_property_reset(&ptr, prop, (all) ? -1 : index)) {
-			/* perform updates required for this property */
-			RNA_property_update(C, &ptr, prop);
-
-			/* as if we pressed the button */
-			uiContextActivePropertyHandle(C);
-
-			success = 1;
-		}
-	}
-
-	/* Since we don't want to undo _all_ edits to settings, eg window
-	 * edits on the screen or on operator settings.
-	 * it might be better to move undo's inline - campbell */
-	if (success) {
-		ID *id = ptr.id.data;
-		if (id && ID_CHECK_UNDO(id)) {
-			/* do nothing, go ahead with undo */
-		}
-		else {
-			return OPERATOR_CANCELLED;
-		}
+		if (RNA_property_reset(&ptr, prop, (all) ? -1 : index))
+			return operator_button_property_finish(C, &ptr, prop);
 	}
-	/* end hack */
 
-	return (success) ? OPERATOR_FINISHED : OPERATOR_CANCELLED;
+	return OPERATOR_CANCELLED;
 }
 
 static void UI_OT_reset_default_button(wmOperatorType *ot)
@@ -503,6 +503,43 @@ static void UI_OT_reset_default_button(wmOperatorType *ot)
 	RNA_def_boolean(ot->srna, "all", 1, "All", "Reset to default values all elements of the array");
 }
 
+/* Unset Property Button Operator ------------------------ */
+
+static int unset_property_button_exec(bContext *C, wmOperator *UNUSED(op))
+{
+	PointerRNA ptr;
+	PropertyRNA *prop;
+	int index;
+
+	/* try to unset the nominated property */
+	uiContextActiveProperty(C, &ptr, &prop, &index);
+
+	/* if there is a valid property that is editable... */
+	if (ptr.data && prop && RNA_property_editable(&ptr, prop)
+	    /*&& RNA_property_is_idprop(prop)*/ && RNA_property_is_set(&ptr, prop))
+	{
+		RNA_property_unset(&ptr, prop);
+		return operator_button_property_finish(C, &ptr, prop);
+	}
+
+	return OPERATOR_CANCELLED;
+}
+
+static void UI_OT_unset_property_button(wmOperatorType *ot)
+{
+	/* identifiers */
+	ot->name = "Unset property";
+	ot->idname = "UI_OT_unset_property_button";
+	ot->description = "Clear the property and use default or generated value in operators";
+
+	/* callbacks */
+	ot->poll = ED_operator_regionactive;
+	ot->exec = unset_property_button_exec;
+
+	/* flags */
+	ot->flag = OPTYPE_UNDO;
+}
+
 /* Copy To Selected Operator ------------------------ */
 
 static int copy_to_selected_list(bContext *C, PointerRNA *ptr, ListBase *lb, int *use_path)
@@ -1081,6 +1118,7 @@ void UI_buttons_operatortypes(void)
 	WM_operatortype_append(UI_OT_reset_default_theme);
 	WM_operatortype_append(UI_OT_copy_data_path_button);
 	WM_operatortype_append(UI_OT_reset_default_button);
+	WM_operatortype_append(UI_OT_unset_property_button);
 	WM_operatortype_append(UI_OT_copy_to_selected_button);
 	WM_operatortype_append(UI_OT_reports_to_textblock);  /* XXX: temp? */
 
diff --git a/source/blender/editors/interface/interface_templates.c b/source/blender/editors/interface/interface_templates.c
index bd3103e499e..6b6b7114c84 100644
--- a/source/blender/editors/interface/interface_templates.c
+++ b/source/blender/editors/interface/interface_templates.c
@@ -3344,7 +3344,7 @@ static void keymap_item_modified(bContext *UNUSED(C), void *kmi_p, void *UNUSED(
 
 static void template_keymap_item_properties(uiLayout *layout, const char *title, PointerRNA *ptr)
 {
-	uiLayout *flow;
+	uiLayout *flow, *box, *row;
 
 	uiItemS(layout);
 
@@ -3356,6 +3356,8 @@ static void template_keymap_item_properties(uiLayout *layout, const char *title,
 	RNA_STRUCT_BEGIN (ptr, prop)
 	{
 		int flag = RNA_property_flag(prop);
+		bool is_set = RNA_property_is_set(ptr, prop);
+		uiBut *but;
 
 		if (flag & PROP_HIDDEN)
 			continue;
@@ -3371,8 +3373,22 @@ static void template_keymap_item_properties(uiLayout *layout, const char *title,
 			}
 		}
 
-		/* add property */
-		uiItemFullR(flow, ptr, prop, -1, 0, 0, NULL, ICON_NONE);
+		box = uiLayoutBox(flow);
+		uiLayoutSetActive(box, is_set);
+		row = uiLayoutRow(box, false);
+
+		/* property value */
+		uiItemFullR(row, ptr, prop, -1, 0, 0, NULL, ICON_NONE);
+
+		if (is_set) {
+			/* unset operator */
+			uiBlock *block = uiLayoutGetBlock(row);
+			uiBlockSetEmboss(block, UI_EMBOSSN);
+			but = uiDefIconButO(block, BUT, "UI_OT_unset_property_button", WM_OP_EXEC_DEFAULT, ICON_X, 0, 0, UI_UNIT_X, UI_UNIT_Y, NULL);
+			but->rnapoin = *ptr;
+			but->rnaprop = prop;
+			uiBlockSetEmboss(block, UI_EMBOSS);
+		}
 	}
 	RNA_STRUCT_END;
 }
diff --git a/source/blender/makesrna/RNA_access.h b/source/blender/makesrna/RNA_access.h
index d18fe7629fd..b4b79059f93 100644
--- a/source/blender/makesrna/RNA_access.h
+++ b/source/blender/makesrna/RNA_access.h
@@ -1119,8 +1119,14 @@ void _RNA_warning(const char *format, ...) ATTR_PRINTF_FORMAT(1, 2);
 /* Equals test (skips pointers and collections)
  * is_strict false assumes uninitialized properties are equal */
 
-bool RNA_property_equals(struct PointerRNA *a, struct PointerRNA *b, struct PropertyRNA *prop, bool is_strict);
-bool RNA_struct_equals(struct PointerRNA *a, struct PointerRNA *b, bool is_strict);
+typedef enum eRNAEqualsMode {
+	RNA_EQ_STRICT,          /* set/unset ignored */
+	RNA_EQ_UNSET_MATCH_ANY, /* unset property matches anything */
+	RNA_EQ_UNSET_MATCH_NONE /* unset property never matches set property */
+} eRNAEqualsMode;
+
+bool RNA_property_equals(struct PointerRNA *a, struct PointerRNA *b, struct PropertyRNA *prop, eRNAEqualsMode mode);
+bool RNA_struct_equals(struct PointerRNA *a, struct PointerRNA *b, eRNAEqualsMode mode);
 
 #ifdef __cplusplus
 }
diff --git a/source/blender/makesrna/intern/rna_access.c b/source/blender/makesrna/intern/rna_access.c
index 918c67b4513..b164a3c3ed9 100644
--- a/source/blender/makesrna/intern/rna_access.c
+++ b/source/blender/makesrna/intern/rna_access.c
@@ -6316,14 +6316,20 @@ void _RNA_warning(const char *format, ...)
 #endif
 }
 
-bool RNA_property_equals(PointerRNA *a, PointerRNA *b, PropertyRNA *prop, bool is_strict)
+bool RNA_property_equals(PointerRNA *a, PointerRNA *b, PropertyRNA *prop, eRNAEqualsMode mode)
 {
 	int len, fromlen;
 
-	/* if not strict, uninitialized properties are assumed to match */
-	if (!is_strict)
-		if (!(RNA_property_is_set(a, prop) && RNA_property_is_set(b, prop)))
+	if (mode == RNA_EQ_UNSET_MATCH_ANY) {
+		/* uninitialized properties are assumed to match anything */
+		if (!RNA_property_is_set(a, prop) || !RNA_property_is_set(b, prop))
 			return true;
+	}
+	else if (mode == RNA_EQ_UNSET_MATCH_NONE) {
+		/* unset properties never match set properties */
+		if (RNA_property_is_set(a, prop) != RNA_property_is_set(b, prop))
+			return false;
+	}
 
 	/* get the length of the array to work with */
 	len = RNA_property_array_length(a, prop);
@@ -6437,7 +6443,7 @@ bool RNA_property_equals(PointerRNA *a, PointerRNA *b, PropertyRNA *prop, bool i
 			if (!STREQ(RNA_property_identifier(prop), "rna_type")) {
 				PointerRNA propptr_a = RNA_property_pointer_get(a, prop);
 				PointerRNA propptr_b = RNA_property_pointer_get(b, prop);
-				return RNA_struct_equals(&propptr_a, &propptr_b, is_strict);
+				return RNA_struct_equals(&propptr_a, &propptr_b, mode);
 			}
 			break;
 		}
@@ -6449,7 +6455,7 @@ bool RNA_property_equals(PointerRNA *a, PointerRNA *b, PropertyRNA *prop, bool i
 	return true;
 }
 
-bool RNA_struct_equals(PointerRNA *a, PointerRNA *b, bool is_strict)
+bool RNA_struct_equals(PointerRNA *a, PointerRNA *b, eRNAEqualsMode mode)
 {
 	CollectionPropertyIterator iter;
 //	CollectionPropertyRNA *citerprop;  /* UNUSED */
@@ -6470,7 +6476,7 @@ bool RNA_struct_equals(PointerRNA *a, PointerRNA *b, bool is_strict)
 	for (; iter.valid; RNA_property_collection_next(&iter)) {
 		PropertyRNA *prop = iter.ptr.data;
 
-		if (!RNA_property_equals(a, b, prop, is_strict)) {
+		if (!RNA_property_equals(a, b, prop, mode)) {
 			equals = false;
 			break;
 		}
diff --git a/source/blender/windowmanager/intern/wm_keymap.c b/source/blender/windowmanager/intern/wm_keymap.c
index ff805579b44..7ba9316fd84 100644
--- a/source/blender/windowmanager/intern/wm_keymap.c
+++ b/source/blender/windowmanager/intern/wm_keymap.c
@@ -101,7 +101,7 @@ static int wm_keymap_item_equals_result(wmKeyMapItem *a, wmKeyMapItem *b)
 	if (strcmp(a->idname, b->idname) != 0)
 		return 0;
 	
-	if (!RNA_struct_equals(a->ptr, b->ptr, true))
+	if (!RNA_struct_equals(a->ptr, b->ptr, RNA_EQ_UNSET_MATCH_NONE))
 		return 0;
 	
 	if ((a->flag & KMI_INACTIVE) != (b->flag & KMI_INACTIVE))