Custom Properties: allow changing the property UI to color picker.

To fully support storing colors as a custom property, it is necessary
to allow switching the property UI to the standard color picker button.
That means in effect supporting custom property subtype values.

Change RNA_property_subtype to look for a 'subtype' string field
in _RNA_UI and parse it as an enum value. To minimize performance
impact, only do it if the property is an array; also, don't use
the custom subtype during RNA path parsing.

On the python side, allow setting some most useful seeming values
from the custom property settings editor.

Also, since some color picker code seems to run into a risk of
buffer overruns if the array size is wrong, check the size in
the UI layout code to be safe.

Reviewers: campbellbarton

Differential Revision: https://developer.blender.org/D5475

5 files changed, 77 insertions, 7 deletions

diff --git a/release/scripts/modules/rna_prop_ui.py b/release/scripts/modules/rna_prop_ui.py
index b568f9835a0..2ff6c3fc1b0 100644
--- a/release/scripts/modules/rna_prop_ui.py
+++ b/release/scripts/modules/rna_prop_ui.py
@@ -159,6 +159,7 @@ def rna_idprop_ui_create(
         soft_min=None, soft_max=None,
         description=None,
         overridable=False,
+        subtype=None,
 ):
     """Create and initialize a custom property with limits, defaults and other settings."""
 
@@ -195,6 +196,9 @@ def rna_idprop_ui_create(
         if default and (not is_array or any(default)):
             rna_ui["default"] = default
 
+        if is_array and subtype and subtype != 'NONE':
+            rna_ui["subtype"] = subtype
+
     # Assign other settings
     if description is not None:
         rna_ui["description"] = description
diff --git a/release/scripts/startup/bl_operators/wm.py b/release/scripts/startup/bl_operators/wm.py
index 97a242772a4..3f25c35973d 100644
--- a/release/scripts/startup/bl_operators/wm.py
+++ b/release/scripts/startup/bl_operators/wm.py
@@ -1085,6 +1085,14 @@ rna_is_overridable_library = BoolProperty(
     default=False,
 )
 
+# Most useful entries of rna_enum_property_subtype_items for number arrays:
+rna_vector_subtype_items = (
+    ('NONE', "Plain Data", "Data values without special behavior"),
+    ('COLOR', "Linear Color", "Color in the linear space"),
+    ('COLOR_GAMMA', "Gamma-Corrected Color", "Color in the gamma corrected space"),
+    ('EULER', "Euler Angles", "Euler rotation angles in radians"),
+    ('QUATERNION', "Quaternion Rotation", "Quaternion rotation (affects NLA blending)"),
+)
 
 class WM_OT_properties_edit(Operator):
     bl_idname = "wm.properties_edit"
@@ -1105,6 +1113,23 @@ class WM_OT_properties_edit(Operator):
     description: StringProperty(
         name="Tooltip",
     )
+    subtype: EnumProperty(
+        name="Subtype",
+        items=lambda self,ctx: WM_OT_properties_edit.subtype_items,
+    )
+
+    subtype_items = rna_vector_subtype_items
+
+    def init_subtype(self, prop_type, is_array, subtype):
+        subtype = subtype or 'NONE'
+        subtype_items = rna_vector_subtype_items
+
+        # Add a temporary enum entry to preserve unknown subtypes
+        if not any(subtype == item[0] for item in subtype_items):
+            subtype_items += ((subtype, subtype, ""),)
+
+        WM_OT_properties_edit.subtype_items = subtype_items
+        self.subtype = subtype
 
     def _cmp_props_get(self):
         # Changing these properties will refresh the UI
@@ -1193,6 +1218,11 @@ class WM_OT_properties_edit(Operator):
                 prop_ui["soft_min"] = prop_type(self.min)
                 prop_ui["soft_max"] = prop_type(self.max)
 
+        if prop_type == float and is_array and self.subtype != 'NONE':
+            prop_ui["subtype"] = self.subtype
+        else:
+            prop_ui.pop("subtype", None)
+
         prop_ui["description"] = self.description
 
         rna_idprop_ui_prop_default_set(item, prop, default_eval)
@@ -1235,7 +1265,11 @@ class WM_OT_properties_edit(Operator):
         return {'FINISHED'}
 
     def invoke(self, context, _event):
-        from rna_prop_ui import rna_idprop_ui_prop_get, rna_idprop_value_to_python
+        from rna_prop_ui import (
+            rna_idprop_ui_prop_get,
+            rna_idprop_value_to_python,
+            rna_idprop_value_item_type
+        )
 
         data_path = self.data_path
 
@@ -1252,7 +1286,7 @@ class WM_OT_properties_edit(Operator):
         self.is_overridable_library = bool(eval(exec_str))
 
         # default default value
-        prop_type = type(self.get_value_eval())
+        prop_type, is_array = rna_idprop_value_item_type(self.get_value_eval())
         if prop_type in {int, float}:
             self.default = str(prop_type(0))
         else:
@@ -1276,6 +1310,12 @@ class WM_OT_properties_edit(Operator):
                 self.max != self.soft_max
             )
 
+            subtype = prop_ui.get("subtype", None)
+        else:
+            subtype = None
+
+        self.init_subtype(prop_type, is_array, subtype)
+
         # store for comparison
         self._cmp_props = self._cmp_props_get()
 
@@ -1341,6 +1381,9 @@ class WM_OT_properties_edit(Operator):
         row.prop(self, "soft_max", text="Soft Max")
         layout.prop(self, "description")
 
+        if is_array and proptype == float:
+            layout.prop(self, "subtype")
+
 
 class WM_OT_properties_add(Operator):
     bl_idname = "wm.properties_add"
diff --git a/source/blender/editors/interface/interface_layout.c b/source/blender/editors/interface/interface_layout.c
index 530689f2d27..78eed98eb77 100644
--- a/source/blender/editors/interface/interface_layout.c
+++ b/source/blender/editors/interface/interface_layout.c
@@ -648,7 +648,7 @@ static void ui_item_array(uiLayout *layout,
      * to work with common cases, but may need to be re-worked */
 
     /* special case, boolean array in a menu, this could be used in a more generic way too */
-    if (ELEM(subtype, PROP_COLOR, PROP_COLOR_GAMMA) && !expand) {
+    if (ELEM(subtype, PROP_COLOR, PROP_COLOR_GAMMA) && !expand && ELEM(len, 3, 4)) {
       uiDefAutoButR(block, ptr, prop, -1, "", ICON_NONE, 0, 0, w, UI_UNIT_Y);
     }
     else {
diff --git a/source/blender/makesrna/RNA_types.h b/source/blender/makesrna/RNA_types.h
index 39889f77a96..38631d1acf2 100644
--- a/source/blender/makesrna/RNA_types.h
+++ b/source/blender/makesrna/RNA_types.h
@@ -105,7 +105,7 @@ typedef enum PropertyUnit {
 #define RNA_STACK_ARRAY 32
 
 /**
- * \note Also update enums in bpy_props.c when adding items here.
+ * \note Also update enums in bpy_props.c and rna_rna.c when adding items here.
  * Watch it: these values are written to files as part of node socket button subtypes!
  */
 typedef enum PropertySubType {
diff --git a/source/blender/makesrna/intern/rna_access.c b/source/blender/makesrna/intern/rna_access.c
index 6d02bfa0987..3e4cb96a8e8 100644
--- a/source/blender/makesrna/intern/rna_access.c
+++ b/source/blender/makesrna/intern/rna_access.c
@@ -1136,12 +1136,34 @@ PropertyType RNA_property_type(PropertyRNA *prop)
 
 PropertySubType RNA_property_subtype(PropertyRNA *prop)
 {
-  return rna_ensure_property(prop)->subtype;
+  PropertyRNA *rna_prop = rna_ensure_property(prop);
+
+  /* For custom properties, find and parse the 'subtype' metadata field. */
+  if (prop->magic != RNA_MAGIC) {
+    IDProperty *idprop = (IDProperty *)prop;
+
+    /* Restrict to arrays only for now for performance reasons. */
+    if (idprop->type == IDP_ARRAY && ELEM(idprop->subtype, IDP_INT, IDP_FLOAT, IDP_DOUBLE)) {
+      IDProperty *idp_ui = rna_idproperty_ui(prop);
+
+      if (idp_ui) {
+        IDProperty *item = IDP_GetPropertyTypeFromGroup(idp_ui, "subtype", IDP_STRING);
+
+        if (item) {
+          int result = PROP_NONE;
+          RNA_enum_value_from_id(rna_enum_property_subtype_items, IDP_String(item), &result);
+          return (PropertySubType)result;
+        }
+      }
+    }
+  }
+
+  return rna_prop->subtype;
 }
 
 PropertyUnit RNA_property_unit(PropertyRNA *prop)
 {
-  return RNA_SUBTYPE_UNIT(rna_ensure_property(prop)->subtype);
+  return RNA_SUBTYPE_UNIT(RNA_property_subtype(prop));
 }
 
 int RNA_property_flag(PropertyRNA *prop)
@@ -1212,7 +1234,7 @@ char RNA_property_array_item_char(PropertyRNA *prop, int index)
   const char *vectoritem = "XYZW";
   const char *quatitem = "WXYZ";
   const char *coloritem = "RGBA";
-  PropertySubType subtype = rna_ensure_property(prop)->subtype;
+  PropertySubType subtype = RNA_property_subtype(prop);
 
   BLI_assert(index >= 0);
 
@@ -1240,6 +1262,7 @@ char RNA_property_array_item_char(PropertyRNA *prop, int index)
 
 int RNA_property_array_item_index(PropertyRNA *prop, char name)
 {
+  /* Don't use custom property subtypes in RNA path lookup. */
   PropertySubType subtype = rna_ensure_property(prop)->subtype;
 
   /* get index based on string name/alias */