Custom Properties: allow changing the property UI to color picker.

To fully support storing colors as a custom property, it is necessary
to allow switching the property UI to the standard color picker button.
That means in effect supporting custom property subtype values.

Change RNA_property_subtype to look for a 'subtype' string field
in _RNA_UI and parse it as an enum value. To minimize performance
impact, only do it if the property is an array; also, don't use
the custom subtype during RNA path parsing.

On the python side, allow setting some most useful seeming values
from the custom property settings editor.

Also, since some color picker code seems to run into a risk of
buffer overruns if the array size is wrong, check the size in
the UI layout code to be safe.

Reviewers: campbellbarton

Differential Revision: https://developer.blender.org/D5475

2 files changed, 27 insertions, 4 deletions

diff --git a/source/blender/makesrna/RNA_types.h b/source/blender/makesrna/RNA_types.h
index 39889f77a96..38631d1acf2 100644
--- a/source/blender/makesrna/RNA_types.h
+++ b/source/blender/makesrna/RNA_types.h
@@ -105,7 +105,7 @@ typedef enum PropertyUnit {
 #define RNA_STACK_ARRAY 32
 
 /**
- * \note Also update enums in bpy_props.c when adding items here.
+ * \note Also update enums in bpy_props.c and rna_rna.c when adding items here.
  * Watch it: these values are written to files as part of node socket button subtypes!
  */
 typedef enum PropertySubType {
diff --git a/source/blender/makesrna/intern/rna_access.c b/source/blender/makesrna/intern/rna_access.c
index 6d02bfa0987..3e4cb96a8e8 100644
--- a/source/blender/makesrna/intern/rna_access.c
+++ b/source/blender/makesrna/intern/rna_access.c
@@ -1136,12 +1136,34 @@ PropertyType RNA_property_type(PropertyRNA *prop)
 
 PropertySubType RNA_property_subtype(PropertyRNA *prop)
 {
-  return rna_ensure_property(prop)->subtype;
+  PropertyRNA *rna_prop = rna_ensure_property(prop);
+
+  /* For custom properties, find and parse the 'subtype' metadata field. */
+  if (prop->magic != RNA_MAGIC) {
+    IDProperty *idprop = (IDProperty *)prop;
+
+    /* Restrict to arrays only for now for performance reasons. */
+    if (idprop->type == IDP_ARRAY && ELEM(idprop->subtype, IDP_INT, IDP_FLOAT, IDP_DOUBLE)) {
+      IDProperty *idp_ui = rna_idproperty_ui(prop);
+
+      if (idp_ui) {
+        IDProperty *item = IDP_GetPropertyTypeFromGroup(idp_ui, "subtype", IDP_STRING);
+
+        if (item) {
+          int result = PROP_NONE;
+          RNA_enum_value_from_id(rna_enum_property_subtype_items, IDP_String(item), &result);
+          return (PropertySubType)result;
+        }
+      }
+    }
+  }
+
+  return rna_prop->subtype;
 }
 
 PropertyUnit RNA_property_unit(PropertyRNA *prop)
 {
-  return RNA_SUBTYPE_UNIT(rna_ensure_property(prop)->subtype);
+  return RNA_SUBTYPE_UNIT(RNA_property_subtype(prop));
 }
 
 int RNA_property_flag(PropertyRNA *prop)
@@ -1212,7 +1234,7 @@ char RNA_property_array_item_char(PropertyRNA *prop, int index)
   const char *vectoritem = "XYZW";
   const char *quatitem = "WXYZ";
   const char *coloritem = "RGBA";
-  PropertySubType subtype = rna_ensure_property(prop)->subtype;
+  PropertySubType subtype = RNA_property_subtype(prop);
 
   BLI_assert(index >= 0);
 
@@ -1240,6 +1262,7 @@ char RNA_property_array_item_char(PropertyRNA *prop, int index)
 
 int RNA_property_array_item_index(PropertyRNA *prop, char name)
 {
+  /* Don't use custom property subtypes in RNA path lookup. */
   PropertySubType subtype = rna_ensure_property(prop)->subtype;
 
   /* get index based on string name/alias */