Materials: add custom object properties as uniform attributes.

This patch allows the user to type a property name into the
Attribute node, which will then output the value of the property
for each individual object, allowing to e.g. customize shaders
by object without duplicating the shader.

In order to make supporting this easier for Eevee, it is necessary
to explicitly choose whether the attribute is varying or uniform
via a dropdown option of the Attribute node. The dropdown also
allows choosing whether instancing should be taken into account.

The Cycles design treats all attributes as one common namespace,
so the Blender interface converts the enum to a name prefix that
can't be entered using keyboard.

In Eevee, the attributes are provided to the shader via a UBO indexed
with resource_id, similar to the existing Object Info data. Unlike it,
however, it is necessary to maintain a separate buffer for every
requested combination of attributes.

This is done using a hash table with the attribute set as the key,
as it is expected that technically different but similar materials
may use the same set of attributes. In addition, in order to minimize
wasted memory, a sparse UBO pool is implemented, so that chunks that
don't contain any data don't have to be allocated.

The back-end Cycles code is already refactored and committed by Brecht.

Differential Revision: https://developer.blender.org/D2057

9 files changed, 218 insertions, 0 deletions

diff --git a/source/blender/gpu/GPU_material.h b/source/blender/gpu/GPU_material.h
index 2f12625acac..67cd1a61aed 100644
--- a/source/blender/gpu/GPU_material.h
+++ b/source/blender/gpu/GPU_material.h
@@ -34,6 +34,7 @@
 extern "C" {
 #endif
 
+struct GHash;
 struct GPUMaterial;
 struct GPUNode;
 struct GPUNodeLink;
@@ -143,6 +144,7 @@ typedef void (*GPUMaterialEvalCallbackFn)(GPUMaterial *mat,
 GPUNodeLink *GPU_constant(const float *num);
 GPUNodeLink *GPU_uniform(const float *num);
 GPUNodeLink *GPU_attribute(GPUMaterial *mat, CustomDataType type, const char *name);
+GPUNodeLink *GPU_uniform_attribute(GPUMaterial *mat, const char *name, bool use_dupli);
 GPUNodeLink *GPU_image(GPUMaterial *mat,
                        struct Image *ima,
                        struct ImageUser *iuser,
@@ -259,6 +261,31 @@ ListBase GPU_material_attributes(GPUMaterial *material);
 ListBase GPU_material_textures(GPUMaterial *material);
 ListBase GPU_material_volume_grids(GPUMaterial *material);
 
+typedef struct GPUUniformAttr {
+  struct GPUUniformAttr *next, *prev;
+
+  /* Meaningful part of the attribute set key. */
+  char name[64]; /* MAX_CUSTOMDATA_LAYER_NAME */
+  bool use_dupli;
+
+  /* Helper fields used by code generation. */
+  short id;
+  int users;
+} GPUUniformAttr;
+
+typedef struct GPUUniformAttrList {
+  ListBase list; /* GPUUniformAttr */
+
+  /* List length and hash code precomputed for fast lookup and comparison. */
+  unsigned int count, hash_code;
+} GPUUniformAttrList;
+
+GPUUniformAttrList *GPU_material_uniform_attributes(GPUMaterial *material);
+
+struct GHash *GPU_uniform_attr_list_hash_new(const char *info);
+void GPU_uniform_attr_list_copy(GPUUniformAttrList *dest, GPUUniformAttrList *src);
+void GPU_uniform_attr_list_free(GPUUniformAttrList *set);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/blender/gpu/GPU_shader.h b/source/blender/gpu/GPU_shader.h
index 9aaa5d4cae8..27a7ea1e6a5 100644
--- a/source/blender/gpu/GPU_shader.h
+++ b/source/blender/gpu/GPU_shader.h
@@ -413,6 +413,9 @@ void GPU_shader_free_builtin_shaders(void);
  * This makes sure the GPUVertexFormat name buffer does not overflow. */
 #define GPU_MAX_ATTR 15
 
+/* Determined by the maximum uniform buffer size divided by chunk size. */
+#define GPU_MAX_UNIFORM_ATTR 8
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/blender/gpu/GPU_uniform_buffer.h b/source/blender/gpu/GPU_uniform_buffer.h
index ebcaa80e6f6..4efac0a8c00 100644
--- a/source/blender/gpu/GPU_uniform_buffer.h
+++ b/source/blender/gpu/GPU_uniform_buffer.h
@@ -53,6 +53,7 @@ void GPU_uniformbuf_unbind(GPUUniformBuf *ubo);
 void GPU_uniformbuf_unbind_all(void);
 
 #define GPU_UBO_BLOCK_NAME "nodeTree"
+#define GPU_ATTRIBUTE_UBO_BLOCK_NAME "uniformAttrs"
 
 #ifdef __cplusplus
 }
diff --git a/source/blender/gpu/GPU_viewport.h b/source/blender/gpu/GPU_viewport.h
index 7b0d8c274d3..d8e4c5377b0 100644
--- a/source/blender/gpu/GPU_viewport.h
+++ b/source/blender/gpu/GPU_viewport.h
@@ -38,6 +38,7 @@ extern "C" {
 #define GPU_INFO_SIZE 512 /* IMA_MAX_RENDER_TEXT */
 #define GLA_PIXEL_OFS 0.375f
 
+typedef struct GHash GHash;
 typedef struct GPUViewport GPUViewport;
 
 struct GPUFrameBuffer;
@@ -57,6 +58,7 @@ typedef struct ViewportMemoryPool {
   struct BLI_memblock *images;
   struct GPUUniformBuf **matrices_ubo;
   struct GPUUniformBuf **obinfos_ubo;
+  struct GHash *obattrs_ubo_pool;
   uint ubo_len;
 } ViewportMemoryPool;
 
diff --git a/source/blender/gpu/intern/gpu_codegen.c b/source/blender/gpu/intern/gpu_codegen.c
index 39c3119cc39..c16fc08d1aa 100644
--- a/source/blender/gpu/intern/gpu_codegen.c
+++ b/source/blender/gpu/intern/gpu_codegen.c
@@ -377,6 +377,19 @@ static int codegen_process_uniforms_functions(GPUMaterial *material,
     BLI_freelistN(&ubo_inputs);
   }
 
+  /* Generate the uniform attribute UBO if necessary. */
+  if (!BLI_listbase_is_empty(&graph->uniform_attrs.list)) {
+    BLI_dynstr_append(ds, "\nstruct UniformAttributes {\n");
+    LISTBASE_FOREACH (GPUUniformAttr *, attr, &graph->uniform_attrs.list) {
+      BLI_dynstr_appendf(ds, "  vec4 attr%d;\n", attr->id);
+    }
+    BLI_dynstr_append(ds, "};\n");
+    BLI_dynstr_appendf(ds, "layout (std140) uniform %s {\n", GPU_ATTRIBUTE_UBO_BLOCK_NAME);
+    BLI_dynstr_append(ds, "  UniformAttributes uniform_attrs[DRW_RESOURCE_CHUNK_LEN];\n");
+    BLI_dynstr_append(ds, "};\n");
+    BLI_dynstr_append(ds, "#define GET_UNIFORM_ATTR(name) (uniform_attrs[resource_id].name)\n");
+  }
+
   BLI_dynstr_append(ds, "\n");
 
   return builtins;
@@ -480,6 +493,9 @@ static void codegen_call_functions(DynStr *ds, GPUNodeGraph *graph, GPUOutput *f
       else if (input->source == GPU_SOURCE_ATTR) {
         codegen_convert_datatype(ds, input->attr->gputype, input->type, "var", input->attr->id);
       }
+      else if (input->source == GPU_SOURCE_UNIFORM_ATTR) {
+        BLI_dynstr_appendf(ds, "GET_UNIFORM_ATTR(attr%d)", input->uniform_attr->id);
+      }
 
       BLI_dynstr_append(ds, ", ");
     }
@@ -799,6 +815,7 @@ GPUPass *GPU_generate_pass(GPUMaterial *material,
   /* Prune the unused nodes and extract attributes before compiling so the
    * generated VBOs are ready to accept the future shader. */
   gpu_node_graph_prune_unused(graph);
+  gpu_node_graph_finalize_uniform_attrs(graph);
 
   int builtins = 0;
   LISTBASE_FOREACH (GPUNode *, node, &graph->nodes) {
diff --git a/source/blender/gpu/intern/gpu_material.c b/source/blender/gpu/intern/gpu_material.c
index 011d14673b4..e676c0b33cd 100644
--- a/source/blender/gpu/intern/gpu_material.c
+++ b/source/blender/gpu/intern/gpu_material.c
@@ -584,6 +584,12 @@ ListBase GPU_material_volume_grids(GPUMaterial *material)
   return material->graph.volume_grids;
 }
 
+GPUUniformAttrList *GPU_material_uniform_attributes(GPUMaterial *material)
+{
+  GPUUniformAttrList *attrs = &material->graph.uniform_attrs;
+  return attrs->count > 0 ? attrs : NULL;
+}
+
 void GPU_material_output_link(GPUMaterial *material, GPUNodeLink *link)
 {
   if (!material->graph.outlink) {
diff --git a/source/blender/gpu/intern/gpu_node_graph.c b/source/blender/gpu/intern/gpu_node_graph.c
index fdf8ba172cb..2a2a51e32b3 100644
--- a/source/blender/gpu/intern/gpu_node_graph.c
+++ b/source/blender/gpu/intern/gpu_node_graph.c
@@ -137,6 +137,10 @@ static void gpu_node_input_link(GPUNode *node, GPUNodeLink *link, const eGPUType
        * can happen if mixing the generic Attribute node with specialized ones. */
       CLAMP_MIN(input->attr->gputype, type);
       break;
+    case GPU_NODE_LINK_UNIFORM_ATTR:
+      input->source = GPU_SOURCE_UNIFORM_ATTR;
+      input->uniform_attr = link->uniform_attr;
+      break;
     case GPU_NODE_LINK_CONSTANT:
       input->source = (type == GPU_CLOSURE) ? GPU_SOURCE_STRUCT : GPU_SOURCE_CONSTANT;
       break;
@@ -262,8 +266,90 @@ static void gpu_node_output(GPUNode *node, const eGPUType type, GPUNodeLink **li
   BLI_addtail(&node->outputs, output);
 }
 
+/* Uniform Attribute Functions */
+
+static int uniform_attr_sort_cmp(const void *a, const void *b)
+{
+  const GPUUniformAttr *attr_a = a, *attr_b = b;
+
+  int cmps = strcmp(attr_a->name, attr_b->name);
+  if (cmps != 0) {
+    return cmps > 0 ? 1 : 0;
+  }
+
+  return (attr_a->use_dupli && !attr_b->use_dupli);
+}
+
+static unsigned int uniform_attr_list_hash(const void *key)
+{
+  const GPUUniformAttrList *attrs = key;
+  return attrs->hash_code;
+}
+
+static bool uniform_attr_list_cmp(const void *a, const void *b)
+{
+  const GPUUniformAttrList *set_a = a, *set_b = b;
+
+  if (set_a->hash_code != set_b->hash_code || set_a->count != set_b->count) {
+    return true;
+  }
+
+  GPUUniformAttr *attr_a = set_a->list.first, *attr_b = set_b->list.first;
+
+  for (; attr_a && attr_b; attr_a = attr_a->next, attr_b = attr_b->next) {
+    if (!STREQ(attr_a->name, attr_b->name) || attr_a->use_dupli != attr_b->use_dupli) {
+      return true;
+    }
+  }
+
+  return attr_a || attr_b;
+}
+
+struct GHash *GPU_uniform_attr_list_hash_new(const char *info)
+{
+  return BLI_ghash_new(uniform_attr_list_hash, uniform_attr_list_cmp, info);
+}
+
+void GPU_uniform_attr_list_copy(GPUUniformAttrList *dest, GPUUniformAttrList *src)
+{
+  dest->count = src->count;
+  dest->hash_code = src->hash_code;
+  BLI_duplicatelist(&dest->list, &src->list);
+}
+
+void GPU_uniform_attr_list_free(GPUUniformAttrList *set)
+{
+  set->count = 0;
+  set->hash_code = 0;
+  BLI_freelistN(&set->list);
+}
+
+void gpu_node_graph_finalize_uniform_attrs(GPUNodeGraph *graph)
+{
+  GPUUniformAttrList *attrs = &graph->uniform_attrs;
+  BLI_assert(attrs->count == BLI_listbase_count(&attrs->list));
+
+  /* Sort the attributes by name to ensure a stable order. */
+  BLI_listbase_sort(&attrs->list, uniform_attr_sort_cmp);
+
+  /* Compute the indices and the hash code. */
+  int next_id = 0;
+  attrs->hash_code = 0;
+
+  LISTBASE_FOREACH (GPUUniformAttr *, attr, &attrs->list) {
+    attr->id = next_id++;
+
+    attrs->hash_code ^= BLI_ghashutil_strhash_p(attr->name);
+
+    if (attr->use_dupli) {
+      attrs->hash_code ^= BLI_ghashutil_uinthash(attr->id);
+    }
+  }
+}
+
 /* Attributes and Textures */
 
+/** Add a new varying attribute of given type and name. Returns NULL if out of slots. */
 static GPUMaterialAttribute *gpu_node_graph_add_attribute(GPUNodeGraph *graph,
                                                           CustomDataType type,
                                                           const char *name)
@@ -299,6 +385,38 @@ static GPUMaterialAttribute *gpu_node_graph_add_attribute(GPUNodeGraph *graph,
   return attr;
 }
 
+/** Add a new uniform attribute of given type and name. Returns NULL if out of slots. */
+static GPUUniformAttr *gpu_node_graph_add_uniform_attribute(GPUNodeGraph *graph,
+                                                            const char *name,
+                                                            bool use_dupli)
+{
+  /* Find existing attribute. */
+  GPUUniformAttrList *attrs = &graph->uniform_attrs;
+  GPUUniformAttr *attr = attrs->list.first;
+
+  for (; attr; attr = attr->next) {
+    if (STREQ(attr->name, name) && attr->use_dupli == use_dupli) {
+      break;
+    }
+  }
+
+  /* Add new requested attribute if it's within GPU limits. */
+  if (attr == NULL && attrs->count < GPU_MAX_UNIFORM_ATTR) {
+    attr = MEM_callocN(sizeof(*attr), __func__);
+    STRNCPY(attr->name, name);
+    attr->use_dupli = use_dupli;
+    attr->id = -1;
+    BLI_addtail(&attrs->list, attr);
+    attrs->count++;
+  }
+
+  if (attr != NULL) {
+    attr->users++;
+  }
+
+  return attr;
+}
+
 static GPUMaterialTexture *gpu_node_graph_add_texture(GPUNodeGraph *graph,
                                                       Image *ima,
                                                       ImageUser *iuser,
@@ -372,6 +490,7 @@ GPUNodeLink *GPU_attribute(GPUMaterial *mat, const CustomDataType type, const ch
   GPUNodeGraph *graph = gpu_material_node_graph(mat);
   GPUMaterialAttribute *attr = gpu_node_graph_add_attribute(graph, type, name);
 
+  /* Dummy fallback if out of slots. */
   if (attr == NULL) {
     static const float zero_data[GPU_MAX_CONSTANT_DATA] = {0.0f};
     return GPU_constant(zero_data);
@@ -383,6 +502,23 @@ GPUNodeLink *GPU_attribute(GPUMaterial *mat, const CustomDataType type, const ch
   return link;
 }
 
+GPUNodeLink *GPU_uniform_attribute(GPUMaterial *mat, const char *name, bool use_dupli)
+{
+  GPUNodeGraph *graph = gpu_material_node_graph(mat);
+  GPUUniformAttr *attr = gpu_node_graph_add_uniform_attribute(graph, name, use_dupli);
+
+  /* Dummy fallback if out of slots. */
+  if (attr == NULL) {
+    static const float zero_data[GPU_MAX_CONSTANT_DATA] = {0.0f};
+    return GPU_constant(zero_data);
+  }
+
+  GPUNodeLink *link = gpu_node_link_create();
+  link->link_type = GPU_NODE_LINK_UNIFORM_ATTR;
+  link->uniform_attr = attr;
+  return link;
+}
+
 GPUNodeLink *GPU_constant(const float *num)
 {
   GPUNodeLink *link = gpu_node_link_create();
@@ -619,6 +755,9 @@ static void gpu_inputs_free(ListBase *inputs)
     if (input->source == GPU_SOURCE_ATTR) {
       input->attr->users--;
     }
+    else if (input->source == GPU_SOURCE_UNIFORM_ATTR) {
+      input->uniform_attr->users--;
+    }
     else if (ELEM(input->source, GPU_SOURCE_TEX, GPU_SOURCE_TEX_TILED_MAPPING)) {
       input->texture->users--;
     }
@@ -674,6 +813,7 @@ void gpu_node_graph_free(GPUNodeGraph *graph)
   BLI_freelistN(&graph->volume_grids);
   BLI_freelistN(&graph->textures);
   BLI_freelistN(&graph->attributes);
+  GPU_uniform_attr_list_free(&graph->uniform_attrs);
 }
 
 /* Prune Unused Nodes */
@@ -738,4 +878,13 @@ void gpu_node_graph_prune_unused(GPUNodeGraph *graph)
       BLI_freelinkN(&graph->volume_grids, grid);
     }
   }
+
+  GPUUniformAttrList *uattrs = &graph->uniform_attrs;
+
+  LISTBASE_FOREACH_MUTABLE (GPUUniformAttr *, attr, &uattrs->list) {
+    if (attr->users == 0) {
+      BLI_freelinkN(&uattrs->list, attr);
+      uattrs->count--;
+    }
+  }
 }
diff --git a/source/blender/gpu/intern/gpu_node_graph.h b/source/blender/gpu/intern/gpu_node_graph.h
index 7265abf4d65..a0e6298cd92 100644
--- a/source/blender/gpu/intern/gpu_node_graph.h
+++ b/source/blender/gpu/intern/gpu_node_graph.h
@@ -42,6 +42,7 @@ typedef enum eGPUDataSource {
   GPU_SOURCE_CONSTANT,
   GPU_SOURCE_UNIFORM,
   GPU_SOURCE_ATTR,
+  GPU_SOURCE_UNIFORM_ATTR,
   GPU_SOURCE_BUILTIN,
   GPU_SOURCE_STRUCT,
   GPU_SOURCE_TEX,
@@ -53,6 +54,7 @@ typedef enum eGPUDataSource {
 typedef enum {
   GPU_NODE_LINK_NONE = 0,
   GPU_NODE_LINK_ATTR,
+  GPU_NODE_LINK_UNIFORM_ATTR,
   GPU_NODE_LINK_BUILTIN,
   GPU_NODE_LINK_COLORBAND,
   GPU_NODE_LINK_CONSTANT,
@@ -96,6 +98,8 @@ struct GPUNodeLink {
     struct GPUOutput *output;
     /* GPU_NODE_LINK_ATTR */
     struct GPUMaterialAttribute *attr;
+    /* GPU_NODE_LINK_UNIFORM_ATTR */
+    struct GPUUniformAttr *uniform_attr;
     /* GPU_NODE_LINK_IMAGE_BLENDER */
     struct GPUMaterialTexture *texture;
   };
@@ -130,6 +134,8 @@ typedef struct GPUInput {
     struct GPUMaterialTexture *texture;
     /* GPU_SOURCE_ATTR */
     struct GPUMaterialAttribute *attr;
+    /* GPU_SOURCE_UNIFORM_ATTR */
+    struct GPUUniformAttr *uniform_attr;
     /* GPU_SOURCE_VOLUME_GRID | GPU_SOURCE_VOLUME_GRID_TRANSFORM */
     struct GPUMaterialVolumeGrid *volume_grid;
   };
@@ -146,11 +152,15 @@ typedef struct GPUNodeGraph {
   ListBase attributes;
   ListBase textures;
   ListBase volume_grids;
+
+  /* The list of uniform attributes. */
+  GPUUniformAttrList uniform_attrs;
 } GPUNodeGraph;
 
 /* Node Graph */
 
 void gpu_node_graph_prune_unused(GPUNodeGraph *graph);
+void gpu_node_graph_finalize_uniform_attrs(GPUNodeGraph *graph);
 void gpu_node_graph_free_nodes(GPUNodeGraph *graph);
 void gpu_node_graph_free(GPUNodeGraph *graph);
 
diff --git a/source/blender/gpu/intern/gpu_viewport.c b/source/blender/gpu/intern/gpu_viewport.c
index 9063c8bdbce..188c8786665 100644
--- a/source/blender/gpu/intern/gpu_viewport.c
+++ b/source/blender/gpu/intern/gpu_viewport.c
@@ -1023,6 +1023,9 @@ void GPU_viewport_free(GPUViewport *viewport)
     }
     BLI_memblock_destroy(viewport->vmempool.images, NULL);
   }
+  if (viewport->vmempool.obattrs_ubo_pool != NULL) {
+    DRW_uniform_attrs_pool_free(viewport->vmempool.obattrs_ubo_pool);
+  }
 
   for (int i = 0; i < viewport->vmempool.ubo_len; i++) {
     GPU_uniformbuf_free(viewport->vmempool.matrices_ubo[i]);