EEVEE: Refactor of eevee_material.c

These are the modifications:

-With DRW modification we reduce the number of passes we need to populate.
-Rename passes for consistent naming.
-Reduce complexity in code compilation
-Cleanup how renderpass accumulation passes are setup, using pass instances.
-Make sculpt mode compatible with shadows
-Make hair passes compatible with SSS
-Error shader and lookdev materials now use standalone materials.
-Support default shader (world and material) using a default nodetree internally.
-Change BLEND_CLIP to be emulated by gpu nodetree. Making less shader variations.
-Use BLI_memblock for cache memory allocation.
-Renderpasses are handled by switching a UBO ref bind.

One major hack in this patch is the use of modified pointer as ghash keys.
This rely on the assumption that the keys will never overlap because the
number of options per key will never be bigger than the pointed struct.

The use of one single nodetree to support default material is also a bit hacky
since it won't support concurent usage of this nodetree.
(see EEVEE_shader_default_surface_nodetree)

Another change is that objects with shader errors now appear solid magenta instead
of shaded magenta. This is only because of code reuse purpose but could be changed
if really needed.

Reviewed By: jbakker

Differential Revision: https://developer.blender.org/D7642

7 files changed, 124 insertions, 78 deletions

diff --git a/source/blender/draw/intern/DRW_render.h b/source/blender/draw/intern/DRW_render.h
index e4088b5654f..df1258e58fb 100644
--- a/source/blender/draw/intern/DRW_render.h
+++ b/source/blender/draw/intern/DRW_render.h
@@ -224,6 +224,7 @@ struct GPUMaterial *DRW_shader_find_from_material(struct Material *ma,
                                                   bool deferred);
 struct GPUMaterial *DRW_shader_create_from_world(struct Scene *scene,
                                                  struct World *wo,
+                                                 struct bNodeTree *ntree,
                                                  const void *engine_type,
                                                  const int options,
                                                  const bool is_volume_shader,
@@ -234,6 +235,7 @@ struct GPUMaterial *DRW_shader_create_from_world(struct Scene *scene,
                                                  bool deferred);
 struct GPUMaterial *DRW_shader_create_from_material(struct Scene *scene,
                                                     struct Material *ma,
+                                                    struct bNodeTree *ntree,
                                                     const void *engine_type,
                                                     const int options,
                                                     const bool is_volume_shader,
@@ -365,6 +367,8 @@ DRWShadingGroup *DRW_shgroup_transform_feedback_create(struct GPUShader *shader,
                                                        DRWPass *pass,
                                                        struct GPUVertBuf *tf_target);
 
+void DRW_shgroup_add_material_resources(DRWShadingGroup *grp, struct GPUMaterial *material);
+
 /* return final visibility */
 typedef bool(DRWCallVisibilityFn)(bool vis_in, void *user_data);
 
@@ -463,15 +467,24 @@ void DRW_shgroup_uniform_texture(DRWShadingGroup *shgroup,
 void DRW_shgroup_uniform_texture_persistent(DRWShadingGroup *shgroup,
                                             const char *name,
                                             const struct GPUTexture *tex);
+void DRW_shgroup_uniform_texture_ref(DRWShadingGroup *shgroup,
+                                     const char *name,
+                                     struct GPUTexture **tex);
+void DRW_shgroup_uniform_texture_ref_persistent(DRWShadingGroup *shgroup,
+                                                const char *name,
+                                                struct GPUTexture **tex);
 void DRW_shgroup_uniform_block(DRWShadingGroup *shgroup,
                                const char *name,
                                const struct GPUUniformBuffer *ubo);
 void DRW_shgroup_uniform_block_persistent(DRWShadingGroup *shgroup,
                                           const char *name,
                                           const struct GPUUniformBuffer *ubo);
-void DRW_shgroup_uniform_texture_ref(DRWShadingGroup *shgroup,
-                                     const char *name,
-                                     struct GPUTexture **tex);
+void DRW_shgroup_uniform_block_ref(DRWShadingGroup *shgroup,
+                                   const char *name,
+                                   struct GPUUniformBuffer **ubo);
+void DRW_shgroup_uniform_block_ref_persistent(DRWShadingGroup *shgroup,
+                                              const char *name,
+                                              struct GPUUniformBuffer **ubo);
 void DRW_shgroup_uniform_float(DRWShadingGroup *shgroup,
                                const char *name,
                                const float *value,
@@ -530,6 +543,8 @@ bool DRW_shgroup_is_empty(DRWShadingGroup *shgroup);
 
 /* Passes */
 DRWPass *DRW_pass_create(const char *name, DRWState state);
+DRWPass *DRW_pass_create_instance(const char *name, DRWPass *original, DRWState state);
+void DRW_pass_link(DRWPass *first, DRWPass *second);
 /* TODO Replace with passes inheritance. */
 void DRW_pass_state_set(DRWPass *pass, DRWState state);
 void DRW_pass_state_add(DRWPass *pass, DRWState state);
@@ -543,6 +558,8 @@ void DRW_pass_sort_shgroup_reverse(DRWPass *pass);
 bool DRW_pass_is_empty(DRWPass *pass);
 
 #define DRW_PASS_CREATE(pass, state) (pass = DRW_pass_create(#pass, state))
+#define DRW_PASS_INSTANCE_CREATE(pass, original, state) \
+  (pass = DRW_pass_create_instance(#pass, (original), state))
 
 /* Views */
 DRWView *DRW_view_create(const float viewmat[4][4],
diff --git a/source/blender/draw/intern/draw_common.h b/source/blender/draw/intern/draw_common.h
index f14cdc0dbde..656d72b2808 100644
--- a/source/blender/draw/intern/draw_common.h
+++ b/source/blender/draw/intern/draw_common.h
@@ -172,23 +172,11 @@ bool DRW_object_axis_orthogonal_to_view(Object *ob, int axis);
 
 /* This creates a shading group with display hairs.
  * The draw call is already added by this function, just add additional uniforms. */
-struct DRWShadingGroup *DRW_shgroup_hair_create(struct Object *object,
-                                                struct ParticleSystem *psys,
-                                                struct ModifierData *md,
-                                                struct DRWPass *hair_pass,
-                                                struct GPUShader *shader);
-
 struct DRWShadingGroup *DRW_shgroup_hair_create_sub(struct Object *object,
                                                     struct ParticleSystem *psys,
                                                     struct ModifierData *md,
                                                     struct DRWShadingGroup *shgrp);
 
-struct DRWShadingGroup *DRW_shgroup_material_hair_create(struct Object *object,
-                                                         struct ParticleSystem *psys,
-                                                         struct ModifierData *md,
-                                                         struct DRWPass *hair_pass,
-                                                         struct GPUMaterial *material);
-
 void DRW_hair_init(void);
 void DRW_hair_update(void);
 void DRW_hair_free(void);
diff --git a/source/blender/draw/intern/draw_hair.c b/source/blender/draw/intern/draw_hair.c
index 27de7cc1c7c..2fdaf0d5345 100644
--- a/source/blender/draw/intern/draw_hair.c
+++ b/source/blender/draw/intern/draw_hair.c
@@ -124,13 +124,10 @@ void DRW_hair_init(void)
   }
 }
 
-static DRWShadingGroup *drw_shgroup_create_hair_procedural_ex(Object *object,
-                                                              ParticleSystem *psys,
-                                                              ModifierData *md,
-                                                              DRWPass *hair_pass,
-                                                              DRWShadingGroup *shgrp_parent,
-                                                              struct GPUMaterial *gpu_mat,
-                                                              GPUShader *gpu_shader)
+DRWShadingGroup *DRW_shgroup_hair_create_sub(Object *object,
+                                             ParticleSystem *psys,
+                                             ModifierData *md,
+                                             DRWShadingGroup *shgrp_parent)
 {
   /* TODO(fclem): Pass the scene as parameter */
   const DRWContextState *draw_ctx = DRW_context_state_get();
@@ -154,24 +151,7 @@ static DRWShadingGroup *drw_shgroup_create_hair_procedural_ex(Object *object,
     need_ft_update = hair_ensure_procedural_data(object, &hair_cache, subdiv, thickness_res);
   }
 
-  DRWShadingGroup *shgrp;
-  if (shgrp_parent) {
-    shgrp = DRW_shgroup_create_sub(shgrp_parent);
-  }
-  else if (gpu_mat) {
-    shgrp = DRW_shgroup_material_create(gpu_mat, hair_pass);
-  }
-  else if (gpu_shader) {
-    shgrp = DRW_shgroup_create(gpu_shader, hair_pass);
-  }
-  else {
-    shgrp = NULL;
-    BLI_assert(0);
-  }
-
-  if (shgrp == NULL) {
-    return NULL;
-  }
+  DRWShadingGroup *shgrp = DRW_shgroup_create_sub(shgrp_parent);
 
   /* TODO optimize this. Only bind the ones GPUMaterial needs. */
   for (int i = 0; i < hair_cache->num_uv_layers; i++) {
@@ -284,29 +264,6 @@ static DRWShadingGroup *drw_shgroup_create_hair_procedural_ex(Object *object,
   return shgrp;
 }
 
-DRWShadingGroup *DRW_shgroup_hair_create(
-    Object *object, ParticleSystem *psys, ModifierData *md, DRWPass *hair_pass, GPUShader *shader)
-{
-  return drw_shgroup_create_hair_procedural_ex(object, psys, md, hair_pass, NULL, NULL, shader);
-}
-
-DRWShadingGroup *DRW_shgroup_hair_create_sub(Object *object,
-                                             ParticleSystem *psys,
-                                             ModifierData *md,
-                                             DRWShadingGroup *shgrp)
-{
-  return drw_shgroup_create_hair_procedural_ex(object, psys, md, NULL, shgrp, NULL, NULL);
-}
-
-DRWShadingGroup *DRW_shgroup_material_hair_create(Object *object,
-                                                  ParticleSystem *psys,
-                                                  ModifierData *md,
-                                                  DRWPass *hair_pass,
-                                                  struct GPUMaterial *material)
-{
-  return drw_shgroup_create_hair_procedural_ex(object, psys, md, hair_pass, NULL, material, NULL);
-}
-
 void DRW_hair_update(void)
 {
 #ifndef USE_TRANSFORM_FEEDBACK
diff --git a/source/blender/draw/intern/draw_manager.h b/source/blender/draw/intern/draw_manager.h
index 796abde140d..295e45c0fb3 100644
--- a/source/blender/draw/intern/draw_manager.h
+++ b/source/blender/draw/intern/draw_manager.h
@@ -279,8 +279,11 @@ typedef enum {
   DRW_UNIFORM_TEXTURE,
   DRW_UNIFORM_TEXTURE_PERSIST,
   DRW_UNIFORM_TEXTURE_REF,
+  DRW_UNIFORM_TEXTURE_REF_PERSIST,
   DRW_UNIFORM_BLOCK,
   DRW_UNIFORM_BLOCK_PERSIST,
+  DRW_UNIFORM_BLOCK_REF,
+  DRW_UNIFORM_BLOCK_REF_PERSIST,
   DRW_UNIFORM_TFEEDBACK_TARGET,
   /** Per drawcall uniforms/UBO */
   DRW_UNIFORM_BLOCK_OBMATS,
@@ -342,6 +345,13 @@ struct DRWPass {
     DRWShadingGroup *last;
   } shgroups;
 
+  /* Draw the shgroups of this pass instead.
+   * This avoid duplicating drawcalls/shgroups
+   * for similar passes. */
+  DRWPass *original;
+  /* Link list of additional passes to render. */
+  DRWPass *next;
+
   DRWResourceHandle handle;
   DRWState state;
   char name[MAX_PASS_NAME];
diff --git a/source/blender/draw/intern/draw_manager_data.c b/source/blender/draw/intern/draw_manager_data.c
index a5df60d66ce..8c89aa826e1 100644
--- a/source/blender/draw/intern/draw_manager_data.c
+++ b/source/blender/draw/intern/draw_manager_data.c
@@ -228,7 +228,11 @@ static void drw_shgroup_uniform(DRWShadingGroup *shgroup,
                                 int arraysize)
 {
   int location;
-  if (ELEM(type, DRW_UNIFORM_BLOCK, DRW_UNIFORM_BLOCK_PERSIST)) {
+  if (ELEM(type,
+           DRW_UNIFORM_BLOCK,
+           DRW_UNIFORM_BLOCK_PERSIST,
+           DRW_UNIFORM_BLOCK_REF,
+           DRW_UNIFORM_BLOCK_REF_PERSIST)) {
     location = GPU_shader_get_uniform_block(shgroup->shader, name);
   }
   else {
@@ -263,6 +267,22 @@ void DRW_shgroup_uniform_texture_persistent(DRWShadingGroup *shgroup,
   drw_shgroup_uniform(shgroup, name, DRW_UNIFORM_TEXTURE_PERSIST, tex, 0, 1);
 }
 
+void DRW_shgroup_uniform_texture_ref(DRWShadingGroup *shgroup, const char *name, GPUTexture **tex)
+{
+  BLI_assert(tex != NULL);
+  drw_shgroup_uniform(shgroup, name, DRW_UNIFORM_TEXTURE_REF, tex, 0, 1);
+}
+
+/* Same as DRW_shgroup_uniform_texture_ref but is guaranteed to be bound if shader does not change
+ * between shgrp. */
+void DRW_shgroup_uniform_texture_ref_persistent(DRWShadingGroup *shgroup,
+                                                const char *name,
+                                                GPUTexture **tex)
+{
+  BLI_assert(tex != NULL);
+  drw_shgroup_uniform(shgroup, name, DRW_UNIFORM_TEXTURE_REF_PERSIST, tex, 0, 1);
+}
+
 void DRW_shgroup_uniform_block(DRWShadingGroup *shgroup,
                                const char *name,
                                const GPUUniformBuffer *ubo)
@@ -281,9 +301,22 @@ void DRW_shgroup_uniform_block_persistent(DRWShadingGroup *shgroup,
   drw_shgroup_uniform(shgroup, name, DRW_UNIFORM_BLOCK_PERSIST, ubo, 0, 1);
 }
 
-void DRW_shgroup_uniform_texture_ref(DRWShadingGroup *shgroup, const char *name, GPUTexture **tex)
+void DRW_shgroup_uniform_block_ref(DRWShadingGroup *shgroup,
+                                   const char *name,
+                                   GPUUniformBuffer **ubo)
 {
-  drw_shgroup_uniform(shgroup, name, DRW_UNIFORM_TEXTURE_REF, tex, 0, 1);
+  BLI_assert(ubo != NULL);
+  drw_shgroup_uniform(shgroup, name, DRW_UNIFORM_BLOCK_REF, ubo, 0, 1);
+}
+
+/* Same as DRW_shgroup_uniform_block_ref but is guaranteed to be bound if shader does not change
+ * between shgrp. */
+void DRW_shgroup_uniform_block_ref_persistent(DRWShadingGroup *shgroup,
+                                              const char *name,
+                                              GPUUniformBuffer **ubo)
+{
+  BLI_assert(ubo != NULL);
+  drw_shgroup_uniform(shgroup, name, DRW_UNIFORM_BLOCK_REF_PERSIST, ubo, 0, 1);
 }
 
 void DRW_shgroup_uniform_bool(DRWShadingGroup *shgroup,
@@ -1290,15 +1323,14 @@ static void drw_shgroup_material_texture(DRWShadingGroup *grp,
                                          int textarget)
 {
   GPUTexture *gputex = GPU_texture_from_blender(tex->ima, tex->iuser, NULL, textarget);
-  DRW_shgroup_uniform_texture(grp, name, gputex);
+  DRW_shgroup_uniform_texture_persistent(grp, name, gputex);
 
   GPUTexture **gputex_ref = BLI_memblock_alloc(DST.vmempool->images);
   *gputex_ref = gputex;
   GPU_texture_ref(gputex);
 }
 
-static DRWShadingGroup *drw_shgroup_material_inputs(DRWShadingGroup *grp,
-                                                    struct GPUMaterial *material)
+void DRW_shgroup_add_material_resources(DRWShadingGroup *grp, struct GPUMaterial *material)
 {
   ListBase textures = GPU_material_textures(material);
 
@@ -1316,7 +1348,7 @@ static DRWShadingGroup *drw_shgroup_material_inputs(DRWShadingGroup *grp,
     }
     else if (tex->colorband) {
       /* Color Ramp */
-      DRW_shgroup_uniform_texture(grp, tex->sampler_name, *tex->colorband);
+      DRW_shgroup_uniform_texture_persistent(grp, tex->sampler_name, *tex->colorband);
     }
   }
 
@@ -1324,8 +1356,6 @@ static DRWShadingGroup *drw_shgroup_material_inputs(DRWShadingGroup *grp,
   if (ubo != NULL) {
     DRW_shgroup_uniform_block(grp, GPU_UBO_BLOCK_NAME, ubo);
   }
-
-  return grp;
 }
 
 GPUVertFormat *DRW_shgroup_instance_format_array(const DRWInstanceAttrFormat attrs[],
@@ -1350,7 +1380,7 @@ DRWShadingGroup *DRW_shgroup_material_create(struct GPUMaterial *material, DRWPa
 
   if (shgroup) {
     drw_shgroup_init(shgroup, GPU_pass_shader_get(gpupass));
-    drw_shgroup_material_inputs(shgroup, material);
+    DRW_shgroup_add_material_resources(shgroup, material);
   }
   return shgroup;
 }
@@ -1901,9 +1931,28 @@ DRWPass *DRW_pass_create(const char *name, DRWState state)
   pass->handle = DST.pass_handle;
   DRW_handle_increment(&DST.pass_handle);
 
+  pass->original = NULL;
+  pass->next = NULL;
+
   return pass;
 }
 
+DRWPass *DRW_pass_create_instance(const char *name, DRWPass *original, DRWState state)
+{
+  DRWPass *pass = DRW_pass_create(name, state);
+  pass->original = original;
+
+  return pass;
+}
+
+/* Link two passes so that they are both rendered if the first one is being drawn. */
+void DRW_pass_link(DRWPass *first, DRWPass *second)
+{
+  BLI_assert(first != second);
+  BLI_assert(first->next == NULL);
+  first->next = second;
+}
+
 bool DRW_pass_is_empty(DRWPass *pass)
 {
   LISTBASE_FOREACH (DRWShadingGroup *, shgroup, &pass->shgroups) {
diff --git a/source/blender/draw/intern/draw_manager_exec.c b/source/blender/draw/intern/draw_manager_exec.c
index cff4df89219..48e0115741f 100644
--- a/source/blender/draw/intern/draw_manager_exec.c
+++ b/source/blender/draw/intern/draw_manager_exec.c
@@ -964,6 +964,12 @@ static void draw_update_uniforms(DRWShadingGroup *shgroup,
           bind_texture(tex, BIND_TEMP);
           GPU_shader_uniform_texture(shgroup->shader, uni->location, tex);
           break;
+        case DRW_UNIFORM_TEXTURE_REF_PERSIST:
+          tex = *((GPUTexture **)uni->pvalue);
+          BLI_assert(tex);
+          bind_texture(tex, BIND_PERSIST);
+          GPU_shader_uniform_texture(shgroup->shader, uni->location, tex);
+          break;
         case DRW_UNIFORM_BLOCK:
           ubo = (GPUUniformBuffer *)uni->pvalue;
           bind_ubo(ubo, BIND_TEMP);
@@ -974,6 +980,16 @@ static void draw_update_uniforms(DRWShadingGroup *shgroup,
           bind_ubo(ubo, BIND_PERSIST);
           GPU_shader_uniform_buffer(shgroup->shader, uni->location, ubo);
           break;
+        case DRW_UNIFORM_BLOCK_REF:
+          ubo = *((GPUUniformBuffer **)uni->pvalue);
+          bind_ubo(ubo, BIND_TEMP);
+          GPU_shader_uniform_buffer(shgroup->shader, uni->location, ubo);
+          break;
+        case DRW_UNIFORM_BLOCK_REF_PERSIST:
+          ubo = *((GPUUniformBuffer **)uni->pvalue);
+          bind_ubo(ubo, BIND_PERSIST);
+          GPU_shader_uniform_buffer(shgroup->shader, uni->location, ubo);
+          break;
         case DRW_UNIFORM_BLOCK_OBMATS:
           state->obmats_loc = uni->location;
           ubo = DST.vmempool->matrices_ubo[0];
@@ -1419,6 +1435,11 @@ static void drw_draw_pass_ex(DRWPass *pass,
                              DRWShadingGroup *start_group,
                              DRWShadingGroup *end_group)
 {
+  if (pass->original) {
+    start_group = pass->original->shgroups.first;
+    end_group = pass->original->shgroups.last;
+  }
+
   if (start_group == NULL) {
     return;
   }
@@ -1504,7 +1525,9 @@ static void drw_draw_pass_ex(DRWPass *pass,
 
 void DRW_draw_pass(DRWPass *pass)
 {
-  drw_draw_pass_ex(pass, pass->shgroups.first, pass->shgroups.last);
+  for (; pass; pass = pass->next) {
+    drw_draw_pass_ex(pass, pass->shgroups.first, pass->shgroups.last);
+  }
 }
 
 /* Draw only a subset of shgroups. Used in special situations as grease pencil strokes */
diff --git a/source/blender/draw/intern/draw_manager_shader.c b/source/blender/draw/intern/draw_manager_shader.c
index 0d6527421d0..087bf7cff9a 100644
--- a/source/blender/draw/intern/draw_manager_shader.c
+++ b/source/blender/draw/intern/draw_manager_shader.c
@@ -398,6 +398,7 @@ GPUMaterial *DRW_shader_find_from_material(Material *ma,
 
 GPUMaterial *DRW_shader_create_from_world(struct Scene *scene,
                                           World *wo,
+                                          struct bNodeTree *ntree,
                                           const void *engine_type,
                                           const int options,
                                           const bool is_volume_shader,
@@ -408,7 +409,7 @@ GPUMaterial *DRW_shader_create_from_world(struct Scene *scene,
                                           bool deferred)
 {
   GPUMaterial *mat = NULL;
-  if (DRW_state_is_image_render()) {
+  if (DRW_state_is_image_render() || !deferred) {
     mat = GPU_material_from_nodetree_find(&wo->gpumaterial, engine_type, options);
   }
 
@@ -416,7 +417,7 @@ GPUMaterial *DRW_shader_create_from_world(struct Scene *scene,
     scene = (Scene *)DEG_get_original_id(&DST.draw_ctx.scene->id);
     mat = GPU_material_from_nodetree(scene,
                                      NULL,
-                                     wo->nodetree,
+                                     ntree,
                                      &wo->gpumaterial,
                                      engine_type,
                                      options,
@@ -437,6 +438,7 @@ GPUMaterial *DRW_shader_create_from_world(struct Scene *scene,
 
 GPUMaterial *DRW_shader_create_from_material(struct Scene *scene,
                                              Material *ma,
+                                             struct bNodeTree *ntree,
                                              const void *engine_type,
                                              const int options,
                                              const bool is_volume_shader,
@@ -447,7 +449,7 @@ GPUMaterial *DRW_shader_create_from_material(struct Scene *scene,
                                              bool deferred)
 {
   GPUMaterial *mat = NULL;
-  if (DRW_state_is_image_render()) {
+  if (DRW_state_is_image_render() || !deferred) {
     mat = GPU_material_from_nodetree_find(&ma->gpumaterial, engine_type, options);
   }
 
@@ -455,7 +457,7 @@ GPUMaterial *DRW_shader_create_from_material(struct Scene *scene,
     scene = (Scene *)DEG_get_original_id(&DST.draw_ctx.scene->id);
     mat = GPU_material_from_nodetree(scene,
                                      ma,
-                                     ma->nodetree,
+                                     ntree,
                                      &ma->gpumaterial,
                                      engine_type,
                                      options,