GPU: Make nodetree GLSL Codegen render engine agnostic

This commit removes all EEVEE specific code from the `gpu_shader_material*.glsl`
files. It defines a clear interface to evaluate the closure nodes leaving
more flexibility to the render engine.

Some of the long standing workaround are fixed:
- bump mapping support is no longer duplicating a lot of node and is instead
  compiled into a function call.
- bump rewiring to Normal socket is no longer needed as we now use a global
  `g_data.N` for that.


Closure sampling with upstread weight eval is now supported if the engine needs
it.

This also makes all the material GLSL sources use `GPUSource` for better
debugging experience. The `GPUFunction` parsing now happens in `GPUSource`
creation.

The whole `GPUCodegen` now uses the `ShaderCreateInfo` and is object type
agnostic. Is has also been rewritten in C++.

This patch changes a view behavior for EEVEE:
- Mix shader node factor imput is now clamped.
- Tangent Vector displacement behavior is now matching cycles.
- The chosen BSDF used for SSR might change.
- Hair shading may have very small changes on very large hairs when using hair
  polygon stripes.
- ShaderToRGB node will remove any SSR and SSS form a shader.
- SSS radius input now is no longer a scaling factor but defines an average
  radius. The SSS kernel "shape" (radii) are still defined by the socket default
  values.

Appart from the listed changes no other regressions are expected.

1 files changed, 194 insertions, 145 deletions

diff --git a/source/blender/gpu/intern/gpu_material.c b/source/blender/gpu/intern/gpu_material.c
index 49732240125..711a3943a25 100644
--- a/source/blender/gpu/intern/gpu_material.c
+++ b/source/blender/gpu/intern/gpu_material.c
@@ -16,7 +16,6 @@
 #include "DNA_scene_types.h"
 #include "DNA_world_types.h"
 
-#include "BLI_ghash.h"
 #include "BLI_listbase.h"
 #include "BLI_math.h"
 #include "BLI_string.h"
@@ -27,6 +26,7 @@
 #include "BKE_material.h"
 #include "BKE_node.h"
 #include "BKE_scene.h"
+#include "BKE_world.h"
 
 #include "NOD_shader.h"
 
@@ -49,56 +49,50 @@ typedef struct GPUColorBandBuilder {
 } GPUColorBandBuilder;
 
 struct GPUMaterial {
-  Scene *scene; /* DEPRECATED was only useful for lights. */
-  Material *ma;
-
+  /* Contains GPUShader and source code for deferred compilation.
+   * Can be shared between similar material (i.e: sharing same nodetree topology). */
+  GPUPass *pass;
+  /** UBOs for this material parameters. */
+  GPUUniformBuf *ubo;
+  /** Compilation status. Do not use if shader is not GPU_MAT_SUCCESS. */
   eGPUMaterialStatus status;
-
-  const void *engine_type; /* attached engine type */
-  int options;             /* to identify shader variations (shadow, probe, world background...) */
-  bool is_volume_shader;   /* is volumetric shader */
-
-  /* Nodes */
+  /** Some flags about the nodetree & the needed resources. */
+  eGPUMaterialFlag flag;
+  /* Identify shader variations (shadow, probe, world background...).
+   * Should be unique even across render engines. */
+  uint64_t uuid;
+  /* Number of generated function. */
+  int generated_function_len;
+  /** Object type for attribute fetching. */
+  bool is_volume_shader;
+
+  /** DEPRECATED Currently only used for deferred compilation. */
+  Scene *scene;
+  /** Source material, might be null. */
+  Material *ma;
+  /** 1D Texture array containing all color bands. */
+  GPUTexture *coba_tex;
+  /** Builder for coba_tex. */
+  GPUColorBandBuilder *coba_builder;
+  /* Low level node graph(s). Also contains resources needed by the material. */
   GPUNodeGraph graph;
 
-  /* for binding the material */
-  GPUPass *pass;
-
-  /* XXX: Should be in Material. But it depends on the output node
-   * used and since the output selection is different for GPUMaterial...
-   */
-  bool has_volume_output;
+  /** DEPRECATED: To remove. */
   bool has_surface_output;
-
-  /* Only used by Eevee to know which BSDF are used. */
-  eGPUMatFlag flag;
-
-  /* Used by 2.8 pipeline */
-  GPUUniformBuf *ubo; /* UBOs for shader uniforms. */
-
-  /* Eevee SSS */
+  bool has_volume_output;
+  /** DEPRECATED: To remove. */
   GPUUniformBuf *sss_profile;  /* UBO containing SSS profile. */
   GPUTexture *sss_tex_profile; /* Texture containing SSS profile. */
-  float sss_enabled;
+  bool sss_enabled;
   float sss_radii[3];
   int sss_samples;
   bool sss_dirty;
 
-  GPUTexture *coba_tex; /* 1D Texture array containing all color bands. */
-  GPUColorBandBuilder *coba_builder;
-
-  GSet *used_libraries;
-
 #ifndef NDEBUG
   char name[64];
 #endif
 };
 
-enum {
-  GPU_USE_SURFACE_OUTPUT = (1 << 0),
-  GPU_USE_VOLUME_OUTPUT = (1 << 1),
-};
-
 /* Functions */
 
 GPUTexture **gpu_material_ramp_texture_row_set(GPUMaterial *mat,
@@ -159,17 +153,15 @@ static void gpu_material_free_single(GPUMaterial *material)
   if (material->ubo != NULL) {
     GPU_uniformbuf_free(material->ubo);
   }
-  if (material->sss_tex_profile != NULL) {
-    GPU_texture_free(material->sss_tex_profile);
+  if (material->coba_tex != NULL) {
+    GPU_texture_free(material->coba_tex);
   }
   if (material->sss_profile != NULL) {
     GPU_uniformbuf_free(material->sss_profile);
   }
-  if (material->coba_tex != NULL) {
-    GPU_texture_free(material->coba_tex);
+  if (material->sss_tex_profile != NULL) {
+    GPU_texture_free(material->sss_tex_profile);
   }
-
-  BLI_gset_free(material->used_libraries, NULL);
 }
 
 void GPU_material_free(ListBase *gpumaterial)
@@ -217,17 +209,40 @@ void GPU_material_uniform_buffer_create(GPUMaterial *material, ListBase *inputs)
   material->ubo = GPU_uniformbuf_create_from_list(inputs, name);
 }
 
+ListBase GPU_material_attributes(GPUMaterial *material)
+{
+  return material->graph.attributes;
+}
+
+ListBase GPU_material_textures(GPUMaterial *material)
+{
+  return material->graph.textures;
+}
+
+ListBase GPU_material_volume_grids(GPUMaterial *material)
+{
+  return material->graph.volume_grids;
+}
+
+GPUUniformAttrList *GPU_material_uniform_attributes(GPUMaterial *material)
+{
+  GPUUniformAttrList *attrs = &material->graph.uniform_attrs;
+  return attrs->count > 0 ? attrs : NULL;
+}
+
+#if 1 /* End of life code. */
 /* Eevee Subsurface scattering. */
 /* Based on Separable SSS. by Jorge Jimenez and Diego Gutierrez */
 
-#define SSS_SAMPLES 65
-#define SSS_EXPONENT 2.0f /* Importance sampling exponent */
+#  define SSS_SAMPLES 65
+#  define SSS_EXPONENT 2.0f /* Importance sampling exponent */
 
 typedef struct GPUSssKernelData {
   float kernel[SSS_SAMPLES][4];
   float param[3], max_radius;
+  float avg_inv_radius;
   int samples;
-  int pad[3];
+  int pad[2];
 } GPUSssKernelData;
 
 BLI_STATIC_ASSERT_ALIGN(GPUSssKernelData, 16)
@@ -243,8 +258,8 @@ static void sss_calculate_offsets(GPUSssKernelData *kd, int count, float exponen
   }
 }
 
-#define BURLEY_TRUNCATE 16.0f
-#define BURLEY_TRUNCATE_CDF 0.9963790093708328f  // cdf(BURLEY_TRUNCATE)
+#  define BURLEY_TRUNCATE 16.0f
+#  define BURLEY_TRUNCATE_CDF 0.9963790093708328f  // cdf(BURLEY_TRUNCATE)
 static float burley_profile(float r, float d)
 {
   float exp_r_3_d = expf(-r / (3.0f * d));
@@ -259,7 +274,7 @@ static float eval_profile(float r, float param)
 }
 
 /* Resolution for each sample of the precomputed kernel profile */
-#define INTEGRAL_RESOLUTION 32
+#  define INTEGRAL_RESOLUTION 32
 static float eval_integral(float x0, float x1, float param)
 {
   const float range = x1 - x0;
@@ -274,7 +289,7 @@ static float eval_integral(float x0, float x1, float param)
 
   return integral;
 }
-#undef INTEGRAL_RESOLUTION
+#  undef INTEGRAL_RESOLUTION
 
 static void compute_sss_kernel(GPUSssKernelData *kd, const float radii[3], int sample_len)
 {
@@ -284,6 +299,8 @@ static void compute_sss_kernel(GPUSssKernelData *kd, const float radii[3], int s
   rad[1] = MAX2(radii[1], 1e-15f);
   rad[2] = MAX2(radii[2], 1e-15f);
 
+  kd->avg_inv_radius = 3.0f / (rad[0] + rad[1] + rad[2]);
+
   /* Christensen-Burley fitting */
   float l[3], d[3];
 
@@ -358,7 +375,7 @@ static void compute_sss_kernel(GPUSssKernelData *kd, const float radii[3], int s
   kd->samples = sample_len;
 }
 
-#define INTEGRAL_RESOLUTION 512
+#  define INTEGRAL_RESOLUTION 512
 static void compute_sss_translucence_kernel(const GPUSssKernelData *kd,
                                             int resolution,
                                             float **output)
@@ -417,10 +434,14 @@ static void compute_sss_translucence_kernel(const GPUSssKernelData *kd,
   mul_v3_fl(texels[resolution - 3], 0.5f);
   mul_v3_fl(texels[resolution - 4], 0.75f);
 }
-#undef INTEGRAL_RESOLUTION
+#  undef INTEGRAL_RESOLUTION
 
-void GPU_material_sss_profile_create(GPUMaterial *material, float radii[3])
+bool GPU_material_sss_profile_create(GPUMaterial *material, float radii[3])
 {
+  /* Enable only once. */
+  if (material->sss_enabled) {
+    return false;
+  }
   copy_v3_v3(material->sss_radii, radii);
   material->sss_dirty = true;
   material->sss_enabled = true;
@@ -429,6 +450,7 @@ void GPU_material_sss_profile_create(GPUMaterial *material, float radii[3])
   if (material->sss_profile == NULL) {
     material->sss_profile = GPU_uniformbuf_create(sizeof(GPUSssKernelData));
   }
+  return true;
 }
 
 struct GPUUniformBuf *GPU_material_sss_profile_get(GPUMaterial *material,
@@ -475,34 +497,37 @@ struct GPUUniformBuf *GPU_material_create_sss_profile_ubo(void)
   return GPU_uniformbuf_create(sizeof(GPUSssKernelData));
 }
 
-#undef SSS_EXPONENT
-#undef SSS_SAMPLES
-
-ListBase GPU_material_attributes(GPUMaterial *material)
-{
-  return material->graph.attributes;
-}
+#  undef SSS_EXPONENT
+#  undef SSS_SAMPLES
+#endif
 
-ListBase GPU_material_textures(GPUMaterial *material)
+void GPU_material_output_surface(GPUMaterial *material, GPUNodeLink *link)
 {
-  return material->graph.textures;
+  if (!material->graph.outlink_surface) {
+    material->graph.outlink_surface = link;
+    material->has_surface_output = true;
+  }
 }
 
-ListBase GPU_material_volume_grids(GPUMaterial *material)
+void GPU_material_output_volume(GPUMaterial *material, GPUNodeLink *link)
 {
-  return material->graph.volume_grids;
+  if (!material->graph.outlink_volume) {
+    material->graph.outlink_volume = link;
+    material->has_volume_output = true;
+  }
 }
 
-GPUUniformAttrList *GPU_material_uniform_attributes(GPUMaterial *material)
+void GPU_material_output_displacement(GPUMaterial *material, GPUNodeLink *link)
 {
-  GPUUniformAttrList *attrs = &material->graph.uniform_attrs;
-  return attrs->count > 0 ? attrs : NULL;
+  if (!material->graph.outlink_displacement) {
+    material->graph.outlink_displacement = link;
+  }
 }
 
-void GPU_material_output_link(GPUMaterial *material, GPUNodeLink *link)
+void GPU_material_output_thickness(GPUMaterial *material, GPUNodeLink *link)
 {
-  if (!material->graph.outlink) {
-    material->graph.outlink = link;
+  if (!material->graph.outlink_thickness) {
+    material->graph.outlink_thickness = link;
   }
 }
 
@@ -514,23 +539,71 @@ void GPU_material_add_output_link_aov(GPUMaterial *material, GPUNodeLink *link,
   BLI_addtail(&material->graph.outlink_aovs, aov_link);
 }
 
+char *GPU_material_split_sub_function(GPUMaterial *material,
+                                      eGPUType return_type,
+                                      GPUNodeLink **link)
+{
+  /* Force cast to return type. */
+  switch (return_type) {
+    case GPU_FLOAT:
+      GPU_link(material, "set_value", *link, link);
+      break;
+    case GPU_VEC3:
+      GPU_link(material, "set_rgb", *link, link);
+      break;
+    case GPU_VEC4:
+      GPU_link(material, "set_rgba", *link, link);
+      break;
+    default:
+      BLI_assert(0);
+      break;
+  }
+
+  GPUNodeGraphFunctionLink *func_link = MEM_callocN(sizeof(GPUNodeGraphFunctionLink), __func__);
+  func_link->outlink = *link;
+  SNPRINTF(func_link->name, "ntree_fn%d", material->generated_function_len++);
+  BLI_addtail(&material->graph.material_functions, func_link);
+
+  /* Set value to break the link with the main graph. */
+  switch (return_type) {
+    case GPU_FLOAT:
+      GPU_link(material, "set_value_one", link);
+      break;
+    case GPU_VEC3:
+      GPU_link(material, "set_rgb_one", link);
+      break;
+    case GPU_VEC4:
+      GPU_link(material, "set_rgba_one", link);
+      break;
+    default:
+      BLI_assert(0);
+      break;
+  }
+  return func_link->name;
+}
+
 GPUNodeGraph *gpu_material_node_graph(GPUMaterial *material)
 {
   return &material->graph;
 }
 
-GSet *gpu_material_used_libraries(GPUMaterial *material)
+eGPUMaterialStatus GPU_material_status(GPUMaterial *mat)
 {
-  return material->used_libraries;
+  return mat->status;
 }
 
-eGPUMaterialStatus GPU_material_status(GPUMaterial *mat)
+void GPU_material_status_set(GPUMaterial *mat, eGPUMaterialStatus status)
 {
-  return mat->status;
+  mat->status = status;
 }
 
 /* Code generation */
 
+bool GPU_material_is_volume_shader(GPUMaterial *mat)
+{
+  return mat->is_volume_shader;
+}
+
 bool GPU_material_has_surface_output(GPUMaterial *mat)
 {
   return mat->has_surface_output;
@@ -541,100 +614,79 @@ bool GPU_material_has_volume_output(GPUMaterial *mat)
   return mat->has_volume_output;
 }
 
-bool GPU_material_is_volume_shader(GPUMaterial *mat)
+void GPU_material_flag_set(GPUMaterial *mat, eGPUMaterialFlag flag)
 {
-  return mat->is_volume_shader;
+  mat->flag |= flag;
 }
 
-void GPU_material_flag_set(GPUMaterial *mat, eGPUMatFlag flag)
+bool GPU_material_flag_get(const GPUMaterial *mat, eGPUMaterialFlag flag)
 {
-  mat->flag |= flag;
+  return (mat->flag & flag) != 0;
 }
 
-bool GPU_material_flag_get(GPUMaterial *mat, eGPUMatFlag flag)
+eGPUMaterialFlag GPU_material_flag(const GPUMaterial *mat)
 {
-  return (mat->flag & flag) != 0;
+  return mat->flag;
 }
 
-GPUMaterial *GPU_material_from_nodetree_find(ListBase *gpumaterials,
-                                             const void *engine_type,
-                                             int options)
+/* Note: Consumes the flags. */
+bool GPU_material_recalc_flag_get(GPUMaterial *mat)
 {
-  LISTBASE_FOREACH (LinkData *, link, gpumaterials) {
-    GPUMaterial *current_material = (GPUMaterial *)link->data;
-    if (current_material->engine_type == engine_type && current_material->options == options) {
-      return current_material;
-    }
-  }
+  bool updated = (mat->flag & GPU_MATFLAG_UPDATED) != 0;
+  mat->flag &= ~GPU_MATFLAG_UPDATED;
+  return updated;
+}
 
-  return NULL;
+uint64_t GPU_material_uuid_get(GPUMaterial *mat)
+{
+  return mat->uuid;
 }
 
 GPUMaterial *GPU_material_from_nodetree(Scene *scene,
-                                        struct Material *ma,
-                                        struct bNodeTree *ntree,
+                                        Material *ma,
+                                        bNodeTree *ntree,
                                         ListBase *gpumaterials,
-                                        const void *engine_type,
-                                        const int options,
-                                        const bool is_volume_shader,
-                                        const char *vert_code,
-                                        const char *geom_code,
-                                        const char *frag_lib,
-                                        const char *defines,
                                         const char *name,
-                                        GPUMaterialEvalCallbackFn callback)
+                                        uint64_t shader_uuid,
+                                        bool is_volume_shader,
+                                        bool is_lookdev,
+                                        GPUCodegenCallbackFn callback,
+                                        void *thunk)
 {
-  LinkData *link;
-  bool has_volume_output, has_surface_output;
-
-  /* Caller must re-use materials. */
-  BLI_assert(GPU_material_from_nodetree_find(gpumaterials, engine_type, options) == NULL);
-
-  /* HACK: Eevee assume this to create #GHash keys. */
-  BLI_assert(sizeof(GPUPass) > 16);
+  /* Search if this material is not already compiled. */
+  LISTBASE_FOREACH (LinkData *, link, gpumaterials) {
+    GPUMaterial *mat = (GPUMaterial *)link->data;
+    if (mat->uuid == shader_uuid) {
+      return mat;
+    }
+  }
 
-  /* allocate material */
   GPUMaterial *mat = MEM_callocN(sizeof(GPUMaterial), "GPUMaterial");
   mat->ma = ma;
   mat->scene = scene;
-  mat->engine_type = engine_type;
-  mat->options = options;
+  mat->uuid = shader_uuid;
+  mat->flag = GPU_MATFLAG_UPDATED;
   mat->is_volume_shader = is_volume_shader;
+  mat->graph.used_libraries = BLI_gset_new(
+      BLI_ghashutil_ptrhash, BLI_ghashutil_ptrcmp, "GPUNodeGraph.used_libraries");
 #ifndef NDEBUG
   BLI_snprintf(mat->name, sizeof(mat->name), "%s", name);
 #else
   UNUSED_VARS(name);
 #endif
+  if (is_lookdev) {
+    mat->flag |= GPU_MATFLAG_LOOKDEV_HACK;
+  }
 
-  mat->used_libraries = BLI_gset_new(
-      BLI_ghashutil_ptrhash, BLI_ghashutil_ptrcmp, "GPUMaterial.used_libraries");
-
-  /* localize tree to create links for reroute and mute */
+  /* Localize tree to create links for reroute and mute. */
   bNodeTree *localtree = ntreeLocalize(ntree);
-  ntreeGPUMaterialNodes(localtree, mat, &has_surface_output, &has_volume_output);
+  ntreeGPUMaterialNodes(localtree, mat);
 
   gpu_material_ramp_texture_build(mat);
 
-  mat->has_surface_output = has_surface_output;
-  mat->has_volume_output = has_volume_output;
-
-  if (mat->graph.outlink) {
-    if (callback) {
-      callback(mat, options, &vert_code, &geom_code, &frag_lib, &defines);
-    }
-    /* HACK: this is only for eevee. We add the define here after the nodetree evaluation. */
-    if (GPU_material_flag_get(mat, GPU_MATFLAG_SSS)) {
-      defines = BLI_string_joinN(defines,
-                                 "#ifndef USE_ALPHA_BLEND\n"
-                                 "#  define USE_SSS\n"
-                                 "#endif\n");
-    }
+  {
     /* Create source code and search pass cache for an already compiled version. */
-    mat->pass = GPU_generate_pass(mat, &mat->graph, vert_code, geom_code, frag_lib, defines);
-
-    if (GPU_material_flag_get(mat, GPU_MATFLAG_SSS)) {
-      MEM_freeN((char *)defines);
-    }
+    mat->pass = GPU_generate_pass(mat, &mat->graph, callback, thunk);
 
     if (mat->pass == NULL) {
       /* We had a cache hit and the shader has already failed to compile. */
@@ -649,26 +701,20 @@ GPUMaterial *GPU_material_from_nodetree(Scene *scene,
         gpu_node_graph_free_nodes(&mat->graph);
       }
       else {
-        mat->status = GPU_MAT_QUEUED;
+        mat->status = GPU_MAT_CREATED;
       }
     }
   }
-  else {
-    mat->status = GPU_MAT_FAILED;
-    gpu_node_graph_free(&mat->graph);
-  }
 
-  /* Only free after GPU_pass_shader_get where GPUUniformBuf
-   * read data from the local tree. */
+  /* Only free after GPU_pass_shader_get where GPUUniformBuf read data from the local tree. */
   ntreeFreeLocalTree(localtree);
   BLI_assert(!localtree->id.py_instance); /* Or call #BKE_libblock_free_data_py. */
   MEM_freeN(localtree);
 
-  /* note that even if building the shader fails in some way, we still keep
+  /* Note that even if building the shader fails in some way, we still keep
    * it to avoid trying to compile again and again, and simply do not use
-   * the actual shader on drawing */
-
-  link = MEM_callocN(sizeof(LinkData), "GPUMaterialLink");
+   * the actual shader on drawing. */
+  LinkData *link = MEM_callocN(sizeof(LinkData), "GPUMaterialLink");
   link->data = mat;
   BLI_addtail(gpumaterials, link);
 
@@ -690,6 +736,8 @@ void GPU_material_compile(GPUMaterial *mat)
   success = GPU_pass_compile(mat->pass, __func__);
 #endif
 
+  mat->flag |= GPU_MATFLAG_UPDATED;
+
   if (success) {
     GPUShader *sh = GPU_pass_shader_get(mat->pass);
     if (sh != NULL) {
@@ -715,5 +763,6 @@ void GPU_materials_free(Main *bmain)
     GPU_material_free(&wo->gpumaterial);
   }
 
+  // BKE_world_defaults_free_gpu();
   BKE_material_defaults_free_gpu();
 }