Cycles: merge of cycles-x branch, a major update to the renderer

This includes much improved GPU rendering performance, viewport interactivity,
new shadow catcher, revamped sampling settings, subsurface scattering anisotropy,
new GPU volume sampling, improved PMJ sampling pattern, and more.

Some features have also been removed or changed, breaking backwards compatibility.
Including the removal of the OpenCL backend, for which alternatives are under
development.

Release notes and code docs:
https://wiki.blender.org/wiki/Reference/Release_Notes/3.0/Cycles
https://wiki.blender.org/wiki/Source/Render/Cycles

Credits:
* Sergey Sharybin
* Brecht Van Lommel
* Patrick Mours (OptiX backend)
* Christophe Hery (subsurface scattering anisotropy)
* William Leeson (PMJ sampling pattern)
* Alaska (various fixes and tweaks)
* Thomas Dinges (various fixes)

For the full commit history, see the cycles-x branch. This squashes together
all the changes since intermediate changes would often fail building or tests.

Ref T87839, T87837, T87836
Fixes T90734, T89353, T80267, T80267, T77185, T69800

11 files changed, 295 insertions, 19 deletions

diff --git a/source/blender/draw/DRW_engine.h b/source/blender/draw/DRW_engine.h
index a125a13eaf9..2e25211ea62 100644
--- a/source/blender/draw/DRW_engine.h
+++ b/source/blender/draw/DRW_engine.h
@@ -176,6 +176,9 @@ void DRW_deferred_shader_remove(struct GPUMaterial *mat);
 struct DrawDataList *DRW_drawdatalist_from_id(struct ID *id);
 void DRW_drawdata_free(struct ID *id);
 
+bool DRW_opengl_context_release(void);
+void DRW_opengl_context_activate(bool test);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/blender/draw/engines/eevee/eevee_cryptomatte.c b/source/blender/draw/engines/eevee/eevee_cryptomatte.c
index 76a1b561972..49780abc6f4 100644
--- a/source/blender/draw/engines/eevee/eevee_cryptomatte.c
+++ b/source/blender/draw/engines/eevee/eevee_cryptomatte.c
@@ -139,8 +139,6 @@ void EEVEE_cryptomatte_renderpasses_init(EEVEE_Data *vedata)
     g_data->cryptomatte_session = session;
 
     g_data->render_passes |= EEVEE_RENDER_PASS_CRYPTOMATTE | EEVEE_RENDER_PASS_VOLUME_LIGHT;
-    g_data->cryptomatte_accurate_mode = (view_layer->cryptomatte_flag &
-                                         VIEW_LAYER_CRYPTOMATTE_ACCURATE) != 0;
   }
 }
 
@@ -405,7 +403,6 @@ void EEVEE_cryptomatte_output_accumulate(EEVEE_ViewLayerData *UNUSED(sldata), EE
 {
   EEVEE_FramebufferList *fbl = vedata->fbl;
   EEVEE_StorageList *stl = vedata->stl;
-  EEVEE_PrivateData *g_data = stl->g_data;
   EEVEE_EffectsInfo *effects = stl->effects;
   EEVEE_PassList *psl = vedata->psl;
   const DRWContextState *draw_ctx = DRW_context_state_get();
@@ -413,10 +410,9 @@ void EEVEE_cryptomatte_output_accumulate(EEVEE_ViewLayerData *UNUSED(sldata), EE
   const int cryptomatte_levels = view_layer->cryptomatte_levels;
   const int current_sample = effects->taa_current_sample;
 
-  /* In accurate mode all render samples are evaluated. In inaccurate mode this is limited to the
-   * number of cryptomatte levels. This will reduce the overhead of downloading the GPU buffer and
-   * integrating it into the accum buffer. */
-  if (g_data->cryptomatte_accurate_mode || current_sample < cryptomatte_levels) {
+  /* Render samples used by cryptomatte are limited to the number of cryptomatte levels. This will
+   * reduce the overhead of downloading the GPU buffer and integrating it into the accum buffer. */
+  if (current_sample < cryptomatte_levels) {
     static float clear_color[4] = {0.0};
     GPU_framebuffer_bind(fbl->cryptomatte_fb);
     GPU_framebuffer_clear_color(fbl->cryptomatte_fb, clear_color);
diff --git a/source/blender/draw/engines/eevee/eevee_engine.c b/source/blender/draw/engines/eevee/eevee_engine.c
index 6a66e8b1a58..f8e1cc9c923 100644
--- a/source/blender/draw/engines/eevee/eevee_engine.c
+++ b/source/blender/draw/engines/eevee/eevee_engine.c
@@ -648,6 +648,8 @@ RenderEngineType DRW_engine_viewport_eevee_type = {
     NULL,
     NULL,
     NULL,
+    NULL,
+    NULL,
     &EEVEE_render_update_passes,
     &draw_engine_eevee_type,
     {NULL, NULL, NULL},
diff --git a/source/blender/draw/engines/eevee/eevee_private.h b/source/blender/draw/engines/eevee/eevee_private.h
index f51b4fa0127..eae5d161cc3 100644
--- a/source/blender/draw/engines/eevee/eevee_private.h
+++ b/source/blender/draw/engines/eevee/eevee_private.h
@@ -1042,7 +1042,6 @@ typedef struct EEVEE_PrivateData {
   int aov_hash;
   int num_aovs_used;
   struct CryptomatteSession *cryptomatte_session;
-  bool cryptomatte_accurate_mode;
   EEVEE_CryptomatteSample *cryptomatte_accum_buffer;
   float *cryptomatte_download_buffer;
 
diff --git a/source/blender/draw/engines/external/external_engine.c b/source/blender/draw/engines/external/external_engine.c
index 89ee3f1b293..cc548a53a8e 100644
--- a/source/blender/draw/engines/external/external_engine.c
+++ b/source/blender/draw/engines/external/external_engine.c
@@ -32,13 +32,19 @@
 #include "BKE_object.h"
 #include "BKE_particle.h"
 
+#include "ED_image.h"
 #include "ED_screen.h"
 
+#include "GPU_batch.h"
+#include "GPU_debug.h"
 #include "GPU_matrix.h"
 #include "GPU_shader.h"
 #include "GPU_state.h"
 #include "GPU_viewport.h"
 
+#include "RE_engine.h"
+#include "RE_pipeline.h"
+
 #include "external_engine.h" /* own include */
 
 /* Shaders */
@@ -137,6 +143,22 @@ static void external_engine_init(void *vedata)
   }
 }
 
+/* Add shading group call which will take care of writing to the depth buffer, so that the
+ * alpha-under overlay will happen for the render buffer. */
+static void external_cache_image_add(DRWShadingGroup *grp)
+{
+  float obmat[4][4];
+  unit_m4(obmat);
+  scale_m4_fl(obmat, 0.5f);
+
+  /* NOTE: Use the same Z-depth value as in the regular image drawing engine. */
+  translate_m4(obmat, 1.0f, 1.0f, 0.75f);
+
+  GPUBatch *geom = DRW_cache_quad_get();
+
+  DRW_shgroup_call_obmat(grp, geom, obmat);
+}
+
 static void external_cache_init(void *vedata)
 {
   EXTERNAL_PassList *psl = ((EXTERNAL_Data *)vedata)->psl;
@@ -162,14 +184,33 @@ static void external_cache_init(void *vedata)
     stl->g_data->depth_shgrp = DRW_shgroup_create(e_data.depth_sh, psl->depth_pass);
   }
 
-  /* Do not draw depth pass when overlays are turned off. */
-  stl->g_data->need_depth = (v3d->flag2 & V3D_HIDE_OVERLAYS) == 0;
+  if (v3d != NULL) {
+    /* Do not draw depth pass when overlays are turned off. */
+    stl->g_data->need_depth = (v3d->flag2 & V3D_HIDE_OVERLAYS) == 0;
+  }
+  else if (draw_ctx->space_data != NULL) {
+    const eSpace_Type space_type = draw_ctx->space_data->spacetype;
+    if (space_type == SPACE_IMAGE) {
+      external_cache_image_add(stl->g_data->depth_shgrp);
+
+      stl->g_data->need_depth = true;
+      stl->g_data->update_depth = true;
+    }
+  }
 }
 
 static void external_cache_populate(void *vedata, Object *ob)
 {
+  const DRWContextState *draw_ctx = DRW_context_state_get();
   EXTERNAL_StorageList *stl = ((EXTERNAL_Data *)vedata)->stl;
 
+  if (draw_ctx->space_data != NULL) {
+    const eSpace_Type space_type = draw_ctx->space_data->spacetype;
+    if (space_type == SPACE_IMAGE) {
+      return;
+    }
+  }
+
   if (!(DRW_object_is_renderable(ob) &&
         DRW_object_visibility_in_active_context(ob) & OB_VISIBLE_SELF)) {
     return;
@@ -210,13 +251,11 @@ static void external_cache_finish(void *UNUSED(vedata))
 {
 }
 
-static void external_draw_scene_do(void *vedata)
+static void external_draw_scene_do_v3d(void *vedata)
 {
   const DRWContextState *draw_ctx = DRW_context_state_get();
-  Scene *scene = draw_ctx->scene;
   RegionView3D *rv3d = draw_ctx->rv3d;
   ARegion *region = draw_ctx->region;
-  const RenderEngineType *type;
 
   DRW_state_reset_ex(DRW_STATE_DEFAULT & ~DRW_STATE_DEPTH_LESS_EQUAL);
 
@@ -229,8 +268,6 @@ static void external_draw_scene_do(void *vedata)
     }
 
     RenderEngine *engine = RE_engine_create(engine_type);
-    engine->tile_x = scene->r.tilex;
-    engine->tile_y = scene->r.tiley;
     engine_type->view_update(engine, draw_ctx->evil_C, draw_ctx->depsgraph);
     rv3d->render_engine = engine;
   }
@@ -241,7 +278,7 @@ static void external_draw_scene_do(void *vedata)
   ED_region_pixelspace(region);
 
   /* Render result draw. */
-  type = rv3d->render_engine->type;
+  const RenderEngineType *type = rv3d->render_engine->type;
   type->view_draw(rv3d->render_engine, draw_ctx->evil_C, draw_ctx->depsgraph);
 
   GPU_bgl_end();
@@ -259,6 +296,116 @@ static void external_draw_scene_do(void *vedata)
   }
 }
 
+/* Configure current matrix stack so that the external engine can use the same drawing code for
+ * both viewport and image editor drawing.
+ *
+ * The engine draws result in the pixel space, and is applying render offset. For image editor we
+ * need to switch from normalized space to pixel space, and "un-apply" offset. */
+static void external_image_space_matrix_set(const RenderEngine *engine)
+{
+  BLI_assert(engine != NULL);
+
+  const DRWContextState *draw_ctx = DRW_context_state_get();
+  const DRWView *view = DRW_view_get_active();
+  struct SpaceImage *space_image = (struct SpaceImage *)draw_ctx->space_data;
+
+  /* Apply current view as transformation matrix.
+   * This will configure drawing for normalized space with current zoom and pan applied. */
+
+  float view_matrix[4][4];
+  DRW_view_viewmat_get(view, view_matrix, false);
+
+  float projection_matrix[4][4];
+  DRW_view_winmat_get(view, projection_matrix, false);
+
+  GPU_matrix_projection_set(projection_matrix);
+  GPU_matrix_set(view_matrix);
+
+  /* Switch from normalized space to pixel space. */
+  {
+    int width, height;
+    ED_space_image_get_size(space_image, &width, &height);
+
+    const float width_inv = width ? 1.0f / width : 0.0f;
+    const float height_inv = height ? 1.0f / height : 0.0f;
+    GPU_matrix_scale_2f(width_inv, height_inv);
+  }
+
+  /* Un-apply render offset. */
+  {
+    Render *render = engine->re;
+    rctf view_rect;
+    rcti render_rect;
+    RE_GetViewPlane(render, &view_rect, &render_rect);
+
+    GPU_matrix_translate_2f(-render_rect.xmin, -render_rect.ymin);
+  }
+}
+
+static void external_draw_scene_do_image(void *UNUSED(vedata))
+{
+  const DRWContextState *draw_ctx = DRW_context_state_get();
+  Scene *scene = draw_ctx->scene;
+  Render *re = RE_GetSceneRender(scene);
+  RenderEngine *engine = RE_engine_get(re);
+
+  /* Is tested before enabling the drawing engine. */
+  BLI_assert(re != NULL);
+  BLI_assert(engine != NULL);
+
+  const DefaultFramebufferList *dfbl = DRW_viewport_framebuffer_list_get();
+
+  /* Clear the depth buffer to the value used by the background overlay so that the overlay is not
+   * happening outside of the drawn image.
+   *
+   * NOTE: The external engine only draws color. The depth is taken care of using the depth pass
+   * which initialized the depth to the values expected by the background overlay. */
+  GPU_framebuffer_clear_depth(dfbl->default_fb, 1.0f);
+
+  GPU_matrix_push_projection();
+  GPU_matrix_push();
+
+  external_image_space_matrix_set(engine);
+
+  GPU_debug_group_begin("External Engine");
+
+  const RenderEngineType *engine_type = engine->type;
+  BLI_assert(engine_type != NULL);
+  BLI_assert(engine_type->draw != NULL);
+
+  engine_type->draw(engine, draw_ctx->evil_C, draw_ctx->depsgraph);
+
+  GPU_debug_group_end();
+
+  GPU_matrix_pop();
+  GPU_matrix_pop_projection();
+
+  DRW_state_reset();
+  GPU_bgl_end();
+
+  RE_engine_draw_release(re);
+}
+
+static void external_draw_scene_do(void *vedata)
+{
+  const DRWContextState *draw_ctx = DRW_context_state_get();
+
+  if (draw_ctx->v3d != NULL) {
+    external_draw_scene_do_v3d(vedata);
+    return;
+  }
+
+  if (draw_ctx->space_data == NULL) {
+    return;
+  }
+
+  const eSpace_Type space_type = draw_ctx->space_data->spacetype;
+  if (space_type == SPACE_IMAGE) {
+    external_draw_scene_do_image(vedata);
+    return;
+  }
+}
+
 static void external_draw_scene(void *vedata)
 {
   const DRWContextState *draw_ctx = DRW_context_state_get();
@@ -297,7 +444,7 @@ static void external_engine_free(void)
 
 static const DrawEngineDataSize external_data_size = DRW_VIEWPORT_DATA_SIZE(EXTERNAL_Data);
 
-static DrawEngineType draw_engine_external_type = {
+DrawEngineType draw_engine_external_type = {
     NULL,
     NULL,
     N_("External"),
@@ -330,8 +477,45 @@ RenderEngineType DRW_engine_viewport_external_type = {
     NULL,
     NULL,
     NULL,
+    NULL,
+    NULL,
     &draw_engine_external_type,
     {NULL, NULL, NULL},
 };
 
+bool DRW_engine_external_acquire_for_image_editor(void)
+{
+  const DRWContextState *draw_ctx = DRW_context_state_get();
+  const SpaceLink *space_data = draw_ctx->space_data;
+  Scene *scene = draw_ctx->scene;
+
+  if (space_data == NULL) {
+    return false;
+  }
+
+  const eSpace_Type space_type = draw_ctx->space_data->spacetype;
+  if (space_type != SPACE_IMAGE) {
+    return false;
+  }
+
+  struct SpaceImage *space_image = (struct SpaceImage *)space_data;
+  const Image *image = ED_space_image(space_image);
+  if (image == NULL || image->type != IMA_TYPE_R_RESULT) {
+    return false;
+  }
+
+  if (image->render_slot != image->last_render_slot) {
+    return false;
+  }
+
+  /* Render is allocated on main thread, so it is safe to access it from here. */
+  Render *re = RE_GetSceneRender(scene);
+
+  if (re == NULL) {
+    return false;
+  }
+
+  return RE_engine_draw_acquire(re);
+}
+
 #undef EXTERNAL_ENGINE
diff --git a/source/blender/draw/engines/external/external_engine.h b/source/blender/draw/engines/external/external_engine.h
index c645fb99e0e..14ec4e2d3c5 100644
--- a/source/blender/draw/engines/external/external_engine.h
+++ b/source/blender/draw/engines/external/external_engine.h
@@ -22,4 +22,12 @@
 
 #pragma once
 
+extern DrawEngineType draw_engine_external_type;
 extern RenderEngineType DRW_engine_viewport_external_type;
+
+/* Check whether an external engine is to be used to draw content of an image editor.
+ * If the drawing is possible, the render engine is "acquired" so that it is not freed by the
+ * render engine for until drawing is finished.
+ *
+ * NOTE: Released by the draw engine when it is done drawing. */
+bool DRW_engine_external_acquire_for_image_editor(void);
diff --git a/source/blender/draw/engines/select/select_engine.c b/source/blender/draw/engines/select/select_engine.c
index 96ab8a28e09..20edd78597b 100644
--- a/source/blender/draw/engines/select/select_engine.c
+++ b/source/blender/draw/engines/select/select_engine.c
@@ -388,6 +388,8 @@ RenderEngineType DRW_engine_viewport_select_type = {
     NULL,
     NULL,
     NULL,
+    NULL,
+    NULL,
     &draw_engine_select_type,
     {NULL, NULL, NULL},
 };
diff --git a/source/blender/draw/engines/workbench/workbench_engine.c b/source/blender/draw/engines/workbench/workbench_engine.c
index f09c019ef8d..635aa7cef25 100644
--- a/source/blender/draw/engines/workbench/workbench_engine.c
+++ b/source/blender/draw/engines/workbench/workbench_engine.c
@@ -651,6 +651,8 @@ RenderEngineType DRW_engine_viewport_workbench_type = {
     NULL,
     NULL,
     NULL,
+    NULL,
+    NULL,
     &workbench_render_update_passes,
     &draw_engine_workbench,
     {NULL, NULL, NULL},
diff --git a/source/blender/draw/intern/DRW_render.h b/source/blender/draw/intern/DRW_render.h
index 660a4adaf51..fb8b8536897 100644
--- a/source/blender/draw/intern/DRW_render.h
+++ b/source/blender/draw/intern/DRW_render.h
@@ -623,6 +623,7 @@ const DRWView *DRW_view_default_get(void);
 void DRW_view_default_set(DRWView *view);
 void DRW_view_reset(void);
 void DRW_view_set_active(DRWView *view);
+const DRWView *DRW_view_get_active(void);
 
 void DRW_view_clip_planes_set(DRWView *view, float (*planes)[4], int plane_len);
 void DRW_view_camtexco_set(DRWView *view, float texco[4]);
diff --git a/source/blender/draw/intern/draw_manager.c b/source/blender/draw/intern/draw_manager.c
index 47adc0acc60..e65fdce5f2e 100644
--- a/source/blender/draw/intern/draw_manager.c
+++ b/source/blender/draw/intern/draw_manager.c
@@ -1197,6 +1197,18 @@ static void drw_engines_enable_basic(void)
   use_drw_engine(&draw_engine_basic_type);
 }
 
+static void drw_engine_enable_image_editor(void)
+{
+  if (DRW_engine_external_acquire_for_image_editor()) {
+    use_drw_engine(&draw_engine_external_type);
+  }
+  else {
+    use_drw_engine(&draw_engine_image_type);
+  }
+
+  use_drw_engine(&draw_engine_overlay_type);
+}
+
 static void drw_engines_enable_editors(void)
 {
   SpaceLink *space_data = DST.draw_ctx.space_data;
@@ -1205,8 +1217,7 @@ static void drw_engines_enable_editors(void)
   }
 
   if (space_data->spacetype == SPACE_IMAGE) {
-    use_drw_engine(&draw_engine_image_type);
-    use_drw_engine(&draw_engine_overlay_type);
+    drw_engine_enable_image_editor();
   }
   else if (space_data->spacetype == SPACE_NODE) {
     /* Only enable when drawing the space image backdrop. */
@@ -3188,3 +3199,66 @@ void DRW_draw_state_init_gtests(eGPUShaderConfig sh_cfg)
 #endif
 
 /** \} */
+
+/* -------------------------------------------------------------------- */
+/** \name Draw manager context release/activation
+ *
+ * These functions are used in cases when an OpenGL context creation is needed during the draw.
+ * This happens, for example, when an external engine needs to create its own OpenGL context from
+ * the engine initialization.
+ *
+ * Example of context creation:
+ *
+ *   const bool drw_state = DRW_opengl_context_release();
+ *   gl_context = WM_opengl_context_create();
+ *   DRW_opengl_context_activate(drw_state);
+ *
+ * Example of context destruction:
+ *
+ *   const bool drw_state = DRW_opengl_context_release();
+ *   WM_opengl_context_activate(gl_context);
+ *   WM_opengl_context_dispose(gl_context);
+ *   DRW_opengl_context_activate(drw_state);
+ *
+ *
+ * NOTE: Will only perform context modification when on main thread. This way these functions can
+ * be used in an engine without check on whether it is a draw manager which manages OpenGL context
+ * on the current thread. The downside of this is that if the engine performs OpenGL creation from
+ * a non-main thread, that thread is supposed to not have OpenGL context ever bound by Blender.
+ *
+ * \{ */
+
+bool DRW_opengl_context_release(void)
+{
+  if (!BLI_thread_is_main()) {
+    return false;
+  }
+
+  if (GPU_context_active_get() != DST.gpu_context) {
+    /* Context release is requested from the outside of the draw manager main draw loop, indicate
+     * this to the `DRW_opengl_context_activate()` so that it restores drawable of the window. */
+    return false;
+  }
+
+  GPU_context_active_set(NULL);
+  WM_opengl_context_release(DST.gl_context);
+
+  return true;
+}
+
+void DRW_opengl_context_activate(bool drw_state)
+{
+  if (!BLI_thread_is_main()) {
+    return;
+  }
+
+  if (drw_state) {
+    WM_opengl_context_activate(DST.gl_context);
+    GPU_context_active_set(DST.gpu_context);
+  }
+  else {
+    wm_window_reset_drawable();
+  }
+}
+
+/** \} */
diff --git a/source/blender/draw/intern/draw_manager_exec.c b/source/blender/draw/intern/draw_manager_exec.c
index 22356a3c57b..aa01ca7a262 100644
--- a/source/blender/draw/intern/draw_manager_exec.c
+++ b/source/blender/draw/intern/draw_manager_exec.c
@@ -367,6 +367,11 @@ void DRW_view_set_active(DRWView *view)
   DST.view_active = (view) ? view : DST.view_default;
 }
 
+const DRWView *DRW_view_get_active(void)
+{
+  return DST.view_active;
+}
+
 /* Return True if the given BoundSphere intersect the current view frustum */
 static bool draw_culling_sphere_test(const BoundSphere *frustum_bsphere,
                                      const float (*frustum_planes)[4],