Paint: prevent RenderResults and Viewers where unappropriate

Using a RenderResult (or a Viewer) was never really working (think you
cant get a real ImBuff from these) -- cannot use it as a clone, stencil
or canvas [Single Image paint texture slot].

In the case of using it as a 2D paint clone image this would also crash
[due to the Image Editor drawing refactor in 2.91].

Now [in the spirit of T73182 / D11179], prevent using these where
unappropriate by using rna pointer polling functions.

Also add a security check for the 2D paint clone image crash in case a
stencil ImBuff cannot be provided for some reason, but generally old
files are now patched in do_versions_after_linking_300 (thx @brecht!).

Fixes T91625.

Maniphest Tasks: T91625

Differential Revision: https://developer.blender.org/D12609

4 files changed, 70 insertions, 24 deletions

diff --git a/source/blender/blenloader/intern/versioning_300.c b/source/blender/blenloader/intern/versioning_300.c
index 9908e231452..0d333ac2edc 100644
--- a/source/blender/blenloader/intern/versioning_300.c
+++ b/source/blender/blenloader/intern/versioning_300.c
@@ -518,6 +518,28 @@ void do_versions_after_linking_300(Main *bmain, ReportList *UNUSED(reports))
   {
     /* Keep this block, even when empty. */
     do_versions_idproperty_ui_data(bmain);
+
+    LISTBASE_FOREACH (Scene *, scene, &bmain->scenes) {
+      ToolSettings *tool_settings = scene->toolsettings;
+      ImagePaintSettings *imapaint = &tool_settings->imapaint;
+      if (imapaint->canvas != NULL &&
+          ELEM(imapaint->canvas->type, IMA_TYPE_R_RESULT, IMA_TYPE_COMPOSITE)) {
+        imapaint->canvas = NULL;
+      }
+      if (imapaint->stencil != NULL &&
+          ELEM(imapaint->stencil->type, IMA_TYPE_R_RESULT, IMA_TYPE_COMPOSITE)) {
+        imapaint->stencil = NULL;
+      }
+      if (imapaint->clone != NULL &&
+          ELEM(imapaint->clone->type, IMA_TYPE_R_RESULT, IMA_TYPE_COMPOSITE)) {
+        imapaint->clone = NULL;
+      }
+    }
+    LISTBASE_FOREACH (Brush *, brush, &bmain->brushes) {
+      if (brush->clone.image != NULL && ELEM(brush->clone.image->type, IMA_TYPE_R_RESULT,
+                                             IMA_TYPE_COMPOSITE)) { brush->clone.image = NULL;
+      }
+    }
   }
 }
 
diff --git a/source/blender/draw/engines/overlay/overlay_edit_uv.c b/source/blender/draw/engines/overlay/overlay_edit_uv.c
index 985f8a6785c..983df1ceac8 100644
--- a/source/blender/draw/engines/overlay/overlay_edit_uv.c
+++ b/source/blender/draw/engines/overlay/overlay_edit_uv.c
@@ -340,34 +340,42 @@ void OVERLAY_edit_uv_cache_init(OVERLAY_Data *vedata)
 
   if (pd->edit_uv.do_stencil_overlay) {
     const Brush *brush = BKE_paint_brush(&ts->imapaint.paint);
-
-    DRW_PASS_CREATE(psl->edit_uv_stencil_ps,
-                    DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_ALWAYS | DRW_STATE_BLEND_ALPHA_PREMUL);
-    GPUShader *sh = OVERLAY_shader_edit_uv_stencil_image();
-    GPUBatch *geom = DRW_cache_quad_get();
-    DRWShadingGroup *grp = DRW_shgroup_create(sh, psl->edit_uv_stencil_ps);
     Image *stencil_image = brush->clone.image;
     ImBuf *stencil_ibuf = BKE_image_acquire_ibuf(stencil_image, NULL, &pd->edit_uv.stencil_lock);
-    pd->edit_uv.stencil_ibuf = stencil_ibuf;
-    pd->edit_uv.stencil_image = stencil_image;
-    GPUTexture *stencil_texture = BKE_image_get_gpu_texture(stencil_image, NULL, stencil_ibuf);
-    DRW_shgroup_uniform_texture(grp, "imgTexture", stencil_texture);
-    DRW_shgroup_uniform_bool_copy(grp, "imgPremultiplied", true);
-    DRW_shgroup_uniform_bool_copy(grp, "imgAlphaBlend", true);
-    DRW_shgroup_uniform_vec4_copy(grp, "color", (float[4]){1.0f, 1.0f, 1.0f, brush->clone.alpha});
-
-    float size_image[2];
-    BKE_image_get_size_fl(image, NULL, size_image);
-    float size_stencil_image[2] = {stencil_ibuf->x, stencil_ibuf->y};
 
-    float obmat[4][4];
-    unit_m4(obmat);
-    obmat[3][1] = brush->clone.offset[1];
-    obmat[3][0] = brush->clone.offset[0];
-    obmat[0][0] = size_stencil_image[0] / size_image[0];
-    obmat[1][1] = size_stencil_image[1] / size_image[1];
+    if (stencil_ibuf == NULL) {
+      pd->edit_uv.stencil_ibuf = NULL;
+      pd->edit_uv.stencil_image = NULL;
+    }
+    else {
+      DRW_PASS_CREATE(psl->edit_uv_stencil_ps,
+                      DRW_STATE_WRITE_COLOR | DRW_STATE_DEPTH_ALWAYS |
+                          DRW_STATE_BLEND_ALPHA_PREMUL);
+      GPUShader *sh = OVERLAY_shader_edit_uv_stencil_image();
+      GPUBatch *geom = DRW_cache_quad_get();
+      DRWShadingGroup *grp = DRW_shgroup_create(sh, psl->edit_uv_stencil_ps);
+      pd->edit_uv.stencil_ibuf = stencil_ibuf;
+      pd->edit_uv.stencil_image = stencil_image;
+      GPUTexture *stencil_texture = BKE_image_get_gpu_texture(stencil_image, NULL, stencil_ibuf);
+      DRW_shgroup_uniform_texture(grp, "imgTexture", stencil_texture);
+      DRW_shgroup_uniform_bool_copy(grp, "imgPremultiplied", true);
+      DRW_shgroup_uniform_bool_copy(grp, "imgAlphaBlend", true);
+      DRW_shgroup_uniform_vec4_copy(
+          grp, "color", (float[4]){1.0f, 1.0f, 1.0f, brush->clone.alpha});
+
+      float size_image[2];
+      BKE_image_get_size_fl(image, NULL, size_image);
+      float size_stencil_image[2] = {stencil_ibuf->x, stencil_ibuf->y};
+
+      float obmat[4][4];
+      unit_m4(obmat);
+      obmat[3][1] = brush->clone.offset[1];
+      obmat[3][0] = brush->clone.offset[0];
+      obmat[0][0] = size_stencil_image[0] / size_image[0];
+      obmat[1][1] = size_stencil_image[1] / size_image[1];
 
-    DRW_shgroup_call_obmat(grp, geom, obmat);
+      DRW_shgroup_call_obmat(grp, geom, obmat);
+    }
   }
   else {
     pd->edit_uv.stencil_ibuf = NULL;
diff --git a/source/blender/makesrna/intern/rna_brush.c b/source/blender/makesrna/intern/rna_brush.c
index 25caa411979..1d3b8cd9f9c 100644
--- a/source/blender/makesrna/intern/rna_brush.c
+++ b/source/blender/makesrna/intern/rna_brush.c
@@ -734,6 +734,12 @@ static void rna_Brush_icon_update(Main *UNUSED(bmain), Scene *UNUSED(scene), Poi
   WM_main_add_notifier(NC_BRUSH | NA_EDITED, br);
 }
 
+static bool rna_Brush_imagetype_poll(PointerRNA *UNUSED(ptr), PointerRNA value)
+{
+  Image *image = (Image *)value.owner_id;
+  return image->type != IMA_TYPE_R_RESULT && image->type != IMA_TYPE_COMPOSITE;
+}
+
 static void rna_TextureSlot_brush_angle_update(bContext *C, PointerRNA *ptr)
 {
   Scene *scene = CTX_data_scene(C);
@@ -3434,6 +3440,7 @@ static void rna_def_brush(BlenderRNA *brna)
   RNA_def_property_flag(prop, PROP_EDITABLE);
   RNA_def_property_ui_text(prop, "Clone Image", "Image for clone tool");
   RNA_def_property_update(prop, NC_SPACE | ND_SPACE_IMAGE, "rna_Brush_update");
+  RNA_def_property_pointer_funcs(prop, NULL, NULL, NULL, "rna_Brush_imagetype_poll");
 
   prop = RNA_def_property(srna, "clone_alpha", PROP_FLOAT, PROP_FACTOR);
   RNA_def_property_float_sdna(prop, NULL, "clone.alpha");
diff --git a/source/blender/makesrna/intern/rna_sculpt_paint.c b/source/blender/makesrna/intern/rna_sculpt_paint.c
index 1da08448c5b..45e85d14865 100644
--- a/source/blender/makesrna/intern/rna_sculpt_paint.c
+++ b/source/blender/makesrna/intern/rna_sculpt_paint.c
@@ -501,6 +501,12 @@ static void rna_ImaPaint_stencil_update(bContext *C, PointerRNA *UNUSED(ptr))
   }
 }
 
+static bool rna_ImaPaint_imagetype_poll(PointerRNA *UNUSED(ptr), PointerRNA value)
+{
+  Image *image = (Image *)value.owner_id;
+  return image->type != IMA_TYPE_R_RESULT && image->type != IMA_TYPE_COMPOSITE;
+}
+
 static void rna_ImaPaint_canvas_update(bContext *C, PointerRNA *UNUSED(ptr))
 {
   Main *bmain = CTX_data_main(C);
@@ -1033,17 +1039,20 @@ static void rna_def_image_paint(BlenderRNA *brna)
   RNA_def_property_flag(prop, PROP_EDITABLE | PROP_CONTEXT_UPDATE);
   RNA_def_property_ui_text(prop, "Stencil Image", "Image used as stencil");
   RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, "rna_ImaPaint_stencil_update");
+  RNA_def_property_pointer_funcs(prop, NULL, NULL, NULL, "rna_ImaPaint_imagetype_poll");
 
   prop = RNA_def_property(srna, "canvas", PROP_POINTER, PROP_NONE);
   RNA_def_property_flag(prop, PROP_EDITABLE | PROP_CONTEXT_UPDATE);
   RNA_def_property_ui_text(prop, "Canvas", "Image used as canvas");
   RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, "rna_ImaPaint_canvas_update");
+  RNA_def_property_pointer_funcs(prop, NULL, NULL, NULL, "rna_ImaPaint_imagetype_poll");
 
   prop = RNA_def_property(srna, "clone_image", PROP_POINTER, PROP_NONE);
   RNA_def_property_pointer_sdna(prop, NULL, "clone");
   RNA_def_property_flag(prop, PROP_EDITABLE);
   RNA_def_property_ui_text(prop, "Clone Image", "Image used as clone source");
   RNA_def_property_update(prop, NC_SCENE | ND_TOOLSETTINGS, NULL);
+  RNA_def_property_pointer_funcs(prop, NULL, NULL, NULL, "rna_ImaPaint_imagetype_poll");
 
   prop = RNA_def_property(srna, "stencil_color", PROP_FLOAT, PROP_COLOR_GAMMA);
   RNA_def_property_range(prop, 0.0, 1.0);