Images: change alpha settings to support channel packing

This also replaces the Use Alpha setting. We now have these alpha modes:

* Straight: store RGB and alpha channels separately with alpha acting as a
  mask, also known as unassociated alpha.
* Premultiplied: transparent RGB pixels are multiplied by the alpha channel.
  The natural format for renders.
* Channel Packed: different images are packed in the RGB and alpha channels,
  and they should not influence each other. Channel packing is commonly used
  by game engines to save memory.
* None: ignore alpha channel from the file and make image fully opaque.

Cycles OSL does not correctly support Channel Packed and None yet, we are
missing fine control over the OpenImageIO texture cache to do that.

Fixes T53672

5 files changed, 56 insertions, 23 deletions

diff --git a/source/blender/imbuf/IMB_imbuf.h b/source/blender/imbuf/IMB_imbuf.h
index 674dc61cd9e..8574f33bce6 100644
--- a/source/blender/imbuf/IMB_imbuf.h
+++ b/source/blender/imbuf/IMB_imbuf.h
@@ -476,6 +476,9 @@ int imb_get_anim_type(const char *name);
  */
 bool IMB_isfloat(struct ImBuf *ibuf);
 
+/* Do byte/float and colorspace conversions need to take alpha into account? */
+bool IMB_alpha_affects_rgb(const struct ImBuf *ibuf);
+
 /* create char buffer, color corrected if necessary, for ImBufs that lack one */
 void IMB_rect_from_float(struct ImBuf *ibuf);
 void IMB_float_from_rect(struct ImBuf *ibuf);
diff --git a/source/blender/imbuf/IMB_imbuf_types.h b/source/blender/imbuf/IMB_imbuf_types.h
index 0f2529e261a..61aa1f401a0 100644
--- a/source/blender/imbuf/IMB_imbuf_types.h
+++ b/source/blender/imbuf/IMB_imbuf_types.h
@@ -285,10 +285,12 @@ enum {
   IB_alphamode_premul = 1 << 12,
   /** if this flag is set, alpha mode would be guessed from file */
   IB_alphamode_detect = 1 << 13,
+  /* alpha channel is unrelated to RGB and should not affect it */
+  IB_alphamode_channel_packed = 1 << 14,
   /** ignore alpha on load and substitute it with 1.0f */
-  IB_ignore_alpha = 1 << 14,
-  IB_thumbnail = 1 << 15,
-  IB_multiview = 1 << 16,
+  IB_alphamode_ignore = 1 << 15,
+  IB_thumbnail = 1 << 16,
+  IB_multiview = 1 << 17,
 };
 
 /** \} */
diff --git a/source/blender/imbuf/intern/colormanagement.c b/source/blender/imbuf/intern/colormanagement.c
index ba0f10446a1..8a10af7e184 100644
--- a/source/blender/imbuf/intern/colormanagement.c
+++ b/source/blender/imbuf/intern/colormanagement.c
@@ -1057,13 +1057,19 @@ void colormanage_imbuf_make_linear(ImBuf *ibuf, const char *from_colorspace)
 
   if (ibuf->rect_float) {
     const char *to_colorspace = global_role_scene_linear;
+    const bool predivide = IMB_alpha_affects_rgb(ibuf);
 
     if (ibuf->rect) {
       imb_freerectImBuf(ibuf);
     }
 
-    IMB_colormanagement_transform(
-        ibuf->rect_float, ibuf->x, ibuf->y, ibuf->channels, from_colorspace, to_colorspace, true);
+    IMB_colormanagement_transform(ibuf->rect_float,
+                                  ibuf->x,
+                                  ibuf->y,
+                                  ibuf->channels,
+                                  from_colorspace,
+                                  to_colorspace,
+                                  predivide);
   }
 }
 
@@ -1405,6 +1411,7 @@ typedef struct DisplayBufferThread {
   int channels;
   float dither;
   bool is_data;
+  bool predivide;
 
   const char *byte_colorspace;
   const char *float_colorspace;
@@ -1469,6 +1476,7 @@ static void display_buffer_init_handle(void *handle_v,
   handle->channels = channels;
   handle->dither = dither;
   handle->is_data = is_data;
+  handle->predivide = IMB_alpha_affects_rgb(ibuf);
 
   handle->byte_colorspace = init_data->byte_colorspace;
   handle->float_colorspace = init_data->float_colorspace;
@@ -1486,6 +1494,7 @@ static void display_buffer_apply_get_linear_buffer(DisplayBufferThread *handle,
 
   bool is_data = handle->is_data;
   bool is_data_display = handle->cm_processor->is_data_result;
+  bool predivide = handle->predivide;
 
   if (!handle->buffer) {
     unsigned char *byte_buffer = handle->byte_buffer;
@@ -1534,7 +1543,7 @@ static void display_buffer_apply_get_linear_buffer(DisplayBufferThread *handle,
 
     if (!is_data && !is_data_display) {
       IMB_colormanagement_transform(
-          linear_buffer, width, height, channels, from_colorspace, to_colorspace, true);
+          linear_buffer, width, height, channels, from_colorspace, to_colorspace, predivide);
     }
 
     *is_straight_alpha = false;
@@ -1590,13 +1599,13 @@ static void *do_display_buffer_apply_thread(void *handle_v)
     }
   }
   else {
-    bool is_straight_alpha, predivide;
+    bool is_straight_alpha;
     float *linear_buffer = MEM_mallocN(((size_t)channels) * width * height * sizeof(float),
                                        "color conversion linear buffer");
 
     display_buffer_apply_get_linear_buffer(handle, height, linear_buffer, &is_straight_alpha);
 
-    predivide = is_straight_alpha == false;
+    bool predivide = handle->predivide && (is_straight_alpha == false);
 
     if (is_data) {
       /* special case for data buffers - no color space conversions,
@@ -2178,6 +2187,7 @@ void IMB_colormanagement_imbuf_to_srgb_texture(unsigned char *out_buffer,
 
   /* TODO(brecht): make this multithreaded, or at least process in batches. */
   const unsigned char *in_buffer = (unsigned char *)ibuf->rect;
+  const bool use_premultiply = IMB_alpha_affects_rgb(ibuf);
 
   for (int y = 0; y < height; y++) {
     const size_t in_offset = (offset_y + y) * ibuf->x + offset_x;
@@ -2192,11 +2202,13 @@ void IMB_colormanagement_imbuf_to_srgb_texture(unsigned char *out_buffer,
         rgba_uchar_to_float(pixel, in);
         OCIO_processorApplyRGB(processor, pixel);
         linearrgb_to_srgb_v3_v3(pixel, pixel);
-        mul_v3_fl(pixel, pixel[3]);
+        if (use_premultiply) {
+          mul_v3_fl(pixel, pixel[3]);
+        }
         rgba_float_to_uchar(out, pixel);
       }
     }
-    else {
+    else if (use_premultiply) {
       /* Premultiply only. */
       for (int x = 0; x < width; x++, in += 4, out += 4) {
         out[0] = (in[0] * in[3]) >> 8;
@@ -2205,6 +2217,15 @@ void IMB_colormanagement_imbuf_to_srgb_texture(unsigned char *out_buffer,
         out[3] = in[3];
       }
     }
+    else {
+      /* Copy only. */
+      for (int x = 0; x < width; x++, in += 4, out += 4) {
+        out[0] = in[0];
+        out[1] = in[1];
+        out[2] = in[2];
+        out[3] = in[3];
+      }
+    }
   }
 }
 
diff --git a/source/blender/imbuf/intern/divers.c b/source/blender/imbuf/intern/divers.c
index 1e16f0975e2..aa49453d68c 100644
--- a/source/blender/imbuf/intern/divers.c
+++ b/source/blender/imbuf/intern/divers.c
@@ -96,6 +96,12 @@ MINLINE void float_to_byte_dither_v4(
   b[3] = unit_float_to_uchar_clamp(f[3]);
 }
 
+/* Test if colorspace conversions of pixels in buffer need to take into account alpha. */
+bool IMB_alpha_affects_rgb(const ImBuf *ibuf)
+{
+  return (ibuf->flags & IB_alphamode_channel_packed) == 0;
+}
+
 /* float to byte pixels, output 4-channel RGBA */
 void IMB_buffer_byte_from_float(uchar *rect_to,
                                 const float *rect_from,
@@ -728,16 +734,19 @@ void IMB_rect_from_float(ImBuf *ibuf)
   buffer = MEM_dupallocN(ibuf->rect_float);
 
   /* first make float buffer in byte space */
+  const bool predivide = IMB_alpha_affects_rgb(ibuf);
   IMB_colormanagement_transform(buffer,
                                 ibuf->x,
                                 ibuf->y,
                                 ibuf->channels,
                                 from_colorspace,
                                 ibuf->rect_colorspace->name,
-                                true);
+                                predivide);
 
   /* convert from float's premul alpha to byte's straight alpha */
-  IMB_unpremultiply_rect_float(buffer, ibuf->channels, ibuf->x, ibuf->y);
+  if (IMB_alpha_affects_rgb(ibuf)) {
+    IMB_unpremultiply_rect_float(buffer, ibuf->channels, ibuf->x, ibuf->y);
+  }
 
   /* convert float to byte */
   IMB_buffer_byte_from_float((unsigned char *)ibuf->rect,
@@ -802,7 +811,9 @@ void IMB_float_from_rect(ImBuf *ibuf)
       rect_float, ibuf->x, ibuf->y, ibuf->channels, ibuf->rect_colorspace, false);
 
   /* byte buffer is straight alpha, float should always be premul */
-  IMB_premultiply_rect_float(rect_float, ibuf->channels, ibuf->x, ibuf->y);
+  if (IMB_alpha_affects_rgb(ibuf)) {
+    IMB_premultiply_rect_float(rect_float, ibuf->channels, ibuf->x, ibuf->y);
+  }
 
   if (ibuf->rect_float == NULL) {
     ibuf->rect_float = rect_float;
diff --git a/source/blender/imbuf/intern/readimage.c b/source/blender/imbuf/intern/readimage.c
index 9297227bc87..d9f3c7ec6bb 100644
--- a/source/blender/imbuf/intern/readimage.c
+++ b/source/blender/imbuf/intern/readimage.c
@@ -61,20 +61,16 @@ static void imb_handle_alpha(ImBuf *ibuf,
   }
 
   bool is_data = (colorspace && IMB_colormanagement_space_name_is_data(colorspace));
-  int alpha_flags;
+  int alpha_flags = (flags & IB_alphamode_detect) ? ibuf->flags : flags;
 
-  if (flags & IB_alphamode_detect) {
-    alpha_flags = ibuf->flags & IB_alphamode_premul;
-  }
-  else {
-    alpha_flags = flags & IB_alphamode_premul;
-  }
-
-  if (is_data) {
+  if (is_data || (flags & IB_alphamode_channel_packed)) {
     /* Don't touch alpha. */
+    ibuf->flags |= IB_alphamode_channel_packed;
   }
-  else if (flags & IB_ignore_alpha) {
+  else if (flags & IB_alphamode_ignore) {
+    /* Make opaque. */
     IMB_rectfill_alpha(ibuf, 1.0f);
+    ibuf->flags |= IB_alphamode_ignore;
   }
   else {
     if (alpha_flags & IB_alphamode_premul) {