IMBUF: Faster JPEG Thumbnails

Make preview thumbnails of JPEG files in less time and with less RAM.

See D14727 for more details.

Differential Revision: https://developer.blender.org/D14727

Reviewed by Brecht Van Lommel

6 files changed, 220 insertions, 46 deletions

diff --git a/source/blender/imbuf/IMB_imbuf.h b/source/blender/imbuf/IMB_imbuf.h
index 0390df06052..8796c99629e 100644
--- a/source/blender/imbuf/IMB_imbuf.h
+++ b/source/blender/imbuf/IMB_imbuf.h
@@ -109,6 +109,14 @@ struct ImBuf *IMB_loadiffname(const char *filepath, int flags, char colorspace[I
 
 /**
  *
+ * \attention Defined in readimage.c
+ */
+struct ImBuf *IMB_thumb_load_image(const char *filepath,
+                                   const size_t max_thumb_size,
+                                   char colorspace[IM_MAX_SPACE]);
+
+/**
+ *
  * \attention Defined in allocimbuf.c
  */
 void IMB_freeImBuf(struct ImBuf *ibuf);
diff --git a/source/blender/imbuf/intern/IMB_filetype.h b/source/blender/imbuf/intern/IMB_filetype.h
index 31f8b3a9505..bcca6f9fd85 100644
--- a/source/blender/imbuf/intern/IMB_filetype.h
+++ b/source/blender/imbuf/intern/IMB_filetype.h
@@ -36,6 +36,15 @@ typedef struct ImFileType {
                         char colorspace[IM_MAX_SPACE]);
   /** Load an image from a file. */
   struct ImBuf *(*load_filepath)(const char *filepath, int flags, char colorspace[IM_MAX_SPACE]);
+  /** Load/Create a thumbnail image from a filepath. `max_thumb_size` is maximum size of either
+   * dimension, so can return less on either or both. Should, if possible and performant, return
+   * dimensions of the full-size image in width_r & height_r. */
+  struct ImBuf *(*load_filepath_thumbnail)(const char *filepath,
+                                           const int flags,
+                                           const size_t max_thumb_size,
+                                           size_t *width_r,
+                                           size_t *height_r,
+                                           char colorspace[IM_MAX_SPACE]);
   /** Save to a file (or memory if #IB_mem is set in `flags` and the format supports it). */
   bool (*save)(struct ImBuf *ibuf, const char *filepath, int flags);
   void (*load_tile)(struct ImBuf *ibuf,
@@ -143,6 +152,12 @@ struct ImBuf *imb_load_jpeg(const unsigned char *buffer,
                             size_t size,
                             int flags,
                             char colorspace[IM_MAX_SPACE]);
+struct ImBuf *imb_thumbnail_jpeg(const char *filepath,
+                                 const int flags,
+                                 const size_t max_thumb_size,
+                                 size_t *width_r,
+                                 size_t *height_r,
+                                 char colorspace[IM_MAX_SPACE]);
 
 /** \} */
 
diff --git a/source/blender/imbuf/intern/filetype.c b/source/blender/imbuf/intern/filetype.c
index 548bc9e120c..74042ef75be 100644
--- a/source/blender/imbuf/intern/filetype.c
+++ b/source/blender/imbuf/intern/filetype.c
@@ -33,6 +33,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_jpeg,
         .load = imb_load_jpeg,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = imb_thumbnail_jpeg,
         .save = imb_savejpeg,
         .load_tile = NULL,
         .flag = 0,
@@ -45,6 +46,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_png,
         .load = imb_loadpng,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_savepng,
         .load_tile = NULL,
         .flag = 0,
@@ -57,6 +59,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_bmp,
         .load = imb_bmp_decode,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_savebmp,
         .load_tile = NULL,
         .flag = 0,
@@ -69,6 +72,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_targa,
         .load = imb_loadtarga,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_savetarga,
         .load_tile = NULL,
         .flag = 0,
@@ -81,6 +85,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_iris,
         .load = imb_loadiris,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_saveiris,
         .load_tile = NULL,
         .flag = 0,
@@ -94,6 +99,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_dpx,
         .load = imb_load_dpx,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_save_dpx,
         .load_tile = NULL,
         .flag = IM_FTYPE_FLOAT,
@@ -106,6 +112,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_cineon,
         .load = imb_load_cineon,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_save_cineon,
         .load_tile = NULL,
         .flag = IM_FTYPE_FLOAT,
@@ -120,6 +127,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_tiff,
         .load = imb_loadtiff,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_savetiff,
         .load_tile = imb_loadtiletiff,
         .flag = 0,
@@ -134,6 +142,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_hdr,
         .load = imb_loadhdr,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_savehdr,
         .load_tile = NULL,
         .flag = IM_FTYPE_FLOAT,
@@ -148,6 +157,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_openexr,
         .load = imb_load_openexr,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_save_openexr,
         .load_tile = NULL,
         .flag = IM_FTYPE_FLOAT,
@@ -162,6 +172,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_jp2,
         .load = imb_load_jp2,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_save_jp2,
         .load_tile = NULL,
         .flag = IM_FTYPE_FLOAT,
@@ -176,6 +187,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_dds,
         .load = imb_load_dds,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = NULL,
         .load_tile = NULL,
         .flag = 0,
@@ -190,6 +202,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_photoshop,
         .load = NULL,
         .load_filepath = imb_load_photoshop,
+        .load_filepath_thumbnail = NULL,
         .save = NULL,
         .load_tile = NULL,
         .flag = IM_FTYPE_FLOAT,
@@ -204,6 +217,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .is_a = imb_is_a_webp,
         .load = imb_loadwebp,
         .load_filepath = NULL,
+        .load_filepath_thumbnail = NULL,
         .save = imb_savewebp,
         .load_tile = NULL,
         .flag = 0,
@@ -211,7 +225,7 @@ const ImFileType IMB_FILE_TYPES[] = {
         .default_save_role = COLOR_ROLE_DEFAULT_BYTE,
     },
 #endif
-    {NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0},
+    {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0},
 };
 
 const ImFileType *IMB_FILE_TYPES_LAST = &IMB_FILE_TYPES[ARRAY_SIZE(IMB_FILE_TYPES) - 1];
diff --git a/source/blender/imbuf/intern/jpeg.c b/source/blender/imbuf/intern/jpeg.c
index 6fb1fb52153..0ec4e5a3ba8 100644
--- a/source/blender/imbuf/intern/jpeg.c
+++ b/source/blender/imbuf/intern/jpeg.c
@@ -39,7 +39,11 @@ static void skip_input_data(j_decompress_ptr cinfo, long num_bytes);
 static void term_source(j_decompress_ptr cinfo);
 static void memory_source(j_decompress_ptr cinfo, const unsigned char *buffer, size_t size);
 static boolean handle_app1(j_decompress_ptr cinfo);
-static ImBuf *ibJpegImageFromCinfo(struct jpeg_decompress_struct *cinfo, int flags);
+static ImBuf *ibJpegImageFromCinfo(struct jpeg_decompress_struct *cinfo,
+                                   int flags,
+                                   int max_size,
+                                   size_t *width_r,
+                                   size_t *height_r);
 
 static const uchar jpeg_default_quality = 75;
 static uchar ibuf_quality;
@@ -246,7 +250,11 @@ static boolean handle_app1(j_decompress_ptr cinfo)
   return true;
 }
 
-static ImBuf *ibJpegImageFromCinfo(struct jpeg_decompress_struct *cinfo, int flags)
+static ImBuf *ibJpegImageFromCinfo(struct jpeg_decompress_struct *cinfo,
+                                   int flags,
+                                   int max_size,
+                                   size_t *width_r,
+                                   size_t *height_r)
 {
   JSAMPARRAY row_pointer;
   JSAMPLE *buffer = NULL;
@@ -264,16 +272,34 @@ static ImBuf *ibJpegImageFromCinfo(struct jpeg_decompress_struct *cinfo, int fla
   jpeg_save_markers(cinfo, JPEG_COM, 0xffff);
 
   if (jpeg_read_header(cinfo, false) == JPEG_HEADER_OK) {
-    x = cinfo->image_width;
-    y = cinfo->image_height;
     depth = cinfo->num_components;
 
     if (cinfo->jpeg_color_space == JCS_YCCK) {
       cinfo->out_color_space = JCS_CMYK;
     }
 
+    if (width_r) {
+      *width_r = cinfo->image_width;
+    }
+    if (height_r) {
+      *height_r = cinfo->image_height;
+    }
+
+    if (max_size > 0) {
+      /* libjpeg can more quickly decompress while scaling down to 1/2, 1/4, 1/8,
+       * while libjpeg-turbo can also do 3/8, 5/8, etc. But max is 1/8. */
+      float scale = (float)max_size / MAX2(cinfo->image_width, cinfo->image_height);
+      cinfo->scale_denom = 8;
+      cinfo->scale_num = MAX2(1, MIN2(8, ceill(scale * (float)cinfo->scale_denom)));
+      cinfo->dct_method = JDCT_FASTEST;
+      cinfo->dither_mode = JDITHER_ORDERED;
+    }
+
     jpeg_start_decompress(cinfo);
 
+    x = cinfo->output_width;
+    y = cinfo->output_height;
+
     if (flags & IB_test) {
       jpeg_abort_decompress(cinfo);
       ibuf = IMB_allocImBuf(x, y, 8 * depth, 0);
@@ -449,11 +475,92 @@ ImBuf *imb_load_jpeg(const unsigned char *buffer,
   jpeg_create_decompress(cinfo);
   memory_source(cinfo, buffer, size);
 
-  ibuf = ibJpegImageFromCinfo(cinfo, flags);
+  ibuf = ibJpegImageFromCinfo(cinfo, flags, -1, NULL, NULL);
+
+  return ibuf;
+}
+
+/* Defines for JPEG Header markers and segment size. */
+#define JPEG_MARKER_MSB (0xFF)
+#define JPEG_MARKER_SOI (0xD8)
+#define JPEG_MARKER_APP1 (0xE1)
+#define JPEG_APP1_MAX (1 << 16)
+
+struct ImBuf *imb_thumbnail_jpeg(const char *filepath,
+                                 const int flags,
+                                 const size_t max_thumb_size,
+                                 size_t *width_r,
+                                 size_t *height_r,
+                                 char colorspace[IM_MAX_SPACE])
+{
+  struct jpeg_decompress_struct _cinfo, *cinfo = &_cinfo;
+  struct my_error_mgr jerr;
+  FILE *infile = NULL;
+
+  colorspace_set_default_role(colorspace, IM_MAX_SPACE, COLOR_ROLE_DEFAULT_BYTE);
+
+  cinfo->err = jpeg_std_error(&jerr.pub);
+  jerr.pub.error_exit = jpeg_error;
+
+  /* Establish the setjmp return context for my_error_exit to use. */
+  if (setjmp(jerr.setjmp_buffer)) {
+    /* If we get here, the JPEG code has signaled an error.
+     * We need to clean up the JPEG object, close the input file, and return.
+     */
+    jpeg_destroy_decompress(cinfo);
+    return NULL;
+  }
+
+  if ((infile = BLI_fopen(filepath, "rb")) == NULL) {
+    fprintf(stderr, "can't open %s\n", filepath);
+    return NULL;
+  }
+
+  /* If file contains an embedded thumbnail, let's return that instead. */
+
+  if ((fgetc(infile) == JPEG_MARKER_MSB) && (fgetc(infile) == JPEG_MARKER_SOI) &&
+      (fgetc(infile) == JPEG_MARKER_MSB) && (fgetc(infile) == JPEG_MARKER_APP1)) {
+    /* This is a JPEG in Exif format (SOI + APP1), not JFIF (SOI + APP0). */
+    unsigned int i = JPEG_APP1_MAX;
+    /* All Exif data is within this 64K header segment. Skip ahead until next SOI for thumbnail. */
+    while (!((fgetc(infile) == JPEG_MARKER_MSB) && (fgetc(infile) == JPEG_MARKER_SOI)) &&
+           !feof(infile) && i--)
+      ;
+    if (i > 0 && !feof(infile)) {
+      /* We found a JPEG thumbnail inside this image. */
+      ImBuf *ibuf = NULL;
+      unsigned char *buffer = (char *)MEM_callocN(JPEG_APP1_MAX, "thumbbuffer");
+      /* Just put SOI directly in buffer rather than seeking back 2 bytes. */
+      buffer[0] = JPEG_MARKER_MSB;
+      buffer[1] = JPEG_MARKER_SOI;
+      if (fread(buffer + 2, JPEG_APP1_MAX - 2, 1, infile) == 1) {
+        ibuf = imb_load_jpeg(buffer, JPEG_APP1_MAX, flags, colorspace);
+      }
+      MEM_SAFE_FREE(buffer);
+      if (ibuf) {
+        fclose(infile);
+        return ibuf;
+      }
+    }
+  }
+
+  /* No embedded thumbnail found, so let's create a new one. */
+
+  fseek(infile, 0, SEEK_SET);
+  jpeg_create_decompress(cinfo);
+
+  jpeg_stdio_src(cinfo, infile);
+  ImBuf *ibuf = ibJpegImageFromCinfo(cinfo, flags, max_thumb_size, width_r, height_r);
+  fclose(infile);
 
   return ibuf;
 }
 
+#undef JPEG_MARKER_MSB
+#undef JPEG_MARKER_SOI
+#undef JPEG_MARKER_APP1
+#undef JPEG_APP1_MAX
+
 static void write_jpeg(struct jpeg_compress_struct *cinfo, struct ImBuf *ibuf)
 {
   JSAMPLE *buffer = NULL;
diff --git a/source/blender/imbuf/intern/readimage.c b/source/blender/imbuf/intern/readimage.c
index df41c0ca757..424ab8e085a 100644
--- a/source/blender/imbuf/intern/readimage.c
+++ b/source/blender/imbuf/intern/readimage.c
@@ -22,6 +22,8 @@
 #include "IMB_filetype.h"
 #include "IMB_imbuf.h"
 #include "IMB_imbuf_types.h"
+#include "IMB_metadata.h"
+#include "IMB_thumbs.h"
 #include "imbuf.h"
 
 #include "IMB_colormanagement.h"
@@ -234,6 +236,61 @@ ImBuf *IMB_loadiffname(const char *filepath, int flags, char colorspace[IM_MAX_S
   return ibuf;
 }
 
+struct ImBuf *IMB_thumb_load_image(const char *filepath,
+                                   size_t max_thumb_size,
+                                   char colorspace[IM_MAX_SPACE])
+{
+  const ImFileType *type = IMB_file_type_from_ftype(IMB_ispic_type(filepath));
+  if (type == NULL) {
+    return NULL;
+  }
+
+  ImBuf *ibuf = NULL;
+  int flags = IB_rect | IB_metadata;
+  /* Size of the original image. */
+  size_t width = 0;
+  size_t height = 0;
+
+  char effective_colorspace[IM_MAX_SPACE] = "";
+  if (colorspace) {
+    BLI_strncpy(effective_colorspace, colorspace, sizeof(effective_colorspace));
+  }
+
+  if (type->load_filepath_thumbnail) {
+    ibuf = type->load_filepath_thumbnail(
+        filepath, flags, max_thumb_size, &width, &height, colorspace);
+  }
+  else {
+    /* Skip images of other types if over 100MB. */
+    const size_t file_size = BLI_file_size(filepath);
+    if (file_size != -1 && file_size > THUMB_SIZE_MAX) {
+      return NULL;
+    }
+    ibuf = IMB_loadiffname(filepath, flags, colorspace);
+    if (ibuf) {
+      width = ibuf->x;
+      height = ibuf->y;
+    }
+  }
+
+  if (ibuf) {
+    imb_handle_alpha(ibuf, flags, colorspace, effective_colorspace);
+
+    if (width > 0 && height > 0) {
+      /* Save dimensions of original image into the thumbnail metadata. */
+      char cwidth[40] = "0";
+      char cheight[40] = "0";
+      BLI_snprintf(cwidth, sizeof(cwidth), "%d", width);
+      BLI_snprintf(cheight, sizeof(cheight), "%d", height);
+      IMB_metadata_ensure(&ibuf->metadata);
+      IMB_metadata_set_field(ibuf->metadata, "Thumb::Image::Width", cwidth);
+      IMB_metadata_set_field(ibuf->metadata, "Thumb::Image::Height", cheight);
+    }
+  }
+
+  return ibuf;
+}
+
 ImBuf *IMB_testiffname(const char *filepath, int flags)
 {
   ImBuf *ibuf;
diff --git a/source/blender/imbuf/intern/thumbs.c b/source/blender/imbuf/intern/thumbs.c
index 37734ebacb2..d546d170568 100644
--- a/source/blender/imbuf/intern/thumbs.c
+++ b/source/blender/imbuf/intern/thumbs.c
@@ -319,11 +319,7 @@ static ImBuf *thumb_create_ex(const char *file_path,
   char tdir[FILE_MAX];
   char temp[FILE_MAX];
   char mtime[40] = "0";  /* in case we can't stat the file */
-  char cwidth[40] = "0"; /* in case images have no data */
-  char cheight[40] = "0";
   short tsize = 128;
-  short ex, ey;
-  float scaledx, scaledy;
   BLI_stat_t info;
 
   switch (size) {
@@ -340,15 +336,6 @@ static ImBuf *thumb_create_ex(const char *file_path,
       return NULL; /* unknown size */
   }
 
-  /* exception, skip images over 100mb */
-  if (source == THB_SOURCE_IMAGE) {
-    const size_t file_size = BLI_file_size(file_path);
-    if (file_size != -1 && file_size > THUMB_SIZE_MAX) {
-      // printf("file too big: %d, skipping %s\n", (int)size, file_path);
-      return NULL;
-    }
-  }
-
   if (get_thumb_dir(tdir, size)) {
     BLI_snprintf(tpath, FILE_MAX, "%s%s", tdir, thumb);
     //      thumb[8] = '\0'; /* shorten for tempname, not needed anymore */
@@ -368,7 +355,7 @@ static ImBuf *thumb_create_ex(const char *file_path,
         if (img == NULL) {
           switch (source) {
             case THB_SOURCE_IMAGE:
-              img = IMB_loadiffname(file_path, IB_rect | IB_metadata, NULL);
+              img = IMB_thumb_load_image(file_path, tsize, NULL);
               break;
             case THB_SOURCE_BLEND:
               img = IMB_thumb_load_blend(file_path, blen_group, blen_id);
@@ -385,8 +372,6 @@ static ImBuf *thumb_create_ex(const char *file_path,
           if (BLI_stat(file_path, &info) != -1) {
             BLI_snprintf(mtime, sizeof(mtime), "%ld", (long int)info.st_mtime);
           }
-          BLI_snprintf(cwidth, sizeof(cwidth), "%d", img->x);
-          BLI_snprintf(cheight, sizeof(cheight), "%d", img->y);
         }
       }
       else if (THB_SOURCE_MOVIE == source) {
@@ -411,28 +396,20 @@ static ImBuf *thumb_create_ex(const char *file_path,
         return NULL;
       }
 
-      if (img->x > img->y) {
-        scaledx = (float)tsize;
-        scaledy = ((float)img->y / (float)img->x) * tsize;
-      }
-      else {
-        scaledy = (float)tsize;
-        scaledx = ((float)img->x / (float)img->y) * tsize;
-      }
-      /* Scaling down must never assign zero width/height, see: T89868. */
-      ex = MAX2(1, (short)scaledx);
-      ey = MAX2(1, (short)scaledy);
-
-      /* save some time by only scaling byte buf */
-      if (img->rect_float) {
-        if (img->rect == NULL) {
-          IMB_rect_from_float(img);
+      if (img->x > tsize || img->y > tsize) {
+        float scale = MIN2((float)tsize / (float)img->x, (float)tsize / (float)img->y);
+        /* Scaling down must never assign zero width/height, see: T89868. */
+        short ex = MAX2(1, (short)(img->x * scale));
+        short ey = MAX2(1, (short)(img->y * scale));
+        /* Save some time by only scaling byte buf */
+        if (img->rect_float) {
+          if (img->rect == NULL) {
+            IMB_rect_from_float(img);
+          }
+          imb_freerectfloatImBuf(img);
         }
-
-        imb_freerectfloatImBuf(img);
+        IMB_scaleImBuf(img, ex, ey);
       }
-
-      IMB_scaleImBuf(img, ex, ey);
     }
     BLI_snprintf(desc, sizeof(desc), "Thumbnail for %s", uri);
     IMB_metadata_ensure(&img->metadata);
@@ -443,10 +420,6 @@ static ImBuf *thumb_create_ex(const char *file_path,
     if (use_hash) {
       IMB_metadata_set_field(img->metadata, "X-Blender::Hash", hash);
     }
-    if (ELEM(source, THB_SOURCE_IMAGE, THB_SOURCE_BLEND, THB_SOURCE_FONT)) {
-      IMB_metadata_set_field(img->metadata, "Thumb::Image::Width", cwidth);
-      IMB_metadata_set_field(img->metadata, "Thumb::Image::Height", cheight);
-    }
     img->ftype = IMB_FTYPE_PNG;
     img->planes = 32;