Baking: new method to generate margin, based on adjacent faces

This significantly reduces discontinuities on UV seams, by giving a better
match of the texture filtered colors on both sides of the seam. It works by
using pixels from adjacent faces across the UV seam.

This new option is called "Adjacent Faces" and is the default. The old option
is called "Extend", and extends border pixels outwards.

Differential Revision: https://developer.blender.org/D13303

2 files changed, 42 insertions, 11 deletions

diff --git a/source/blender/editors/object/object_bake.c b/source/blender/editors/object/object_bake.c
index 1b6b0c78037..a5e6e7f0852 100644
--- a/source/blender/editors/object/object_bake.c
+++ b/source/blender/editors/object/object_bake.c
@@ -108,8 +108,10 @@ typedef struct {
   ListBase data;
   /** Clear the images before baking */
   bool bake_clear;
-  /** Bake-filter, aka margin */
-  int bake_filter;
+  /** margin size in pixels*/
+  int bake_margin;
+  /** margin type */
+  char bake_margin_type;
   /** mode of baking (displacement, normals, AO) */
   short mode;
   /** Use low-resolution mesh when baking displacement maps */
@@ -372,7 +374,8 @@ static int multiresbake_image_exec_locked(bContext *C, wmOperator *op)
 
     /* copy data stored in job descriptor */
     bkr.scene = scene;
-    bkr.bake_filter = scene->r.bake_filter;
+    bkr.bake_margin = scene->r.bake_margin;
+    bkr.bake_margin_type = scene->r.bake_margin_type;
     bkr.mode = scene->r.bake_mode;
     bkr.use_lores_mesh = scene->r.bake_flag & R_BAKE_LORES_MESH;
     bkr.bias = scene->r.bake_biasdist;
@@ -416,7 +419,8 @@ static void init_multiresbake_job(bContext *C, MultiresBakeJob *bkj)
 
   /* backup scene settings, so their changing in UI would take no effect on baker */
   bkj->scene = scene;
-  bkj->bake_filter = scene->r.bake_filter;
+  bkj->bake_margin = scene->r.bake_margin;
+  bkj->bake_margin_type = scene->r.bake_margin_type;
   bkj->mode = scene->r.bake_mode;
   bkj->use_lores_mesh = scene->r.bake_flag & R_BAKE_LORES_MESH;
   bkj->bake_clear = scene->r.bake_flag & R_BAKE_CLEAR;
@@ -477,7 +481,8 @@ static void multiresbake_startjob(void *bkv, short *stop, short *do_update, floa
 
     /* copy data stored in job descriptor */
     bkr.scene = bkj->scene;
-    bkr.bake_filter = bkj->bake_filter;
+    bkr.bake_margin = bkj->bake_margin;
+    bkr.bake_margin_type = bkj->bake_margin_type;
     bkr.mode = bkj->mode;
     bkr.use_lores_mesh = bkj->use_lores_mesh;
     bkr.user_scale = bkj->user_scale;
diff --git a/source/blender/editors/object/object_bake_api.c b/source/blender/editors/object/object_bake_api.c
index 0de34e21462..d56d0edd5a2 100644
--- a/source/blender/editors/object/object_bake_api.c
+++ b/source/blender/editors/object/object_bake_api.c
@@ -89,6 +89,7 @@ typedef struct BakeAPIRender {
   eScenePassType pass_type;
   int pass_filter;
   int margin;
+  eBakeMarginType margin_type;
 
   bool is_clear;
   bool is_selected_to_active;
@@ -184,8 +185,11 @@ static bool write_internal_bake_pixels(Image *image,
                                        const int width,
                                        const int height,
                                        const int margin,
+                                       const char margin_type,
                                        const bool is_clear,
-                                       const bool is_noncolor)
+                                       const bool is_noncolor,
+                                       Mesh const *mesh,
+                                       char const *uv_layer)
 {
   ImBuf *ibuf;
   void *lock;
@@ -281,7 +285,7 @@ static bool write_internal_bake_pixels(Image *image,
 
   /* margins */
   if (margin > 0) {
-    RE_bake_margin(ibuf, mask_buffer, margin);
+    RE_bake_margin(ibuf, mask_buffer, margin, margin_type, mesh, uv_layer);
   }
 
   ibuf->userflags |= IB_DISPLAY_BUFFER_INVALID;
@@ -327,8 +331,11 @@ static bool write_external_bake_pixels(const char *filepath,
                                        const int width,
                                        const int height,
                                        const int margin,
+                                       const int margin_type,
                                        ImageFormatData *im_format,
-                                       const bool is_noncolor)
+                                       const bool is_noncolor,
+                                       Mesh const *mesh,
+                                       char const *uv_layer)
 {
   ImBuf *ibuf = NULL;
   bool ok = false;
@@ -385,7 +392,7 @@ static bool write_external_bake_pixels(const char *filepath,
 
     mask_buffer = MEM_callocN(sizeof(char) * num_pixels, "Bake Mask");
     RE_bake_mask_fill(pixel_array, num_pixels, mask_buffer);
-    RE_bake_margin(ibuf, mask_buffer, margin);
+    RE_bake_margin(ibuf, mask_buffer, margin, margin_type, mesh, uv_layer);
 
     if (mask_buffer) {
       MEM_freeN(mask_buffer);
@@ -770,6 +777,7 @@ static bool bake_targets_output_internal(const BakeAPIRender *bkr,
                                          ReportList *reports)
 {
   bool all_ok = true;
+  const Mesh *me = (Mesh *)ob->data;
 
   for (int i = 0; i < targets->num_images; i++) {
     BakeImage *bk_image = &targets->images[i];
@@ -780,8 +788,11 @@ static bool bake_targets_output_internal(const BakeAPIRender *bkr,
                                                bk_image->width,
                                                bk_image->height,
                                                bkr->margin,
+                                               bkr->margin_type,
                                                bkr->is_clear,
-                                               targets->is_noncolor);
+                                               targets->is_noncolor,
+                                               me,
+                                               bkr->uv_layer);
 
     /* might be read by UI to set active image for display */
     bake_update_image(bkr->area, bk_image->image);
@@ -895,8 +906,11 @@ static bool bake_targets_output_external(const BakeAPIRender *bkr,
                                                bk_image->width,
                                                bk_image->height,
                                                bkr->margin,
+                                               bkr->margin_type,
                                                &bake->im_format,
-                                               targets->is_noncolor);
+                                               targets->is_noncolor,
+                                               me,
+                                               bkr->uv_layer);
 
     if (!ok) {
       BKE_reportf(reports, RPT_ERROR, "Problem saving baked map in \"%s\"", name);
@@ -1625,6 +1639,7 @@ static void bake_init_api_data(wmOperator *op, bContext *C, BakeAPIRender *bkr)
   bkr->pass_type = RNA_enum_get(op->ptr, "type");
   bkr->pass_filter = RNA_enum_get(op->ptr, "pass_filter");
   bkr->margin = RNA_int_get(op->ptr, "margin");
+  bkr->margin_type = RNA_enum_get(op->ptr, "margin_type");
 
   bkr->save_mode = (eBakeSaveMode)RNA_enum_get(op->ptr, "save_mode");
   bkr->target = (eBakeTarget)RNA_enum_get(op->ptr, "target");
@@ -1818,6 +1833,11 @@ static void bake_set_props(wmOperator *op, Scene *scene)
     RNA_property_int_set(op->ptr, prop, bake->margin);
   }
 
+  prop = RNA_struct_find_property(op->ptr, "margin_type");
+  if (!RNA_property_is_set(op->ptr, prop)) {
+    RNA_property_enum_set(op->ptr, prop, bake->margin_type);
+  }
+
   prop = RNA_struct_find_property(op->ptr, "use_selected_to_active");
   if (!RNA_property_is_set(op->ptr, prop)) {
     RNA_property_boolean_set(op->ptr, prop, (bake->flag & R_BAKE_TO_ACTIVE) != 0);
@@ -2008,6 +2028,12 @@ void OBJECT_OT_bake(wmOperatorType *ot)
               "Extends the baked result as a post process filter",
               0,
               64);
+  RNA_def_enum(ot->srna,
+               "margin_type",
+               rna_enum_bake_margin_type_items,
+               R_BAKE_EXTEND,
+               "Margin Type",
+               "Which algorithm to use to generate the margin");
   RNA_def_boolean(ot->srna,
                   "use_selected_to_active",
                   false,