* Bakeing fix for apricot, stop pixels being extended for parts of the image that are not being baked (useful for multiple objects)

* Added support for baking alpha, as well as extending alpha margins.

4 files changed, 160 insertions, 22 deletions

diff --git a/source/blender/imbuf/IMB_imbuf.h b/source/blender/imbuf/IMB_imbuf.h
index dbbddd2a070..7cbdfbe32d1 100644
--- a/source/blender/imbuf/IMB_imbuf.h
+++ b/source/blender/imbuf/IMB_imbuf.h
@@ -321,7 +321,7 @@ void IMB_antialias(struct ImBuf * ibuf);
  */
 void IMB_filter(struct ImBuf *ibuf);
 void IMB_filterN(struct ImBuf *out, struct ImBuf *in);
-void IMB_filter_extend(struct ImBuf *ibuf);
+void IMB_filter_extend(struct ImBuf *ibuf, char *mask);
 void IMB_makemipmap(struct ImBuf *ibuf, int use_filter);
 
 /**
diff --git a/source/blender/imbuf/IMB_imbuf_types.h b/source/blender/imbuf/IMB_imbuf_types.h
index 91ffa188fcc..8a642a79889 100644
--- a/source/blender/imbuf/IMB_imbuf_types.h
+++ b/source/blender/imbuf/IMB_imbuf_types.h
@@ -95,7 +95,7 @@ typedef struct ImBuf {
 	int	userflags;			/**< Used to set imbuf to Dirty and other stuff */
 	int	*zbuf;				/**< z buffer data, original zbuffer */
 	float *zbuf_float;		/**< z buffer data, camera coordinates */
-	void *userdata;	
+	void *userdata;			/**< temporary storage, only used by baking at the moment */
 	unsigned char *encodedbuffer;     /**< Compressed image only used with png currently */
 	unsigned int   encodedsize;       /**< Size of data written to encodedbuffer */
 	unsigned int   encodedbuffersize; /**< Size of encodedbuffer */
diff --git a/source/blender/imbuf/intern/filter.c b/source/blender/imbuf/intern/filter.c
index 156e26fd5d6..1be5cf99e19 100644
--- a/source/blender/imbuf/intern/filter.c
+++ b/source/blender/imbuf/intern/filter.c
@@ -240,13 +240,16 @@ void IMB_filter(struct ImBuf *ibuf)
 	imb_filterx(ibuf);
 }
 
-#define EXTEND_PIXEL(a, w)	if((a)[3]) {r+= w*(a)[0]; g+= w*(a)[1]; b+= w*(a)[2]; tot+=w;}
+#define EXTEND_PIXEL(color, w) if((color)[3]) {r+= w*(color)[0]; g+= w*(color)[1]; b+= w*(color)[2]; a+= w*(color)[3]; tot+=w;}
 
-/* if alpha is zero, it checks surrounding pixels and averages color. sets new alphas to 1.0 */
-void IMB_filter_extend(struct ImBuf *ibuf)
+/* if alpha is zero, it checks surrounding pixels and averages color. sets new alphas to 1.0
+ * 
+ * When a mask is given, only effect pixels with a mask value of 1, defined as BAKE_MASK_MARGIN in rendercore.c
+ * */
+void IMB_filter_extend(struct ImBuf *ibuf, char *mask)
 {
 	register char *row1, *row2, *row3;
-	register char *cp;
+	register char *cp; 
 	int rowlen, x, y;
 	
 	rowlen= ibuf->x;
@@ -269,11 +272,11 @@ void IMB_filter_extend(struct ImBuf *ibuf)
 				row3f= row2f;
 			
 			fp= (float *)(ibuf->rect_float + (y-1)*rowlen*4);
-			
+				
 			for(x=0; x<rowlen; x++) {
-				if(fp[3]==0.0f) {
+				if((mask==NULL && fp[3]==0.0f) || (mask && mask[((y-1)*rowlen)+x]==1)) {
 					int tot= 0;
-					float r=0.0f, g=0.0f, b=0.0f;
+					float r=0.0f, g=0.0f, b=0.0f, a=0.0f;
 					
 					EXTEND_PIXEL(row1f, 1);
 					EXTEND_PIXEL(row2f, 2);
@@ -289,7 +292,7 @@ void IMB_filter_extend(struct ImBuf *ibuf)
 						fp[0]= r/tot;
 						fp[1]= g/tot;
 						fp[2]= b/tot;
-						fp[3]= 1.0;
+						fp[3]= a/tot;
 					}
 				}
 				fp+=4; 
@@ -321,8 +324,9 @@ void IMB_filter_extend(struct ImBuf *ibuf)
 			cp= (char *)(ibuf->rect + (y-1)*rowlen);
 			
 			for(x=0; x<rowlen; x++) {
-				if(cp[3]==0) {
-					int tot= 0, r=0, g=0, b=0;
+				/*if(cp[3]==0) {*/
+				if((mask==NULL && cp[3]==0) || (mask && mask[((y-1)*rowlen)+x]==1)) {
+					int tot= 0, r=0, g=0, b=0, a=0;
 					
 					EXTEND_PIXEL(row1, 1);
 					EXTEND_PIXEL(row2, 2);
@@ -338,10 +342,10 @@ void IMB_filter_extend(struct ImBuf *ibuf)
 						cp[0]= r/tot;
 						cp[1]= g/tot;
 						cp[2]= b/tot;
-						cp[3]= 255;
+						cp[3]= a/tot;
 					}
 				}
-				cp+=4; 
+				cp+=4;
 				
 				if(x!=0) {
 					row1+=4; row2+=4; row3+=4;
diff --git a/source/blender/render/intern/source/rendercore.c b/source/blender/render/intern/source/rendercore.c
index 32210080acb..26a237139a3 100644
--- a/source/blender/render/intern/source/rendercore.c
+++ b/source/blender/render/intern/source/rendercore.c
@@ -1797,8 +1797,79 @@ typedef struct BakeShade {
 	
 	unsigned int *rect;
 	float *rect_float;
+	
+	int usemask;
+	char *rect_mask; /* bake pixel mask */
 } BakeShade;
 
+/* bake uses a char mask to know what has been baked */
+#define BAKE_MASK_NULL		0
+#define BAKE_MASK_MARGIN	1
+#define BAKE_MASK_BAKED		2
+static void bake_mask_filter_extend( char *mask, int width, int height )
+{
+	char *row1, *row2, *row3;
+	int rowlen, x, y;
+	char *temprect;
+	
+	rowlen= width;
+	
+	/* make a copy, to prevent flooding */
+	temprect= MEM_dupallocN(mask);
+	
+	for(y=1; y<=height; y++) {
+		/* setup rows */
+		row1= (char *)(temprect + (y-2)*rowlen);
+		row2= row1 + rowlen;
+		row3= row2 + rowlen;
+		if(y==1)
+			row1= row2;
+		else if(y==height)
+			row3= row2;
+		
+		for(x=0; x<rowlen; x++) {
+			if (mask[((y-1)*rowlen)+x]==0) {
+				if (*row1 || *row2 || *row3 || *(row1+1) || *(row3+1) ) {
+					mask[((y-1)*rowlen)+x] = BAKE_MASK_MARGIN;
+				} else if((x!=rowlen-1) && (*(row1+2) || *(row2+2) || *(row3+2)) ) {
+					mask[((y-1)*rowlen)+x] = BAKE_MASK_MARGIN;
+				}
+			}
+			
+			if(x!=0) {
+				row1++; row2++; row3++;
+			}
+		}
+	}
+	MEM_freeN(temprect);
+}
+
+static void bake_mask_clear( ImBuf *ibuf, char *mask, char val )
+{
+	int x,y;
+	if (ibuf->rect_float) {
+		for(x=0; x<ibuf->x; x++) {
+			for(y=0; y<ibuf->y; y++) {
+				if (mask[ibuf->x*y + x] == val) {
+					float *col= ibuf->rect_float + 4*(ibuf->x*y + x);
+					col[0] = col[1] = col[2] = col[3] = 0.0f;
+				}
+			}
+		}
+		
+	} else {
+		/* char buffer */
+		for(x=0; x<ibuf->x; x++) {
+			for(y=0; y<ibuf->y; y++) {
+				if (mask[ibuf->x*y + x] == val) {
+					char *col= (char *)(ibuf->rect + ibuf->x*y + x);
+					col[0] = col[1] = col[2] = col[3] = 0;
+				}
+			}
+		}
+	}
+}
+
 static void bake_set_shade_input(ObjectInstanceRen *obi, VlakRen *vlr, ShadeInput *shi, int quad, int isect, int x, int y, float u, float v)
 {
 	if(isect) {
@@ -1918,14 +1989,28 @@ static void bake_shade(void *handle, Object *ob, ShadeInput *shi, int quad, int
 	if(bs->rect_float) {
 		float *col= bs->rect_float + 4*(bs->rectx*y + x);
 		VECCOPY(col, shr.combined);
-		col[3]= 1.0f;
+		if (bs->type==RE_BAKE_ALL) {
+			col[3]= shr.alpha;
+		} else {
+			col[3]= 1.0;
+		}
 	}
 	else {
 		char *col= (char *)(bs->rect + bs->rectx*y + x);
 		col[0]= FTOCHAR(shr.combined[0]);
 		col[1]= FTOCHAR(shr.combined[1]);
 		col[2]= FTOCHAR(shr.combined[2]);
-		col[3]= 255;
+		
+		
+		if (bs->type==RE_BAKE_ALL) {
+			col[3]= FTOCHAR(shr.alpha);
+		} else {
+			col[3]= 255;
+		}
+	}
+	
+	if (bs->rect_mask) {
+		bs->rect_mask[bs->rectx*y + x] = BAKE_MASK_BAKED;
 	}
 }
 
@@ -1951,6 +2036,9 @@ static void bake_displacement(void *handle, ShadeInput *shi, float dist, int x,
 		col[2]= FTOCHAR(disp);
 		col[3]= 255;
 	}
+	if (bs->rect_mask) {
+		bs->rect_mask[bs->rectx*y + x] = BAKE_MASK_BAKED;
+	}
 }
 
 static int bake_check_intersect(Isect *is, int ob, RayFace *face)
@@ -2191,6 +2279,13 @@ static void shade_tface(BakeShade *bs)
 	bs->rect_float= bs->ibuf->rect_float;
 	bs->quad= 0;
 	
+	if (bs->usemask) {
+		if (bs->ibuf->userdata==NULL) {
+			bs->ibuf->userdata = (void *)MEM_callocN(sizeof(char)*bs->rectx*bs->recty, "BakeMask");
+			bs->rect_mask= (char *)bs->ibuf->userdata;
+		}
+	}
+	
 	/* get pixel level vertex coordinates */
 	for(a=0; a<4; a++) {
 		vec[a][0]= tface->uv[a][0]*(float)bs->rectx - 0.5f;
@@ -2232,8 +2327,8 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob)
 	BakeShade handles[BLENDER_MAX_THREADS];
 	ListBase threads;
 	Image *ima;
-	int a, vdone=0;
-
+	int a, vdone=0, usemask=0;
+	
 	/* initialize render global */
 	R= *re;
 	R.bakebuf= NULL;
@@ -2241,9 +2336,18 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob)
 	/* initialize static vars */
 	get_next_bake_face(NULL);
 	
+	/* do we need a mask? */
+	
+	/*if ((re->r.bake_mode==RE_BAKE_ALL) && (re->r.bake_filter) && (re->r.bake_flag & R_BAKE_CLEAR)==0)*/
+	if (re->r.bake_filter && (re->r.bake_flag & R_BAKE_CLEAR)==0)
+		usemask = 1;
+	
 	/* baker uses this flag to detect if image was initialized */
-	for(ima= G.main->image.first; ima; ima= ima->id.next)
+	for(ima= G.main->image.first; ima; ima= ima->id.next) {
+		ImBuf *ibuf= BKE_image_get_ibuf(ima, NULL);
 		ima->id.flag |= LIB_DOIT;
+		ibuf->userdata = NULL; /* use for masking if needed */
+	}
 	
 	BLI_init_threads(&threads, do_bake_thread, re->r.threads);
 
@@ -2262,6 +2366,8 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob)
 		handles[a].actob= actob;
 		handles[a].zspan= MEM_callocN(sizeof(ZSpan), "zspan for bake");
 		
+		handles[a].usemask = usemask;
+		
 		BLI_insert_thread(&threads, &handles[a]);
 	}
 	
@@ -2280,8 +2386,36 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob)
 	for(ima= G.main->image.first; ima; ima= ima->id.next) {
 		if((ima->id.flag & LIB_DOIT)==0) {
 			ImBuf *ibuf= BKE_image_get_ibuf(ima, NULL);
-			for(a=0; a<re->r.bake_filter; a++)
-				IMB_filter_extend(ibuf);
+			if (re->r.bake_filter) {
+				if (usemask) {
+					/* extend the mask +2 pixels from the image,
+					 * this is so colors dont blend in from outside */
+					char *temprect;
+					
+					for(a=0; a<re->r.bake_filter; a++)
+						bake_mask_filter_extend((char *)ibuf->userdata, ibuf->x, ibuf->y);
+					
+					temprect = MEM_dupallocN(ibuf->userdata);
+					
+					/* expand twice to clear this many pixels, so they blend back in */
+					bake_mask_filter_extend(temprect, ibuf->x, ibuf->y);
+					bake_mask_filter_extend(temprect, ibuf->x, ibuf->y);
+					
+					/* clear all pixels in the margin*/
+					bake_mask_clear(ibuf, temprect, BAKE_MASK_MARGIN);
+					MEM_freeN(temprect);
+				}
+				
+				for(a=0; a<re->r.bake_filter; a++) {
+					/*the mask, ibuf->userdata - can be null, in this case only zero alpha is used */
+					IMB_filter_extend(ibuf, (char *)ibuf->userdata);
+				}
+				
+				if (ibuf->userdata) {
+					MEM_freeN(ibuf->userdata);
+					ibuf->userdata= NULL;
+				}
+			}
 			ibuf->userflags |= IB_BITMAPDIRTY;
 			if (ibuf->rect_float) IMB_rect_from_float(ibuf);
 		}
@@ -2293,7 +2427,7 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob)
 		
 		zbuf_free_span(handles[a].zspan);
 		MEM_freeN(handles[a].zspan);
-	}
+ 	}
 	
 	BLI_end_threads(&threads);
 	return vdone;