Fix #27120: Bake to image used for rendering glitch (bake feedback loop)

Detect feedback loop and do not bake to images detected in this loop and show
nice warning message in such cases.

It's a way which wouldn't overcomplicate code trying to duplicate images and so
without real benefit.

3 files changed, 24 insertions, 2 deletions

diff --git a/source/blender/render/extern/include/RE_shader_ext.h b/source/blender/render/extern/include/RE_shader_ext.h
index dd0a0f308ce..2f585f91d44 100644
--- a/source/blender/render/extern/include/RE_shader_ext.h
+++ b/source/blender/render/extern/include/RE_shader_ext.h
@@ -210,4 +210,8 @@ int RE_bake_shade_all_selected(struct Render *re, int type, struct Object *actob
 struct Image *RE_bake_shade_get_image(void);
 void RE_bake_ibuf_filter(struct ImBuf *ibuf, char *mask, const int filter);
 
+#define BAKE_RESULT_OK			0
+#define BAKE_RESULT_NO_OBJECTS		1
+#define BAKE_RESULT_FEEDBACK_LOOP	2
+
 #endif /* RE_SHADER_EXT_H */
diff --git a/source/blender/render/intern/source/imagetexture.c b/source/blender/render/intern/source/imagetexture.c
index 0b6e1b8ee01..2d64828b6b3 100644
--- a/source/blender/render/intern/source/imagetexture.c
+++ b/source/blender/render/intern/source/imagetexture.c
@@ -131,6 +131,8 @@ int imagewrap(Tex *tex, Image *ima, ImBuf *ibuf, const float texvec[3], TexResul
 			return retval;
 		
 		ibuf= BKE_image_get_ibuf(ima, &tex->iuser);
+
+		ima->flag|= IMA_USED_FOR_RENDER;
 	}
 	if(ibuf==NULL || (ibuf->rect==NULL && ibuf->rect_float==NULL))
 		return retval;
@@ -1441,6 +1443,8 @@ int imagewraposa(Tex *tex, Image *ima, ImBuf *ibuf, const float texvec[3], const
 			return retval;
 		
 		ibuf= BKE_image_get_ibuf(ima, &tex->iuser); 
+
+		ima->flag|= IMA_USED_FOR_RENDER;
 	}
 	if(ibuf==NULL || (ibuf->rect==NULL && ibuf->rect_float==NULL))
 		return retval;
@@ -1812,6 +1816,8 @@ void image_sample(Image *ima, float fx, float fy, float dx, float dy, float *res
 	
 	if( (R.flag & R_SEC_FIELD) && (ibuf->flags & IB_fields) )
 		ibuf->rect-= (ibuf->x*ibuf->y);
+
+	ima->flag|= IMA_USED_FOR_RENDER;
 }
 
 void ibuf_sample(ImBuf *ibuf, float fx, float fy, float dx, float dy, float *result)
diff --git a/source/blender/render/intern/source/rendercore.c b/source/blender/render/intern/source/rendercore.c
index 773af96671a..2d3837acbfa 100644
--- a/source/blender/render/intern/source/rendercore.c
+++ b/source/blender/render/intern/source/rendercore.c
@@ -2440,6 +2440,11 @@ static int get_next_bake_face(BakeShade *bs)
 					if(ibuf->rect_float && !(ibuf->channels==0 || ibuf->channels==4))
 						continue;
 					
+					if(ima->flag & IMA_USED_FOR_RENDER) {
+						ima->id.flag &= ~LIB_DOIT;
+						continue;
+					}
+					
 					/* find the image for the first time? */
 					if(ima->id.flag & LIB_DOIT) {
 						ima->id.flag &= ~LIB_DOIT;
@@ -2584,7 +2589,7 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_up
 	BakeShade *handles;
 	ListBase threads;
 	Image *ima;
-	int a, vdone=0, usemask=0;
+	int a, vdone=0, usemask=0, result=BAKE_RESULT_OK;
 	
 	/* initialize render global */
 	R= *re;
@@ -2601,6 +2606,7 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_up
 	for(ima= G.main->image.first; ima; ima= ima->id.next) {
 		ImBuf *ibuf= BKE_image_get_ibuf(ima, NULL);
 		ima->id.flag |= LIB_DOIT;
+		ima->flag&= ~IMA_USED_FOR_RENDER;
 		if(ibuf) {
 			ibuf->userdata = NULL; /* use for masking if needed */
 			if(ibuf->rect_float)
@@ -2659,6 +2665,9 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_up
 		if((ima->id.flag & LIB_DOIT)==0) {
 			ImBuf *ibuf= BKE_image_get_ibuf(ima, NULL);
 
+			if(ima->flag & IMA_USED_FOR_RENDER)
+				result= BAKE_RESULT_FEEDBACK_LOOP;
+
 			if(!ibuf)
 				continue;
 
@@ -2679,7 +2688,10 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_up
 	
 	BLI_end_threads(&threads);
 
-	return vdone;
+	if(vdone==0)
+		result= BAKE_RESULT_NO_OBJECTS;
+
+	return result;
 }
 
 struct Image *RE_bake_shade_get_image(void)