Fix #27120: Bake to image used for rendering glitch (bake feedback loop)

Detect feedback loop and do not bake to images detected in this loop and show
nice warning message in such cases.

It's a way which wouldn't overcomplicate code trying to duplicate images and so
without real benefit.

5 files changed, 38 insertions, 8 deletions

diff --git a/source/blender/editors/object/object_bake.c b/source/blender/editors/object/object_bake.c
index 6ec778d7ee9..2fe2e71f2d9 100644
--- a/source/blender/editors/object/object_bake.c
+++ b/source/blender/editors/object/object_bake.c
@@ -1255,7 +1255,7 @@ typedef struct BakeRender {
 	Main *main;
 	Scene *scene;
 	struct Object *actob;
-	int tot, ready;
+	int result, ready;
 
 	ReportList *reports;
 
@@ -1328,7 +1328,7 @@ static void finish_bake_internal(BakeRender *bkr)
 		if(bkr->prev_r_raytrace == 0)
 			bkr->scene->r.mode &= ~R_RAYTRACE;
 
-	if(bkr->tot) {
+	if(bkr->result==BAKE_RESULT_OK) {
 		Image *ima;
 		/* force OpenGL reload and mipmap recalc */
 		for(ima= G.main->image.first; ima; ima= ima->id.next) {
@@ -1355,7 +1355,7 @@ static void *do_bake_render(void *bake_v)
 {
 	BakeRender *bkr= bake_v;
 
-	bkr->tot= RE_bake_shade_all_selected(bkr->re, bkr->scene->r.bake_mode, bkr->actob, NULL, bkr->progress);
+	bkr->result= RE_bake_shade_all_selected(bkr->re, bkr->scene->r.bake_mode, bkr->actob, NULL, bkr->progress);
 	bkr->ready= 1;
 
 	return NULL;
@@ -1377,7 +1377,7 @@ static void bake_startjob(void *bkv, short *stop, short *do_update, float *progr
 	RE_Database_Baking(bkr->re, bmain, scene, scene->lay, scene->r.bake_mode, bkr->actob);
 
 	/* baking itself is threaded, cannot use test_break in threads. we also update optional imagewindow */
-	bkr->tot= RE_bake_shade_all_selected(bkr->re, scene->r.bake_mode, bkr->actob, bkr->do_update, bkr->progress);
+	bkr->result= RE_bake_shade_all_selected(bkr->re, scene->r.bake_mode, bkr->actob, bkr->do_update, bkr->progress);
 }
 
 static void bake_update(void *bkv)
@@ -1396,7 +1396,11 @@ static void bake_freejob(void *bkv)
 	BakeRender *bkr= bkv;
 	finish_bake_internal(bkr);
 
-	if(bkr->tot==0) BKE_report(bkr->reports, RPT_ERROR, "No objects or images found to bake to");
+	if(bkr->result==BAKE_RESULT_NO_OBJECTS)
+		BKE_report(bkr->reports, RPT_ERROR, "No objects or images found to bake to");
+	else if(bkr->result==BAKE_RESULT_FEEDBACK_LOOP)
+		BKE_report(bkr->reports, RPT_WARNING, "Feedback loop detected");
+
 	MEM_freeN(bkr);
 	G.rendering = 0;
 }
@@ -1513,7 +1517,10 @@ static int bake_image_exec(bContext *C, wmOperator *op)
 			}
 			BLI_end_threads(&threads);
 
-			if(bkr.tot==0) BKE_report(op->reports, RPT_ERROR, "No valid images found to bake to");
+			if(bkr.result==BAKE_RESULT_NO_OBJECTS)
+				BKE_report(op->reports, RPT_ERROR, "No valid images found to bake to");
+			else if(bkr.result==BAKE_RESULT_FEEDBACK_LOOP)
+				BKE_report(op->reports, RPT_ERROR, "Feedback loop detected");
 
 			finish_bake_internal(&bkr);
 
diff --git a/source/blender/makesdna/DNA_image_types.h b/source/blender/makesdna/DNA_image_types.h
index 110ac59edc9..efe6c4d2586 100644
--- a/source/blender/makesdna/DNA_image_types.h
+++ b/source/blender/makesdna/DNA_image_types.h
@@ -120,6 +120,7 @@ typedef struct Image {
 #define IMA_DEPRECATED		64
 #define IMA_OLD_PREMUL		128
 #define IMA_CM_PREDIVIDE	256
+#define IMA_USED_FOR_RENDER	512
 
 /* Image.tpageflag */
 #define IMA_TILES			1
diff --git a/source/blender/render/extern/include/RE_shader_ext.h b/source/blender/render/extern/include/RE_shader_ext.h
index dd0a0f308ce..2f585f91d44 100644
--- a/source/blender/render/extern/include/RE_shader_ext.h
+++ b/source/blender/render/extern/include/RE_shader_ext.h
@@ -210,4 +210,8 @@ int RE_bake_shade_all_selected(struct Render *re, int type, struct Object *actob
 struct Image *RE_bake_shade_get_image(void);
 void RE_bake_ibuf_filter(struct ImBuf *ibuf, char *mask, const int filter);
 
+#define BAKE_RESULT_OK			0
+#define BAKE_RESULT_NO_OBJECTS		1
+#define BAKE_RESULT_FEEDBACK_LOOP	2
+
 #endif /* RE_SHADER_EXT_H */
diff --git a/source/blender/render/intern/source/imagetexture.c b/source/blender/render/intern/source/imagetexture.c
index 0b6e1b8ee01..2d64828b6b3 100644
--- a/source/blender/render/intern/source/imagetexture.c
+++ b/source/blender/render/intern/source/imagetexture.c
@@ -131,6 +131,8 @@ int imagewrap(Tex *tex, Image *ima, ImBuf *ibuf, const float texvec[3], TexResul
 			return retval;
 		
 		ibuf= BKE_image_get_ibuf(ima, &tex->iuser);
+
+		ima->flag|= IMA_USED_FOR_RENDER;
 	}
 	if(ibuf==NULL || (ibuf->rect==NULL && ibuf->rect_float==NULL))
 		return retval;
@@ -1441,6 +1443,8 @@ int imagewraposa(Tex *tex, Image *ima, ImBuf *ibuf, const float texvec[3], const
 			return retval;
 		
 		ibuf= BKE_image_get_ibuf(ima, &tex->iuser); 
+
+		ima->flag|= IMA_USED_FOR_RENDER;
 	}
 	if(ibuf==NULL || (ibuf->rect==NULL && ibuf->rect_float==NULL))
 		return retval;
@@ -1812,6 +1816,8 @@ void image_sample(Image *ima, float fx, float fy, float dx, float dy, float *res
 	
 	if( (R.flag & R_SEC_FIELD) && (ibuf->flags & IB_fields) )
 		ibuf->rect-= (ibuf->x*ibuf->y);
+
+	ima->flag|= IMA_USED_FOR_RENDER;
 }
 
 void ibuf_sample(ImBuf *ibuf, float fx, float fy, float dx, float dy, float *result)
diff --git a/source/blender/render/intern/source/rendercore.c b/source/blender/render/intern/source/rendercore.c
index 773af96671a..2d3837acbfa 100644
--- a/source/blender/render/intern/source/rendercore.c
+++ b/source/blender/render/intern/source/rendercore.c
@@ -2440,6 +2440,11 @@ static int get_next_bake_face(BakeShade *bs)
 					if(ibuf->rect_float && !(ibuf->channels==0 || ibuf->channels==4))
 						continue;
 					
+					if(ima->flag & IMA_USED_FOR_RENDER) {
+						ima->id.flag &= ~LIB_DOIT;
+						continue;
+					}
+					
 					/* find the image for the first time? */
 					if(ima->id.flag & LIB_DOIT) {
 						ima->id.flag &= ~LIB_DOIT;
@@ -2584,7 +2589,7 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_up
 	BakeShade *handles;
 	ListBase threads;
 	Image *ima;
-	int a, vdone=0, usemask=0;
+	int a, vdone=0, usemask=0, result=BAKE_RESULT_OK;
 	
 	/* initialize render global */
 	R= *re;
@@ -2601,6 +2606,7 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_up
 	for(ima= G.main->image.first; ima; ima= ima->id.next) {
 		ImBuf *ibuf= BKE_image_get_ibuf(ima, NULL);
 		ima->id.flag |= LIB_DOIT;
+		ima->flag&= ~IMA_USED_FOR_RENDER;
 		if(ibuf) {
 			ibuf->userdata = NULL; /* use for masking if needed */
 			if(ibuf->rect_float)
@@ -2659,6 +2665,9 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_up
 		if((ima->id.flag & LIB_DOIT)==0) {
 			ImBuf *ibuf= BKE_image_get_ibuf(ima, NULL);
 
+			if(ima->flag & IMA_USED_FOR_RENDER)
+				result= BAKE_RESULT_FEEDBACK_LOOP;
+
 			if(!ibuf)
 				continue;
 
@@ -2679,7 +2688,10 @@ int RE_bake_shade_all_selected(Render *re, int type, Object *actob, short *do_up
 	
 	BLI_end_threads(&threads);
 
-	return vdone;
+	if(vdone==0)
+		result= BAKE_RESULT_NO_OBJECTS;
+
+	return result;
 }
 
 struct Image *RE_bake_shade_get_image(void)