Phase one of better masking support while rendering.

Problem: artist wants character to walk in grass, but still have all rendered
in seperate render-layers, for postpro effects and vblur. How to efficiently
create a mask image you can put *over* the character for the grass?

Solution has two parts; this commits allows any layer inside of the renderlayers
to become a Z-mask (Z values for solid gets filled in, but not rendered).

Second part of commit is render option "Only render stuff that's in front of
a zbuffer value that was filled in (saves render time)

7 files changed, 21 insertions, 14 deletions

diff --git a/source/blender/makesdna/DNA_scene_types.h b/source/blender/makesdna/DNA_scene_types.h
index b342bc5afe8..58c15ade575 100644
--- a/source/blender/makesdna/DNA_scene_types.h
+++ b/source/blender/makesdna/DNA_scene_types.h
@@ -120,7 +120,11 @@ typedef struct SceneRenderLayer {
 	struct Group *light_override;
 	
 	unsigned int lay;		/* scene->lay itself has priority over this */
+	unsigned int lay_zmask;	/* has to be after lay, this is for Z-masking */
 	int layflag;
+	
+	int pad;
+	
 	int passflag;			/* pass_xor has to be after passflag */
 	int pass_xor;
 } SceneRenderLayer;
diff --git a/source/blender/render/extern/include/RE_pipeline.h b/source/blender/render/extern/include/RE_pipeline.h
index 846b0867a76..de12f75300e 100644
--- a/source/blender/render/extern/include/RE_pipeline.h
+++ b/source/blender/render/extern/include/RE_pipeline.h
@@ -73,7 +73,7 @@ typedef struct RenderLayer {
 	
 	/* copy of RenderData */
 	char name[RE_MAXNAME];		
-	unsigned int lay;			
+	unsigned int lay, lay_zmask;
 	int layflag, passflag, pass_xor;		
 	
 	struct Material *mat_override;
diff --git a/source/blender/render/intern/include/zbuf.h b/source/blender/render/intern/include/zbuf.h
index 1d202fa45fa..8feb18a7ab8 100644
--- a/source/blender/render/intern/include/zbuf.h
+++ b/source/blender/render/intern/include/zbuf.h
@@ -48,7 +48,7 @@ void projectverto(float *v1, float winmat[][4], float *adr);
 int testclip(float *v); 
 
 void zbuffer_shadow(struct Render *re, float winmat[][4], struct LampRen *lar, int *rectz, int size, float jitx, float jity);
-void zbuffer_solid(struct RenderPart *pa, unsigned int layer, short layflag, void (*fillfunc)(struct RenderPart*, struct ZSpan*, int, void*), void *data);
+void zbuffer_solid(struct RenderPart *pa, struct RenderLayer *rl, void (*fillfunc)(struct RenderPart*, struct ZSpan*, int, void*), void *data);
 
 unsigned short *zbuffer_transp_shade(struct RenderPart *pa, struct RenderLayer *rl, float *pass, struct ListBase *psmlist);
 unsigned short *zbuffer_strands_shade(struct Render *re, struct RenderPart *pa, struct RenderLayer *rl, float *pass);
diff --git a/source/blender/render/intern/source/pipeline.c b/source/blender/render/intern/source/pipeline.c
index 43f0a0d13e7..2011be226fa 100644
--- a/source/blender/render/intern/source/pipeline.c
+++ b/source/blender/render/intern/source/pipeline.c
@@ -505,6 +505,7 @@ static RenderResult *new_render_result(Render *re, rcti *partrct, int crop, int
 		
 		strcpy(rl->name, srl->name);
 		rl->lay= srl->lay;
+		rl->lay_zmask= srl->lay_zmask;
 		rl->layflag= srl->layflag;
 		rl->passflag= srl->passflag;
 		rl->pass_xor= srl->pass_xor;
diff --git a/source/blender/render/intern/source/rendercore.c b/source/blender/render/intern/source/rendercore.c
index 50f3142f95a..531594e2ece 100644
--- a/source/blender/render/intern/source/rendercore.c
+++ b/source/blender/render/intern/source/rendercore.c
@@ -897,7 +897,7 @@ void zbufshadeDA_tile(RenderPart *pa)
 			sdata.rl= rl;
 			sdata.psmlist= &psmlist;
 			sdata.edgerect= edgerect;
-			zbuffer_solid(pa, rl->lay, rl->layflag, make_pixelstructs, &sdata);
+			zbuffer_solid(pa, rl, make_pixelstructs, &sdata);
 			if(R.test_break()) break; 
 		}
 		
@@ -1063,7 +1063,7 @@ void zbufshade_tile(RenderPart *pa)
 		shade_sample_initialize(&ssamp, pa, rl);
 		addpassflag= rl->passflag & ~(SCE_PASS_Z|SCE_PASS_COMBINED);
 		
-		zbuffer_solid(pa, rl->lay, rl->layflag, NULL, NULL);
+		zbuffer_solid(pa, rl, NULL, NULL);
 		
 		if(!R.test_break()) {	/* NOTE: this if() is not consistant */
 			
diff --git a/source/blender/render/intern/source/zbuf.c b/source/blender/render/intern/source/zbuf.c
index a8b6dfec667..3198b771815 100644
--- a/source/blender/render/intern/source/zbuf.c
+++ b/source/blender/render/intern/source/zbuf.c
@@ -1925,7 +1925,7 @@ void zbufclip4(ZSpan *zspan, int obi, int zvlnr, float *f1, float *f2, float *f3
 
 /* ***************** ZBUFFER MAIN ROUTINES **************** */
 
-void zbuffer_solid(RenderPart *pa, unsigned int lay, short layflag, void(*fillfunc)(RenderPart*, ZSpan*, int, void*), void *data)
+void zbuffer_solid(RenderPart *pa, RenderLayer *rl, void(*fillfunc)(RenderPart*, ZSpan*, int, void*), void *data)
 {
 	ZbufProjectCache cache[ZBUF_PROJECT_CACHE_SIZE];
 	ZSpan zspans[16], *zspan; /* 16 = RE_MAX_OSA */
@@ -1935,7 +1935,9 @@ void zbuffer_solid(RenderPart *pa, unsigned int lay, short layflag, void(*fillfu
 	ObjectInstanceRen *obi;
 	ObjectRen *obr;
 	float winmat[4][4], bounds[4], ho1[4], ho2[4], ho3[4], ho4[4]={0};
+	unsigned int lay= rl->lay, lay_zmask= rl->lay_zmask;
 	int i, v, zvlnr, zsample, samples, c1, c2, c3, c4=0;
+	short layflag= rl->layflag;
 	short nofill=0, env=0, wire=0, all_z= layflag & SCE_LAY_ALL_Z;
 
 	samples= (R.osa? R.osa: 1);
@@ -2016,7 +2018,7 @@ void zbuffer_solid(RenderPart *pa, unsigned int lay, short layflag, void(*fillfu
 					}
 				}
 			}
-			else if(all_z) {
+			else if(all_z || (obr->lay & lay_zmask)) {
 				env= 1;
 				nofill= 0;
 				ma= NULL; 
diff --git a/source/blender/src/buttons_scene.c b/source/blender/src/buttons_scene.c
index 0dc6c485db5..3ab19958405 100644
--- a/source/blender/src/buttons_scene.c
+++ b/source/blender/src/buttons_scene.c
@@ -2556,29 +2556,29 @@ static char *scene_layer_menu(void)
 	return str;
 }
 
-static void draw_3d_layer_buttons(uiBlock *block, unsigned int *poin, short xco, short yco, short dx, short dy)
+static void draw_3d_layer_buttons(uiBlock *block, int type, unsigned int *poin, short xco, short yco, short dx, short dy, char *tip)
 {
 	uiBut *bt;
 	long a;
 	
 	uiBlockBeginAlign(block);
 	for(a=0; a<5; a++) {
-		bt= uiDefButBitI(block, TOG, 1<<a, B_NOP, "",	(short)(xco+a*(dx/2)), yco+dy/2, (short)(dx/2), (short)(dy/2), (int *)poin, 0, 0, 0, 0, "");
+		bt= uiDefButBitI(block, type, 1<<a, B_NOP, "",	(short)(xco+a*(dx/2)), yco+dy/2, (short)(dx/2), (short)(dy/2), (int *)poin, 0, 0, 0, 0, tip);
 		uiButSetFunc(bt, layer_copy_func, (void *)a, poin);
 	}
 	for(a=0; a<5; a++) {
-		bt=uiDefButBitI(block, TOG, 1<<(a+10), B_NOP, "",	(short)(xco+a*(dx/2)), yco, (short)(dx/2), (short)(dy/2), (int *)poin, 0, 0, 0, 0, "");
+		bt=uiDefButBitI(block, type, 1<<(a+10), B_NOP, "",	(short)(xco+a*(dx/2)), yco, (short)(dx/2), (short)(dy/2), (int *)poin, 0, 0, 0, 0, tip);
 		uiButSetFunc(bt, layer_copy_func, (void *)(a+10), poin);
 	}
 	
 	xco+= 7;
 	uiBlockBeginAlign(block);
 	for(a=5; a<10; a++) {
-		bt=uiDefButBitI(block, TOG, 1<<a, B_NOP, "",	(short)(xco+a*(dx/2)), yco+dy/2, (short)(dx/2), (short)(dy/2), (int *)poin, 0, 0, 0, 0, "");
+		bt=uiDefButBitI(block, type, 1<<a, B_NOP, "",	(short)(xco+a*(dx/2)), yco+dy/2, (short)(dx/2), (short)(dy/2), (int *)poin, 0, 0, 0, 0, tip);
 		uiButSetFunc(bt, layer_copy_func, (void *)a, poin);
 	}
 	for(a=5; a<10; a++) {
-		bt=uiDefButBitI(block, TOG, 1<<(a+10), B_NOP, "",	(short)(xco+a*(dx/2)), yco, (short)(dx/2), (short)(dy/2), (int *)poin, 0, 0, 0, 0, "");
+		bt=uiDefButBitI(block, type, 1<<(a+10), B_NOP, "",	(short)(xco+a*(dx/2)), yco, (short)(dx/2), (short)(dy/2), (int *)poin, 0, 0, 0, 0, tip);
 		uiButSetFunc(bt, layer_copy_func, (void *)(a+10), poin);
 	}
 	
@@ -2603,7 +2603,7 @@ static void render_panel_layers(void)
 	
 	/* first, as reminder, the scene layers */
 	uiDefBut(block, LABEL, 0, "Scene:",				10,170,100,20, NULL, 0, 0, 0, 0, "");
-	draw_3d_layer_buttons(block, &G.scene->lay,		130, 170, 35, 30);
+	draw_3d_layer_buttons(block, TOG, &G.scene->lay,		130, 170, 35, 30, "Scene layers to render");
 	
 	/* layer disable, menu, name, delete button */
 	uiBlockBeginAlign(block);
@@ -2623,10 +2623,10 @@ static void render_panel_layers(void)
 
 	/* RenderLayer visible-layers */
 	uiDefBut(block, LABEL, 0, "Layer:",			10,110,100,20, NULL, 0, 0, 0, 0, "");
-	draw_3d_layer_buttons(block, &srl->lay,		130,110, 35, 30);
+	draw_3d_layer_buttons(block, BUT_TOGDUAL, &srl->lay,		130,110, 35, 30, "Scene-layers included in this render-layer (Hold CTRL for Z-mask)");
 	
 	uiBlockBeginAlign(block);
-	uiDefButBitI(block, TOG, SCE_LAY_ALL_Z, B_NOP,"AllZ",	10, 85, 40, 20, &srl->layflag, 0, 0, 0, 0, "Fill in Z values for all not-rendered faces, for masking");	
+	uiDefButBitI(block, TOG, SCE_LAY_ALL_Z, B_NOP,"AllZ",	10, 85, 40, 20, &srl->layflag, 0, 0, 0, 0, "Fill in Z values for faces in invisible layers, for masking");	
 	uiBlockBeginAlign(block);
 	uiDefButBitI(block, TOG, SCE_LAY_SOLID, B_NOP,"Solid",	50,  85, 45, 20, &srl->layflag, 0, 0, 0, 0, "Render Solid faces in this Layer");	
 	uiDefButBitI(block, TOG, SCE_LAY_HALO, B_NOP,"Halo",	95,  85, 40, 20, &srl->layflag, 0, 0, 0, 0, "Render Halos in this Layer (on top of Solid)");