Plumiferos fix-fix :)

Code to allow "Env" material to mask out ztransp gave bad AA on edges of
solid faces, when transparant was behind it.

Recoded "Env" to use index -1 in the polygon index buffer, and restored
code that caused the bad AA.

4 files changed, 18 insertions, 23 deletions

diff --git a/source/blender/render/intern/include/render_types.h b/source/blender/render/intern/include/render_types.h
index 5a655c089fd..ab30935276c 100644
--- a/source/blender/render/intern/include/render_types.h
+++ b/source/blender/render/intern/include/render_types.h
@@ -65,7 +65,7 @@ typedef struct RenderPart
 	/* result of part rendering */
 	RenderResult *result;
 	
-	unsigned int *rectp;			/* polygon index table */
+	int *rectp;			/* polygon index table */
 	int *rectz;						/* zbuffer */
 	long *rectdaps;					/* delta acum buffer for pixel structs */
 	
diff --git a/source/blender/render/intern/source/rendercore.c b/source/blender/render/intern/source/rendercore.c
index 7cdf05c4958..d75cd78da07 100644
--- a/source/blender/render/intern/source/rendercore.c
+++ b/source/blender/render/intern/source/rendercore.c
@@ -2359,9 +2359,6 @@ void *shadepixel(ShadePixelInfo *shpi, float x, float y, int z, volatile int fac
 	ShadeInput shi;
 	VlakRen *vlr=NULL;
 	
-	if(facenr< 0) {	/* error */
-		return NULL;
-	}
 	/* currently in use for dithering (soft shadow) node preview */
 	shi.xs= (int)(x+0.5f);
 	shi.ys= (int)(y+0.5f);
@@ -2374,7 +2371,7 @@ void *shadepixel(ShadePixelInfo *shpi, float x, float y, int z, volatile int fac
 	shi.mask= mask;
 	shi.depth= 0;	// means first hit, not raytracing
 	
-	if(facenr==0) {	/* sky */
+	if(facenr<=0) {	/* sky or env */
 		memset(shr, 0, sizeof(ShadeResult));
 		rco[0]= rco[1]= rco[2]= 0.0f;
 	}
@@ -2639,7 +2636,7 @@ void *shadepixel(ShadePixelInfo *shpi, float x, float y, int z, volatile int fac
 	if(R.flag & R_LAMPHALO) {
 		if(shpi->layflag & SCE_LAY_HALO) {
 
-			if(facenr<=0) {	/* calc view vector and put shi.co at far */
+			if(facenr<=0) {	/* sky or env, calc view vector and put shi.co at far */
 				if(R.r.mode & R_ORTHO) {
 					/* x and y 3d coordinate can be derived from pixel coord and winmat */
 					float fx= 2.0/(R.rectx*R.winmat[0][0]);
@@ -2947,7 +2944,7 @@ static void shadeDA_tile(RenderPart *pa, RenderLayer *rl)
 				}
 				
 				/* check osa level */
-				if(face==0) full_osa= 0;
+				if(face<=0) full_osa= 0;
 				else {
 					VlakRen *vlr= RE_findOrAddVlak(&R, (face-1) & RE_QUAD_MASK);
 					full_osa= (vlr->flag & R_FULL_OSA);
diff --git a/source/blender/render/intern/source/zbuf.c b/source/blender/render/intern/source/zbuf.c
index dd6d15eea2e..74375c409bd 100644
--- a/source/blender/render/intern/source/zbuf.c
+++ b/source/blender/render/intern/source/zbuf.c
@@ -1675,7 +1675,7 @@ void zbuffer_solid(RenderPart *pa, unsigned int lay, short layflag)
 				
 				if((partclip & clipmask)==0) {
 					
-					if(env) zvlnr= 0;
+					if(env) zvlnr= -1;
 					else zvlnr= v+1;
 					
 					if(wire) zbufclipwire(&zspan, zvlnr, vlr);
@@ -1687,7 +1687,7 @@ void zbuffer_solid(RenderPart *pa, unsigned int lay, short layflag)
 						else {
 							zbufclip(&zspan, zvlnr, vlr->v1->ho, vlr->v2->ho, vlr->v3->ho, vlr->v1->clip, vlr->v2->clip, vlr->v3->clip);
 							if(vlr->v4) {
-								if(zvlnr) zvlnr+= RE_QUAD_OFFS;
+								if(zvlnr>0) zvlnr+= RE_QUAD_OFFS;
 								zbufclip(&zspan, zvlnr, vlr->v1->ho, vlr->v3->ho, vlr->v4->ho, vlr->v1->clip, vlr->v3->clip, vlr->v4->clip);
 							}
 						}
@@ -2355,13 +2355,11 @@ static void copyto_abufz(RenderPart *pa, int *arectz, int sample)
 	int x, y, *rza;
 	long *rd;
 	
-	/* now, first copy existing zbuffer (has env z values too!
-	   in OSA the pixstructs contain all faces filled in */
+	if(R.osa==0) {
+		memcpy(arectz, pa->rectz, 4*pa->rectx*pa->recty);
+		return;
+	}
 	
-	memcpy(arectz, pa->rectz, 4*pa->rectx*pa->recty);
-
-	if(R.osa==0) return;
-		
 	rza= arectz;
 	rd= pa->rectdaps;
 
@@ -2370,16 +2368,14 @@ static void copyto_abufz(RenderPart *pa, int *arectz, int sample)
 	for(y=0; y<pa->recty; y++) {
 		for(x=0; x<pa->rectx; x++) {
 			
+			*rza= 0x7FFFFFFF;
 			if(*rd) {	
-				ps= (PixStr *)(*rd);
-
-				while(ps) {
+				/* when there's a sky pixstruct, fill in sky-Z, otherwise solid Z */
+				for(ps= (PixStr *)(*rd); ps; ps= ps->next) {
 					if(sample & ps->mask) {
-						//printf("filled xy %d %d mask %d\n", x, y, sample);
 						*rza= ps->z;
 						break;
 					}
-					ps= ps->next;
 				}
 			}
 			
@@ -2595,7 +2591,9 @@ static void shadetrapixel(ShadePixelInfo *shpi, float x, float y, int z, int fac
 {
 	float rco[3];
 	
-	if( (facenr & RE_QUAD_MASK) > R.totvlak) {
+	if(facenr<0)
+		return;
+	else if( (facenr & RE_QUAD_MASK) > R.totvlak) {
 		printf("error in shadetrapixel nr: %d\n", (facenr & RE_QUAD_MASK));
 		return;
 	}
diff --git a/source/blender/src/buttons_shading.c b/source/blender/src/buttons_shading.c
index 77b4519198b..f7270b05342 100644
--- a/source/blender/src/buttons_shading.c
+++ b/source/blender/src/buttons_shading.c
@@ -3091,8 +3091,8 @@ static void material_panel_material(Material *ma)
 			uiDefButBitI(block, TOG, MA_VERTEXCOLP, B_REDR, "VCol Paint",	82,166,74,20, &(ma->mode), 0, 0, 0, 0, "Replaces material's colours with vertex colours");
 			uiDefButBitI(block, TOG, MA_FACETEXTURE, B_REDR, "TexFace",		156,166,74,20, &(ma->mode), 0, 0, 0, 0, "Sets UV-Editor assigned texture as color and texture info for faces");
 			uiDefButBitI(block, TOG, MA_SHLESS, B_MATPRV, "Shadeless",	230,166,73,20, &(ma->mode), 0, 0, 0, 0, "Makes material insensitive to light or shadow");
-			uiDefButBitI(block, TOG, MA_NOMIST, 0,	"No Mist",	8,146,146,20, &(ma->mode), 0, 0, 0, 0, "Sets the material to ignore mist values");
-			uiDefButBitI(block, TOG, MA_ENV, 0,	"Env",			158,146,145,20, &(ma->mode), 0, 0, 0, 0, "Causes faces to render with alpha zero: allows sky/backdrop to show through (only for solid faces)");
+			uiDefButBitI(block, TOG, MA_NOMIST, B_NOP,	"No Mist",	8,146,146,20, &(ma->mode), 0, 0, 0, 0, "Sets the material to ignore mist values");
+			uiDefButBitI(block, TOG, MA_ENV, B_MATPRV,	"Env",			158,146,145,20, &(ma->mode), 0, 0, 0, 0, "Causes faces to render with alpha zero: allows sky/backdrop to show through (only for solid faces)");
 		}
 		uiBlockSetCol(block, TH_AUTO);
 		uiBlockBeginAlign(block);