Alpha premul pipeline cleanup

This assumptions are now made:
- Internally float buffers are always linear alpha-premul colors
- Readers should worry about delivering float buffers with that
  assumptions.
- There's an input image setting to say whether it's stored with
  straight/premul alpha on the disk.
- Byte buffers are now assumed have straight alpha, readers should
  deliver straight alpha.

Some implementation details:

- Removed scene's color unpremultiply setting, which was very
  much confusing and was wrong for default settings.
  Now all renderers assumes to deliver premultiplied alpha.

- IMB_buffer_byte_from_float will now linearize alpha when
  converting from buffer.

- Sequencer's effects were changed to assume bytes have got
  straight alpha. Most of effects will work with bytes still,
  however for glow it was more tricky to avoid data loss, so
  there's a commented out glow implementation which converts
  byte buffer to floats first, operates on floats and returns
  bytes back. It's slower and not sure if it should actually
  be used -- who're using glow on alpha anyway?

- Sequencer modifiers should also be working nice with straight
  bytes now.

- GLSL preview will predivide float textures to make nice shading,
  shading with byte textures worked nice (GLSL was assuming straight
  alpha).

- Blender Internal will set alpha=1 to the whole sky. The same
  happens in Cycles and there's no way to avoid this -- sky is
  neither straight nor premul and doesn't fit color pipeline well.

- Straight alpha mode for render result was also eliminated.

- Conversion to correct alpha need to be done before linearizing
  float buffer.

- TIFF will now load and save files with proper alpha mode setting
  in file meta data header.

- Remove Use Alpha from texture mapping and replaced with image
  datablock setting.

  Behaves much more predictable and clear from code point of view
  and solves possible regressions when non-premultiplied images were
  used as textures with ignoring alpha channel.

5 files changed, 25 insertions, 57 deletions

diff --git a/source/blender/render/intern/include/render_result.h b/source/blender/render/intern/include/render_result.h
index 5d61417cbaf..61b39a59b0b 100644
--- a/source/blender/render/intern/include/render_result.h
+++ b/source/blender/render/intern/include/render_result.h
@@ -91,7 +91,7 @@ void render_result_rect_from_ibuf(struct RenderResult *rr, struct RenderData *rd
 	struct ImBuf *ibuf);
 
 void render_result_rect_fill_zero(struct RenderResult *rr);
-void render_result_rect_get_pixels(struct RenderResult *rr, struct RenderData *rd,
+void render_result_rect_get_pixels(struct RenderResult *rr,
 	unsigned int *rect, int rectx, int recty,
 	const struct ColorManagedViewSettings *view_settings,
 	const struct ColorManagedDisplaySettings *display_settings);
diff --git a/source/blender/render/intern/source/imagetexture.c b/source/blender/render/intern/source/imagetexture.c
index 7c14e0e5465..4aaa6247478 100644
--- a/source/blender/render/intern/source/imagetexture.c
+++ b/source/blender/render/intern/source/imagetexture.c
@@ -102,6 +102,11 @@ static void ibuf_get_color(float col[4], struct ImBuf *ibuf, int x, int y)
 		col[1] = ((float)rect[1])*(1.0f/255.0f);
 		col[2] = ((float)rect[2])*(1.0f/255.0f);
 		col[3] = ((float)rect[3])*(1.0f/255.0f);
+
+		/* bytes are internally straight, however render pipeline seems to expect premul */
+		col[0] *= col[3];
+		col[1] *= col[3];
+		col[2] *= col[3];
 	}
 }
 
@@ -219,10 +224,8 @@ int imagewrap(Tex *tex, Image *ima, ImBuf *ibuf, const float texvec[3], TexResul
 	}
 
 	/* keep this before interpolation [#29761] */
-	if (tex->imaflag & TEX_USEALPHA) {
-		if ((tex->imaflag & TEX_CALCALPHA) == 0) {
-			texres->talpha = TRUE;
-		}
+	if ((tex->imaflag & TEX_CALCALPHA) == 0) {
+		texres->talpha = TRUE;
 	}
 
 	/* interpolate */
@@ -710,9 +713,10 @@ static int ibuf_get_color_clip(float col[4], ImBuf *ibuf, int x, int y, int extf
 	}
 	else {
 		char *rect = (char *)(ibuf->rect + x + y*ibuf->x);
-		col[0] = rect[0]*(1.f/255.f);
-		col[1] = rect[1]*(1.f/255.f);
-		col[2] = rect[2]*(1.f/255.f);
+		float inv_alpha_fac = (1.0f / 255.0f) * rect[3] * (1.0f / 255.0f);
+		col[0] = rect[0] * inv_alpha_fac;
+		col[1] = rect[1] * inv_alpha_fac;
+		col[2] = rect[2] * inv_alpha_fac;
 		col[3] = clip ? 0.f : rect[3]*(1.f/255.f);
 	}
 	return clip;
@@ -1088,7 +1092,8 @@ static int imagewraposa_aniso(Tex *tex, Image *ima, ImBuf *ibuf, const float tex
 	/* mipmap test */
 	image_mipmap_test(tex, ibuf);
 	
-	if ((tex->imaflag & TEX_USEALPHA) && ((tex->imaflag & TEX_CALCALPHA) == 0)) texres->talpha = 1;
+	if ((tex->imaflag & TEX_CALCALPHA) == 0)
+		texres->talpha = 1;
 	texr.talpha = texres->talpha;
 
 	if (tex->imaflag & TEX_IMAROT) {
@@ -1501,13 +1506,8 @@ int imagewraposa(Tex *tex, Image *ima, ImBuf *ibuf, const float texvec[3], const
 	/* mipmap test */
 	image_mipmap_test(tex, ibuf);
 
-	if (tex->imaflag & TEX_USEALPHA) {
-		if (tex->imaflag & TEX_CALCALPHA) {
-			/* pass */
-		}
-		else {
-			texres->talpha = TRUE;
-		}
+	if ((tex->imaflag & TEX_CALCALPHA) == 0) {
+		texres->talpha = TRUE;
 	}
 	
 	texr.talpha= texres->talpha;
diff --git a/source/blender/render/intern/source/pipeline.c b/source/blender/render/intern/source/pipeline.c
index 0545d2940ab..f2db84c47a9 100644
--- a/source/blender/render/intern/source/pipeline.c
+++ b/source/blender/render/intern/source/pipeline.c
@@ -336,7 +336,7 @@ void RE_ResultGet32(Render *re, unsigned int *rect)
 	RenderResult rres;
 	
 	RE_AcquireResultImage(re, &rres);
-	render_result_rect_get_pixels(&rres, &re->r, rect, re->rectx, re->recty, &re->scene->view_settings, &re->scene->display_settings);
+	render_result_rect_get_pixels(&rres, rect, re->rectx, re->recty, &re->scene->view_settings, &re->scene->display_settings);
 	RE_ReleaseResultImage(re);
 }
 
@@ -833,7 +833,7 @@ static void threaded_tile_processor(Render *re)
 	
 	if (re->result == NULL)
 		return;
-	
+
 	/* warning; no return here without closing exr file */
 	
 	RE_parts_init(re, TRUE);
@@ -1092,7 +1092,7 @@ static void do_render_blur_3d(Render *re)
 		
 		blurfac = 1.0f / (float)(re->r.mblur_samples - blur);
 		
-		merge_renderresult_blur(rres, re->result, blurfac, re->r.alphamode & R_ALPHAKEY);
+		merge_renderresult_blur(rres, re->result, blurfac, FALSE);
 		if (re->test_break(re->tbh)) break;
 	}
 	
diff --git a/source/blender/render/intern/source/render_result.c b/source/blender/render/intern/source/render_result.c
index f8281586038..78750cfd1ca 100644
--- a/source/blender/render/intern/source/render_result.c
+++ b/source/blender/render/intern/source/render_result.c
@@ -434,7 +434,7 @@ RenderResult *render_result_new(Render *re, rcti *partrct, int crop, int savebuf
 	rr->renrect.xmin = 0; rr->renrect.xmax = rectx - 2 * crop;
 	/* crop is one or two extra pixels rendered for filtering, is used for merging and display too */
 	rr->crop = crop;
-	
+
 	/* tilerect is relative coordinates within render disprect. do not subtract crop yet */
 	rr->tilerect.xmin = partrct->xmin - re->disprect.xmin;
 	rr->tilerect.xmax = partrct->xmax - re->disprect.xmin;
@@ -1077,8 +1077,7 @@ int render_result_exr_file_read_path(RenderResult *rr, RenderLayer *rl_single, c
 
 ImBuf *render_result_rect_to_ibuf(RenderResult *rr, RenderData *rd)
 {
-	int flags = (rd->color_mgt_flag & R_COLOR_MANAGEMENT_PREDIVIDE) ? IB_cm_predivide : 0;
-	ImBuf *ibuf = IMB_allocImBuf(rr->rectx, rr->recty, rd->im_format.planes, flags);
+	ImBuf *ibuf = IMB_allocImBuf(rr->rectx, rr->recty, rd->im_format.planes, 0);
 	
 	/* if not exists, BKE_imbuf_write makes one */
 	ibuf->rect = (unsigned int *)rr->rect32;
@@ -1148,17 +1147,15 @@ void render_result_rect_fill_zero(RenderResult *rr)
 		rr->rect32 = MEM_callocN(sizeof(int) * rr->rectx * rr->recty, "render_seq rect");
 }
 
-void render_result_rect_get_pixels(RenderResult *rr, RenderData *rd, unsigned int *rect, int rectx, int recty,
+void render_result_rect_get_pixels(RenderResult *rr, unsigned int *rect, int rectx, int recty,
                                    const ColorManagedViewSettings *view_settings, const ColorManagedDisplaySettings *display_settings)
 {
 	if (rr->rect32) {
 		memcpy(rect, rr->rect32, sizeof(int) * rr->rectx * rr->recty);
 	}
 	else if (rr->rectf) {
-		int predivide = (rd->color_mgt_flag & R_COLOR_MANAGEMENT_PREDIVIDE);
-
 		IMB_display_buffer_transform_apply((unsigned char *) rect, rr->rectf, rr->rectx, rr->recty, 4,
-		                                   view_settings, display_settings, predivide);
+		                                   view_settings, display_settings, TRUE);
 	}
 	else
 		/* else fill with black */
diff --git a/source/blender/render/intern/source/rendercore.c b/source/blender/render/intern/source/rendercore.c
index 9f171ba71db..9b08d6c07e9 100644
--- a/source/blender/render/intern/source/rendercore.c
+++ b/source/blender/render/intern/source/rendercore.c
@@ -710,9 +710,11 @@ static void sky_tile(RenderPart *pa, RenderLayer *rl)
 					
 					if (pass[3]==0.0f) {
 						copy_v4_v4(pass, col);
+						pass[3] = 1.0f;
 					}
 					else {
 						addAlphaUnderFloat(pass, col);
+						pass[3] = 1.0f;
 					}
 				}
 			}
@@ -981,29 +983,6 @@ static void edge_enhance_add(RenderPart *pa, float *rectf, float *arect)
 	}
 }
 
-static void convert_to_key_alpha(RenderPart *pa, RenderLayer *rl)
-{
-	RenderLayer *rlpp[RE_MAX_OSA];
-	int y, sample, totsample;
-	
-	totsample= get_sample_layers(pa, rl, rlpp);
-	
-	for (sample= 0; sample<totsample; sample++) {
-		float *rectf= rlpp[sample]->rectf;
-		
-		for (y= pa->rectx*pa->recty; y>0; y--, rectf+=4) {
-			if (rectf[3] >= 1.0f) {
-				/* pass */
-			}
-			else if (rectf[3] > 0.0f) {
-				rectf[0] /= rectf[3];
-				rectf[1] /= rectf[3];
-				rectf[2] /= rectf[3];
-			}
-		}
-	}
-}
-
 /* clamp alpha and RGB to 0..1 and 0..inf, can go outside due to filter */
 static void clamp_alpha_rgb_range(RenderPart *pa, RenderLayer *rl)
 {
@@ -1312,10 +1291,6 @@ void zbufshadeDA_tile(RenderPart *pa)
 		/* clamp alpha to 0..1 range, can go outside due to filter */
 		clamp_alpha_rgb_range(pa, rl);
 		
-		/* de-premul alpha */
-		if (R.r.alphamode & R_ALPHAKEY)
-			convert_to_key_alpha(pa, rl);
-		
 		/* free stuff within loop! */
 		MEM_freeN(pa->rectdaps); pa->rectdaps= NULL;
 		freeps(&psmlist);
@@ -1476,10 +1451,6 @@ void zbufshade_tile(RenderPart *pa)
 		if (rl->passflag & SCE_PASS_VECTOR)
 			reset_sky_speed(pa, rl);
 		
-		/* de-premul alpha */
-		if (R.r.alphamode & R_ALPHAKEY)
-			convert_to_key_alpha(pa, rl);
-		
 		if (edgerect) MEM_freeN(edgerect);
 		edgerect= NULL;