Bunch of fixes for GLSL display transform

- GLSL shader wasn't aware of alpha predivide option,
  always assuming alpha is straight. Gave wrong results
  when displaying transparent float buffers.

- GLSL display wasn't aware of float buffers with number
  of channels different from 4, crashing when trying to
  display image with different number of channels.

  This required a bit larger changes, namely now it's
  possible to pass format (GL_RGB, GL_RGBAm GL_LUMINANCE)
  to glaDrawPixelsTex, This also implied adding format to
  glaDrawPixelsAuto and modifying all places where this
  functions are called.

  Now GLSL will handle both 3 and 4 channels buffers,
  single channel images are handled by CPU.

- Replaced hack for render result displaying with a bit
  different hack.

  Namely CPU conversion will happen only during render,
  once render is done GLSL would be used for displaying
  render result on a screen.

  This is so because of the way renderer updates parts
  of the image -- it happens without respect to active
  render layer in image user. This is harmless because
  only display buffer is modifying, but this is tricky
  because we don't have original buffer opened during
  rendering.

  One more related fix here was about when rendering
  multiple layers, wrong image would be displaying when
  rendering is done. Added a signal to invalidate
  display buffer once rendering is done (only happens
  when using multiple layers). This solves issue with
  wrong buffer stuck on the display when using regular
  CPU display space transform and if GLSL is available
  it'll make image displayed with a GLSL shader.

- As an additional change, byte buffers now also uses
  GLSL display transform.

  So now only dutehr and RGB curves are stoppers for
  using GLSL for all kind of display transforms.

1 files changed, 108 insertions, 32 deletions

diff --git a/source/blender/editors/screen/glutil.c b/source/blender/editors/screen/glutil.c
index b5c2b106a15..ec1a085f9da 100644
--- a/source/blender/editors/screen/glutil.c
+++ b/source/blender/editors/screen/glutil.c
@@ -42,6 +42,7 @@
 #include "BLI_threads.h"
 
 #include "BKE_blender.h"
+#include "BKE_global.h"
 #include "BKE_colortools.h"
 #include "BKE_context.h"
 
@@ -488,7 +489,7 @@ static int get_cached_work_texture(int *w_r, int *h_r)
 	return texid;
 }
 
-void glaDrawPixelsTexScaled(float x, float y, int img_w, int img_h, int format, int zoomfilter, void *rect, float scaleX, float scaleY)
+void glaDrawPixelsTexScaled(float x, float y, int img_w, int img_h, int format, int type, int zoomfilter, void *rect, float scaleX, float scaleY)
 {
 	unsigned char *uc_rect = (unsigned char *) rect;
 	float *f_rect = (float *)rect;
@@ -498,7 +499,8 @@ void glaDrawPixelsTexScaled(float x, float y, int img_w, int img_h, int format,
 	int subpart_x, subpart_y, tex_w, tex_h;
 	int seamless, offset_x, offset_y, nsubparts_x, nsubparts_y;
 	int texid = get_cached_work_texture(&tex_w, &tex_h);
-	
+	int components;
+
 	/* Specify the color outside this function, and tex will modulate it.
 	 * This is useful for changing alpha without using glPixelTransferf()
 	 */
@@ -525,13 +527,22 @@ void glaDrawPixelsTexScaled(float x, float y, int img_w, int img_h, int format,
 	nsubparts_x = (img_w + (offset_x - 1)) / (offset_x);
 	nsubparts_y = (img_h + (offset_y - 1)) / (offset_y);
 
-	if (format == GL_FLOAT) {
+	if (format == GL_RGBA)
+		components = 4;
+	else if (format == GL_RGB)
+		components = 3;
+	else if (format == GL_LUMINANCE)
+		components = 1;
+	else
+		BLI_assert(!"Incompatible format passed to glaDrawPixelsTexScaled");
+
+	if (type == GL_FLOAT) {
 		/* need to set internal format to higher range float */
-		glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA16F, tex_w, tex_h, 0, GL_RGBA, GL_FLOAT, NULL);
+		glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA16F_ARB, tex_w, tex_h, 0, format, GL_FLOAT, NULL);
 	}
 	else {
 		/* switch to 8bit RGBA for byte buffer  */
-		glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, tex_w, tex_h, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
+		glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, tex_w, tex_h, 0, format, GL_UNSIGNED_BYTE, NULL);
 	}
 
 	for (subpart_y = 0; subpart_y < nsubparts_y; subpart_y++) {
@@ -551,26 +562,26 @@ void glaDrawPixelsTexScaled(float x, float y, int img_w, int img_h, int format,
 			if (subpart_w <= seamless || subpart_h <= seamless)
 				continue;
 			
-			if (format == GL_FLOAT) {
-				glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, subpart_w, subpart_h, GL_RGBA, GL_FLOAT, &f_rect[subpart_y * offset_y * img_w * 4 + subpart_x * offset_x * 4]);
+			if (type == GL_FLOAT) {
+				glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, subpart_w, subpart_h, format, GL_FLOAT, &f_rect[subpart_y * offset_y * img_w * components + subpart_x * offset_x * components]);
 				
 				/* add an extra border of pixels so linear looks ok at edges of full image. */
 				if (subpart_w < tex_w)
-					glTexSubImage2D(GL_TEXTURE_2D, 0, subpart_w, 0, 1, subpart_h, GL_RGBA, GL_FLOAT, &f_rect[subpart_y * offset_y * img_w * 4 + (subpart_x * offset_x + subpart_w - 1) * 4]);
+					glTexSubImage2D(GL_TEXTURE_2D, 0, subpart_w, 0, 1, subpart_h, format, GL_FLOAT, &f_rect[subpart_y * offset_y * img_w * components + (subpart_x * offset_x + subpart_w - 1) * components]);
 				if (subpart_h < tex_h)
-					glTexSubImage2D(GL_TEXTURE_2D, 0, 0, subpart_h, subpart_w, 1, GL_RGBA, GL_FLOAT, &f_rect[(subpart_y * offset_y + subpart_h - 1) * img_w * 4 + subpart_x * offset_x * 4]);
+					glTexSubImage2D(GL_TEXTURE_2D, 0, 0, subpart_h, subpart_w, 1, format, GL_FLOAT, &f_rect[(subpart_y * offset_y + subpart_h - 1) * img_w * components + subpart_x * offset_x * components]);
 				if (subpart_w < tex_w && subpart_h < tex_h)
-					glTexSubImage2D(GL_TEXTURE_2D, 0, subpart_w, subpart_h, 1, 1, GL_RGBA, GL_FLOAT, &f_rect[(subpart_y * offset_y + subpart_h - 1) * img_w * 4 + (subpart_x * offset_x + subpart_w - 1) * 4]);
+					glTexSubImage2D(GL_TEXTURE_2D, 0, subpart_w, subpart_h, 1, 1, format, GL_FLOAT, &f_rect[(subpart_y * offset_y + subpart_h - 1) * img_w * components + (subpart_x * offset_x + subpart_w - 1) * components]);
 			}
 			else {
-				glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, subpart_w, subpart_h, GL_RGBA, GL_UNSIGNED_BYTE, &uc_rect[subpart_y * offset_y * img_w * 4 + subpart_x * offset_x * 4]);
+				glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, subpart_w, subpart_h, format, GL_UNSIGNED_BYTE, &uc_rect[subpart_y * offset_y * img_w * components + subpart_x * offset_x * components]);
 				
 				if (subpart_w < tex_w)
-					glTexSubImage2D(GL_TEXTURE_2D, 0, subpart_w, 0, 1, subpart_h, GL_RGBA, GL_UNSIGNED_BYTE, &uc_rect[subpart_y * offset_y * img_w * 4 + (subpart_x * offset_x + subpart_w - 1) * 4]);
+					glTexSubImage2D(GL_TEXTURE_2D, 0, subpart_w, 0, 1, subpart_h, format, GL_UNSIGNED_BYTE, &uc_rect[subpart_y * offset_y * img_w * components + (subpart_x * offset_x + subpart_w - 1) * components]);
 				if (subpart_h < tex_h)
-					glTexSubImage2D(GL_TEXTURE_2D, 0, 0, subpart_h, subpart_w, 1, GL_RGBA, GL_UNSIGNED_BYTE, &uc_rect[(subpart_y * offset_y + subpart_h - 1) * img_w * 4 + subpart_x * offset_x * 4]);
+					glTexSubImage2D(GL_TEXTURE_2D, 0, 0, subpart_h, subpart_w, 1, format, GL_UNSIGNED_BYTE, &uc_rect[(subpart_y * offset_y + subpart_h - 1) * img_w * components + subpart_x * offset_x * components]);
 				if (subpart_w < tex_w && subpart_h < tex_h)
-					glTexSubImage2D(GL_TEXTURE_2D, 0, subpart_w, subpart_h, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, &uc_rect[(subpart_y * offset_y + subpart_h - 1) * img_w * 4 + (subpart_x * offset_x + subpart_w - 1) * 4]);
+					glTexSubImage2D(GL_TEXTURE_2D, 0, subpart_w, subpart_h, 1, 1, format, GL_UNSIGNED_BYTE, &uc_rect[(subpart_y * offset_y + subpart_h - 1) * img_w * components + (subpart_x * offset_x + subpart_w - 1) * components]);
 			}
 
 			glEnable(GL_TEXTURE_2D);
@@ -601,9 +612,9 @@ void glaDrawPixelsTexScaled(float x, float y, int img_w, int img_h, int format,
 #endif
 }
 
-void glaDrawPixelsTex(float x, float y, int img_w, int img_h, int format, int zoomfilter, void *rect)
+void glaDrawPixelsTex(float x, float y, int img_w, int img_h, int format, int type, int zoomfilter, void *rect)
 {
-	glaDrawPixelsTexScaled(x, y, img_w, img_h, format, zoomfilter, rect, 1.0f, 1.0f);
+	glaDrawPixelsTexScaled(x, y, img_w, img_h, format, type, zoomfilter, rect, 1.0f, 1.0f);
 }
 
 void glaDrawPixelsSafe(float x, float y, int img_w, int img_h, int row_w, int format, int type, void *rect)
@@ -686,7 +697,7 @@ void glaDrawPixelsSafe(float x, float y, int img_w, int img_h, int row_w, int fo
 }
 
 /* uses either DrawPixelsSafe or DrawPixelsTex, based on user defined maximum */
-void glaDrawPixelsAuto(float x, float y, int img_w, int img_h, int format, int zoomfilter, void *rect)
+void glaDrawPixelsAuto(float x, float y, int img_w, int img_h, int format, int type, int zoomfilter, void *rect)
 {
 	if (U.image_gpubuffer_limit) {
 		/* Megapixels, use float math to prevent overflow */
@@ -694,11 +705,11 @@ void glaDrawPixelsAuto(float x, float y, int img_w, int img_h, int format, int z
 		
 		if (U.image_gpubuffer_limit > (int)img_size) {
 			glColor4f(1.0, 1.0, 1.0, 1.0);
-			glaDrawPixelsTex(x, y, img_w, img_h, format, zoomfilter, rect);
+			glaDrawPixelsTex(x, y, img_w, img_h, format, type, zoomfilter, rect);
 			return;
 		}
 	}
-	glaDrawPixelsSafe(x, y, img_w, img_h, img_w, GL_RGBA, format, rect);
+	glaDrawPixelsSafe(x, y, img_w, img_h, img_w, format, type, rect);
 }
 
 /* 2D Drawing Assistance */
@@ -1017,22 +1028,83 @@ void bglFlush(void)
 /* Draw given image buffer on a screen using GLSL for display transform */
 void glaDrawImBuf_glsl_ctx(const bContext *C, ImBuf *ibuf, float x, float y, int zoomfilter)
 {
+	bool force_fallback = false;
 	bool need_fallback = true;
 
-	/* Bytes and dithering are not supported on GLSL yet */
+	/* Early out */
+	if (ibuf->rect == NULL && ibuf->rect_float == NULL)
+		return;
+
+	/* Dithering is not supported on GLSL yet */
+	force_fallback = ibuf->dither != 0.0f;
+
+	/* Single channel images could not be transformed using GLSL yet */
+	force_fallback = ibuf->channels == 1;
 
-	/* WORKAROUND: only use GLSL if there's no byte buffer at all,
-	 *             this is because of how render results are handled,
-	 *             they're not updating image buffer's float buffer,
-	 *             but writes data directly to it's byte buffer and
-	 *             modifies display buffer.
+	/* This is actually lots of crap, but currently not sure about
+	 * more clear way to bypass partial buffer update crappyness
+	 * while rendering.
+	 *
+	 * The thing is -- render engines are only updating byte and
+	 * display buffers for active render result opened in image
+	 * editor. This works fine to show render progress without
+	 * switching render layers in image editor user, but this is
+	 * completely useless for GLSL display, where we need to have
+	 * original buffer which we could color manage.
+	 *
+	 * For the time of rendering, we'll stick back to slower CPU
+	 * display buffer update. GLSL could be used as soon as some
+	 * fixes (?) are done in render itself, so we'll always have
+	 * image buffer with relevant float buffer opened while
+	 * rendering.
+	 *
+	 * On the other hand, when using Cycles, stressing GPU with
+	 * GLSL could backfire on a performance.
+	 *                                         - sergey -
 	 */
-	if (ibuf->rect == NULL && ibuf->rect_float && ibuf->dither == 0.0f) {
-		if (IMB_colormanagement_setup_glsl_draw_from_ctx(C)) {
+	if (G.is_rendering) {
+		/* Try to detect whether we're drawing render result,
+		 * other images could have both rect and rect_float
+		 * but they'll be synchronized
+		 */
+		if (ibuf->rect_float && ibuf->rect &&
+		    ((ibuf->mall & IB_rectfloat) == 0))
+		{
+			force_fallback = true;
+		}
+	}
+
+	/* Try to draw buffer using GLSL display transform */
+	if (force_fallback == false) {
+		int ok;
+
+		if (ibuf->rect_float)
+			ok = IMB_colormanagement_setup_glsl_draw_ctx(C, TRUE);
+		else
+			ok = IMB_colormanagement_setup_glsl_draw_from_space_ctx(C, ibuf->rect_colorspace, FALSE);
+
+		if (ok) {
 			glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE);
 			glColor4f(1.0, 1.0, 1.0, 1.0);
 
-			glaDrawPixelsTex(x, y, ibuf->x, ibuf->y, GL_FLOAT, zoomfilter, ibuf->rect_float);
+			if (ibuf->rect_float) {
+				int format;
+
+				if (ibuf->channels == 3)
+					format = GL_RGB;
+				else if (ibuf->channels == 4)
+					format = GL_RGBA;
+				else
+					BLI_assert(!"Incompatible number of channels for GLSL display");
+
+				glaDrawPixelsTex(x, y, ibuf->x, ibuf->y, format, GL_FLOAT,
+				                 zoomfilter, ibuf->rect_float);
+			}
+			else if (ibuf->rect) {
+				/* ibuf->rect is always RGBA */
+				glaDrawPixelsTex(x, y, ibuf->x, ibuf->y, GL_RGBA, GL_UNSIGNED_BYTE,
+				                 zoomfilter, ibuf->rect);
+			}
 
 			IMB_colormanagement_finish_glsl_draw();
 
@@ -1040,6 +1112,7 @@ void glaDrawImBuf_glsl_ctx(const bContext *C, ImBuf *ibuf, float x, float y, int
 		}
 	}
 
+	/* In case GLSL failed or not usable, fallback to glaDrawPixelsAuto */
 	if (need_fallback) {
 		unsigned char *display_buffer;
 		void *cache_handle;
@@ -1047,7 +1120,8 @@ void glaDrawImBuf_glsl_ctx(const bContext *C, ImBuf *ibuf, float x, float y, int
 		display_buffer = IMB_display_buffer_acquire_ctx(C, ibuf, &cache_handle);
 
 		if (display_buffer)
-			glaDrawPixelsAuto(x, y, ibuf->x, ibuf->y, GL_UNSIGNED_BYTE, zoomfilter, display_buffer);
+			glaDrawPixelsAuto(x, y, ibuf->x, ibuf->y, GL_RGBA, GL_UNSIGNED_BYTE,
+			                  zoomfilter, display_buffer);
 
 		IMB_display_buffer_release(cache_handle);
 	}
@@ -1057,6 +1131,8 @@ void glaDrawImBuf_glsl_ctx(const bContext *C, ImBuf *ibuf, float x, float y, int
  *
  * See IMB_colormanagement_setup_transform_from_role_glsl description for
  * some more details
+ *
+ * NOTE: this only works for RGBA buffers!
  */
 int glaBufferTransformFromRole_glsl(float *buffer, int width, int height, int role)
 {
@@ -1071,7 +1147,7 @@ int glaBufferTransformFromRole_glsl(float *buffer, int width, int height, int ro
 
 	GPU_offscreen_bind(ofs);
 
-	if (!IMB_colormanagement_setup_transform_from_role_glsl(role)) {
+	if (!IMB_colormanagement_setup_transform_from_role_glsl(role, TRUE)) {
 		GPU_offscreen_unbind(ofs);
 		GPU_offscreen_free(ofs);
 		return FALSE;
@@ -1085,9 +1161,9 @@ int glaBufferTransformFromRole_glsl(float *buffer, int width, int height, int ro
 	glPushMatrix();
 
 	glaDefine2DArea(&display_rect);
-	glLoadIdentity();
 
-	glaDrawPixelsTex(0, 0, width, height, GL_FLOAT, GL_NEAREST, buffer);
+	glaDrawPixelsTex(0, 0, width, height, GL_RGBA, GL_FLOAT,
+	                 GL_NEAREST, buffer);
 
 	glMatrixMode(GL_PROJECTION);
 	glPopMatrix();