Bunch of fixes for GLSL display transform

- GLSL shader wasn't aware of alpha predivide option,
  always assuming alpha is straight. Gave wrong results
  when displaying transparent float buffers.

- GLSL display wasn't aware of float buffers with number
  of channels different from 4, crashing when trying to
  display image with different number of channels.

  This required a bit larger changes, namely now it's
  possible to pass format (GL_RGB, GL_RGBAm GL_LUMINANCE)
  to glaDrawPixelsTex, This also implied adding format to
  glaDrawPixelsAuto and modifying all places where this
  functions are called.

  Now GLSL will handle both 3 and 4 channels buffers,
  single channel images are handled by CPU.

- Replaced hack for render result displaying with a bit
  different hack.

  Namely CPU conversion will happen only during render,
  once render is done GLSL would be used for displaying
  render result on a screen.

  This is so because of the way renderer updates parts
  of the image -- it happens without respect to active
  render layer in image user. This is harmless because
  only display buffer is modifying, but this is tricky
  because we don't have original buffer opened during
  rendering.

  One more related fix here was about when rendering
  multiple layers, wrong image would be displaying when
  rendering is done. Added a signal to invalidate
  display buffer once rendering is done (only happens
  when using multiple layers). This solves issue with
  wrong buffer stuck on the display when using regular
  CPU display space transform and if GLSL is available
  it'll make image displayed with a GLSL shader.

- As an additional change, byte buffers now also uses
  GLSL display transform.

  So now only dutehr and RGB curves are stoppers for
  using GLSL for all kind of display transforms.

5 files changed, 29 insertions, 10 deletions

diff --git a/intern/opencolorio/fallback_impl.cc b/intern/opencolorio/fallback_impl.cc
index 4c19c454aee..37f624e1f8b 100644
--- a/intern/opencolorio/fallback_impl.cc
+++ b/intern/opencolorio/fallback_impl.cc
@@ -381,7 +381,7 @@ void FallbackImpl::matrixTransformScale(float * , float * , const float *)
 {
 }
 
-bool FallbackImpl::setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor)
+bool FallbackImpl::setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor, bool predivide)
 {
 	return false;
 }
diff --git a/intern/opencolorio/ocio_capi.cc b/intern/opencolorio/ocio_capi.cc
index 8e831bb0736..c8db2c2b531 100644
--- a/intern/opencolorio/ocio_capi.cc
+++ b/intern/opencolorio/ocio_capi.cc
@@ -283,9 +283,9 @@ void OCIO_matrixTransformScale(float * m44, float * offset4, const float *scale4
 	impl->matrixTransformScale(m44, offset4, scale4f);
 }
 
-int OCIO_setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor)
+int OCIO_setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor, int predivide)
 {
-	return (int) impl->setupGLSLDraw(state_r, processor);
+	return (int) impl->setupGLSLDraw(state_r, processor, (bool) predivide);
 }
 
 void OCIO_finishGLSLDraw(struct OCIO_GLSLDrawState *state)
diff --git a/intern/opencolorio/ocio_capi.h b/intern/opencolorio/ocio_capi.h
index 8d8db161077..3632a0da1c6 100644
--- a/intern/opencolorio/ocio_capi.h
+++ b/intern/opencolorio/ocio_capi.h
@@ -121,7 +121,7 @@ void OCIO_matrixTransformRelease(OCIO_MatrixTransformRcPtr *mt);
 
 void OCIO_matrixTransformScale(float * m44, float * offset4, const float * scale4);
 
-int OCIO_setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor);
+int OCIO_setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor, int predivide);
 void OCIO_finishGLSLDraw(struct OCIO_GLSLDrawState *state);
 void OCIO_freeOGLState(struct OCIO_GLSLDrawState *state);
 
diff --git a/intern/opencolorio/ocio_impl.h b/intern/opencolorio/ocio_impl.h
index 3a1207d0870..a328470ccb5 100644
--- a/intern/opencolorio/ocio_impl.h
+++ b/intern/opencolorio/ocio_impl.h
@@ -96,7 +96,7 @@ public:
 
 	virtual void matrixTransformScale(float * m44, float * offset4, const float * scale4) = 0;
 
-	virtual bool setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor) = 0;
+	virtual bool setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor, bool predivide) = 0;
 	virtual void finishGLSLDraw(struct OCIO_GLSLDrawState *state) = 0;
 	virtual void freeGLState(struct OCIO_GLSLDrawState *state_r) = 0;
 };
@@ -169,7 +169,7 @@ public:
 
 	void matrixTransformScale(float * m44, float * offset4, const float * scale4);
 
-	bool setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor);
+	bool setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor, bool predivide);
 	void finishGLSLDraw(struct OCIO_GLSLDrawState *state);
 	void freeGLState(struct OCIO_GLSLDrawState *state_r);
 };
@@ -243,7 +243,7 @@ public:
 
 	void matrixTransformScale(float * m44, float * offset4, const float * scale4);
 
-	bool setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor);
+	bool setupGLSLDraw(struct OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor, bool predivide);
 	void finishGLSLDraw(struct OCIO_GLSLDrawState *state);
 	void freeGLState(struct OCIO_GLSLDrawState *state_r);
 };
diff --git a/intern/opencolorio/ocio_impl_glsl.cc b/intern/opencolorio/ocio_impl_glsl.cc
index 59845abca33..9343a13e888 100644
--- a/intern/opencolorio/ocio_impl_glsl.cc
+++ b/intern/opencolorio/ocio_impl_glsl.cc
@@ -77,15 +77,33 @@ typedef struct OCIO_GLSLDrawState {
 	GLint last_texture, last_texture_unit;
 } OCIO_GLSLDrawState;
 
-static const char * g_fragShaderText = ""
+/* Hardcoded to do alpha predivide before color space conversion */
+static const char *g_fragShaderText = ""
 "\n"
 "uniform sampler2D tex1;\n"
 "uniform sampler3D tex2;\n"
+"uniform bool predivide;\n"
 "\n"
 "void main()\n"
 "{\n"
 "    vec4 col = texture2D(tex1, gl_TexCoord[0].st);\n"
-"    gl_FragColor = OCIODisplay(col, tex2);\n"
+"    if (predivide == false || col[3] == 1.0f || col[3] == 0.0f) {\n"
+"      gl_FragColor = OCIODisplay(col, tex2);\n"
+"    } else {\n"
+"      float alpha = col[3];\n"
+"      float inv_alpha = 1.0f / alpha;\n"
+"\n"
+"      col[0] *= inv_alpha;\n"
+"      col[1] *= inv_alpha;\n"
+"      col[2] *= inv_alpha;\n"
+"\n"
+"      gl_FragColor = OCIODisplay(col, tex2);\n"
+"\n"
+"      col[0] *= alpha;\n"
+"      col[1] *= alpha;\n"
+"      col[2] *= alpha;\n"
+"    }\n"
+"\n"
 "}\n";
 
 static GLuint compileShaderText(GLenum shaderType, const char *text)
@@ -187,7 +205,7 @@ static void ensureLUT3DAllocated(OCIO_GLSLDrawState *state)
  * When all drawing is finished, finishGLSLDraw shall be called to
  * restore OpenGL context to it's pre-GLSL draw state.
  */
-bool OCIOImpl::setupGLSLDraw(OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor)
+bool OCIOImpl::setupGLSLDraw(OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRcPtr *processor, bool predivide)
 {
 	ConstProcessorRcPtr ocio_processor = *(ConstProcessorRcPtr *) processor;
 
@@ -252,6 +270,7 @@ bool OCIOImpl::setupGLSLDraw(OCIO_GLSLDrawState **state_r, OCIO_ConstProcessorRc
 		glUseProgram(state->program);
 		glUniform1i(glGetUniformLocation(state->program, "tex1"), 0);
 		glUniform1i(glGetUniformLocation(state->program, "tex2"), 1);
+		glUniform1i(glGetUniformLocation(state->program, "predivide"), predivide);
 
 		return true;
 	}