Added support for transparent stroke colors.

In Freestyle, strokes are represented with triangle strips, and stroke
colors are realized using vertex colors in order to enable variable
stroke colors (i.e., each triangle has a different color).  Stroke
colors in Freestyle are in the RGBA format, while vertex colors in
Blender do not have an alpha component.  Therefore, we here employ a
2-pass rendering approach as follows.  First, the alpha component of
an image is rendered by using the red component of vertex colors as
the alpha component of stroke colors (1st pass).  The render result is
saved into a temporary buffer.  Then, the vertex colors of stroke
meshes are replaced with RGB values, and the RGB components of the
image is rendered (2nd pass).  Finally, the RGB and alpha components
are merged to produce the render result in the RGBA format.

1 files changed, 94 insertions, 12 deletions

diff --git a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
index 95a0f642c82..a220340efcc 100644
--- a/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
+++ b/source/blender/freestyle/intern/blender_interface/BlenderStrokeRenderer.cpp
@@ -27,6 +27,8 @@ extern "C" {
 
 #include "RE_pipeline.h"
 
+#include "renderpipeline.h"
+
 #ifdef __cplusplus
 }
 #endif
@@ -215,7 +217,9 @@ void BlenderStrokeRenderer::RenderStrokeRepBasic(StrokeRep *iStrokeRep) const{
 		mesh->mface = (MFace*) CustomData_add_layer( &mesh->fdata, CD_MFACE, CD_CALLOC, NULL, mesh->totface);
 		
 		// colors allocation  - me.vertexColors = True
-		mesh->mcol = (MCol *) CustomData_add_layer( &mesh->fdata, CD_MCOL, CD_CALLOC, NULL, mesh->totface );
+		MCol* alphas = (MCol *) CustomData_add_layer_named( &mesh->fdata, CD_MCOL, CD_CALLOC, NULL, mesh->totface, "A" );
+		MCol* colors = (MCol *) CustomData_add_layer_named( &mesh->fdata, CD_MCOL, CD_CALLOC, NULL, mesh->totface, "RGB" );
+		mesh->mcol = alphas;
 
 		////////////////////
 		//  Data copy
@@ -223,7 +227,6 @@ void BlenderStrokeRenderer::RenderStrokeRepBasic(StrokeRep *iStrokeRep) const{
 		
 		MVert* vertices = mesh->mvert;
 		MFace* faces = mesh->mface;
-		MCol* colors = mesh->mcol;
 		
 	    v[0] = strip_vertices.begin();
 	    v[1] = v[0]; ++(v[1]);
@@ -280,25 +283,37 @@ void BlenderStrokeRenderer::RenderStrokeRepBasic(StrokeRep *iStrokeRep) const{
 				color[1] = svRep[1]->color();
 				color[2] = svRep[2]->color();
 
+				colors->a = 0;
 				colors->r = (short)(255.0f*(color[0])[2]);
 				colors->g = (short)(255.0f*(color[0])[1]);
 				colors->b = (short)(255.0f*(color[0])[0]);
-				colors->a = (short)(255.0f*svRep[0]->alpha());
-				++colors;
+				alphas->a = 0;
+				alphas->r = 0;
+				alphas->g = 0;
+				alphas->b = (short)(255.0f*svRep[0]->alpha());
+				++colors; ++alphas;
 				
+				colors->a = 0;
 				colors->r = (short)(255.0f*(color[1])[2]);
 				colors->g = (short)(255.0f*(color[1])[1]);
 				colors->b = (short)(255.0f*(color[1])[0]);
-				colors->a = (short)(255.0f*svRep[1]->alpha());
-				++colors;
+				alphas->a = 0;
+				alphas->r = 0;
+				alphas->g = 0;
+				alphas->b = (short)(255.0f*svRep[1]->alpha());
+				++colors; ++alphas;
 				
+				colors->a = 0;
 				colors->r = (short)(255.0f*(color[2])[2]);
 				colors->g = (short)(255.0f*(color[2])[1]);
 				colors->b = (short)(255.0f*(color[2])[0]);
-				colors->a = (short)(255.0f*svRep[2]->alpha());
-				++colors;
+				alphas->a = 0;
+				alphas->r = 0;
+				alphas->g = 0;
+				alphas->b = (short)(255.0f*svRep[2]->alpha());
+				++colors; ++alphas;
 
-				++faces; ++vertices; ++colors;
+				++faces; ++vertices; ++colors; ++alphas;
 				++vertex_index;
 			}
 			++v[0]; ++v[1]; ++v[2];
@@ -310,13 +325,80 @@ void BlenderStrokeRenderer::RenderStrokeRepBasic(StrokeRep *iStrokeRep) const{
 }
 
 Render* BlenderStrokeRenderer::RenderScene( Render *re ) {
+    Render* freestyle_render;
+    RenderLayer *rl;
+    float *rectf, *alpha;
+	int x, y, rectx, recty;
+	
 	freestyle_scene->r.mode &= ~( R_EDGE_FRS | R_SHADOW | R_SSS | R_PANORAMA | R_ENVMAP | R_MBLUR );
 	freestyle_scene->r.scemode &= ~( R_SINGLE_LAYER );
 	freestyle_scene->r.planes = R_PLANES32;
 	freestyle_scene->r.imtype = R_PNG;
 	
-	Render* freestyle_render = RE_NewRender(freestyle_scene->id.name);
-	
+    // Pass 1 - render only the alpha component
+	freestyle_render = RE_NewRender(freestyle_scene->id.name);
 	RE_BlenderFrame( freestyle_render, freestyle_scene, NULL, 1);
-	return freestyle_render;
+
+    // save the alpha component of the render into a buffer
+	rl = render_get_active_layer( freestyle_render, freestyle_render->result );
+    if (!rl || rl->rectf == NULL) {
+        cout << "Cannot find Freestyle result image" << endl;
+    	RE_FreeRender(freestyle_render);
+        return NULL;
+    }
+	rectf = rl->rectf;
+	rectx = re->rectx;
+	recty = re->recty;
+    alpha = new float[rectx * recty];
+    for (y = 0; y < recty; y++)
+        for (x = 0; x < rectx; x++)
+            alpha[rectx * y + x] = rectf[4 * (rectx * y + x)];
+
+	RE_FreeRender(freestyle_render);
+
+    // replace the mesh vertex colors
+    LinkData *link = (LinkData *)objects.first;
+    while (link) {
+		Object *ob = (Object *)link->data;
+        if (ob->type == OB_MESH) {
+        	Mesh *mesh = (Mesh *)ob->data;
+            int index = CustomData_get_named_layer_index(&mesh->fdata, CD_MCOL, "A");
+            if (index < 0) {
+                cout << "Cannot find mesh MCol layer (A)" << endl;
+                goto error;
+            }
+            CustomData_free_layer(&mesh->fdata, CD_MCOL, mesh->totface, index);
+        	mesh->mcol = (MCol *)CustomData_get_layer_named(&mesh->fdata, CD_MCOL, "RGB");
+            if (!mesh->mcol) {
+                cout << "Cannot find mesh MCol layer (RGB)" << endl;
+                goto error;
+            }
+		}
+        link = link->next;
+	}
+
+    // Pass 2 - render the RGB components
+	freestyle_render = RE_NewRender(freestyle_scene->id.name);
+    RE_BlenderFrame( freestyle_render, freestyle_scene, NULL, 1);
+
+    // merge the saved alpha component into the 2nd render
+	rl = render_get_active_layer( freestyle_render, freestyle_render->result );
+    if (!rl || rl->rectf == NULL) {
+        cout << "Cannot find Freestyle result image" << endl;
+    	RE_FreeRender(freestyle_render);
+        goto error;
+    }
+	rectf = rl->rectf;
+    for (y = 0; y < recty; y++)
+        for (x = 0; x < rectx; x++)
+            rectf[4 * (rectx * y + x) + 3] = alpha[rectx * y + x];
+
+    delete [] alpha;
+
+    return freestyle_render;
+
+error:
+    delete [] alpha;
+
+    return NULL;
 }