Changes to keying nodes:

- Replace FastGaussian blur with GaussianBokeh blur which should give better results.
- Changes a bit formula of saturation which in some cases gives better result.
  Also included (commented out) original formula which was also checked by Brecht
  and which gave better result in some other cases.
- Made clipping white/black temporal dependent, so hopefully it wouldn't destroy
  gradients on edges.

7 files changed, 199 insertions, 45 deletions

diff --git a/source/blender/compositor/CMakeLists.txt b/source/blender/compositor/CMakeLists.txt
index c71168b2d2e..6fc938c192c 100644
--- a/source/blender/compositor/CMakeLists.txt
+++ b/source/blender/compositor/CMakeLists.txt
@@ -333,6 +333,8 @@ set(SRC
 	operations/COM_KeyingScreenOperation.h
 	operations/COM_KeyingDespillOperation.cpp
 	operations/COM_KeyingDespillOperation.h
+	operations/COM_KeyingClipOperation.cpp
+	operations/COM_KeyingClipOperation.h
 
 operations/COM_ColorSpillOperation.cpp
 operations/COM_ColorSpillOperation.h
diff --git a/source/blender/compositor/nodes/COM_KeyingNode.cpp b/source/blender/compositor/nodes/COM_KeyingNode.cpp
index 9e29a99293e..2bc502a5f86 100644
--- a/source/blender/compositor/nodes/COM_KeyingNode.cpp
+++ b/source/blender/compositor/nodes/COM_KeyingNode.cpp
@@ -27,12 +27,13 @@
 
 #include "COM_KeyingOperation.h"
 #include "COM_KeyingDespillOperation.h"
+#include "COM_KeyingClipOperation.h"
 
 #include "COM_SeparateChannelOperation.h"
 #include "COM_CombineChannelsOperation.h"
 #include "COM_ConvertRGBToYCCOperation.h"
 #include "COM_ConvertYCCToRGBOperation.h"
-#include "COM_FastGaussianBlurOperation.h"
+#include "COM_GaussianBokehBlurOperation.h"
 #include "COM_SetValueOperation.h"
 
 #include "COM_DilateErodeOperation.h"
@@ -72,8 +73,9 @@ OutputSocket *KeyingNode::setupPreBlur(ExecutionSystem *graph, InputSocket *inpu
 			setValueOperation->setValue(1.0f);
 			graph->addOperation(setValueOperation);
 
-			FastGaussianBlurOperation *blurOperation = new FastGaussianBlurOperation();
+			GaussianBokehBlurOperation *blurOperation = new GaussianBokehBlurOperation();
 			blurOperation->setData(&preBlurData);
+			blurOperation->setQuality(COM_QUALITY_HIGH);
 
 			addLink(graph, separateOperation->getOutputSocket(0), blurOperation->getInputSocket(0));
 			addLink(graph, setValueOperation->getOutputSocket(0), blurOperation->getInputSocket(1));
@@ -104,8 +106,9 @@ OutputSocket *KeyingNode::setupPostBlur(ExecutionSystem *graph, OutputSocket *po
 	setValueOperation->setValue(1.0f);
 	graph->addOperation(setValueOperation);
 
-	FastGaussianBlurOperation *blurOperation = new FastGaussianBlurOperation();
+	GaussianBokehBlurOperation *blurOperation = new GaussianBokehBlurOperation();
 	blurOperation->setData(&postBlurData);
+	blurOperation->setQuality(COM_QUALITY_HIGH);
 
 	addLink(graph, postBLurInput, blurOperation->getInputSocket(0));
 	addLink(graph, setValueOperation->getOutputSocket(0), blurOperation->getInputSocket(1));
@@ -149,6 +152,20 @@ OutputSocket *KeyingNode::setupDespill(ExecutionSystem *graph, OutputSocket *des
 	return despillOperation->getOutputSocket(0);
 }
 
+OutputSocket *KeyingNode::setupClip(ExecutionSystem *graph, OutputSocket *clipInput, float clipBlack, float clipWhite)
+{
+	KeyingClipOperation *clipOperation = new KeyingClipOperation();
+
+	clipOperation->setClipBlack(clipBlack);
+	clipOperation->setClipWhite(clipWhite);
+
+	addLink(graph, clipInput, clipOperation->getInputSocket(0));
+
+	graph->addOperation(clipOperation);
+
+	return clipOperation->getOutputSocket(0);
+}
+
 void KeyingNode::convertToOperations(ExecutionSystem *graph, CompositorContext *context)
 {
 	InputSocket *inputImage = this->getInputSocket(0);
@@ -163,9 +180,6 @@ void KeyingNode::convertToOperations(ExecutionSystem *graph, CompositorContext *
 	/* keying operation */
 	KeyingOperation *keyingOperation = new KeyingOperation();
 
-	keyingOperation->setClipBlack(keying_data->clip_black);
-	keyingOperation->setClipWhite(keying_data->clip_white);
-
 	inputScreen->relinkConnections(keyingOperation->getInputSocket(1), 1, graph);
 
 	if (keying_data->blur_pre) {
@@ -180,11 +194,14 @@ void KeyingNode::convertToOperations(ExecutionSystem *graph, CompositorContext *
 
 	graph->addOperation(keyingOperation);
 
+	postprocessedMatte = keyingOperation->getOutputSocket();
+
+	if (keying_data->clip_black > 0.0f || keying_data->clip_white< 1.0f)
+		postprocessedMatte = setupClip(graph, postprocessedMatte, keying_data->clip_black, keying_data->clip_white);
+
 	/* apply blur on matte if needed */
 	if (keying_data->blur_post)
-		postprocessedMatte = setupPostBlur(graph, keyingOperation->getOutputSocket(), keying_data->blur_post);
-	else
-		postprocessedMatte = keyingOperation->getOutputSocket();
+		postprocessedMatte = setupPostBlur(graph, postprocessedMatte, keying_data->blur_post);
 
 	/* matte dilate/erode */
 	if (keying_data->dilate_distance != 0) {
diff --git a/source/blender/compositor/nodes/COM_KeyingNode.h b/source/blender/compositor/nodes/COM_KeyingNode.h
index 894d0ddc095..9d4067ce599 100644
--- a/source/blender/compositor/nodes/COM_KeyingNode.h
+++ b/source/blender/compositor/nodes/COM_KeyingNode.h
@@ -37,6 +37,7 @@ protected:
 	OutputSocket *setupPostBlur(ExecutionSystem *graph, OutputSocket *postBLurInput, int size);
 	OutputSocket *setupDilateErode(ExecutionSystem *graph, OutputSocket *dilateErodeInput, int distance);
 	OutputSocket *setupDespill(ExecutionSystem *graph, OutputSocket *despillInput, InputSocket *inputSrceen, float factor);
+	OutputSocket *setupClip(ExecutionSystem *graph, OutputSocket *clipInput, float clipBlack, float clipWhite);
 public:
 	KeyingNode(bNode *editorNode);
 	void convertToOperations(ExecutionSystem *graph, CompositorContext *context);
diff --git a/source/blender/compositor/operations/COM_KeyingClipOperation.cpp b/source/blender/compositor/operations/COM_KeyingClipOperation.cpp
new file mode 100644
index 00000000000..559d282f613
--- /dev/null
+++ b/source/blender/compositor/operations/COM_KeyingClipOperation.cpp
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2012, Blender Foundation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * Contributor:
+ *		Jeroen Bakker
+ *		Monique Dewanchand
+ *		Sergey Sharybin
+ */
+
+#include "COM_KeyingClipOperation.h"
+
+#include "MEM_guardedalloc.h"
+
+#include "BLI_listbase.h"
+#include "BLI_math.h"
+
+KeyingClipOperation::KeyingClipOperation(): NodeOperation()
+{
+	this->addInputSocket(COM_DT_VALUE);
+	this->addOutputSocket(COM_DT_VALUE);
+
+	this->clipBlack = 0.0f;
+	this->clipWhite = 1.0f;
+
+	this->pixelReader = NULL;
+}
+
+void KeyingClipOperation::initExecution()
+{
+	this->pixelReader = this->getInputSocketReader(0);
+}
+
+void KeyingClipOperation::deinitExecution()
+{
+	this->pixelReader = NULL;
+}
+
+void KeyingClipOperation::executePixel(float *color, float x, float y, PixelSampler sampler, MemoryBuffer *inputBuffers[])
+{
+	const int delta = 3;
+
+	float pixelColor[4];
+	int width = this->getWidth(), height = this->getHeight();
+	int count_black = 0, count_white = 0;
+	int i, j;
+
+	this->pixelReader->read(pixelColor, x, y, sampler, inputBuffers);
+
+	for (i = -delta + 1; i < delta; i++) {
+		for (j = -delta + 1; j < delta; j++) {
+			int cx = x + j, cy = y + i;
+
+			if (i == 0 && j == 0)
+				continue;
+
+			if (cx >= 0 && cx < width && cy >= 0 && cy < height) {
+				float value[4];
+
+				this->pixelReader->read(value, cx, cy, sampler, inputBuffers);
+
+				if (value[0] < 0.4f)
+					count_black++;
+				else if (value[0] > 0.6f)
+					count_white++;
+			}
+		}
+	}
+
+	color[0] = pixelColor[0];
+
+	if (count_black >= 22 || count_white >= 22) {
+		if (count_black >= 4 || count_white >= 4) {
+			if (color[0] < this->clipBlack)
+				color[0] = 0.0f;
+			else if (color[0] >= this->clipWhite)
+				color[0] = 1.0f;
+			else
+				color[0] = (color[0] - this->clipBlack) / (this->clipWhite - this->clipBlack);
+		}
+	}
+}
diff --git a/source/blender/compositor/operations/COM_KeyingClipOperation.h b/source/blender/compositor/operations/COM_KeyingClipOperation.h
new file mode 100644
index 00000000000..1141e0b47ab
--- /dev/null
+++ b/source/blender/compositor/operations/COM_KeyingClipOperation.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2012, Blender Foundation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * Contributor:
+ *		Jeroen Bakker
+ *		Monique Dewanchand
+ *		Sergey Sharybin
+ */
+
+#ifndef _COM_KeyingClipOperation_h
+#define _COM_KeyingClipOperation_h
+
+#include "COM_NodeOperation.h"
+
+/**
+  * Class with implementation of black/white clipping for keying node
+  */
+class KeyingClipOperation : public NodeOperation {
+protected:
+	SocketReader *pixelReader;
+	float clipBlack;
+	float clipWhite;
+
+public:
+	KeyingClipOperation();
+
+	void initExecution();
+	void deinitExecution();
+
+	void setClipBlack(float value) {this->clipBlack = value;}
+	void setClipWhite(float value) {this->clipWhite = value;}
+
+	void executePixel(float *color, float x, float y, PixelSampler sampler, MemoryBuffer *inputBuffers[]);
+};
+
+#endif
diff --git a/source/blender/compositor/operations/COM_KeyingOperation.cpp b/source/blender/compositor/operations/COM_KeyingOperation.cpp
index 5f1c9d0640d..7d7ac378bed 100644
--- a/source/blender/compositor/operations/COM_KeyingOperation.cpp
+++ b/source/blender/compositor/operations/COM_KeyingOperation.cpp
@@ -28,26 +28,33 @@
 #include "BLI_listbase.h"
 #include "BLI_math.h"
 
-static int get_pixel_primary_channel(float *pixel)
+static int get_pixel_primary_channel(float pixelColor[4])
 {
-	float max_value = MAX3(pixel[0], pixel[1], pixel[2]);
+	float max_value = MAX3(pixelColor[0], pixelColor[1], pixelColor[2]);
 
-	if (max_value == pixel[0])
+	if (max_value == pixelColor[0])
 		return 0;
-	else if (max_value == pixel[1])
+	else if (max_value == pixelColor[1])
 		return 1;
 
 	return 2;
 }
 
-static float get_pixel_saturation(float *pixel, float screen_balance)
+static float get_pixel_saturation(float pixelColor[4], float screen_balance, int primary_channel)
 {
-	float min = MIN3(pixel[0], pixel[1], pixel[2]);
-	float max = MAX3(pixel[0], pixel[1], pixel[2]);
-	float mid = pixel[0] + pixel[1] + pixel[2] - min - max;
-	float val = (1.0f - screen_balance) * min + screen_balance * mid;
+	int other_1 = (primary_channel + 1) % 3;
+	int other_2 = (primary_channel + 2) % 3;
 
-	return (max - val) * (1.0f - val) * (1.0f - val);
+	float min = MIN2(pixelColor[other_1], pixelColor[other_2]);
+	float max = MAX2(pixelColor[other_1], pixelColor[other_2]);
+	float val = screen_balance * min + (1.0f - screen_balance) * max;
+
+	// original formula, also used by brecht
+	// works a bit crappy in areas with values > 1.0
+	// return (pixelColor[primary_channel] - val) * fabsf(1.0f - val);
+
+	// sergey's test formula
+	return pixelColor[1] - (pixelColor[0] + pixelColor[1]) * 0.5f;
 }
 
 KeyingOperation::KeyingOperation(): NodeOperation()
@@ -57,8 +64,6 @@ KeyingOperation::KeyingOperation(): NodeOperation()
 	this->addOutputSocket(COM_DT_VALUE);
 
 	this->screenBalance = 0.5f;
-	this->clipBlack = 0.0f;
-	this->clipWhite = 1.0f;
 
 	this->pixelReader = NULL;
 	this->screenReader = NULL;
@@ -84,31 +89,20 @@ void KeyingOperation::executePixel(float *color, float x, float y, PixelSampler
 	this->pixelReader->read(pixelColor, x, y, sampler, inputBuffers);
 	this->screenReader->read(screenColor, x, y, sampler, inputBuffers);
 
-	float saturation = get_pixel_saturation(pixelColor, this->screenBalance);
-	float screen_saturation = get_pixel_saturation(screenColor, this->screenBalance);
-	int primary_channel = get_pixel_primary_channel(pixelColor);
-	int screen_primary_channel = get_pixel_primary_channel(screenColor);
+	int primary_channel = get_pixel_primary_channel(screenColor);
+
+	float saturation = get_pixel_saturation(pixelColor, this->screenBalance, primary_channel);
+	float screen_saturation = get_pixel_saturation(screenColor, this->screenBalance, primary_channel);
 
-	if (primary_channel != screen_primary_channel) {
-		/* different main channel means pixel is on foreground */
+	if (saturation < 0) {
 		color[0] = 1.0f;
 	}
-        else if (saturation >= screen_saturation) {
-		/* saturation of main channel is more than screen, definitely a background */
+	else if (saturation >= screen_saturation) {
 		color[0] = 0.0f;
 	}
 	else {
-		float distance;
-
-		distance = 1.0f - saturation / screen_saturation;
-
-		color[0] = distance * distance;
+		float distance = 1.0f - saturation / screen_saturation;
 
-		if (color[0] < this->clipBlack)
-			color[0] = 0.0f;
-		else if (color[0] >= this->clipWhite)
-			color[0] = 1.0f;
-		else
-			color[0] = (color[0] - this->clipBlack) / (this->clipWhite - this->clipBlack);
+		color[0] = distance;
 	}
 }
diff --git a/source/blender/compositor/operations/COM_KeyingOperation.h b/source/blender/compositor/operations/COM_KeyingOperation.h
index 546ff355573..0ce6481b835 100644
--- a/source/blender/compositor/operations/COM_KeyingOperation.h
+++ b/source/blender/compositor/operations/COM_KeyingOperation.h
@@ -39,8 +39,6 @@ protected:
 	SocketReader *pixelReader;
 	SocketReader *screenReader;
 	float screenBalance;
-	float clipBlack;
-	float clipWhite;
 
 public:
 	KeyingOperation();
@@ -48,9 +46,6 @@ public:
 	void initExecution();
 	void deinitExecution();
 
-	void setClipBlack(float value) {this->clipBlack = value;}
-	void setClipWhite(float value) {this->clipWhite = value;}
-
 	void executePixel(float *color, float x, float y, PixelSampler sampler, MemoryBuffer *inputBuffers[]);
 };