Keying: apply garbage / core mattes after clamping

Applying this mattes before clamping produced ugly outline around
matte boundaries.

3 files changed, 38 insertions, 22 deletions

diff --git a/source/blender/compositor/nodes/COM_KeyingNode.cpp b/source/blender/compositor/nodes/COM_KeyingNode.cpp
index efd50a44a51..6bc9afba32c 100644
--- a/source/blender/compositor/nodes/COM_KeyingNode.cpp
+++ b/source/blender/compositor/nodes/COM_KeyingNode.cpp
@@ -30,6 +30,8 @@
 #include "COM_KeyingDespillOperation.h"
 #include "COM_KeyingClipOperation.h"
 
+#include "COM_MathBaseOperation.h"
+
 #include "COM_SeparateChannelOperation.h"
 #include "COM_CombineChannelsOperation.h"
 #include "COM_ConvertRGBToYCCOperation.h"
@@ -239,8 +241,6 @@ void KeyingNode::convertToOperations(ExecutionSystem *graph, CompositorContext *
 	keyingOperation->setScreenBalance(keying_data->screen_balance);
 
 	inputScreen->relinkConnections(keyingOperation->getInputSocket(1), 1, graph);
-	inputGarbageMatte->relinkConnections(keyingOperation->getInputSocket(2), 2, graph);
-	inputCoreMatte->relinkConnections(keyingOperation->getInputSocket(3), 3, graph);
 
 	if (keying_data->blur_pre) {
 		/* chroma preblur operation for input of keying operation  */
@@ -256,18 +256,54 @@ void KeyingNode::convertToOperations(ExecutionSystem *graph, CompositorContext *
 
 	postprocessedMatte = keyingOperation->getOutputSocket();
 
+	/* black / white clipping */
 	if (keying_data->clip_black > 0.0f || keying_data->clip_white < 1.0f) {
 		postprocessedMatte = setupClip(graph, postprocessedMatte,
 		                               keying_data->edge_kernel_radius, keying_data->edge_kernel_tolerance,
 		                               keying_data->clip_black, keying_data->clip_white, false);
 	}
 
+	/* output edge matte */
 	if (outputEdges->isConnected()) {
 		edgesMatte = setupClip(graph, postprocessedMatte,
 		                       keying_data->edge_kernel_radius, keying_data->edge_kernel_tolerance,
 		                       keying_data->clip_black, keying_data->clip_white, true);
 	}
 
+	/* apply garbage matte */
+	if (inputGarbageMatte->isConnected()) {
+		SetValueOperation *valueOperation = new SetValueOperation();
+		MathSubtractOperation *subtractOperation = new MathSubtractOperation();
+		MathMinimumOperation *minOperation = new MathMinimumOperation();
+
+		valueOperation->setValue(1.0f);
+
+		addLink(graph, valueOperation->getOutputSocket(), subtractOperation->getInputSocket(0));
+		inputGarbageMatte->relinkConnections(subtractOperation->getInputSocket(1), 0, graph);
+
+		addLink(graph, subtractOperation->getOutputSocket(), minOperation->getInputSocket(0));
+		addLink(graph, postprocessedMatte, minOperation->getInputSocket(1));
+
+		postprocessedMatte = minOperation->getOutputSocket();
+
+		graph->addOperation(valueOperation);
+		graph->addOperation(subtractOperation);
+		graph->addOperation(minOperation);
+	}
+
+	/* apply core matte */
+	if (inputCoreMatte->isConnected()) {
+		MathMaximumOperation *maxOperation = new MathMaximumOperation();
+
+		inputCoreMatte->relinkConnections(maxOperation->getInputSocket(0), 0, graph);
+
+		addLink(graph, postprocessedMatte, maxOperation->getInputSocket(1));
+
+		postprocessedMatte = maxOperation->getOutputSocket();
+
+		graph->addOperation(maxOperation);
+	}
+
 	/* apply blur on matte if needed */
 	if (keying_data->blur_post)
 		postprocessedMatte = setupPostBlur(graph, postprocessedMatte, keying_data->blur_post);
diff --git a/source/blender/compositor/operations/COM_KeyingOperation.cpp b/source/blender/compositor/operations/COM_KeyingOperation.cpp
index 5912c206a84..ed9e4f1df16 100644
--- a/source/blender/compositor/operations/COM_KeyingOperation.cpp
+++ b/source/blender/compositor/operations/COM_KeyingOperation.cpp
@@ -57,45 +57,33 @@ KeyingOperation::KeyingOperation() : NodeOperation()
 {
 	this->addInputSocket(COM_DT_COLOR);
 	this->addInputSocket(COM_DT_COLOR);
-	this->addInputSocket(COM_DT_VALUE);
-	this->addInputSocket(COM_DT_VALUE);
 	this->addOutputSocket(COM_DT_VALUE);
 
 	this->m_screenBalance = 0.5f;
 
 	this->m_pixelReader = NULL;
 	this->m_screenReader = NULL;
-	this->m_garbageReader = NULL;
-	this->m_coreReader = NULL;
 }
 
 void KeyingOperation::initExecution()
 {
 	this->m_pixelReader = this->getInputSocketReader(0);
 	this->m_screenReader = this->getInputSocketReader(1);
-	this->m_garbageReader = this->getInputSocketReader(2);
-	this->m_coreReader = this->getInputSocketReader(3);
 }
 
 void KeyingOperation::deinitExecution()
 {
 	this->m_pixelReader = NULL;
 	this->m_screenReader = NULL;
-	this->m_garbageReader = NULL;
-	this->m_coreReader = NULL;
 }
 
 void KeyingOperation::executePixel(float *color, float x, float y, PixelSampler sampler)
 {
 	float pixelColor[4];
 	float screenColor[4];
-	float garbageValue[4];
-	float coreValue[4];
 
 	this->m_pixelReader->read(pixelColor, x, y, sampler);
 	this->m_screenReader->read(screenColor, x, y, sampler);
-	this->m_garbageReader->read(garbageValue, x, y, sampler);
-	this->m_coreReader->read(coreValue, x, y, sampler);
 
 	int primary_channel = get_pixel_primary_channel(screenColor);
 
@@ -130,10 +118,4 @@ void KeyingOperation::executePixel(float *color, float x, float y, PixelSampler
 			color[0] = distance;
 		}
 	}
-
-	/* apply garbage matte */
-	color[0] = MIN2(color[0], 1.0f - garbageValue[0]);
-
-	/* apply core matte */
-	color[0] = MAX2(color[0], coreValue[0]);
 }
diff --git a/source/blender/compositor/operations/COM_KeyingOperation.h b/source/blender/compositor/operations/COM_KeyingOperation.h
index 413aaf6a81e..5ca4db7fc75 100644
--- a/source/blender/compositor/operations/COM_KeyingOperation.h
+++ b/source/blender/compositor/operations/COM_KeyingOperation.h
@@ -38,8 +38,6 @@ class KeyingOperation : public NodeOperation {
 protected:
 	SocketReader *m_pixelReader;
 	SocketReader *m_screenReader;
-	SocketReader *m_garbageReader;
-	SocketReader *m_coreReader;
 
 	float m_screenBalance;