Keying node: assume overexposured pixels as foreground

Screens are usually doesn't have overexposured pixels and all
saturation / gradient math was written assuming that all channels
are withing 0 .. 1 range and in cases when some channel exceeds
this range matte could be completely wrong.

Added special check for overesposure and assume such pixels as
definitely foreground.

Also fixed minimal value for edge kernel size.

1 files changed, 30 insertions, 11 deletions

diff --git a/source/blender/compositor/operations/COM_KeyingOperation.cpp b/source/blender/compositor/operations/COM_KeyingOperation.cpp
index 0a450cc3bf8..ae2913350f9 100644
--- a/source/blender/compositor/operations/COM_KeyingOperation.cpp
+++ b/source/blender/compositor/operations/COM_KeyingOperation.cpp
@@ -98,23 +98,42 @@ void KeyingOperation::executePixel(float *color, float x, float y, PixelSampler
 
 	int primary_channel = get_pixel_primary_channel(screenColor);
 
-	float saturation = get_pixel_saturation(pixelColor, this->screenBalance, primary_channel);
-	float screen_saturation = get_pixel_saturation(screenColor, this->screenBalance, primary_channel);
-
-	if (saturation < 0) {
+	if (pixelColor[primary_channel] > 1.0f) {
+		/* overexposure doesn't happen on screen itself and usually happens
+		 * on light sources in the shot, this need to be checked separately
+		 * because saturation and falloff calculation is based on the fact
+		 * that pixels are not overexposured
+		 */
 		color[0] = 1.0f;
 	}
-	else if (saturation >= screen_saturation) {
-		color[0] = 0.0f;
-	}
 	else {
-		float distance = 1.0f - saturation / screen_saturation;
-
-		color[0] = distance;
+		float saturation = get_pixel_saturation(pixelColor, this->screenBalance, primary_channel);
+		float screen_saturation = get_pixel_saturation(screenColor, this->screenBalance, primary_channel);
+
+		if (saturation < 0) {
+			/* means main channel of pixel is different from screen,
+			 * assume this is completely a foreground
+			 */
+			color[0] = 1.0f;
+		}
+		else if (saturation >= screen_saturation) {
+			/* matched main channels and higher saturation on pixel
+			 * is treated as completely background
+			 */
+			color[0] = 0.0f;
+		}
+		else {
+			/* nice alpha falloff on edges */
+			float distance = 1.0f - saturation / screen_saturation;
+
+			color[0] = distance;
+		}
 	}
 
-	color[0] *= (1.0f - garbageValue[0]);
+	/* apply garbage matte */
+	color[0] = MIN2(color[0], 1.0f - garbageValue[0]);
 
+	/* apply core matte */
 	color[0] = MAX2(color[0], coreValue[0]);
 }