Compositor: Ensure keying node result is pre-multiplied

Historically the result of the keying node was violating alpha
pre-multiplication rules in Blender: it was simply overriding
the alpha channel of input.

This change makes it so keying node mixes alpha into the input,
which solves the following issues:

- The result is properly pre-multiplied, no need in separate
  alpha-convert node anymore.

- Allows to more easily stack keying nodes.
  This usecase was never really investigated, but since previously
  alpha is always overwritten it was never possible to easily stack
  nodes. Now it is at something to be tried.

Unfortunately, this breaks compatibility with existing files, where
alpha-convert node is to be manually removed.

From implementation side this is done as a dedicated operation since
there was no ready-to-use operation. Maybe in the future it might
be replaced with some sort of vector math node.

Reviewed By: brecht

Differential Revision: https://developer.blender.org/D9211

4 files changed, 98 insertions, 2 deletions

diff --git a/source/blender/compositor/CMakeLists.txt b/source/blender/compositor/CMakeLists.txt
index d524f4a0e1e..54dd121952b 100644
--- a/source/blender/compositor/CMakeLists.txt
+++ b/source/blender/compositor/CMakeLists.txt
@@ -353,6 +353,8 @@ set(SRC
   operations/COM_KeyingDespillOperation.h
   operations/COM_KeyingOperation.cpp
   operations/COM_KeyingOperation.h
+  operations/COM_KeyingSetAlphaOperation.cpp
+  operations/COM_KeyingSetAlphaOperation.h
 
   operations/COM_ColorSpillOperation.cpp
   operations/COM_ColorSpillOperation.h
diff --git a/source/blender/compositor/nodes/COM_KeyingNode.cpp b/source/blender/compositor/nodes/COM_KeyingNode.cpp
index b75d06763b7..a0ccaf0b9da 100644
--- a/source/blender/compositor/nodes/COM_KeyingNode.cpp
+++ b/source/blender/compositor/nodes/COM_KeyingNode.cpp
@@ -32,7 +32,7 @@
 
 #include "COM_DilateErodeOperation.h"
 
-#include "COM_SetAlphaOperation.h"
+#include "COM_KeyingSetAlphaOperation.h"
 
 #include "COM_GaussianAlphaXBlurOperation.h"
 #include "COM_GaussianAlphaYBlurOperation.h"
@@ -322,7 +322,7 @@ void KeyingNode::convertToOperations(NodeConverter &converter,
   }
 
   /* set alpha channel to output image */
-  SetAlphaOperation *alphaOperation = new SetAlphaOperation();
+  KeyingSetAlphaOperation *alphaOperation = new KeyingSetAlphaOperation();
   converter.addOperation(alphaOperation);
 
   converter.mapInputSocket(inputImage, alphaOperation->getInputSocket(0));
diff --git a/source/blender/compositor/operations/COM_KeyingSetAlphaOperation.cpp b/source/blender/compositor/operations/COM_KeyingSetAlphaOperation.cpp
new file mode 100644
index 00000000000..dff88d49e80
--- /dev/null
+++ b/source/blender/compositor/operations/COM_KeyingSetAlphaOperation.cpp
@@ -0,0 +1,55 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * Copyright 2020, Blender Foundation.
+ */
+
+#include "COM_KeyingSetAlphaOperation.h"
+
+KeyingSetAlphaOperation::KeyingSetAlphaOperation() : NodeOperation()
+{
+  this->addInputSocket(COM_DT_COLOR);
+  this->addInputSocket(COM_DT_VALUE);
+  this->addOutputSocket(COM_DT_COLOR);
+
+  this->m_inputColor = NULL;
+  this->m_inputAlpha = NULL;
+}
+
+void KeyingSetAlphaOperation::initExecution()
+{
+  this->m_inputColor = getInputSocketReader(0);
+  this->m_inputAlpha = getInputSocketReader(1);
+}
+
+void KeyingSetAlphaOperation::executePixelSampled(float output[4],
+                                                  float x,
+                                                  float y,
+                                                  PixelSampler sampler)
+{
+  float color_input[4];
+  float alpha_input[4];
+
+  this->m_inputColor->readSampled(color_input, x, y, sampler);
+  this->m_inputAlpha->readSampled(alpha_input, x, y, sampler);
+
+  mul_v4_v4fl(output, color_input, alpha_input[0]);
+}
+
+void KeyingSetAlphaOperation::deinitExecution()
+{
+  this->m_inputColor = NULL;
+  this->m_inputAlpha = NULL;
+}
diff --git a/source/blender/compositor/operations/COM_KeyingSetAlphaOperation.h b/source/blender/compositor/operations/COM_KeyingSetAlphaOperation.h
new file mode 100644
index 00000000000..b786240f215
--- /dev/null
+++ b/source/blender/compositor/operations/COM_KeyingSetAlphaOperation.h
@@ -0,0 +1,39 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * Copyright 2020, Blender Foundation.
+ */
+
+#pragma once
+
+#include "COM_NodeOperation.h"
+
+/**
+ * Operation which is used by keying node to modify image's alpha channels.
+ * It keeps color properly pre-multiplied.
+ */
+class KeyingSetAlphaOperation : public NodeOperation {
+ private:
+  SocketReader *m_inputColor;
+  SocketReader *m_inputAlpha;
+
+ public:
+  KeyingSetAlphaOperation();
+
+  void executePixelSampled(float output[4], float x, float y, PixelSampler sampler);
+
+  void initExecution();
+  void deinitExecution();
+};