Fix T39799: Backdrop (compositor) ignores alpha.

This issue is because of a somewhat "special" behavior in old code, which got lost during rB09874df:

There was a variant of the `relinkConnections` function which would leave the socket completely unconnected. This is not a valid state really (given that each unconnected input must otherwise connected to a constant `Set` type node), but was used as a way to distinguish connected alpha/depth sockets in composite and viewer output nodes.
https://developer.blender.org/diffusion/B/browse/master/source/blender/compositor/intern/COM_InputSocket.cpp;28a829893c702918afc5ac1945a06eaefa611594$69

After the large cleanup patch ({D309}) every socket is now automatically connected to a constant, such that `getInputSocketReader` will never return a NULL pointer. This breaks the previous test method, which needs to be replaced by more explicit flags. Luckily this was done only for very few output nodes (Composite, Viewer, Output-File). These now use the regular SetValueOperation default in case "use alpha" is disabled, but set this to an explicit 1.0 value instead of mapping to the node socket.

12 files changed, 87 insertions, 46 deletions

diff --git a/source/blender/compositor/intern/COM_NodeConverter.cpp b/source/blender/compositor/intern/COM_NodeConverter.cpp
index 833fcedbc53..5965eade389 100644
--- a/source/blender/compositor/intern/COM_NodeConverter.cpp
+++ b/source/blender/compositor/intern/COM_NodeConverter.cpp
@@ -103,6 +103,33 @@ NodeOperationInput *NodeConverter::addOutputProxy(NodeOutput *output)
 	return proxy->getInputSocket(0);
 }
 
+void NodeConverter::addInputValue(NodeOperationInput *input, float value)
+{
+	SetValueOperation *operation = new SetValueOperation();
+	operation->setValue(value);
+	
+	m_builder->addOperation(operation);
+	m_builder->addLink(operation->getOutputSocket(), input);
+}
+
+void NodeConverter::addInputColor(NodeOperationInput *input, const float value[4])
+{
+	SetColorOperation *operation = new SetColorOperation();
+	operation->setChannels(value);
+	
+	m_builder->addOperation(operation);
+	m_builder->addLink(operation->getOutputSocket(), input);
+}
+
+void NodeConverter::addInputVector(NodeOperationInput *input, const float value[3])
+{
+	SetVectorOperation *operation = new SetVectorOperation();
+	operation->setVector(value);
+	
+	m_builder->addOperation(operation);
+	m_builder->addLink(operation->getOutputSocket(), input);
+}
+
 void NodeConverter::addOutputValue(NodeOutput *output, float value)
 {
 	SetValueOperation *operation = new SetValueOperation();
diff --git a/source/blender/compositor/intern/COM_NodeConverter.h b/source/blender/compositor/intern/COM_NodeConverter.h
index a67dafde5e1..cad8408a2bf 100644
--- a/source/blender/compositor/intern/COM_NodeConverter.h
+++ b/source/blender/compositor/intern/COM_NodeConverter.h
@@ -75,6 +75,13 @@ public:
 	 */
 	NodeOperationInput *addOutputProxy(NodeOutput *output);
 	
+	/** Define a constant input value. */
+	void addInputValue(NodeOperationInput *input, float value);
+	/** Define a constant input color. */
+	void addInputColor(NodeOperationInput *input, const float value[4]);
+	/** Define a constant input vector. */
+	void addInputVector(NodeOperationInput *input, const float value[3]);
+	
 	/** Define a constant output value. */
 	void addOutputValue(NodeOutput *output, float value);
 	/** Define a constant output color. */
diff --git a/source/blender/compositor/intern/COM_NodeOperationBuilder.cpp b/source/blender/compositor/intern/COM_NodeOperationBuilder.cpp
index fdea3206d8e..a90bac847c0 100644
--- a/source/blender/compositor/intern/COM_NodeOperationBuilder.cpp
+++ b/source/blender/compositor/intern/COM_NodeOperationBuilder.cpp
@@ -85,7 +85,11 @@ void NodeOperationBuilder::convertToOperations(ExecutionSystem *system)
 		const OpInputs &op_to_list = find_operation_inputs(inverse_input_map, to);
 		if (!op_from || op_to_list.empty()) {
 			/* XXX allow this? error/debug message? */
-			BLI_assert(false);
+			//BLI_assert(false);
+			/* XXX note: this can happen with certain nodes (e.g. OutputFile)
+			 * which only generate operations in certain circumstances (rendering)
+			 * just let this pass silently for now ...
+			 */
 			continue;
 		}
 		
diff --git a/source/blender/compositor/nodes/COM_CompositorNode.cpp b/source/blender/compositor/nodes/COM_CompositorNode.cpp
index 868528ade81..3d79eb693ea 100644
--- a/source/blender/compositor/nodes/COM_CompositorNode.cpp
+++ b/source/blender/compositor/nodes/COM_CompositorNode.cpp
@@ -34,6 +34,7 @@ void CompositorNode::convertToOperations(NodeConverter &converter, const Composi
 	bNode *editorNode = this->getbNode();
 	bool is_active = (editorNode->flag & NODE_DO_OUTPUT_RECALC) ||
 	                 context.isRendering();
+	bool ignore_alpha = editorNode->custom2 & CMP_NODE_OUTPUT_IGNORE_ALPHA;
 
 	NodeInput *imageSocket = this->getInputSocket(0);
 	NodeInput *alphaSocket = this->getInputSocket(1);
@@ -43,12 +44,17 @@ void CompositorNode::convertToOperations(NodeConverter &converter, const Composi
 	compositorOperation->setSceneName(context.getScene()->id.name);
 	compositorOperation->setRenderData(context.getRenderData());
 	compositorOperation->setbNodeTree(context.getbNodeTree());
-	compositorOperation->setIgnoreAlpha(editorNode->custom2 & CMP_NODE_OUTPUT_IGNORE_ALPHA);
+	/* alpha socket gives either 1 or a custom alpha value if "use alpha" is enabled */
+	compositorOperation->setUseAlphaInput(ignore_alpha || alphaSocket->isLinked());
 	compositorOperation->setActive(is_active);
 	
 	converter.addOperation(compositorOperation);
 	converter.mapInputSocket(imageSocket, compositorOperation->getInputSocket(0));
-	converter.mapInputSocket(alphaSocket, compositorOperation->getInputSocket(1));
+	/* only use alpha link if "use alpha" is enabled */
+	if (ignore_alpha)
+		converter.addInputValue(compositorOperation->getInputSocket(1), 1.0f);
+	else
+		converter.mapInputSocket(alphaSocket, compositorOperation->getInputSocket(1));
 	converter.mapInputSocket(depthSocket, compositorOperation->getInputSocket(2));
 	
 	converter.addNodeInputPreview(imageSocket);
diff --git a/source/blender/compositor/nodes/COM_OutputFileNode.cpp b/source/blender/compositor/nodes/COM_OutputFileNode.cpp
index 83b138c1b07..92fa74b9a2e 100644
--- a/source/blender/compositor/nodes/COM_OutputFileNode.cpp
+++ b/source/blender/compositor/nodes/COM_OutputFileNode.cpp
@@ -56,7 +56,8 @@ void OutputFileNode::convertToOperations(NodeConverter &converter, const Composi
 			NodeInput *input = getInputSocket(i);
 			NodeImageMultiFileSocket *sockdata = (NodeImageMultiFileSocket *)input->getbNodeSocket()->storage;
 			
-			outputOperation->add_layer(sockdata->layer, input->getDataType());
+			/* note: layer becomes an empty placeholder if the input is not linked */
+			outputOperation->add_layer(sockdata->layer, input->getDataType(), input->isLinked());
 			
 			converter.mapInputSocket(input, outputOperation->getInputSocket(i));
 			
diff --git a/source/blender/compositor/nodes/COM_ViewerNode.cpp b/source/blender/compositor/nodes/COM_ViewerNode.cpp
index 5e37bf06dc2..09a3cea2da1 100644
--- a/source/blender/compositor/nodes/COM_ViewerNode.cpp
+++ b/source/blender/compositor/nodes/COM_ViewerNode.cpp
@@ -36,6 +36,7 @@ void ViewerNode::convertToOperations(NodeConverter &converter, const CompositorC
 	bNode *editorNode = this->getbNode();
 	bool is_active = (editorNode->flag & NODE_DO_OUTPUT_RECALC || context.isRendering()) &&
 	                 ((editorNode->flag & NODE_DO_OUTPUT) && this->isInActiveGroup());
+	bool ignore_alpha = editorNode->custom2 & CMP_NODE_OUTPUT_IGNORE_ALPHA;
 
 	NodeInput *imageSocket = this->getInputSocket(0);
 	NodeInput *alphaSocket = this->getInputSocket(1);
@@ -50,7 +51,8 @@ void ViewerNode::convertToOperations(NodeConverter &converter, const CompositorC
 	viewerOperation->setChunkOrder((OrderOfChunks)editorNode->custom1);
 	viewerOperation->setCenterX(editorNode->custom3);
 	viewerOperation->setCenterY(editorNode->custom4);
-	viewerOperation->setIgnoreAlpha(editorNode->custom2 & CMP_NODE_OUTPUT_IGNORE_ALPHA);
+	/* alpha socket gives either 1 or a custom alpha value if "use alpha" is enabled */
+	viewerOperation->setUseAlphaInput(ignore_alpha || alphaSocket->isLinked());
 
 	viewerOperation->setViewSettings(context.getViewSettings());
 	viewerOperation->setDisplaySettings(context.getDisplaySettings());
@@ -64,7 +66,11 @@ void ViewerNode::convertToOperations(NodeConverter &converter, const CompositorC
 
 	converter.addOperation(viewerOperation);
 	converter.mapInputSocket(imageSocket, viewerOperation->getInputSocket(0));
-	converter.mapInputSocket(alphaSocket, viewerOperation->getInputSocket(1));
+	/* only use alpha link if "use alpha" is enabled */
+	if (ignore_alpha)
+		converter.addInputValue(viewerOperation->getInputSocket(1), 1.0f);
+	else
+		converter.mapInputSocket(alphaSocket, viewerOperation->getInputSocket(1));
 	converter.mapInputSocket(depthSocket, viewerOperation->getInputSocket(2));
 
 	converter.addNodeInputPreview(imageSocket);
diff --git a/source/blender/compositor/operations/COM_CompositorOperation.cpp b/source/blender/compositor/operations/COM_CompositorOperation.cpp
index 14aba267a23..ef331a50dfd 100644
--- a/source/blender/compositor/operations/COM_CompositorOperation.cpp
+++ b/source/blender/compositor/operations/COM_CompositorOperation.cpp
@@ -48,7 +48,7 @@ CompositorOperation::CompositorOperation() : NodeOperation()
 	this->m_alphaInput = NULL;
 	this->m_depthInput = NULL;
 
-	this->m_ignoreAlpha = false;
+	this->m_useAlphaInput = false;
 	this->m_active = false;
 
 	this->m_sceneName[0] = '\0';
@@ -188,21 +188,14 @@ void CompositorOperation::executeRegion(rcti *rect, unsigned int tileNumber)
 			int input_x = x + dx, input_y = y + dy;
 
 			this->m_imageInput->readSampled(color, input_x, input_y, COM_PS_NEAREST);
-			if (this->m_ignoreAlpha) {
-				color[3] = 1.0f;
-			}
-			else {
-				if (this->m_alphaInput != NULL) {
-					this->m_alphaInput->readSampled(&(color[3]), input_x, input_y, COM_PS_NEAREST);
-				}
+			if (this->m_useAlphaInput) {
+				this->m_alphaInput->readSampled(&(color[3]), input_x, input_y, COM_PS_NEAREST);
 			}
 
 			copy_v4_v4(buffer + offset4, color);
 
-			if (this->m_depthInput != NULL) {
-				this->m_depthInput->readSampled(color, input_x, input_y, COM_PS_NEAREST);
-				zbuffer[offset] = color[0];
-			}
+			this->m_depthInput->readSampled(color, input_x, input_y, COM_PS_NEAREST);
+			zbuffer[offset] = color[0];
 			offset4 += COM_NUMBER_OF_CHANNELS;
 			offset++;
 			if (isBreaked()) {
diff --git a/source/blender/compositor/operations/COM_CompositorOperation.h b/source/blender/compositor/operations/COM_CompositorOperation.h
index d33e89ed742..771c32ffd12 100644
--- a/source/blender/compositor/operations/COM_CompositorOperation.h
+++ b/source/blender/compositor/operations/COM_CompositorOperation.h
@@ -69,7 +69,7 @@ private:
 	/**
 	 * @brief Ignore any alpha input
 	 */
-	bool m_ignoreAlpha;
+	bool m_useAlphaInput;
 
 	/**
 	 * @brief operation is active for calculating final compo result
@@ -86,7 +86,7 @@ public:
 	void deinitExecution();
 	const CompositorPriority getRenderPriority() const { return COM_PRIORITY_MEDIUM; }
 	void determineResolution(unsigned int resolution[2], unsigned int preferredResolution[2]);
-	void setIgnoreAlpha(bool value) { this->m_ignoreAlpha = value; }
+	void setUseAlphaInput(bool value) { this->m_useAlphaInput = value; }
 	void setActive(bool active) { this->m_active = active; }
 };
 #endif
diff --git a/source/blender/compositor/operations/COM_OutputFileOperation.cpp b/source/blender/compositor/operations/COM_OutputFileOperation.cpp
index db2f8445db7..92e8f309ea1 100644
--- a/source/blender/compositor/operations/COM_OutputFileOperation.cpp
+++ b/source/blender/compositor/operations/COM_OutputFileOperation.cpp
@@ -155,10 +155,12 @@ void OutputSingleLayerOperation::deinitExecution()
 }
 
 
-OutputOpenExrLayer::OutputOpenExrLayer(const char *name_, DataType datatype_)
+OutputOpenExrLayer::OutputOpenExrLayer(const char *name_, DataType datatype_, bool use_layer_)
 {
 	BLI_strncpy(this->name, name_, sizeof(this->name));
 	this->datatype = datatype_;
+	this->use_layer = use_layer_;
+	
 	/* these are created in initExecution */
 	this->outputBuffer = 0;
 	this->imageInput = 0;
@@ -174,21 +176,20 @@ OutputOpenExrMultiLayerOperation::OutputOpenExrMultiLayerOperation(
 	this->m_exr_codec = exr_codec;
 }
 
-void OutputOpenExrMultiLayerOperation::add_layer(const char *name, DataType datatype)
+void OutputOpenExrMultiLayerOperation::add_layer(const char *name, DataType datatype, bool use_layer)
 {
 	this->addInputSocket(datatype);
-	this->m_layers.push_back(OutputOpenExrLayer(name, datatype));
+	this->m_layers.push_back(OutputOpenExrLayer(name, datatype, use_layer));
 }
 
 void OutputOpenExrMultiLayerOperation::initExecution()
 {
 	for (unsigned int i = 0; i < this->m_layers.size(); ++i) {
-		SocketReader *reader = getInputSocketReader(i);
-		this->m_layers[i].imageInput = reader;
-		if (reader)
+		if (this->m_layers[i].use_layer) {
+			SocketReader *reader = getInputSocketReader(i);
+			this->m_layers[i].imageInput = reader;
 			this->m_layers[i].outputBuffer = init_buffer(this->getWidth(), this->getHeight(), this->m_layers[i].datatype);
-		else
-			this->m_layers[i].outputBuffer = NULL;
+		}
 	}
 }
 
diff --git a/source/blender/compositor/operations/COM_OutputFileOperation.h b/source/blender/compositor/operations/COM_OutputFileOperation.h
index 6bb2e7d7320..03278c5b149 100644
--- a/source/blender/compositor/operations/COM_OutputFileOperation.h
+++ b/source/blender/compositor/operations/COM_OutputFileOperation.h
@@ -62,11 +62,14 @@ public:
 
 /* extra info for OpenEXR layers */
 struct OutputOpenExrLayer {
-	OutputOpenExrLayer(const char *name, DataType datatype);
+	OutputOpenExrLayer(const char *name, DataType datatype, bool use_layer);
 	
 	char name[EXR_TOT_MAXNAME - 2];
-	float *outputBuffer;
 	DataType datatype;
+	bool use_layer;
+	
+	/* internals */
+	float *outputBuffer;
 	SocketReader *imageInput;
 };
 
@@ -85,7 +88,7 @@ private:
 public:
 	OutputOpenExrMultiLayerOperation(const RenderData *rd, const bNodeTree *tree, const char *path, char exr_codec);
 	
-	void add_layer(const char *name, DataType datatype);
+	void add_layer(const char *name, DataType datatype, bool use_layer);
 	
 	void executeRegion(rcti *rect, unsigned int tileNumber);
 	bool isOutputOperation(bool rendering) const { return true; }
diff --git a/source/blender/compositor/operations/COM_ViewerOperation.cpp b/source/blender/compositor/operations/COM_ViewerOperation.cpp
index 6579eb268a6..53c0acd781a 100644
--- a/source/blender/compositor/operations/COM_ViewerOperation.cpp
+++ b/source/blender/compositor/operations/COM_ViewerOperation.cpp
@@ -48,7 +48,7 @@ ViewerOperation::ViewerOperation() : NodeOperation()
 	this->m_doDepthBuffer = false;
 	this->m_viewSettings = NULL;
 	this->m_displaySettings = NULL;
-	this->m_ignoreAlpha = false;
+	this->m_useAlphaInput = false;
 	
 	this->addInputSocket(COM_DT_COLOR);
 	this->addInputSocket(COM_DT_VALUE);
@@ -101,19 +101,12 @@ void ViewerOperation::executeRegion(rcti *rect, unsigned int tileNumber)
 	for (y = y1; y < y2 && (!breaked); y++) {
 		for (x = x1; x < x2; x++) {
 			this->m_imageInput->readSampled(&(buffer[offset4]), x, y, COM_PS_NEAREST);
-			if (this->m_ignoreAlpha) {
-				buffer[offset4 + 3] = 1.0f;
-			}
-			else {
-				if (this->m_alphaInput != NULL) {
-					this->m_alphaInput->readSampled(alpha, x, y, COM_PS_NEAREST);
-					buffer[offset4 + 3] = alpha[0];
-				}
-			}
-			if (m_depthInput) {
-				this->m_depthInput->readSampled(depth, x, y, COM_PS_NEAREST);
-				depthbuffer[offset] = depth[0];
+			if (this->m_useAlphaInput) {
+				this->m_alphaInput->readSampled(alpha, x, y, COM_PS_NEAREST);
+				buffer[offset4 + 3] = alpha[0];
 			}
+			this->m_depthInput->readSampled(depth, x, y, COM_PS_NEAREST);
+			depthbuffer[offset] = depth[0];
 
 			offset ++;
 			offset4 += 4;
diff --git a/source/blender/compositor/operations/COM_ViewerOperation.h b/source/blender/compositor/operations/COM_ViewerOperation.h
index 34954382b82..dcc7ffa3730 100644
--- a/source/blender/compositor/operations/COM_ViewerOperation.h
+++ b/source/blender/compositor/operations/COM_ViewerOperation.h
@@ -39,7 +39,7 @@ private:
 	OrderOfChunks m_chunkOrder;
 	bool m_doDepthBuffer;
 	ImBuf *m_ibuf;
-	bool m_ignoreAlpha;
+	bool m_useAlphaInput;
 	
 	const ColorManagedViewSettings *m_viewSettings;
 	const ColorManagedDisplaySettings *m_displaySettings;
@@ -66,7 +66,7 @@ public:
 	OrderOfChunks getChunkOrder() const { return this->m_chunkOrder; }
 	const CompositorPriority getRenderPriority() const;
 	bool isViewerOperation() const { return true; }
-	void setIgnoreAlpha(bool value) { this->m_ignoreAlpha = value; }
+	void setUseAlphaInput(bool value) { this->m_useAlphaInput = value; }
 
 	void setViewSettings(const ColorManagedViewSettings *viewSettings) { this->m_viewSettings = viewSettings; }
 	void setDisplaySettings(const ColorManagedDisplaySettings *displaySettings) { this->m_displaySettings = displaySettings; }