Python API: add Python-defined node groups for shaders and compositing.

This was already supported for Cycles shader nodes, but now also works for
Eevee and compositing nodes. Instead of a generic NodeCustomGroup, now
there is ShaderNodeCustomGroup and CompositorNodeCustomGroup that can be
subclassed and registered.

Differential Revision: https://developer.blender.org/D4370

17 files changed, 107 insertions, 24 deletions

diff --git a/intern/cycles/blender/blender_shader.cpp b/intern/cycles/blender/blender_shader.cpp
index b04a0394141..44ff29d4bf8 100644
--- a/intern/cycles/blender/blender_shader.cpp
+++ b/intern/cycles/blender/blender_shader.cpp
@@ -1050,13 +1050,18 @@ static void add_nodes(Scene *scene,
 				graph->add(proxy);
 			}
 		}
-		else if(b_node->is_a(&RNA_ShaderNodeGroup) || b_node->is_a(&RNA_NodeCustomGroup)) {
+		else if(b_node->is_a(&RNA_ShaderNodeGroup) ||
+		        b_node->is_a(&RNA_NodeCustomGroup) ||
+		        b_node->is_a(&RNA_ShaderNodeCustomGroup)) {
 
 			BL::ShaderNodeTree b_group_ntree(PointerRNA_NULL);
 			if(b_node->is_a(&RNA_ShaderNodeGroup))
 				b_group_ntree = BL::ShaderNodeTree(((BL::NodeGroup)(*b_node)).node_tree());
-			else
+			else if (b_node->is_a(&RNA_NodeCustomGroup))
 				b_group_ntree = BL::ShaderNodeTree(((BL::NodeCustomGroup)(*b_node)).node_tree());
+			else
+				b_group_ntree = BL::ShaderNodeTree(((BL::ShaderNodeCustomGroup)(*b_node)).node_tree());
+
 			ProxyMap group_proxy_input_map, group_proxy_output_map;
 
 			/* Add a proxy node for each socket
diff --git a/release/scripts/startup/nodeitems_builtins.py b/release/scripts/startup/nodeitems_builtins.py
index 99df0167557..60772062dcf 100644
--- a/release/scripts/startup/nodeitems_builtins.py
+++ b/release/scripts/startup/nodeitems_builtins.py
@@ -99,7 +99,9 @@ def node_group_items(context):
         # filter out recursive groups
         if contains_group(group, ntree):
             continue
-
+        # filter out hidden nodetrees
+        if group.name.startswith('.'):
+            continue
         yield NodeItem(node_tree_group_type[group.bl_idname],
                        group.name,
                        {"node_tree": "bpy.data.node_groups[%r]" % group.name})
diff --git a/source/blender/blenkernel/BKE_node.h b/source/blender/blenkernel/BKE_node.h
index 951962d26f1..80b0f9a023f 100644
--- a/source/blender/blenkernel/BKE_node.h
+++ b/source/blender/blenkernel/BKE_node.h
@@ -618,6 +618,7 @@ bool BKE_node_is_connected_to_output(struct bNodeTree *ntree, struct bNode *node
 #define NODE_REROUTE	6
 #define NODE_GROUP_INPUT	7
 #define NODE_GROUP_OUTPUT	8
+#define NODE_CUSTOM_GROUP	9
 
 void BKE_node_tree_unlink_id(ID *id, struct bNodeTree *ntree);
 
diff --git a/source/blender/blenkernel/intern/material.c b/source/blender/blenkernel/intern/material.c
index 90b25e8fd49..88a96ff098e 100644
--- a/source/blender/blenkernel/intern/material.c
+++ b/source/blender/blenkernel/intern/material.c
@@ -1038,7 +1038,7 @@ static int count_texture_nodes_recursive(bNodeTree *nodetree)
 		if (node->typeinfo->nclass == NODE_CLASS_TEXTURE && node->typeinfo->type == SH_NODE_TEX_IMAGE && node->id) {
 			tex_nodes++;
 		}
-		else if (node->type == NODE_GROUP && node->id) {
+		else if (ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP) && node->id) {
 			/* recurse into the node group and see if it contains any textures */
 			tex_nodes += count_texture_nodes_recursive((bNodeTree *)node->id);
 		}
@@ -1073,7 +1073,7 @@ static void fill_texpaint_slots_recursive(bNodeTree *nodetree, bNode *active_nod
 			}
 			(*index)++;
 		}
-		else if (node->type == NODE_GROUP && node->id) {
+		else if (ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP) && node->id) {
 			/* recurse into the node group and see if it contains any textures */
 			fill_texpaint_slots_recursive((bNodeTree *)node->id, active_node, ma, index);
 		}
diff --git a/source/blender/blenkernel/intern/node.c b/source/blender/blenkernel/intern/node.c
index 57dcbcbebf8..d4dd74a0e2d 100644
--- a/source/blender/blenkernel/intern/node.c
+++ b/source/blender/blenkernel/intern/node.c
@@ -1804,7 +1804,7 @@ static void free_localized_node_groups(bNodeTree *ntree)
 		return;
 
 	for (node = ntree->nodes.first; node; node = node->next) {
-		if (node->type == NODE_GROUP && node->id) {
+		if ((ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) && node->id) {
 			bNodeTree *ngroup = (bNodeTree *)node->id;
 			ntreeFreeTree(ngroup);
 			MEM_freeN(ngroup);
@@ -2046,7 +2046,7 @@ bNodeTree *ntreeLocalize(bNodeTree *ntree)
 		         LIB_ID_COPY_NO_ANIMDATA));
 
 		for (node = ltree->nodes.first; node; node = node->next) {
-			if (node->type == NODE_GROUP && node->id) {
+			if ((ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) && node->id) {
 				node->id = (ID *)ntreeLocalize((bNodeTree *)node->id);
 			}
 		}
@@ -2369,7 +2369,7 @@ bool ntreeHasTree(const bNodeTree *ntree, const bNodeTree *lookup)
 		return true;
 
 	for (node = ntree->nodes.first; node; node = node->next)
-		if (node->type == NODE_GROUP && node->id)
+		if (ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP) && node->id)
 			if (ntreeHasTree((bNodeTree *)node->id, lookup))
 				return true;
 
diff --git a/source/blender/compositor/intern/COM_NodeGraph.cpp b/source/blender/compositor/intern/COM_NodeGraph.cpp
index ba0d225fdd7..f23b186416d 100644
--- a/source/blender/compositor/intern/COM_NodeGraph.cpp
+++ b/source/blender/compositor/intern/COM_NodeGraph.cpp
@@ -128,7 +128,7 @@ void NodeGraph::add_bNode(const CompositorContext &context, bNodeTree *b_ntree,
 	}
 
 	/* special node types */
-	if (b_node->type == NODE_GROUP) {
+	if (ELEM(b_node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) {
 		add_proxies_group(context, b_node, key);
 	}
 	else if (b_node->type == NODE_REROUTE) {
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc b/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
index ecb2dad292a..09a79db2dbd 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_nodes.cc
@@ -1505,7 +1505,7 @@ void DepsgraphNodeBuilder::build_nodetree(bNodeTree *ntree)
 		else if (id_type == ID_MC) {
 			build_movieclip((MovieClip *)id);
 		}
-		else if (bnode->type == NODE_GROUP) {
+		else if (ELEM(bnode->type, NODE_GROUP, NODE_CUSTOM_GROUP)) {
 			bNodeTree *group_ntree = (bNodeTree *)id;
 			build_nodetree(group_ntree);
 		}
diff --git a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
index ef5ec6a9949..3c70cdd5bfd 100644
--- a/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
+++ b/source/blender/depsgraph/intern/builder/deg_builder_relations.cc
@@ -2420,7 +2420,7 @@ void DepsgraphRelationBuilder::build_nodetree(bNodeTree *ntree)
 		else if (id_type == ID_MC) {
 			build_movieclip((MovieClip *)id);
 		}
-		else if (bnode->type == NODE_GROUP) {
+		else if (ELEM(bnode->type, NODE_GROUP, NODE_CUSTOM_GROUP)) {
 			bNodeTree *group_ntree = (bNodeTree *)id;
 			build_nodetree(group_ntree);
 			ComponentKey group_shading_key(&group_ntree->id,
diff --git a/source/blender/editors/space_node/node_relationships.c b/source/blender/editors/space_node/node_relationships.c
index e0fce2a5367..d81c764af81 100644
--- a/source/blender/editors/space_node/node_relationships.c
+++ b/source/blender/editors/space_node/node_relationships.c
@@ -124,7 +124,7 @@ static bool node_group_has_output_dfs(bNode *node)
 
 static bool node_group_has_output(Main *bmain, bNode *node)
 {
-	BLI_assert(node->type == NODE_GROUP);
+	BLI_assert(ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP));
 	bNodeTree *ntree = (bNodeTree *)node->id;
 	if (ntree == NULL) {
 		return false;
@@ -153,7 +153,7 @@ bool node_connected_to_output(Main *bmain, bNodeTree *ntree, bNode *node)
 		 * We could make check more grained here by taking which socket the node
 		 * is connected to and so eventually.
 		 */
-		if (current_node->type == NODE_GROUP) {
+		if (ELEM(current_node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) {
 			if (current_node->id != NULL &&
 			    ntree_has_drivers((bNodeTree *)current_node->id))
 			{
diff --git a/source/blender/editors/space_node/node_templates.c b/source/blender/editors/space_node/node_templates.c
index b3edf98c470..10cb3ba8d99 100644
--- a/source/blender/editors/space_node/node_templates.c
+++ b/source/blender/editors/space_node/node_templates.c
@@ -68,7 +68,7 @@ typedef struct NodeLinkItem {
  */
 static bool node_link_item_compare(bNode *node, NodeLinkItem *item)
 {
-	if (node->type == NODE_GROUP) {
+	if (ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) {
 		return (node->id == (ID *)item->ngroup);
 	}
 	else
@@ -77,7 +77,7 @@ static bool node_link_item_compare(bNode *node, NodeLinkItem *item)
 
 static void node_link_item_apply(Main *bmain, bNode *node, NodeLinkItem *item)
 {
-	if (node->type == NODE_GROUP) {
+	if (ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) {
 		node->id = (ID *)item->ngroup;
 		ntreeUpdateTree(bmain, item->ngroup);
 	}
diff --git a/source/blender/makesrna/intern/rna_nodetree.c b/source/blender/makesrna/intern/rna_nodetree.c
index 74f47f76dc3..591cabac98f 100644
--- a/source/blender/makesrna/intern/rna_nodetree.c
+++ b/source/blender/makesrna/intern/rna_nodetree.c
@@ -192,6 +192,8 @@ static const EnumPropertyItem prop_shader_output_target_items[] = {
 
 #include "NOD_common.h"
 #include "NOD_socket.h"
+#include "NOD_shader.h"
+#include "NOD_composite.h"
 
 #include "RE_engine.h"
 #include "RE_pipeline.h"
@@ -451,6 +453,13 @@ static const EnumPropertyItem *rna_node_static_type_itemf(bContext *UNUSED(C), P
 	tmp.icon = ICON_NONE;
 	RNA_enum_item_add(&item, &totitem, &tmp);
 
+	tmp.value = NODE_CUSTOM_GROUP;
+	tmp.identifier = "CUSTOM GROUP";
+	tmp.name = "CustomGroup";
+	tmp.description = "Custom Group Node";
+	tmp.icon = ICON_NONE;
+	RNA_enum_item_add(&item, &totitem, &tmp);
+
 	tmp.value = NODE_UNDEFINED;
 	tmp.identifier = "UNDEFINED";
 	tmp.name = "UNDEFINED";
@@ -2460,6 +2469,50 @@ static StructRNA *rna_NodeCustomGroup_register(
 	return nt->ext.srna;
 }
 
+static StructRNA *rna_ShaderNodeCustomGroup_register(
+		Main * bmain, ReportList *reports,
+		void *data, const char *identifier,
+		StructValidateFunc validate, StructCallbackFunc call, StructFreeFunc free)
+{
+	bNodeType * nt = rna_Node_register_base(bmain, reports, &RNA_ShaderNodeCustomGroup, data, identifier, validate, call, free);
+
+	if (!nt)
+		return NULL;
+
+	nt->verifyfunc = node_group_verify;
+	nt->type = NODE_CUSTOM_GROUP;
+
+	register_node_type_sh_custom_group(nt);
+
+	nodeRegisterType(nt);
+
+	WM_main_add_notifier(NC_NODE | NA_EDITED, NULL);
+
+	return nt->ext.srna;
+}
+
+static StructRNA *rna_CompositorNodeCustomGroup_register(
+		Main *bmain, ReportList *reports,
+		void *data, const char *identifier,
+		StructValidateFunc validate, StructCallbackFunc call, StructFreeFunc free)
+{
+	bNodeType *nt = rna_Node_register_base(bmain, reports, &RNA_CompositorNodeCustomGroup, data, identifier, validate, call, free);
+	if (!nt)
+		return NULL;
+
+	nt->verifyfunc = node_group_verify;
+	nt->type = NODE_CUSTOM_GROUP;
+
+	register_node_type_cmp_custom_group(nt);
+
+	nodeRegisterType(nt);
+
+	WM_main_add_notifier(NC_NODE | NA_EDITED, NULL);
+
+	return nt->ext.srna;
+}
+
+
 static void rna_CompositorNode_tag_need_exec(bNode *node)
 {
 	node->need_exec = true;
@@ -3410,15 +3463,16 @@ static void def_group(StructRNA *srna)
 	RNA_def_property_ui_text(prop, "Interface", "Interface socket data");
 }
 
-static void def_custom_group(BlenderRNA *brna)
+static void def_custom_group(BlenderRNA *brna, const char *struct_name, const char *base_name,
+						const char *ui_name, const char *ui_desc, const char *reg_func)
 {
 	StructRNA *srna;
 
-	srna = RNA_def_struct(brna, "NodeCustomGroup", "Node");
-	RNA_def_struct_ui_text(srna, "Custom Group", "Base node type for custom registered node group types");
+	srna = RNA_def_struct(brna, struct_name, base_name);
+	RNA_def_struct_ui_text(srna, ui_name, ui_desc);
 	RNA_def_struct_sdna(srna, "bNode");
 
-	RNA_def_struct_register_funcs(srna, "rna_NodeCustomGroup_register", "rna_Node_unregister", NULL);
+	RNA_def_struct_register_funcs(srna, reg_func, "rna_Node_unregister", NULL);
 
 	def_group(srna);
 }
@@ -8616,7 +8670,9 @@ void RNA_def_nodetree(BlenderRNA *brna)
 	define_specific_node(brna, "ShaderNodeGroup", "ShaderNode", "Group", "", def_group);
 	define_specific_node(brna, "CompositorNodeGroup", "CompositorNode", "Group", "", def_group);
 	define_specific_node(brna, "TextureNodeGroup", "TextureNode", "Group", "", def_group);
-	def_custom_group(brna);
+	def_custom_group(brna, "ShaderNodeCustomGroup", "ShaderNode", "Shader Custom Group", "Custom Shader Group Node for Python nodes", "rna_ShaderNodeCustomGroup_register");
+	def_custom_group(brna, "CompositorNodeCustomGroup", "CompositorNode", "Compositor Custom Group", "Custom Compositor Group Node for Python nodes", "rna_CompositorNodeCustomGroup_register");
+	def_custom_group(brna, "NodeCustomGroup", "Node", "Custom Group", "Base node type for custom registered node group types", "rna_NodeCustomGroup_register"); 
 
 	/* special socket types */
 	rna_def_cmp_output_file_slot_file(brna);
diff --git a/source/blender/nodes/NOD_composite.h b/source/blender/nodes/NOD_composite.h
index 1e43b5011e6..85f4c81610b 100644
--- a/source/blender/nodes/NOD_composite.h
+++ b/source/blender/nodes/NOD_composite.h
@@ -136,4 +136,6 @@ void node_cmp_rlayers_outputs(struct bNodeTree *ntree, struct bNode *node);
 void node_cmp_rlayers_register_pass(struct bNodeTree *ntree, struct bNode *node, struct Scene *scene, struct ViewLayer *view_layer, const char *name, int type);
 const char *node_cmp_rlayers_sock_to_pass(int sock_index);
 
+void register_node_type_cmp_custom_group(bNodeType *ntype);
+
 #endif
diff --git a/source/blender/nodes/NOD_shader.h b/source/blender/nodes/NOD_shader.h
index e88f7a49d30..9e6fa0706d1 100644
--- a/source/blender/nodes/NOD_shader.h
+++ b/source/blender/nodes/NOD_shader.h
@@ -131,4 +131,6 @@ void register_node_type_sh_tex_checker(void);
 void register_node_type_sh_bump(void);
 void register_node_type_sh_tex_ies(void);
 
+void register_node_type_sh_custom_group(bNodeType *ntype);
+
 #endif
diff --git a/source/blender/nodes/composite/nodes/node_composite_common.c b/source/blender/nodes/composite/nodes/node_composite_common.c
index 9d72335d720..f9a6961c47b 100644
--- a/source/blender/nodes/composite/nodes/node_composite_common.c
+++ b/source/blender/nodes/composite/nodes/node_composite_common.c
@@ -56,3 +56,9 @@ void register_node_type_cmp_group(void)
 
 	nodeRegisterType(&ntype);
 }
+
+void register_node_type_cmp_custom_group(bNodeType *ntype)
+{
+	ntype->insert_link = node_insert_link_default;
+	ntype->update_internal_links = node_update_internal_links_default;
+}
diff --git a/source/blender/nodes/shader/node_shader_tree.c b/source/blender/nodes/shader/node_shader_tree.c
index 50bec047222..a4356dabfc0 100644
--- a/source/blender/nodes/shader/node_shader_tree.c
+++ b/source/blender/nodes/shader/node_shader_tree.c
@@ -290,7 +290,7 @@ static bNode *ntree_shader_output_node_from_group(bNodeTree *ntree, int target)
 
 	/* Search if node groups do not contain valid output nodes (recursively). */
 	for (bNode *node = ntree->nodes.first; node; node = node->next) {
-		if (!ELEM(node->type, NODE_GROUP)) {
+		if (!ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) {
 			continue;
 		}
 		if (node->id != NULL) {
@@ -453,7 +453,7 @@ static void ntree_shader_groups_expand_inputs(bNodeTree *localtree)
 
 	for (group_node = localtree->nodes.first; group_node; group_node = group_node->next) {
 
-		if (group_node->type != NODE_GROUP || group_node->id == NULL)
+		if (!(ELEM(group_node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) || group_node->id == NULL)
 			continue;
 
 		/* Do it recursively. */
@@ -674,7 +674,7 @@ static void ntree_shader_link_builtin_normal(bNodeTree *ntree,
 			/* Don't connect node itself! */
 			continue;
 		}
-		if (node->type == NODE_GROUP && node->id) {
+		if ((ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) && node->id) {
 			/* Special re-linking for group nodes. */
 			ntree_shader_link_builtin_group_normal(ntree,
 			                                       node,
@@ -765,6 +765,7 @@ static bool ntree_tag_bsdf_cb(bNode *fromnode, bNode *UNUSED(tonode), void *user
 
 	switch (fromnode->type) {
 		case NODE_GROUP:
+		case NODE_CUSTOM_GROUP:
 			/* Recursive */
 			if (fromnode->id != NULL) {
 				bNodeTree *ntree = (bNodeTree *)fromnode->id;
diff --git a/source/blender/nodes/shader/nodes/node_shader_common.c b/source/blender/nodes/shader/nodes/node_shader_common.c
index c238a3d946d..0c2b8d939f5 100644
--- a/source/blender/nodes/shader/nodes/node_shader_common.c
+++ b/source/blender/nodes/shader/nodes/node_shader_common.c
@@ -239,3 +239,11 @@ void register_node_type_sh_group(void)
 
 	nodeRegisterType(&ntype);
 }
+
+void register_node_type_sh_custom_group(bNodeType *ntype)
+{
+	ntype->insert_link = node_insert_link_default;
+	ntype->update_internal_links = node_update_internal_links_default;
+	node_type_exec(ntype, group_initexec, group_freeexec, group_execute);
+	node_type_gpu(ntype, gpu_group_execute);
+}
diff --git a/source/blender/render/intern/source/pipeline.c b/source/blender/render/intern/source/pipeline.c
index bd8d390b23b..22e798e04c8 100644
--- a/source/blender/render/intern/source/pipeline.c
+++ b/source/blender/render/intern/source/pipeline.c
@@ -1769,7 +1769,7 @@ static bool node_tree_has_composite_output(bNodeTree *ntree)
 		if (ELEM(node->type, CMP_NODE_COMPOSITE, CMP_NODE_OUTPUT_FILE)) {
 			return true;
 		}
-		else if (node->type == NODE_GROUP) {
+		else if (ELEM(node->type, NODE_GROUP, NODE_CUSTOM_GROUP)) {
 			if (node->id) {
 				if (node_tree_has_composite_output((bNodeTree *)node->id)) {
 					return true;