Fix #20486: blender hangs upon import attempt of an .obj with >40k polys.

Added automatic generation of lookup_int callbacks for collections, for
quicker lookup by index instead of looping over the whole thing. Import
is still quite slow, though now it only takes a few seconds.

The next bottleneck seems to be running update (depsgraph, notifiers, ..)
on setting every property. I fixed part of that by avoiding a notifier
to be added each time, now it checks for duplicates.

3 files changed, 110 insertions, 0 deletions

diff --git a/source/blender/makesrna/intern/makesrna.c b/source/blender/makesrna/intern/makesrna.c
index 15d5658160d..6d3d78907f1 100644
--- a/source/blender/makesrna/intern/makesrna.c
+++ b/source/blender/makesrna/intern/makesrna.c
@@ -846,6 +846,98 @@ static char *rna_def_property_begin_func(FILE *f, StructRNA *srna, PropertyRNA *
 	return func;
 }
 
+static char *rna_def_property_lookup_int_func(FILE *f, StructRNA *srna, PropertyRNA *prop, PropertyDefRNA *dp, char *manualfunc, char *nextfunc)
+{
+	char *func;
+
+	if(prop->flag & PROP_IDPROPERTY)
+		return NULL;
+
+	if(!manualfunc) {
+		if(!dp->dnastructname || !dp->dnaname)
+			return NULL;
+
+		/* only supported in case of standard next functions */
+		if(strcmp(nextfunc, "rna_iterator_array_next") == 0);
+		else if(strcmp(nextfunc, "rna_iterator_listbase_next") == 0);
+		else return NULL;
+	}
+
+	func= rna_alloc_function_name(srna->identifier, prop->identifier, "lookup_int");
+
+	fprintf(f, "PointerRNA %s(PointerRNA *ptr, int index)\n", func);
+	fprintf(f, "{\n");
+
+	if(manualfunc) {
+		fprintf(f, "\n	return %s(ptr, index);\n", manualfunc);
+		fprintf(f, "}\n\n");
+		return func;
+	}
+
+	fprintf(f, "	PointerRNA r_ptr;\n");
+	fprintf(f, "	CollectionPropertyIterator iter;\n\n");
+
+	fprintf(f, "	%s_%s_begin(&iter, ptr);\n\n", srna->identifier, prop->identifier);
+	fprintf(f, "	{\n");
+
+	if(strcmp(nextfunc, "rna_iterator_array_next") == 0) {
+		fprintf(f, "		ArrayIterator *internal= iter.internal;\n");
+		fprintf(f, "		if(internal->skip) {\n");
+		fprintf(f, "			while(index-- > 0) {\n");
+		fprintf(f, "				do {\n");
+		fprintf(f, "					internal->ptr += internal->itemsize;\n");
+		fprintf(f, "				} while(internal->skip(&iter, internal->ptr));\n");
+		fprintf(f, "			}\n");
+		fprintf(f, "		}\n");
+		fprintf(f, "		else {\n");
+		fprintf(f, "			internal->ptr += internal->itemsize*index;\n");
+		fprintf(f, "		}\n");
+	}
+	else if(strcmp(nextfunc, "rna_iterator_listbase_next") == 0) {
+		fprintf(f, "		ListBaseIterator *internal= iter.internal;\n");
+		fprintf(f, "		if(internal->skip) {\n");
+		fprintf(f, "			while(index-- > 0) {\n");
+		fprintf(f, "				do {\n");
+		fprintf(f, "					internal->link= internal->link->next;\n");
+		fprintf(f, "				} while(internal->skip(&iter, internal->link));\n");
+		fprintf(f, "			}\n");
+		fprintf(f, "		}\n");
+		fprintf(f, "		else {\n");
+		fprintf(f, "			while(index-- > 0)\n");
+		fprintf(f, "				internal->link= internal->link->next;\n");
+		fprintf(f, "		}\n");
+	}
+
+	fprintf(f, "	}\n\n");
+
+	fprintf(f, "	r_ptr = %s_%s_get(&iter);\n", srna->identifier, prop->identifier);
+	fprintf(f, "	%s_%s_end(&iter);\n\n", srna->identifier, prop->identifier);
+
+	fprintf(f, "	return r_ptr;\n");
+
+#if 0
+	rna_print_data_get(f, dp);
+	item_type= (cprop->item_type)? (char*)cprop->item_type: "UnknownType";
+
+	if(dp->dnalengthname || dp->dnalengthfixed) {
+		if(dp->dnalengthname)
+			fprintf(f, "\n	rna_array_lookup_int(ptr, &RNA_%s, data->%s, sizeof(data->%s[0]), data->%s, index);\n", item_type, dp->dnaname, dp->dnaname, dp->dnalengthname);
+		else
+			fprintf(f, "\n	rna_array_lookup_int(ptr, &RNA_%s, data->%s, sizeof(data->%s[0]), %d, index);\n", item_type, dp->dnaname, dp->dnaname, dp->dnalengthfixed);
+	}
+	else {
+		if(dp->dnapointerlevel == 0)
+			fprintf(f, "\n	return rna_listbase_lookup_int(ptr, &RNA_%s, &data->%s, index);\n", item_type, dp->dnaname);
+		else
+			fprintf(f, "\n	return rna_listbase_lookup_int(ptr, &RNA_%s, data->%s, index);\n", item_type, dp->dnaname);
+	}
+#endif
+
+	fprintf(f, "}\n\n");
+
+	return func;
+}
+
 static char *rna_def_property_next_func(FILE *f, StructRNA *srna, PropertyRNA *prop, PropertyDefRNA *dp, char *manualfunc)
 {
 	char *func, *getfunc;
@@ -1015,6 +1107,7 @@ static void rna_def_property_funcs(FILE *f, StructRNA *srna, PropertyDefRNA *dp)
 		}
 		case PROP_COLLECTION: {
 			CollectionPropertyRNA *cprop= (CollectionPropertyRNA*)prop;
+			char *nextfunc= (char*)cprop->next;
 
 			if(dp->dnatype && strcmp(dp->dnatype, "ListBase")==0);
 			else if(dp->dnalengthname || dp->dnalengthfixed)
@@ -1031,6 +1124,7 @@ static void rna_def_property_funcs(FILE *f, StructRNA *srna, PropertyDefRNA *dp)
 			cprop->begin= (void*)rna_def_property_begin_func(f, srna, prop, dp, (char*)cprop->begin);
 			cprop->next= (void*)rna_def_property_next_func(f, srna, prop, dp, (char*)cprop->next);
 			cprop->end= (void*)rna_def_property_end_func(f, srna, prop, dp, (char*)cprop->end);
+			cprop->lookupint= (void*)rna_def_property_lookup_int_func(f, srna, prop, dp, (char*)cprop->lookupint, nextfunc);
 
 			if(!(prop->flag & PROP_IDPROPERTY)) {
 				if(!cprop->begin) {
diff --git a/source/blender/makesrna/intern/rna_access.c b/source/blender/makesrna/intern/rna_access.c
index 6bd99793933..672ca5d7b90 100644
--- a/source/blender/makesrna/intern/rna_access.c
+++ b/source/blender/makesrna/intern/rna_access.c
@@ -2736,6 +2736,12 @@ void rna_iterator_listbase_end(CollectionPropertyIterator *iter)
 	iter->internal= NULL;
 }
 
+PointerRNA rna_listbase_lookup_int(PointerRNA *ptr, StructRNA *type, struct ListBase *lb, int index)
+{
+	void *data= BLI_findlink(lb, index);
+	return rna_pointer_inherit_refine(ptr, type, data);
+}
+
 void rna_iterator_array_begin(CollectionPropertyIterator *iter, void *ptr, int itemsize, int length, int free_ptr, IteratorSkipFunc skip)
 {
 	ArrayIterator *internal;
@@ -2800,6 +2806,14 @@ void rna_iterator_array_end(CollectionPropertyIterator *iter)
 	iter->internal= NULL;
 }
 
+PointerRNA rna_array_lookup_int(PointerRNA *ptr, StructRNA *type, void *data, int itemsize, int length, int index)
+{
+	if(index < 0 || index >= length)
+		return PointerRNA_NULL;
+
+	return rna_pointer_inherit_refine(ptr, type, ((char*)data) + index*itemsize);
+}
+
 /* RNA Path - Experiment */
 
 static char *rna_path_token(const char **path, char *fixedbuf, int fixedlen, int bracket)
diff --git a/source/blender/makesrna/intern/rna_internal.h b/source/blender/makesrna/intern/rna_internal.h
index 06e0a96deed..68a7ca29a1f 100644
--- a/source/blender/makesrna/intern/rna_internal.h
+++ b/source/blender/makesrna/intern/rna_internal.h
@@ -297,6 +297,7 @@ void rna_iterator_listbase_begin(struct CollectionPropertyIterator *iter, struct
 void rna_iterator_listbase_next(struct CollectionPropertyIterator *iter);
 void *rna_iterator_listbase_get(struct CollectionPropertyIterator *iter);
 void rna_iterator_listbase_end(struct CollectionPropertyIterator *iter);
+PointerRNA rna_listbase_lookup_int(PointerRNA *ptr, StructRNA *type, struct ListBase *lb, int index);
 
 typedef struct ArrayIterator {
 	char *ptr;
@@ -311,6 +312,7 @@ void rna_iterator_array_next(struct CollectionPropertyIterator *iter);
 void *rna_iterator_array_get(struct CollectionPropertyIterator *iter);
 void *rna_iterator_array_dereference_get(struct CollectionPropertyIterator *iter);
 void rna_iterator_array_end(struct CollectionPropertyIterator *iter);
+PointerRNA rna_array_lookup_int(PointerRNA *ptr, StructRNA *type, void *data, int itemsize, int length, int index);
 
 /* Duplicated code since we can't link in blenlib */