Fix T44908: Blender crashes when trying to use cycles experimental displacement

The issue was caused by the reshuffle needed to make objects flags have proper
object's bounding box to solve regressions in SSS objects intersecting volumes.

There's actually a feedback loop happening here, which is now solved in quite
naive way -- for the true displacement we consider all objects are capable of
intersecting volumes, synchronize object flags prior to displacement shader
tasks runs and then re-update object flags for proper bounding box.

Not sure what will be the proper solution here, we can't do preliminary check
of intersection for displacement shader, but on the other hand we don't really
need this flag for displacement shader anyway.

3 files changed, 44 insertions, 10 deletions

diff --git a/intern/cycles/render/mesh.cpp b/intern/cycles/render/mesh.cpp
index 19ca159d03f..45685fe5927 100644
--- a/intern/cycles/render/mesh.cpp
+++ b/intern/cycles/render/mesh.cpp
@@ -1227,6 +1227,7 @@ void MeshManager::device_update(Device *device, DeviceScene *dscene, Scene *scen
 
 	/* Update images needed for true displacement. */
 	bool need_displacement_images = false;
+	bool old_need_object_flags_update = false;
 	foreach(Mesh *mesh, scene->meshes) {
 		if(mesh->need_update &&
 		   mesh->displacement_method != Mesh::DISPLACE_BUMP)
@@ -1238,6 +1239,12 @@ void MeshManager::device_update(Device *device, DeviceScene *dscene, Scene *scen
 	if(need_displacement_images) {
 		VLOG(1) << "Updating images used for true displacement.";
 		device_update_displacement_images(device, dscene, scene, progress);
+		old_need_object_flags_update = scene->object_manager->need_flags_update;
+		scene->object_manager->device_update_flags(device,
+		                                           dscene,
+		                                           scene,
+		                                           progress,
+		                                           false);
 	}
 
 	/* device update */
@@ -1314,6 +1321,16 @@ void MeshManager::device_update(Device *device, DeviceScene *dscene, Scene *scen
 	device_update_bvh(device, dscene, scene, progress);
 
 	need_update = false;
+
+	if(need_displacement_images) {
+		/* Re-tag flags for update, so they're re-evaluated
+		 * for meshes with correct bounding boxes.
+		 *
+		 * This wouldn't cause wrong results, just true
+		 * displacement might be less optimal ot calculate.
+		 */
+		scene->object_manager->need_flags_update = old_need_object_flags_update;
+	}
 }
 
 void MeshManager::device_free(Device *device, DeviceScene *dscene)
diff --git a/intern/cycles/render/object.cpp b/intern/cycles/render/object.cpp
index da9bc23ccc4..ae72d728c8c 100644
--- a/intern/cycles/render/object.cpp
+++ b/intern/cycles/render/object.cpp
@@ -410,7 +410,8 @@ void ObjectManager::device_update(Device *device, DeviceScene *dscene, Scene *sc
 void ObjectManager::device_update_flags(Device *device,
                                         DeviceScene *dscene,
                                         Scene *scene,
-                                        Progress& /*progress*/)
+                                        Progress& /*progress*/,
+                                        bool bounds_valid)
 {
 	if(!need_update && !need_flags_update)
 		return;
@@ -425,9 +426,13 @@ void ObjectManager::device_update_flags(Device *device,
 	uint *object_flag = dscene->object_flag.get_data();
 
 	vector<Object *> volume_objects;
+	bool has_volume_objects = false;
 	foreach(Object *object, scene->objects) {
 		if(object->mesh->has_volume) {
-			volume_objects.push_back(object);
+			if(bounds_valid) {
+				volume_objects.push_back(object);
+			}
+			has_volume_objects = true;
 		}
 	}
 
@@ -440,15 +445,23 @@ void ObjectManager::device_update_flags(Device *device,
 			object_flag[object_index] &= ~SD_OBJECT_HAS_VOLUME;
 		}
 
-		foreach(Object *volume_object, volume_objects) {
-			if(object == volume_object) {
-				continue;
-			}
-			if(object->bounds.intersects(volume_object->bounds)) {
-				object_flag[object_index] |= SD_OBJECT_INTERSECTS_VOLUME;
-				break;
+		if(bounds_valid) {
+			foreach(Object *volume_object, volume_objects) {
+				if(object == volume_object) {
+					continue;
+				}
+				if(object->bounds.intersects(volume_object->bounds)) {
+					object_flag[object_index] |= SD_OBJECT_INTERSECTS_VOLUME;
+					break;
+				}
 			}
 		}
+		else if(has_volume_objects) {
+			/* Not really valid, but can't make more reliable in the case
+			 * of bounds not being up to date.
+			 */
+			object_flag[object_index] |= SD_OBJECT_INTERSECTS_VOLUME;
+		}
 		++object_index;
 	}
 
diff --git a/intern/cycles/render/object.h b/intern/cycles/render/object.h
index acc08a0e204..379d1748cdd 100644
--- a/intern/cycles/render/object.h
+++ b/intern/cycles/render/object.h
@@ -77,7 +77,11 @@ public:
 
 	void device_update(Device *device, DeviceScene *dscene, Scene *scene, Progress& progress);
 	void device_update_transforms(Device *device, DeviceScene *dscene, Scene *scene, uint *object_flag, Progress& progress);
-	void device_update_flags(Device *device, DeviceScene *dscene, Scene *scene, Progress& progress);
+	void device_update_flags(Device *device,
+	                         DeviceScene *dscene,
+	                         Scene *scene,
+	                         Progress& progress,
+	                         bool bounds_valid = true);
 	void device_free(Device *device, DeviceScene *dscene);
 
 	void tag_update(Scene *scene);