Fix T41258: Crash when entering edit mode while viewport render is enabled

The issue was caused by the render engine loading edit mesh, which re-allocates
mesh array which might be referenced by other object's derived meshed.

Worst thing about this is that updating render engine happens from the end of
scene update function, after all the objects are updated and so. This is needed
so render engine gets the update objects which is correct.

The only proper way to solve the issue is to make it so viewport engine does not
leave objects in inconsistent state, meaning nobody will reference to freed data.

In order to reach this we do edit mesh loading before running objects update so
all the objects which uses that mesh will have proper references in the derived
mesh.

This also solves old creepyness which happened before when having single object
in edit mode. tweaking it will calculate derived mesh as a part of scene update,
then this derived mesh will be freed by edit mesh loading and viewport will be
creating derived mesh again.

Now render engine is expected to do nothing with meshes which are in edit mode,
but they still need to load edit data for non0meshes. It's not really easy to
do from the BKE level because needed functions are implemented in the editor.

Thanks Campbell for the review!

Differential Revision: https://developer.blender.org/D697

7 files changed, 80 insertions, 8 deletions

diff --git a/intern/cycles/blender/blender_mesh.cpp b/intern/cycles/blender/blender_mesh.cpp
index c532f07d89a..f07f0fde888 100644
--- a/intern/cycles/blender/blender_mesh.cpp
+++ b/intern/cycles/blender/blender_mesh.cpp
@@ -568,7 +568,13 @@ Mesh *BlenderSync::sync_mesh(BL::Object b_ob, bool object_updated, bool hide_tri
 	mesh->name = ustring(b_ob_data.name().c_str());
 
 	if(render_layer.use_surfaces || render_layer.use_hair) {
-		if(preview)
+		/* mesh objects does have special handle in the dependency graph,
+		 * they're ensured to have properly updated.
+		 *
+		 * updating meshes here will end up having derived mesh referencing
+		 * freed data from the blender side.
+		 */
+		if(preview && b_ob.type() != BL::Object::type_MESH)
 			b_ob.update_from_editmode();
 
 		bool need_undeformed = mesh->need_attribute(scene, ATTR_STD_GENERATED);
diff --git a/source/blender/blenkernel/intern/scene.c b/source/blender/blenkernel/intern/scene.c
index 18a408167e2..a6e23edba56 100644
--- a/source/blender/blenkernel/intern/scene.c
+++ b/source/blender/blenkernel/intern/scene.c
@@ -39,6 +39,7 @@
 #include "DNA_anim_types.h"
 #include "DNA_group_types.h"
 #include "DNA_linestyle_types.h"
+#include "DNA_mesh_types.h"
 #include "DNA_node_types.h"
 #include "DNA_object_types.h"
 #include "DNA_rigidbody_types.h"
@@ -46,6 +47,8 @@
 #include "DNA_screen_types.h"
 #include "DNA_sequence_types.h"
 #include "DNA_space_types.h"
+#include "DNA_view3d_types.h"
+#include "DNA_windowmanager_types.h"
 
 #include "BLI_math.h"
 #include "BLI_blenlib.h"
@@ -62,6 +65,7 @@
 #include "BKE_action.h"
 #include "BKE_colortools.h"
 #include "BKE_depsgraph.h"
+#include "BKE_editmesh.h"
 #include "BKE_fcurve.h"
 #include "BKE_freestyle.h"
 #include "BKE_global.h"
@@ -86,6 +90,8 @@
 
 #include "IMB_colormanagement.h"
 
+#include "bmesh.h"
+
 //XXX #include "BIF_previewrender.h"
 //XXX #include "BIF_editseq.h"
 
@@ -1554,6 +1560,53 @@ static void scene_update_tagged_recursive(EvaluationContext *eval_ctx, Main *bma
 	
 }
 
+static bool check_rendered_viewport_visible(Main *bmain)
+{
+	wmWindowManager *wm = bmain->wm.first;
+	wmWindow *window;
+	for (window = wm->windows.first; window != NULL; window = window->next) {
+		bScreen *screen = window->screen;
+		ScrArea *area;
+		for (area = screen->areabase.first; area != NULL; area = area->next) {
+			View3D *v3d = area->spacedata.first;
+			if (area->spacetype != SPACE_VIEW3D) {
+				continue;
+			}
+			if (v3d->drawtype == OB_RENDER) {
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+static void prepare_mesh_for_viewport_render(Main *bmain, Scene *scene)
+{
+	/* This is needed to prepare mesh to be used by the render
+	 * engine from the viewport rendering. We do loading here
+	 * so all the objects which shares the same mesh datablock
+	 * are nicely tagged for update and updated.
+	 *
+	 * This makes it so viewport render engine doesn't need to
+	 * call loading of the edit data for the mesh objects.
+	 */
+
+	Object *obedit = scene->obedit;
+	if (obedit) {
+		Mesh *mesh = obedit->data;
+		/* TODO(sergey): Check object recalc flags as well? */
+		if ((obedit->type == OB_MESH) &&
+		    (mesh->id.flag & (LIB_ID_RECALC | LIB_ID_RECALC_DATA)))
+		{
+			if (check_rendered_viewport_visible(bmain)) {
+				BMesh *bm = mesh->edit_btmesh->bm;
+				BM_mesh_bm_to_me(bm, mesh, false);
+				DAG_id_tag_update(&mesh->id, 0);
+			}
+		}
+	}
+}
+
 void BKE_scene_update_tagged(EvaluationContext *eval_ctx, Main *bmain, Scene *scene)
 {
 	Scene *sce_iter;
@@ -1565,6 +1618,9 @@ void BKE_scene_update_tagged(EvaluationContext *eval_ctx, Main *bmain, Scene *sc
 	for (sce_iter = scene; sce_iter; sce_iter = sce_iter->set)
 		DAG_scene_relations_update(bmain, sce_iter);
 
+	/* flush editing data if needed */
+	prepare_mesh_for_viewport_render(bmain, scene);
+
 	/* flush recalc flags to dependencies */
 	DAG_ids_flush_tagged(bmain);
 
diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index b0ea984d6a6..430c80446b5 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -365,6 +365,6 @@ void ED_view3d_operator_properties_viewmat_get(struct wmOperator *op, int *winx,
 #endif
 
 /* render */
-void ED_view3d_shade_update(struct Main *bmain, struct View3D *v3d, struct ScrArea *sa);
+void ED_view3d_shade_update(struct Main *bmain, struct Scene *scene, struct View3D *v3d, struct ScrArea *sa);
 
 #endif /* __ED_VIEW3D_H__ */
diff --git a/source/blender/editors/object/object_edit.c b/source/blender/editors/object/object_edit.c
index 20e31e3909e..3b60bbbb37e 100644
--- a/source/blender/editors/object/object_edit.c
+++ b/source/blender/editors/object/object_edit.c
@@ -363,6 +363,11 @@ static bool ED_object_editmode_load_ex(Object *obedit, const bool freedata)
 		if (freedata) free_editMball(obedit);
 	}
 
+	/* Tag update so no access to freed data referenced from
+	 * derived cache will happen.
+	 */
+	DAG_id_tag_update((ID *)obedit->data, 0);
+
 	return true;
 }
 
diff --git a/source/blender/editors/space_view3d/space_view3d.c b/source/blender/editors/space_view3d/space_view3d.c
index 1273b905016..8b76ec3a56d 100644
--- a/source/blender/editors/space_view3d/space_view3d.c
+++ b/source/blender/editors/space_view3d/space_view3d.c
@@ -43,6 +43,7 @@
 #include "BLI_utildefines.h"
 
 #include "BKE_context.h"
+#include "BKE_depsgraph.h"
 #include "BKE_icons.h"
 #include "BKE_library.h"
 #include "BKE_main.h"
@@ -286,7 +287,7 @@ static void view3d_stop_render_preview(wmWindowManager *wm, ARegion *ar)
 	}
 }
 
-void ED_view3d_shade_update(Main *bmain, View3D *v3d, ScrArea *sa)
+void ED_view3d_shade_update(Main *bmain, Scene *scene, View3D *v3d, ScrArea *sa)
 {
 	wmWindowManager *wm = bmain->wm.first;
 
@@ -298,6 +299,10 @@ void ED_view3d_shade_update(Main *bmain, View3D *v3d, ScrArea *sa)
 				view3d_stop_render_preview(wm, ar);
 		}
 	}
+	else if (scene->obedit != NULL && scene->obedit->type == OB_MESH) {
+		/* Tag mesh to load edit data. */
+		DAG_id_tag_update(scene->obedit->data, 0);
+	}
 }
 
 /* ******************** default callbacks for view3d space ***************** */
diff --git a/source/blender/editors/space_view3d/view3d_view.c b/source/blender/editors/space_view3d/view3d_view.c
index e51cf371a64..6d28accbc7b 100644
--- a/source/blender/editors/space_view3d/view3d_view.c
+++ b/source/blender/editors/space_view3d/view3d_view.c
@@ -1305,7 +1305,7 @@ static bool view3d_localview_init(
 	return ok;
 }
 
-static void restore_localviewdata(wmWindowManager *wm, wmWindow *win, Main *bmain, ScrArea *sa, const int smooth_viewtx)
+static void restore_localviewdata(wmWindowManager *wm, wmWindow *win, Main *bmain, Scene *scene, ScrArea *sa, const int smooth_viewtx)
 {
 	const bool free = true;
 	ARegion *ar;
@@ -1355,7 +1355,7 @@ static void restore_localviewdata(wmWindowManager *wm, wmWindow *win, Main *bmai
 				}
 			}
 
-			ED_view3d_shade_update(bmain, v3d, sa);
+			ED_view3d_shade_update(bmain, scene, v3d, sa);
 		}
 	}
 }
@@ -1372,7 +1372,7 @@ static bool view3d_localview_exit(
 		
 		locallay = v3d->lay & 0xFF000000;
 
-		restore_localviewdata(wm, win, bmain, sa, smooth_viewtx);
+		restore_localviewdata(wm, win, bmain, scene, sa, smooth_viewtx);
 
 		/* for when in other window the layers have changed */
 		if (v3d->scenelock) v3d->lay = scene->lay;
diff --git a/source/blender/makesrna/intern/rna_space.c b/source/blender/makesrna/intern/rna_space.c
index 3e20c593a78..0ddc63e57cf 100644
--- a/source/blender/makesrna/intern/rna_space.c
+++ b/source/blender/makesrna/intern/rna_space.c
@@ -466,12 +466,12 @@ static void rna_SpaceView3D_layer_update(Main *bmain, Scene *UNUSED(scene), Poin
 	DAG_on_visible_update(bmain, false);
 }
 
-static void rna_SpaceView3D_viewport_shade_update(Main *bmain, Scene *UNUSED(scene), PointerRNA *ptr)
+static void rna_SpaceView3D_viewport_shade_update(Main *bmain, Scene *scene, PointerRNA *ptr)
 {
 	View3D *v3d = (View3D *)(ptr->data);
 	ScrArea *sa = rna_area_from_space(ptr);
 
-	ED_view3d_shade_update(bmain, v3d, sa);
+	ED_view3d_shade_update(bmain, scene, v3d, sa);
 }
 
 static void rna_SpaceView3D_matcap_update(Main *UNUSED(bmain), Scene *UNUSED(scene), PointerRNA *ptr)