Fix T69809, T69810: sculpt gone or crashing after render

Make a distinction between flush sculpt changes for rendering, and forcing
sculpt data structures to be rebuilt after mesh changes. Also don't use PBVH
for renders.

14 files changed, 43 insertions, 46 deletions

diff --git a/source/blender/blenkernel/BKE_multires.h b/source/blender/blenkernel/BKE_multires.h
index c5955a9af8d..7513717df41 100644
--- a/source/blender/blenkernel/BKE_multires.h
+++ b/source/blender/blenkernel/BKE_multires.h
@@ -53,8 +53,8 @@ void multires_mark_as_modified(struct Depsgraph *depsgraph,
                                struct Object *object,
                                enum MultiresModifiedFlags flags);
 
-void multires_force_update(struct Object *ob);
-void multires_force_render_update(struct Object *ob);
+void multires_flush_sculpt_updates(struct Object *ob);
+void multires_force_sculpt_rebuild(struct Object *ob);
 void multires_force_external_reload(struct Object *ob);
 
 /* internal, only called in subsurf_ccg.c */
diff --git a/source/blender/blenkernel/intern/library_remap.c b/source/blender/blenkernel/intern/library_remap.c
index 61f506a8a6c..7554dad5fb0 100644
--- a/source/blender/blenkernel/intern/library_remap.c
+++ b/source/blender/blenkernel/intern/library_remap.c
@@ -389,7 +389,7 @@ static void libblock_remap_data_postprocess_obdata_relink(Main *bmain, Object *o
   if (ob->data == new_id) {
     switch (GS(new_id->name)) {
       case ID_ME:
-        multires_force_update(ob);
+        multires_force_sculpt_rebuild(ob);
         break;
       case ID_CU:
         BKE_curve_type_test(ob);
diff --git a/source/blender/blenkernel/intern/mesh.c b/source/blender/blenkernel/intern/mesh.c
index ae9c3400ace..c108ff90850 100644
--- a/source/blender/blenkernel/intern/mesh.c
+++ b/source/blender/blenkernel/intern/mesh.c
@@ -1080,12 +1080,12 @@ void BKE_mesh_assign_object(Main *bmain, Object *ob, Mesh *me)
 {
   Mesh *old = NULL;
 
-  multires_force_update(ob);
-
   if (ob == NULL) {
     return;
   }
 
+  multires_force_sculpt_rebuild(ob);
+
   if (ob->type == OB_MESH) {
     old = ob->data;
     if (old) {
diff --git a/source/blender/blenkernel/intern/multires.c b/source/blender/blenkernel/intern/multires.c
index 8cf960a7f14..82dfe68dbd6 100644
--- a/source/blender/blenkernel/intern/multires.c
+++ b/source/blender/blenkernel/intern/multires.c
@@ -406,24 +406,24 @@ void multires_mark_as_modified(Depsgraph *depsgraph, Object *object, MultiresMod
   multires_ccg_mark_as_modified(subdiv_ccg, flags);
 }
 
-void multires_force_update(Object *ob)
+void multires_flush_sculpt_updates(Object *ob)
 {
-  if (ob == NULL) {
-    return;
-  }
-  SculptSession *sculpt_session = ob->sculpt;
-  if (sculpt_session != NULL && sculpt_session->pbvh != NULL) {
-    PBVH *pbvh = sculpt_session->pbvh;
-    if (BKE_pbvh_type(pbvh) == PBVH_GRIDS) {
+  if (ob && ob->sculpt && ob->sculpt->pbvh != NULL) {
+    SculptSession *sculpt_session = ob->sculpt;
+    if (BKE_pbvh_type(sculpt_session->pbvh) == PBVH_GRIDS) {
       Mesh *mesh = ob->data;
       multiresModifier_reshapeFromCCG(
           sculpt_session->multires->totlvl, mesh, sculpt_session->subdiv_ccg);
     }
-    else {
-      /* NOTE: Disabled for until OpenSubdiv is enabled by default. */
-      // BLI_assert(!"multires_force_update is used on non-grids PBVH");
-    }
-    BKE_pbvh_free(pbvh);
+  }
+}
+
+void multires_force_sculpt_rebuild(Object *ob)
+{
+  multires_flush_sculpt_updates(ob);
+
+  if (ob && ob->sculpt && ob->sculpt->pbvh != NULL) {
+    BKE_pbvh_free(ob->sculpt->pbvh);
     ob->sculpt->pbvh = NULL;
   }
 }
@@ -433,14 +433,7 @@ void multires_force_external_reload(Object *ob)
   Mesh *me = BKE_mesh_from_object(ob);
 
   CustomData_external_reload(&me->ldata, &me->id, CD_MASK_MDISPS, me->totloop);
-  multires_force_update(ob);
-}
-
-void multires_force_render_update(Object *ob)
-{
-  if (ob && (ob->mode & OB_MODE_SCULPT) && modifiers_findByType(ob, eModifierType_Multires)) {
-    multires_force_update(ob);
-  }
+  multires_force_sculpt_rebuild(ob);
 }
 
 /* reset the multires levels to match the number of mdisps */
@@ -624,7 +617,7 @@ static void multires_del_higher(MultiresModifierData *mmd, Object *ob, int lvl)
   mdisps = CustomData_get_layer(&me->ldata, CD_MDISPS);
   gpm = CustomData_get_layer(&me->ldata, CD_GRID_PAINT_MASK);
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
 
   if (mdisps && levels > 0) {
     if (lvl > 0) {
@@ -689,7 +682,7 @@ void multiresModifier_del_levels(MultiresModifierData *mmd,
   CustomData_external_read(&me->ldata, &me->id, CD_MASK_MDISPS, me->totloop);
   mdisps = CustomData_get_layer(&me->ldata, CD_MDISPS);
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
 
   if (mdisps && levels > 0 && direction == 1) {
     multires_del_higher(mmd, ob, lvl);
@@ -781,7 +774,7 @@ void multiresModifier_base_apply(MultiresModifierData *mmd, Scene *scene, Object
   float(*origco)[3];
   int i, j, k, offset, totlvl;
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
 
   me = BKE_mesh_from_object(ob);
   totlvl = mmd->totlvl;
@@ -928,7 +921,7 @@ static void multires_subdivide(
 
   BLI_assert(totlvl > lvl);
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
 
   mdisps = CustomData_get_layer(&me->ldata, CD_MDISPS);
   if (!mdisps) {
diff --git a/source/blender/draw/engines/basic/basic_engine.c b/source/blender/draw/engines/basic/basic_engine.c
index f548bd15bf4..188e252a285 100644
--- a/source/blender/draw/engines/basic/basic_engine.c
+++ b/source/blender/draw/engines/basic/basic_engine.c
@@ -162,7 +162,8 @@ static void basic_cache_populate(void *vedata, Object *ob)
     }
   }
 
-  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                               !DRW_state_is_image_render();
   const bool do_cull = (draw_ctx->v3d &&
                         (draw_ctx->v3d->shading.flag & V3D_SHADING_BACKFACE_CULLING));
   DRWShadingGroup *shgrp = (do_cull) ? stl->g_data->depth_shgrp_cull : stl->g_data->depth_shgrp;
diff --git a/source/blender/draw/engines/eevee/eevee_materials.c b/source/blender/draw/engines/eevee/eevee_materials.c
index 2158d395d84..f01c6363ccb 100644
--- a/source/blender/draw/engines/eevee/eevee_materials.c
+++ b/source/blender/draw/engines/eevee/eevee_materials.c
@@ -1454,7 +1454,8 @@ void EEVEE_materials_cache_populate(EEVEE_Data *vedata,
   Scene *scene = draw_ctx->scene;
   GHash *material_hash = stl->g_data->material_hash;
 
-  bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+  bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                         !DRW_state_is_image_render();
 
   /* First get materials for this mesh. */
   if (ELEM(ob->type, OB_MESH, OB_CURVE, OB_SURF, OB_FONT, OB_MBALL)) {
diff --git a/source/blender/draw/engines/workbench/workbench_deferred.c b/source/blender/draw/engines/workbench/workbench_deferred.c
index e41cba1d3bd..15522ba0dfb 100644
--- a/source/blender/draw/engines/workbench/workbench_deferred.c
+++ b/source/blender/draw/engines/workbench/workbench_deferred.c
@@ -945,7 +945,8 @@ static void workbench_cache_populate_texture_paint_mode(WORKBENCH_Data *vedata,
   const DRWContextState *draw_ctx = DRW_context_state_get();
 
   Scene *scene = draw_ctx->scene;
-  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                               !DRW_state_is_image_render();
   WORKBENCH_MaterialData *material;
 
   /* Force workbench to render active object textured when in texture paint mode */
diff --git a/source/blender/draw/engines/workbench/workbench_forward.c b/source/blender/draw/engines/workbench/workbench_forward.c
index 890359e8fda..d731b167c06 100644
--- a/source/blender/draw/engines/workbench/workbench_forward.c
+++ b/source/blender/draw/engines/workbench/workbench_forward.c
@@ -560,7 +560,8 @@ static void workbench_forward_cache_populate_texture_paint_mode(WORKBENCH_Data *
   const DRWContextState *draw_ctx = DRW_context_state_get();
 
   Scene *scene = draw_ctx->scene;
-  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+  const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                               !DRW_state_is_image_render();
   WORKBENCH_MaterialData *material;
 
   /* Force workbench to render active object textured when in texture paint mode */
@@ -635,7 +636,8 @@ void workbench_forward_cache_populate(WORKBENCH_Data *vedata, Object *ob)
 
   WORKBENCH_MaterialData *material;
   if (ELEM(ob->type, OB_MESH, OB_CURVE, OB_SURF, OB_FONT, OB_MBALL)) {
-    const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+    const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                                 !DRW_state_is_image_render();
     const int materials_len = MAX2(1, ob->totcol);
     const Mesh *me = (ob->type == OB_MESH) ? ob->data : NULL;
     const bool use_texture_paint_drawing = !(DRW_state_is_image_render() &&
diff --git a/source/blender/draw/modes/overlay_mode.c b/source/blender/draw/modes/overlay_mode.c
index 2433f4c1df5..587046589ee 100644
--- a/source/blender/draw/modes/overlay_mode.c
+++ b/source/blender/draw/modes/overlay_mode.c
@@ -394,7 +394,8 @@ static void overlay_cache_populate(void *vedata, Object *ob)
     if ((!pd->show_overlays) ||
         (((ob != draw_ctx->object_edit) && !is_edit_mode) || has_edit_mesh_cage) ||
         ob->type != OB_MESH) {
-      const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d);
+      const bool use_sculpt_pbvh = BKE_sculptsession_use_pbvh_draw(ob, draw_ctx->v3d) &&
+                                   !DRW_state_is_image_render();
       const bool all_wires = (ob->dtx & OB_DRAW_ALL_EDGES);
       const bool is_wire = (ob->dt < OB_SOLID);
       const bool is_xray = (ob->dtx & OB_DRAWXRAY);
diff --git a/source/blender/editors/object/object_bake.c b/source/blender/editors/object/object_bake.c
index 70a9870e6ae..bc79521ee9b 100644
--- a/source/blender/editors/object/object_bake.c
+++ b/source/blender/editors/object/object_bake.c
@@ -371,7 +371,7 @@ static int multiresbake_image_exec_locked(bContext *C, wmOperator *op)
 
     ob = base->object;
 
-    multires_force_update(ob);
+    multires_flush_sculpt_updates(ob);
 
     /* copy data stored in job descriptor */
     bkr.scene = scene;
@@ -435,7 +435,7 @@ static void init_multiresbake_job(bContext *C, MultiresBakeJob *bkj)
 
     ob = base->object;
 
-    multires_force_update(ob);
+    multires_flush_sculpt_updates(ob);
 
     data = MEM_callocN(sizeof(MultiresBakerJobData), "multiresBaker derivedMesh_data");
 
diff --git a/source/blender/editors/object/object_modifier.c b/source/blender/editors/object/object_modifier.c
index 88d01936882..c437f00684d 100644
--- a/source/blender/editors/object/object_modifier.c
+++ b/source/blender/editors/object/object_modifier.c
@@ -677,7 +677,7 @@ static int modifier_apply_obdata(
 
     /* Multires: ensure that recent sculpting is applied */
     if (md_eval->type == eModifierType_Multires) {
-      multires_force_update(ob);
+      multires_force_sculpt_rebuild(ob);
     }
 
     if (mmd && mmd->totlvl && mti->type == eModifierTypeType_OnlyDeform) {
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 83b9abe5f3e..f9a7e59c509 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -1099,7 +1099,7 @@ static bool sculpt_brush_test_cyl(SculptBrushTest *test,
 static bool sculpt_automasking_enabled(SculptSession *ss, const Brush *br)
 {
   // REMOVE WITH PBVH_GRIDS
-  if (BKE_pbvh_type(ss->pbvh) == PBVH_GRIDS) {
+  if (ss->pbvh && BKE_pbvh_type(ss->pbvh) == PBVH_GRIDS) {
     return false;
   }
 
@@ -7529,13 +7529,11 @@ void ED_object_sculptmode_exit_ex(Main *bmain, Depsgraph *depsgraph, Scene *scen
   const int mode_flag = OB_MODE_SCULPT;
   Mesh *me = BKE_mesh_from_object(ob);
 
-  MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
-  if (mmd) {
-    multires_force_update(ob);
-  }
+  multires_flush_sculpt_updates(ob);
 
   /* Not needed for now. */
 #if 0
+  MultiresModifierData *mmd = BKE_sculpt_multires_active(scene, ob);
   const int flush_recalc = ed_object_sculptmode_flush_recalc_flag(scene, ob, mmd);
 #endif
 
diff --git a/source/blender/editors/util/ed_util.c b/source/blender/editors/util/ed_util.c
index c2566d111cf..f5548119e0a 100644
--- a/source/blender/editors/util/ed_util.c
+++ b/source/blender/editors/util/ed_util.c
@@ -245,7 +245,7 @@ bool ED_editors_flush_edits(Main *bmain, bool for_render)
        * may cause a flush on saving: T53986. */
       if ((ob->sculpt && ob->sculpt->cache) == 0) {
         /* flush multires changes (for sculpt) */
-        multires_force_update(ob);
+        multires_flush_sculpt_updates(ob);
         has_edited = true;
 
         if (for_render) {
diff --git a/source/blender/makesrna/intern/rna_modifier.c b/source/blender/makesrna/intern/rna_modifier.c
index 712fe2a9984..eebb09a2bc3 100644
--- a/source/blender/makesrna/intern/rna_modifier.c
+++ b/source/blender/makesrna/intern/rna_modifier.c
@@ -915,7 +915,7 @@ static void rna_MultiresModifier_type_set(PointerRNA *ptr, int value)
   Object *ob = (Object *)ptr->owner_id;
   MultiresModifierData *mmd = (MultiresModifierData *)ptr->data;
 
-  multires_force_update(ob);
+  multires_force_sculpt_rebuild(ob);
   mmd->simple = value;
 }