Enable dyntopo re-entering sculpt mode

1 files changed, 91 insertions, 31 deletions

diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 0ab8203c85f..67609d98683 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -5105,13 +5105,19 @@ static int sculpt_dynamic_topology_toggle_exec(bContext *C, wmOperator *UNUSED(o
 	return OPERATOR_FINISHED;
 }
 
+enum eDynTopoWarnFlag {
+	DYNTOPO_WARN_VDATA = (1 << 0),
+	DYNTOPO_WARN_EDATA = (1 << 1),
+	DYNTOPO_WARN_LDATA = (1 << 2),
+	DYNTOPO_WARN_MODIFIER = (1 << 3),
+};
 
-static int dyntopo_warning_popup(bContext *C, wmOperatorType *ot, bool vdata, bool modifiers)
+static int dyntopo_warning_popup(bContext *C, wmOperatorType *ot, enum eDynTopoWarnFlag flag)
 {
 	uiPopupMenu *pup = UI_popup_menu_begin(C, IFACE_("Warning!"), ICON_ERROR);
 	uiLayout *layout = UI_popup_menu_layout(pup);
 
-	if (vdata) {
+	if (flag & (DYNTOPO_WARN_VDATA | DYNTOPO_WARN_EDATA | DYNTOPO_WARN_LDATA)) {
 		const char *msg_error = TIP_("Vertex Data Detected!");
 		const char *msg = TIP_("Dyntopo will not preserve vertex colors, UVs, or other customdata");
 		uiItemL(layout, msg_error, ICON_INFO);
@@ -5119,7 +5125,7 @@ static int dyntopo_warning_popup(bContext *C, wmOperatorType *ot, bool vdata, bo
 		uiItemS(layout);
 	}
 
-	if (modifiers) {
+	if (flag & DYNTOPO_WARN_MODIFIER) {
 		const char *msg_error = TIP_("Generative Modifiers Detected!");
 		const char *msg = TIP_("Keeping the modifiers will increase polycount when returning to object mode");
 
@@ -5135,33 +5141,35 @@ static int dyntopo_warning_popup(bContext *C, wmOperatorType *ot, bool vdata, bo
 	return OPERATOR_INTERFACE;
 }
 
-
-static int sculpt_dynamic_topology_toggle_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(event))
+static enum eDynTopoWarnFlag sculpt_dynamic_topology_check(bContext *C)
 {
 	Object *ob = CTX_data_active_object(C);
 	Mesh *me = ob->data;
 	SculptSession *ss = ob->sculpt;
 
-	if (!ss->bm) {
-		Scene *scene = CTX_data_scene(C);
-		ModifierData *md;
-		VirtualModifierData virtualModifierData;
-		int i;
-		bool vdata = false;
-		bool modifiers = false;
-
-		for (i = 0; i < CD_NUMTYPES; i++) {
-			if (!ELEM(i, CD_MVERT, CD_MEDGE, CD_MFACE, CD_MLOOP, CD_MPOLY, CD_PAINT_MASK, CD_ORIGINDEX) &&
-			    (CustomData_has_layer(&me->vdata, i) ||
-			     CustomData_has_layer(&me->edata, i) ||
-			     CustomData_has_layer(&me->ldata, i)))
-			{
-				vdata = true;
-				break;
+	Scene *scene = CTX_data_scene(C);
+	enum eDynTopoWarnFlag flag = 0;
+
+	BLI_assert(ss->bm == NULL);
+	UNUSED_VARS_NDEBUG(ss);
+
+	for (int i = 0; i < CD_NUMTYPES; i++) {
+		if (!ELEM(i, CD_MVERT, CD_MEDGE, CD_MFACE, CD_MLOOP, CD_MPOLY, CD_PAINT_MASK, CD_ORIGINDEX)) {
+			if (CustomData_has_layer(&me->vdata, i)) {
+				flag |= DYNTOPO_WARN_VDATA;
+			}
+			if (CustomData_has_layer(&me->edata, i)) {
+				flag |= DYNTOPO_WARN_EDATA;
+			}
+			if (CustomData_has_layer(&me->ldata, i)) {
+				flag |= DYNTOPO_WARN_LDATA;
 			}
 		}
+	}
 
-		md = modifiers_getVirtualModifierList(ob, &virtualModifierData);
+	{
+		VirtualModifierData virtualModifierData;
+		ModifierData *md = modifiers_getVirtualModifierList(ob, &virtualModifierData);
 
 		/* exception for shape keys because we can edit those */
 		for (; md; md = md->next) {
@@ -5169,14 +5177,26 @@ static int sculpt_dynamic_topology_toggle_invoke(bContext *C, wmOperator *op, co
 			if (!modifier_isEnabled(scene, md, eModifierMode_Realtime)) continue;
 
 			if (mti->type == eModifierTypeType_Constructive) {
-				modifiers = true;
+				flag |= DYNTOPO_WARN_MODIFIER;
 				break;
 			}
 		}
+	}
+
+	return flag;
+}
+
+static int sculpt_dynamic_topology_toggle_invoke(bContext *C, wmOperator *op, const wmEvent *UNUSED(event))
+{
+	Object *ob = CTX_data_active_object(C);
+	SculptSession *ss = ob->sculpt;
+
+	if (!ss->bm) {
+		enum eDynTopoWarnFlag flag = sculpt_dynamic_topology_check(C);
 
-		if (vdata || modifiers) {
+		if (flag) {
 			/* The mesh has customdata that will be lost, let the user confirm this is OK */
-			return dyntopo_warning_popup(C, op->type, vdata, modifiers);
+			return dyntopo_warning_popup(C, op->type, flag);
 		}
 	}
 
@@ -5329,6 +5349,9 @@ static int sculpt_mode_toggle_exec(bContext *C, wmOperator *op)
 			 * mode to ensure the undo stack stays in a consistent
 			 * state */
 			sculpt_dynamic_topology_toggle_exec(C, NULL);
+
+			/* store so we know to re-enable when entering sculpt mode */
+			me->flag |= ME_SCULPT_DYNAMIC_TOPOLOGY;
 		}
 
 		/* Leave sculptmode */
@@ -5342,12 +5365,6 @@ static int sculpt_mode_toggle_exec(bContext *C, wmOperator *op)
 		/* Enter sculptmode */
 		ob->mode |= mode_flag;
 
-		/* Remove dynamic-topology flag; this will be enabled if the
-		 * file was saved with dynamic topology on, but we don't
-		 * automatically re-enter dynamic-topology mode when loading a
-		 * file. */
-		me->flag &= ~ME_SCULPT_DYNAMIC_TOPOLOGY;
-
 		if (flush_recalc)
 			DAG_id_tag_update(&ob->id, OB_RECALC_DATA);
 
@@ -5401,6 +5418,49 @@ static int sculpt_mode_toggle_exec(bContext *C, wmOperator *op)
 		BKE_paint_init(scene, ePaintSculpt, PAINT_CURSOR_SCULPT);
 
 		paint_cursor_start(C, sculpt_poll_view3d);
+
+		/* Check dynamic-topology flag; re-enter dynamic-topology mode when changing modes,
+		 * As long as no data was added that is not supported. */
+		if (me->flag & ME_SCULPT_DYNAMIC_TOPOLOGY) {
+			const char *message_unsupported = NULL;
+			if (me->totloop != me->totpoly * 3) {
+				message_unsupported = TIP_("non-triangle face");
+			}
+			else if (mmd != NULL) {
+				message_unsupported = TIP_("multi-res modifier");
+			}
+			else {
+				enum eDynTopoWarnFlag flag = sculpt_dynamic_topology_check(C);
+				if (flag == 0) {
+					/* pass */
+				}
+				else if (flag & DYNTOPO_WARN_VDATA) {
+					message_unsupported = TIP_("vertex data");
+				}
+				else if (flag & DYNTOPO_WARN_EDATA) {
+					message_unsupported = TIP_("edge data");
+				}
+				else if (flag & DYNTOPO_WARN_LDATA) {
+					message_unsupported = TIP_("face data");
+				}
+				else if (flag & DYNTOPO_WARN_MODIFIER) {
+					message_unsupported = TIP_("constructive modifier");
+				}
+				else {
+					BLI_assert(0);
+				}
+			}
+
+			if (message_unsupported == NULL) {
+				sculpt_dynamic_topology_enable(C);
+			}
+			else {
+				BKE_reportf(op->reports, RPT_WARNING,
+				            "Dynamic Topology found: %s, disabled",
+				            message_unsupported);
+				me->flag &= ~ME_SCULPT_DYNAMIC_TOPOLOGY;
+			}
+		}
 	}
 
 	if (ob->derivedFinal) /* VBO no longer valid */