Improve Solidify/Bevel Modifier cooperation

Adds a slider to solidify which allows the user to add bevel weight on the outside
or remove bevel weight from the inside.

Also includes a very small improvment for working with subsurface modifier where
the rim edge in complex solidify will now also have a chance to get a crease if
there is only two adjacent edges.

Differential Revision: https://developer.blender.org/D7334

Reviewing and minor cleanups: Bastien Montagne (@mont29).

3 files changed, 223 insertions, 95 deletions

diff --git a/source/blender/modifiers/intern/MOD_solidify.c b/source/blender/modifiers/intern/MOD_solidify.c
index 74d9df7d093..fc9719a5fb4 100644
--- a/source/blender/modifiers/intern/MOD_solidify.c
+++ b/source/blender/modifiers/intern/MOD_solidify.c
@@ -54,6 +54,7 @@ static void initData(ModifierData *md)
   smd->nonmanifold_offset_mode = MOD_SOLIDIFY_NONMANIFOLD_OFFSET_MODE_CONSTRAINTS;
   smd->nonmanifold_boundary_mode = MOD_SOLIDIFY_NONMANIFOLD_BOUNDARY_MODE_NONE;
   smd->merge_tolerance = 0.0001f;
+  smd->bevel_convex = 0.0f;
 }
 
 static void requiredDataMask(Object *UNUSED(ob),
diff --git a/source/blender/modifiers/intern/MOD_solidify_extrude.c b/source/blender/modifiers/intern/MOD_solidify_extrude.c
index ecd62e5e1fe..56ec7796b0d 100644
--- a/source/blender/modifiers/intern/MOD_solidify_extrude.c
+++ b/source/blender/modifiers/intern/MOD_solidify_extrude.c
@@ -224,15 +224,18 @@ Mesh *MOD_solidify_extrude_applyModifier(ModifierData *md,
 
   const bool need_poly_normals = (smd->flag & MOD_SOLIDIFY_NORMAL_CALC) ||
                                  (smd->flag & MOD_SOLIDIFY_EVEN) ||
-                                 (smd->flag & MOD_SOLIDIFY_OFFSET_ANGLE_CLAMP);
+                                 (smd->flag & MOD_SOLIDIFY_OFFSET_ANGLE_CLAMP) ||
+                                 (smd->bevel_convex != 0);
 
   const float ofs_orig = -(((-smd->offset_fac + 1.0f) * 0.5f) * smd->offset);
   const float ofs_new = smd->offset + ofs_orig;
   const float offset_fac_vg = smd->offset_fac_vg;
   const float offset_fac_vg_inv = 1.0f - smd->offset_fac_vg;
+  const float bevel_convex = smd->bevel_convex;
   const bool do_flip = (smd->flag & MOD_SOLIDIFY_FLIP) != 0;
   const bool do_clamp = (smd->offset_clamp != 0.0f);
-  const bool do_angle_clamp = (smd->flag & MOD_SOLIDIFY_OFFSET_ANGLE_CLAMP) != 0;
+  const bool do_angle_clamp = do_clamp && (smd->flag & MOD_SOLIDIFY_OFFSET_ANGLE_CLAMP) != 0;
+  const bool do_bevel_convex = bevel_convex != 0.0f;
   const bool do_shell = ((smd->flag & MOD_SOLIDIFY_RIM) && (smd->flag & MOD_SOLIDIFY_NOSHELL)) ==
                         0;
 
@@ -510,62 +513,75 @@ Mesh *MOD_solidify_extrude_applyModifier(ModifierData *md,
     const float offset = fabsf(smd->offset) * smd->offset_clamp;
     const float offset_sq = offset * offset;
 
-    if (do_clamp) {
-      uint i;
+    /* for bevel weight */
+    float *edge_angs = NULL;
 
+    if (do_clamp) {
       vert_lens = MEM_malloc_arrayN(numVerts, sizeof(float), "vert_lens");
       copy_vn_fl(vert_lens, (int)numVerts, FLT_MAX);
-      for (i = 0; i < numEdges; i++) {
+      for (uint i = 0; i < numEdges; i++) {
         const float ed_len_sq = len_squared_v3v3(mvert[medge[i].v1].co, mvert[medge[i].v2].co);
         vert_lens[medge[i].v1] = min_ff(vert_lens[medge[i].v1], ed_len_sq);
         vert_lens[medge[i].v2] = min_ff(vert_lens[medge[i].v2], ed_len_sq);
       }
+    }
+
+    if (do_angle_clamp || do_bevel_convex) {
+      uint eidx;
       if (do_angle_clamp) {
-        uint eidx;
         vert_angs = MEM_malloc_arrayN(numVerts, sizeof(float), "vert_angs");
         copy_vn_fl(vert_angs, (int)numVerts, 0.5f * M_PI);
-        uint(*edge_user_pairs)[2] = MEM_malloc_arrayN(
-            numEdges, sizeof(*edge_user_pairs), "edge_user_pairs");
-        for (eidx = 0; eidx < numEdges; eidx++) {
-          edge_user_pairs[eidx][0] = INVALID_UNUSED;
-          edge_user_pairs[eidx][1] = INVALID_UNUSED;
-        }
-        for (i = 0, mp = orig_mpoly; i < numPolys; i++, mp++) {
-          ml = orig_mloop + mp->loopstart;
-          MLoop *ml_prev = ml + (mp->totloop - 1);
-
-          for (int j = 0; j < mp->totloop; j++, ml++) {
-            /* add edge user */
-            eidx = ml_prev->e;
-            ed = orig_medge + eidx;
-            BLI_assert(ELEM(ml_prev->v, ed->v1, ed->v2) && ELEM(ml->v, ed->v1, ed->v2));
-            char flip = (char)((ml_prev->v > ml->v) == (ed->v1 < ed->v2));
-            if (edge_user_pairs[eidx][flip] == INVALID_UNUSED) {
-              edge_user_pairs[eidx][flip] = i;
-            }
-            else {
-              edge_user_pairs[eidx][0] = INVALID_PAIR;
-              edge_user_pairs[eidx][1] = INVALID_PAIR;
-            }
-            ml_prev = ml;
+      }
+      if (do_bevel_convex) {
+        edge_angs = MEM_malloc_arrayN(numEdges, sizeof(float), "edge_angs");
+      }
+      uint(*edge_user_pairs)[2] = MEM_malloc_arrayN(
+          numEdges, sizeof(*edge_user_pairs), "edge_user_pairs");
+      for (eidx = 0; eidx < numEdges; eidx++) {
+        edge_user_pairs[eidx][0] = INVALID_UNUSED;
+        edge_user_pairs[eidx][1] = INVALID_UNUSED;
+      }
+      mp = orig_mpoly;
+      for (uint i = 0; i < numPolys; i++, mp++) {
+        ml = orig_mloop + mp->loopstart;
+        MLoop *ml_prev = ml + (mp->totloop - 1);
+
+        for (uint j = 0; j < mp->totloop; j++, ml++) {
+          /* add edge user */
+          eidx = ml_prev->e;
+          ed = orig_medge + eidx;
+          BLI_assert(ELEM(ml_prev->v, ed->v1, ed->v2) && ELEM(ml->v, ed->v1, ed->v2));
+          char flip = (char)((ml_prev->v > ml->v) == (ed->v1 < ed->v2));
+          if (edge_user_pairs[eidx][flip] == INVALID_UNUSED) {
+            edge_user_pairs[eidx][flip] = i;
           }
+          else {
+            edge_user_pairs[eidx][0] = INVALID_PAIR;
+            edge_user_pairs[eidx][1] = INVALID_PAIR;
+          }
+          ml_prev = ml;
         }
-        ed = orig_medge;
-        float e[3];
-        for (i = 0; i < numEdges; i++, ed++) {
-          if (!ELEM(edge_user_pairs[i][0], INVALID_UNUSED, INVALID_PAIR) &&
-              !ELEM(edge_user_pairs[i][1], INVALID_UNUSED, INVALID_PAIR)) {
-            const float *n0 = poly_nors[edge_user_pairs[i][0]];
-            const float *n1 = poly_nors[edge_user_pairs[i][1]];
-            sub_v3_v3v3(e, orig_mvert[ed->v1].co, orig_mvert[ed->v2].co);
-            normalize_v3(e);
-            const float angle = angle_signed_on_axis_v3v3_v3(n0, n1, e);
+      }
+      ed = orig_medge;
+      float e[3];
+      for (uint i = 0; i < numEdges; i++, ed++) {
+        if (!ELEM(edge_user_pairs[i][0], INVALID_UNUSED, INVALID_PAIR) &&
+            !ELEM(edge_user_pairs[i][1], INVALID_UNUSED, INVALID_PAIR)) {
+          const float *n0 = poly_nors[edge_user_pairs[i][0]];
+          const float *n1 = poly_nors[edge_user_pairs[i][1]];
+          sub_v3_v3v3(e, orig_mvert[ed->v1].co, orig_mvert[ed->v2].co);
+          normalize_v3(e);
+          const float angle = angle_signed_on_axis_v3v3_v3(n0, n1, e);
+          if (do_angle_clamp) {
             vert_angs[ed->v1] = max_ff(vert_angs[ed->v1], angle);
             vert_angs[ed->v2] = max_ff(vert_angs[ed->v2], angle);
           }
+          if (do_bevel_convex) {
+            edge_angs[i] = angle;
+          }
         }
-        MEM_freeN(edge_user_pairs);
       }
+      MEM_freeN(edge_user_pairs);
     }
 
     if (ofs_new != 0.0f) {
@@ -661,6 +677,30 @@ Mesh *MOD_solidify_extrude_applyModifier(ModifierData *md,
       }
     }
 
+    if (do_bevel_convex) {
+      for (uint i = 0; i < numEdges; i++) {
+        if (edge_users[i] == INVALID_PAIR) {
+          float angle = edge_angs[i];
+          medge[i].bweight = (char)clamp_i(
+              (int)medge[i].bweight + (int)((angle < M_PI ? clamp_f(bevel_convex, 0.0f, 1.0f) :
+                                                            clamp_f(bevel_convex, -1.0f, 0.0f)) *
+                                            255),
+              0,
+              255);
+          if (do_shell) {
+            medge[i + numEdges].bweight = (char)clamp_i(
+                (int)medge[i + numEdges].bweight +
+                    (int)((angle > M_PI ? clamp_f(bevel_convex, 0.0f, 1.0f) :
+                                          clamp_f(bevel_convex, -1.0f, 0.0f)) *
+                          255),
+                0,
+                255);
+          }
+        }
+      }
+      MEM_freeN(edge_angs);
+    }
+
     if (do_clamp) {
       MEM_freeN(vert_lens);
       if (do_angle_clamp) {
@@ -757,6 +797,68 @@ Mesh *MOD_solidify_extrude_applyModifier(ModifierData *md,
       }
     }
 
+    /* for angle clamp */
+    float *vert_angs = NULL;
+    /* for bevel convex */
+    float *edge_angs = NULL;
+
+    if (do_angle_clamp || do_bevel_convex) {
+      uint eidx;
+      if (do_angle_clamp) {
+        vert_angs = MEM_malloc_arrayN(numVerts, sizeof(float), "vert_angs even");
+        copy_vn_fl(vert_angs, (int)numVerts, 0.5f * M_PI);
+      }
+      if (do_bevel_convex) {
+        edge_angs = MEM_malloc_arrayN(numEdges, sizeof(float), "edge_angs even");
+      }
+      uint(*edge_user_pairs)[2] = MEM_malloc_arrayN(
+          numEdges, sizeof(*edge_user_pairs), "edge_user_pairs");
+      for (eidx = 0; eidx < numEdges; eidx++) {
+        edge_user_pairs[eidx][0] = INVALID_UNUSED;
+        edge_user_pairs[eidx][1] = INVALID_UNUSED;
+      }
+      for (i = 0, mp = orig_mpoly; i < numPolys; i++, mp++) {
+        ml = orig_mloop + mp->loopstart;
+        MLoop *ml_prev = ml + (mp->totloop - 1);
+
+        for (int j = 0; j < mp->totloop; j++, ml++) {
+          /* add edge user */
+          eidx = ml_prev->e;
+          ed = orig_medge + eidx;
+          BLI_assert(ELEM(ml_prev->v, ed->v1, ed->v2) && ELEM(ml->v, ed->v1, ed->v2));
+          char flip = (char)((ml_prev->v > ml->v) == (ed->v1 < ed->v2));
+          if (edge_user_pairs[eidx][flip] == INVALID_UNUSED) {
+            edge_user_pairs[eidx][flip] = i;
+          }
+          else {
+            edge_user_pairs[eidx][0] = INVALID_PAIR;
+            edge_user_pairs[eidx][1] = INVALID_PAIR;
+          }
+          ml_prev = ml;
+        }
+      }
+      ed = orig_medge;
+      float e[3];
+      for (i = 0; i < numEdges; i++, ed++) {
+        if (!ELEM(edge_user_pairs[i][0], INVALID_UNUSED, INVALID_PAIR) &&
+            !ELEM(edge_user_pairs[i][1], INVALID_UNUSED, INVALID_PAIR)) {
+          const float *n0 = poly_nors[edge_user_pairs[i][0]];
+          const float *n1 = poly_nors[edge_user_pairs[i][1]];
+          if (do_angle_clamp) {
+            const float angle = M_PI - angle_normalized_v3v3(n0, n1);
+            vert_angs[ed->v1] = max_ff(vert_angs[ed->v1], angle);
+            vert_angs[ed->v2] = max_ff(vert_angs[ed->v2], angle);
+          }
+          if (do_bevel_convex) {
+            sub_v3_v3v3(e, orig_mvert[ed->v1].co, orig_mvert[ed->v2].co);
+            normalize_v3(e);
+            edge_angs[i] = angle_signed_on_axis_v3v3_v3(n0, n1, e);
+          }
+        }
+      }
+      MEM_freeN(edge_user_pairs);
+    }
+
     if (do_clamp) {
       const float clamp_fac = 1 + (do_angle_clamp ? fabsf(smd->offset_fac) : 0);
       const float offset = fabsf(smd->offset) * smd->offset_clamp * clamp_fac;
@@ -770,48 +872,6 @@ Mesh *MOD_solidify_extrude_applyModifier(ModifierData *md,
           vert_lens_sq[medge[i].v2] = min_ff(vert_lens_sq[medge[i].v2], ed_len);
         }
         if (do_angle_clamp) {
-          uint eidx;
-          float *vert_angs = MEM_malloc_arrayN(numVerts, sizeof(float), "vert_angs even");
-          copy_vn_fl(vert_angs, (int)numVerts, 0.5f * M_PI);
-          uint(*edge_user_pairs)[2] = MEM_malloc_arrayN(
-              numEdges, sizeof(*edge_user_pairs), "edge_user_pairs");
-          for (eidx = 0; eidx < numEdges; eidx++) {
-            edge_user_pairs[eidx][0] = INVALID_UNUSED;
-            edge_user_pairs[eidx][1] = INVALID_UNUSED;
-          }
-          for (i = 0, mp = orig_mpoly; i < numPolys; i++, mp++) {
-            ml = orig_mloop + mp->loopstart;
-            MLoop *ml_prev = ml + (mp->totloop - 1);
-
-            for (int j = 0; j < mp->totloop; j++, ml++) {
-              /* add edge user */
-              eidx = ml_prev->e;
-              ed = orig_medge + eidx;
-              BLI_assert(ELEM(ml_prev->v, ed->v1, ed->v2) && ELEM(ml->v, ed->v1, ed->v2));
-              char flip = (char)((ml_prev->v > ml->v) == (ed->v1 < ed->v2));
-              if (edge_user_pairs[eidx][flip] == INVALID_UNUSED) {
-                edge_user_pairs[eidx][flip] = i;
-              }
-              else {
-                edge_user_pairs[eidx][0] = INVALID_PAIR;
-                edge_user_pairs[eidx][1] = INVALID_PAIR;
-              }
-              ml_prev = ml;
-            }
-          }
-          ed = orig_medge;
-          for (i = 0; i < numEdges; i++, ed++) {
-            if (!ELEM(edge_user_pairs[i][0], INVALID_UNUSED, INVALID_PAIR) &&
-                !ELEM(edge_user_pairs[i][1], INVALID_UNUSED, INVALID_PAIR)) {
-              const float *n0 = poly_nors[edge_user_pairs[i][0]];
-              const float *n1 = poly_nors[edge_user_pairs[i][1]];
-              const float angle = M_PI - angle_normalized_v3v3(n0, n1);
-              vert_angs[ed->v1] = max_ff(vert_angs[ed->v1], angle);
-              vert_angs[ed->v2] = max_ff(vert_angs[ed->v2], angle);
-            }
-          }
-          MEM_freeN(edge_user_pairs);
-
           for (i = 0; i < numVerts; i++) {
             float cos_ang = cosf(vert_angs[i] * 0.5f);
             if (cos_ang > 0) {
@@ -835,6 +895,30 @@ Mesh *MOD_solidify_extrude_applyModifier(ModifierData *md,
       }
     }
 
+    if (do_bevel_convex) {
+      for (i = 0; i < numEdges; i++) {
+        if (edge_users[i] == INVALID_PAIR) {
+          float angle = edge_angs[i];
+          medge[i].bweight = (char)clamp_i(
+              (int)medge[i].bweight + (int)((angle < M_PI ? clamp_f(bevel_convex, 0, 1) :
+                                                            clamp_f(bevel_convex, -1, 0)) *
+                                            255),
+              0,
+              255);
+          if (do_shell) {
+            medge[i + numEdges].bweight = (char)clamp_i(
+                (int)medge[i + numEdges].bweight +
+                    (int)((angle > M_PI ? clamp_f(bevel_convex, 0, 1) :
+                                          clamp_f(bevel_convex, -1, 0)) *
+                          255),
+                0,
+                255);
+          }
+        }
+      }
+      MEM_freeN(edge_angs);
+    }
+
 #undef INVALID_UNUSED
 #undef INVALID_PAIR
 
diff --git a/source/blender/modifiers/intern/MOD_solidify_nonmanifold.c b/source/blender/modifiers/intern/MOD_solidify_nonmanifold.c
index 1cf9def46fa..70aec2d797b 100644
--- a/source/blender/modifiers/intern/MOD_solidify_nonmanifold.c
+++ b/source/blender/modifiers/intern/MOD_solidify_nonmanifold.c
@@ -175,6 +175,8 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
                         0;
   const bool do_clamp = (smd->offset_clamp != 0.0f);
 
+  const float bevel_convex = smd->bevel_convex;
+
   MDeformVert *dvert;
   const bool defgrp_invert = (smd->flag & MOD_SOLIDIFY_VGROUP_INV) != 0;
   int defgrp_index;
@@ -1798,6 +1800,11 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
   int *origindex_edge = CustomData_get_layer(&result->edata, CD_ORIGINDEX);
   int *origindex_poly = CustomData_get_layer(&result->pdata, CD_ORIGINDEX);
 
+  if (bevel_convex != 0.0f) {
+    /* make sure bweight is enabled */
+    result->cd_flag |= ME_CDFLAG_EDGE_BWEIGHT;
+  }
+
   /* Checks that result has dvert data. */
   if (shell_defgrp_index != -1 || rim_defgrp_index != -1) {
     dvert = CustomData_duplicate_referenced_layer(&result->vdata, CD_MDEFORMVERT, result->totvert);
@@ -1863,6 +1870,17 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
             medge[insert].flag = orig_medge[(*l)->old_edge].flag | ME_EDGEDRAW | ME_EDGERENDER;
             medge[insert].crease = orig_medge[(*l)->old_edge].crease;
             medge[insert].bweight = orig_medge[(*l)->old_edge].bweight;
+            if (bevel_convex != 0.0f && (*l)->faces[1] != NULL) {
+              medge[insert].bweight = (char)clamp_i(
+                  (int)medge[insert].bweight + (int)(((*l)->angle > M_PI + FLT_EPSILON ?
+                                                          clamp_f(bevel_convex, 0.0f, 1.0f) :
+                                                          ((*l)->angle < M_PI - FLT_EPSILON ?
+                                                               clamp_f(bevel_convex, -1.0f, 0.0f) :
+                                                               0)) *
+                                                     255),
+                  0,
+                  255);
+            }
             (*l)->new_edge = insert;
           }
         }
@@ -1914,7 +1932,8 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
   /* Make boundary edges/faces. */
   {
     gs_ptr = orig_vert_groups_arr;
-    for (uint i = 0; i < numVerts; i++, gs_ptr++) {
+    mv = orig_mvert;
+    for (uint i = 0; i < numVerts; i++, gs_ptr++, mv++) {
       EdgeGroup *gs = *gs_ptr;
       if (gs) {
         EdgeGroup *g = gs;
@@ -1936,15 +1955,37 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
             max_crease = 0;
             max_bweight = 0;
             flag = 0;
-            for (uint k = 1; k < g->edges_len - 1; k++) {
-              ed = orig_medge + g->edges[k]->old_edge;
-              if (ed->crease > max_crease) {
-                max_crease = ed->crease;
+
+            BLI_assert(g->edges_len >= 2);
+
+            if (g->edges_len == 2) {
+              max_crease = min_cc(orig_medge[g->edges[0]->old_edge].crease,
+                                  orig_medge[g->edges[1]->old_edge].crease);
+            }
+            else {
+              for (uint k = 1; k < g->edges_len - 1; k++) {
+                ed = orig_medge + g->edges[k]->old_edge;
+                if (ed->crease > max_crease) {
+                  max_crease = ed->crease;
+                }
+                char bweight = medge[g->edges[k]->new_edge].bweight;
+                if (bweight > max_bweight) {
+                  max_bweight = bweight;
+                }
+                flag |= ed->flag;
               }
-              if (ed->bweight > max_bweight) {
-                max_bweight = ed->bweight;
+            }
+
+            const char bweight_open_edge = min_cc(
+                orig_medge[g->edges[0]->old_edge].bweight,
+                orig_medge[g->edges[g->edges_len - 1]->old_edge].bweight);
+            if (bweight_open_edge > 0) {
+              max_bweight = min_cc(bweight_open_edge, max_bweight);
+            }
+            else {
+              if (bevel_convex < 0.0f) {
+                max_bweight = 0;
               }
-              flag |= ed->flag;
             }
             if (!first_g) {
               first_g = g;
@@ -1966,8 +2007,9 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
               medge[edge_index].v2 = g->new_vert;
               medge[edge_index].flag = ME_EDGEDRAW | ME_EDGERENDER |
                                        ((last_flag | flag) & (ME_SEAM | ME_SHARP));
-              medge[edge_index].crease = MAX2(last_max_crease, max_crease);
-              medge[edge_index++].bweight = MAX2(last_max_bweight, max_bweight);
+              medge[edge_index].crease = min_cc(last_max_crease, max_crease);
+              medge[edge_index++].bweight = max_cc(mv->bweight,
+                                                   min_cc(last_max_bweight, max_bweight));
             }
             last_g = g;
             last_max_crease = max_crease;
@@ -1993,8 +2035,9 @@ Mesh *MOD_solidify_nonmanifold_applyModifier(ModifierData *md,
               medge[edge_index].v2 = first_g->new_vert;
               medge[edge_index].flag = ME_EDGEDRAW | ME_EDGERENDER |
                                        ((last_flag | first_flag) & (ME_SEAM | ME_SHARP));
-              medge[edge_index].crease = MAX2(last_max_crease, first_max_crease);
-              medge[edge_index++].bweight = MAX2(last_max_bweight, first_max_bweight);
+              medge[edge_index].crease = min_cc(last_max_crease, first_max_crease);
+              medge[edge_index++].bweight = max_cc(mv->bweight,
+                                                   min_cc(last_max_bweight, first_max_bweight));
 
               /* Loop data. */
               int *loops = MEM_malloc_arrayN(j, sizeof(*loops), "loops in solidify");