Mesh calc smooth group: several fixes.

* Consider non-manifold edges as sharp, as in split normals handling.
* Consider edges from sharp polys as sharp!!!
* Fix returned number of groups (was off-by-one for non-bitflags grouping, could also
  be wrong in case of id overflow).

Note about using sharp polys too to define groups: Only current use of this function
(Obj exporter) does not need that, because it does its own check for sharp faces.
However, we might reuse that func in other places in future (e.g. in custom split
normals area), so better to get a consistent behavior!

1 files changed, 13 insertions, 2 deletions

diff --git a/source/blender/blenkernel/intern/mesh_mapping.c b/source/blender/blenkernel/intern/mesh_mapping.c
index 82065750791..53d1aae104c 100644
--- a/source/blender/blenkernel/intern/mesh_mapping.c
+++ b/source/blender/blenkernel/intern/mesh_mapping.c
@@ -416,18 +416,22 @@ int *BKE_mesh_calc_smoothgroups(const MEdge *medge, const int totedge,
 		while (ps_curr_idx != ps_end_idx) {
 			const MPoly *mp;
 			const MLoop *ml;
+			bool sharp_poly;
 			int j;
 
 			poly = poly_stack[ps_curr_idx++];
 			BLI_assert(poly_groups[poly] == poly_group_id);
 
 			mp = &mpoly[poly];
+			sharp_poly = !(mp->flag & ME_SMOOTH);
 			for (ml = &mloop[mp->loopstart], j = mp->totloop; j--; ml++) {
 				/* loop over poly users */
 				const MeshElemMap *map_ele = &edge_poly_map[ml->e];
 				const int *p = map_ele->indices;
 				int i = map_ele->count;
-				if (!(medge[ml->e].flag & ME_SHARP)) {
+				/* Edge is smooth only if its poly is not sharp, edge is not sharp,
+				 * and edge is used by exactly two polygons. */
+				if (!sharp_poly && !(medge[ml->e].flag & ME_SHARP) && i == 2) {
 					for (; i--; p++) {
 						/* if we meet other non initialized its a bug */
 						BLI_assert(ELEM(poly_groups[*p], 0, poly_group_id));
@@ -482,6 +486,11 @@ int *BKE_mesh_calc_smoothgroups(const MEdge *medge, const int totedge,
 		}
 	}
 
+	if (use_bitflags) {
+		/* used bits are zero-based. */
+		tot_group++;
+	}
+
 	if (UNLIKELY(group_id_overflow)) {
 		int i = totpoly, *gid = poly_groups;
 		for (; i--; gid++) {
@@ -489,13 +498,15 @@ int *BKE_mesh_calc_smoothgroups(const MEdge *medge, const int totedge,
 				*gid = 0;
 			}
 		}
+		/* Using 0 as group id adds one more group! */
+		tot_group++;
 	}
 
 	MEM_freeN(edge_poly_map);
 	MEM_freeN(edge_poly_mem);
 	MEM_freeN(poly_stack);
 
-	*r_totgroup = tot_group + 1;
+	*r_totgroup = tot_group;
 
 	return poly_groups;
 }