BMesh: avoid incorrect/invalid matrix calculation

- `BKE_object_scale_to_mat3` was used to get the worldspace scale,
  without taking constraints, parenting etc into account.
- Don't pass object's into BMesh API, (prefer matrices instead).
- Avoid matrix invert for each edge-angle calculation.
- Avoid 2x matrix multiplies when looping over edge pairs.

1 files changed, 20 insertions, 6 deletions

diff --git a/source/blender/editors/mesh/editmesh_select_similar.c b/source/blender/editors/mesh/editmesh_select_similar.c
index 85b59cb78c3..1124b58e116 100644
--- a/source/blender/editors/mesh/editmesh_select_similar.c
+++ b/source/blender/editors/mesh/editmesh_select_similar.c
@@ -245,6 +245,9 @@ static int similar_face_select_exec(bContext *C, wmOperator *op)
 			continue;
 		}
 
+		float ob_m3[3][3];
+		copy_m3_m4(ob_m3, ob->obmat);
+
 		switch (type) {
 			case SIMFACE_MATERIAL:
 			{
@@ -293,14 +296,14 @@ static int similar_face_select_exec(bContext *C, wmOperator *op)
 					}
 					case SIMFACE_AREA:
 					{
-						float area = BM_face_calc_area_worldspace(ob, face);
+						float area = BM_face_calc_area_with_mat3(face, ob_m3);
 						float dummy[3] = {area, 0.0f, 0.0f};
 						BLI_kdtree_insert(tree, tree_index++, dummy);
 						break;
 					}
 					case SIMFACE_PERIMETER:
 					{
-						float perimeter = BM_face_calc_perimeter_worldspace(ob, face);
+						float perimeter = BM_face_calc_perimeter_with_mat3(face, ob_m3);
 						float dummy[3] = {perimeter, 0.0f, 0.0f};
 						BLI_kdtree_insert(tree, tree_index++, dummy);
 						break;
@@ -371,6 +374,9 @@ static int similar_face_select_exec(bContext *C, wmOperator *op)
 		Material ***material_array = NULL;
 		int custom_data_offset;
 
+		float ob_m3[3][3];
+		copy_m3_m4(ob_m3, ob->obmat);
+
 		bool has_custom_data_layer = false;
 		switch (type) {
 			case SIMFACE_MATERIAL:
@@ -440,7 +446,7 @@ static int similar_face_select_exec(bContext *C, wmOperator *op)
 					}
 					case SIMFACE_AREA:
 					{
-						float area = BM_face_calc_area_worldspace(ob, face);
+						float area = BM_face_calc_area_with_mat3(face, ob_m3);
 						if (ED_select_similar_compare_float_tree(tree, area, thresh, compare)) {
 							select = true;
 						}
@@ -448,7 +454,7 @@ static int similar_face_select_exec(bContext *C, wmOperator *op)
 					}
 					case SIMFACE_PERIMETER:
 					{
-						float perimeter = BM_face_calc_perimeter_worldspace(ob, face);
+						float perimeter = BM_face_calc_perimeter_with_mat3(face, ob_m3);
 						if (ED_select_similar_compare_float_tree(tree, perimeter, thresh, compare)) {
 							select = true;
 						}
@@ -748,6 +754,10 @@ static int similar_edge_select_exec(bContext *C, wmOperator *op)
 			}
 		}
 
+		float ob_m3[3][3], ob_m3_inv[3][3];
+		copy_m3_m4(ob_m3, ob->obmat);
+		invert_m3_m3(ob_m3_inv, ob_m3);
+
 		BMEdge *edge; /* Mesh edge. */
 		BMIter iter; /* Selected edges iterator. */
 
@@ -774,7 +784,7 @@ static int similar_edge_select_exec(bContext *C, wmOperator *op)
 					case SIMEDGE_FACE_ANGLE:
 					{
 						if (BM_edge_face_count_at_most(edge, 2) == 2) {
-							float angle = BM_edge_calc_face_angle_worldspace(ob, edge);
+							float angle = BM_edge_calc_face_angle_with_imat3(edge, ob_m3_inv);
 							float dummy[3] = {angle, 0.0f, 0.0f};
 							BLI_kdtree_insert(tree, tree_index++, dummy);
 						}
@@ -855,6 +865,10 @@ static int similar_edge_select_exec(bContext *C, wmOperator *op)
 			}
 		}
 
+		float ob_m3[3][3], ob_m3_inv[3][3];
+		copy_m3_m4(ob_m3, ob->obmat);
+		invert_m3_m3(ob_m3_inv, ob_m3);
+
 		BMEdge *edge; /* Mesh edge. */
 		BMIter iter; /* Selected edges iterator. */
 
@@ -904,7 +918,7 @@ static int similar_edge_select_exec(bContext *C, wmOperator *op)
 					case SIMEDGE_FACE_ANGLE:
 					{
 						if (BM_edge_face_count_at_most(edge, 2) == 2) {
-							float angle = BM_edge_calc_face_angle_worldspace(ob, edge);
+							float angle = BM_edge_calc_face_angle_with_imat3(edge, ob_m3_inv);
 							if (ED_select_similar_compare_float_tree(tree, angle, thresh, SIM_CMP_EQ)) {
 								select = true;
 							}