Transform: improve normal orientation

- when 3 verts are selected ensure the normal is flipped on the side of existing vert normals.
  also use the most distant 2 verts to define the tangent.

- when 2 vertices are selected, the normal wasn't aligned with the vert normal.

1 files changed, 32 insertions, 7 deletions

diff --git a/source/blender/editors/transform/transform_orientations.c b/source/blender/editors/transform/transform_orientations.c
index 82c2d380d2e..6dc685c0148 100644
--- a/source/blender/editors/transform/transform_orientations.c
+++ b/source/blender/editors/transform/transform_orientations.c
@@ -577,7 +577,6 @@ int getTransformOrientation(const bContext *C, float normal[3], float plane[3],
 				else if (em->bm->totvertsel == 3) {
 					BMVert *v1 = NULL, *v2 = NULL, *v3 = NULL;
 					BMIter iter;
-					float cotangent[3];
 					
 					BM_ITER_MESH (eve, &iter, em->bm, BM_VERTS_OF_MESH) {
 						if (BM_elem_flag_test(eve, BM_ELEM_SELECT)) {
@@ -588,11 +587,37 @@ int getTransformOrientation(const bContext *C, float normal[3], float plane[3],
 								v2 = eve;
 							}
 							else {
+								float no_test[3];
+
+								float tan_a[3], tan_b[3], tan_c[3];
+								float len_a, len_b, len_c;
+								const float *tan_best;
+
+
 								v3 = eve;
+								sub_v3_v3v3(tan_a, v2->co, v1->co);
+								sub_v3_v3v3(tan_b, v3->co, v2->co);
+								sub_v3_v3v3(tan_c, v1->co, v3->co);
+								cross_v3_v3v3(normal, tan_b, tan_a);
+
+								/* check if the normal is pointing opposite to vert normals */
+								no_test[0] = v1->no[0] + v2->no[0] + v3->no[0];
+								no_test[1] = v1->no[1] + v2->no[1] + v3->no[1];
+								no_test[2] = v1->no[2] + v2->no[2] + v3->no[2];
+								if (dot_v3v3(no_test, normal) < 0.0f) {
+									negate_v3(normal);
+								}
+
+								/* always give the plane to the 2 most distant verts */
+								len_a = len_squared_v3(tan_a);
+								len_b = len_squared_v3(tan_b);
+								len_c = len_squared_v3(tan_c);
+
+								tan_best = MAX3_PAIR(len_a, len_b, len_c,
+								                     tan_a, tan_b, tan_c);
+
+								copy_v3_v3(plane, tan_best);
 
-								sub_v3_v3v3(plane, v2->co, v1->co);
-								sub_v3_v3v3(cotangent, v3->co, v2->co);
-								cross_v3_v3v3(normal, cotangent, plane);
 								break;
 							}
 						}
@@ -639,9 +664,9 @@ int getTransformOrientation(const bContext *C, float normal[3], float plane[3],
 							else {
 								v2 = eve;
 								
-								copy_v3_v3(plane, v1->no);
-								add_v3_v3(plane, v2->no);
-								sub_v3_v3v3(normal, v2->co, v1->co);
+								copy_v3_v3(normal, v1->no);
+								add_v3_v3(normal, v2->no);
+								sub_v3_v3v3(plane, v2->co, v1->co);
 								break; 
 							}
 						}