Sculpts calc_area_normal() could result in a zero length vector if all faces were pointing away from the view.

deal with this by making 2 average vectors, one for front pointing faces, and another for back pointing faces,
also removed an unneeded acos().

1 files changed, 18 insertions, 8 deletions

diff --git a/source/blender/src/sculptmode.c b/source/blender/src/sculptmode.c
index 217fd9f3705..273c9e910bb 100644
--- a/source/blender/src/sculptmode.c
+++ b/source/blender/src/sculptmode.c
@@ -418,14 +418,17 @@ void sculpt_clip(const BrushAction *a, float *co, const float val[3])
 	}		
 }
 
-void add_norm_if(const BrushAction *a, float out[3], const short no[3])
+static void add_norm_if(const BrushAction *a, float out[3], float out_flip[3], const short no[3])
 {
 	float fno[3] = {no[0], no[1], no[2]};
 
 	Normalize(fno);
 
-	if(acos(Inpf(((BrushAction*)a)->symm.out, fno)) < M_PI_2)
+	if((Inpf(((BrushAction*)a)->symm.out, fno)) < 0) {
 		VecAddf(out, out, fno);
+	} else if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
+		VecAddf(out_flip, out_flip, fno); /* out_flip is used when out is {0,0,0} */
+	}
 }
 
 /* Currently only for the draw brush; finds average normal for all active
@@ -435,18 +438,23 @@ void calc_area_normal(float out[3], const BrushAction *a, const float *outdir, c
 	Mesh *me = get_mesh(OBACT);
 	ActiveData *node = active_verts->first;
 	const int view = sculpt_data()->brush_type==DRAW_BRUSH ? sculptmode_brush()->view : 0;
-
-	out[0] = out[1] = out[2] = 0;
+	float out_flip[3];
+	
+	out[0]=out[1]=out[2] = out_flip[0]=out_flip[1]=out_flip[2] = 0;
 
 	if(sculptmode_brush()->flag & SCULPT_BRUSH_ANCHORED) {
 		for(; node; node = node->next)
-			add_norm_if(a, out, a->orig_norms[node->Index]);
+			add_norm_if(a, out, out_flip, a->orig_norms[node->Index]);
 	}
 	else {
 		for(; node; node = node->next)
-			add_norm_if(a, out, me->mvert[node->Index].no);
+			add_norm_if(a, out, out_flip, me->mvert[node->Index].no);
 	}
 
+	if (out[0]==0.0 && out[1]==0.0 && out[2]==0.0) {
+		VECCOPY(out, out_flip);
+	}
+	
 	Normalize(out);
 
 	if(outdir) {
@@ -690,9 +698,11 @@ void do_flatten_brush(const BrushAction *a, const ListBase *active_verts)
 		VecSubf(val, intr, co);
 		VecMulf(val, node->Fade);
 		VecAddf(val, val, co);
-		                     
-		sculpt_clip(a, co, val);
 		
+		sculpt_clip(a, co, val);
+		if (isnan(co[0]) || isnan(co[1]) || isnan(co[2])) {
+			printf("NAN\n");
+		} 
 		node= node->next;
 	}
 }