Factor out some generic parts of the sculpting PBVH redraw planes code.

One function converts bounding boxes to screen space, the other
converts a screen-space rectangle to 3D clipping planes.

Also const-ified some parameters in the ED_view3d API.

6 files changed, 121 insertions, 68 deletions

diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index cf109768a43..0562f400700 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -201,7 +201,7 @@ void ED_view3d_depth_tag_update(struct RegionView3D *rv3d);
 /* Projection */
 #define IS_CLIPPED        12000
 
-void ED_view3d_calc_clipping(struct BoundBox *bb, float planes[4][4], struct bglMats *mats, struct rcti *rect);
+void ED_view3d_calc_clipping(struct BoundBox *bb, float planes[4][4], struct bglMats *mats, const struct rcti *rect);
 
 void project_short(struct ARegion *ar, const float vec[3], short adr[2]);
 void project_short_noclip(struct ARegion *ar, const float vec[3], short adr[2]);
@@ -215,7 +215,7 @@ void project_float_noclip(struct ARegion *ar, const float vec[3], float adr[2]);
 int ED_view3d_clip_range_get(struct View3D *v3d, struct RegionView3D *rv3d, float *clipsta, float *clipend);
 int ED_view3d_viewplane_get(struct View3D *v3d, struct RegionView3D *rv3d, int winxi, int winyi, struct rctf *viewplane, float *clipsta, float *clipend);
 void ED_view3d_ob_project_mat_get(struct RegionView3D *v3d, struct Object *ob, float pmat[4][4]);
-void ED_view3d_project_float(struct ARegion *a, const float vec[3], float adr[2], float mat[4][4]);
+void ED_view3d_project_float(const struct ARegion *a, const float vec[3], float adr[2], float mat[4][4]);
 void ED_view3d_calc_camera_border(struct Scene *scene, struct ARegion *ar, struct View3D *v3d, struct RegionView3D *rv3d, struct rctf *viewborder_r, short no_shift);
 void ED_view3d_calc_camera_border_size(struct Scene *scene, struct ARegion *ar, struct View3D *v3d, struct RegionView3D *rv3d, float size_r[2]);
 
@@ -257,7 +257,7 @@ void view3d_set_viewcontext(struct bContext *C, struct ViewContext *vc);
 void view3d_operator_needs_opengl(const struct bContext *C);
 void view3d_region_operator_needs_opengl(struct wmWindow *win, struct ARegion *ar);
 int view3d_get_view_aligned_coordinate(struct ViewContext *vc, float fp[3], const int mval[2], const short do_fallback);
-void view3d_get_transformation(struct ARegion *ar, struct RegionView3D *rv3d, struct Object *ob, struct bglMats *mats);
+void view3d_get_transformation(const struct ARegion *ar, struct RegionView3D *rv3d, struct Object *ob, struct bglMats *mats);
 
 /* XXX should move to BLI_math */
 int edge_inside_circle(short centx, short centy, short rad, short x1, short y1, short x2, short y2);
diff --git a/source/blender/editors/sculpt_paint/paint_intern.h b/source/blender/editors/sculpt_paint/paint_intern.h
index 3b001a8fcdc..a5e68f9a244 100644
--- a/source/blender/editors/sculpt_paint/paint_intern.h
+++ b/source/blender/editors/sculpt_paint/paint_intern.h
@@ -41,6 +41,7 @@ struct Mesh;
 struct Object;
 struct PaintStroke;
 struct PointerRNA;
+struct rcti;
 struct Scene;
 struct VPaint;
 struct ViewContext;
@@ -104,6 +105,26 @@ void PAINT_OT_image_from_view(struct wmOperatorType *ot);
 
 
 /* paint_utils.c */
+
+/* Convert the object-space axis-aligned bounding box (expressed as
+   its minimum and maximum corners) into a screen-space rectangle,
+   returns zero if the result is empty */
+int paint_convert_bb_to_rect(struct rcti *rect,
+							 const float bb_min[3],
+							 const float bb_max[3],
+							 const struct ARegion *ar,
+							 struct RegionView3D *rv3d,
+							 struct Object *ob);
+
+/* Get four planes in object-space that describe the projection of
+   screen_rect from screen into object-space (essentially converting a
+   2D screens-space bounding box into four 3D planes) */
+void paint_calc_redraw_planes(float planes[4][4],
+							  const struct ARegion *ar,
+							  struct RegionView3D *rv3d,
+							  struct Object *ob,
+							  const struct rcti *screen_rect);
+
 void projectf(struct bglMats *mats, const float v[3], float p[2]);
 float paint_calc_object_space_radius(struct ViewContext *vc, float center[3], float pixel_radius);
 float paint_get_tex_pixel(struct Brush* br, float u, float v);
diff --git a/source/blender/editors/sculpt_paint/paint_utils.c b/source/blender/editors/sculpt_paint/paint_utils.c
index ed2956ff054..c5e860c41ec 100644
--- a/source/blender/editors/sculpt_paint/paint_utils.c
+++ b/source/blender/editors/sculpt_paint/paint_utils.c
@@ -67,6 +67,77 @@
 
 #include "paint_intern.h"
 
+/* Convert the object-space axis-aligned bounding box (expressed as
+   its minimum and maximum corners) into a screen-space rectangle,
+   returns zero if the result is empty */
+int paint_convert_bb_to_rect(rcti *rect,
+							 const float bb_min[3],
+							 const float bb_max[3],
+							 const ARegion *ar,
+							 RegionView3D *rv3d,
+							 Object *ob)
+{
+	float projection_mat[4][4];
+	int i, j, k;
+
+	rect->xmin = rect->ymin = INT_MAX;
+	rect->xmax = rect->ymax = INT_MIN;
+
+	/* return zero if the bounding box has non-positive volume */
+	if(bb_min[0] > bb_max[0] || bb_min[1] > bb_max[1] || bb_min[2] > bb_max[2])
+		return 0;
+
+	ED_view3d_ob_project_mat_get(rv3d, ob, projection_mat);
+
+	for(i = 0; i < 2; ++i) {
+		for(j = 0; j < 2; ++j) {
+			for(k = 0; k < 2; ++k) {
+				float vec[3], proj[2];
+				vec[0] = i ? bb_min[0] : bb_max[0];
+				vec[1] = j ? bb_min[1] : bb_max[1];
+				vec[2] = k ? bb_min[2] : bb_max[2];
+				/* convert corner to screen space */
+				ED_view3d_project_float(ar, vec, proj,
+										projection_mat);
+				/* expand 2D rectangle */
+				rect->xmin = MIN2(rect->xmin, proj[0]);
+				rect->xmax = MAX2(rect->xmax, proj[0]);
+				rect->ymin = MIN2(rect->ymin, proj[1]);
+				rect->ymax = MAX2(rect->ymax, proj[1]);
+			}
+		}
+	}
+
+	/* return false if the rectangle has non-positive area */
+	return rect->xmin < rect->xmax && rect->ymin < rect->ymax;
+}
+
+/* Get four planes in object-space that describe the projection of
+   screen_rect from screen into object-space (essentially converting a
+   2D screens-space bounding box into four 3D planes) */
+void paint_calc_redraw_planes(float planes[4][4],
+							  const ARegion *ar,
+							  RegionView3D *rv3d,
+							  Object *ob,
+							  const rcti *screen_rect)
+{
+	BoundBox bb;
+	bglMats mats;
+	rcti rect;
+
+	memset(&bb, 0, sizeof(BoundBox));
+	view3d_get_transformation(ar, rv3d, ob, &mats);
+
+	/* use some extra space just in case */
+	rect.xmin -= 2;
+	rect.xmax += 2;
+	rect.ymin -= 2;
+	rect.ymax += 2;
+
+	ED_view3d_calc_clipping(&bb, planes, &mats, screen_rect);
+	mul_m4_fl(planes, -1.0f);
+}
+
 /* convert a point in model coordinates to 2D screen coordinates */
 /* TODO: can be deleted once all calls are replaced with
    view3d_project_float() */
diff --git a/source/blender/editors/sculpt_paint/sculpt.c b/source/blender/editors/sculpt_paint/sculpt.c
index 1cdc7732b21..9fb992d969c 100644
--- a/source/blender/editors/sculpt_paint/sculpt.c
+++ b/source/blender/editors/sculpt_paint/sculpt.c
@@ -248,90 +248,51 @@ typedef struct StrokeCache {
 
 /* Get a screen-space rectangle of the modified area */
 static int sculpt_get_redraw_rect(ARegion *ar, RegionView3D *rv3d,
-				Object *ob, rcti *rect)
+								  Object *ob, rcti *rect)
 {
+	SculptSession *ss;
 	PBVH *pbvh= ob->sculpt->pbvh;
-	float bb_min[3], bb_max[3], pmat[4][4];
-	int i, j, k;
-
-	ED_view3d_ob_project_mat_get(rv3d, ob, pmat);
+	float bb_min[3], bb_max[3];
 
 	if(!pbvh)
 		return 0;
 
 	BLI_pbvh_redraw_BB(pbvh, bb_min, bb_max);
 
-	rect->xmin = rect->ymin = INT_MAX;
-	rect->xmax = rect->ymax = INT_MIN;
-
-	if(bb_min[0] > bb_max[0] || bb_min[1] > bb_max[1] || bb_min[2] > bb_max[2])
+	/* convert 3D bounding box to screen space */
+	if(!paint_convert_bb_to_rect(rect,
+								 bb_min,
+								 bb_max,
+								 ar,
+								 rv3d,
+								 ob)) {
 		return 0;
-
-	for(i = 0; i < 2; ++i) {
-		for(j = 0; j < 2; ++j) {
-			for(k = 0; k < 2; ++k) {
-				float vec[3], proj[2];
-				vec[0] = i ? bb_min[0] : bb_max[0];
-				vec[1] = j ? bb_min[1] : bb_max[1];
-				vec[2] = k ? bb_min[2] : bb_max[2];
-				ED_view3d_project_float(ar, vec, proj, pmat);
-				rect->xmin = MIN2(rect->xmin, proj[0]);
-				rect->xmax = MAX2(rect->xmax, proj[0]);
-				rect->ymin = MIN2(rect->ymin, proj[1]);
-				rect->ymax = MAX2(rect->ymax, proj[1]);
-			}
-		}
 	}
-	
-	if (rect->xmin < rect->xmax && rect->ymin < rect->ymax) {
-		/* expand redraw rect with redraw rect from previous step to prevent
-		   partial-redraw issues caused by fast strokes. This is needed here (not in sculpt_flush_update)
-		   as it was before because redraw rectangle should be the same in both of
-		   optimized PBVH draw function and 3d view redraw (if not -- some mesh parts could
-		   disapper from screen (sergey) */
-		SculptSession *ss = ob->sculpt;
 
-		if (ss->cache) {
-			if (!BLI_rcti_is_empty(&ss->cache->previous_r))
-				BLI_union_rcti(rect, &ss->cache->previous_r);
-		}
-
-		return 1;
+	/* expand redraw rect with redraw rect from previous step to
+	   prevent partial-redraw issues caused by fast strokes. This is
+	   needed here (not in sculpt_flush_update) as it was before
+	   because redraw rectangle should be the same in both of
+	   optimized PBVH draw function and 3d view redraw (if not -- some
+	   mesh parts could disapper from screen (sergey) */
+	ss = ob->sculpt;
+	if(ss->cache) {
+		if(!BLI_rcti_is_empty(&ss->cache->previous_r))
+			BLI_union_rcti(rect, &ss->cache->previous_r);
 	}
 
-	return 0;
+	return 1;
 }
 
 void sculpt_get_redraw_planes(float planes[4][4], ARegion *ar,
 				  RegionView3D *rv3d, Object *ob)
 {
 	PBVH *pbvh= ob->sculpt->pbvh;
-	BoundBox bb;
-	bglMats mats;
 	rcti rect;
 
-	memset(&bb, 0, sizeof(BoundBox));
-
-	view3d_get_transformation(ar, rv3d, ob, &mats);
-	sculpt_get_redraw_rect(ar, rv3d,ob, &rect);
-
-#if 1
-	/* use some extra space just in case */
-	rect.xmin -= 2;
-	rect.xmax += 2;
-	rect.ymin -= 2;
-	rect.ymax += 2;
-#else
-	/* it was doing this before, allows to redraw a smaller
-	   part of the screen but also gives artifaces .. */
-	rect.xmin += 2;
-	rect.xmax -= 2;
-	rect.ymin += 2;
-	rect.ymax -= 2;
-#endif
+	sculpt_get_redraw_rect(ar, rv3d, ob, &rect);
 
-	ED_view3d_calc_clipping(&bb, planes, &mats, &rect);
-	mul_m4_fl(planes, -1.0f);
+	paint_calc_redraw_planes(planes, ar, rv3d, ob, &rect);
 
 	/* clear redraw flag from nodes */
 	if(pbvh)
diff --git a/source/blender/editors/space_view3d/view3d_select.c b/source/blender/editors/space_view3d/view3d_select.c
index 1be19f14618..886db3a5f0e 100644
--- a/source/blender/editors/space_view3d/view3d_select.c
+++ b/source/blender/editors/space_view3d/view3d_select.c
@@ -135,7 +135,7 @@ int view3d_get_view_aligned_coordinate(ViewContext *vc, float fp[3], const int m
 /*
  * ob == NULL if you want global matrices
  * */
-void view3d_get_transformation(ARegion *ar, RegionView3D *rv3d, Object *ob, bglMats *mats)
+void view3d_get_transformation(const ARegion *ar, RegionView3D *rv3d, Object *ob, bglMats *mats)
 {
 	float cpy[4][4];
 	int i, j;
diff --git a/source/blender/editors/space_view3d/view3d_view.c b/source/blender/editors/space_view3d/view3d_view.c
index 65ad45495db..aa444125287 100644
--- a/source/blender/editors/space_view3d/view3d_view.c
+++ b/source/blender/editors/space_view3d/view3d_view.c
@@ -512,7 +512,7 @@ void VIEW3D_OT_object_as_camera(wmOperatorType *ot)
 
 /* ********************************** */
 
-void ED_view3d_calc_clipping(BoundBox *bb, float planes[4][4], bglMats *mats, rcti *rect)
+void ED_view3d_calc_clipping(BoundBox *bb, float planes[4][4], bglMats *mats, const rcti *rect)
 {
 	float modelview[4][4];
 	double xs, ys, p[3];
@@ -760,7 +760,7 @@ void view3d_unproject(bglMats *mats, float out[3], const short x, const short y,
 #endif
 
 /* use view3d_get_object_project_mat to get projecting mat */
-void ED_view3d_project_float(ARegion *ar, const float vec[3], float adr[2], float mat[4][4])
+void ED_view3d_project_float(const ARegion *ar, const float vec[3], float adr[2], float mat[4][4])
 {
 	float vec4[4];