Add WM_framebuffer_to_index_array

Convert buffer to index in one loop,
also minor cleanup to backbuf/selection functions.

- Use IMB_rectcpy instead of inline pixel copy.
- Redundant WM_framebuffer_to_index call.

8 files changed, 145 insertions, 85 deletions

diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index be4204e7cb7..c3999d29f89 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -269,7 +269,7 @@ void drawcircball(int mode, const float cent[3], float rad, float tmat[4][4]);
 
 /* backbuffer select and draw support */
 void          ED_view3d_backbuf_validate(struct ViewContext *vc);
-struct ImBuf *ED_view3d_backbuf_read(struct ViewContext *vc, short xmin, short ymin, short xmax, short ymax);
+struct ImBuf *ED_view3d_backbuf_read(struct ViewContext *vc, int xmin, int ymin, int xmax, int ymax);
 unsigned int  ED_view3d_backbuf_sample_rect(
         struct ViewContext *vc, const int mval[2], int size,
         unsigned int min, unsigned int max, float *r_dist);
diff --git a/source/blender/editors/mesh/editface.c b/source/blender/editors/mesh/editface.c
index 35fc4483702..de37d5c0a0c 100644
--- a/source/blender/editors/mesh/editface.c
+++ b/source/blender/editors/mesh/editface.c
@@ -426,13 +426,15 @@ int do_paintface_box_select(ViewContext *vc, rcti *rect, bool select, bool exten
 	unsigned int *rt;
 	char *selar;
 	int a, index;
-	int sx = BLI_rcti_size_x(rect) + 1;
-	int sy = BLI_rcti_size_y(rect) + 1;
+	const int size[2] = {
+	    BLI_rcti_size_x(rect) + 1,
+	    BLI_rcti_size_y(rect) + 1};
 	
 	me = BKE_mesh_from_object(ob);
 
-	if (me == NULL || me->totpoly == 0 || sx * sy <= 0)
+	if ((me == NULL) || (me->totpoly == 0) || (size[0] * size[1] <= 0)) {
 		return OPERATOR_CANCELLED;
+	}
 
 	selar = MEM_callocN(me->totpoly + 1, "selar");
 
@@ -448,16 +450,21 @@ int do_paintface_box_select(ViewContext *vc, rcti *rect, bool select, bool exten
 
 	ED_view3d_backbuf_validate(vc);
 
-	ibuf = IMB_allocImBuf(sx, sy, 32, IB_rect);
+	ibuf = IMB_allocImBuf(size[0], size[1], 32, IB_rect);
 	rt = ibuf->rect;
-	view3d_opengl_read_pixels(vc->ar, rect->xmin, rect->ymin, sx, sy, GL_RGBA, GL_UNSIGNED_BYTE,  ibuf->rect);
-	if (ENDIAN_ORDER == B_ENDIAN) IMB_convert_rgba_to_abgr(ibuf);
+	view3d_opengl_read_pixels(vc->ar, rect->xmin, rect->ymin, size[0], size[1], GL_RGBA, GL_UNSIGNED_BYTE,  ibuf->rect);
+	if (ENDIAN_ORDER == B_ENDIAN) {
+		IMB_convert_rgba_to_abgr(ibuf);
+	}
+	WM_framebuffer_to_index_array(ibuf->rect, size[0] * size[1]);
 
-	a = sx * sy;
+	a = size[0] * size[1];
 	while (a--) {
 		if (*rt) {
-			index = WM_framebuffer_to_index(*rt);
-			if (index <= me->totpoly) selar[index] = 1;
+			index = *rt;
+			if (index <= me->totpoly) {
+				selar[index] = 1;
+			}
 		}
 		rt++;
 	}
diff --git a/source/blender/editors/mesh/editmesh_select.c b/source/blender/editors/mesh/editmesh_select.c
index 8c2bc154e30..0e4630f7029 100644
--- a/source/blender/editors/mesh/editmesh_select.c
+++ b/source/blender/editors/mesh/editmesh_select.c
@@ -203,8 +203,9 @@ bool EDBM_backbuf_border_init(ViewContext *vc, short xmin, short ymin, short xma
 	}
 	
 	buf = ED_view3d_backbuf_read(vc, xmin, ymin, xmax, ymax);
-	if (buf == NULL) return false;
-	if (bm_vertoffs == 0) return false;
+	if ((buf == NULL) || (bm_vertoffs == 0)) {
+		return false;
+	}
 
 	dr = buf->rect;
 	
@@ -278,8 +279,9 @@ bool EDBM_backbuf_border_mask_init(ViewContext *vc, const int mcords[][2], short
 	}
 
 	buf = ED_view3d_backbuf_read(vc, xmin, ymin, xmax, ymax);
-	if (buf == NULL) return false;
-	if (bm_vertoffs == 0) return false;
+	if ((buf == NULL) || (bm_vertoffs == 0)) {
+		return false;
+	}
 
 	dr = buf->rect;
 
@@ -329,8 +331,9 @@ bool EDBM_backbuf_circle_init(ViewContext *vc, short xs, short ys, short rads)
 	xmin = xs - rads; xmax = xs + rads;
 	ymin = ys - rads; ymax = ys + rads;
 	buf = ED_view3d_backbuf_read(vc, xmin, ymin, xmax, ymax);
-	if (bm_vertoffs == 0) return false;
-	if (buf == NULL) return false;
+	if ((buf == NULL) || (bm_vertoffs == 0)) {
+		return false;
+	}
 
 	dr = buf->rect;
 	
diff --git a/source/blender/editors/sculpt_paint/paint_vertex.c b/source/blender/editors/sculpt_paint/paint_vertex.c
index 5af327e7b49..6116903935b 100644
--- a/source/blender/editors/sculpt_paint/paint_vertex.c
+++ b/source/blender/editors/sculpt_paint/paint_vertex.c
@@ -869,7 +869,7 @@ static unsigned int vpaint_blend(VPaint *vp, unsigned int col, unsigned int colo
 }
 
 
-static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x, int y, float size)
+static int sample_backbuf_area(ViewContext *vc, int *indexar, int totpoly, int x, int y, float size)
 {
 	struct ImBuf *ibuf;
 	int a, tot = 0, index;
@@ -882,22 +882,25 @@ static int sample_backbuf_area(ViewContext *vc, int *indexar, int totface, int x
 	if (ibuf) {
 		unsigned int *rt = ibuf->rect;
 
-		memset(indexar, 0, sizeof(int) * (totface + 1));
+		memset(indexar, 0, sizeof(int) * (totpoly + 1));
 		
 		size = ibuf->x * ibuf->y;
 		while (size--) {
 				
 			if (*rt) {
-				index = WM_framebuffer_to_index(*rt);
-				if (index > 0 && index <= totface)
+				index = *rt;
+				if (index > 0 && index <= totpoly) {
 					indexar[index] = 1;
+				}
 			}
 		
 			rt++;
 		}
 		
-		for (a = 1; a <= totface; a++) {
-			if (indexar[a]) indexar[tot++] = a;
+		for (a = 1; a <= totpoly; a++) {
+			if (indexar[a]) {
+				indexar[tot++] = a;
+			}
 		}
 
 		IMB_freeImBuf(ibuf);
diff --git a/source/blender/editors/space_view3d/view3d_draw.c b/source/blender/editors/space_view3d/view3d_draw.c
index 33c883822a0..45d5b2f88a3 100644
--- a/source/blender/editors/space_view3d/view3d_draw.c
+++ b/source/blender/editors/space_view3d/view3d_draw.c
@@ -1492,62 +1492,61 @@ unsigned int ED_view3d_backbuf_sample(ViewContext *vc, int x, int y)
 }
 
 /* reads full rect, converts indices */
-ImBuf *ED_view3d_backbuf_read(ViewContext *vc, short xmin, short ymin, short xmax, short ymax)
+ImBuf *ED_view3d_backbuf_read(ViewContext *vc, int xmin, int ymin, int xmax, int ymax)
 {
-	unsigned int *dr, *rd;
-	struct ImBuf *ibuf, *ibuf1;
-	int a;
-	short xminc, yminc, xmaxc, ymaxc, xs, ys;
-	
+	struct ImBuf *ibuf_clip;
 	/* clip */
-	xminc = max_ii(xmin, 0);
-	yminc = max_ii(ymin, 0);
-	xmaxc = min_ii(xmax, vc->ar->winx - 1);
-	ymaxc = min_ii(ymax, vc->ar->winy - 1);
-
-	if (UNLIKELY((xminc > xmaxc) || (yminc > ymaxc))) {
+	const rcti clip = {
+	    max_ii(xmin, 0), min_ii(xmax, vc->ar->winx - 1),
+	    max_ii(ymin, 0), min_ii(ymax, vc->ar->winy - 1)};
+	const int size_clip[2] = {
+	    BLI_rcti_size_x(&clip) + 1,
+	    BLI_rcti_size_y(&clip) + 1};
+
+	if (UNLIKELY((clip.xmin > clip.xmax) ||
+	             (clip.ymin > clip.ymax)))
+	{
 		return NULL;
 	}
 
-	ibuf = IMB_allocImBuf((xmaxc - xminc + 1), (ymaxc - yminc + 1), 32, IB_rect);
+	ibuf_clip = IMB_allocImBuf(size_clip[0], size_clip[1], 32, IB_rect);
 
 	ED_view3d_backbuf_validate(vc);
 
-	view3d_opengl_read_pixels(vc->ar,
-	             xminc, yminc,
-	             (xmaxc - xminc + 1),
-	             (ymaxc - yminc + 1),
-	             GL_RGBA, GL_UNSIGNED_BYTE, ibuf->rect);
+	view3d_opengl_read_pixels(vc->ar, clip.xmin, clip.ymin, size_clip[0], size_clip[1], GL_RGBA, GL_UNSIGNED_BYTE, ibuf_clip->rect);
 
 	glReadBuffer(GL_BACK);
 
-	if (ENDIAN_ORDER == B_ENDIAN) IMB_convert_rgba_to_abgr(ibuf);
-
-	a = (xmaxc - xminc + 1) * (ymaxc - yminc + 1);
-	dr = ibuf->rect;
-	while (a--) {
-		if (*dr) *dr = WM_framebuffer_to_index(*dr);
-		dr++;
+	if (ENDIAN_ORDER == B_ENDIAN) {
+		IMB_convert_rgba_to_abgr(ibuf_clip);
 	}
-	
-	/* put clipped result back, if needed */
-	if (xminc == xmin && xmaxc == xmax && yminc == ymin && ymaxc == ymax)
-		return ibuf;
-	
-	ibuf1 = IMB_allocImBuf( (xmax - xmin + 1), (ymax - ymin + 1), 32, IB_rect);
-	rd = ibuf->rect;
-	dr = ibuf1->rect;
 
-	for (ys = ymin; ys <= ymax; ys++) {
-		for (xs = xmin; xs <= xmax; xs++, dr++) {
-			if (xs >= xminc && xs <= xmaxc && ys >= yminc && ys <= ymaxc) {
-				*dr = *rd;
-				rd++;
-			}
-		}
+	WM_framebuffer_to_index_array(ibuf_clip->rect, size_clip[0] * size_clip[1]);
+	
+	if ((clip.xmin == xmin) &&
+	    (clip.xmax == xmax) &&
+	    (clip.ymin == ymin) &&
+	    (clip.ymax == ymax))
+	{
+		return ibuf_clip;
+	}
+	else {
+		/* put clipped result into a non-clipped buffer */
+		struct ImBuf *ibuf_full;
+		const int size[2] = {
+		    (xmax - xmin + 1),
+		    (ymax - ymin + 1)};
+
+		ibuf_full = IMB_allocImBuf(size[0], size[1], 32, IB_rect);
+
+		IMB_rectcpy(
+		        ibuf_full, ibuf_clip,
+		        clip.xmin - xmin, clip.ymin - ymin,
+		        0, 0,
+		        size_clip[0], size_clip[1]);
+		IMB_freeImBuf(ibuf_clip);
+		return ibuf_full;
 	}
-	IMB_freeImBuf(ibuf);
-	return ibuf1;
 }
 
 /* smart function to sample a rect spiralling outside, nice for backbuf selection */
diff --git a/source/blender/editors/space_view3d/view3d_select.c b/source/blender/editors/space_view3d/view3d_select.c
index 49e42cf164a..5a35e9fcad1 100644
--- a/source/blender/editors/space_view3d/view3d_select.c
+++ b/source/blender/editors/space_view3d/view3d_select.c
@@ -1648,14 +1648,15 @@ static int do_paintvert_box_select(ViewContext *vc, rcti *rect, bool select, boo
 	unsigned int *rt;
 	int a, index;
 	char *selar;
-	int sx = BLI_rcti_size_x(rect) + 1;
-	int sy = BLI_rcti_size_y(rect) + 1;
+	const int size[2] = {
+	    BLI_rcti_size_x(rect) + 1,
+	    BLI_rcti_size_y(rect) + 1};
 
 	me = vc->obact->data;
 
-	if (me == NULL || me->totvert == 0 || sx * sy <= 0)
+	if ((me == NULL) || (me->totvert == 0) || (size[0] * size[1] <= 0)) {
 		return OPERATOR_CANCELLED;
-
+	}
 
 	if (extend == false && select)
 		paintvert_deselect_all_visible(vc->obact, SEL_DESELECT, false);
@@ -1664,16 +1665,24 @@ static int do_paintvert_box_select(ViewContext *vc, rcti *rect, bool select, boo
 		selar = MEM_callocN(me->totvert + 1, "selar");
 		ED_view3d_backbuf_validate(vc);
 
-		ibuf = IMB_allocImBuf(sx, sy, 32, IB_rect);
+		ibuf = IMB_allocImBuf(size[0], size[1], 32, IB_rect);
 		rt = ibuf->rect;
-		glReadPixels(rect->xmin + vc->ar->winrct.xmin,  rect->ymin + vc->ar->winrct.ymin, sx, sy, GL_RGBA, GL_UNSIGNED_BYTE,  ibuf->rect);
-		if (ENDIAN_ORDER == B_ENDIAN) IMB_convert_rgba_to_abgr(ibuf);
+		glReadPixels(
+		        rect->xmin + vc->ar->winrct.xmin,
+		        rect->ymin + vc->ar->winrct.ymin,
+		        size[0], size[1], GL_RGBA, GL_UNSIGNED_BYTE,  ibuf->rect);
+		if (ENDIAN_ORDER == B_ENDIAN) {
+			IMB_convert_rgba_to_abgr(ibuf);
+		}
+		WM_framebuffer_to_index_array(ibuf->rect, size[0] * size[1]);
 
-		a = sx * sy;
+		a = size[0] * size[1];
 		while (a--) {
 			if (*rt) {
-				index = WM_framebuffer_to_index(*rt);
-				if (index <= me->totvert) selar[index] = 1;
+				index = *rt;
+				if (index <= me->totvert) {
+					selar[index] = 1;
+				}
 			}
 			rt++;
 		}
diff --git a/source/blender/windowmanager/WM_api.h b/source/blender/windowmanager/WM_api.h
index 9e27190de02..fa9357f4eec 100644
--- a/source/blender/windowmanager/WM_api.h
+++ b/source/blender/windowmanager/WM_api.h
@@ -393,6 +393,7 @@ void		wmOrtho2_pixelspace(const float x, const float y);
 			/* utilities */
 void		WM_framebuffer_index_set(int index);
 int			WM_framebuffer_to_index(unsigned int col);
+void		WM_framebuffer_to_index_array(unsigned int *col, const unsigned int size);
 
 			/* threaded Jobs Manager */
 enum {
diff --git a/source/blender/windowmanager/intern/wm_subwindow.c b/source/blender/windowmanager/intern/wm_subwindow.c
index d081644fa61..9209fa49f74 100644
--- a/source/blender/windowmanager/intern/wm_subwindow.c
+++ b/source/blender/windowmanager/intern/wm_subwindow.c
@@ -461,25 +461,63 @@ void WM_framebuffer_index_set(int index)
 	cpack(col);
 }
 
+#define INDEX_FROM_BUF_8(col)     (((col & 0xC00000) >> 18) + ((col & 0xC000) >> 12) + ((col & 0xC0) >> 6))
+#define INDEX_FROM_BUF_12(col)    (((col & 0xF00000) >> 12) + ((col & 0xF000) >> 8)  + ((col & 0xF0) >> 4))
+#define INDEX_FROM_BUF_15_16(col) (((col & 0xF80000) >> 9)  + ((col & 0xF800) >> 6)  + ((col & 0xF8) >> 3))
+#define INDEX_FROM_BUF_18(col)    (((col & 0xFC0000) >> 6)  + ((col & 0xFC00) >> 4)  + ((col & 0xFC) >> 2))
+#define INDEX_FROM_BUF_24(col)      (col & 0xFFFFFF)
+
 int WM_framebuffer_to_index(unsigned int col)
 {
-	if (col == 0) return 0;
+	if (col == 0) {
+		return 0;
+	}
 
 	switch (GPU_color_depth()) {
-		case 8:
-			return ((col & 0xC00000) >> 18) + ((col & 0xC000) >> 12) + ((col & 0xC0) >> 6);
-		case 12:
-			return ((col & 0xF00000) >> 12) + ((col & 0xF000) >> 8) + ((col & 0xF0) >> 4);
+		case  8: return INDEX_FROM_BUF_8(col);
+		case 12: return INDEX_FROM_BUF_12(col);
 		case 15:
-		case 16:
-			return ((col & 0xF80000) >> 9) + ((col & 0xF800) >> 6) + ((col & 0xF8) >> 3);
-		case 24:
-			return col & 0xFFFFFF;
-		default: // 18 bits...
-			return ((col & 0xFC0000) >> 6) + ((col & 0xFC00) >> 4) + ((col & 0xFC) >> 2);
+		case 16: return INDEX_FROM_BUF_15_16(col);
+		case 24: return INDEX_FROM_BUF_24(col);
+		default: return INDEX_FROM_BUF_18(col);
 	}
 }
 
+void WM_framebuffer_to_index_array(unsigned int *col, const unsigned int size)
+{
+#define INDEX_BUF_ARRAY(INDEX_FROM_BUF_BITS) \
+	for (i = size; i--; col++) { \
+		if ((c = *col)) { \
+			*col = INDEX_FROM_BUF_BITS(c); \
+		} \
+	} ((void)0)
+
+	if (size > 0) {
+		unsigned int i, c;
+
+		switch (GPU_color_depth()) {
+			case  8:
+				INDEX_BUF_ARRAY(INDEX_FROM_BUF_8);
+				break;
+			case 12:
+				INDEX_BUF_ARRAY(INDEX_FROM_BUF_12);
+				break;
+			case 15:
+			case 16:
+				INDEX_BUF_ARRAY(INDEX_FROM_BUF_15_16);
+				break;
+			case 24:
+				INDEX_BUF_ARRAY(INDEX_FROM_BUF_24);
+				break;
+			default:
+				INDEX_BUF_ARRAY(INDEX_FROM_BUF_18);
+				break;
+		}
+	}
+
+#undef INDEX_BUF_ARRAY
+}
+
 
 /* ********** END MY WINDOW ************** */