Fix: multisample viewport drawing didn't work well with selection or particle

brushes, due to issues with color coded drawing or slow/buggy reading from such
a buffer on some systems.

In case multisample is enabled now, it uses an offscreen buffer for such drawing,
which is not multisampled and so should not cause issues. This does mean there is
some extra GPU memory usage when multisample is enabled, and we could optimize
triple buffer to work together here somehow to share buffers, but it's better than
having selection not working.

10 files changed, 80 insertions, 36 deletions

diff --git a/source/blender/blenloader/intern/readfile.c b/source/blender/blenloader/intern/readfile.c
index cef53b63397..335e80eac15 100644
--- a/source/blender/blenloader/intern/readfile.c
+++ b/source/blender/blenloader/intern/readfile.c
@@ -5917,6 +5917,7 @@ static void direct_link_region(FileData *fd, ARegion *ar, int spacetype)
 			rv3d->clipbb = newdataadr(fd, rv3d->clipbb);
 			
 			rv3d->depths = NULL;
+			rv3d->gpuoffscreen = NULL;
 			rv3d->ri = NULL;
 			rv3d->render_engine = NULL;
 			rv3d->sms = NULL;
diff --git a/source/blender/editors/include/ED_view3d.h b/source/blender/editors/include/ED_view3d.h
index c2eb7f42bb0..470bbe616cf 100644
--- a/source/blender/editors/include/ED_view3d.h
+++ b/source/blender/editors/include/ED_view3d.h
@@ -260,6 +260,7 @@ void view3d_set_viewcontext(struct bContext *C, struct ViewContext *vc);
 void view3d_operator_needs_opengl(const struct bContext *C);
 void view3d_region_operator_needs_opengl(struct wmWindow *win, struct ARegion *ar);
 bool view3d_get_view_aligned_coordinate(struct ARegion *ar, float fp[3], const int mval[2], const bool do_fallback);
+void view3d_opengl_read_pixels(struct ARegion *ar, int x, int y, int w, int h, int format, int type, void *data);
 void view3d_get_transformation(const struct ARegion *ar, struct RegionView3D *rv3d, struct Object *ob, struct bglMats *mats);
 
 /* XXX should move to BLI_math */
diff --git a/source/blender/editors/mesh/editface.c b/source/blender/editors/mesh/editface.c
index 9b26959ea28..ec146380111 100644
--- a/source/blender/editors/mesh/editface.c
+++ b/source/blender/editors/mesh/editface.c
@@ -570,7 +570,7 @@ int do_paintface_box_select(ViewContext *vc, rcti *rect, int select, int extend)
 
 	ibuf = IMB_allocImBuf(sx, sy, 32, IB_rect);
 	rt = ibuf->rect;
-	glReadPixels(rect->xmin + vc->ar->winrct.xmin,  rect->ymin + vc->ar->winrct.ymin, sx, sy, GL_RGBA, GL_UNSIGNED_BYTE,  ibuf->rect);
+	view3d_opengl_read_pixels(vc->ar, rect->xmin, rect->ymin, sx, sy, GL_RGBA, GL_UNSIGNED_BYTE,  ibuf->rect);
 	if (ENDIAN_ORDER == B_ENDIAN) IMB_convert_rgba_to_abgr(ibuf);
 
 	a = sx * sy;
diff --git a/source/blender/editors/physics/particle_edit.c b/source/blender/editors/physics/particle_edit.c
index 6c80f9377df..329f1f67c4a 100644
--- a/source/blender/editors/physics/particle_edit.c
+++ b/source/blender/editors/physics/particle_edit.c
@@ -430,15 +430,6 @@ static int key_test_depth(PEData *data, const float co[3], const int screen_co[2
 	gluProject(co[0], co[1], co[2], data->mats.modelview, data->mats.projection,
 	           (GLint *)data->mats.viewport, &ux, &uy, &uz);
 
-#if 0 /* works well but too slow on some systems [#23118] */
-	screen_co[0] += (short)data->vc.ar->winrct.xmin;
-	screen_co[1] += (short)data->vc.ar->winrct.ymin;
-
-	/* PE_set_view3d_data calls this. no need to call here */
-	/* view3d_validate_backbuf(&data->vc); */
-	glReadPixels(screen_co[0], screen_co[1], 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &depth);
-#else /* faster to use depths, these are calculated in PE_set_view3d_data */
-
 	/* check if screen_co is within bounds because brush_cut uses out of screen coords */
 	if (screen_co[0] >= 0 && screen_co[0] < vd->w && screen_co[1] >= 0 && screen_co[1] < vd->h) {
 		BLI_assert(vd && vd->depths);
@@ -447,7 +438,6 @@ static int key_test_depth(PEData *data, const float co[3], const int screen_co[2
 	}
 	else
 		return 0;
-#endif
 
 	if ((float)uz - 0.00001f > depth)
 		return 0;
diff --git a/source/blender/editors/space_view3d/drawobject.c b/source/blender/editors/space_view3d/drawobject.c
index fa09b93ee4d..da3ebf586e9 100644
--- a/source/blender/editors/space_view3d/drawobject.c
+++ b/source/blender/editors/space_view3d/drawobject.c
@@ -808,17 +808,6 @@ void view3d_cached_text_draw_end(View3D *v3d, ARegion *ar, int depth_write, floa
 		}
 		
 		for (vos = strings->first; vos; vos = vos->next) {
-			/* too slow, reading opengl info while drawing is very bad,
-			 * better to see if we can use the zbuffer while in pixel space - campbell */
-#if 0
-			if (v3d->zbuf && (vos->flag & V3D_CACHE_TEXT_ZBUF)) {
-				gluProject(vos->vec[0], vos->vec[1], vos->vec[2], mats.modelview, mats.projection, (GLint *)mats.viewport, &ux, &uy, &uz);
-				glReadPixels(ar->winrct.xmin + vos->mval[0] + vos->xoffs, ar->winrct.ymin + vos->mval[1], 1, 1, GL_DEPTH_COMPONENT, GL_FLOAT, &depth);
-
-				if (uz > depth)
-					continue;
-			}
-#endif
 			if (vos->sco[0] != IS_CLIPPED) {
 				const char *str = (char *)(vos + 1);
 
diff --git a/source/blender/editors/space_view3d/space_view3d.c b/source/blender/editors/space_view3d/space_view3d.c
index a3f5055c284..2eba28858bc 100644
--- a/source/blender/editors/space_view3d/space_view3d.c
+++ b/source/blender/editors/space_view3d/space_view3d.c
@@ -53,6 +53,7 @@
 #include "ED_screen.h"
 #include "ED_object.h"
 
+#include "GPU_extensions.h"
 #include "GPU_material.h"
 
 #include "BIF_gl.h"
@@ -478,6 +479,11 @@ static void view3d_main_area_exit(wmWindowManager *wm, ARegion *ar)
 		RE_engine_free(rv3d->render_engine);
 		rv3d->render_engine = NULL;
 	}
+
+	if (rv3d->gpuoffscreen) {
+		GPU_offscreen_free(rv3d->gpuoffscreen);
+		rv3d->gpuoffscreen = NULL;
+	}
 }
 
 static int view3d_ob_drop_poll(bContext *UNUSED(C), wmDrag *drag, const wmEvent *UNUSED(event))
@@ -626,6 +632,10 @@ static void view3d_main_area_free(ARegion *ar)
 		if (rv3d->sms) {
 			MEM_freeN(rv3d->sms);
 		}
+		if (rv3d->gpuoffscreen) {
+			GPU_offscreen_free(rv3d->gpuoffscreen);
+		}
+
 		MEM_freeN(rv3d);
 		ar->regiondata = NULL;
 	}
@@ -644,6 +654,7 @@ static void *view3d_main_area_duplicate(void *poin)
 			new->clipbb = MEM_dupallocN(rv3d->clipbb);
 		
 		new->depths = NULL;
+		new->gpuoffscreen = NULL;
 		new->ri = NULL;
 		new->render_engine = NULL;
 		new->gpd = NULL;
diff --git a/source/blender/editors/space_view3d/view3d_draw.c b/source/blender/editors/space_view3d/view3d_draw.c
index 5b1e1e75f85..252a27947fb 100644
--- a/source/blender/editors/space_view3d/view3d_draw.c
+++ b/source/blender/editors/space_view3d/view3d_draw.c
@@ -1348,7 +1348,34 @@ static void backdrawview3d(Scene *scene, ARegion *ar, View3D *v3d)
 	if (multisample_enabled)
 		glDisable(GL_MULTISAMPLE_ARB);
 
-	glScissor(ar->winrct.xmin, ar->winrct.ymin, BLI_rcti_size_x(&ar->winrct), BLI_rcti_size_y(&ar->winrct));
+	if (U.ogl_multisamples != USER_MULTISAMPLE_NONE) {
+		/* for multisample we use an offscreen FBO. multisample drawing can fail
+		 * with color coded selection drawing, and reading back depths from such
+		 * a buffer can also cause a few seconds freeze on OS X / NVidia. */
+		int w = BLI_rcti_size_x(&ar->winrct);
+		int h = BLI_rcti_size_y(&ar->winrct);
+		char error[256];
+
+		if (rv3d->gpuoffscreen) {
+			if (GPU_offscreen_width(rv3d->gpuoffscreen) != w ||
+			    GPU_offscreen_height(rv3d->gpuoffscreen) != h) {
+				GPU_offscreen_free(rv3d->gpuoffscreen);
+				rv3d->gpuoffscreen = NULL;
+			}
+		}
+
+		if (!rv3d->gpuoffscreen) {
+			rv3d->gpuoffscreen = GPU_offscreen_create(w, h, error);
+
+			if (!rv3d->gpuoffscreen)
+				fprintf(stderr, "Failed to create offscreen selection buffer for multisample: %s\n", error);
+		}
+	}
+
+	if (rv3d->gpuoffscreen)
+		GPU_offscreen_bind(rv3d->gpuoffscreen);
+	else
+		glScissor(ar->winrct.xmin, ar->winrct.ymin, BLI_rcti_size_x(&ar->winrct), BLI_rcti_size_y(&ar->winrct));
 
 	glClearColor(0.0, 0.0, 0.0, 0.0); 
 	if (v3d->zbuf) {
@@ -1367,9 +1394,13 @@ static void backdrawview3d(Scene *scene, ARegion *ar, View3D *v3d)
 	
 	if (base && (base->lay & v3d->lay))
 		draw_object_backbufsel(scene, v3d, rv3d, base->object);
+	
+	if (rv3d->gpuoffscreen)
+		GPU_offscreen_unbind(rv3d->gpuoffscreen);
+	else
+		ar->swap = 0; /* mark invalid backbuf for wm draw */
 
 	v3d->flag &= ~V3D_INVALID_BACKBUF;
-	ar->swap = 0; /* mark invalid backbuf for wm draw */
 
 	G.f &= ~G_BACKBUFSEL;
 	v3d->zbuf = FALSE;
@@ -1386,6 +1417,21 @@ static void backdrawview3d(Scene *scene, ARegion *ar, View3D *v3d)
 
 }
 
+void view3d_opengl_read_pixels(ARegion *ar, int x, int y, int w, int h, int format, int type, void *data)
+{
+	RegionView3D *rv3d = ar->regiondata;
+
+	if (rv3d->gpuoffscreen) {
+		GPU_offscreen_bind(rv3d->gpuoffscreen);
+		glReadBuffer(GL_COLOR_ATTACHMENT0_EXT);
+		glReadPixels(x, y, w, h, format, type, data);
+		GPU_offscreen_unbind(rv3d->gpuoffscreen);
+	}
+	else {
+		glReadPixels(ar->winrct.xmin + x, ar->winrct.ymin + y, w, h, format, type, data);
+	}
+}
+
 void view3d_validate_backbuf(ViewContext *vc)
 {
 	if (vc->v3d->flag & V3D_INVALID_BACKBUF)
@@ -1401,12 +1447,9 @@ unsigned int view3d_sample_backbuf(ViewContext *vc, int x, int y)
 		return 0;
 	}
 
-	x += vc->ar->winrct.xmin;
-	y += vc->ar->winrct.ymin;
-	
 	view3d_validate_backbuf(vc);
 
-	glReadPixels(x, y, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, &col);
+	view3d_opengl_read_pixels(vc->ar, x, y, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, &col);
 	glReadBuffer(GL_BACK);
 	
 	if (ENDIAN_ORDER == B_ENDIAN) {
@@ -1437,8 +1480,8 @@ ImBuf *view3d_read_backbuf(ViewContext *vc, short xmin, short ymin, short xmax,
 
 	view3d_validate_backbuf(vc);
 
-	glReadPixels(vc->ar->winrct.xmin + xminc,
-	             vc->ar->winrct.ymin + yminc,
+	view3d_opengl_read_pixels(vc->ar,
+	             xminc, yminc,
 	             (xmaxc - xminc + 1),
 	             (ymaxc - yminc + 1),
 	             GL_RGBA, GL_UNSIGNED_BYTE, ibuf->rect);
@@ -2104,7 +2147,7 @@ void view3d_update_depths_rect(ARegion *ar, ViewDepths *d, rcti *rect)
 	}
 
 	if (d->damaged) {
-		glReadPixels(ar->winrct.xmin + d->x, ar->winrct.ymin + d->y, d->w, d->h, GL_DEPTH_COMPONENT, GL_FLOAT, d->depths);
+		view3d_opengl_read_pixels(ar, d->x, d->y, d->w, d->h, GL_DEPTH_COMPONENT, GL_FLOAT, d->depths);
 		glGetDoublev(GL_DEPTH_RANGE, d->depth_range);
 		d->damaged = FALSE;
 	}
@@ -2132,9 +2175,7 @@ void ED_view3d_depth_update(ARegion *ar)
 		}
 		
 		if (d->damaged) {
-			glReadPixels(ar->winrct.xmin, ar->winrct.ymin, d->w, d->h,
-			             GL_DEPTH_COMPONENT, GL_FLOAT, d->depths);
-			
+			view3d_opengl_read_pixels(ar, 0, 0, d->w, d->h, GL_DEPTH_COMPONENT, GL_FLOAT, d->depths);
 			glGetDoublev(GL_DEPTH_RANGE, d->depth_range);
 			
 			d->damaged = 0;
diff --git a/source/blender/gpu/GPU_extensions.h b/source/blender/gpu/GPU_extensions.h
index 07269d3e2b5..0ae45775473 100644
--- a/source/blender/gpu/GPU_extensions.h
+++ b/source/blender/gpu/GPU_extensions.h
@@ -157,6 +157,8 @@ void GPU_offscreen_free(GPUOffScreen *ofs);
 void GPU_offscreen_bind(GPUOffScreen *ofs);
 void GPU_offscreen_unbind(GPUOffScreen *ofs);
 void GPU_offscreen_read_pixels(GPUOffScreen *ofs, int type, void *pixels);
+int GPU_offscreen_width(GPUOffScreen *ofs);
+int GPU_offscreen_height(GPUOffScreen *ofs);
 
 /* GPU Shader
  * - only for fragment shaders now
diff --git a/source/blender/gpu/intern/gpu_extensions.c b/source/blender/gpu/intern/gpu_extensions.c
index f5eaa716dfa..c7a421a49fc 100644
--- a/source/blender/gpu/intern/gpu_extensions.c
+++ b/source/blender/gpu/intern/gpu_extensions.c
@@ -905,10 +905,8 @@ void GPU_framebuffer_texture_bind(GPUFrameBuffer *UNUSED(fb), GPUTexture *tex, i
 
 	glMatrixMode(GL_PROJECTION);
 	glPushMatrix();
-	glLoadIdentity();
 	glMatrixMode(GL_MODELVIEW);
 	glPushMatrix();
-	glLoadIdentity();
 }
 
 void GPU_framebuffer_texture_unbind(GPUFrameBuffer *UNUSED(fb), GPUTexture *UNUSED(tex))
@@ -1096,6 +1094,16 @@ void GPU_offscreen_read_pixels(GPUOffScreen *ofs, int type, void *pixels)
 	glReadPixels(0, 0, ofs->w, ofs->h, GL_RGBA, type, pixels);
 }
 
+int GPU_offscreen_width(GPUOffScreen *ofs)
+{
+	return ofs->w;
+}
+
+int GPU_offscreen_height(GPUOffScreen *ofs)
+{
+	return ofs->h;
+}
+
 /* GPUShader */
 
 struct GPUShader {
diff --git a/source/blender/makesdna/DNA_view3d_types.h b/source/blender/makesdna/DNA_view3d_types.h
index 5397b06dc15..294c47aa9db 100644
--- a/source/blender/makesdna/DNA_view3d_types.h
+++ b/source/blender/makesdna/DNA_view3d_types.h
@@ -107,6 +107,7 @@ typedef struct RegionView3D {
 	struct RenderInfo *ri;
 	struct RenderEngine *render_engine;
 	struct ViewDepths *depths;
+	void *gpuoffscreen;
 
 	/* animated smooth view */
 	struct SmoothView3DStore *sms;