Fix #35960, #36044: blender internal viewport rendering crash while editing data.

Now the viewport rendering thread will lock the main thread while it is exporting
objects to render data. This is not ideal if you have big scenes that might block
the UI, but Cycles does the same, and it's fairly quick because the same evaluated
mesh can be used as for viewport drawing. It's the only way to get things stable
until the thread safe dependency graph is here.

This adds a mechanism to the job system for jobs to lock the main thread, using a
new 'ticket mutex lock' which is a mutex lock that gives priority to the first
thread that tries to lock the mutex.

Still to solve: undo/redo crashes.

8 files changed, 146 insertions, 18 deletions

diff --git a/source/blender/blenlib/BLI_threads.h b/source/blender/blenlib/BLI_threads.h
index 331cac3ed76..154986936a2 100644
--- a/source/blender/blenlib/BLI_threads.h
+++ b/source/blender/blenlib/BLI_threads.h
@@ -131,6 +131,18 @@ void BLI_rw_mutex_free(ThreadRWMutex *mutex);
 void BLI_rw_mutex_lock(ThreadRWMutex *mutex, int mode);
 void BLI_rw_mutex_unlock(ThreadRWMutex *mutex);
 
+/* Ticket Mutex Lock
+ *
+ * This is a 'fair' mutex in that it will grant the lock to the first thread
+ * that requests it. */
+
+typedef struct TicketMutex TicketMutex;
+
+TicketMutex *BLI_ticket_mutex_alloc(void);
+void BLI_ticket_mutex_free(TicketMutex *ticket);
+void BLI_ticket_mutex_lock(TicketMutex *ticket);
+void BLI_ticket_mutex_unlock(TicketMutex *ticket);
+
 /* ThreadedWorker
  *
  * A simple tool for dispatching work to a limited number of threads
diff --git a/source/blender/blenlib/intern/threads.c b/source/blender/blenlib/intern/threads.c
index e0ea3bbf685..2b6fb52c49c 100644
--- a/source/blender/blenlib/intern/threads.c
+++ b/source/blender/blenlib/intern/threads.c
@@ -508,6 +508,52 @@ void BLI_rw_mutex_free(ThreadRWMutex *mutex)
 	MEM_freeN(mutex);
 }
 
+/* Ticket Mutex Lock */
+
+struct TicketMutex {
+	pthread_cond_t cond;
+	pthread_mutex_t mutex;
+	unsigned int queue_head, queue_tail;
+};
+
+TicketMutex *BLI_ticket_mutex_alloc(void)
+{
+	TicketMutex *ticket = MEM_callocN(sizeof(TicketMutex), "TicketMutex");
+
+	pthread_cond_init(&ticket->cond, NULL);
+	pthread_mutex_init(&ticket->mutex, NULL);
+
+	return ticket;
+}
+
+void BLI_ticket_mutex_free(TicketMutex *ticket)
+{
+	pthread_mutex_destroy(&ticket->mutex);
+	pthread_cond_destroy(&ticket->cond);
+	MEM_freeN(ticket);
+}
+
+void BLI_ticket_mutex_lock(TicketMutex *ticket)
+{
+	unsigned int queue_me;
+
+	pthread_mutex_lock(&ticket->mutex);
+	queue_me = ticket->queue_tail++;
+
+	while (queue_me != ticket->queue_head)
+		pthread_cond_wait(&ticket->cond, &ticket->mutex);
+
+	pthread_mutex_unlock(&ticket->mutex);
+}
+
+void BLI_ticket_mutex_unlock(TicketMutex *ticket)
+{
+	pthread_mutex_lock(&ticket->mutex);
+	ticket->queue_head++;
+	pthread_cond_broadcast(&ticket->cond);
+	pthread_mutex_unlock(&ticket->mutex);
+}
+
 /* ************************************************ */
 
 typedef struct ThreadedWorker {
diff --git a/source/blender/editors/render/render_internal.c b/source/blender/editors/render/render_internal.c
index 09138a5523a..03ccb2496a1 100644
--- a/source/blender/editors/render/render_internal.c
+++ b/source/blender/editors/render/render_internal.c
@@ -733,6 +733,7 @@ typedef struct RenderPreview {
 	/* from wmJob */
 	void *owner;
 	short *stop, *do_update;
+	wmJob *job;
 	
 	Scene *scene;
 	ScrArea *sa;
@@ -913,8 +914,15 @@ static void render_view3d_startjob(void *customdata, short *stop, short *do_upda
 		else lay = rp->v3d->lay;
 		
 		RE_SetView(re, rp->viewmat);
-		
+
+		/* copying blender data while main thread is locked, to avoid crashes */
+		WM_job_main_thread_lock_acquire(rp->job);
 		RE_Database_FromScene(re, rp->bmain, rp->scene, lay, 0);		// 0= dont use camera view
+		WM_job_main_thread_lock_release(rp->job);
+
+		/* do preprocessing like building raytree, shadows, volumes, SSS */
+		RE_Database_Preprocess(re);
+
 		// printf("dbase update\n");
 	}
 	else {
@@ -958,7 +966,8 @@ static void render_view3d_do(RenderEngine *engine, const bContext *C, int keep_d
 	wm_job = WM_jobs_get(CTX_wm_manager(C), CTX_wm_window(C), CTX_wm_region(C), "Render Preview",
 	                     WM_JOB_EXCL_RENDER, WM_JOB_TYPE_RENDER_PREVIEW);
 	rp = MEM_callocN(sizeof(RenderPreview), "render preview");
-	
+	rp->job = wm_job;
+
 	/* customdata for preview thread */
 	rp->scene = scene;
 	rp->engine = engine;
diff --git a/source/blender/render/extern/include/RE_pipeline.h b/source/blender/render/extern/include/RE_pipeline.h
index da53bc5a819..e154fd42119 100644
--- a/source/blender/render/extern/include/RE_pipeline.h
+++ b/source/blender/render/extern/include/RE_pipeline.h
@@ -207,6 +207,7 @@ void RE_GetViewPlane(struct Render *re, rctf *viewplane, rcti *disprect);
 
 /* make or free the dbase */
 void RE_Database_FromScene(struct Render *re, struct Main *bmain, struct Scene *scene, unsigned int lay, int use_camera_view);
+void RE_Database_Preprocess(struct Render *re);
 void RE_Database_Free(struct Render *re);
 
 /* project dbase again, when viewplane/perspective changed */
diff --git a/source/blender/render/intern/source/convertblender.c b/source/blender/render/intern/source/convertblender.c
index 6e70b4bcfc9..bd2d292f633 100644
--- a/source/blender/render/intern/source/convertblender.c
+++ b/source/blender/render/intern/source/convertblender.c
@@ -5321,14 +5321,10 @@ void RE_Database_FromScene(Render *re, Main *bmain, Scene *scene, unsigned int l
 	database_init_objects(re, lay, 0, 0, 0, 0);
 	
 	if (!re->test_break(re->tbh)) {
-		int tothalo;
-
 		set_material_lightgroups(re);
 		for (sce= re->scene; sce; sce= sce->set)
 			set_renderlayer_lightgroups(re, sce);
 		
-		slurph_opt= 1;
-		
 		/* for now some clumsy copying still */
 		re->i.totvert= re->totvert;
 		re->i.totface= re->totvlak;
@@ -5336,7 +5332,16 @@ void RE_Database_FromScene(Render *re, Main *bmain, Scene *scene, unsigned int l
 		re->i.tothalo= re->tothalo;
 		re->i.totlamp= re->totlamp;
 		re->stats_draw(re->sdh, &re->i);
-		
+	}
+
+	slurph_opt= 1;
+}
+
+void RE_Database_Preprocess(Render *re)
+{
+	if (!re->test_break(re->tbh)) {
+		int tothalo;
+
 		/* don't sort stars */
 		tothalo= re->tothalo;
 		if (!re->test_break(re->tbh)) {
@@ -5392,7 +5397,6 @@ void RE_Database_FromScene(Render *re, Main *bmain, Scene *scene, unsigned int l
 		if (!re->test_break(re->tbh))
 			if (re->r.mode & R_RAYTRACE)
 				volume_precache(re);
-		
 	}
 	
 	if (re->test_break(re->tbh))
@@ -5866,8 +5870,10 @@ void RE_Database_FromScene_Vectors(Render *re, Main *bmain, Scene *sce, unsigned
 	RE_Database_Free(re);
 	re->strandsurface= strandsurface;
 	
-	if (!re->test_break(re->tbh))
+	if (!re->test_break(re->tbh)) {
 		RE_Database_FromScene(re, bmain, sce, lay, 1);
+		RE_Database_Preprocess(re);
+	}
 	
 	if (!re->test_break(re->tbh)) {
 		int vectorlay= get_vector_renderlayers(re->scene);
diff --git a/source/blender/render/intern/source/pipeline.c b/source/blender/render/intern/source/pipeline.c
index dfdfe973241..848e94c8d4b 100644
--- a/source/blender/render/intern/source/pipeline.c
+++ b/source/blender/render/intern/source/pipeline.c
@@ -1142,10 +1142,13 @@ static void do_render_3d(Render *re)
 		re->draw_lock(re->dlh, 1);
 	
 	/* make render verts/faces/halos/lamps */
-	if (render_scene_needs_vector(re))
+	if (render_scene_needs_vector(re)) {
 		RE_Database_FromScene_Vectors(re, re->main, re->scene, re->lay);
-	else
+	}
+	else {
 		RE_Database_FromScene(re, re->main, re->scene, re->lay, 1);
+		RE_Database_Preprocess(re);
+	}
 	
 	/* clear UI drawing locks */
 	if (re->draw_lock)
diff --git a/source/blender/windowmanager/WM_api.h b/source/blender/windowmanager/WM_api.h
index 230784d7273..7cb24daaaef 100644
--- a/source/blender/windowmanager/WM_api.h
+++ b/source/blender/windowmanager/WM_api.h
@@ -404,6 +404,9 @@ void		WM_jobs_kill_type(struct wmWindowManager *wm, int job_type);
 
 int			WM_jobs_has_running(struct wmWindowManager *wm);
 
+void		WM_job_main_thread_lock_acquire(struct wmJob *job);
+void		WM_job_main_thread_lock_release(struct wmJob *job);
+
 			/* clipboard */
 char       *WM_clipboard_text_get(int selection);
 void        WM_clipboard_text_set(char *buf, int selection);
diff --git a/source/blender/windowmanager/intern/wm_jobs.c b/source/blender/windowmanager/intern/wm_jobs.c
index 03af5e9e8a6..c6e067dc2f9 100644
--- a/source/blender/windowmanager/intern/wm_jobs.c
+++ b/source/blender/windowmanager/intern/wm_jobs.c
@@ -128,8 +128,43 @@ struct wmJob {
 	ListBase threads;
 
 	double start_time;
+
+	/* ticket mutex for main thread locking while some job accesses
+	 * data that the main thread might modify at the same time */
+	TicketMutex *main_thread_mutex;
+	bool main_thread_mutex_ending;
 };
 
+/* Main thread locking */
+
+void WM_job_main_thread_lock_acquire(wmJob *wm_job)
+{
+	BLI_ticket_mutex_lock(wm_job->main_thread_mutex);
+
+	/* if BLI_end_threads is being called to stop the job before it's finished,
+	 * we no longer need to lock to get access to the main thread as it's
+	 * waiting and can't respond */
+	if (wm_job->main_thread_mutex_ending)
+		BLI_ticket_mutex_unlock(wm_job->main_thread_mutex);
+}
+
+void WM_job_main_thread_lock_release(wmJob *wm_job)
+{
+	if (!wm_job->main_thread_mutex_ending)
+		BLI_ticket_mutex_unlock(wm_job->main_thread_mutex);
+}
+
+static void wm_job_main_thread_yield(wmJob *wm_job, bool ending)
+{
+	if (ending)
+		wm_job->main_thread_mutex_ending = true;
+
+	/* unlock and lock the ticket mutex. because it's a fair mutex any job that
+	 * is waiting to acquire the lock will get it first, before we can lock */
+	BLI_ticket_mutex_unlock(wm_job->main_thread_mutex);
+	BLI_ticket_mutex_lock(wm_job->main_thread_mutex);
+}
+
 /* finds:
  * if type, compare for it, otherwise any matching job 
  */
@@ -162,13 +197,16 @@ wmJob *WM_jobs_get(wmWindowManager *wm, wmWindow *win, void *owner, const char *
 	
 	if (wm_job == NULL) {
 		wm_job = MEM_callocN(sizeof(wmJob), "new job");
-	
+
 		BLI_addtail(&wm->jobs, wm_job);
 		wm_job->win = win;
 		wm_job->owner = owner;
 		wm_job->flag = flag;
 		wm_job->job_type = job_type;
 		BLI_strncpy(wm_job->name, name, sizeof(wm_job->name));
+
+		wm_job->main_thread_mutex = BLI_ticket_mutex_alloc();
+		BLI_ticket_mutex_lock(wm_job->main_thread_mutex);
 	}
 	/* else: a running job, be careful */
 	
@@ -369,12 +407,21 @@ void WM_jobs_start(wmWindowManager *wm, wmJob *wm_job)
 	}
 }
 
+static void wm_job_free(wmWindowManager *wm, wmJob *wm_job)
+{
+	BLI_remlink(&wm->jobs, wm_job);
+	BLI_ticket_mutex_unlock(wm_job->main_thread_mutex);
+	BLI_ticket_mutex_free(wm_job->main_thread_mutex);
+	MEM_freeN(wm_job);
+}
+
 /* stop job, end thread, free data completely */
 static void wm_jobs_kill_job(wmWindowManager *wm, wmJob *wm_job)
 {
 	if (wm_job->running) {
 		/* signal job to end */
 		wm_job->stop = TRUE;
+		wm_job_main_thread_yield(wm_job, true);
 		BLI_end_threads(&wm_job->threads);
 
 		if (wm_job->endjob)
@@ -389,9 +436,7 @@ static void wm_jobs_kill_job(wmWindowManager *wm, wmJob *wm_job)
 		wm_job->run_free(wm_job->run_customdata);
 	
 	/* remove wm_job */
-	BLI_remlink(&wm->jobs, wm_job);
-	MEM_freeN(wm_job);
-	
+	wm_job_free(wm, wm_job);
 }
 
 /* wait until every job ended */
@@ -483,7 +528,6 @@ void wm_jobs_timer(const bContext *C, wmWindowManager *wm, wmTimer *wt)
 	float total_progress = 0.f;
 	float jobs_progress = 0;
 	
-	
 	for (wm_job = wm->jobs.first; wm_job; wm_job = wm_jobnext) {
 		wm_jobnext = wm_job->next;
 		
@@ -491,6 +535,9 @@ void wm_jobs_timer(const bContext *C, wmWindowManager *wm, wmTimer *wt)
 			
 			/* running threads */
 			if (wm_job->threads.first) {
+
+				/* let threads get temporary lock over main thread if needed */
+				wm_job_main_thread_yield(wm_job, false);
 				
 				/* always call note and update when ready */
 				if (wm_job->do_update || wm_job->ready) {
@@ -522,7 +569,9 @@ void wm_jobs_timer(const bContext *C, wmWindowManager *wm, wmTimer *wt)
 					}
 
 					wm_job->running = FALSE;
+					wm_job_main_thread_yield(wm_job, true);
 					BLI_end_threads(&wm_job->threads);
+					wm_job->main_thread_mutex_ending = false;
 					
 					if (wm_job->endnote)
 						WM_event_add_notifier(C, wm_job->endnote, NULL);
@@ -539,8 +588,7 @@ void wm_jobs_timer(const bContext *C, wmWindowManager *wm, wmTimer *wt)
 						wm_job->wt = NULL;
 						
 						/* remove wm_job */
-						BLI_remlink(&wm->jobs, wm_job);
-						MEM_freeN(wm_job);
+						wm_job_free(wm, wm_job);
 					}
 				}
 				else if (wm_job->flag & WM_JOB_PROGRESS) {