Fix T102292: deadlock in geometry nodes evaluation with task isolation

As described in the comment on `BLI_task_isolate`, deadlocks can happen
when isolation is used with threading primitives that separate spawning tasks
from executing them. All threads are waiting the tasks to complete but no
thread is able to continue working due to task isolation.

The fix is to not pass lazy-threading hints through task isolations. This way
isolated regions can't create new tasks in a scheduler further up the call stack.
This may lead to minor slowdowns because less threading may be used.
It's generally possible to get rid of the slowdown again by sending the
lazy-threading hint before entering the isolated region.

4 files changed, 39 insertions, 5 deletions

diff --git a/source/blender/blenlib/BLI_lazy_threading.hh b/source/blender/blenlib/BLI_lazy_threading.hh
index b5a15919c89..4d04fe9e908 100644
--- a/source/blender/blenlib/BLI_lazy_threading.hh
+++ b/source/blender/blenlib/BLI_lazy_threading.hh
@@ -80,4 +80,15 @@ class HintReceiver {
   ~HintReceiver();
 };
 
+/**
+ * Used to make sure that lazy-threading hints don't propagate through task isolation. This is
+ * necessary to avoid deadlocks when isolated regions are used together with e.g. task pools. For
+ * more info see the comment on #BLI_task_isolate.
+ */
+class ReceiverIsolation {
+ public:
+  ReceiverIsolation();
+  ~ReceiverIsolation();
+};
+
 }  // namespace blender::lazy_threading
diff --git a/source/blender/blenlib/BLI_task.hh b/source/blender/blenlib/BLI_task.hh
index 9f9a57be634..e7d9a21439a 100644
--- a/source/blender/blenlib/BLI_task.hh
+++ b/source/blender/blenlib/BLI_task.hh
@@ -129,6 +129,7 @@ void parallel_invoke(const bool use_threading, Functions &&...functions)
 template<typename Function> void isolate_task(const Function &function)
 {
 #ifdef WITH_TBB
+  lazy_threading::ReceiverIsolation isolation;
   tbb::this_task_arena::isolate(function);
 #else
   function();
diff --git a/source/blender/blenlib/intern/lazy_threading.cc b/source/blender/blenlib/intern/lazy_threading.cc
index 803fd81a96d..4f6d3a75ecc 100644
--- a/source/blender/blenlib/intern/lazy_threading.cc
+++ b/source/blender/blenlib/intern/lazy_threading.cc
@@ -1,30 +1,50 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 
 #include "BLI_lazy_threading.hh"
+#include "BLI_stack.hh"
 #include "BLI_vector.hh"
 
 namespace blender::lazy_threading {
 
 /**
- * This is a #RawVector so that it can be destructed after Blender checks for memory leaks.
+ * This uses a "raw" stack and vector so that it can be destructed after Blender checks for memory
+ * leaks. A new list of receivers is created whenever an isolated region is entered to avoid
+ * deadlocks.
  */
-thread_local RawVector<FunctionRef<void()>, 0> hint_receivers;
+using HintReceivers = RawStack<RawVector<FunctionRef<void()>, 0>, 0>;
+thread_local HintReceivers hint_receivers = []() {
+  HintReceivers receivers;
+  /* Make sure there is always at least one vector. */
+  receivers.push_as();
+  return receivers;
+}();
 
 void send_hint()
 {
-  for (const FunctionRef<void()> &fn : hint_receivers) {
+  for (const FunctionRef<void()> &fn : hint_receivers.peek()) {
     fn();
   }
 }
 
 HintReceiver::HintReceiver(const FunctionRef<void()> fn)
 {
-  hint_receivers.append(fn);
+  hint_receivers.peek().append(fn);
 }
 
 HintReceiver::~HintReceiver()
 {
-  hint_receivers.pop_last();
+  hint_receivers.peek().pop_last();
+}
+
+ReceiverIsolation::ReceiverIsolation()
+{
+  hint_receivers.push_as();
+}
+
+ReceiverIsolation::~ReceiverIsolation()
+{
+  BLI_assert(hint_receivers.peek().is_empty());
+  hint_receivers.pop();
 }
 
 }  // namespace blender::lazy_threading
diff --git a/source/blender/blenlib/intern/task_scheduler.cc b/source/blender/blenlib/intern/task_scheduler.cc
index 1f7747453c1..5b056df78b4 100644
--- a/source/blender/blenlib/intern/task_scheduler.cc
+++ b/source/blender/blenlib/intern/task_scheduler.cc
@@ -8,6 +8,7 @@
 
 #include "MEM_guardedalloc.h"
 
+#include "BLI_lazy_threading.hh"
 #include "BLI_task.h"
 #include "BLI_threads.h"
 
@@ -67,6 +68,7 @@ int BLI_task_scheduler_num_threads()
 void BLI_task_isolate(void (*func)(void *userdata), void *userdata)
 {
 #ifdef WITH_TBB
+  blender::lazy_threading::ReceiverIsolation isolation;
   tbb::this_task_arena::isolate([&] { func(userdata); });
 #else
   func(userdata);