BLI_task: add TLS support to BLI_task_parallel_mempool

Support thread local storage for BLI_task_parallel_mempool,
as well as support for the reduce and free callbacks.

mempool_iter_threadsafe_* functions have been moved into a private
header thats only shared between task_iterator.c and BLI_mempool.c
so the TLS can be made part of the iterator array without having to
rely on passing in struct offsets.

Add test task.MempoolIterTLS that ensures reduce and free
are working as expected.

Reviewed By: mont29

Ref D11548

11 files changed, 295 insertions, 77 deletions

diff --git a/source/blender/blenkernel/intern/armature_deform.c b/source/blender/blenkernel/intern/armature_deform.c
index 8711a001e32..bca5503c8d2 100644
--- a/source/blender/blenkernel/intern/armature_deform.c
+++ b/source/blender/blenkernel/intern/armature_deform.c
@@ -444,7 +444,9 @@ static void armature_vert_task(void *__restrict userdata,
   armature_vert_task_with_dvert(data, i, dvert);
 }
 
-static void armature_vert_task_editmesh(void *__restrict userdata, MempoolIterData *iter)
+static void armature_vert_task_editmesh(void *__restrict userdata,
+                                        MempoolIterData *iter,
+                                        const TaskParallelTLS *__restrict UNUSED(tls))
 {
   const ArmatureUserdata *data = userdata;
   BMVert *v = (BMVert *)iter;
@@ -452,7 +454,9 @@ static void armature_vert_task_editmesh(void *__restrict userdata, MempoolIterDa
   armature_vert_task_with_dvert(data, BM_elem_index_get(v), dvert);
 }
 
-static void armature_vert_task_editmesh_no_dvert(void *__restrict userdata, MempoolIterData *iter)
+static void armature_vert_task_editmesh_no_dvert(void *__restrict userdata,
+                                                 MempoolIterData *iter,
+                                                 const TaskParallelTLS *__restrict UNUSED(tls))
 {
   const ArmatureUserdata *data = userdata;
   BMVert *v = (BMVert *)iter;
@@ -593,12 +597,16 @@ static void armature_deform_coords_impl(const Object *ob_arm,
      * have already been properly set. */
     BM_mesh_elem_index_ensure(em_target->bm, BM_VERT);
 
+    TaskParallelSettings settings;
+    BLI_parallel_mempool_settings_defaults(&settings);
+
     if (use_dverts) {
-      BLI_task_parallel_mempool(em_target->bm->vpool, &data, armature_vert_task_editmesh, true);
+      BLI_task_parallel_mempool(
+          em_target->bm->vpool, &data, armature_vert_task_editmesh, &settings);
     }
     else {
       BLI_task_parallel_mempool(
-          em_target->bm->vpool, &data, armature_vert_task_editmesh_no_dvert, true);
+          em_target->bm->vpool, &data, armature_vert_task_editmesh_no_dvert, &settings);
     }
   }
   else {
diff --git a/source/blender/blenkernel/intern/lattice_deform.c b/source/blender/blenkernel/intern/lattice_deform.c
index 4a926ee3d96..a8126cb5538 100644
--- a/source/blender/blenkernel/intern/lattice_deform.c
+++ b/source/blender/blenkernel/intern/lattice_deform.c
@@ -320,7 +320,9 @@ static void lattice_deform_vert_task(void *__restrict userdata,
   lattice_deform_vert_with_dvert(data, index, data->dvert ? &data->dvert[index] : NULL);
 }
 
-static void lattice_vert_task_editmesh(void *__restrict userdata, MempoolIterData *iter)
+static void lattice_vert_task_editmesh(void *__restrict userdata,
+                                       MempoolIterData *iter,
+                                       const TaskParallelTLS *__restrict UNUSED(tls))
 {
   const LatticeDeformUserdata *data = userdata;
   BMVert *v = (BMVert *)iter;
@@ -328,7 +330,9 @@ static void lattice_vert_task_editmesh(void *__restrict userdata, MempoolIterDat
   lattice_deform_vert_with_dvert(data, BM_elem_index_get(v), dvert);
 }
 
-static void lattice_vert_task_editmesh_no_dvert(void *__restrict userdata, MempoolIterData *iter)
+static void lattice_vert_task_editmesh_no_dvert(void *__restrict userdata,
+                                                MempoolIterData *iter,
+                                                const TaskParallelTLS *__restrict UNUSED(tls))
 {
   const LatticeDeformUserdata *data = userdata;
   BMVert *v = (BMVert *)iter;
@@ -397,12 +401,16 @@ static void lattice_deform_coords_impl(const Object *ob_lattice,
      * have already been properly set. */
     BM_mesh_elem_index_ensure(em_target->bm, BM_VERT);
 
+    TaskParallelSettings settings;
+    BLI_parallel_mempool_settings_defaults(&settings);
+
     if (cd_dvert_offset != -1) {
-      BLI_task_parallel_mempool(em_target->bm->vpool, &data, lattice_vert_task_editmesh, true);
+      BLI_task_parallel_mempool(
+          em_target->bm->vpool, &data, lattice_vert_task_editmesh, &settings);
     }
     else {
       BLI_task_parallel_mempool(
-          em_target->bm->vpool, &data, lattice_vert_task_editmesh_no_dvert, true);
+          em_target->bm->vpool, &data, lattice_vert_task_editmesh_no_dvert, &settings);
     }
   }
   else {
diff --git a/source/blender/blenlib/BLI_mempool.h b/source/blender/blenlib/BLI_mempool.h
index 3ee6f182593..29fb6a2e233 100644
--- a/source/blender/blenlib/BLI_mempool.h
+++ b/source/blender/blenlib/BLI_mempool.h
@@ -88,11 +88,6 @@ enum {
 void BLI_mempool_iternew(BLI_mempool *pool, BLI_mempool_iter *iter) ATTR_NONNULL();
 void *BLI_mempool_iterstep(BLI_mempool_iter *iter) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
 
-BLI_mempool_iter *BLI_mempool_iter_threadsafe_create(BLI_mempool *pool,
-                                                     const size_t num_iter) ATTR_WARN_UNUSED_RESULT
-    ATTR_NONNULL();
-void BLI_mempool_iter_threadsafe_free(BLI_mempool_iter *iter_arr) ATTR_NONNULL();
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/blender/blenlib/BLI_task.h b/source/blender/blenlib/BLI_task.h
index 339bb256819..59020c9eb1e 100644
--- a/source/blender/blenlib/BLI_task.h
+++ b/source/blender/blenlib/BLI_task.h
@@ -261,11 +261,14 @@ void BLI_task_parallel_listbase(struct ListBase *listbase,
                                 const TaskParallelSettings *settings);
 
 typedef struct MempoolIterData MempoolIterData;
-typedef void (*TaskParallelMempoolFunc)(void *userdata, MempoolIterData *iter);
+
+typedef void (*TaskParallelMempoolFunc)(void *userdata,
+                                        MempoolIterData *iter,
+                                        const TaskParallelTLS *__restrict tls);
 void BLI_task_parallel_mempool(struct BLI_mempool *mempool,
                                void *userdata,
                                TaskParallelMempoolFunc func,
-                               const bool use_threading);
+                               const TaskParallelSettings *settings);
 
 /* TODO(sergey): Think of a better place for this. */
 BLI_INLINE void BLI_parallel_range_settings_defaults(TaskParallelSettings *settings)
@@ -276,6 +279,12 @@ BLI_INLINE void BLI_parallel_range_settings_defaults(TaskParallelSettings *setti
   settings->min_iter_per_thread = 0;
 }
 
+BLI_INLINE void BLI_parallel_mempool_settings_defaults(TaskParallelSettings *settings)
+{
+  memset(settings, 0, sizeof(*settings));
+  settings->use_threading = true;
+}
+
 /* Don't use this, store any thread specific data in tls->userdata_chunk instead.
  * Only here for code to be removed. */
 int BLI_task_parallel_thread_id(const TaskParallelTLS *tls);
diff --git a/source/blender/blenlib/CMakeLists.txt b/source/blender/blenlib/CMakeLists.txt
index f3dc343ee20..e04f3c1b19d 100644
--- a/source/blender/blenlib/CMakeLists.txt
+++ b/source/blender/blenlib/CMakeLists.txt
@@ -145,6 +145,9 @@ set(SRC
   intern/winstuff.c
   intern/winstuff_dir.c
 
+  # Private headers.
+  intern/BLI_mempool_private.h
+
   # Header as source (included in C files above).
   intern/kdtree_impl.h
   intern/list_sort_impl.h
diff --git a/source/blender/blenlib/intern/BLI_mempool.c b/source/blender/blenlib/intern/BLI_mempool.c
index 75f934c1fb8..cf0021609b5 100644
--- a/source/blender/blenlib/intern/BLI_mempool.c
+++ b/source/blender/blenlib/intern/BLI_mempool.c
@@ -36,7 +36,8 @@
 
 #include "BLI_utildefines.h"
 
-#include "BLI_mempool.h" /* own include */
+#include "BLI_mempool.h"         /* own include */
+#include "BLI_mempool_private.h" /* own include */
 
 #include "MEM_guardedalloc.h"
 
@@ -558,33 +559,31 @@ void BLI_mempool_iternew(BLI_mempool *pool, BLI_mempool_iter *iter)
  *
  * See BLI_task_parallel_mempool implementation for detailed usage example.
  */
-BLI_mempool_iter *BLI_mempool_iter_threadsafe_create(BLI_mempool *pool, const size_t num_iter)
+ParallelMempoolTaskData *mempool_iter_threadsafe_create(BLI_mempool *pool, const size_t num_iter)
 {
   BLI_assert(pool->flag & BLI_MEMPOOL_ALLOW_ITER);
 
-  BLI_mempool_iter *iter_arr = MEM_mallocN(sizeof(*iter_arr) * num_iter, __func__);
+  ParallelMempoolTaskData *iter_arr = MEM_mallocN(sizeof(*iter_arr) * num_iter, __func__);
   BLI_mempool_chunk **curchunk_threaded_shared = MEM_mallocN(sizeof(void *), __func__);
 
-  BLI_mempool_iternew(pool, iter_arr);
-
-  *curchunk_threaded_shared = iter_arr->curchunk;
-  iter_arr->curchunk_threaded_shared = curchunk_threaded_shared;
+  BLI_mempool_iternew(pool, &iter_arr->iter);
 
+  *curchunk_threaded_shared = iter_arr->iter.curchunk;
+  iter_arr->iter.curchunk_threaded_shared = curchunk_threaded_shared;
   for (size_t i = 1; i < num_iter; i++) {
-    iter_arr[i] = iter_arr[0];
-    *curchunk_threaded_shared = iter_arr[i].curchunk = ((*curchunk_threaded_shared) ?
-                                                            (*curchunk_threaded_shared)->next :
-                                                            NULL);
+    iter_arr[i].iter = iter_arr[0].iter;
+    *curchunk_threaded_shared = iter_arr[i].iter.curchunk =
+        ((*curchunk_threaded_shared) ? (*curchunk_threaded_shared)->next : NULL);
   }
 
   return iter_arr;
 }
 
-void BLI_mempool_iter_threadsafe_free(BLI_mempool_iter *iter_arr)
+void mempool_iter_threadsafe_destroy(ParallelMempoolTaskData *iter_arr)
 {
-  BLI_assert(iter_arr->curchunk_threaded_shared != NULL);
+  BLI_assert(iter_arr->iter.curchunk_threaded_shared != NULL);
 
-  MEM_freeN(iter_arr->curchunk_threaded_shared);
+  MEM_freeN(iter_arr->iter.curchunk_threaded_shared);
   MEM_freeN(iter_arr);
 }
 
diff --git a/source/blender/blenlib/intern/BLI_mempool_private.h b/source/blender/blenlib/intern/BLI_mempool_private.h
new file mode 100644
index 00000000000..60994624ecd
--- /dev/null
+++ b/source/blender/blenlib/intern/BLI_mempool_private.h
@@ -0,0 +1,45 @@
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2008 Blender Foundation.
+ * All rights reserved.
+ */
+
+#pragma once
+
+/** \file
+ * \ingroup bli
+ *
+ * Shared logic for #BLI_task_parallel_mempool to create a threaded iterator,
+ * without exposing the these functions publicly.
+ */
+
+#include "BLI_compiler_attrs.h"
+
+#include "BLI_mempool.h"
+#include "BLI_task.h"
+
+typedef struct ParallelMempoolTaskData {
+  BLI_mempool_iter iter;
+  TaskParallelTLS tls;
+} ParallelMempoolTaskData;
+
+ParallelMempoolTaskData *mempool_iter_threadsafe_create(BLI_mempool *pool, const size_t num_iter)
+    ATTR_WARN_UNUSED_RESULT ATTR_NONNULL();
+void mempool_iter_threadsafe_destroy(ParallelMempoolTaskData *iter_arr) ATTR_NONNULL();
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/source/blender/blenlib/intern/task_iterator.c b/source/blender/blenlib/intern/task_iterator.c
index 463d3675cbe..3914a074d11 100644
--- a/source/blender/blenlib/intern/task_iterator.c
+++ b/source/blender/blenlib/intern/task_iterator.c
@@ -29,6 +29,7 @@
 #include "BLI_listbase.h"
 #include "BLI_math.h"
 #include "BLI_mempool.h"
+#include "BLI_mempool_private.h"
 #include "BLI_task.h"
 #include "BLI_threads.h"
 
@@ -348,9 +349,6 @@ void BLI_task_parallel_listbase(ListBase *listbase,
   task_parallel_iterator_do(settings, &state);
 }
 
-#undef MALLOCA
-#undef MALLOCA_FREE
-
 typedef struct ParallelMempoolState {
   void *userdata;
   TaskParallelMempoolFunc func;
@@ -359,11 +357,12 @@ typedef struct ParallelMempoolState {
 static void parallel_mempool_func(TaskPool *__restrict pool, void *taskdata)
 {
   ParallelMempoolState *__restrict state = BLI_task_pool_user_data(pool);
-  BLI_mempool_iter *iter = taskdata;
-  MempoolIterData *item;
+  BLI_mempool_iter *iter = &((ParallelMempoolTaskData *)taskdata)->iter;
+  TaskParallelTLS *tls = &((ParallelMempoolTaskData *)taskdata)->tls;
 
+  MempoolIterData *item;
   while ((item = BLI_mempool_iterstep(iter)) != NULL) {
-    state->func(state->userdata, item);
+    state->func(state->userdata, item, tls);
   }
 }
 
@@ -373,16 +372,14 @@ static void parallel_mempool_func(TaskPool *__restrict pool, void *taskdata)
  * \param mempool: The iterable BLI_mempool to loop over.
  * \param userdata: Common userdata passed to all instances of \a func.
  * \param func: Callback function.
- * \param use_threading: If \a true, actually split-execute loop in threads,
- * else just do a sequential for loop
- * (allows caller to use any kind of test to switch on parallelization or not).
+ * \param settings: See public API doc of TaskParallelSettings for description of all settings.
  *
  * \note There is no static scheduling here.
  */
 void BLI_task_parallel_mempool(BLI_mempool *mempool,
                                void *userdata,
                                TaskParallelMempoolFunc func,
-                               const bool use_threading)
+                               const TaskParallelSettings *settings)
 {
   TaskPool *task_pool;
   ParallelMempoolState state;
@@ -392,14 +389,34 @@ void BLI_task_parallel_mempool(BLI_mempool *mempool,
     return;
   }
 
-  if (!use_threading) {
+  void *userdata_chunk = settings->userdata_chunk;
+  const size_t userdata_chunk_size = settings->userdata_chunk_size;
+  void *userdata_chunk_local = NULL;
+  void *userdata_chunk_array = NULL;
+  const bool use_userdata_chunk = (userdata_chunk_size != 0) && (userdata_chunk != NULL);
+
+  if (!settings->use_threading) {
+    TaskParallelTLS tls = {NULL};
+    if (use_userdata_chunk) {
+      userdata_chunk_local = MALLOCA(userdata_chunk_size);
+      memcpy(userdata_chunk_local, userdata_chunk, userdata_chunk_size);
+      tls.userdata_chunk = userdata_chunk_local;
+    }
+
     BLI_mempool_iter iter;
     BLI_mempool_iternew(mempool, &iter);
 
-    for (void *item = BLI_mempool_iterstep(&iter); item != NULL;
-         item = BLI_mempool_iterstep(&iter)) {
-      func(userdata, item);
+    void *item;
+    while ((item = BLI_mempool_iterstep(&iter))) {
+      func(userdata, item, &tls);
     }
+
+    if (settings->func_free != NULL) {
+      /* `func_free` should only free data that was created during execution of `func`. */
+      settings->func_free(userdata, userdata_chunk_local);
+    }
+
+    MALLOCA_FREE(userdata_chunk_local, userdata_chunk_size);
     return;
   }
 
@@ -415,16 +432,44 @@ void BLI_task_parallel_mempool(BLI_mempool *mempool,
   state.userdata = userdata;
   state.func = func;
 
-  BLI_mempool_iter *mempool_iterators = BLI_mempool_iter_threadsafe_create(mempool,
-                                                                           (size_t)num_tasks);
+  if (use_userdata_chunk) {
+    userdata_chunk_array = MALLOCA(userdata_chunk_size * num_tasks);
+  }
+
+  ParallelMempoolTaskData *mempool_iterator_data = mempool_iter_threadsafe_create(
+      mempool, (size_t)num_tasks);
 
   for (i = 0; i < num_tasks; i++) {
+    if (use_userdata_chunk) {
+      userdata_chunk_local = (char *)userdata_chunk_array + (userdata_chunk_size * i);
+      memcpy(userdata_chunk_local, userdata_chunk, userdata_chunk_size);
+    }
+    mempool_iterator_data[i].tls.userdata_chunk = userdata_chunk_local;
+
     /* Use this pool's pre-allocated tasks. */
-    BLI_task_pool_push(task_pool, parallel_mempool_func, &mempool_iterators[i], false, NULL);
+    BLI_task_pool_push(task_pool, parallel_mempool_func, &mempool_iterator_data[i], false, NULL);
   }
 
   BLI_task_pool_work_and_wait(task_pool);
   BLI_task_pool_free(task_pool);
 
-  BLI_mempool_iter_threadsafe_free(mempool_iterators);
+  if (use_userdata_chunk) {
+    if ((settings->func_free != NULL) || (settings->func_reduce != NULL)) {
+      for (i = 0; i < num_tasks; i++) {
+        if (settings->func_reduce) {
+          settings->func_reduce(
+              userdata, userdata_chunk, mempool_iterator_data[i].tls.userdata_chunk);
+        }
+        if (settings->func_free) {
+          settings->func_free(userdata, mempool_iterator_data[i].tls.userdata_chunk);
+        }
+      }
+    }
+    MALLOCA_FREE(userdata_chunk_array, userdata_chunk_size * num_tasks);
+  }
+
+  mempool_iter_threadsafe_destroy(mempool_iterator_data);
 }
+
+#undef MALLOCA
+#undef MALLOCA_FREE
diff --git a/source/blender/blenlib/tests/BLI_task_test.cc b/source/blender/blenlib/tests/BLI_task_test.cc
index fce3e56d105..75d89656a0f 100644
--- a/source/blender/blenlib/tests/BLI_task_test.cc
+++ b/source/blender/blenlib/tests/BLI_task_test.cc
@@ -67,7 +67,9 @@ TEST(task, RangeIter)
 
 /* *** Parallel iterations over mempool items. *** */
 
-static void task_mempool_iter_func(void *userdata, MempoolIterData *item)
+static void task_mempool_iter_func(void *userdata,
+                                   MempoolIterData *item,
+                                   const TaskParallelTLS *__restrict UNUSED(tls))
 {
   int *data = (int *)item;
   int *count = (int *)userdata;
@@ -119,7 +121,10 @@ TEST(task, MempoolIter)
     }
   }
 
-  BLI_task_parallel_mempool(mempool, &num_items, task_mempool_iter_func, true);
+  TaskParallelSettings settings;
+  BLI_parallel_mempool_settings_defaults(&settings);
+
+  BLI_task_parallel_mempool(mempool, &num_items, task_mempool_iter_func, &settings);
 
   /* Those checks should ensure us all items of the mempool were processed once, and only once - as
    * expected. */
@@ -134,6 +139,99 @@ TEST(task, MempoolIter)
   BLI_threadapi_exit();
 }
 
+/* *** Parallel iterations over mempool items with TLS. *** */
+
+typedef struct TaskMemPool_Chunk {
+  ListBase *accumulate_items;
+} TaskMemPool_Chunk;
+
+static void task_mempool_iter_tls_func(void *UNUSED(userdata),
+                                       MempoolIterData *item,
+                                       const TaskParallelTLS *__restrict tls)
+{
+  TaskMemPool_Chunk *task_data = (TaskMemPool_Chunk *)tls->userdata_chunk;
+  int *data = (int *)item;
+
+  EXPECT_TRUE(data != nullptr);
+  if (task_data->accumulate_items == nullptr) {
+    task_data->accumulate_items = (ListBase *)MEM_callocN(sizeof(ListBase), __func__);
+  }
+
+  /* Flip to prove this has been touched. */
+  *data = -*data;
+
+  BLI_addtail(task_data->accumulate_items, BLI_genericNodeN(data));
+}
+
+static void task_mempool_iter_tls_reduce(const void *__restrict UNUSED(userdata),
+                                         void *__restrict chunk_join,
+                                         void *__restrict chunk)
+{
+  TaskMemPool_Chunk *join_chunk = (TaskMemPool_Chunk *)chunk_join;
+  TaskMemPool_Chunk *data_chunk = (TaskMemPool_Chunk *)chunk;
+
+  if (data_chunk->accumulate_items != nullptr) {
+    if (join_chunk->accumulate_items == nullptr) {
+      join_chunk->accumulate_items = (ListBase *)MEM_callocN(sizeof(ListBase), __func__);
+    }
+    BLI_movelisttolist(join_chunk->accumulate_items, data_chunk->accumulate_items);
+  }
+}
+
+static void task_mempool_iter_tls_free(const void *UNUSED(userdata),
+                                       void *__restrict userdata_chunk)
+{
+  TaskMemPool_Chunk *task_data = (TaskMemPool_Chunk *)userdata_chunk;
+  MEM_freeN(task_data->accumulate_items);
+}
+
+TEST(task, MempoolIterTLS)
+{
+  int *data[NUM_ITEMS];
+  BLI_threadapi_init();
+  BLI_mempool *mempool = BLI_mempool_create(
+      sizeof(*data[0]), NUM_ITEMS, 32, BLI_MEMPOOL_ALLOW_ITER);
+
+  int i;
+
+  /* Add numbers negative `1..NUM_ITEMS` inclusive. */
+  int num_items = 0;
+  for (i = 0; i < NUM_ITEMS; i++) {
+    data[i] = (int *)BLI_mempool_alloc(mempool);
+    *data[i] = -(i + 1);
+    num_items++;
+  }
+
+  TaskParallelSettings settings;
+  BLI_parallel_mempool_settings_defaults(&settings);
+
+  TaskMemPool_Chunk tls_data;
+
+  settings.userdata_chunk = &tls_data;
+  settings.userdata_chunk_size = sizeof(tls_data);
+
+  settings.func_free = task_mempool_iter_tls_free;
+  settings.func_reduce = task_mempool_iter_tls_reduce;
+
+  BLI_task_parallel_mempool(mempool, nullptr, task_mempool_iter_tls_func, &settings);
+
+  EXPECT_EQ(BLI_listbase_count(tls_data.accumulate_items), NUM_ITEMS);
+
+  /* Check that all elements are added into the list once. */
+  int num_accum = 0;
+  for (LinkData *link = (LinkData *)tls_data.accumulate_items->first; link; link = link->next) {
+    int *data = (int *)link->data;
+    num_accum += *data;
+  }
+  EXPECT_EQ(num_accum, (NUM_ITEMS * (NUM_ITEMS + 1)) / 2);
+
+  BLI_freelistN(tls_data.accumulate_items);
+  MEM_freeN(tls_data.accumulate_items);
+
+  BLI_mempool_destroy(mempool);
+  BLI_threadapi_exit();
+}
+
 /* *** Parallel iterations over double-linked list items. *** */
 
 static void task_listbase_iter_func(void *userdata,
diff --git a/source/blender/bmesh/intern/bmesh_iterators_inline.h b/source/blender/bmesh/intern/bmesh_iterators_inline.h
index c384fb03cd9..81b6a58e58b 100644
--- a/source/blender/bmesh/intern/bmesh_iterators_inline.h
+++ b/source/blender/bmesh/intern/bmesh_iterators_inline.h
@@ -188,18 +188,18 @@ BLI_INLINE void BM_iter_parallel(BMesh *bm,
                                  const char itype,
                                  TaskParallelMempoolFunc func,
                                  void *userdata,
-                                 const bool use_threading)
+                                 const TaskParallelSettings *settings)
 {
   /* inlining optimizes out this switch when called with the defined type */
   switch ((BMIterType)itype) {
     case BM_VERTS_OF_MESH:
-      BLI_task_parallel_mempool(bm->vpool, userdata, func, use_threading);
+      BLI_task_parallel_mempool(bm->vpool, userdata, func, settings);
       break;
     case BM_EDGES_OF_MESH:
-      BLI_task_parallel_mempool(bm->epool, userdata, func, use_threading);
+      BLI_task_parallel_mempool(bm->epool, userdata, func, settings);
       break;
     case BM_FACES_OF_MESH:
-      BLI_task_parallel_mempool(bm->fpool, userdata, func, use_threading);
+      BLI_task_parallel_mempool(bm->fpool, userdata, func, settings);
       break;
     default:
       /* should never happen */
diff --git a/source/blender/bmesh/intern/bmesh_mesh_normals.c b/source/blender/bmesh/intern/bmesh_mesh_normals.c
index f0a791bae19..a3eae6dabe8 100644
--- a/source/blender/bmesh/intern/bmesh_mesh_normals.c
+++ b/source/blender/bmesh/intern/bmesh_mesh_normals.c
@@ -57,7 +57,9 @@ typedef struct BMEdgesCalcVectorsData {
   float (*edgevec)[3];
 } BMEdgesCalcVectorsData;
 
-static void bm_edge_calc_vectors_cb(void *userdata, MempoolIterData *mp_e)
+static void bm_edge_calc_vectors_cb(void *userdata,
+                                    MempoolIterData *mp_e,
+                                    const TaskParallelTLS *__restrict UNUSED(tls))
 {
   BMEdge *e = (BMEdge *)mp_e;
   /* The edge vector will not be needed when the edge has no radial. */
@@ -69,7 +71,9 @@ static void bm_edge_calc_vectors_cb(void *userdata, MempoolIterData *mp_e)
   }
 }
 
-static void bm_edge_calc_vectors_with_coords_cb(void *userdata, MempoolIterData *mp_e)
+static void bm_edge_calc_vectors_with_coords_cb(void *userdata,
+                                                MempoolIterData *mp_e,
+                                                const TaskParallelTLS *__restrict UNUSED(tls))
 {
   BMEdge *e = (BMEdge *)mp_e;
   /* The edge vector will not be needed when the edge has no radial. */
@@ -86,20 +90,19 @@ static void bm_mesh_edges_calc_vectors(BMesh *bm, float (*edgevec)[3], const flo
 {
   BM_mesh_elem_index_ensure(bm, BM_EDGE | (vcos ? BM_VERT : 0));
 
+  TaskParallelSettings settings;
+  BLI_parallel_mempool_settings_defaults(&settings);
+  settings.use_threading = bm->totedge >= BM_OMP_LIMIT;
+
   if (vcos == NULL) {
-    BM_iter_parallel(
-        bm, BM_EDGES_OF_MESH, bm_edge_calc_vectors_cb, edgevec, bm->totedge >= BM_OMP_LIMIT);
+    BM_iter_parallel(bm, BM_EDGES_OF_MESH, bm_edge_calc_vectors_cb, edgevec, &settings);
   }
   else {
     BMEdgesCalcVectorsData data = {
         .edgevec = edgevec,
         .vcos = vcos,
     };
-    BM_iter_parallel(bm,
-                     BM_EDGES_OF_MESH,
-                     bm_edge_calc_vectors_with_coords_cb,
-                     &data,
-                     bm->totedge >= BM_OMP_LIMIT);
+    BM_iter_parallel(bm, BM_EDGES_OF_MESH, bm_edge_calc_vectors_with_coords_cb, &data, &settings);
   }
 }
 
@@ -161,7 +164,9 @@ static void bm_vert_calc_normals_impl(const float (*edgevec)[3], BMVert *v)
   normalize_v3_v3(v_no, v->co);
 }
 
-static void bm_vert_calc_normals_cb(void *userdata, MempoolIterData *mp_v)
+static void bm_vert_calc_normals_cb(void *userdata,
+                                    MempoolIterData *mp_v,
+                                    const TaskParallelTLS *__restrict UNUSED(tls))
 {
   const float(*edgevec)[3] = userdata;
   BMVert *v = (BMVert *)mp_v;
@@ -198,7 +203,9 @@ static void bm_vert_calc_normals_with_coords(BMVert *v, BMVertsCalcNormalsWithCo
   normalize_v3_v3(v_no, data->vcos[BM_elem_index_get(v)]);
 }
 
-static void bm_vert_calc_normals_with_coords_cb(void *userdata, MempoolIterData *mp_v)
+static void bm_vert_calc_normals_with_coords_cb(void *userdata,
+                                                MempoolIterData *mp_v,
+                                                const TaskParallelTLS *__restrict UNUSED(tls))
 {
   BMVertsCalcNormalsWithCoordsData *data = userdata;
   BMVert *v = (BMVert *)mp_v;
@@ -213,12 +220,12 @@ static void bm_mesh_verts_calc_normals(BMesh *bm,
 {
   BM_mesh_elem_index_ensure(bm, (BM_EDGE | BM_FACE) | ((vnos || vcos) ? BM_VERT : 0));
 
+  TaskParallelSettings settings;
+  BLI_parallel_mempool_settings_defaults(&settings);
+  settings.use_threading = bm->totvert >= BM_OMP_LIMIT;
+
   if (vcos == NULL) {
-    BM_iter_parallel(bm,
-                     BM_VERTS_OF_MESH,
-                     bm_vert_calc_normals_cb,
-                     (void *)edgevec,
-                     bm->totvert >= BM_OMP_LIMIT);
+    BM_iter_parallel(bm, BM_VERTS_OF_MESH, bm_vert_calc_normals_cb, (void *)edgevec, &settings);
   }
   else {
     BLI_assert(!ELEM(NULL, fnos, vnos));
@@ -228,15 +235,13 @@ static void bm_mesh_verts_calc_normals(BMesh *bm,
         .vcos = vcos,
         .vnos = vnos,
     };
-    BM_iter_parallel(bm,
-                     BM_VERTS_OF_MESH,
-                     bm_vert_calc_normals_with_coords_cb,
-                     &data,
-                     bm->totvert >= BM_OMP_LIMIT);
+    BM_iter_parallel(bm, BM_VERTS_OF_MESH, bm_vert_calc_normals_with_coords_cb, &data, &settings);
   }
 }
 
-static void bm_face_calc_normals_cb(void *UNUSED(userdata), MempoolIterData *mp_f)
+static void bm_face_calc_normals_cb(void *UNUSED(userdata),
+                                    MempoolIterData *mp_f,
+                                    const TaskParallelTLS *__restrict UNUSED(tls))
 {
   BMFace *f = (BMFace *)mp_f;
 
@@ -256,8 +261,11 @@ void BM_mesh_normals_update(BMesh *bm)
   BM_mesh_elem_index_ensure(bm, (BM_EDGE | BM_FACE));
 
   /* Calculate all face normals. */
-  BM_iter_parallel(
-      bm, BM_FACES_OF_MESH, bm_face_calc_normals_cb, NULL, bm->totface >= BM_OMP_LIMIT);
+  TaskParallelSettings settings;
+  BLI_parallel_mempool_settings_defaults(&settings);
+  settings.use_threading = bm->totedge >= BM_OMP_LIMIT;
+
+  BM_iter_parallel(bm, BM_FACES_OF_MESH, bm_face_calc_normals_cb, NULL, &settings);
 
   bm_mesh_edges_calc_vectors(bm, edgevec, NULL);