[libc] Add implementation of pthread_once.

The existing thrd_once function has been refactored so that the
implementation can be shared between thrd_once and pthread_once
functions.

Reviewed By: michaelrj

Differential Revision: https://reviews.llvm.org/D134716

20 files changed, 357 insertions, 17 deletions

diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 03559879b353..50baea0157c5 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -234,6 +234,7 @@ def ThreadsAPI : PublicAPI<"threads.h"> {
 
 def PThreadAPI : PublicAPI<"pthread.h"> {
   let Types = [
+      "__pthread_once_func_t",
       "__pthread_start_t",
       "__pthread_tss_dtor_t",
       "pthread_attr_t",
@@ -241,6 +242,7 @@ def PThreadAPI : PublicAPI<"pthread.h"> {
       "pthread_mutexattr_t",
       "pthread_t",
       "pthread_key_t",
+      "pthread_once_t",
   ];
 }
 
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 2a48c90b967d..b7d122b60d6f 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -321,6 +321,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.pthread.pthread_mutexattr_setpshared
     libc.src.pthread.pthread_mutexattr_setrobust
     libc.src.pthread.pthread_mutexattr_settype
+    libc.src.pthread.pthread_once
     libc.src.pthread.pthread_setspecific
 
     # stdio.h entrypoints
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index a8aac4cfbd31..e8db6b72df70 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -188,6 +188,8 @@ add_gen_header(
     .llvm-libc-types.pthread_key_t
     .llvm-libc-types.pthread_mutex_t
     .llvm-libc-types.pthread_mutexattr_t
+    .llvm-libc-types.pthread_once_t
+    .llvm-libc-types.__pthread_once_func_t
     .llvm-libc-types.pthread_t
 )
 
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index 5fde05a3665a..5f0a5796042c 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -4,6 +4,7 @@ add_header(__bsearchcompare_t HDR __bsearchcompare_t.h)
 add_header(__call_once_func_t HDR __call_once_func_t.h)
 add_header(__futex_word HDR __futex_word.h)
 add_header(__mutex_type HDR __mutex_type.h DEPENDS .__futex_word)
+add_header(__pthread_once_func_t HDR __pthread_once_func_t.h)
 add_header(__pthread_start_t HDR __pthread_start_t.h)
 add_header(__pthread_tss_dtor_t HDR __pthread_tss_dtor_t.h)
 add_header(__qsortcompare_t HDR __qsortcompare_t.h)
@@ -38,6 +39,7 @@ add_header(pthread_key_t HDR pthread_key_t.h)
 add_header(pthread_mutex_t HDR pthread_mutex_t.h DEPENDS .__futex_word .__mutex_type)
 add_header(pthread_t HDR pthread_t.h DEPENDS .__thread_type)
 add_header(pthread_mutexattr_t HDR pthread_mutexattr_t.h)
+add_header(pthread_once_t HDR pthread_once_t.h DEPENDS .__futex_word)
 add_header(rlim_t HDR rlim_t.h)
 add_header(struct_rlimit HDR struct_rlimit.h DEPENDS .rlim_t)
 add_header(ssize_t HDR ssize_t.h)
diff --git a/libc/include/llvm-libc-types/__pthread_once_func_t.h b/libc/include/llvm-libc-types/__pthread_once_func_t.h
new file mode 100644
index 000000000000..5ace5cb7f151
--- /dev/null
+++ b/libc/include/llvm-libc-types/__pthread_once_func_t.h
@@ -0,0 +1,14 @@
+//===-- Definition of __pthread_once_func_t type --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_TYPES_PTHREAD_ONCE_FUNC_T_H__
+#define __LLVM_LIBC_TYPES_PTHREAD_ONCE_FUNC_T_H__
+
+typedef void (*__pthread_once_func_t)(void);
+
+#endif // __LLVM_LIBC_TYPES_PTHREAD_ONCE_FUNC_T_H__
diff --git a/libc/include/llvm-libc-types/pthread_once_t.h b/libc/include/llvm-libc-types/pthread_once_t.h
new file mode 100644
index 000000000000..be5c139d0c0e
--- /dev/null
+++ b/libc/include/llvm-libc-types/pthread_once_t.h
@@ -0,0 +1,20 @@
+//===-- Definition of pthread_once_t type ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H__
+#define __LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H__
+
+#include <llvm-libc-types/__futex_word.h>
+
+#ifdef __unix__
+typedef __futex_word pthread_once_t;
+#else
+#error "Once flag type not defined for the target platform."
+#endif
+
+#endif // __LLVM_LIBC_TYPES_PTHREAD_ONCE_T_H__
diff --git a/libc/include/pthread.h.def b/libc/include/pthread.h.def
index 60f257e7078d..18352c053938 100644
--- a/libc/include/pthread.h.def
+++ b/libc/include/pthread.h.def
@@ -13,6 +13,8 @@
 
 #define PTHREAD_STACK_MIN (1 << 14) // 16KB
 
+#define PTHREAD_ONCE_INIT {0}
+
 enum {
   PTHREAD_CREATE_JOINABLE = 0x0,
   PTHREAD_CREATE_DETACHED = 0x1,
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index d93ba16de9fa..131adf96da93 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -14,6 +14,9 @@ def PThreadStartT : NamedType<"__pthread_start_t">;
 def PThreadTSSDtorT : NamedType<"__pthread_tss_dtor_t">;
 def PThreadKeyT : NamedType<"pthread_key_t">;
 def PThreadKeyTPtr : PtrType<PThreadKeyT>;
+def PThreadOnceT : NamedType<"pthread_once_t">;
+def PThreadOnceTPtr : PtrType<PThreadOnceT>;
+def PThreadOnceCallback : NamedType<"__pthread_once_func_t">;
 
 def InoT : NamedType<"ino_t">;
 def UidT : NamedType<"uid_t">;
@@ -675,6 +678,8 @@ def POSIX : StandardSpec<"POSIX"> {
         PThreadKeyT,
         PThreadMutexAttrTType,
         PThreadMutexTType,
+        PThreadOnceCallback,
+        PThreadOnceT,
         PThreadStartT,
         PThreadTSSDtorT,
         PThreadTType,
@@ -861,6 +866,11 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<VoidPtr>,
           [ArgSpec<PThreadKeyT>, ArgSpec<ConstVoidPtr>]
       >,
+      FunctionSpec<
+          "pthread_once",
+          RetValSpec<IntType>,
+          [ArgSpec<PThreadOnceTPtr>, ArgSpec<PThreadOnceCallback>]
+      >,
     ]
   >;
 
diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt
index 413989c5879d..5ef1cd3a5f16 100644
--- a/libc/src/__support/threads/CMakeLists.txt
+++ b/libc/src/__support/threads/CMakeLists.txt
@@ -43,3 +43,17 @@ if(TARGET libc.src.__support.threads.${LIBC_TARGET_OS}.thread)
       libc.src.__support.CPP.optional
   )
 endif()
+
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}/callonce.cpp)
+  add_object_library(
+    callonce
+    SRCS
+      ${LIBC_TARGET_OS}/callonce.cpp
+    HDRS
+      callonce.h
+    DEPENDS
+      libc.include.sys_syscall
+      libc.src.__support.CPP.atomic
+      libc.src.__support.OSUtil.osutil
+  )
+endif()
diff --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h
new file mode 100644
index 000000000000..4ffff2a2c71a
--- /dev/null
+++ b/libc/src/__support/threads/callonce.h
@@ -0,0 +1,16 @@
+//===-- Types related to the callonce function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+namespace __llvm_libc {
+
+struct CallOnceFlag;
+using CallOnceCallback = void(void);
+
+int callonce(CallOnceFlag *flag, CallOnceCallback *callback);
+
+} // namespace __llvm_libc
diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp
new file mode 100644
index 000000000000..1aab6fc9c449
--- /dev/null
+++ b/libc/src/__support/threads/linux/callonce.cpp
@@ -0,0 +1,55 @@
+//===-- Linux implementation of the callonce function ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "futex_word.h"
+
+#include "include/sys/syscall.h" // For syscall numbers.
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/OSUtil/syscall.h" // For syscall functions.
+#include "src/__support/threads/callonce.h"
+
+#include <limits.h>
+#include <linux/futex.h>
+
+namespace __llvm_libc {
+
+static constexpr FutexWordType NOT_CALLED = 0x0;
+static constexpr FutexWordType START = 0x11;
+static constexpr FutexWordType WAITING = 0x22;
+static constexpr FutexWordType FINISH = 0x33;
+
+int callonce(CallOnceFlag *flag, CallOnceCallback *func) {
+  auto *futex_word = reinterpret_cast<cpp::Atomic<FutexWordType> *>(flag);
+
+  FutexWordType not_called = NOT_CALLED;
+
+  // The call_once call can return only after the called function |func|
+  // returns. So, we use futexes to synchronize calls with the same flag value.
+  if (futex_word->compare_exchange_strong(not_called, START)) {
+    func();
+    auto status = futex_word->exchange(FINISH);
+    if (status == WAITING) {
+      __llvm_libc::syscall(SYS_futex, &futex_word->val, FUTEX_WAKE_PRIVATE,
+                           INT_MAX, // Wake all waiters.
+                           0, 0, 0);
+    }
+    return 0;
+  }
+
+  FutexWordType status = START;
+  if (futex_word->compare_exchange_strong(status, WAITING) ||
+      status == WAITING) {
+    __llvm_libc::syscall(SYS_futex, &futex_word->val, FUTEX_WAIT_PRIVATE,
+                         WAITING, // Block only if status is still |WAITING|.
+                         0, 0, 0);
+  }
+
+  return 0;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/pthread/CMakeLists.txt b/libc/src/pthread/CMakeLists.txt
index e55d654e2307..7062ed733679 100644
--- a/libc/src/pthread/CMakeLists.txt
+++ b/libc/src/pthread/CMakeLists.txt
@@ -386,3 +386,14 @@ add_entrypoint_object(
     libc.include.pthread
     libc.src.__support.threads.thread
 )
+
+add_entrypoint_object(
+  pthread_once
+  SRCS
+    pthread_once.cpp
+  HDRS
+    pthread_once.h
+  DEPENDS
+    libc.include.pthread
+    libc.src.__support.threads.callonce
+)
diff --git a/libc/src/pthread/pthread_once.cpp b/libc/src/pthread/pthread_once.cpp
new file mode 100644
index 000000000000..4d141a672036
--- /dev/null
+++ b/libc/src/pthread/pthread_once.cpp
@@ -0,0 +1,23 @@
+//===-- Linux implementation of the pthread_once function -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "pthread_once.h"
+#include "src/__support/common.h"
+#include "src/__support/threads/callonce.h"
+
+#include <pthread.h> // For pthread_once_t and __pthread_once_func_t definitions.
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(int, pthread_once,
+                   (pthread_once_t * flag, __pthread_once_func_t func)) {
+  return callonce(reinterpret_cast<CallOnceFlag *>(flag),
+                  reinterpret_cast<CallOnceCallback *>(func));
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/pthread/pthread_once.h b/libc/src/pthread/pthread_once.h
new file mode 100644
index 000000000000..35e706ebc240
--- /dev/null
+++ b/libc/src/pthread/pthread_once.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for pthread_once function ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_THREADS_PTHREAD_ONCE_H
+#define LLVM_LIBC_SRC_THREADS_PTHREAD_ONCE_H
+
+#include <pthread.h>
+
+namespace __llvm_libc {
+
+int pthread_once(pthread_once_t *flag, __pthread_once_func_t func);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_THREADS_PTHREAD_ONCE_H
diff --git a/libc/src/threads/CMakeLists.txt b/libc/src/threads/CMakeLists.txt
index db011caa5776..b3b22e7f5c73 100644
--- a/libc/src/threads/CMakeLists.txt
+++ b/libc/src/threads/CMakeLists.txt
@@ -4,9 +4,13 @@ endif()
 
 add_entrypoint_object(
   call_once
-  ALIAS
+  SRCS
+    call_once.cpp
+  HDRS
+    call_once.h
   DEPENDS
-    .${LIBC_TARGET_OS}.call_once
+    libc.include.threads
+    libc.src.__support.threads.callonce
 )
 
 add_entrypoint_object(
diff --git a/libc/src/threads/call_once.cpp b/libc/src/threads/call_once.cpp
new file mode 100644
index 000000000000..67ab34509a4c
--- /dev/null
+++ b/libc/src/threads/call_once.cpp
@@ -0,0 +1,23 @@
+//===-- Linux implementation of the call_once function --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/threads/call_once.h"
+#include "src/__support/common.h"
+#include "src/__support/threads/callonce.h"
+
+#include <threads.h> // For once_flag and __call_once_func_t definitions.
+
+namespace __llvm_libc {
+
+LLVM_LIBC_FUNCTION(void, call_once,
+                   (once_flag * flag, __call_once_func_t func)) {
+  callonce(reinterpret_cast<CallOnceFlag *>(flag),
+           reinterpret_cast<CallOnceCallback *>(func));
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/threads/call_once.h b/libc/src/threads/call_once.h
index f6602df68197..c1e1a71da2d5 100644
--- a/libc/src/threads/call_once.h
+++ b/libc/src/threads/call_once.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_THREADS_CALL_ONCE_H
 #define LLVM_LIBC_SRC_THREADS_CALL_ONCE_H
 
-#include "include/threads.h"
+#include <threads.h>
 
 namespace __llvm_libc {
 
diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt
index 339c2b5dc3d5..be5407031aad 100644
--- a/libc/src/threads/linux/CMakeLists.txt
+++ b/libc/src/threads/linux/CMakeLists.txt
@@ -1,17 +1,3 @@
-add_entrypoint_object(
-  call_once
-  SRCS
-    call_once.cpp
-  HDRS
-    ../call_once.h
-  DEPENDS
-    .threads_utils
-    libc.include.sys_syscall
-    libc.include.threads
-    libc.src.__support.CPP.atomic
-    libc.src.__support.OSUtil.osutil
-)
-
 add_header_library(
   threads_utils
   HDRS
diff --git a/libc/test/integration/src/pthread/CMakeLists.txt b/libc/test/integration/src/pthread/CMakeLists.txt
index 31838fb04dfe..7215893d0869 100644
--- a/libc/test/integration/src/pthread/CMakeLists.txt
+++ b/libc/test/integration/src/pthread/CMakeLists.txt
@@ -110,3 +110,23 @@ add_integration_test(
     libc.src.pthread.pthread_getspecific
     libc.src.pthread.pthread_setspecific
 )
+
+add_integration_test(
+  pthread_once_test
+  SUITE
+    libc-pthread-integration-tests
+  SRCS
+    pthread_once_test.cpp
+  LOADER
+    libc.loader.linux.crt1
+  DEPENDS
+    libc.include.pthread
+    libc.src.pthread.pthread_once
+    libc.src.pthread.pthread_mutex_destroy
+    libc.src.pthread.pthread_mutex_init
+    libc.src.pthread.pthread_mutex_lock
+    libc.src.pthread.pthread_mutex_unlock
+    libc.src.pthread.pthread_create
+    libc.src.pthread.pthread_join
+    libc.src.__support.CPP.atomic
+)
diff --git a/libc/test/integration/src/pthread/pthread_once_test.cpp b/libc/test/integration/src/pthread/pthread_once_test.cpp
new file mode 100644
index 000000000000..86ad6496ad5b
--- /dev/null
+++ b/libc/test/integration/src/pthread/pthread_once_test.cpp
@@ -0,0 +1,115 @@
+//===-- Tests for pthread_once --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/CPP/atomic.h"
+#include "src/pthread/pthread_create.h"
+#include "src/pthread/pthread_join.h"
+#include "src/pthread/pthread_mutex_destroy.h"
+#include "src/pthread/pthread_mutex_init.h"
+#include "src/pthread/pthread_mutex_lock.h"
+#include "src/pthread/pthread_mutex_unlock.h"
+#include "src/pthread/pthread_once.h"
+
+#include "utils/IntegrationTest/test.h"
+
+#include <pthread.h>
+
+static constexpr unsigned int NUM_THREADS = 5;
+static __llvm_libc::cpp::Atomic<unsigned int> thread_count;
+
+static unsigned int call_count;
+static void pthread_once_func() { ++call_count; }
+
+static void *func(void *) {
+  static pthread_once_t flag = PTHREAD_ONCE_INIT;
+  __llvm_libc::pthread_once(&flag, pthread_once_func);
+
+  thread_count.fetch_add(1);
+
+  return nullptr;
+}
+
+void call_from_5_threads() {
+  // Ensure the call count and thread count are 0 to begin with.
+  call_count = 0;
+  thread_count = 0;
+
+  pthread_t threads[NUM_THREADS];
+  for (unsigned int i = 0; i < NUM_THREADS; ++i) {
+    ASSERT_EQ(__llvm_libc::pthread_create(threads + i, nullptr, func, nullptr),
+              0);
+  }
+
+  for (unsigned int i = 0; i < NUM_THREADS; ++i) {
+    void *retval;
+    ASSERT_EQ(__llvm_libc::pthread_join(threads[i], &retval), 0);
+    ASSERT_EQ(uintptr_t(retval), uintptr_t(0));
+  }
+
+  EXPECT_EQ(thread_count.val, 5U);
+  EXPECT_EQ(call_count, 1U);
+}
+
+static pthread_mutex_t once_func_blocker;
+static void blocking_once_func() {
+  __llvm_libc::pthread_mutex_lock(&once_func_blocker);
+  __llvm_libc::pthread_mutex_unlock(&once_func_blocker);
+}
+
+static __llvm_libc::cpp::Atomic<unsigned int> start_count;
+static __llvm_libc::cpp::Atomic<unsigned int> done_count;
+static void *once_func_caller(void *) {
+  static pthread_once_t flag;
+  start_count.fetch_add(1);
+  __llvm_libc::pthread_once(&flag, blocking_once_func);
+  done_count.fetch_add(1);
+  return nullptr;
+}
+
+// Test the synchronization aspect of the pthread_once function.
+// This is not a fool proof test, but something which might be
+// useful when we add a flakiness detection scheme to UnitTest.
+void test_synchronization() {
+  start_count = 0;
+  done_count = 0;
+
+  ASSERT_EQ(__llvm_libc::pthread_mutex_init(&once_func_blocker, nullptr), 0);
+  // Lock the blocking mutex so that the once func blocks.
+  ASSERT_EQ(__llvm_libc::pthread_mutex_lock(&once_func_blocker), 0);
+
+  pthread_t t1, t2;
+  ASSERT_EQ(
+      __llvm_libc::pthread_create(&t1, nullptr, once_func_caller, nullptr), 0);
+  ASSERT_EQ(
+      __llvm_libc::pthread_create(&t2, nullptr, once_func_caller, nullptr), 0);
+
+  while (start_count.load() != 2)
+    ; // Spin until both threads start.
+
+  // Since the once func is blocked, the threads should not be done yet.
+  EXPECT_EQ(done_count.val, 0U);
+
+  // Unlock the blocking mutex so that the once func blocks.
+  ASSERT_EQ(__llvm_libc::pthread_mutex_unlock(&once_func_blocker), 0);
+
+  void *retval;
+  ASSERT_EQ(__llvm_libc::pthread_join(t1, &retval), uintptr_t(0));
+  ASSERT_EQ(uintptr_t(retval), 0);
+  ASSERT_EQ(__llvm_libc::pthread_join(t2, &retval), uintptr_t(0));
+  ASSERT_EQ(uintptr_t(retval), 0);
+
+  ASSERT_EQ(done_count.val, 2U);
+
+  __llvm_libc::pthread_mutex_destroy(&once_func_blocker);
+}
+
+TEST_MAIN() {
+  call_from_5_threads();
+  test_synchronization();
+  return 0;
+}