Bug 1774458 - Use undocumented, non-public adaptive spinlocks on macOS 10.15+,...

Bug 1774458 - Use undocumented, non-public adaptive spinlocks on macOS 10.15+, revert to user-space spinlocks on older versions r=pbone

Differential Revision: https://phabricator.services.mozilla.com/D149599

memory/build/Mutex.cpp

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "Mutex.h"
+
+#if defined(XP_DARWIN)
+
+// static
+bool Mutex::UseUnfairLocks() {
+  if (__builtin_available(macOS 10.15, *)) {
+    return true;
+  }
+
+  return false;
+}
+
+// static
+bool Mutex::gFallbackToOSSpinLock = !UseUnfairLocks();
+
+#endif  // defined(XP_DARWIN)

memory/build/Mutex.h

@@ -10,21 +10,48 @@
 #if defined(XP_WIN)
 #  include <windows.h>
 #elif defined(XP_DARWIN)
+#  include <libkern/OSAtomic.h>
 #  include <os/lock.h>
 #else
 #  include <pthread.h>
 #endif
 #include "mozilla/Attributes.h"
 
+#if defined(XP_DARWIN)
+// For information about the following undocumented flags and functions see
+// https://github.com/apple/darwin-xnu/blob/main/bsd/sys/ulock.h and
+// https://github.com/apple/darwin-libplatform/blob/main/private/os/lock_private.h
+#  define OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION (0x00010000)
+#  define OS_UNFAIR_LOCK_ADAPTIVE_SPIN (0x00040000)
+
+extern "C" {
+
+typedef uint32_t os_unfair_lock_options_t;
+OS_UNFAIR_LOCK_AVAILABILITY
+OS_EXPORT OS_NOTHROW OS_NONNULL_ALL void os_unfair_lock_lock_with_options(
+    os_unfair_lock_t lock, os_unfair_lock_options_t options);
+}
+
+static_assert(OS_UNFAIR_LOCK_INIT._os_unfair_lock_opaque == OS_SPINLOCK_INIT,
+              "OS_UNFAIR_LOCK_INIT and OS_SPINLOCK_INIT have the same "
+              "value");
+static_assert(sizeof(os_unfair_lock) == sizeof(OSSpinLock),
+              "os_unfair_lock and OSSpinLock are the same size");
+#endif  // defined(XP_DARWIN)
+
 // Mutexes based on spinlocks.  We can't use normal pthread spinlocks in all
-// places, because they require malloc()ed memory, which causes bootstrapping
-// issues in some cases.  We also can't use constructors, because for statics,
-// they would fire after the first use of malloc, resetting the locks.
+// places, because they require malloc()ed memory, which causes
+// bootstrapping issues in some cases.  We also can't use constructors,
+// because for statics, they would fire after the first use of malloc,
+// resetting the locks.
 struct Mutex {
 #if defined(XP_WIN)
   CRITICAL_SECTION mMutex;
 #elif defined(XP_DARWIN)
-  os_unfair_lock mMutex;
+  union {
+    os_unfair_lock mUnfairLock;
+    OSSpinLock mSpinLock;
+  } mMutex;
 #else
   pthread_mutex_t mMutex;
 #endif
@@ -36,7 +63,10 @@ struct Mutex {
       return false;
     }
 #elif defined(XP_DARWIN)
-    mMutex = OS_UNFAIR_LOCK_INIT;
+    // The hack below works because both OS_UNFAIR_LOCK_INIT and
+    // OS_SPINLOCK_INIT initialize the lock to 0 and in both case it's a 32-bit
+    // integer.
+    mMutex.mUnfairLock = OS_UNFAIR_LOCK_INIT;
 #elif defined(XP_LINUX) && !defined(ANDROID)
     pthread_mutexattr_t attr;
     if (pthread_mutexattr_init(&attr) != 0) {
@@ -60,7 +90,20 @@ struct Mutex {
 #if defined(XP_WIN)
     EnterCriticalSection(&mMutex);
 #elif defined(XP_DARWIN)
-    os_unfair_lock_lock(&mMutex);
+    if (Mutex::gFallbackToOSSpinLock) {
+      OSSpinLockLock(&mMutex.mSpinLock);
+    } else {
+      // We rely on a non-public function to improve performance here.
+      // The OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION flag informs the kernel that
+      // the calling thread is able to make progress even in absence of actions
+      // from other threads and the OS_UNFAIR_LOCK_ADAPTIVE_SPIN one causes the
+      // kernel to spin on a contested lock if the owning thread is running on
+      // the same physical core (presumably only on x86 CPUs given that ARM
+      // macs don't have cores capable of SMT).
+      os_unfair_lock_lock_with_options(
+          &mMutex.mUnfairLock,
+          OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION | OS_UNFAIR_LOCK_ADAPTIVE_SPIN);
+    }
 #else
     pthread_mutex_lock(&mMutex);
 #endif
@@ -70,11 +113,20 @@ struct Mutex {
 #if defined(XP_WIN)
     LeaveCriticalSection(&mMutex);
 #elif defined(XP_DARWIN)
-    os_unfair_lock_unlock(&mMutex);
+    if (Mutex::gFallbackToOSSpinLock) {
+      OSSpinLockUnlock(&mMutex.mSpinLock);
+    } else {
+      os_unfair_lock_unlock(&mMutex.mUnfairLock);
+    }
 #else
     pthread_mutex_unlock(&mMutex);
 #endif
   }
+
+#if defined(XP_DARWIN)
+  static bool UseUnfairLocks();
+  static bool gFallbackToOSSpinLock;
+#endif  // XP_DARWIN
 };
 
 // Mutex that can be used for static initialization.
@@ -101,7 +153,10 @@ struct StaticMutex {
 typedef Mutex StaticMutex;
 
 #  if defined(XP_DARWIN)
-#    define STATIC_MUTEX_INIT OS_UNFAIR_LOCK_INIT
+// The hack below works because both OS_UNFAIR_LOCK_INIT and OS_SPINLOCK_INIT
+// initialize the lock to 0 and in both case it's a 32-bit integer.
+#    define STATIC_MUTEX_INIT \
+      { .mUnfairLock = OS_UNFAIR_LOCK_INIT }
 #  elif defined(XP_LINUX) && !defined(ANDROID)
 #    define STATIC_MUTEX_INIT PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
 #  else

memory/build/moz.build

@@ -37,6 +37,7 @@ if CONFIG["OS_TARGET"] == "Darwin" and (
     CONFIG["MOZ_REPLACE_MALLOC"] or CONFIG["MOZ_MEMORY"]
 ):
     SOURCES += [
+        "Mutex.cpp",
         "zone.c",
     ]