Verified Commit a4f37bbf authored by Gabriele Svelto's avatar Gabriele Svelto Committed by Pier Angelo Vendrame
Browse files

Bug 1774458 - Use undocumented, non-public adaptive spinlocks on macOS 10.15+,... 

Bug 1774458 - Use undocumented, non-public adaptive spinlocks on macOS 10.15+, revert to user-space spinlocks on older versions r=pbone

Differential Revision: https://phabricator.services.mozilla.com/D149599
parent 39e4cc51

memory/build/Mutex.cpp

0 → 100644
+21 −0
Original line number Diff line number Diff line 
/* This Source Code Form is subject to the terms of the Mozilla Public

 * License, v. 2.0. If a copy of the MPL was not distributed with this

 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */



#include "Mutex.h"



#if defined(XP_DARWIN)



// static

bool Mutex::UseUnfairLocks() {

  if (__builtin_available(macOS 10.15, *)) {

    return true;

  }



  return false;

}



// static

bool Mutex::gFallbackToOSSpinLock = !UseUnfairLocks();



#endif  // defined(XP_DARWIN)

memory/build/Mutex.h

+63 −8
Original line number Diff line number Diff line
@@ -10,21 +10,48 @@ 
#if defined(XP_WIN)

#  include <windows.h>

#elif defined(XP_DARWIN)

#  include <libkern/OSAtomic.h>

#  include <os/lock.h>

#else

#  include <pthread.h>

#endif

#include "mozilla/Attributes.h"



#if defined(XP_DARWIN)

// For information about the following undocumented flags and functions see

// https://github.com/apple/darwin-xnu/blob/main/bsd/sys/ulock.h and

// https://github.com/apple/darwin-libplatform/blob/main/private/os/lock_private.h

#  define OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION (0x00010000)

#  define OS_UNFAIR_LOCK_ADAPTIVE_SPIN (0x00040000)



extern "C" {



typedef uint32_t os_unfair_lock_options_t;

OS_UNFAIR_LOCK_AVAILABILITY

OS_EXPORT OS_NOTHROW OS_NONNULL_ALL void os_unfair_lock_lock_with_options(

    os_unfair_lock_t lock, os_unfair_lock_options_t options);

}



static_assert(OS_UNFAIR_LOCK_INIT._os_unfair_lock_opaque == OS_SPINLOCK_INIT,

              "OS_UNFAIR_LOCK_INIT and OS_SPINLOCK_INIT have the same "

              "value");

static_assert(sizeof(os_unfair_lock) == sizeof(OSSpinLock),

              "os_unfair_lock and OSSpinLock are the same size");

#endif  // defined(XP_DARWIN)



// Mutexes based on spinlocks.  We can't use normal pthread spinlocks in all

// places, because they require malloc()ed memory, which causes bootstrapping

// issues in some cases.  We also can't use constructors, because for statics,

// they would fire after the first use of malloc, resetting the locks.

// places, because they require malloc()ed memory, which causes

// bootstrapping issues in some cases.  We also can't use constructors,

// because for statics, they would fire after the first use of malloc,

// resetting the locks.

struct Mutex {

#if defined(XP_WIN)

  CRITICAL_SECTION mMutex;

#elif defined(XP_DARWIN)

  os_unfair_lock mMutex;

  union {

    os_unfair_lock mUnfairLock;

    OSSpinLock mSpinLock;

  } mMutex;

#else

  pthread_mutex_t mMutex;

#endif
@@ -36,7 +63,10 @@ struct Mutex { 
      return false;

    }

#elif defined(XP_DARWIN)

    mMutex = OS_UNFAIR_LOCK_INIT;

    // The hack below works because both OS_UNFAIR_LOCK_INIT and

    // OS_SPINLOCK_INIT initialize the lock to 0 and in both case it's a 32-bit

    // integer.

    mMutex.mUnfairLock = OS_UNFAIR_LOCK_INIT;

#elif defined(XP_LINUX) && !defined(ANDROID)

    pthread_mutexattr_t attr;

    if (pthread_mutexattr_init(&attr) != 0) {
@@ -60,7 +90,20 @@ struct Mutex { 
#if defined(XP_WIN)

    EnterCriticalSection(&mMutex);

#elif defined(XP_DARWIN)

    os_unfair_lock_lock(&mMutex);

    if (Mutex::gFallbackToOSSpinLock) {

      OSSpinLockLock(&mMutex.mSpinLock);

    } else {

      // We rely on a non-public function to improve performance here.

      // The OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION flag informs the kernel that

      // the calling thread is able to make progress even in absence of actions

      // from other threads and the OS_UNFAIR_LOCK_ADAPTIVE_SPIN one causes the

      // kernel to spin on a contested lock if the owning thread is running on

      // the same physical core (presumably only on x86 CPUs given that ARM

      // macs don't have cores capable of SMT).

      os_unfair_lock_lock_with_options(

          &mMutex.mUnfairLock,

          OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION | OS_UNFAIR_LOCK_ADAPTIVE_SPIN);

    }

#else

    pthread_mutex_lock(&mMutex);

#endif
@@ -70,11 +113,20 @@ struct Mutex { 
#if defined(XP_WIN)

    LeaveCriticalSection(&mMutex);

#elif defined(XP_DARWIN)

    os_unfair_lock_unlock(&mMutex);

    if (Mutex::gFallbackToOSSpinLock) {

      OSSpinLockUnlock(&mMutex.mSpinLock);

    } else {

      os_unfair_lock_unlock(&mMutex.mUnfairLock);

    }

#else

    pthread_mutex_unlock(&mMutex);

#endif

  }



#if defined(XP_DARWIN)

  static bool UseUnfairLocks();

  static bool gFallbackToOSSpinLock;

#endif  // XP_DARWIN

};



// Mutex that can be used for static initialization.
@@ -101,7 +153,10 @@ struct StaticMutex { 
typedef Mutex StaticMutex;



#  if defined(XP_DARWIN)

#    define STATIC_MUTEX_INIT OS_UNFAIR_LOCK_INIT

// The hack below works because both OS_UNFAIR_LOCK_INIT and OS_SPINLOCK_INIT

// initialize the lock to 0 and in both case it's a 32-bit integer.

#    define STATIC_MUTEX_INIT \

      { .mUnfairLock = OS_UNFAIR_LOCK_INIT }

#  elif defined(XP_LINUX) && !defined(ANDROID)

#    define STATIC_MUTEX_INIT PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP

#  else

memory/build/moz.build

+1 −0
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ if CONFIG["OS_TARGET"] == "Darwin" and ( 
    CONFIG["MOZ_REPLACE_MALLOC"] or CONFIG["MOZ_MEMORY"]

):

    SOURCES += [

        "Mutex.cpp",

        "zone.c",

    ]