Bug 1400940 - Fix WebAuthn deadlock when cancelling a request on tab switch r=jcj

This should be an easy solution. We can't stop the sign() or register()
runloop from calling the callback, so we need the callback to simply return
early when the U2FHIDTokenManager shuts down.

Bug #: 1400940

Differential Revision: https://phabricator.services.mozilla.com/D67

dom/webauthn/U2FHIDTokenManager.cpp

@@ -18,7 +18,7 @@ static void
 u2f_register_callback(uint64_t aTransactionId, rust_u2f_result* aResult)
 {
   StaticMutexAutoLock lock(gInstanceMutex);
-  if (NS_WARN_IF(!gInstance || !gPBackgroundThread)) {
+  if (!gInstance || NS_WARN_IF(!gPBackgroundThread)) {
     return;
   }
 
@@ -35,7 +35,7 @@ static void
 u2f_sign_callback(uint64_t aTransactionId, rust_u2f_result* aResult)
 {
   StaticMutexAutoLock lock(gInstanceMutex);
-  if (NS_WARN_IF(!gInstance || !gPBackgroundThread)) {
+  if (!gInstance || NS_WARN_IF(!gPBackgroundThread)) {
     return;
   }
 
@@ -63,15 +63,21 @@ U2FHIDTokenManager::U2FHIDTokenManager() : mTransactionId(0)
 
 U2FHIDTokenManager::~U2FHIDTokenManager()
 {
-  StaticMutexAutoLock lock(gInstanceMutex);
-  MOZ_ASSERT(NS_GetCurrentThread() == gPBackgroundThread);
+  {
+    StaticMutexAutoLock lock(gInstanceMutex);
+    MOZ_ASSERT(NS_GetCurrentThread() == gPBackgroundThread);
 
-  mRegisterPromise.RejectIfExists(NS_ERROR_DOM_UNKNOWN_ERR, __func__);
-  mSignPromise.RejectIfExists(NS_ERROR_DOM_UNKNOWN_ERR, __func__);
+    mRegisterPromise.RejectIfExists(NS_ERROR_DOM_UNKNOWN_ERR, __func__);
+    mSignPromise.RejectIfExists(NS_ERROR_DOM_UNKNOWN_ERR, __func__);
+
+    gInstance = nullptr;
+  }
 
+  // Release gInstanceMutex before we call U2FManager::drop(). It will wait
+  // for the work queue thread to join, and that requires the
+  // u2f_{register,sign}_callback to lock and return.
   rust_u2f_mgr_free(mU2FManager);
   mU2FManager = nullptr;
-  gInstance = nullptr;
 }
 
 // A U2F Register operation causes a new key pair to be generated by the token.