Bug 1684837 - Adjust pointer lock request handling; r=smaug, a=jcristau

Differential Revision: https://phabricator.services.mozilla.com/D100960

dom/base/Document.cpp

@@ -14664,64 +14664,130 @@ static void ChangePointerLockedElement(Element* aElement, Document* aDocument,
   DispatchPointerLockChange(aDocument);
 }
-MOZ_CAN_RUN_SCRIPT_BOUNDARY static void StartSetPointerLock(
-    Element* aElement, Document* aDocument, bool aUserInputOrChromeCaller) {
+MOZ_CAN_RUN_SCRIPT_BOUNDARY static bool StartSetPointerLock(
+    Element* aElement, Document* aDocument) {
+  if (!aDocument->SetPointerLock(aElement, StyleCursorKind::None)) {
+    DispatchPointerLockError(aDocument, "PointerLockDeniedFailedToLock");
+    return false;
+  }
+
+  ChangePointerLockedElement(aElement, aDocument, nullptr);
+  nsContentUtils::DispatchEventOnlyToChrome(
+      aDocument, ToSupports(aElement), u"MozDOMPointerLock:Entered"_ns,
+      CanBubble::eYes, Cancelable::eNo, /* DefaultAction */ nullptr);
+
+  return true;
+}
+
+class PointerLockRequest final : public Runnable {
+ public:
+  PointerLockRequest(Element* aElement, bool aUserInputOrChromeCaller)
+      : mozilla::Runnable("PointerLockRequest"),
+        mElement(do_GetWeakReference(aElement)),
+        mDocument(do_GetWeakReference(aElement->OwnerDoc())),
+        mUserInputOrChromeCaller(aUserInputOrChromeCaller) {}
+
+  NS_IMETHOD Run() final {
+    nsCOMPtr<Element> element = do_QueryReferent(mElement);
+    nsCOMPtr<Document> document = do_QueryReferent(mDocument);
+
     const char* error = nullptr;
-  if (!aElement || !aDocument || !aElement->GetComposedDoc()) {
+    if (!element || !document || !element->GetComposedDoc()) {
       error = "PointerLockDeniedNotInDocument";
-  } else if (aElement->GetComposedDoc() != aDocument) {
+    } else if (element->GetComposedDoc() != document) {
       error = "PointerLockDeniedMovedDocument";
     }
     if (!error) {
       nsCOMPtr<Element> pointerLockedElement =
           do_QueryReferent(EventStateManager::sPointerLockedElement);
-    if (aElement == pointerLockedElement) {
-      DispatchPointerLockChange(aDocument);
-      return;
+      if (element == pointerLockedElement) {
+        DispatchPointerLockChange(document);
+        return NS_OK;
       }
       // Note, we must bypass focus change, so pass true as the last parameter!
-    error = GetPointerLockError(aElement, pointerLockedElement, true);
+      error = GetPointerLockError(element, pointerLockedElement, true);
       // Another element in the same document is requesting pointer lock,
       // just grant it without user input check.
       if (!error && pointerLockedElement) {
-      ChangePointerLockedElement(aElement, aDocument, pointerLockedElement);
-      return;
+        ChangePointerLockedElement(element, document, pointerLockedElement);
+        return NS_OK;
       }
     }
     // If it is neither user input initiated, nor requested in fullscreen,
     // it should be rejected.
-  if (!error && !aUserInputOrChromeCaller &&
-      !aDocument->GetUnretargetedFullScreenElement()) {
+    if (!error && !mUserInputOrChromeCaller &&
+        !document->GetUnretargetedFullScreenElement()) {
       error = "PointerLockDeniedNotInputDriven";
     }
-  if (!error && !aDocument->SetPointerLock(aElement, StyleCursorKind::None)) {
-    error = "PointerLockDeniedFailedToLock";
+
+    if (error) {
+      DispatchPointerLockError(document, error);
+      return NS_OK;
     }
+
+    if (BrowserChild* browserChild =
+            BrowserChild::GetFrom(document->GetDocShell())) {
+      nsWeakPtr e = do_GetWeakReference(element);
+      nsWeakPtr doc = do_GetWeakReference(element->OwnerDoc());
+      nsWeakPtr bc = do_GetWeakReference(browserChild);
+      browserChild->SendRequestPointerLock(
+          [e, doc, bc](const nsCString& aError) {
+            nsCOMPtr<Document> document = do_QueryReferent(doc);
+            if (!aError.IsEmpty()) {
+              DispatchPointerLockError(document, aError.get());
+              return;
+            }
+
+            const char* error = nullptr;
+            auto autoCleanup = MakeScopeExit([&] {
               if (error) {
-    DispatchPointerLockError(aDocument, error);
+                DispatchPointerLockError(document, error);
+                // If we are failed to set pointer lock, notify parent to stop
+                // redirect mouse event to this process.
+                if (nsCOMPtr<nsIBrowserChild> browserChild =
+                        do_QueryReferent(bc)) {
+                  static_cast<BrowserChild*>(browserChild.get())
+                      ->SendReleasePointerLock();
+                }
+              }
+            });
+
+            nsCOMPtr<Element> element = do_QueryReferent(e);
+            if (!element || !document || !element->GetComposedDoc()) {
+              error = "PointerLockDeniedNotInDocument";
               return;
             }
-  ChangePointerLockedElement(aElement, aDocument, nullptr);
-  nsContentUtils::DispatchEventOnlyToChrome(
-      aDocument, ToSupports(aElement), u"MozDOMPointerLock:Entered"_ns,
-      CanBubble::eYes, Cancelable::eNo, /* DefaultAction */ nullptr);
-}
-class PointerLockRequest final : public Runnable {
- public:
-  PointerLockRequest(Element* aElement, bool aUserInputOrChromeCaller)
-      : mozilla::Runnable("PointerLockRequest"),
-        mElement(do_GetWeakReference(aElement)),
-        mDocument(do_GetWeakReference(aElement->OwnerDoc())),
-        mUserInputOrChromeCaller(aUserInputOrChromeCaller) {
-    MOZ_ASSERT(XRE_IsParentProcess());
+            if (element->GetComposedDoc() != document) {
+              error = "PointerLockDeniedMovedDocument";
+              return;
+            }
+            nsCOMPtr<Element> pointerLockedElement =
+                do_QueryReferent(EventStateManager::sPointerLockedElement);
+            error = GetPointerLockError(element, pointerLockedElement, true);
+            if (error) {
+              return;
+            }
+
+            if (!StartSetPointerLock(element, document)) {
+              error = "PointerLockDeniedFailedToLock";
+              return;
+            }
+          },
+          [doc](mozilla::ipc::ResponseRejectReason) {
+            // IPC layer error
+            nsCOMPtr<Document> document = do_QueryReferent(doc);
+            if (!document) {
+              return;
+            }
+
+            DispatchPointerLockError(document, "PointerLockDeniedFailedToLock");
+          });
+    } else {
+      StartSetPointerLock(element, document);
     }
-  NS_IMETHOD Run() final {
-    nsCOMPtr<Element> element = do_QueryReferent(mElement);
-    nsCOMPtr<Document> document = do_QueryReferent(mDocument);
-    StartSetPointerLock(element, document, mUserInputOrChromeCaller);
     return NS_OK;
   };
@@ -14749,30 +14815,9 @@ void Document::RequestPointerLock(Element* aElement, CallerType aCallerType) {
   bool userInputOrSystemCaller = HasValidTransientUserGestureActivation() ||
                                  aCallerType == CallerType::System;
-  if (BrowserChild* browserChild = BrowserChild::GetFrom(GetDocShell())) {
-    nsWeakPtr e = do_GetWeakReference(aElement);
-    nsWeakPtr doc = do_GetWeakReference(aElement->OwnerDoc());
-    browserChild->SendRequestPointerLock(
-        [e, doc, userInputOrSystemCaller](const nsCString& aError) {
-          nsCOMPtr<Document> document = do_QueryReferent(doc);
-          if (!aError.IsEmpty()) {
-            DispatchPointerLockError(document, aError.get());
-            return;
-          }
-
-          nsCOMPtr<Element> element = do_QueryReferent(e);
-          StartSetPointerLock(element, document, userInputOrSystemCaller);
-        },
-        [doc](mozilla::ipc::ResponseRejectReason) {
-          // IPC layer error
-          nsCOMPtr<Document> document = do_QueryReferent(doc);
-          DispatchPointerLockError(document, "PointerLockDeniedFailedToLock");
-        });
-  } else {
   nsCOMPtr<nsIRunnable> request =
       new PointerLockRequest(aElement, userInputOrSystemCaller);
   Dispatch(TaskCategory::Other, request.forget());
-  }
 }
 bool Document::SetPointerLock(Element* aElement, StyleCursorKind aCursorStyle) {