Bug 1782579 - Pass languages to the text recognition api; r=nordzilla,emilio (3c686461) · Commits · The Tor Project / Applications / Mullvad Browser

dom/base/nsImageLoadingContent.cpp

+92 −52

Original line number	Diff line number	Diff line
		@@ -52,6 +52,8 @@
		#include "mozilla/dom/ImageTextBinding.h"
		#include "mozilla/dom/ImageTracker.h"
		#include "mozilla/dom/ScriptSettings.h"
		#include "mozilla/intl/LocaleService.h"
		#include "mozilla/intl/Locale.h"
		#include "mozilla/net/UrlClassifierFeatureFactory.h"
		#include "mozilla/widget/TextRecognition.h"

		@@ -1232,13 +1234,49 @@ already_AddRefed<Promise> nsImageLoadingContent::RecognizeCurrentImageText(
		return nullptr;
		}

		TextRecognition::FindText(*image)->Then(
		// The list of ISO 639-1 language tags to pass to the text recognition API.
		AutoTArray<nsCString, 4> languages;
		{
		// The document's locale should be the top language to use. Parse the BCP 47
		// locale and extract the ISO 639-1 language tag. e.g. "en-US" -> "en".
		nsAutoCString elementLanguage;
		nsAtom* imgLanguage = AsContent()->GetLang();
		intl::Locale locale;
		if (imgLanguage) {
		imgLanguage->ToUTF8String(elementLanguage);
		auto result = intl::LocaleParser::TryParse(elementLanguage, locale);
		if (result.isOk()) {
		languages.AppendElement(locale.Language().Span());
		}
		}
		}

		{
		// The app locales should also be included after the document's locales.
		// Extract the language tag like above.
		nsTArray<nsCString> appLocales;
		intl::LocaleService::GetInstance()->GetAppLocalesAsBCP47(appLocales);

		for (const auto& localeString : appLocales) {
		intl::Locale locale;
		auto result = intl::LocaleParser::TryParse(localeString, locale);
		if (result.isErr()) {
		NS_WARNING("Could not parse an app locale string, ignoring it.");
		continue;
		}
		languages.AppendElement(locale.Language().Span());
		}
		}

		TextRecognition::FindText(*image, languages)
		->Then(
		GetCurrentSerialEventTarget(), __func__,
		[weak = RefPtr{do_GetWeakReference(this)},
		request = RefPtr{mCurrentRequest}, domPromise](
		TextRecognition::NativePromise::ResolveOrRejectValue&& aValue) {
		if (aValue.IsReject()) {
		domPromise->MaybeRejectWithNotSupportedError(aValue.RejectValue());
		domPromise->MaybeRejectWithNotSupportedError(
		aValue.RejectValue());
		return;
		}
		RefPtr<nsIImageLoadingContent> iilc = do_QueryReferent(weak.get());
		@@ -1249,14 +1287,15 @@ already_AddRefed<Promise> nsImageLoadingContent::RecognizeCurrentImageText(
		}
		auto* ilc = static_cast<nsImageLoadingContent*>(iilc.get());
		if (ilc->mCurrentRequest != request) {
		domPromise->MaybeRejectWithInvalidStateError("Request not current");
		domPromise->MaybeRejectWithInvalidStateError(
		"Request not current");
		return;
		}
		auto& textRecognitionResult = aValue.ResolveValue();
		Element* el = ilc->AsContent()->AsElement();

		// When enabled, this feature will place the recognized text as spans
		// inside of the shadow dom of the img element. These are then
		// When enabled, this feature will place the recognized text as
		// spans inside of the shadow dom of the img element. These are then
		// positioned so that the user can select the text.
		if (Preferences::GetBool("dom.text-recognition.shadow-dom-enabled",
		false)) {
		@@ -1266,7 +1305,8 @@ already_AddRefed<Promise> nsImageLoadingContent::RecognizeCurrentImageText(
		el->NotifyUAWidgetSetupOrChange();
		}

		nsTArray<ImageText> imageTexts(textRecognitionResult.quads().Length());
		nsTArray<ImageText> imageTexts(
		textRecognitionResult.quads().Length());
		nsIGlobalObject* global = el->OwnerDoc()->GetOwnerGlobal();

		for (const auto& quad : textRecognitionResult.quads()) {

dom/ipc/ContentParent.cpp

+13 −11

Original line number	Diff line number	Diff line
		@@ -4965,14 +4965,16 @@ mozilla::ipc::IPCResult ContentParent::RecvCopyFavicon(
		}

		mozilla::ipc::IPCResult ContentParent::RecvFindImageText(
		ShmemImage&& aImage, FindImageTextResolver&& aResolver) {
		ShmemImage&& aImage, nsTArray<nsCString>&& aLanguages,
		FindImageTextResolver&& aResolver) {
		RefPtr<DataSourceSurface> surf =
		nsContentUtils::IPCImageToSurface(std::move(aImage), this);
		if (!surf) {
		aResolver(TextRecognitionResultOrError("Failed to read image"_ns));
		return IPC_OK();
		}
		TextRecognition::FindText(*surf)->Then(
		TextRecognition::FindText(*surf, aLanguages)
		->Then(
		GetCurrentSerialEventTarget(), __func__,
		[resolver = std::move(aResolver)](
		TextRecognition::NativePromise::ResolveOrRejectValue&& aValue) {

dom/ipc/ContentParent.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -1110,7 +1110,7 @@ class ContentParent final : public PContentParent,
		mozilla::ipc::IPCResult RecvCopyFavicon(nsIURI* aOldURI, nsIURI* aNewURI,
		const bool& aInPrivateBrowsing);

		mozilla::ipc::IPCResult RecvFindImageText(ShmemImage&&,
		mozilla::ipc::IPCResult RecvFindImageText(ShmemImage&&, nsTArray<nsCString>&&,
		FindImageTextResolver&&);

		virtual void ProcessingError(Result aCode, const char* aMsgName) override;

dom/ipc/PContent.ipdl

+2 −1

Original line number	Diff line number	Diff line
		@@ -1237,7 +1237,8 @@ parent:

		async CopyFavicon(nsIURI oldURI, nsIURI newURI, bool isPrivate);

		async FindImageText(ShmemImage image) returns (TextRecognitionResultOrError result);
		async FindImageText(ShmemImage image, nsCString[] languages)
		returns (TextRecognitionResultOrError result);

		/**
		* Notifies the parent about a recording device is starting or shutdown.

toolkit/content/tests/widgets/chrome.ini

+5 −0

Original line number	Diff line number	Diff line
		@@ -2,6 +2,7 @@
		skip-if = os == 'android'
		support-files =
		image.png
		image-zh.png
		tree_shared.js
		popup_shared.js
		window_label_checkbox.xhtml
		@@ -15,6 +16,10 @@ skip-if = os == 'linux' # Bug 1115088
		[test_editor_currentURI.xhtml]
		[test_image_recognition.html]
		run-if = os == "mac" # Mac only feature.
		[test_image_recognition_unsupported.html]
		skip-if = os == 'mac'
		[test_image_recognition_zh.html]
		run-if = os == "mac" && os_version != "10.15" # Mac only feature, requires > 10.15 to support multilingual results.
		[test_label_checkbox.xhtml]
		[test_menubar.xhtml]
		skip-if = os == 'mac'