Bug 1752183 - Add support for downloading and presenting remote images r=Mardak (551bbc49) · Commits · clairehurst / Tor Browser

browser/components/newtab/lib/RemoteImages.jsm

0 → 100644

+227 −0

Original line number	Diff line number	Diff line
		/* This Source Code Form is subject to the terms of the Mozilla Public
		* License, v. 2.0. If a copy of the MPL was not distributed with this
		* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
		"use strict";

		ChromeUtils.defineModuleGetter(
		this,
		"Downloader",
		"resource://services-settings/Attachments.jsm"
		);

		ChromeUtils.defineModuleGetter(
		this,
		"KintoHttpClient",
		"resource://services-common/kinto-http-client.js"
		);

		ChromeUtils.defineModuleGetter(
		this,
		"Services",
		"resource://gre/modules/Services.jsm"
		);

		const RS_SERVER_PREF = "services.settings.server";
		const RS_MAIN_BUCKET = "main";
		const RS_COLLECTION = "ms-images";
		const RS_DOWNLOAD_MAX_RETRIES = 2;

		const CLEANUP_FINISHED_TOPIC = "remote-images:cleanup-finished";

		const IMAGE_EXPIRY_DURATION = 30 * 24 * 60 * 60; // 30 days in seconds.

		const EXTENSIONS = new Map([
		["avif", "image/avif"],
		["png", "image/png"],
		["svg", "image/svg+xml"],
		]);

		class _RemoteImages {
		#cleaningUp;

		constructor() {
		this.#cleaningUp = false;

		// Piggy back off of RemoteSettings timer, triggering image cleanup every
		// 24h.
		Services.obs.addObserver(
		() => this.#cleanup(),
		"remote-settings:changes-poll-end"
		);
		}

		/**
		* The directory where images are cached.
		*/
		get imagesDir() {
		const value = PathUtils.join(
		PathUtils.localProfileDir,
		"settings",
		RS_MAIN_BUCKET,
		RS_COLLECTION
		);

		Object.defineProperty(this, "imagesDir", { value });
		return value;
		}

		/**
		* Load a remote image.
		*
		* If the image has not been previously downloaded then the image will be
		* downloaded from RemoteSettings.
		*
		* @param imageId The unique image ID.
		*
		* @returns A promise that resolves with a blob URL for the given image, or
		* rejects with an error.
		*
		* After the caller is finished with the image, they must call
		* \|RemoteImages.unload()\| on the returned URL.
		*/
		async load(imageId) {
		let blob;
		try {
		blob = await this.#readFromDisk(imageId);
		} catch (e) {
		if (
		!(
		e instanceof Components.Exception &&
		e.name === "NS_ERROR_FILE_NOT_FOUND"
		)
		) {
		throw e;
		}

		// If we have not downloaded the file then download the file from
		// RemoteSettings.
		blob = await this.#download(imageId);
		}

		return URL.createObjectURL(blob);
		}

		/**
		* Unload a URL returned by RemoteImages
		*
		* @param url The URL to unload.
		**/
		unload(url) {
		URL.revokeObjectURL(url);
		}

		/**
		* Clean up all files that haven't been touched in 30d.
		*/
		async #cleanup() {
		// The observer service doesn't await observers, so guard against re-entry.
		if (this.#cleaningUp) {
		return;
		}
		this.#cleaningUp = true;

		try {
		const now = Date.now();
		const children = await IOUtils.getChildren(this.imagesDir);
		for (const child of children) {
		const stat = await IOUtils.stat(child);

		if (now - stat.lastModified >= IMAGE_EXPIRY_DURATION) {
		await IOUtils.remove(child);
		}
		}
		} finally {
		this.#cleaningUp = false;
		Services.obs.notifyObservers(null, CLEANUP_FINISHED_TOPIC);
		}
		}

		/**
		* Return the record ID and mimetype given an imageId.
		*
		* Remote Settings does not support records with periods in their names, but
		* we use a full filename (e.g., `${recordId}.png`) for image IDs so that we
		* don't also have to carry the mimetype as a separate field and can instead
		* infer it from the imageId.
		*
		* @param imageId The image ID, including its extension.
		*
		* @returns An object containing the \|recordId\| and the \|mimetype\|
		*/
		#getRecordIdAndMimetype(imageId) {
		const idx = imageId.lastIndexOf(".");
		if (idx === -1) {
		throw new TypeError("imageId must include extension");
		}

		const recordId = imageId.substring(0, idx);
		const ext = imageId.substring(idx + 1);
		const mimetype = EXTENSIONS.get(ext);

		if (!mimetype) {
		throw new TypeError(
		`Unsupported extension '.${ext}' for remote image ${imageId}`
		);
		}

		return { recordId, mimetype };
		}

		/**
		* Read the image from disk
		*
		* @param imageId The image ID.
		*
		* @returns A promise that resolves to a blob, or rejects with an Error.
		*/
		async #readFromDisk(imageId) {
		const { mimetype } = this.#getRecordIdAndMimetype(imageId);
		const path = PathUtils.join(this.imagesDir, imageId);
		const blob = await File.createFromFileName(path, { type: mimetype });

		// Update the modification time so that we don't cleanup this image.
		await IOUtils.setModificationTime(path);

		return blob;
		}

		/**
		* Download an image from RemoteSettings.
		*
		* @param imageId The image ID.
		*
		* @returns A promise that resolves with a Blob of the image data or rejects
		* with an Error.
		*/
		async #download(imageId) {
		const client = new KintoHttpClient(
		Services.prefs.getStringPref(RS_SERVER_PREF)
		);

		const { recordId } = this.#getRecordIdAndMimetype(imageId);
		const record = await client
		.bucket(RS_MAIN_BUCKET)
		.collection(RS_COLLECTION)
		.getRecord(recordId);

		const downloader = new Downloader(RS_MAIN_BUCKET, RS_COLLECTION);

		const arrayBuffer = await downloader.downloadAsBytes(record.data, {
		retries: RS_DOWNLOAD_MAX_RETRIES,
		});

		// Cache to disk.
		const path = PathUtils.join(this.imagesDir, imageId);

		// We do not await this promise because any other attempt to interact with
		// the file via IOUtils will have to synchronize via the IOUtils event queue
		// anyway.
		IOUtils.write(path, new Uint8Array(arrayBuffer));

		return new Blob([arrayBuffer], { type: record.data.attachment.mimetype });
		}
		}

		const RemoteImages = new _RemoteImages();

		const EXPORTED_SYMBOLS = ["RemoteImages"];

browser/components/newtab/moz.build

+4 −0

Original line number	Diff line number	Diff line
		@@ -12,6 +12,10 @@ BROWSER_CHROME_MANIFESTS += [
		"test/browser/browser.ini",
		]

		TESTING_JS_MODULES += [
		"test/RemoteImagesTestUtils.jsm",
		]

		SPHINX_TREES["docs"] = "docs"
		SPHINX_TREES["content-src/asrouter/docs"] = "content-src/asrouter/docs"

browser/components/newtab/test/RemoteImagesTestUtils.jsm

0 → 100644

+143 −0

Original line number	Diff line number	Diff line
		/* Any copyright is dedicated to the Public Domain.
		http://creativecommons.org/publicdomain/zero/1.0/ */

		"use strict";

		const { BrowserUtils } = ChromeUtils.import(
		"resource://gre/modules/BrowserUtils.jsm"
		);
		const { FileUtils } = ChromeUtils.import(
		"resource://gre/modules/FileUtils.jsm"
		);
		const { HttpServer } = ChromeUtils.import("resource://testing-common/httpd.js");
		const { RemoteImages } = ChromeUtils.import(
		"resource://activity-stream/lib/RemoteImages.jsm"
		);
		const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");

		const RS_SERVER_PREF = "services.settings.server";

		const RemoteImagesTestUtils = {
		/**
		* Serve a mock Remote Settings server with content for Remote Images
		*
		* @params imageInfo An entry describing the image. Should be one of
		* \|RemoteImagesTestUtils.images\|.
		*
		* @returns A promise yielding a cleanup function. This function will stop the
		* internal HTTP server and clean up all Remote Images state.
		*/
		async serveRemoteImages(imageInfo) {
		const { imageId, recordId, mimetype, hash, path, size } = imageInfo;

		const server = new HttpServer();
		server.start(-1);

		const baseURL = `http://localhost:${server.identity.primaryPort}/`;

		server.registerPathHandler("/v1/", (request, response) => {
		response.write(
		JSON.stringify({
		capabilities: {
		attachments: {
		base_url: `${baseURL}cdn`,
		},
		},
		})
		);

		response.setHeader("Content-Type", "application/json; charset=UTF-8");
		response.setStatusLine(null, 200, "OK");
		});

		server.registerPathHandler(
		`/v1/buckets/main/collections/ms-images/records/${recordId}`,
		(request, response) => {
		response.write(
		JSON.stringify({
		data: {
		attachment: {
		filename: imageId,
		location: `main/ms-images/${imageId}`,
		hash,
		mimetype,
		size,
		},
		},
		id: "image-id",
		last_modified: Date.now(),
		})
		);

		response.setHeader("Content-Type", "application/json; charset=UTF-8");
		response.setStatusLine(null, 200, "OK");
		}
		);

		server.registerPathHandler(
		`/cdn/main/ms-images/${imageId}`,
		async (request, response) => {
		const file = new FileUtils.File(path);
		const stream = Cc[
		"@mozilla.org/network/file-input-stream;1"
		].createInstance(Ci.nsIFileInputStream);
		stream.init(file, -1, -1, 0);

		response.bodyOutputStream.writeFrom(stream, size);
		response.setHeader("Content-Type", mimetype);
		response.setStatusLine(null, 200, "OK");
		}
		);

		Services.prefs.setCharPref(RS_SERVER_PREF, `${baseURL}v1`);

		return async () => {
		await new Promise(resolve => server.stop(() => resolve()));
		Services.prefs.clearUserPref(RS_SERVER_PREF);
		await RemoteImagesTestUtils.wipeCache();
		};
		},

		/**
		* Wipe the Remote Images cache.
		*/
		async wipeCache() {
		const children = await IOUtils.getChildren(RemoteImages.imagesDir);
		for (const child of children) {
		await IOUtils.remove(child);
		}
		},

		/**
		* Trigger RemoteImages cleanup.
		*
		* Remote Images cleanup is triggered by an observer, but its cleanup function
		* is async, so we must wait for a notification that cleanup is complete.
		*/
		async triggerCleanup() {
		Services.obs.notifyObservers(null, "remote-settings:changes-poll-end");
		await BrowserUtils.promiseObserved("remote-images:cleanup-finished");
		},

		/**
		* Remote Image entries.
		*/
		images: {
		AboutRobots: {
		imageId: "about-robots.png",
		recordId: "about-robots",
		mimetype: "image/png",
		hash: "29f1fe2cb5181152d2c01c0b2f12e5d9bb3379a61b94fb96de0f734eb360da62",
		path: PathUtils.join(
		PathUtils.parent(Services.dirsvc.get("CurWorkD", Ci.nsIFile).path, 4),
		"browser",
		"base",
		"content",
		"aboutRobots-icon.png"
		),
		size: 7599,
		},
		},
		};

		const EXPORTED_SYMBOLS = ["RemoteImagesTestUtils"];

browser/components/newtab/test/browser/browser.ini

+3 −0

Original line number	Diff line number	Diff line
		@@ -61,6 +61,9 @@ tags = remote-settings
		[browser_asrouter_group_frequency.js]
		https_first_disabled = true
		[browser_asrouter_group_userprefs.js]
		[browser_asrouter_remoteimages.js]
		support-files=
		../../../../base/content/aboutRobots-icon.png
		[browser_trigger_listeners.js]
		https_first_disabled = true
		[browser_customize_menu_render.js]

browser/components/newtab/test/browser/browser_asrouter_remoteimages.js

0 → 100644

+98 −0

Original line number	Diff line number	Diff line
		/* Any copyright is dedicated to the Public Domain.
		http://creativecommons.org/publicdomain/zero/1.0/ */

		"use strict";

		const { ObjectUtils } = ChromeUtils.import(
		"resource://gre/modules/ObjectUtils.jsm"
		);
		const { RemoteImages } = ChromeUtils.import(
		"resource://activity-stream/lib/RemoteImages.jsm"
		);
		const { RemoteImagesTestUtils } = ChromeUtils.import(
		"resource://testing-common/RemoteImagesTestUtils.jsm"
		);

		add_task(async function test_remoteImages_load() {
		const imageInfo = RemoteImagesTestUtils.images.AboutRobots;
		const cleanup = await RemoteImagesTestUtils.serveRemoteImages(imageInfo);

		registerCleanupFunction(cleanup);

		const url = await RemoteImages.load(imageInfo.imageId);
		ok(url.startsWith("blob:"), "RemoteImages.load() returns blob URL");
		ok(
		await IOUtils.exists(
		PathUtils.join(RemoteImages.imagesDir, imageInfo.imageId)
		),
		"RemoteImages caches the image"
		);

		const data = new Uint8Array(
		await fetch(url, { credentials: "omit" }).then(rsp => rsp.arrayBuffer())
		);
		const expected = await IOUtils.read(imageInfo.path);

		is(
		data.length,
		expected.length,
		"Data read from blob: URL has correct length"
		);
		ok(
		ObjectUtils.deepEqual(data, expected),
		"Data read from blob: URL matches expected"
		);

		RemoteImages.unload(url);
		});

		add_task(async function test_remoteImages_expire() {
		const THIRTY_ONE_DAYS = 31 * 24 * 60 * 60;

		const imageInfo = RemoteImagesTestUtils.images.AboutRobots;

		const cleanup = await RemoteImagesTestUtils.serveRemoteImages(imageInfo);
		registerCleanupFunction(cleanup);

		const path = PathUtils.join(RemoteImages.imagesDir, imageInfo.imageId);

		info("Doing initial load() of image");
		{
		const url = await RemoteImages.load(imageInfo.imageId);
		RemoteImages.unload(url);
		}

		ok(
		await IOUtils.exists(path),
		"Remote image exists after load() and unload()"
		);

		await RemoteImagesTestUtils.triggerCleanup();
		ok(
		await IOUtils.exists(path),
		"Remote image cache still exists after cleanup because it has not expired"
		);

		info("Setting last modified time beyond expiration date and reloading");
		{
		await IOUtils.setModificationTime(path, Date.now() - THIRTY_ONE_DAYS);

		const url = await RemoteImages.load(imageInfo.imageId);
		RemoteImages.unload(url);
		}

		await RemoteImagesTestUtils.triggerCleanup();
		ok(
		await IOUtils.exists(path),
		"Remote image cache still exists after cleanup becuase load() refreshes the expiry"
		);

		info("Setting last mofieid time beyond expiration");
		await IOUtils.setModificationTime(path, Date.now() - THIRTY_ONE_DAYS);

		await RemoteImagesTestUtils.triggerCleanup();
		ok(
		!(await IOUtils.exists(path)),
		"Remote image cache no longer exists after clenaup beyond expiry"
		);
		});