Bug 1717068 - Option to load from JSON dump if it's newer than local data r=leplatrem (b5259917) · Commits · The Tor Project / Applications / Mullvad Browser

services/settings/RemoteSettingsClient.jsm

+30 −4

Original line number	Diff line number	Diff line
		@@ -341,6 +341,7 @@ class RemoteSettingsClient extends EventEmitter {
		* @param {Object} options.filters Filter the results (default: `{}`).
		* @param {String} options.order The order to apply (eg. `"-last_modified"`).
		* @param {boolean} options.dumpFallback Fallback to dump data if read of local DB fails (default: `true`).
		* @param {boolean} options.loadDumpIfNewer Use dump data if it is newer than local data (default: `false`).
		* @param {boolean} options.syncIfEmpty Synchronize from server if local data is empty (default: `true`).
		* @param {boolean} options.verifySignature Verify the signature of the local data (default: `false`).
		* @return {Promise}
		@@ -350,13 +351,15 @@ class RemoteSettingsClient extends EventEmitter {
		filters = {},
		order = "", // not sorted by default.
		dumpFallback = true,
		loadDumpIfNewer = false, // TODO bug 1718083: should default to true.
		syncIfEmpty = true,
		} = options;
		let { verifySignature = false } = options;

		let data;
		try {
		let hasLocalData = await Utils.hasLocalData(this);
		let lastModified = await this.db.getLastModified();
		let hasLocalData = lastModified !== null;

		if (syncIfEmpty && !hasLocalData) {
		// .get() was called before we had the chance to synchronize the local database.
		@@ -375,9 +378,35 @@ class RemoteSettingsClient extends EventEmitter {
		await this.sync({ loadDump: false });
		}
		})();
		} else {
		console.debug(`${this.identifier} Awaiting existing import.`);
		}
		} else if (hasLocalData && loadDumpIfNewer) {
		// Check whether the local data is older than the packaged dump.
		// If it is, load the packaged dump (which overwrites the local data).
		let lastModifiedDump = await Utils.getLocalDumpLastModified(
		this.bucketName,
		this.collectionName
		);
		if (lastModified < lastModifiedDump) {
		console.debug(
		`${this.identifier} Local DB is stale (${lastModified}), using dump instead (${lastModifiedDump})`
		);
		if (!this._importingPromise) {
		// As part of importing, any existing data is wiped.
		this._importingPromise = this._importJSONDump();
		} else {
		console.debug(`${this.identifier} Awaiting existing import.`);
		}
		}
		}

		if (this._importingPromise) {
		try {
		await this._importingPromise;
		// No need to verify signature, because either we've just load a trusted
		// dump (here or in a parallel call), or it was verified during sync.
		verifySignature = false;
		} catch (e) {
		// Report error, but continue because there could have been data
		// loaded from a parrallel call.
		@@ -386,9 +415,6 @@ class RemoteSettingsClient extends EventEmitter {
		// then delete this promise again, as now we should have local data:
		delete this._importingPromise;
		}
		// No need to verify signature, because either we've just load a trusted
		// dump (here or in a parallel call), or it was verified during sync.
		verifySignature = false;
		}

		// Read from the local DB.

services/settings/Utils.jsm

+30 −0

Original line number	Diff line number	Diff line
		@@ -126,6 +126,36 @@ var Utils = {
		}
		},

		/**
		* Look up the last modification time of the JSON dump.
		*
		* @param {String} bucket
		* @param {String} collection
		* @return {int} The last modification time of the dump. -1 if non-existent.
		*/
		async getLocalDumpLastModified(bucket, collection) {
		if (!this._dumpStats) {
		this._dumpStats = {};
		}
		const identifier = `${bucket}/${collection}`;
		let lastModified = this._dumpStats[identifier];
		if (lastModified === undefined) {
		try {
		let res = await fetch(
		`resource://app/defaults/settings/${bucket}/${collection}.json`
		);
		let records = (await res.json()).data;
		// Records in dumps are sorted by last_modified, newest first.
		// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
		lastModified = records[0]?.last_modified \|\| 0;
		} catch (e) {
		lastModified = -1;
		}
		this._dumpStats[identifier] = lastModified;
		}
		return lastModified;
		},

		/**
		* Fetch the list of remote collections and their timestamp.
		* ```

services/settings/test/unit/test_remote_settings_offline.js

0 → 100644

+127 −0

Original line number	Diff line number	Diff line
		const { RemoteSettingsClient } = ChromeUtils.import(
		"resource://services-settings/RemoteSettingsClient.jsm"
		);
		const { RemoteSettingsWorker } = ChromeUtils.import(
		"resource://services-settings/RemoteSettingsWorker.jsm"
		);
		const { SharedUtils } = ChromeUtils.import(
		"resource://services-settings/SharedUtils.jsm"
		);

		// A collection with a dump that's packaged on all builds where this test runs,
		// including on Android at mobile/android/installer/package-manifest.in
		const TEST_BUCKET = "main";
		const TEST_COLLECTION = "password-recipes";

		let client;
		let DUMP_RECORDS;
		let DUMP_LAST_MODIFIED;

		add_task(async function setup() {
		// "services.settings.server" pref is not set.
		// Test defaults to an unreachable server,
		// and will only load from the dump if any.

		client = new RemoteSettingsClient(TEST_COLLECTION, {
		bucketName: TEST_BUCKET,
		});

		DUMP_RECORDS = (await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION))
		.data;
		DUMP_LAST_MODIFIED = DUMP_RECORDS.reduce(
		(max, { last_modified }) => Math.max(last_modified, max),
		-Infinity
		);
		// Dumps are fetched via the following, which sorts the records, newest first.
		// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
		equal(
		DUMP_LAST_MODIFIED,
		DUMP_RECORDS[0].last_modified,
		"records in dump ought to be sorted by last_modified"
		);
		});

		async function importData(records) {
		await RemoteSettingsWorker._execute("_test_only_import", [
		TEST_BUCKET,
		TEST_COLLECTION,
		records,
		]);
		}

		async function clear_state() {
		await client.db.clear();
		}

		add_task(async function test_load_from_dump_when_offline() {
		// Baseline: verify that the collection is empty at first,
		// but non-empty after loading from the dump.
		const before = await client.get({ syncIfEmpty: false });
		equal(before.length, 0, "collection empty when offline");

		// should import from dump since collection was not initialized.
		const after = await client.get();
		equal(after.length, DUMP_RECORDS.length, "collection loaded from dump");
		equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
		});
		add_task(clear_state);

		add_task(async function test_skip_dump_after_empty_import() {
		// clear_state should have wiped the database.
		const before = await client.get({ syncIfEmpty: false });
		equal(before.length, 0, "collection empty after clearing");

		// Verify that the dump is not imported again by client.get()
		// when the database is initialized with an empty dump.
		await importData([]); // <-- Empty set of records.

		const after = await client.get();
		equal(after.length, 0, "collection still empty due to import");
		equal(await client.getLastModified(), 0, "Empty dump has no timestamp");
		});
		add_task(clear_state);

		add_task(async function test_skip_dump_after_non_empty_import() {
		await importData([{ last_modified: 1234, id: "dummy" }]);

		const after = await client.get();
		equal(after.length, 1, "Imported dummy data");
		equal(await client.getLastModified(), 1234, "Expected timestamp of import");

		await importData([]);
		const after2 = await client.get();
		equal(after2.length, 0, "Previous data wiped on duplicate import");
		equal(await client.getLastModified(), 0, "Timestamp of empty collection");
		});
		add_task(clear_state);

		add_task(async function test_load_dump_after_empty_import() {
		await importData([]); // <-- Empty set of records, i.e. last_modified = 0.

		const after = await client.get({ loadDumpIfNewer: true });
		equal(after.length, DUMP_RECORDS.length, "Imported dump");
		equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
		});
		add_task(clear_state);

		add_task(async function test_load_dump_after_non_empty_import() {
		// Dump is updated regularly, verify that the dump matches our expectations
		// before running the test.
		ok(DUMP_LAST_MODIFIED > 1234, "Assuming dump to be newer than dummy 1234");

		await importData([{ last_modified: 1234, id: "dummy" }]);

		const after = await client.get({ loadDumpIfNewer: true });
		equal(after.length, DUMP_RECORDS.length, "Imported dump");
		equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "dump's timestamp");
		});
		add_task(clear_state);

		add_task(async function test_skip_dump_if_same_last_modified() {
		await importData([{ last_modified: DUMP_LAST_MODIFIED, id: "dummy" }]);

		const after = await client.get({ loadDumpIfNewer: true });
		equal(after.length, 1, "Not importing dump when time matches");
		equal(await client.getLastModified(), DUMP_LAST_MODIFIED, "Same timestamp");
		});
		add_task(clear_state);

services/settings/test/unit/xpcshell.ini

+1 −0

Original line number	Diff line number	Diff line
		@@ -9,6 +9,7 @@ skip-if = appname == "thunderbird" # Bug 1662758 - these tests don't pass with a
		[test_attachments_downloader.js]
		support-files = test_attachments_downloader/**
		[test_remote_settings.js]
		[test_remote_settings_offline.js]
		[test_remote_settings_poll.js]
		[test_remote_settings_worker.js]
		[test_remote_settings_jexl_filters.js]