merge autoland to mozilla-central a=merge (b02bbd40) · Commits · The Tor Project / Applications / Mullvad Browser

browser/components/migration/AutoMigrate.jsm

+180 −105

Original line number	Diff line number	Diff line
		@@ -11,29 +11,33 @@ const { classes: Cc, interfaces: Ci, results: Cr, utils: Cu } = Components;
		const kAutoMigrateEnabledPref = "browser.migrate.automigrate.enabled";
		const kUndoUIEnabledPref = "browser.migrate.automigrate.ui.enabled";

		const kAutoMigrateStartedPref = "browser.migrate.automigrate.started";
		const kAutoMigrateFinishedPref = "browser.migrate.automigrate.finished";
		const kAutoMigrateBrowserPref = "browser.migrate.automigrate.browser";

		const kAutoMigrateLastUndoPromptDateMsPref = "browser.migrate.automigrate.lastUndoPromptDateMs";
		const kAutoMigrateDaysToOfferUndoPref = "browser.migrate.automigrate.daysToOfferUndo";

		const kPasswordManagerTopic = "passwordmgr-storage-changed";
		const kPasswordManagerTopicTypes = new Set([
		"addLogin",
		"modifyLogin",
		]);

		const kSyncTopic = "fxaccounts:onlogin";

		const kNotificationId = "abouthome-automigration-undo";

		Cu.import("resource:///modules/MigrationUtils.jsm");
		Cu.import("resource://gre/modules/Preferences.jsm");
		Cu.import("resource://gre/modules/PlacesUtils.jsm");
		Cu.import("resource://gre/modules/Services.jsm");
		Cu.import("resource://gre/modules/Task.jsm");
		Cu.import("resource://gre/modules/XPCOMUtils.jsm");
		XPCOMUtils.defineLazyModuleGetter(this, "AsyncShutdown",
		"resource://gre/modules/AsyncShutdown.jsm");
		XPCOMUtils.defineLazyModuleGetter(this, "LoginHelper",
		"resource://gre/modules/LoginHelper.jsm");
		XPCOMUtils.defineLazyModuleGetter(this, "OS",
		"resource://gre/modules/osfile.jsm");
		XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
		"resource://gre/modules/PlacesUtils.jsm");

		Cu.importGlobalProperties(["URL"]);

		/* globals kUndoStateFullPath */
		XPCOMUtils.defineLazyGetter(this, "kUndoStateFullPath", function() {
		return OS.Path.join(OS.Constants.Path.profileDir, "initialMigrationMetadata.jsonlz4");
		});

		const AutoMigrate = {
		get resourceTypesToUse() {
		@@ -63,39 +67,6 @@ const AutoMigrate = {

		init() {
		this.enabled = this._checkIfEnabled();
		if (this.enabled) {
		this.maybeInitUndoObserver();
		}
		},

		maybeInitUndoObserver() {
		if (!this.canUndo()) {
		return;
		}
		// Now register places, password and sync observers:
		this.onItemAdded = this.onItemMoved = this.onItemChanged =
		this.removeUndoOption.bind(this, this.UNDO_REMOVED_REASON_BOOKMARK_CHANGE);
		PlacesUtils.addLazyBookmarkObserver(this, true);
		for (let topic of [kSyncTopic, kPasswordManagerTopic]) {
		Services.obs.addObserver(this, topic, true);
		}
		},

		observe(subject, topic, data) {
		if (topic == kPasswordManagerTopic) {
		// As soon as any login gets added or modified, disable undo:
		// (Note that this ignores logins being removed as that doesn't
		// impair the 'undo' functionality of the import.)
		if (kPasswordManagerTopicTypes.has(data)) {
		// Ignore chrome:// things like sync credentials:
		let loginInfo = subject.QueryInterface(Ci.nsILoginInfo);
		if (!loginInfo.hostname \|\| !loginInfo.hostname.startsWith("chrome://")) {
		this.removeUndoOption(this.UNDO_REMOVED_REASON_PASSWORD_CHANGE);
		}
		}
		} else if (topic == kSyncTopic) {
		this.removeUndoOption(this.UNDO_REMOVED_REASON_SYNC_SIGNIN);
		}
		},

		/**
		@@ -132,21 +103,18 @@ const AutoMigrate = {
		}
		Services.obs.removeObserver(migrationObserver, "Migration:Ended");
		Services.obs.removeObserver(migrationObserver, "Migration:ItemError");
		Services.prefs.setCharPref(kAutoMigrateStartedPref, startTime.toString());
		Services.prefs.setCharPref(kAutoMigrateFinishedPref, Date.now().toString());
		Services.prefs.setCharPref(kAutoMigrateBrowserPref, pickedKey);
		// Need to manually start listening to new bookmarks/logins being created,
		// because, when we were initialized, there wasn't the possibility to
		// 'undo' anything.
		this.maybeInitUndoObserver();
		// Save the undo history and block shutdown on that save completing.
		AsyncShutdown.profileBeforeChange.addBlocker(
		"AutoMigrate Undo saving", this.saveUndoState(), () => {
		return {state: this._saveUndoStateTrackerForShutdown};
		});
		}
		};

		MigrationUtils.initializeUndoData();
		Services.obs.addObserver(migrationObserver, "Migration:Ended", false);
		Services.obs.addObserver(migrationObserver, "Migration:ItemError", false);
		// We'll save this when the migration has finished, at which point the pref
		// service will be available.
		let startTime = Date.now();
		migrator.migrate(this.resourceTypesToUse, profileStartup, profileToMigrate);
		histogram.add(20);
		},
		@@ -206,81 +174,53 @@ const AutoMigrate = {
		return profiles ? profiles[0] : null;
		},

		getUndoRange() {
		let start, finish;
		canUndo: Task.async(function* () {
		if (this._savingPromise) {
		yield this._savingPromise;
		}
		let fileExists = false;
		try {
		start = parseInt(Preferences.get(kAutoMigrateStartedPref, "0"), 10);
		finish = parseInt(Preferences.get(kAutoMigrateFinishedPref, "0"), 10);
		fileExists = yield OS.File.exists(kUndoStateFullPath);
		} catch (ex) {
		Cu.reportError(ex);
		}
		if (!finish \|\| !start) {
		return null;
		}
		return [new Date(start), new Date(finish)];
		},

		canUndo() {
		return !!this.getUndoRange();
		},
		return fileExists;
		}),

		undo: Task.async(function* () {
		let histogram = Services.telemetry.getHistogramById("FX_STARTUP_MIGRATION_AUTOMATED_IMPORT_UNDO");
		histogram.add(0);
		if (!this.canUndo()) {
		if (!(yield this.canUndo())) {
		histogram.add(5);
		throw new Error("Can't undo!");
		}

		histogram.add(10);

		yield PlacesUtils.bookmarks.eraseEverything();
		let readPromise = OS.File.read(kUndoStateFullPath, {
		encoding: "utf-8",
		compression: "lz4",
		});
		let stateData = this._dejsonifyUndoState(yield readPromise);
		yield this._removeUnchangedBookmarks(stateData.get("bookmarks"));
		histogram.add(15);

		// NB: we drop the start time of the migration for now. This is because
		// imported history will always end up being 'backdated' to the actual
		// visit time recorded by the browser from which we imported. As a result,
		// a lower bound on this item doesn't really make sense.
		// Note that for form data this could be different, but we currently don't
		// support form data import from any non-Firefox browser, so it isn't
		// imported from other browsers by the automigration code, nor do we
		// remove it here.
		let range = this.getUndoRange();
		yield PlacesUtils.history.removeVisitsByFilter({
		beginDate: new Date(0),
		endDate: range[1]
		});
		yield this._removeSomeVisits(stateData.get("visits"));
		histogram.add(20);

		try {
		Services.logins.removeAllLogins();
		} catch (ex) {
		// ignore failure.
		}
		yield this._removeUnchangedLogins(stateData.get("logins"));
		histogram.add(25);

		this.removeUndoOption(this.UNDO_REMOVED_REASON_UNDO_USED);
		histogram.add(30);
		}),

		removeUndoOption(reason) {
		// Remove observers, and ensure that exceptions doing so don't break
		// removing the pref.
		for (let topic of [kSyncTopic, kPasswordManagerTopic]) {
		try {
		Services.obs.removeObserver(this, topic);
		} catch (ex) {
		Cu.reportError("Error removing observer for " + topic + ": " + ex);
		}
		}
		try {
		PlacesUtils.removeLazyBookmarkObserver(this);
		} catch (ex) {
		Cu.reportError("Error removing lazy bookmark observer: " + ex);
		}
		// We don't wait for the off-main-thread removal to complete. OS.File will
		// ensure it happens before shutdown.
		OS.File.remove(kUndoStateFullPath, {ignoreAbsent: true});

		let migrationBrowser = Preferences.get(kAutoMigrateBrowserPref, "unknown");
		Services.prefs.clearUserPref(kAutoMigrateStartedPref);
		Services.prefs.clearUserPref(kAutoMigrateFinishedPref);
		Services.prefs.clearUserPref(kAutoMigrateBrowserPref);

		let browserWindows = Services.wm.getEnumerator("navigator:browser");
		@@ -309,9 +249,13 @@ const AutoMigrate = {
		return null;
		},

		maybeShowUndoNotification(target) {
		maybeShowUndoNotification: Task.async(function* (target) {
		if (!(yield this.canUndo())) {
		return;
		}

		// The tab might have navigated since we requested the undo state:
		if (!this.canUndo() \|\| target.currentURI.spec != "about:home" \|\|
		if (target.currentURI.spec != "about:home" \|\|
		!Preferences.get(kUndoUIEnabledPref, false)) {
		return;
		}
		@@ -360,7 +304,7 @@ const AutoMigrate = {
		);
		let remainingDays = Preferences.get(kAutoMigrateDaysToOfferUndoPref, 0);
		Services.telemetry.getHistogramById("FX_STARTUP_MIGRATION_UNDO_OFFERED").add(4 - remainingDays);
		},
		}),

		shouldStillShowUndoPrompt() {
		let today = new Date();
		@@ -390,6 +334,137 @@ const AutoMigrate = {
		UNDO_REMOVED_REASON_OFFER_EXPIRED: 4,
		UNDO_REMOVED_REASON_OFFER_REJECTED: 5,

		_jsonifyUndoState(state) {
		if (!state) {
		return "null";
		}
		// Deal with date serialization.
		let bookmarks = state.get("bookmarks");
		for (let bm of bookmarks) {
		bm.lastModified = bm.lastModified.getTime();
		}
		let serializableState = {
		bookmarks,
		logins: state.get("logins"),
		visits: state.get("visits"),
		};
		return JSON.stringify(serializableState);
		},

		_dejsonifyUndoState(state) {
		state = JSON.parse(state);
		for (let bm of state.bookmarks) {
		bm.lastModified = new Date(bm.lastModified);
		}
		return new Map([
		["bookmarks", state.bookmarks],
		["logins", state.logins],
		["visits", state.visits],
		]);
		},

		_saveUndoStateTrackerForShutdown: "not running",
		saveUndoState: Task.async(function* () {
		let resolveSavingPromise;
		this._saveUndoStateTrackerForShutdown = "processing undo history";
		this._savingPromise = new Promise(resolve => { resolveSavingPromise = resolve });
		let state = yield MigrationUtils.stopAndRetrieveUndoData();
		this._saveUndoStateTrackerForShutdown = "writing undo history";
		this._undoSavePromise = OS.File.writeAtomic(
		kUndoStateFullPath, this._jsonifyUndoState(state), {
		encoding: "utf-8",
		compression: "lz4",
		tmpPath: kUndoStateFullPath + ".tmp",
		});
		this._undoSavePromise.then(
		rv => {
		resolveSavingPromise(rv);
		delete this._savingPromise;
		},
		e => {
		Cu.reportError("Could not write undo state for automatic migration.");
		throw e;
		});
		return this._undoSavePromise;
		}),

		_removeUnchangedBookmarks: Task.async(function* (bookmarks) {
		if (!bookmarks.length) {
		return;
		}

		let guidToLMMap = new Map(bookmarks.map(b => [b.guid, b.lastModified]));
		let bookmarksFromDB = [];
		let bmPromises = Array.from(guidToLMMap.keys()).map(guid => {
		// Ignore bookmarks where the promise doesn't resolve (ie that are missing)
		// Also check that the bookmark fetch returns isn't null before adding it.
		return PlacesUtils.bookmarks.fetch(guid).then(bm => bm && bookmarksFromDB.push(bm), () => {});
		});
		// We can't use the result of Promise.all because that would include nulls
		// for bookmarks that no longer exist (which we're catching above).
		yield Promise.all(bmPromises);
		let unchangedBookmarks = bookmarksFromDB.filter(bm => {
		return bm.lastModified.getTime() == guidToLMMap.get(bm.guid).getTime();
		});

		// We need to remove items with no ancestors first, followed by their
		// parents, etc. In order to do this, find out how many ancestors each item
		// has that also appear in our list of things to remove, and sort the items
		// by those numbers. This ensures that children are always removed before
		// their parents.
		function determineAncestorCount(bm) {
		if (bm._ancestorCount) {
		return bm._ancestorCount;
		}
		let myCount = 0;
		let parentBM = unchangedBookmarks.find(item => item.guid == bm.parentGuid);
		if (parentBM) {
		myCount = determineAncestorCount(parentBM) + 1;
		}
		bm._ancestorCount = myCount;
		return myCount;
		}
		unchangedBookmarks.forEach(determineAncestorCount);
		unchangedBookmarks.sort((a, b) => b._ancestorCount - a._ancestorCount);
		for (let {guid} of unchangedBookmarks) {
		yield PlacesUtils.bookmarks.remove(guid, {preventRemovalOfNonEmptyFolders: true}).catch(err => {
		if (err && err.message != "Cannot remove a non-empty folder.") {
		Cu.reportError(err);
		}
		});
		}
		}),

		_removeUnchangedLogins: Task.async(function* (logins) {
		for (let login of logins) {
		let foundLogins = LoginHelper.searchLoginsWithObject({guid: login.guid});
		if (foundLogins.length) {
		let foundLogin = foundLogins[0];
		foundLogin.QueryInterface(Ci.nsILoginMetaInfo);
		if (foundLogin.timePasswordChanged == login.timePasswordChanged) {
		Services.logins.removeLogin(foundLogin);
		}
		}
		}
		}),

		_removeSomeVisits: Task.async(function* (visits) {
		for (let urlVisits of visits) {
		let urlObj;
		try {
		urlObj = new URL(urlVisits.url);
		} catch (ex) {
		continue;
		}
		yield PlacesUtils.history.removeVisitsByFilter({
		url: urlObj,
		beginDate: PlacesUtils.toDate(urlVisits.first),
		endDate: PlacesUtils.toDate(urlVisits.last),
		limit: urlVisits.visitCount,
		});
		}
		}),

		QueryInterface: XPCOMUtils.generateQI(
		[Ci.nsIObserver, Ci.nsINavBookmarkObserver, Ci.nsISupportsWeakReference]
		),

browser/components/migration/ChromeProfileMigrator.js

+2 −2

Original line number	Diff line number	Diff line
		@@ -448,7 +448,7 @@ function GetWindowsPasswordsResource(aProfileFolder) {
		decryptData(crypto.arrayToString(row.getResultByName("password_value")),
		null),
		hostname: NetUtil.newURI(row.getResultByName("origin_url")).prePath,
		submitURL: null,
		formSubmitURL: null,
		httpRealm: null,
		usernameElement: row.getResultByName("username_element"),
		passwordElement: row.getResultByName("password_element"),
		@@ -459,7 +459,7 @@ function GetWindowsPasswordsResource(aProfileFolder) {
		try {
		switch (row.getResultByName("scheme")) {
		case AUTH_TYPE.SCHEME_HTML:
		loginInfo.submitURL = NetUtil.newURI(row.getResultByName("action_url")).prePath;
		loginInfo.formSubmitURL = NetUtil.newURI(row.getResultByName("action_url")).prePath;
		break;
		case AUTH_TYPE.SCHEME_BASIC:
		case AUTH_TYPE.SCHEME_DIGEST:

browser/components/migration/MigrationUtils.jsm

+94 −2

Original line number	Diff line number	Diff line
		@@ -16,6 +16,8 @@ Cu.import("resource://gre/modules/Services.jsm");
		Cu.import("resource://gre/modules/Task.jsm");
		Cu.import("resource://gre/modules/XPCOMUtils.jsm");

		Cu.importGlobalProperties(["URL"]);

		XPCOMUtils.defineLazyModuleGetter(this, "AutoMigrate",
		"resource:///modules/AutoMigrate.jsm");
		XPCOMUtils.defineLazyModuleGetter(this, "BookmarkHTMLUtils",
		@@ -38,6 +40,9 @@ var gProfileStartup = null;
		var gMigrationBundle = null;
		var gPreviousDefaultBrowserKey = "";

		let gKeepUndoData = false;
		let gUndoData = null;

		XPCOMUtils.defineLazyGetter(this, "gAvailableMigratorKeys", function() {
		if (AppConstants.platform == "win") {
		return [
		@@ -948,17 +953,104 @@ this.MigrationUtils = Object.freeze({

		insertBookmarkWrapper(bookmark) {
		this._importQuantities.bookmarks++;
		return PlacesUtils.bookmarks.insert(bookmark);
		let insertionPromise = PlacesUtils.bookmarks.insert(bookmark);
		if (!gKeepUndoData) {
		return insertionPromise;
		}
		// If we keep undo data, add a promise handler that stores the undo data once
		// the bookmark has been inserted in the DB, and then returns the bookmark.
		let {parentGuid} = bookmark;
		return insertionPromise.then(bm => {
		let {guid, lastModified, type} = bm;
		gUndoData.get("bookmarks").push({
		parentGuid, guid, lastModified, type
		});
		return bm;
		});
		},

		insertVisitsWrapper(places, options) {
		this._importQuantities.history += places.length;
		if (gKeepUndoData) {
		this._updateHistoryUndo(places);
		}
		return PlacesUtils.asyncHistory.updatePlaces(places, options);
		},

		insertLoginWrapper(login) {
		this._importQuantities.logins++;
		return LoginHelper.maybeImportLogin(login);
		let insertedLogin = LoginHelper.maybeImportLogin(login);
		// Note that this means that if we import a login that has a newer password
		// than we know about, we will update the login, and an undo of the import
		// will not revert this. This seems preferable over removing the login
		// outright or storing the old password in the undo file.
		if (insertedLogin && gKeepUndoData) {
		let {guid, timePasswordChanged} = insertedLogin;
		gUndoData.get("logins").push({guid, timePasswordChanged});
		}
		},

		initializeUndoData() {
		gKeepUndoData = true;
		gUndoData = new Map([["bookmarks", []], ["visits", []], ["logins", []]]);
		},

		_postProcessUndoData: Task.async(function*(state) {
		if (!state) {
		return state;
		}
		let bookmarkFolders = state.get("bookmarks").filter(b => b.type == PlacesUtils.bookmarks.TYPE_FOLDER);

		let bookmarkFolderData = [];
		let bmPromises = bookmarkFolders.map(({guid}) => {
		// Ignore bookmarks where the promise doesn't resolve (ie that are missing)
		// Also check that the bookmark fetch returns isn't null before adding it.
		return PlacesUtils.bookmarks.fetch(guid).then(bm => bm && bookmarkFolderData.push(bm), () => {});
		});

		yield Promise.all(bmPromises);
		let folderLMMap = new Map(bookmarkFolderData.map(b => [b.guid, b.lastModified]));
		for (let bookmark of bookmarkFolders) {
		let lastModified = folderLMMap.get(bookmark.guid);
		// If the bookmark was deleted, the map will be returning null, so check:
		if (lastModified) {
		bookmark.lastModified = lastModified;
		}
		}
		return state;
		}),

		stopAndRetrieveUndoData() {
		let undoData = gUndoData;
		gUndoData = null;
		gKeepUndoData = false;
		return this._postProcessUndoData(undoData);
		},

		_updateHistoryUndo(places) {
		let visits = gUndoData.get("visits");
		let visitMap = new Map(visits.map(v => [v.url, v]));
		for (let place of places) {
		let visitCount = place.visits.length;
		let first = Math.min.apply(Math, place.visits.map(v => v.visitDate));
		let last = Math.max.apply(Math, place.visits.map(v => v.visitDate));
		let url = place.uri.spec;
		try {
		new URL(url);
		} catch (ex) {
		// This won't save and we won't need to 'undo' it, so ignore this URL.
		continue;
		}
		if (!visitMap.has(url)) {
		visitMap.set(url, {url, visitCount, first, last});
		} else {
		let currentData = visitMap.get(url);
		currentData.visitCount += visitCount;
		currentData.first = Math.min(currentData.first, first);
		currentData.last = Math.max(currentData.last, last);
		}
		}
		gUndoData.set("visits", Array.from(visitMap.values()));
		},

		/**

browser/components/migration/tests/unit/head_migration.js

+10 −5

Original line number	Diff line number	Diff line
		@@ -6,15 +6,20 @@ var { classes: Cc, interfaces: Ci, results: Cr, utils: Cu } = Components;

		Cu.importGlobalProperties([ "URL" ]);

		Cu.import("resource://gre/modules/XPCOMUtils.jsm");
		Cu.import("resource:///modules/MigrationUtils.jsm");
		Cu.import("resource://gre/modules/LoginHelper.jsm");
		Cu.import("resource://gre/modules/NetUtil.jsm");
		Cu.import("resource://gre/modules/PlacesUtils.jsm");
		Cu.import("resource://gre/modules/Preferences.jsm");
		Cu.import("resource://gre/modules/PromiseUtils.jsm");
		Cu.import("resource://gre/modules/Task.jsm");
		Cu.import("resource://gre/modules/Services.jsm");
		Cu.import("resource://gre/modules/XPCOMUtils.jsm");
		Cu.import("resource://testing-common/TestUtils.jsm");
		Cu.import("resource://testing-common/PlacesTestUtils.jsm");

		XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
		"resource://gre/modules/PlacesUtils.jsm");
		XPCOMUtils.defineLazyModuleGetter(this, "FileUtils",
		"resource://gre/modules/FileUtils.jsm");
		XPCOMUtils.defineLazyModuleGetter(this, "MigrationUtils",
		"resource:///modules/MigrationUtils.jsm");

		// Initialize profile.
		var gProfD = do_get_profile();