Commit cb18367f authored by Jan-Erik Rediger's avatar Jan-Erik Rediger
Browse files

Bug 1481812 - Refactor memory gathering into its own module r=chutten

This avoids loading the remaining parts of TelemetrySession in a content process.
This saves around 10 kb of memory.

Differential Revision: https://phabricator.services.mozilla.com/D8378

--HG--
extra : moz-landing-system : lando
parent 0324774e
......@@ -64,7 +64,7 @@ const whitelist = {
// Telemetry
"resource://gre/modules/TelemetryController.jsm", // bug 1470339
"resource://gre/modules/TelemetrySession.jsm", // bug 1470339
"resource://gre/modules/MemoryTelemetry.jsm", // bug 1481812
"resource://gre/modules/TelemetryUtils.jsm", // bug 1470339
// Extensions
......
......@@ -57,6 +57,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.jsm",
TelemetryArchive: "resource://gre/modules/TelemetryArchive.jsm",
TelemetrySession: "resource://gre/modules/TelemetrySession.jsm",
MemoryTelemetry: "resource://gre/modules/MemoryTelemetry.jsm",
TelemetrySend: "resource://gre/modules/TelemetrySend.jsm",
TelemetryReportingPolicy: "resource://gre/modules/TelemetryReportingPolicy.jsm",
TelemetryModules: "resource://gre/modules/ModulesPing.jsm",
......@@ -631,6 +632,7 @@ var Impl = {
// Perform a lightweight, early initialization for the component, just registering
// a few observers and initializing the session.
TelemetrySession.earlyInit(this._testMode);
MemoryTelemetry.earlyInit(this._testMode);
// Annotate crash reports so that we get pings for startup crashes
TelemetrySend.earlyInit();
......@@ -672,6 +674,7 @@ var Impl = {
// Perform TelemetrySession delayed init.
await TelemetrySession.delayedInit();
await MemoryTelemetry.delayedInit();
if (Services.prefs.getBoolPref(TelemetryUtils.Preferences.NewProfilePingEnabled, false) &&
!TelemetrySession.newProfilePingSent) {
......@@ -728,7 +731,7 @@ var Impl = {
this._log.trace("setupContentTelemetry - Content process recording disabled.");
return;
}
TelemetrySession.setupContent(testing);
MemoryTelemetry.setupContent(testing);
},
// Do proper shutdown waiting and cleanup.
......@@ -761,6 +764,7 @@ var Impl = {
await TelemetryHealthPing.shutdown();
await TelemetrySession.shutdown();
await MemoryTelemetry.shutdown();
// First wait for clients processing shutdown.
await this._shutdownBarrier.wait();
......
......@@ -98,6 +98,7 @@ EXTRA_JS_MODULES += [
'app/TelemetryTimestamps.jsm',
'app/TelemetryUtils.jsm',
'other/GCTelemetry.jsm',
'other/MemoryTelemetry.jsm',
'other/UITelemetry.jsm',
'pings/EventPing.jsm',
'pings/HealthPing.jsm',
......
/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
ChromeUtils.import("resource://gre/modules/Log.jsm");
ChromeUtils.import("resource://gre/modules/Services.jsm", this);
ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm", this);
ChromeUtils.import("resource://gre/modules/DeferredTask.jsm", this);
ChromeUtils.import("resource://gre/modules/Timer.jsm");
ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this);
ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
XPCOMUtils.defineLazyModuleGetters(this, {
GCTelemetry: "resource://gre/modules/GCTelemetry.jsm",
});
const Utils = TelemetryUtils;
const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "MemoryTelemetry" + (Utils.isContentProcess ? "#content::" : "::");
const MESSAGE_TELEMETRY_USS = "Telemetry:USS";
const MESSAGE_TELEMETRY_GET_CHILD_USS = "Telemetry:GetChildUSS";
// Do not gather data more than once a minute (ms)
const TELEMETRY_INTERVAL = 60 * 1000;
// Delay before intializing telemetry (ms)
const TELEMETRY_DELAY = Services.prefs.getIntPref("toolkit.telemetry.initDelay", 60) * 1000;
// Delay before initializing telemetry if we're testing (ms)
const TELEMETRY_TEST_DELAY = 1;
const TOPIC_CYCLE_COLLECTOR_BEGIN = "cycle-collector-begin";
// How long to wait in millis for all the child memory reports to come in
const TOTAL_MEMORY_COLLECTOR_TIMEOUT = 200;
var gLastMemoryPoll = null;
XPCOMUtils.defineLazyServiceGetters(this, {
Telemetry: ["@mozilla.org/base/telemetry;1", "nsITelemetry"],
});
var EXPORTED_SYMBOLS = ["MemoryTelemetry"];
var MemoryTelemetry = Object.freeze({
/**
* Pull values from about:memory into corresponding histograms
*/
gatherMemory() {
return Impl.gatherMemory();
},
/**
* Triggers shutdown of the module.
*/
shutdown() {
return Impl.shutdown();
},
/**
* Sets up components used in the content process.
*/
setupContent(testing = false) {
return Impl.setupContent(testing);
},
/**
* Lightweight init function, called as soon as Firefox starts.
*/
earlyInit(aTesting = false) {
return Impl.earlyInit(aTesting);
},
/**
* Does the "heavy" Telemetry initialization later on, so we
* don't impact startup performance.
* @return {Promise} Resolved when the initialization completes.
*/
delayedInit() {
return Impl.delayedInit();
},
});
var Impl = {
_initialized: false,
_logger: null,
_prevValues: {},
// A task performing delayed initialization of the chrome process
_delayedInitTask: null,
// Need a timeout in case children are tardy in giving back their memory reports.
_totalMemoryTimeout: undefined,
_testing: false,
// An accumulator of total memory across all processes. Only valid once the final child reports.
_totalMemory: null,
// A Set of outstanding USS report ids
_childrenToHearFrom: null,
// monotonically-increasing id for USS reports
_nextTotalMemoryId: 1,
_USSFromChildProcesses: null,
// Keep track of the active observers
_observedTopics: new Set(),
addObserver(aTopic) {
Services.obs.addObserver(this, aTopic);
this._observedTopics.add(aTopic);
},
removeObserver(aTopic) {
Services.obs.removeObserver(this, aTopic);
this._observedTopics.delete(aTopic);
},
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
}
return this._logger;
},
/**
* Pull values from about:memory into corresponding histograms
*/
gatherMemory: function gatherMemory() {
let mgr;
try {
mgr = Cc["@mozilla.org/memory-reporter-manager;1"].
getService(Ci.nsIMemoryReporterManager);
} catch (e) {
// OK to skip memory reporters in xpcshell
return;
}
let histogram = Telemetry.getHistogramById("TELEMETRY_MEMORY_REPORTER_MS");
let startTime = new Date();
// Get memory measurements from distinguished amount attributes. We used
// to measure "explicit" too, but it could cause hangs, and the data was
// always really noisy anyway. See bug 859657.
//
// test_TelemetrySession.js relies on some of these histograms being
// here. If you remove any of the following histograms from here, you'll
// have to modify test_TelemetrySession.js:
//
// * MEMORY_JS_GC_HEAP, and
// * MEMORY_JS_COMPARTMENTS_SYSTEM.
//
// The distinguished amount attribute names don't match the telemetry id
// names in some cases due to a combination of (a) historical reasons, and
// (b) the fact that we can't change telemetry id names without breaking
// data continuity.
//
let boundHandleMemoryReport = this.handleMemoryReport.bind(this);
let h = (id, units, amountName) => {
try {
// If mgr[amountName] throws an exception, just move on -- some amounts
// aren't available on all platforms. But if the attribute simply
// isn't present, that indicates the distinguished amounts have changed
// and this file hasn't been updated appropriately.
let amount = mgr[amountName];
if (amount === undefined) {
this._log.error("gatherMemory - telemetry accessed an unknown distinguished amount");
}
boundHandleMemoryReport(id, units, amount);
} catch (e) {
}
};
let b = (id, n) => h(id, Ci.nsIMemoryReporter.UNITS_BYTES, n);
let c = (id, n) => h(id, Ci.nsIMemoryReporter.UNITS_COUNT, n);
let cc = (id, n) => h(id, Ci.nsIMemoryReporter.UNITS_COUNT_CUMULATIVE, n);
let p = (id, n) => h(id, Ci.nsIMemoryReporter.UNITS_PERCENTAGE, n);
// GHOST_WINDOWS is opt-out as of Firefox 55
c("GHOST_WINDOWS", "ghostWindows");
if (!Telemetry.canRecordExtended) {
return;
}
b("MEMORY_VSIZE", "vsize");
if (!Services.appinfo.is64Bit || AppConstants.platform !== "win") {
b("MEMORY_VSIZE_MAX_CONTIGUOUS", "vsizeMaxContiguous");
}
b("MEMORY_RESIDENT_FAST", "residentFast");
b("MEMORY_UNIQUE", "residentUnique");
p("MEMORY_HEAP_OVERHEAD_FRACTION", "heapOverheadFraction");
b("MEMORY_JS_GC_HEAP", "JSMainRuntimeGCHeap");
c("MEMORY_JS_COMPARTMENTS_SYSTEM", "JSMainRuntimeRealmsSystem");
c("MEMORY_JS_COMPARTMENTS_USER", "JSMainRuntimeRealmsUser");
b("MEMORY_IMAGES_CONTENT_USED_UNCOMPRESSED", "imagesContentUsedUncompressed");
b("MEMORY_STORAGE_SQLITE", "storageSQLite");
cc("LOW_MEMORY_EVENTS_VIRTUAL", "lowMemoryEventsVirtual");
cc("LOW_MEMORY_EVENTS_COMMIT_SPACE", "lowMemoryEventsCommitSpace");
cc("LOW_MEMORY_EVENTS_PHYSICAL", "lowMemoryEventsPhysical");
cc("PAGE_FAULTS_HARD", "pageFaultsHard");
try {
mgr.getHeapAllocatedAsync(heapAllocated => {
boundHandleMemoryReport("MEMORY_HEAP_ALLOCATED",
Ci.nsIMemoryReporter.UNITS_BYTES,
heapAllocated);
});
} catch (e) {
}
if (!Utils.isContentProcess && !this._totalMemoryTimeout) {
// Only the chrome process should gather total memory
// total = parent RSS + sum(child USS)
this._totalMemory = mgr.residentFast;
if (Services.ppmm.childCount > 1) {
// Do not report If we time out waiting for the children to call
this._totalMemoryTimeout = setTimeout(
() => {
this._totalMemoryTimeout = undefined;
this._childrenToHearFrom.clear();
},
TOTAL_MEMORY_COLLECTOR_TIMEOUT);
this._USSFromChildProcesses = [];
this._childrenToHearFrom = new Set();
for (let i = 1; i < Services.ppmm.childCount; i++) {
let child = Services.ppmm.getChildAt(i);
try {
child.sendAsyncMessage(MESSAGE_TELEMETRY_GET_CHILD_USS, {id: this._nextTotalMemoryId});
this._childrenToHearFrom.add(this._nextTotalMemoryId);
this._nextTotalMemoryId++;
} catch (ex) {
// If a content process has just crashed, then attempting to send it
// an async message will throw an exception.
Cu.reportError(ex);
}
}
} else {
boundHandleMemoryReport(
"MEMORY_TOTAL",
Ci.nsIMemoryReporter.UNITS_BYTES,
this._totalMemory);
}
}
histogram.add(new Date() - startTime);
},
handleMemoryReport(id, units, amount, key) {
let val;
if (units == Ci.nsIMemoryReporter.UNITS_BYTES) {
val = Math.floor(amount / 1024);
} else if (units == Ci.nsIMemoryReporter.UNITS_PERCENTAGE) {
// UNITS_PERCENTAGE amounts are 100x greater than their raw value.
val = Math.floor(amount / 100);
} else if (units == Ci.nsIMemoryReporter.UNITS_COUNT) {
val = amount;
} else if (units == Ci.nsIMemoryReporter.UNITS_COUNT_CUMULATIVE) {
// If the reporter gives us a cumulative count, we'll report the
// difference in its value between now and our previous ping.
if (!(id in this._prevValues)) {
// If this is the first time we're reading this reporter, store its
// current value but don't report it in the telemetry ping, so we
// ignore the effect startup had on the reporter.
this._prevValues[id] = amount;
return;
}
val = amount - this._prevValues[id];
this._prevValues[id] = amount;
} else {
this._log.error("handleMemoryReport - Can't handle memory reporter with units " + units);
return;
}
if (key) {
Telemetry.getKeyedHistogramById(id).add(key, val);
return;
}
Telemetry.getHistogramById(id).add(val);
},
attachObservers: function attachObservers() {
if (!this._initialized)
return;
if (Telemetry.canRecordExtended) {
this.addObserver(TOPIC_CYCLE_COLLECTOR_BEGIN);
}
},
/**
* Lightweight init function, called as soon as Firefox starts.
*/
earlyInit(testing) {
this._log.trace("earlyInit");
this._initStarted = true;
this._testing = testing;
if (this._initialized && !testing) {
this._log.error("earlyInit - already initialized");
return;
}
if (!Telemetry.canRecordBase && !testing) {
this._log.config("earlyInit - Telemetry recording is disabled, skipping Chrome process setup.");
return;
}
Services.ppmm.addMessageListener(MESSAGE_TELEMETRY_USS, this);
},
/**
* Does the "heavy" Telemetry initialization later on, so we
* don't impact startup performance.
* @return {Promise} Resolved when the initialization completes.
*/
delayedInit() {
this._log.trace("delayedInit");
this._delayedInitTask = (async () => {
try {
this._initialized = true;
this.attachObservers();
this.gatherMemory();
if (Telemetry.canRecordExtended) {
GCTelemetry.init();
}
this._delayedInitTask = null;
} catch (e) {
this._delayedInitTask = null;
throw e;
}
})();
return this._delayedInitTask;
},
/**
* Initializes telemetry for a content process.
*/
setupContent: function setupContent(testing) {
this._log.trace("setupContent");
this._testing = testing;
if (!Telemetry.canRecordBase) {
this._log.trace("setupContent - base recording is disabled, not initializing");
return;
}
this.addObserver("content-child-shutdown");
Services.cpmm.addMessageListener(MESSAGE_TELEMETRY_GET_CHILD_USS, this);
let delayedTask = new DeferredTask(() => {
this._initialized = true;
this.attachObservers();
this.gatherMemory();
if (Telemetry.canRecordExtended) {
GCTelemetry.init();
}
}, testing ? TELEMETRY_TEST_DELAY : TELEMETRY_DELAY,
testing ? 0 : undefined);
delayedTask.arm();
},
getOpenTabsCount: function getOpenTabsCount() {
let tabCount = 0;
for (let win of Services.wm.getEnumerator("navigator:browser")) {
tabCount += win.gBrowser.tabs.length;
}
return tabCount;
},
receiveMessage: function receiveMessage(message) {
this._log.trace("receiveMessage - Message name " + message.name);
switch (message.name) {
case MESSAGE_TELEMETRY_USS:
{
// In parent process, receive the USS report from the child
if (this._totalMemoryTimeout && this._childrenToHearFrom.delete(message.data.id)) {
let uss = message.data.bytes;
this._totalMemory += uss;
this._USSFromChildProcesses.push(uss);
if (this._childrenToHearFrom.size == 0) {
clearTimeout(this._totalMemoryTimeout);
this._totalMemoryTimeout = undefined;
this.handleMemoryReport(
"MEMORY_TOTAL",
Ci.nsIMemoryReporter.UNITS_BYTES,
this._totalMemory);
let length = this._USSFromChildProcesses.length;
if (length > 1) {
// Mean of the USS of all the content processes.
let mean = this._USSFromChildProcesses.reduce((a, b) => a + b, 0) / length;
// Absolute error of USS for each content process, normalized by the mean (*100 to get it in percentage).
// 20% means for a content process that it is using 20% more or 20% less than the mean.
let diffs = this._USSFromChildProcesses.map(value => Math.floor(Math.abs(value - mean) * 100 / mean));
let tabsCount = this.getOpenTabsCount();
let key;
if (tabsCount < 11) {
key = "0 - 10 tabs";
} else if (tabsCount < 501) {
key = "11 - 500 tabs";
} else {
key = "more tabs";
}
diffs.forEach(value => {
this.handleMemoryReport(
"MEMORY_DISTRIBUTION_AMONG_CONTENT",
Ci.nsIMemoryReporter.UNITS_COUNT,
value,
key);
});
// This notification is for testing only.
Services.obs.notifyObservers(null, "gather-memory-telemetry-finished");
}
this._USSFromChildProcesses = undefined;
}
} else {
this._log.trace("Child USS report was missed");
}
break;
}
case MESSAGE_TELEMETRY_GET_CHILD_USS:
{
// In child process, send the requested USS report
this.sendContentProcessUSS(message.data.id);
break;
}
default:
throw new Error("Telemetry.receiveMessage: bad message name");
}
},
sendContentProcessUSS: function sendContentProcessUSS(aMessageId) {
this._log.trace("sendContentProcessUSS");
let mgr;
try {
mgr = Cc["@mozilla.org/memory-reporter-manager;1"].
getService(Ci.nsIMemoryReporterManager);
} catch (e) {
// OK to skip memory reporters in xpcshell
return;
}
Services.cpmm.sendAsyncMessage(
MESSAGE_TELEMETRY_USS,
{bytes: mgr.residentUnique, id: aMessageId}
);
},
/**
* Do some shutdown work that is common to all process types.
*/
uninstall() {
for (let topic of this._observedTopics) {
try {
// Tests may flip Telemetry.canRecordExtended on and off. It can be the case
// that the observer TOPIC_CYCLE_COLLECTOR_BEGIN was not added.
this.removeObserver(topic);
} catch (e) {
this._log.warn("uninstall - Failed to remove " + topic, e);
}
}
GCTelemetry.shutdown();
},
observe(aSubject, aTopic, aData) {
// Prevent the cycle collector begin topic from cluttering the log.
if (aTopic != TOPIC_CYCLE_COLLECTOR_BEGIN) {
this._log.trace("observe - " + aTopic + " notified.");
}
switch (aTopic) {
case "content-child-shutdown":
// content-child-shutdown is only registered for content processes.
this.uninstall();
Telemetry.flushBatchedChildTelemetry();
break;
case TOPIC_CYCLE_COLLECTOR_BEGIN:
let now = new Date();
if (!gLastMemoryPoll
|| (TELEMETRY_INTERVAL <= now - gLastMemoryPoll)) {
gLastMemoryPoll = now;
this._log.trace("Dispatching idle gatherMemory task");
Services.tm.idleDispatchToMainThread(() => {
this._log.trace("Running idle gatherMemory task");
this.gatherMemory();
return true;
});
}
break;
}
return undefined;
},
shutdown() {
this._log.trace("shutdown");
let cleanup = () => {
this.uninstall();
this._initStarted = false;
this._initialized = false;
};
// We can be in one the following states here:
// 1) delayedInit was never called
// or it was called and
// 2) _delayedInitTask is running now.
// 3) _delayedInitTask finished running already.
// This handles 1).
if (!this._initStarted) {
return Promise.resolve();
}
// This handles 3).
if (!this._delayedInitTask) {
// We already ran the delayed initialization.
return cleanup();
}
// This handles 2).
return this._delayedInitTask.then(cleanup);
},
};
......@@ -8,7 +8,6 @@
ChromeUtils.import("resource://gre/modules/Log.jsm");
ChromeUtils.import("resource://gre/modules/Services.jsm", this);
ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm", this);
ChromeUtils.import("resource://gre/modules/DeferredTask.jsm", this);
ChromeUtils.import("resource://gre/modules/Timer.jsm");
ChromeUtils.import("resource://gre/modules/TelemetryUtils.jsm", this);
ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
......@@ -17,6 +16,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
AddonManagerPrivate: "resource://gre/modules/AddonManager.jsm",
TelemetryController: "resource://gre/modules/TelemetryController.jsm",
TelemetryStorage: "resource://gre/modules/TelemetryStorage.jsm",
MemoryTelemetry: "resource://gre/modules/MemoryTelemetry.jsm",
UITelemetry: "resource://gre/modules/UITelemetry.jsm",
GCTelemetry: "resource://gre/modules/GCTelemetry.jsm",
TelemetryEnvironment: "resource://gre/modules/TelemetryEnvironment.jsm",
......@@ -48,19 +48,10 @@ const MIN_SUBSESSION_LENGTH_MS = Services.prefs.getIntPref("toolkit.telemetry.mi
const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetrySession" + (Utils.isContentProcess ? "#content::" : "::");
const MESSAGE_TELEMETRY_USS = "Telemetry:USS";
const MESSAGE_TELEMETRY_GET_CHILD_USS = "Telemetry:GetChildUSS";
// Whether the FHR/Telemetry unification features are enabled.
// Changing this pref requires a restart.
const IS_UNIFIED_TELEMETRY = Services.prefs.getBoolPref(TelemetryUtils.Preferences.Unified, false);
// Do not gather data more than once a minute (ms)
const TELEMETRY_INTERVAL = 60 * 1000;
// Delay before intializing telemetry (ms)
const TELEMETRY_DELAY = Services.prefs.getIntPref("toolkit.telemetry.initDelay", 60) * 1000;
// Delay before initializing telemetry if we're testing (ms)
const TELEMETRY_TEST_DELAY = 1;
// Execute a scheduler tick every 5 minutes.