Commit 7d9a9985 authored by Nihanth Subramanya's avatar Nihanth Subramanya
Browse files

Bug 1690105 - Implement RemoteSettings client in DoHConfig.jsm for provider...

Bug 1690105 - Implement RemoteSettings client in DoHConfig.jsm for provider and config data collections. r=dragana,preferences-reviewers,Gijs

Differential Revision: https://phabricator.services.mozilla.com/D103714
parent 384c77f3
Loading
Loading
Loading
Loading
+257 −42
Original line number Diff line number Diff line
@@ -5,72 +5,287 @@
"use strict";

/*
 * This module provides an interface to acces DoH config settings - e.g. whether
 * DoH is enabled, whether capabilities are enabled, etc. Currently this just
 * provides getters for prefs, but imminently will be extended to read config
 * from a Remote Settings collection and filter by client region etc.
 * This module provides an interface to access DoH configuration - e.g. whether
 * DoH is enabled, whether capabilities are enabled, etc. The configuration is
 * sourced from either Remote Settings or pref values, with Remote Settings
 * being preferred.
 */
var EXPORTED_SYMBOLS = ["Config"];
var EXPORTED_SYMBOLS = ["DoHConfigController"];

const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");

ChromeUtils.defineModuleGetter(
  this,
  "Preferences",
  "resource://gre/modules/Preferences.jsm"
const { XPCOMUtils } = ChromeUtils.import(
  "resource://gre/modules/XPCOMUtils.jsm"
);

const kEnabledPref = "doh-rollout.enabled";
XPCOMUtils.defineLazyModuleGetters(this, {
  Preferences: "resource://gre/modules/Preferences.jsm",
  Region: "resource://gre/modules/Region.jsm",
  RemoteSettings: "resource://services-settings/remote-settings.js",
  Services: "resource://gre/modules/Services.jsm",
});

const kGlobalPrefBranch = "doh-rollout";
var kRegionPrefBranch;

const kEnabledPref = "enabled";

const kTRRSelectionEnabledPref = "doh-rollout.trr-selection.enabled";
const kTRRSelectionCommitResultPref = "doh-rollout.trr-selection.commit-result";
const kProvidersPref = "provider-list";

const kProviderSteeringEnabledPref = "doh-rollout.provider-steering.enabled";
const kProviderSteeringListPref = "doh-rollout.provider-steering.provider-list";
const kTRRSelectionEnabledPref = "trr-selection.enabled";
const kTRRSelectionProvidersPref = "trr-selection.provider-list";
const kTRRSelectionCommitResultPref = "trr-selection.commit-result";

const kProviderSteeringEnabledPref = "provider-steering.enabled";
const kProviderSteeringListPref = "provider-steering.provider-list";

const kPrefChangedTopic = "nsPref:changed";

const Config = {
  init() {
    Preferences.observe(kEnabledPref, this);
  },
const gProvidersCollection = RemoteSettings("doh-providers");
const gConfigCollection = RemoteSettings("doh-config");

  observe(subject, topic, data) {
    switch (topic) {
      case kPrefChangedTopic:
        this.notifyNewConfig();
        break;
function getPrefValueRegionFirst(prefName, defaultValue) {
  return (
    Preferences.get(`${kRegionPrefBranch}.${prefName}`) ||
    Preferences.get(`${kGlobalPrefBranch}.${prefName}`, defaultValue)
  );
}

function getProviderListFromPref(prefName) {
  try {
    return JSON.parse(getPrefValueRegionFirst(prefName, "[]"));
  } catch (e) {
    Cu.reportError(`DoH provider list not a valid JSON array: ${prefName}`);
  }
  return [];
}

// Generate a base config object with getters that return pref values. When
// Remote Settings values become available, a new config object will be
// generated from this and specific fields will be replaced by the RS value.
// If we use a class to store base config and instantiate new config objects
// from it, we lose the ability to override getters because they are defined
// as non-configureable properties on class instances. So just use a function.
function makeBaseConfigObject() {
  return {
    get enabled() {
      return getPrefValueRegionFirst(kEnabledPref, false);
    },

  kConfigUpdateTopic: "doh-config-updated",
  notifyNewConfig() {
    Services.obs.notifyObservers(null, this.kConfigUpdateTopic);
    get providerList() {
      return getProviderListFromPref(kProvidersPref);
    },

  get enabled() {
    return Preferences.get(kEnabledPref, false);
    get fallbackProviderURI() {
      return this.providerList[0]?.uri;
    },

    trrSelection: {
      get enabled() {
      return Preferences.get(kTRRSelectionEnabledPref, false);
        return getPrefValueRegionFirst(kTRRSelectionEnabledPref, false);
      },

      get commitResult() {
      return Preferences.get(kTRRSelectionCommitResultPref, false);
        return getPrefValueRegionFirst(kTRRSelectionCommitResultPref, false);
      },

      get providerList() {
        return getProviderListFromPref(kTRRSelectionProvidersPref);
      },
    },

    providerSteering: {
      get enabled() {
      return Preferences.get(kProviderSteeringEnabledPref, false);
        return getPrefValueRegionFirst(kProviderSteeringEnabledPref, false);
      },

      get providerList() {
      return Preferences.get(kProviderSteeringListPref, "[]");
        return getProviderListFromPref(kProviderSteeringListPref);
      },
    },
  };
}

const DoHConfigController = {
  initComplete: null,
  _resolveInitComplete: null,

  // This field always contains the current config state, for
  // consumer use.
  currentConfig: makeBaseConfigObject(),

  // Loads the client's region via Region.jsm. This might mean waiting
  // until the region is available.
  async loadRegion() {
    await new Promise(resolve => {
      let homeRegion = Preferences.get(`${kGlobalPrefBranch}.home-region`);
      if (homeRegion) {
        kRegionPrefBranch = `${kGlobalPrefBranch}.${homeRegion.toLowerCase()}`;
        resolve();
        return;
      }

      let updateRegionAndResolve = () => {
        kRegionPrefBranch = `${kGlobalPrefBranch}.${Region.home.toLowerCase()}`;
        Preferences.set(`${kGlobalPrefBranch}.home-region`, Region.home);
        resolve();
      };

      if (Region.home) {
        updateRegionAndResolve();
        return;
      }

      Services.obs.addObserver(function obs(sub, top, data) {
        Services.obs.removeObserver(obs, Region.REGION_TOPIC);
        updateRegionAndResolve();
      }, Region.REGION_TOPIC);
    });

    // Finally, reload config.
    await this.updateFromRemoteSettings();
  },

  async init() {
    await this.loadRegion();

    Services.prefs.addObserver(`${kGlobalPrefBranch}.`, this, true);

    gProvidersCollection.on("sync", this.updateFromRemoteSettings);
    gConfigCollection.on("sync", this.updateFromRemoteSettings);

    this._resolveInitComplete();
  },

  // Useful for tests to set prior state before init()
  async _uninit() {
    await this.initComplete;

    Services.prefs.removeObserver(`${kGlobalPrefBranch}`, this);

    gProvidersCollection.off("sync", this.updateFromRemoteSettings);
    gConfigCollection.off("sync", this.updateFromRemoteSettings);

    this.initComplete = new Promise(resolve => {
      this._resolveInitComplete = resolve;
    });
  },

  observe(subject, topic, data) {
    switch (topic) {
      case kPrefChangedTopic:
        if (
          !data.startsWith(kRegionPrefBranch) &&
          data != `${kGlobalPrefBranch}.${kEnabledPref}` &&
          data != `${kGlobalPrefBranch}.${kProvidersPref}`
        ) {
          break;
        }
        this.notifyNewConfig();
        break;
    }
  },

  QueryInterface: ChromeUtils.generateQI([
    "nsIObserver",
    "nsISupportsWeakReference",
  ]),

  // Creates new config object from currently available
  // Remote Settings values.
  async updateFromRemoteSettings() {
    let providers = await gProvidersCollection.get();
    let config = await gConfigCollection.get();

    let providersById = new Map();
    providers.forEach(p => providersById.set(p.id, p));

    let configByRegion = new Map();
    config.forEach(c => {
      c.id = c.id.toLowerCase();
      configByRegion.set(c.id, c);
    });

    let homeRegion = Preferences.get(`${kGlobalPrefBranch}.home-region`);
    let localConfig =
      configByRegion.get(homeRegion?.toLowerCase()) ||
      configByRegion.get("global");

    // Make a new config object first, mutate it as needed, then synchronously
    // replace the currentConfig object at the end to ensure atomicity.
    let newConfig = makeBaseConfigObject();

    if (!localConfig) {
      DoHConfigController.currentConfig = newConfig;
      DoHConfigController.notifyNewConfig();
      return;
    }

    if (localConfig.rolloutEnabled) {
      delete newConfig.enabled;
      newConfig.enabled = true;
    }

    let parseProviderList = (list, checkFn) => {
      let parsedList = [];
      list?.split(",")?.forEach(p => {
        p = p.trim();
        if (!p.length) {
          return;
        }
        p = providersById.get(p);
        if (!p || (checkFn && !checkFn(p))) {
          return;
        }
        parsedList.push(p);
      });
      return parsedList;
    };

    let regionalProviders = parseProviderList(localConfig.providers);
    if (regionalProviders?.length) {
      delete newConfig.providerList;
      newConfig.providerList = regionalProviders;
    }

    if (localConfig.steeringEnabled) {
      let steeringProviders = parseProviderList(
        localConfig.steeringProviders,
        p => p.canonicalName?.length
      );
      if (steeringProviders?.length) {
        delete newConfig.providerSteering.providerList;
        newConfig.providerSteering.providerList = steeringProviders;

        delete newConfig.providerSteering.enabled;
        newConfig.providerSteering.enabled = true;
      }
    }

    if (localConfig.autoDefaultEnabled) {
      let defaultProviders = parseProviderList(
        localConfig.autoDefaultProviders
      );
      if (defaultProviders?.length) {
        delete newConfig.trrSelection.providerList;
        newConfig.trrSelection.providerList = defaultProviders;

        delete newConfig.trrSelection.enabled;
        newConfig.trrSelection.enabled = true;
      }
    }

    // Finally, update the currentConfig object synchronously.
    DoHConfigController.currentConfig = newConfig;

    DoHConfigController.notifyNewConfig();
  },

  kConfigUpdateTopic: "doh-config-updated",
  notifyNewConfig() {
    Services.obs.notifyObservers(null, this.kConfigUpdateTopic);
  },
};

Config.init();
DoHConfigController.initComplete = new Promise(resolve => {
  DoHConfigController._resolveInitComplete = resolve;
});
DoHConfigController.init();
+41 −23
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ XPCOMUtils.defineLazyModuleGetters(this, {
  AsyncShutdown: "resource://gre/modules/AsyncShutdown.jsm",
  ClientID: "resource://gre/modules/ClientID.jsm",
  ExtensionStorageIDB: "resource://gre/modules/ExtensionStorageIDB.jsm",
  Config: "resource:///modules/DoHConfig.jsm",
  DoHConfigController: "resource:///modules/DoHConfig.jsm",
  Heuristics: "resource:///modules/DoHHeuristics.jsm",
  Preferences: "resource://gre/modules/Preferences.jsm",
  setTimeout: "resource://gre/modules/Timer.jsm",
@@ -114,8 +114,6 @@ const BREADCRUMB_PREF = "doh-rollout.self-enabled";
const NETWORK_TRR_MODE_PREF = "network.trr.mode";
const NETWORK_TRR_URI_PREF = "network.trr.uri";

const TRR_LIST_PREF = "network.trr.resolvers";

const ROLLOUT_MODE_PREF = "doh-rollout.mode";
const ROLLOUT_URI_PREF = "doh-rollout.uri";

@@ -168,11 +166,13 @@ const DoHController = {
      true
    );

    Services.obs.addObserver(this, Config.kConfigUpdateTopic);
    await DoHConfigController.initComplete;

    Services.obs.addObserver(this, DoHConfigController.kConfigUpdateTopic);
    Preferences.observe(NETWORK_TRR_MODE_PREF, this);
    Preferences.observe(NETWORK_TRR_URI_PREF, this);

    if (Config.enabled) {
    if (DoHConfigController.currentConfig.enabled) {
      await this.maybeEnableHeuristics();
    } else if (Preferences.get(FIRST_RUN_PREF, false)) {
      await this.rollback();
@@ -193,7 +193,7 @@ const DoHController = {
  // Also used by tests to reset DoHController state (prefs are not cleared
  // here - tests do that when needed between _uninit and init).
  async _uninit() {
    Services.obs.removeObserver(this, Config.kConfigUpdateTopic);
    Services.obs.removeObserver(this, DoHConfigController.kConfigUpdateTopic);
    Preferences.ignore(NETWORK_TRR_MODE_PREF, this);
    Preferences.ignore(NETWORK_TRR_URI_PREF, this);
    AsyncShutdown.profileBeforeChange.removeBlocker(this._asyncShutdownBlocker);
@@ -201,9 +201,15 @@ const DoHController = {
  },

  // Called to reset state when a new config is available.
  resetPromise: Promise.resolve(),
  async reset() {
    this.resetPromise = this.resetPromise.then(async () => {
      await this._uninit();
      await this.init();
      Services.obs.notifyObservers(null, "doh:controller-reloaded");
    });

    return this.resetPromise;
  },

  async migrateLocalStoragePrefs() {
@@ -328,6 +334,14 @@ const DoHController = {
    }

    await this.runTRRSelection();
    // If we enter this branch it means that no automatic selection was possible.
    // In this case, we try to set a fallback (as defined by DoHConfigController).
    if (!Preferences.isSet(ROLLOUT_URI_PREF)) {
      Preferences.set(
        ROLLOUT_URI_PREF,
        DoHConfigController.currentConfig.fallbackProviderURI
      );
    }
    this.runHeuristicsThrottled("startup");
    Services.obs.addObserver(this, kLinkStatusChangedTopic);
    Services.obs.addObserver(this, kConnectivityTopic);
@@ -560,22 +574,26 @@ const DoHController = {
  async runTRRSelection() {
    // If persisting the selection is disabled, clear the existing
    // selection.
    if (!Config.trrSelection.commitResult) {
    if (!DoHConfigController.currentConfig.trrSelection.commitResult) {
      Preferences.reset(ROLLOUT_URI_PREF);
    }

    if (!Config.trrSelection.enabled) {
    if (!DoHConfigController.currentConfig.trrSelection.enabled) {
      return;
    }

    if (Preferences.isSet(ROLLOUT_URI_PREF)) {
    if (
      Preferences.isSet(ROLLOUT_URI_PREF) &&
      Preferences.get(ROLLOUT_URI_PREF) ==
        Preferences.get(TRR_SELECT_DRY_RUN_RESULT_PREF)
    ) {
      return;
    }

    await this.runTRRSelectionDryRun();

    // If persisting the selection is disabled, don't commit the value.
    if (!Config.trrSelection.commitResult) {
    if (!DoHConfigController.currentConfig.trrSelection.commitResult) {
      return;
    }

@@ -590,31 +608,28 @@ const DoHController = {
      // Check whether the existing dry-run-result is in the default
      // list of TRRs. If it is, all good. Else, run the dry run again.
      let dryRunResult = Preferences.get(TRR_SELECT_DRY_RUN_RESULT_PREF);
      let defaultTRRs = JSON.parse(
        Services.prefs.getDefaultBranch("").getCharPref(TRR_LIST_PREF)
      );
      let dryRunResultIsValid = defaultTRRs.some(
        trr => trr.url == dryRunResult
      let dryRunResultIsValid = DoHConfigController.currentConfig.providerList.some(
        trr => trr.uri == dryRunResult
      );
      if (dryRunResultIsValid) {
        return;
      }
    }

    let setDryRunResultAndRecordTelemetry = trr => {
      Preferences.set(TRR_SELECT_DRY_RUN_RESULT_PREF, trr);
    let setDryRunResultAndRecordTelemetry = trrUri => {
      Preferences.set(TRR_SELECT_DRY_RUN_RESULT_PREF, trrUri);
      Services.telemetry.recordEvent(
        TRRSELECT_TELEMETRY_CATEGORY,
        "trrselect",
        "dryrunresult",
        trr.substring(0, 40) // Telemetry payload max length
        trrUri.substring(0, 40) // Telemetry payload max length
      );
    };

    if (kIsInAutomation) {
      // For mochitests, just record telemetry with a dummy result.
      // TRRPerformance.jsm is tested in xpcshell.
      setDryRunResultAndRecordTelemetry("https://dummytrr.com/query");
      setDryRunResultAndRecordTelemetry("https://example.com/dns-query");
      return;
    }

@@ -624,10 +639,13 @@ const DoHController = {
      "resource:///modules/TRRPerformance.jsm"
    );
    await new Promise(resolve => {
      let trrList = DoHConfigController.currentConfig.trrSelection.providerList.map(
        trr => trr.uri
      );
      let racer = new TRRRacer(() => {
        setDryRunResultAndRecordTelemetry(racer.getFastestTRR(true));
        resolve();
      });
      }, trrList);
      racer.run();
    });
  },
@@ -643,7 +661,7 @@ const DoHController = {
      case kPrefChangedTopic:
        this.onPrefChanged(data);
        break;
      case Config.kConfigUpdateTopic:
      case DoHConfigController.kConfigUpdateTopic:
        this.reset();
        break;
    }
+4 −9
Original line number Diff line number Diff line
@@ -39,7 +39,7 @@ XPCOMUtils.defineLazyServiceGetter(

ChromeUtils.defineModuleGetter(
  this,
  "Config",
  "DoHConfigController",
  "resource:///modules/DoHConfig.jsm"
);

@@ -363,7 +363,7 @@ async function platform() {
// provider if the check is successful, else null. Currently we only support
// this for Comcast networks.
async function providerSteering() {
  if (!Config.providerSteering.enabled) {
  if (!DoHConfigController.currentConfig.providerSteering.enabled) {
    return null;
  }
  const TEST_DOMAIN = "doh.test.";
@@ -371,13 +371,8 @@ async function providerSteering() {
  // Array of { name, canonicalName, uri } where name is an identifier for
  // telemetry, canonicalName is the expected CNAME when looking up doh.test,
  // and uri is the provider's DoH endpoint.
  let steeredProviders = Config.providerSteering.providerList;
  try {
    steeredProviders = JSON.parse(steeredProviders);
  } catch (e) {
    console.log("Provider list is invalid JSON, moving on.");
    return null;
  }
  let steeredProviders =
    DoHConfigController.currentConfig.providerSteering.providerList;

  if (!steeredProviders || !steeredProviders.length) {
    return null;
+142 −0
Original line number Diff line number Diff line
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

"use strict";

var EXPORTED_SYMBOLS = ["DoHTestUtils"];

ChromeUtils.defineModuleGetter(
  this,
  "RemoteSettings",
  "resource://services-settings/remote-settings.js"
);

ChromeUtils.defineModuleGetter(
  this,
  "TestUtils",
  "resource://testing-common/TestUtils.jsm"
);

const kConfigCollectionKey = "doh-config";
const kProviderCollectionKey = "doh-providers";

const kConfigUpdateTopic = "doh-config-updated";
const kControllerReloadedTopic = "doh:controller-reloaded";

/*
 * Some helpers for loading and modifying DoH config in
 * Remote Settings. Call resetRemoteSettingsConfig to set up
 * basic default config that omits external URLs. Use
 * waitForConfigFlush to wait for DoH actors to pick up changes.
 *
 * Some tests need to load/reset config while DoH actors are
 * uninitialized. Pass waitForConfigFlushes = false in these cases.
 */
const DoHTestUtils = {
  providers: [
    {
      uri: "https://example.com/1",
      UIName: "Example 1",
      autoDefault: false,
      canonicalName: "",
      id: "example-1",
    },
    {
      uri: "https://example.com/2",
      UIName: "Example 2",
      autoDefault: false,
      canonicalName: "",
      id: "example-2",
    },
  ],

  async loadRemoteSettingsProviders(providers, waitForConfigFlushes = true) {
    let configFlushedPromise = this.waitForConfigFlush(waitForConfigFlushes);

    let providerRS = RemoteSettings(kProviderCollectionKey);
    let db = await providerRS.db;
    await db.importChanges({}, Date.now(), providers, { clear: true });

    // Trigger a sync.
    await this.triggerSync(providerRS);

    await configFlushedPromise;
  },

  async loadRemoteSettingsConfig(config, waitForConfigFlushes = true) {
    let configFlushedPromise = this.waitForConfigFlush(waitForConfigFlushes);

    let configRS = RemoteSettings(kConfigCollectionKey);
    let db = await configRS.db;
    await db.importChanges({}, Date.now(), [config]);

    // Trigger a sync.
    await this.triggerSync(configRS);

    await configFlushedPromise;
  },

  // Loads default config for testing without clearing existing entries.
  async loadDefaultRemoteSettingsConfig(waitForConfigFlushes = true) {
    await this.loadRemoteSettingsProviders(
      this.providers,
      waitForConfigFlushes
    );

    await this.loadRemoteSettingsConfig(
      {
        providers: "example-1, example-2",
        rolloutEnabled: false,
        steeringEnabled: false,
        steeringProviders: "",
        autoDefaultEnabled: false,
        autoDefaultProviders: "",
        id: "global",
      },
      waitForConfigFlushes
    );
  },

  // Clears existing config AND loads defaults.
  async resetRemoteSettingsConfig(waitForConfigFlushes = true) {
    let providerRS = RemoteSettings(kProviderCollectionKey);
    let configRS = RemoteSettings(kConfigCollectionKey);
    for (let rs of [providerRS, configRS]) {
      let configFlushedPromise = this.waitForConfigFlush(waitForConfigFlushes);
      await rs.db.importChanges({}, 1234567, [], { clear: true });
      // Trigger a sync to clear.
      await this.triggerSync(rs);
      await configFlushedPromise;
    }

    await this.loadDefaultRemoteSettingsConfig(waitForConfigFlushes);
  },

  triggerSync(rs) {
    return rs.emit("sync", {
      data: {
        current: [],
      },
    });
  },

  waitForConfigUpdate() {
    return TestUtils.topicObserved(kConfigUpdateTopic);
  },

  waitForControllerReload() {
    return TestUtils.topicObserved(kControllerReloadedTopic);
  },

  waitForConfigFlush(shouldWait = true) {
    if (!shouldWait) {
      return Promise.resolve();
    }

    return Promise.all([
      this.waitForConfigUpdate(),
      this.waitForControllerReload(),
    ]);
  },
};
+10 −10
Original line number Diff line number Diff line
@@ -58,11 +58,6 @@ XPCOMUtils.defineLazyServiceGetter(
  "nsIUUIDGenerator"
);

// The list of participating TRRs.
const kTRRs = JSON.parse(
  Services.prefs.getDefaultBranch("").getCharPref("network.trr.resolvers")
).map(trr => trr.url);

// The canonical domain whose subdomains we will be resolving.
XPCOMUtils.defineLazyPreferenceGetter(
  this,
@@ -154,8 +149,9 @@ DNSLookup.prototype.QueryInterface = ChromeUtils.generateQI(["nsIDNSListener"]);
// triggered and the results aggregated before telemetry is sent. If aborted,
// any aggregated results are discarded.
class LookupAggregator {
  constructor(onCompleteCallback) {
  constructor(onCompleteCallback, trrList) {
    this.onCompleteCallback = onCompleteCallback;
    this.trrList = trrList;
    this.aborted = false;
    this.networkUnstable = false;
    this.captivePortal = false;
@@ -166,7 +162,7 @@ class LookupAggregator {
      this.domains.push(null);
    }
    this.domains.push(...kPopularDomains);
    this.totalLookups = kTRRs.length * this.domains.length;
    this.totalLookups = this.trrList.length * this.domains.length;
    this.completedLookups = 0;
    this.results = [];
  }
@@ -178,7 +174,7 @@ class LookupAggregator {
    }

    this._ran = true;
    for (let trr of kTRRs) {
    for (let trr of this.trrList) {
      for (let domain of this.domains) {
        new DNSLookup(
          domain,
@@ -256,11 +252,12 @@ class LookupAggregator {
// spawned next time we get a link, up to 5 times. On the fifth time, we just
// let the aggegator complete and mark it as tainted.
class TRRRacer {
  constructor(onCompleteCallback) {
  constructor(onCompleteCallback, trrList) {
    this._aggregator = null;
    this._retryCount = 0;
    this._complete = false;
    this._onCompleteCallback = onCompleteCallback;
    this._trrList = trrList;
  }

  run() {
@@ -365,7 +362,10 @@ class TRRRacer {
  }

  _runNewAggregator() {
    this._aggregator = new LookupAggregator(() => this.onComplete());
    this._aggregator = new LookupAggregator(
      () => this.onComplete(),
      this._trrList
    );
    this._aggregator.run();
    this._retryCount++;
  }
Loading