Commit e9c2babf authored by Erik Nordin's avatar Erik Nordin
Browse files

Bug 1836436 - Add telemetry tests for translation failure r=gregtatum

Depends on D179770

Differential Revision: https://phabricator.services.mozilla.com/D179771
parent 21070825
Loading
Loading
Loading
Loading
+1 −0
Original line number Diff line number Diff line
@@ -15,4 +15,5 @@ support-files =
[browser_translations_panel_retry.js]
[browser_translations_panel_switch_languages.js]
[browser_translations_telemetry_open_panel.js]
[browser_translations_telemetry_translation_failure.js]
[browser_translations_telemetry_translation_request.js]
+195 −0
Original line number Diff line number Diff line
/* Any copyright is dedicated to the Public Domain.
   http://creativecommons.org/publicdomain/zero/1.0/ */

"use strict";

const { PromiseTestUtils } = ChromeUtils.importESModule(
  "resource://testing-common/PromiseTestUtils.sys.mjs"
);

/**
 * Tests the telemetry event for a manual translation request failure.
 */
add_task(
  async function test_translations_telemetry_manual_translation_failure() {
    PromiseTestUtils.expectUncaughtRejection(
      /Intentionally rejecting downloads./
    );

    const { cleanup, rejectDownloads, runInPage } = await loadTestPage({
      page: SPANISH_PAGE_URL,
      languagePairs: LANGUAGE_PAIRS,
    });

    const { button } = await assertTranslationsButton(
      { button: true, circleArrows: false, locale: false, icon: true },
      "The button is available."
    );

    await runInPage(async TranslationsTest => {
      const { getH1 } = TranslationsTest.getSelectors();
      await TranslationsTest.assertTranslationResult(
        "The page's H1 is in Spanish.",
        getH1,
        "Don Quijote de La Mancha"
      );
    });

    await TestTranslationsTelemetry.assertCounter(
      "RequestCount",
      Glean.translations.requestsCount,
      0
    );
    await TestTranslationsTelemetry.assertRate(
      "ErrorRate",
      Glean.translations.errorRate,
      {
        expectedNumerator: 0,
        expectedDenominator: 0,
      }
    );
    await TestTranslationsTelemetry.assertEvent(
      "TranslationRequest",
      Glean.translations.translationRequest,
      {
        expectedLength: 0,
      }
    );

    await waitForTranslationsPopupEvent("popupshown", () => {
      click(button, "Opening the popup");
    });

    await waitForTranslationsPopupEvent("popuphidden", () => {
      click(
        getByL10nId("translations-panel-translate-button"),
        "Start translating by clicking the translate button."
      );
    });

    await assertTranslationsButton(
      { button: true, circleArrows: true, locale: false, icon: true },
      "The icon presents the loading indicator."
    );

    await rejectDownloads(1);

    await runInPage(async TranslationsTest => {
      const { getH1 } = TranslationsTest.getSelectors();
      await TranslationsTest.assertTranslationResult(
        "The page's H1 is in Spanish.",
        getH1,
        "Don Quijote de La Mancha"
      );
    });

    await TestTranslationsTelemetry.assertCounter(
      "RequestCount",
      Glean.translations.requestsCount,
      1
    );
    await TestTranslationsTelemetry.assertRate(
      "ErrorRate",
      Glean.translations.errorRate,
      {
        expectedNumerator: 1,
        expectedDenominator: 1,
      }
    );
    await TestTranslationsTelemetry.assertEvent(
      "Error",
      Glean.translations.error,
      {
        expectedLength: 1,
        finalValuePredicates: [
          value =>
            value.extra.reason === "Error: Intentionally rejecting downloads.",
        ],
      }
    );
    await TestTranslationsTelemetry.assertEvent(
      "TranslationRequest",
      Glean.translations.translationRequest,
      {
        expectedLength: 1,
        finalValuePredicates: [
          value => value.extra.from_language === "es",
          value => value.extra.to_language === "en",
          value => value.extra.auto_translate === "false",
        ],
      }
    );

    await cleanup();
  }
);

/**
 * Tests the telemetry event for an automatic translation request failure.
 */
add_task(async function test_translations_telemetry_auto_translation_failure() {
  PromiseTestUtils.expectUncaughtRejection(
    /Intentionally rejecting downloads./
  );

  const { cleanup, rejectDownloads, runInPage } = await loadTestPage({
    page: SPANISH_PAGE_URL,
    languagePairs: LANGUAGE_PAIRS,
    prefs: [["browser.translations.alwaysTranslateLanguages", "es"]],
  });

  await assertTranslationsButton(
    { button: true, circleArrows: true, locale: false, icon: true },
    "The icon presents the loading indicator."
  );

  await rejectDownloads(1);

  await runInPage(async TranslationsTest => {
    const { getH1 } = TranslationsTest.getSelectors();
    await TranslationsTest.assertTranslationResult(
      "The page's H1 is in Spanish.",
      getH1,
      "Don Quijote de La Mancha"
    );
  });

  await TestTranslationsTelemetry.assertCounter(
    "RequestCount",
    Glean.translations.requestsCount,
    1
  );
  await TestTranslationsTelemetry.assertRate(
    "ErrorRate",
    Glean.translations.errorRate,
    {
      expectedNumerator: 1,
      expectedDenominator: 1,
    }
  );
  await TestTranslationsTelemetry.assertEvent(
    "Error",
    Glean.translations.error,
    {
      expectedLength: 1,
      finalValuePredicates: [
        value =>
          value.extra.reason === "Error: Intentionally rejecting downloads.",
      ],
    }
  );
  await TestTranslationsTelemetry.assertEvent(
    "TranslationRequest",
    Glean.translations.translationRequest,
    {
      expectedLength: 1,
      finalValuePredicates: [
        value => value.extra.from_language === "es",
        value => value.extra.to_language === "en",
        value => value.extra.auto_translate === "true",
      ],
    }
  );

  await cleanup();
});
+39 −0
Original line number Diff line number Diff line
@@ -26,6 +26,19 @@ add_task(async function test_translations_telemetry_manual_translation() {
    );
  });

  await TestTranslationsTelemetry.assertCounter(
    "RequestCount",
    Glean.translations.requestsCount,
    0
  );
  await TestTranslationsTelemetry.assertRate(
    "ErrorRate",
    Glean.translations.errorRate,
    {
      expectedNumerator: 0,
      expectedDenominator: 0,
    }
  );
  await TestTranslationsTelemetry.assertEvent(
    "TranslationRequest",
    Glean.translations.translationRequest,
@@ -68,6 +81,19 @@ add_task(async function test_translations_telemetry_manual_translation() {
    );
  });

  await TestTranslationsTelemetry.assertCounter(
    "RequestCount",
    Glean.translations.requestsCount,
    1
  );
  await TestTranslationsTelemetry.assertRate(
    "ErrorRate",
    Glean.translations.errorRate,
    {
      expectedNumerator: 0,
      expectedDenominator: 1,
    }
  );
  await TestTranslationsTelemetry.assertEvent(
    "TranslationRequest",
    Glean.translations.translationRequest,
@@ -117,6 +143,19 @@ add_task(async function test_translations_telemetry_auto_translation() {
    );
  });

  await TestTranslationsTelemetry.assertCounter(
    "RequestCount",
    Glean.translations.requestsCount,
    1
  );
  await TestTranslationsTelemetry.assertRate(
    "ErrorRate",
    Glean.translations.errorRate,
    {
      expectedNumerator: 0,
      expectedDenominator: 1,
    }
  );
  await TestTranslationsTelemetry.assertEvent(
    "TranslationRequest",
    Glean.translations.translationRequest,
+60 −1
Original line number Diff line number Diff line
@@ -429,6 +429,16 @@ async function loadTestPage({
      );
    },

    /**
     * @param {number} count - Count of the language pairs expected.
     */
    async rejectDownloads(count) {
      await remoteClients.translationsWasm.rejectPendingDownloads(1);
      await remoteClients.translationModels.rejectPendingDownloads(
        FILES_PER_LANGUAGE_PAIR * count
      );
    },

    async resolveLanguageIdDownloads() {
      await remoteClients.translationsWasm.resolvePendingDownloads(1);
      await remoteClients.languageIdModels.resolvePendingDownloads(1);
@@ -888,10 +898,29 @@ async function mockLocales({ systemLocales, appLocales, webLanguages }) {
 * Helpful test functions for translations telemetry
 */
class TestTranslationsTelemetry {
  /**
   * Asserts qualities about a counter telemetry metric.
   *
   * @param {string} name - The name of the metric.
   * @param {Object} counter - The Glean counter object.
   * @param {Object} expectedCount - The expected value of the counter.
   */
  static async assertCounter(name, counter, expectedCount) {
    // Ensures that glean metrics are collected from all child processes
    // so that calls to testGetValue() are up to date.
    await Services.fog.testFlushAllChildren();
    const count = counter.testGetValue() ?? 0;
    is(
      count,
      expectedCount,
      `Telemetry counter ${name} should have expected count`
    );
  }

  /**
   * Asserts qualities about an event telemetry metric.
   *
   * @param {string} name - The name of the event.
   * @param {string} name - The name of the metric.
   * @param {Object} event - The Glean event object.
   * @param {Object} expectations - The test expectations.
   * @param {number} expectations.expectedLength - The expected length of the event.
@@ -949,4 +978,34 @@ class TestTranslationsTelemetry {
      }
    }
  }

  /**
   * Asserts qualities about a rate telemetry metric.
   *
   * @param {string} name - The name of the metric.
   * @param {Object} rate - The Glean rate object.
   * @param {Object} expectations - The test expectations.
   * @param {number} expectations.expectedNumerator - The expected value of the numerator.
   * @param {number} expectations.expectedDenominator - The expected value of the denominator.
   */
  static async assertRate(
    name,
    rate,
    { expectedNumerator, expectedDenominator }
  ) {
    // Ensures that glean metrics are collected from all child processes
    // so that calls to testGetValue() are up to date.
    await Services.fog.testFlushAllChildren();
    const { numerator = 0, denominator = 0 } = rate.testGetValue() ?? {};
    is(
      numerator,
      expectedNumerator,
      `Telemetry rate ${name} should have expected numerator`
    );
    is(
      denominator,
      expectedDenominator,
      `Telemetry rate ${name} should have expected denominator`
    );
  }
}