Bug 1776870 - Integrate esmify script into mach. r=yulia,firefox-static-analysis-reviewers,andi

Differential Revision: https://phabricator.services.mozilla.com/D150494

.gitignore

@@ -193,3 +193,9 @@ config/external/icu4x
 # Ignore Storybook generated files
 browser/components/storybook/node_modules/
 browser/components/storybook/storybook-static/
+
+# Ignore jscodeshift installed by mach esmify on windows
+tools/esmify/jscodeshift
+tools/esmify/jscodeshift.cmd
+tools/esmify/jscodeshift.ps1
+tools/esmify/package-lock.json

.hgignore

@@ -157,6 +157,7 @@ _OPT\.OBJ/
 ^tools/browsertime/node_modules/
 ^tools/lint/eslint/eslint-plugin-mozilla/node_modules/
 ^browser/components/newtab/node_modules/
+^tools/esmify/node_modules/
 
 # Ignore talos virtualenv and tp5n files.
 # The tp5n set is supposed to be decompressed at
@@ -246,3 +247,9 @@ toolkit/components/certviewer/content/package-lock.json
 # Ignore Storybook generated files
 ^browser/components/storybook/node_modules/
 ^browser/components/storybook/storybook-static/
+
+# Ignore jscodeshift installed by mach esmify on windows
+^tools/esmify/jscodeshift
+^tools/esmify/jscodeshift.cmd
+^tools/esmify/jscodeshift.ps1
+^tools/esmify/package-lock.json

build/mach_initialize.py

@@ -358,6 +358,7 @@ def initialize(topsrcdir):
         "test-interventions": MachCommandReference(
             "testing/webcompat/mach_commands.py"
         ),
+        "esmify": MachCommandReference("tools/esmify/mach_commands.py"),
     }
 
     # Set a reasonable limit to the number of open files.

tools/esmify/exported_symbols-to-declarations.js

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// jscodeshift rule to replace EXPORTED_SYMBOLS with export declarations.
+
+/* global module */
+
+module.exports = function(fileInfo, api) {
+  const { jscodeshift } = api;
+  const root = jscodeshift(fileInfo.source);
+  doTranslate(fileInfo.path, jscodeshift, root);
+  return root.toSource({ lineTerminator: "\n" });
+};
+
+module.exports.doTranslate = doTranslate;
+
+function IsIdentifier(node, name) {
+  if (node.type !== "Identifier") {
+    return false;
+  }
+  if (node.name !== name) {
+    return false;
+  }
+  return true;
+}
+
+function moveComments(inputFile, path) {
+  const parent = path.parent;
+  if (parent.node.type !== "Program") {
+    warnForPath(
+      inputFile,
+      path,
+      `EXPORTED_SYMBOLS has comments and it cannot be preserved`
+    );
+    return;
+  }
+
+  const index = parent.node.body.findIndex(n => n == path.node);
+  if (index === -1) {
+    warnForPath(
+      inputFile,
+      path,
+      `EXPORTED_SYMBOLS has comments and it cannot be preserved`
+    );
+    return;
+  }
+
+  if (index + 1 < parent.node.body.length) {
+    const next = parent.node.body[index + 1];
+    if (next.comments) {
+      next.comments = [...path.node.comments, ...next.comments];
+    } else {
+      next.comments = path.node.comments;
+    }
+    path.node.comments = [];
+  } else if (index > 0) {
+    const prev = parent.node.body[index - 1];
+
+    path.node.comments.forEach(c => {
+      c.leading = false;
+      c.trailing = true;
+    });
+
+    if (prev.comments) {
+      prev.comments = [...prev.comments, ...path.node.comments];
+    } else {
+      prev.comments = path.node.comments;
+    }
+    path.node.comments = [];
+  }
+}
+
+function warnForPath(inputFile, path, message) {
+  const loc = path.node.loc;
+  console.log(
+    `WARNING: ${inputFile}:${loc.start.line}:${loc.start.column} : ${message}`
+  );
+}
+
+function collectAndRemoveExportedSymbols(inputFile, root) {
+  const nodes = root.findVariableDeclarators("EXPORTED_SYMBOLS");
+  if (!nodes.length) {
+    throw Error(`EXPORTED_SYMBOLS not found`);
+  }
+
+  let path = nodes.get(0);
+  const obj = nodes.get(0).node.init;
+  if (!obj) {
+    throw Error(`EXPORTED_SYMBOLS is not statically known`);
+  }
+
+  if (path.parent.node.declarations.length !== 1) {
+    throw Error(`EXPORTED_SYMBOLS shouldn't be declared with other variables`);
+  }
+
+  if (path.parent.node.comments && path.parent.node.comments.length) {
+    moveComments(inputFile, path.parent);
+  }
+
+  path.parent.prune();
+
+  const EXPORTED_SYMBOLS = new Set();
+  if (obj.type !== "ArrayExpression") {
+    throw Error(`EXPORTED_SYMBOLS is not statically known`);
+  }
+
+  for (const elem of obj.elements) {
+    if (elem.type !== "Literal") {
+      throw Error(`EXPORTED_SYMBOLS is not statically known`);
+    }
+    var name = elem.value;
+    if (typeof name !== "string") {
+      throw Error(`EXPORTED_SYMBOLS item must be a string`);
+    }
+    EXPORTED_SYMBOLS.add(name);
+  }
+
+  return EXPORTED_SYMBOLS;
+}
+
+function isTopLevel(path) {
+  return path.parent.node.type === "Program";
+}
+
+function convertToExport(jscodeshift, path, name) {
+  const e = jscodeshift.exportNamedDeclaration(path.node);
+  e.comments = [];
+  e.comments = path.node.comments;
+  path.node.comments = [];
+
+  path.replace(e);
+}
+
+function doTranslate(inputFile, jscodeshift, root) {
+  const EXPORTED_SYMBOLS = collectAndRemoveExportedSymbols(inputFile, root);
+
+  root.find(jscodeshift.FunctionDeclaration).forEach(path => {
+    if (!isTopLevel(path)) {
+      return;
+    }
+    const name = path.node.id.name;
+    if (!EXPORTED_SYMBOLS.has(name)) {
+      return;
+    }
+    EXPORTED_SYMBOLS.delete(name);
+    convertToExport(jscodeshift, path, name);
+  });
+
+  root.find(jscodeshift.ClassDeclaration).forEach(path => {
+    if (!isTopLevel(path)) {
+      return;
+    }
+    const name = path.node.id.name;
+    if (!EXPORTED_SYMBOLS.has(name)) {
+      return;
+    }
+    EXPORTED_SYMBOLS.delete(name);
+    convertToExport(jscodeshift, path, name);
+  });
+
+  root.find(jscodeshift.VariableDeclaration).forEach(path => {
+    if (!isTopLevel(path)) {
+      return;
+    }
+
+    let exists = false;
+    let name;
+    for (const decl of path.node.declarations) {
+      if (decl.id.type === "Identifier") {
+        name = decl.id.name;
+        if (EXPORTED_SYMBOLS.has(name)) {
+          exists = true;
+          break;
+        }
+      }
+
+      if (decl.id.type === "ObjectPattern") {
+        if (decl.id.properties.length === 1) {
+          const prop = decl.id.properties[0];
+          if (prop.shorthand) {
+            if (prop.key.type === "Identifier") {
+              name = prop.key.name;
+              if (EXPORTED_SYMBOLS.has(name)) {
+                exists = true;
+                break;
+              }
+            }
+          }
+        }
+      }
+    }
+    if (!exists) {
+      return;
+    }
+
+    if (path.node.declarations.length !== 1) {
+      throw Error(
+        `exported variable shouldn't be declared with other variables`
+      );
+    }
+
+    EXPORTED_SYMBOLS.delete(name);
+    convertToExport(jscodeshift, path, name);
+  });
+
+  if (EXPORTED_SYMBOLS.size !== 0) {
+    throw Error(
+      `exported symbols ${[...EXPORTED_SYMBOLS].join(", ")} not found`
+    );
+  }
+}

tools/esmify/import-to-import_esmodule.js

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// jscodeshift rule to replace import calls for JSM with import calls for ESM
+// or static import for ESM.
+
+/* global require, __dirname, process */
+
+const _path = require("path");
+const { isESMified } = require(_path.resolve(__dirname, "./is-esmified.js"));
+
+/* global module */
+
+module.exports = function(fileInfo, api) {
+  const { jscodeshift } = api;
+  const root = jscodeshift(fileInfo.source);
+  doTranslate(fileInfo.path, jscodeshift, root);
+  return root.toSource({ lineTerminator: "\n" });
+};
+
+module.exports.doTranslate = doTranslate;
+
+function isESMifiedAndTarget(resourceURI) {
+  const files = [];
+  if (!isESMified(resourceURI, files)) {
+    return false;
+  }
+
+  if ("ESMIFY_TARGET_PREFIX" in process.env) {
+    const targetPrefix = process.env.ESMIFY_TARGET_PREFIX;
+    for (const esm of files) {
+      if (esm.startsWith(targetPrefix)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  return true;
+}
+
+function IsIdentifier(node, name) {
+  if (node.type !== "Identifier") {
+    return false;
+  }
+  if (node.name !== name) {
+    return false;
+  }
+  return true;
+}
+
+function calleeToString(node) {
+  if (node.type === "Identifier") {
+    return node.name;
+  }
+
+  if (node.type === "MemberExpression" && !node.computed) {
+    return calleeToString(node.object) + "." + node.property.name;
+  }
+
+  return "???";
+}
+
+function isImportCall(node) {
+  const s = calleeToString(node.callee);
+  return ["Cu.import", "ChromeUtils.import"].includes(s);
+}
+
+function isLazyGetterCall(node) {
+  const s = calleeToString(node.callee);
+  return [
+    "XPCOMUtils.defineLazyModuleGetter",
+    "ChromeUtils.defineModuleGetter",
+  ].includes(s);
+}
+
+function isLazyGettersCall(node) {
+  const s = calleeToString(node.callee);
+  return ["XPCOMUtils.defineLazyModuleGetters"].includes(s);
+}
+
+function warnForPath(inputFile, path, message) {
+  const loc = path.node.loc;
+  console.log(
+    `WARNING: ${inputFile}:${loc.start.line}:${loc.start.column} : ${message}`
+  );
+}
+
+const extPattern = /\.(jsm|js|jsm\.js)$/;
+
+function esmify(path) {
+  return path.replace(extPattern, ".sys.mjs");
+}
+
+function isString(node) {
+  return node.type === "Literal" && typeof node.value === "string";
+}
+
+function tryReplacingWithStatciImport(
+  jscodeshift,
+  inputFile,
+  path,
+  resourceURINode
+) {
+  if (!inputFile.endsWith(".sys.mjs")) {
+    // Static import is available only in system ESM.
+    return false;
+  }
+
+  if (path.parent.node.type !== "VariableDeclarator") {
+    return false;
+  }
+
+  if (path.parent.parent.node.type !== "VariableDeclaration") {
+    return false;
+  }
+
+  const decls = path.parent.parent.node;
+  if (decls.declarations.length !== 1) {
+    return false;
+  }
+
+  if (path.parent.parent.parent.node.type !== "Program") {
+    return false;
+  }
+
+  if (path.node.arguments.length !== 1) {
+    return false;
+  }
+
+  const resourceURI = resourceURINode.value;
+
+  const specs = [];
+
+  if (path.parent.node.id.type === "Identifier") {
+    specs.push(jscodeshift.importNamespaceSpecifier(path.parent.node.id));
+  } else if (path.parent.node.id.type === "ObjectPattern") {
+    for (const prop of path.parent.node.id.properties) {
+      if (prop.shorthand) {
+        specs.push(jscodeshift.importSpecifier(prop.key));
+      } else if (prop.value.type === "Identifier") {
+        specs.push(jscodeshift.importSpecifier(prop.key, prop.value));
+      } else {
+        return false;
+      }
+    }
+  } else {
+    return false;
+  }
+
+  resourceURINode.value = esmify(resourceURI);
+
+  const e = jscodeshift.importDeclaration(specs, resourceURINode);
+  e.comments = path.parent.parent.node.comments;
+  path.parent.parent.node.comments = [];
+  path.parent.parent.replace(e);
+
+  return true;
+}
+
+function replaceImportCall(inputFile, jscodeshift, path) {
+  if (path.node.arguments.length !== 1) {
+    warnForPath(inputFile, path, `import call should have only one argument`);
+    return;
+  }
+
+  const resourceURINode = path.node.arguments[0];
+  if (!isString(resourceURINode)) {
+    warnForPath(inputFile, path, `resource URI should be a string`);
+    return;
+  }
+
+  const resourceURI = resourceURINode.value;
+  if (!resourceURI.match(extPattern)) {
+    warnForPath(inputFile, path, `Non-jsm: ${resourceURI}`);
+    return;
+  }
+
+  if (!isESMifiedAndTarget(resourceURI)) {
+    return;
+  }
+
+  if (
+    !tryReplacingWithStatciImport(jscodeshift, inputFile, path, resourceURINode)
+  ) {
+    path.node.callee.object.name = "ChromeUtils";
+    path.node.callee.property.name = "importESModule";
+    resourceURINode.value = esmify(resourceURI);
+  }
+}
+
+function replaceLazyGetterCall(inputFile, jscodeshift, path) {
+  if (path.node.arguments.length !== 3) {
+    warnForPath(inputFile, path, `lazy getter call should have 3 arguments`);
+    return;
+  }
+
+  const nameNode = path.node.arguments[1];
+  if (!isString(nameNode)) {
+    warnForPath(inputFile, path, `name should be a string`);
+    return;
+  }
+
+  const resourceURINode = path.node.arguments[2];
+  if (!isString(resourceURINode)) {
+    warnForPath(inputFile, path, `resource URI should be a string`);
+    return;
+  }
+
+  const resourceURI = resourceURINode.value;
+  if (!resourceURI.match(extPattern)) {
+    warnForPath(inputFile, path, `Non-js/jsm: ${resourceURI}`);
+    return;
+  }
+
+  if (!isESMifiedAndTarget(resourceURI)) {
+    return;
+  }
+
+  path.node.callee.object.name = "ChromeUtils";
+  path.node.callee.property.name = "defineESModuleGetters";
+  resourceURINode.value = esmify(resourceURI);
+  path.node.arguments = [
+    path.node.arguments[0],
+    jscodeshift.objectExpression([
+      jscodeshift.property("init", nameNode, resourceURINode),
+    ]),
+  ];
+}
+
+function replaceLazyGettersCall(inputFile, jscodeshift, path) {
+  if (path.node.arguments.length !== 2) {
+    warnForPath(inputFile, path, `lazy getters call should have 2 arguments`);
+    return;
+  }
+
+  const modulesNode = path.node.arguments[1];
+  if (modulesNode.type !== "ObjectExpression") {
+    warnForPath(inputFile, path, `modules parameter should be an object`);
+    return;
+  }
+
+  const esmProps = [];
+  const jsmProps = [];
+
+  for (const prop of modulesNode.properties) {
+    const resourceURINode = prop.value;
+    if (!isString(resourceURINode)) {
+      warnForPath(inputFile, path, `resource URI should be a string`);
+      jsmProps.push(prop);
+      continue;
+    }
+
+    const resourceURI = resourceURINode.value;
+    if (!resourceURI.match(extPattern)) {
+      warnForPath(inputFile, path, `Non-js/jsm: ${resourceURI}`);
+      jsmProps.push(prop);
+      continue;
+    }
+
+    if (!isESMifiedAndTarget(resourceURI)) {
+      jsmProps.push(prop);
+      continue;
+    }
+
+    esmProps.push(prop);
+  }
+
+  if (esmProps.length === 0) {
+    return;
+  }
+
+  if (jsmProps.length === 0) {
+    path.node.callee.object.name = "ChromeUtils";
+    path.node.callee.property.name = "defineESModuleGetters";
+    for (const prop of esmProps) {
+      const resourceURINode = prop.value;
+      resourceURINode.value = esmify(resourceURINode.value);
+    }
+  } else {
+    if (path.parent.node.type !== "ExpressionStatement") {
+      warnForPath(inputFile, path, `lazy getters call in unexpected context`);
+      return;
+    }
+
+    for (const prop of esmProps) {
+      const resourceURINode = prop.value;
+      resourceURINode.value = esmify(resourceURINode.value);
+    }
+
+    const callStmt = jscodeshift.expressionStatement(
+      jscodeshift.callExpression(
+        jscodeshift.memberExpression(
+          jscodeshift.identifier("ChromeUtils"),
+          jscodeshift.identifier("defineESModuleGetters")
+        ),
+        [path.node.arguments[0], jscodeshift.objectExpression(esmProps)]
+      )
+    );
+
+    callStmt.comments = path.parent.node.comments;
+    path.parent.node.comments = [];
+    path.parent.insertBefore(callStmt);
+
+    path.node.arguments[1].properties = jsmProps;
+  }
+}
+
+function getProp(obj, key) {
+  if (obj.type !== "ObjectExpression") {
+    return null;
+  }
+
+  for (const prop of obj.properties) {
+    if (prop.computed) {
+      continue;
+    }
+
+    if (!prop.key) {
+      continue;
+    }
+
+    if (IsIdentifier(prop.key, key)) {
+      return prop;
+    }
+  }
+
+  return null;
+}
+
+function tryReplaceActorDefinition(inputFile, path, name) {
+  const obj = path.node;
+
+  const prop = getProp(obj, name);
+  if (!prop) {
+    return;
+  }
+
+  const moduleURIProp = getProp(prop.value, "moduleURI");
+  if (!moduleURIProp) {
+    return;
+  }
+
+  if (!isString(moduleURIProp.value)) {
+    warnForPath(inputFile, path, `${name} moduleURI should be a string`);
+    return;
+  }
+
+  const moduleURI = moduleURIProp.value.value;
+  if (!moduleURI.match(extPattern)) {
+    warnForPath(inputFile, path, `${name} Non-js/jsm: ${moduleURI}`);
+    return;
+  }
+
+  if (!isESMifiedAndTarget(moduleURI)) {
+    return;
+  }
+
+  moduleURIProp.key.name = "esModuleURI";
+  moduleURIProp.value.value = esmify(moduleURI);
+}
+
+function doTranslate(inputFile, jscodeshift, root) {
+  root.find(jscodeshift.CallExpression).forEach(path => {
+    if (isImportCall(path.node)) {
+      replaceImportCall(inputFile, jscodeshift, path);
+    } else if (isLazyGetterCall(path.node)) {
+      replaceLazyGetterCall(inputFile, jscodeshift, path);
+    } else if (isLazyGettersCall(path.node)) {
+      replaceLazyGettersCall(inputFile, jscodeshift, path);
+    }
+  });
+
+  root.find(jscodeshift.ObjectExpression).forEach(path => {
+    tryReplaceActorDefinition(inputFile, path, "parent");
+    tryReplaceActorDefinition(inputFile, path, "child");
+  });
+}

tools/esmify/is-esmified.js

+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// A utility to check if given JSM is already ESM-ified.
+
+/* global require, __dirname */
+
+const fs = require("fs");
+const _path = require("path");
+
+/* global exports */
+
+const uri_map = JSON.parse(
+  fs.readFileSync(_path.resolve(__dirname, "./map.json"))
+);
+
+function esmify(path) {
+  return path.replace(/\.(jsm|js|jsm\.js)$/, ".sys.mjs");
+}
+
+function isESMifiedSlow(resourceURI) {
+  if (!(resourceURI in uri_map)) {
+    console.log(`WARNING: Unknown module: ${resourceURI}`);
+    return { result: false, jsms: [] };
+  }
+
+  let jsms = uri_map[resourceURI];
+  if (typeof jsms === "string") {
+    jsms = [jsms];
+  }
+
+  const prefix = "../../";
+  for (const jsm of jsms) {
+    if (fs.existsSync(prefix + jsm)) {
+      return { result: false, jsms };
+    }
+    const esm = esmify(jsm);
+    if (!fs.existsSync(prefix + esm)) {
+      return { result: false, jsms };
+    }
+  }
+
+  return { result: true, jsms };
+}
+
+const isESMified_memo = {};
+function isESMified(resourceURI, files) {
+  if (!(resourceURI in isESMified_memo)) {
+    isESMified_memo[resourceURI] = isESMifiedSlow(resourceURI);
+  }
+
+  for (const jsm of isESMified_memo[resourceURI].jsms) {
+    files.push(esmify(jsm));
+  }
+
+  return isESMified_memo[resourceURI].result;
+}
+
+exports.isESMified = isESMified;

tools/esmify/package.json

+{
+  "devDependencies": {
+    "jscodeshift": "^0.13.1"
+  },
+  "scripts": {
+    "rewrite_exports": "jscodeshift -t exported_symbols-to-declarations.js --stdin --verbose=2",
+    "rewrite_imports": "jscodeshift -t import-to-import_esmodule.js --stdin --verbose=2"
+  }
+}