Bug 1730712: Split site manager into command and mach managers r=ahal

The command site manager needs to be able to do ad-hoc pip
installations, while the Mach site manager needs to manage
the system `sys.path` and conditionally create an on-disk
virtualenv.

By splitting the class into two, we can now give each use case the
attention it deserves.

Differential Revision: https://phabricator.services.mozilla.com/D129529

build/mach_initialize.py

@@ -8,8 +8,6 @@ import math
 import os
 import platform
 import shutil
-import site
-import subprocess
 import sys
 
 if sys.version_info[0] < 3:
@@ -168,14 +166,9 @@ install a recent enough Python 3.
 """.strip()
 
 
-def _scrub_system_site_packages():
-    site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
-    sys.path = [path for path in sys.path if path not in site_paths]
-
-
 def _activate_python_environment(topsrcdir, state_dir):
-    # We need the "mach" module to access the logic to parse site
-    # requirements. Since that depends on "packaging" (and, transitively,
+    # We need the "mach" module to access the logic to activate the top-level
+    # Mach site. Since that depends on "packaging" (and, transitively,
     # "pyparsing"), we add those to the path too.
     sys.path[0:0] = [
         os.path.join(topsrcdir, module)
@@ -187,98 +180,24 @@ def _activate_python_environment(topsrcdir, state_dir):
     ]
 
     from mach.site import (
-        MozSiteMetadata,
+        MachSiteManager,
+        VirtualenvOutOfDateException,
         MozSiteMetadataOutOfDateError,
-        MozSiteManager,
     )
 
     try:
-        mach_site = MozSiteManager(
+        mach_environment = MachSiteManager.from_environment(
             topsrcdir,
-            os.path.join(state_dir, "_virtualenvs"),
-            "mach",
-        )
-        active_site_metadata = MozSiteMetadata.from_runtime()
-        is_mach_site = active_site_metadata and active_site_metadata.site_name == "mach"
-    except MozSiteMetadataOutOfDateError as e:
-        print(e)
-        print('This should be resolved by running "./mach create-mach-environment".')
-        sys.exit(1)
-
-    requirements = mach_site.requirements()
-    if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"):
-        env_var = (
-            "MOZ_AUTOMATION"
-            if os.environ.get("MOZ_AUTOMATION")
-            else "MACH_USE_SYSTEM_PYTHON"
+            # normpath state_dir to normalize msys-style slashes.
+            os.path.normpath(state_dir),
         )
-
-        has_pip = (
-            subprocess.run(
-                [sys.executable, "-c", "import pip"], stderr=subprocess.DEVNULL
-            ).returncode
-            == 0
+        mach_environment.activate()
+    except (VirtualenvOutOfDateException, MozSiteMetadataOutOfDateError):
+        print(
+            'The "mach" virtualenv is not up-to-date, please run '
+            '"./mach create-mach-environment"'
         )
-        # There are environments in CI that aren't prepared to provide any Mach dependency
-        # packages. Changing this is a nontrivial endeavour, so guard against having
-        # non-optional Mach requirements.
-        assert (
-            not requirements.pypi_requirements
-        ), "Mach pip package requirements must be optional."
-        if has_pip:
-            pip = [sys.executable, "-m", "pip"]
-            check_result = subprocess.run(
-                pip + ["check"],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                universal_newlines=True,
-            )
-            if check_result.returncode:
-                print(check_result.stdout, file=sys.stderr)
-                subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
-                raise Exception(
-                    'According to "pip check", the current Python '
-                    "environment has package-compatibility issues."
-                )
-
-            package_result = requirements.validate_environment_packages(pip)
-            if not package_result.has_all_packages:
-                print(
-                    "Skipping automatic management of Python dependencies since "
-                    f"the '{env_var}' environment variable is set.\n"
-                    "The following issues were found while validating your Python "
-                    "environment:"
-                )
-                print(package_result.report())
-                sys.exit(1)
-        else:
-            # Pip isn't installed to the system Python environment, so we can't use
-            # it to verify compatibility with Mach. Remove the system site-packages
-            # from the import scope so that Mach behaves as though all of its
-            # (optional) dependencies are not installed.
-            _scrub_system_site_packages()
-
-        sys.path[0:0] = requirements.pths_as_absolute(topsrcdir)
-    elif is_mach_site:
-        # We're running in the Mach virtualenv - check that it's up-to-date.
-        # Note that the "pip package check" exists to ensure that a virtualenv isn't
-        # corrupted by ad-hoc pip installs. Since the Mach virtualenv is unlikely
-        # to be affected by such installs, and since it takes ~400ms to get the list
-        # of installed pip packages (a *lot* of time to wait during Mach init), we
-        # skip verifying that our pip packages exist.
-        if not mach_site.up_to_date():
-            print(
-                'The "mach" virtualenv is not up-to-date, please run '
-                '"./mach create-mach-environment"'
-            )
-            sys.exit(1)
-    else:
-        # We're in an environment where we normally *would* use the Mach virtualenv,
-        # but we're running a "nativecmd" such as "create-mach-environment".
-        # Remove global site packages from sys.path to improve isolation accordingly.
-        _scrub_system_site_packages()
-
-        sys.path[0:0] = requirements.pths_as_absolute(topsrcdir)
+        sys.exit(1)
 
 
 def initialize(topsrcdir):

configure.py

@@ -22,7 +22,7 @@ sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
-from mach.site import MozSiteManager
+from mach.site import CommandSiteManager
 from mozbuild.configure import (
     ConfigureSandbox,
     TRACE,
@@ -232,14 +232,13 @@ def _activate_build_virtualenv():
     topobjdir = os.path.realpath(".")
     topsrcdir = os.path.realpath(os.path.dirname(__file__))
 
-    build_site = MozSiteManager(
+    build_site = CommandSiteManager(
         topsrcdir,
         os.path.join(topobjdir, "_virtualenvs"),
         "build",
     )
-    if not build_site.up_to_date():
-        print("Creating Python 3 virtualenv")
-        build_site.build()
+    if not build_site.ensure():
+        print("Created Python 3 virtualenv")
     build_site.activate()
 
 

python/mozbuild/mozbuild/base.py

@@ -289,10 +289,10 @@ class MozbuildObject(ProcessExecutionMixin):
 
     @property
     def virtualenv_manager(self):
-        from mach.site import MozSiteManager
+        from mach.site import CommandSiteManager
 
         if self._virtualenv_manager is None:
-            self._virtualenv_manager = MozSiteManager(
+            self._virtualenv_manager = CommandSiteManager(
                 self.topsrcdir,
                 os.path.join(self.topobjdir, "_virtualenvs"),
                 self._virtualenv_name,

python/mozbuild/mozbuild/mach_commands.py

@@ -2457,29 +2457,19 @@ def package_l10n(command_context, verbose=False, locales=[]):
 )
 def create_mach_environment(command_context, force=False):
     """Create the mach virtualenv."""
-    from mozboot.util import get_mach_virtualenv_root
-    from mach.site import MozSiteManager
+    from mozboot.util import get_state_dir
+    from mach.site import MachSiteManager
 
-    virtualenv_path = get_mach_virtualenv_root()
-    if sys.executable.startswith(virtualenv_path):
-        print(
-            "You can only create a mach environment with the system "
-            "Python. Re-run this `mach` command with the system Python.",
-            file=sys.stderr,
-        )
-        return 1
-
-    manager = MozSiteManager(
+    manager = MachSiteManager.from_environment(
         command_context.topsrcdir,
-        os.path.dirname(virtualenv_path),
-        "mach",
+        get_state_dir(),
+        is_mach_create_mach_env_command=True,
     )
 
-    if manager.up_to_date() and not force:
-        print("virtualenv at %s is already up to date." % virtualenv_path)
+    if manager.ensure(force=force):
+        print("Mach environment is already up to date.")
     else:
-        manager.build()
-    print("Mach environment created.")
+        print("Mach environment created.")
 
 
 def _prepend_debugger_args(args, debugger, debugger_args):

python/mozbuild/mozbuild/sphinx.py

@@ -197,7 +197,7 @@ class MozbuildSymbols(Directive):
 
 
 def setup(app):
-    from mach.site import MozSiteManager
+    from mach.site import CommandSiteManager
     from moztreedocs import manager
 
     app.add_directive("mozbuildsymbols", MozbuildSymbols)
@@ -214,7 +214,7 @@ def setup(app):
     # We need to adjust sys.path in order for Python API docs to get generated
     # properly. We leverage the in-tree virtualenv for this.
     topsrcdir = manager.topsrcdir
-    site = MozSiteManager(
+    site = CommandSiteManager(
         topsrcdir,
         os.path.join(app.outdir, "_venv"),
         "common",

python/mozbuild/mozbuild/test/test_vendor.py

@@ -39,7 +39,7 @@ def test_up_to_date_vendor():
         work_vendored = os.path.join(work_dir, "third_party", "python")
         shutil.copytree(existing_vendored, work_vendored)
 
-        # Copy "mach" module so that `MozSiteManager` can populate itself.
+        # Copy "mach" module so that `CommandSiteManager` can populate itself.
         # This is needed because "topsrcdir" is used in this test both for determining
         # import paths and for acting as a "work dir".
         existing_mach = os.path.join(topsrcdir, "python", "mach")