Bug 1723031: In CI, assert Mach pypi package deps using system Python r=ahal (99ad911a) · Commits · The Tor Project / Applications / Tor Browser

build/mach_initialize.py

+65 −6

Original line number	Diff line number	Diff line
		@@ -9,6 +9,7 @@ import os
		import platform
		import shutil
		import site
		import subprocess
		import sys

		if sys.version_info[0] < 3:
		@@ -167,6 +168,11 @@ install a recent enough Python 3.
		""".strip()


		def _scrub_system_site_packages():
		site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
		sys.path = [path for path in sys.path if path not in site_paths]


		def _activate_python_environment(topsrcdir):
		# We need the "mach" module to access the logic to parse virtualenv
		# requirements. Since that depends on "packaging" (and, transitively,
		@@ -193,6 +199,65 @@ def _activate_python_environment(topsrcdir):
		True,
		os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
		)

		if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"):
		env_var = (
		"MOZ_AUTOMATION"
		if os.environ.get("MOZ_AUTOMATION")
		else "MACH_USE_SYSTEM_PYTHON"
		)

		has_pip = (
		subprocess.run(
		[sys.executable, "-c", "import pip"], stderr=subprocess.DEVNULL
		).returncode
		== 0
		)
		# There are environments in CI that aren't prepared to provide any Mach dependency
		# packages. Changing this is a nontrivial endeavour, so guard against having
		# non-optional Mach requirements.
		assert (
		not requirements.pypi_requirements
		), "Mach pip package requirements must be optional."
		if has_pip:
		pip = [sys.executable, "-m", "pip"]
		check_result = subprocess.run(
		pip + ["check"],
		stdout=subprocess.PIPE,
		stderr=subprocess.STDOUT,
		universal_newlines=True,
		)
		if check_result.returncode:
		print(check_result.stdout, file=sys.stderr)
		subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
		raise Exception(
		'According to "pip check", the current Python '
		"environment has package-compatibility issues."
		)

		package_result = requirements.validate_environment_packages(pip)
		if not package_result.has_all_packages:
		print(
		"Skipping automatic management of Python dependencies since "
		f"the '{env_var}' environment variable is set.\n"
		"The following issues were found while validating your Python "
		"environment:"
		)
		print(package_result.report())
		sys.exit(1)
		else:
		# Pip isn't installed to the system Python environment, so we can't use
		# it to verify compatibility with Mach. Remove the system site-packages
		# from the import scope so that Mach behaves as though all of its
		# (optional) dependencies are not installed.
		_scrub_system_site_packages()

		elif sys.prefix == sys.base_prefix:
		# We're in an environment where we normally use the Mach virtualenv,
		# but we're running a "nativecmd" such as "create-mach-environment".
		# Remove global site packages from sys.path to improve isolation accordingly.
		_scrub_system_site_packages()

		sys.path[0:0] = [
		os.path.join(topsrcdir, pth.path)
		for pth in requirements.pth_requirements + requirements.vendored_requirements
		@@ -222,12 +287,6 @@ def initialize(topsrcdir):
		if os.path.exists(deleted_dir):
		shutil.rmtree(deleted_dir, ignore_errors=True)

		if sys.prefix == sys.base_prefix:
		# We are not in a virtualenv. Remove global site packages
		# from sys.path.
		site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
		sys.path = [path for path in sys.path if path not in site_paths]

		state_dir = _create_state_dir()
		_activate_python_environment(topsrcdir)

python/mach/mach/requirements.py

+51 −1

Original line number	Diff line number	Diff line
		# This Source Code Form is subject to the terms of the Mozilla Public
		# License, v. 2.0. If a copy of the MPL was not distributed with this
		# file, You can obtain one at http://mozilla.org/MPL/2.0/.

		import json
		import os
		from pathlib import Path
		import subprocess

		from packaging.requirements import Requirement

		@@ -13,6 +14,26 @@ Thunderbird requirements definitions cannot include PyPI packages.
		""".strip()


		class EnvironmentPackageValidationResult:
		def __init__(self):
		self._package_discrepancies = []
		self.has_all_packages = True

		def add_discrepancy(self, requirement, found):
		self._package_discrepancies.append((requirement, found))
		self.has_all_packages = False

		def report(self):
		lines = []
		for requirement, found in self._package_discrepancies:
		if found:
		error = f'Installed with unexpected version "{found}"'
		else:
		error = "Not installed"
		lines.append(f"{requirement}: {error}")
		return "\n".join(lines)


		class PthSpecifier:
		def __init__(self, path):
		self.path = path
		@@ -61,6 +82,35 @@ class MachEnvRequirements:
		self.pypi_optional_requirements = []
		self.vendored_requirements = []

		def validate_environment_packages(self, pip_command):
		result = EnvironmentPackageValidationResult()
		if not self.pypi_requirements and not self.pypi_optional_requirements:
		return result

		pip_json = subprocess.check_output(
		pip_command + ["list", "--format", "json"], universal_newlines=True
		)

		installed_packages = json.loads(pip_json)
		installed_packages = {
		package["name"]: package["version"] for package in installed_packages
		}
		for pkg in self.pypi_requirements:
		installed_version = installed_packages.get(pkg.requirement.name)
		if not installed_version or not pkg.requirement.specifier.contains(
		installed_version
		):
		result.add_discrepancy(pkg.requirement, installed_version)

		for pkg in self.pypi_optional_requirements:
		installed_version = installed_packages.get(pkg.requirement.name)
		if installed_version and not pkg.requirement.specifier.contains(
		installed_version
		):
		result.add_discrepancy(pkg.requirement, installed_version)

		return result

		@classmethod
		def from_requirements_definition(
		cls,

python/mozbuild/mozbuild/virtualenv.py

+4 −23

Original line number	Diff line number	Diff line
		@@ -222,28 +222,9 @@ class VirtualenvManager(VirtualenvHelper):
		if current_paths != required_paths:
		return False

		if (
		env_requirements.pypi_requirements
		or env_requirements.pypi_optional_requirements
		):
		pip_json = self._run_pip(
		["list", "--format", "json"], stdout=subprocess.PIPE
		).stdout
		installed_packages = json.loads(pip_json)
		installed_packages = {
		package["name"]: package["version"] for package in installed_packages
		}
		for pkg in env_requirements.pypi_requirements:
		if not pkg.requirement.specifier.contains(
		installed_packages.get(pkg.requirement.name, None)
		):
		return False

		for pkg in env_requirements.pypi_optional_requirements:
		installed_version = installed_packages.get(pkg.requirement.name, None)
		if installed_version and not pkg.requirement.specifier.contains(
		installed_packages.get(pkg.requirement.name, None)
		):
		pip = os.path.join(self.bin_path, "pip")
		package_result = env_requirements.validate_environment_packages([pip])
		if not package_result.has_all_packages:
		return False

		return True