Commit 74b09f92 authored by Mitchell Hentges's avatar Mitchell Hentges
Browse files

Bug 1712151: Add test to verify virtualenv compatibility r=ahal 

This adds two main compatibility guarantees:
1. Vendored dependencies <=> Pypi-downloaded dependencies
2. Global Mach dependencies <=> command-specific dependencies

As part of this, a new `vendored:` action was added to the virtualenv
definition format. Otherwise similar to `pth:` paths, `vendored:`
packages are assumed to be "pip install"-able.

Some validation (the `.dist-info`/`PKG-INFO` checks) was added to
`requirements.py` to verify that `pth:` and `vendored:` are correctly
used.

Differential Revision: https://phabricator.services.mozilla.com/D122900
parent b9d05720

build/build_virtualenv_packages.txt

+1 −1
Original line number Diff line number Diff line 
packages.txt:build/common_virtualenv_packages.txt

pth:third_party/python/glean_parser
 
vendored:third_party/python/glean_parser

build/common_virtualenv_packages.txt

+73 −75
Original line number Diff line number Diff line
@@ -44,82 +44,80 @@ pth:testing/mozbase/mozversion 
pth:testing/raptor

pth:testing/talos

pth:testing/web-platform

pth:testing/web-platform/tests/tools/third_party/certifi

pth:testing/web-platform/tests/tools/third_party/h2

pth:testing/web-platform/tests/tools/third_party/hpack

pth:testing/web-platform/tests/tools/third_party/html5lib

pth:testing/web-platform/tests/tools/third_party/hyperframe

pth:testing/web-platform/tests/tools/third_party/pywebsocket3

pth:testing/web-platform/tests/tools/third_party/webencodings

pth:testing/web-platform/tests/tools/wptserve

pth:testing/web-platform/tests/tools/wptrunner

vendored:testing/web-platform/tests/tools/third_party/certifi

vendored:testing/web-platform/tests/tools/third_party/h2

vendored:testing/web-platform/tests/tools/third_party/hpack

vendored:testing/web-platform/tests/tools/third_party/html5lib

vendored:testing/web-platform/tests/tools/third_party/hyperframe

vendored:testing/web-platform/tests/tools/third_party/pywebsocket3

vendored:testing/web-platform/tests/tools/third_party/webencodings

vendored:testing/web-platform/tests/tools/wptserve

vendored:testing/web-platform/tests/tools/wptrunner

pth:testing/xpcshell

pth:third_party/python/aiohttp

pth:third_party/python/appdirs

pth:third_party/python/async_timeout

pth:third_party/python/atomicwrites

pth:third_party/python/attrs

pth:third_party/python/blessings

pth:third_party/python/cbor2

pth:third_party/python/chardet

pth:third_party/python/Click

pth:third_party/python/compare_locales

pth:third_party/python/cookies

pth:third_party/python/cram

pth:third_party/python/diskcache

pth:third_party/python/distro

pth:third_party/python/dlmanager

pth:third_party/python/ecdsa

pth:third_party/python/esprima

pth:third_party/python/fluent.migrate

pth:third_party/python/fluent.syntax

pth:third_party/python/funcsigs

pth:third_party/python/gyp/pylib

pth:third_party/python/idna

pth:third_party/python/idna-ssl

pth:third_party/python/importlib_metadata

pth:third_party/python/iso8601

pth:third_party/python/Jinja2

pth:third_party/python/jsmin

pth:third_party/python/json-e

pth:third_party/python/jsonschema

pth:third_party/python/MarkupSafe/src

pth:third_party/python/mohawk

pth:third_party/python/more_itertools

pth:third_party/python/mozilla_version

pth:third_party/python/multidict

pth:third_party/python/packaging

pth:third_party/python/pathspec

pth:third_party/python/pip_tools

pth:third_party/python/pluggy

pth:third_party/python/ply

pth:third_party/python/py

pth:third_party/python/pyasn1

pth:third_party/python/pyasn1_modules

pth:third_party/python/pylru

pth:third_party/python/pyparsing

pth:third_party/python/pyrsistent

pth:third_party/python/pystache

pth:third_party/python/pytest

pth:third_party/python/python-hglib

pth:third_party/python/pytoml

pth:third_party/python/PyYAML/lib3/

pth:third_party/python/redo

pth:third_party/python/requests

pth:third_party/python/requests_unixsocket

pth:third_party/python/responses

pth:third_party/python/rsa

pth:third_party/python/sentry_sdk

pth:third_party/python/six

pth:third_party/python/slugid

pth:third_party/python/taskcluster

pth:third_party/python/taskcluster_urls

pth:third_party/python/typing_extensions

pth:third_party/python/urllib3

pth:third_party/python/voluptuous

pth:third_party/python/yamllint

pth:third_party/python/yarl

pth:third_party/python/zipp

vendored:third_party/python/aiohttp

vendored:third_party/python/appdirs

vendored:third_party/python/async_timeout

vendored:third_party/python/atomicwrites

vendored:third_party/python/attrs

vendored:third_party/python/blessings

vendored:third_party/python/cbor2

vendored:third_party/python/chardet

vendored:third_party/python/Click

vendored:third_party/python/compare_locales

vendored:third_party/python/cookies

vendored:third_party/python/cram

vendored:third_party/python/diskcache

vendored:third_party/python/distro

vendored:third_party/python/dlmanager

vendored:third_party/python/ecdsa

vendored:third_party/python/esprima

vendored:third_party/python/fluent.migrate

vendored:third_party/python/fluent.syntax

vendored:third_party/python/funcsigs

vendored:third_party/python/gyp/pylib

vendored:third_party/python/idna

vendored:third_party/python/idna-ssl

vendored:third_party/python/importlib_metadata

vendored:third_party/python/iso8601

vendored:third_party/python/Jinja2

vendored:third_party/python/jsmin

vendored:third_party/python/json-e

vendored:third_party/python/jsonschema

vendored:third_party/python/MarkupSafe/src

vendored:third_party/python/mohawk

vendored:third_party/python/more_itertools

vendored:third_party/python/mozilla_version

vendored:third_party/python/multidict

vendored:third_party/python/pathspec

vendored:third_party/python/pip_tools

vendored:third_party/python/pluggy

vendored:third_party/python/ply

vendored:third_party/python/py

vendored:third_party/python/pyasn1

vendored:third_party/python/pyasn1_modules

vendored:third_party/python/pylru

vendored:third_party/python/pyrsistent

vendored:third_party/python/pystache

vendored:third_party/python/pytest

vendored:third_party/python/python-hglib

vendored:third_party/python/pytoml

vendored:third_party/python/PyYAML/lib3/

vendored:third_party/python/redo

vendored:third_party/python/requests

vendored:third_party/python/requests_unixsocket

vendored:third_party/python/responses

vendored:third_party/python/rsa

vendored:third_party/python/sentry_sdk

vendored:third_party/python/six

vendored:third_party/python/slugid

vendored:third_party/python/taskcluster

vendored:third_party/python/taskcluster_urls

vendored:third_party/python/typing_extensions

vendored:third_party/python/urllib3

vendored:third_party/python/voluptuous

vendored:third_party/python/yamllint

vendored:third_party/python/yarl

vendored:third_party/python/zipp

pth:toolkit/components/telemetry/tests/marionette/harness

pth:tools

pth:tools/moztreedocs

build/mach_initialize.py

+2 −1
Original line number Diff line number Diff line
@@ -185,7 +185,8 @@ def _activate_python_environment(topsrcdir): 
        os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),

    )

    sys.path[0:0] = [

        os.path.join(topsrcdir, pth.path) for pth in requirements.pth_requirements

        os.path.join(topsrcdir, pth.path)

        for pth in requirements.pth_requirements + requirements.vendored_requirements

    ]

build/python-test_virtualenv_packages.txt

+2 −1
Original line number Diff line number Diff line 
packages.txt:build/common_virtualenv_packages.txt

pth:third_party/python/glean_parser
 
vendored:third_party/python/glean_parser

python/mach/mach/requirements.py

+32 −3
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
# file, You can obtain one at http://mozilla.org/MPL/2.0/.



import os

from pathlib import Path





THUNDERBIRD_PYPI_ERROR = """
@@ -60,6 +61,7 @@ class MachEnvRequirements: 
        self.pth_requirements = []

        self.pypi_requirements = []

        self.pypi_optional_requirements = []

        self.vendored_requirements = []



    @classmethod

    def from_requirements_definition(
@@ -75,14 +77,39 @@ class MachEnvRequirements: 
def _parse_mach_env_requirements(

    requirements_output, root_requirements_path, topsrcdir, is_thunderbird

):

    def _parse_requirements_line(line, is_thunderbird_packages_txt):

    topsrcdir = Path(topsrcdir)



    def _parse_requirements_line(

        current_requirements_path, line, line_number, is_thunderbird_packages_txt

    ):

        line = line.strip()

        if not line or line.startswith("#"):

            return



        action, params = line.rstrip().split(":", maxsplit=1)

        if action == "pth":

            path = topsrcdir / params

            if not path.exists():

                # In sparse checkouts, not all paths will be populated.

                return



            for child in path.iterdir():

                if child.name.endswith(".dist-info"):

                    raise Exception(

                        f'The "pth:" pointing to "{path}" has a ".dist-info" file.\n'

                        f'Perhaps "{current_requirements_path}:{line_number}" '

                        'should change to start with "vendored:" instead of "pth:".'

                    )

                if child.name == "PKG-INFO":

                    raise Exception(

                        f'The "pth:" pointing to "{path}" has a "PKG-INFO" file.\n'

                        f'Perhaps "{current_requirements_path}:{line_number}" '

                        'should change to start with "vendored:" instead of "pth:".'

                    )



            requirements_output.pth_requirements.append(PthSpecifier(params))

        elif action == "vendored":

            requirements_output.vendored_requirements.append(PthSpecifier(params))

        elif action == "packages.txt":

            _parse_requirements_definition_file(

                os.path.join(topsrcdir, params),
@@ -129,8 +156,10 @@ def _parse_mach_env_requirements( 
        with open(requirements_path, "r") as requirements_file:

            lines = [line for line in requirements_file]



        for line in lines:

            _parse_requirements_line(line, is_thunderbird_packages_txt)

        for number, line in enumerate(lines, start=1):

            _parse_requirements_line(

                requirements_path, line, number, is_thunderbird_packages_txt

            )



    _parse_requirements_definition_file(root_requirements_path, False)