Commit 7cd8a546 authored by brizental's avatar brizental
Browse files

fixup! BB 45308: Implement ./mach yeet command 

Address review comments
parent 014bb2ac

.gitignore

+3 −4
Changes for .gitignore: 3 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -426,11 +426,10 @@ CLAUDE.local.md 
# Other files modified at build time

mobile/android/.experimenter.json



# GitLab project and token saved by `mach yeet auth`

/.yeet.json



# Tor libraries for local builds

mobile/android/fenix/tools/nimbus-fml

mobile/android/fenix/app/tor-expert-bundle.aar

mobile/android/fenix/app/src/main/assets/extensions/{73a6fe31-595d-460b-a920-fcc0f8843232}.xpi
 


# GitLab project and token saved by `mach yeet auth`

/.yeet.json

tools/yeet/auth.py

+55 −65
Changes for tools/yeet/auth.py: 55 added lines, 65 removed lines.
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ 


import json

import os

import re

import subprocess

import urllib.error

import urllib.parse
@@ -14,38 +13,30 @@ from pathlib import Path 
from yeet.common import (

    CANONICAL_GITLAB_PROJECT,

    CANONICAL_GITLAB_REPO_NAME,

    DEFAULT_SSH_HOST,

    GITLAB_API_BASE,

    SSH_HOST,

    TERM,

    YEET_CONFIG_FILENAME,

    prompt,

)



_GITLAB_USERNAME_RE = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]*")





def _resolve_project(raw_username):

    """Build a fork project path out of a GitLab username, or return an

    error message if it's empty, malformed or the canonical repo's own

    namespace.

    """Build a fork project path out of a GitLab username. Raises ValueError

    if it's empty or the canonical repo's own namespace.

    """

    username = raw_username.strip()

    if not username:

        return None, "A GitLab username is required."



    if not _GITLAB_USERNAME_RE.fullmatch(username):

        return None, (

            f"Invalid GitLab username {username!r}. Enter just the username, "

            "not a URL or project path."

        )

        raise ValueError("A GitLab username is required.")



    if username.lower() == CANONICAL_GITLAB_PROJECT.split("/", 1)[0].lower():

        return None, (

    project = f"{username}/{CANONICAL_GITLAB_REPO_NAME}"

    if project.lower() == CANONICAL_GITLAB_PROJECT.lower():

        raise ValueError(

            "That's the canonical repo's namespace, not a fork. Use your "

            "own GitLab username instead."

        )



    return f"{username}/{CANONICAL_GITLAB_REPO_NAME}", None

    return project





def _prompt_fork():
@@ -69,11 +60,10 @@ empty.""" 
    print()



    while True:

        project, error = _resolve_project(prompt("Your GitLab username: "))

        if error:

            print(TERM.red(f"ERROR! {error}"))

            continue

        return project

        try:

            return _resolve_project(prompt("Your GitLab username: "))

        except ValueError as e:

            print(TERM.red(f"ERROR! {e}"))





def _gitlab_token_instructions(project):
@@ -96,28 +86,15 @@ and are not committed to it. 
""".strip()





_SSH_USERNAME_RE = re.compile(r"[a-z_][a-z0-9_-]*")





def _ssh_username_error(username):

    if not username:

        return "Username cannot be empty."

    if not _SSH_USERNAME_RE.fullmatch(username):

        return (

            f"Invalid username {username!r}. Use only lowercase letters, "

            "digits, '_' and '-', not starting with a digit or '-'."

        )

    return None





def _prompt_ssh_username():

def _print_ssh_instructions():

    print()

    print(

        TERM.yellow(

            f"""`mach yeet` also uploads to and runs commands on

{SSH_HOST} over SSH. Uploaded files are served from

your account's public_html there, at:

  https://{SSH_HOST}/~<username>

            f"""`mach yeet` also uploads to and runs commands on a

build server over SSH ({DEFAULT_SSH_HOST} by

default). Uploaded files are served from your

account's public_html there, at:

  https://<host>/~<username>



Anything special about the connection -- port, jump

host, a hardware key's PKCS11 provider, agent
@@ -129,12 +106,17 @@ too.""" 
    )

    print()





def _prompt_ssh_host():

    return prompt(f"SSH host [{DEFAULT_SSH_HOST}]: ").strip() or DEFAULT_SSH_HOST





def _prompt_ssh_username(ssh_host):

    while True:

        username = prompt(f"Your username on {SSH_HOST}: ").strip()

        error = _ssh_username_error(username)

        if not error:

        username = prompt(f"Your username on {ssh_host}: ").strip()

        if username:

            return username

        print(TERM.red(f"ERROR! {error}"))

        print(TERM.red("ERROR! Username cannot be empty."))





def _prompt_token():
@@ -149,13 +131,10 @@ def _verify_ssh(target): 
    """Try an actual SSH connection. stdin is /dev/null so ssh can't swallow

    input buffered for us; its touch/PIN/passphrase prompts go through

    /dev/tty and still reach the user.



    A short ConnectTimeout only bounds the initial TCP connection, so it

    won't cut off someone who's slow to respond to that prompt.

    """

    try:

        result = subprocess.run(

            ["ssh", "-o", "ConnectTimeout=15", target, "true"],

            ["ssh", target, "true"],

            stdin=subprocess.DEVNULL,

            check=False,

        )
@@ -189,21 +168,18 @@ def _verify_auth(project, token): 
        return True





def run(

    command_context,

    verify=True,

    gitlab_username=None,

    gitlab_token=None,

    ssh_username=None,

):

def run(command_context, **kwargs):

    gitlab_username = kwargs.get("gitlab_username")

    if gitlab_username is not None:

        project, error = _resolve_project(gitlab_username)

        if error:

            print(TERM.red(f"ERROR! --gitlab-username: {error}"))

        try:

            project = _resolve_project(gitlab_username)

        except ValueError as e:

            print(TERM.red(f"ERROR! --gitlab-username: {e}"))

            return 1

    else:

        project = _prompt_fork()



    gitlab_token = kwargs.get("gitlab_token")

    if gitlab_token is not None:

        token = gitlab_token.strip()

        if not token:
@@ -216,6 +192,7 @@ def run( 
        print()

        token = _prompt_token()



    verify = kwargs.get("verify", True)

    if verify:

        print()

        print("Verifying credentials with GitLab...")
@@ -237,18 +214,31 @@ def run( 
            return 1

        print(TERM.green("Credentials verified."))



    ssh_host = kwargs.get("ssh_host")

    ssh_username = kwargs.get("ssh_username")

    if ssh_host is None or ssh_username is None:

        _print_ssh_instructions()



    if ssh_host is not None:

        ssh_host = ssh_host.strip()

        if not ssh_host:

            print()

            print(TERM.red("ERROR! --ssh-host was empty."))

            return 1

    else:

        ssh_host = _prompt_ssh_host()



    if ssh_username is not None:

        ssh_username = ssh_username.strip()

        error = _ssh_username_error(ssh_username)

        if error:

        if not ssh_username:

            print()

            print(TERM.red(f"ERROR! --ssh-user: {error}"))

            print(TERM.red("ERROR! --ssh-user was empty."))

            return 1

    else:

        ssh_username = _prompt_ssh_username()

        ssh_username = _prompt_ssh_username(ssh_host)



    if verify:

        ssh_target = f"{ssh_username}@{SSH_HOST}"

        ssh_target = f"{ssh_username}@{ssh_host}"

        print()

        print(f"Verifying SSH access to {ssh_target}...")

        if not _verify_ssh(ssh_target):
@@ -271,7 +261,7 @@ def run( 
            {

                "project": project,

                "token": token,

                "ssh_host": SSH_HOST,

                "ssh_host": ssh_host,

                "ssh_user": ssh_username,

            },

            f,

tools/yeet/common.py

+6 −6
Changes for tools/yeet/common.py: 6 added lines, 6 removed lines.
Original line number Diff line number Diff line
@@ -15,16 +15,16 @@ GITLAB_API_BASE = "https://gitlab.torproject.org/api/v4" 


# The canonical repo. `mach yeet remote` must never target this directly --

# only a fork of it, so CI runs don't pile up on the shared project.

CANONICAL_GITLAB_PROJECT = "tpo/applications/tor-browser-bundle-testsuite"

CANONICAL_GITLAB_REPO_NAME = CANONICAL_GITLAB_PROJECT.rsplit("/", 1)[-1]

CANONICAL_GITLAB_NAMESPACE = "tpo/applications"

CANONICAL_GITLAB_REPO_NAME = "tor-browser-bundle-testsuite"

CANONICAL_GITLAB_PROJECT = f"{CANONICAL_GITLAB_NAMESPACE}/{CANONICAL_GITLAB_REPO_NAME}"



# Project and token saved by `mach yeet auth`.

YEET_CONFIG_FILENAME = ".yeet.json"



# The server `mach yeet` uploads to and runs commands on over SSH. Not

# user-configurable -- uploaded files are served from this host's

# public_html, so it can't be swapped out per-user.

SSH_HOST = "tb-build-03.torproject.org"

# The default server `mach yeet` uploads to and runs commands on over SSH.

# Uploaded files are served from the user's public_html on it.

DEFAULT_SSH_HOST = "tb-build-03.torproject.org"





def prompt(message, secret=False):

tools/yeet/mach_commands.py

+18 −45
Changes for tools/yeet/mach_commands.py: 18 added lines, 45 removed lines.
Original line number Diff line number Diff line
@@ -74,9 +74,9 @@ def yeet(command_context): 
    "--tag",

    dest="tags",

    action="append",

    default=["tor", "base-browser"],

    default=None,

    help="Test tag to filter tests by (can be passed multiple times, e.g. "

    "--tag tor --tag base-browser). Defaults to 'tor'.",

    "--tag tor --tag base-browser). Defaults to '--tag tor --tag base-browser'.",

)

@CommandArgument(

    "--dry-run",
@@ -92,34 +92,12 @@ def yeet(command_context): 
    default=False,

    help="Upload a local installer without asking for confirmation.",

)

def yeet_remote(

    command_context,

    platform,

    channel,

    installer_url,

    sha256sums_url,

    artifacts_url,

    mozharness_url,

    version,

    ref,

    tags,

    dry_run,

    assume_yes,

):

    return remote.run(

        command_context,

        platform,

        channel,

        installer_url,

        sha256sums_url,

        artifacts_url,

        mozharness_url,

        version,

        ref,

        tags,

        dry_run,

        assume_yes=assume_yes,

    )

def yeet_remote(command_context, **kwargs):

    # With action="append", argparse appends to a non-None default instead of

    # replacing it, so the default tags are applied here only if none were given.

    if kwargs["tags"] is None:

        kwargs["tags"] = ["tor", "base-browser"]

    return remote.run(command_context, **kwargs)





@SubCommand(
@@ -129,7 +107,8 @@ def yeet_remote( 
)

@CommandArgument(

    "--no-verify",

    action="store_true",

    dest="verify",

    action="store_false",

    help="Skip checking the GitLab and SSH credentials before saving them.",

)

@CommandArgument(
@@ -147,22 +126,16 @@ def yeet_remote( 
)

@CommandArgument(

    "--ssh-user",

    dest="ssh_username",

    default=None,

    help="Username on the build server, skipping that prompt.",

)

def yeet_auth(

    command_context,

    no_verify=False,

    gitlab_username=None,

    gitlab_token=None,

    ssh_user=None,

):

@CommandArgument(

    "--ssh-host",

    default=None,

    help="Build server to upload to and run commands on over SSH, skipping that prompt.",

)

def yeet_auth(command_context, **kwargs):

    from yeet import auth



    return auth.run(

        command_context,

        verify=not no_verify,

        gitlab_username=gitlab_username,

        gitlab_token=gitlab_token,

        ssh_username=ssh_user,

    )
 
    return auth.run(command_context, **kwargs)

tools/yeet/remote.py

+47 −42
Changes for tools/yeet/remote.py: 47 added lines, 42 removed lines.
Original line number Diff line number Diff line
@@ -14,6 +14,8 @@ from yeet.common import GITLAB_API_BASE, TERM, prompt, read_config 


PLATFORMS = ["debian_x86_64", "windows_x86_64", "macos_x86_64", "android_x86_64"]



GITLAB_REQUEST_TIMEOUT_SECONDS = 30



TOOLCHAINS_URL_BASE = "https://nightlies.tbb.torproject.org/nightly-builds/toolchains"



PLATFORM_TOOLCHAIN_DIR = {
@@ -73,21 +75,19 @@ def _curl_command(url, headers, data): 
    return shlex.join(args)





def _resolve_installer_url(command_context, installer, assume_yes, dry_run):

def _resolve_installer_url(command_context, installer, platform, assume_yes, dry_run):

    """Accept either an installer URL directly, or a local file path to

    upload first. On a dry run a local file is not uploaded, and a

    placeholder URL is returned instead. Returns (url, error).

    placeholder URL is returned instead. Raises whatever

    `upload.run` raises on failure.

    """

    if urllib.parse.urlparse(installer).scheme in ("http", "https"):

        return installer, None

        return installer

    if dry_run:

        file_path, error = upload.resolve_file(installer)

        if error:

            print(TERM.red(f"ERROR! {error}"))

            return None, error

        file_path = upload.resolve_file(installer)

        print(TERM.yellow(f"Dry run: not uploading {file_path}."))

        return f"<URL of uploaded {file_path.name}>", None

    return upload.run(command_context, installer, assume_yes=assume_yes)

        return f"<URL of uploaded {file_path.name}>"

    return upload.run(command_context, installer, platform, assume_yes=assume_yes)





def _prompt_platform():
@@ -128,49 +128,50 @@ def _prompt_channel(): 
        )





def run(

    command_context,

    platform,

    channel,

    installer_url,

    sha256sums_url,

    artifacts_url,

    mozharness_url,

    version,

    ref,

    tags,

    dry_run,

    assume_yes=False,

):

    # TODO: this runs whatever the pipeline runs, indiscriminately -- fine

    # for now since marionette is the only suite it has, but once others

    # (xpcshell, cppunittest, mochitest, ...) exist, add a --suite argument

    # here and thread it into `inputs` so a run can be scoped to just one.

def run(command_context, **kwargs):

    # TODO(tor-browser-bundle-testsuite#40120): this runs whatever the pipeline

    # runs, indiscriminately -- fine for now since marionette is the only suite

    # it has, but once others (xpcshell, cppunittest, mochitest, ...) exist, add

    # a --suite argument here and thread it into `inputs` so a run can be scoped

    # to just one.

    config = read_config(command_context)

    if config is None:

        print(TERM.red("No GitLab fork/token found, run `mach yeet auth` first."))

        return 1



    installer_url, error = _resolve_installer_url(

        command_context, installer_url, assume_yes, dry_run

    )

    if error:

    installer_url = kwargs["installer_url"].strip()

    if not installer_url:

        print(TERM.red("ERROR! The installer URL or path was empty."))

        return 1



    if not platform:

        platform = _prompt_platform()

    platform = kwargs.get("platform") or _prompt_platform()

    dry_run = kwargs.get("dry_run", False)



    try:

        installer_url = _resolve_installer_url(

            command_context,

            installer_url,

            platform,

            kwargs.get("assume_yes", False),

            dry_run,

        )

    except (FileNotFoundError, RuntimeError) as e:

        print()

        print(TERM.red(f"ERROR! {e}"))

        return 1



    package_name = None

    if platform == "android_x86_64":

        if not channel:

            channel = _prompt_channel()

        channel = kwargs.get("channel") or _prompt_channel()

        if channel not in ANDROID_CHANNEL_PACKAGE_NAMES:

            print(TERM.red(f"ERROR! Unknown channel {channel!r}."))

            return 1

        package_name = ANDROID_CHANNEL_PACKAGE_NAMES[channel]



    artifacts_url = kwargs.get("artifacts_url")

    mozharness_url = kwargs.get("mozharness_url")

    if not artifacts_url or not mozharness_url:

        version = _resolve_version(command_context, version)

        version = _resolve_version(command_context, kwargs.get("version"))

    if not artifacts_url:

        artifacts_url = _artifacts_url(version, platform)

    if not mozharness_url:
@@ -183,17 +184,19 @@ def run( 
    }

    if package_name:

        inputs[f"{platform}_package_name"] = package_name

    if sha256sums_url:

    if sha256sums_url := kwargs.get("sha256sums_url"):

        inputs[f"{platform}_sha256sums_url"] = sha256sums_url

    if tags:

    if tags := kwargs.get("tags"):

        inputs["tags"] = shlex.join(arg for tag in tags for arg in ("--tag", tag))



    project_id = urllib.parse.quote(config["project"], safe="")

    url = f"{GITLAB_API_BASE}/projects/{project_id}/trigger/pipeline"

    headers = {"Content-Type": "application/json"}

    data = json.dumps({"token": config["token"], "ref": ref, "inputs": inputs}).encode(

        "utf-8"

    )

    data = json.dumps({

        "token": config["token"],

        "ref": kwargs.get("ref", "main"),

        "inputs": inputs,

    }).encode("utf-8")



    if dry_run:

        print()
@@ -204,7 +207,9 @@ def run( 
    request = urllib.request.Request(url, data=data, headers=headers, method="POST")



    try:

        with urllib.request.urlopen(request, timeout=30) as response:

        with urllib.request.urlopen(

            request, timeout=GITLAB_REQUEST_TIMEOUT_SECONDS

        ) as response:

            result = json.load(response)

    except urllib.error.HTTPError as e:

        body = e.read().decode("utf-8", "replace")