Backed out changeset 9a33551de4cc (bug 1798589) for causing python failures CLOSED TREE

    scriptworkers in order to bundle things up into shippable formats, such as a

    .dmg on OSX or an installer exe on Windows.

    """

    print("Usage: ./mach repackage [dmg|pkg|installer|mar] [args...]")

    print("Usage: ./mach repackage [dmg|installer|mar] [args...]")

@SubCommand("repackage", "dmg", description="Repackage a tar file into a .dmg for OSX")

    repackage_dmg(input, output)

@SubCommand("repackage", "pkg", description="Repackage a tar file into a .pkg for OSX")

@CommandArgument("--input", "-i", type=str, required=True, help="Input filename")

@CommandArgument("--output", "-o", type=str, required=True, help="Output filename")

def repackage_pkg(command_context, input, output):

    if not os.path.exists(input):

        print("Input file does not exist: %s" % input)

        return 1

    from mozbuild.repackaging.pkg import repackage_pkg

    repackage_pkg(input, output)

@SubCommand(

    "repackage", "installer", description="Repackage into a Windows installer exe"

)

# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this file,

# You can obtain one at http://mozilla.org/MPL/2.0/.

import shutil

import tarfile

from pathlib import Path

import mozfile

from mozbuild.bootstrap import bootstrap_toolchain

from mozpack.pkg import create_pkg

def repackage_pkg(infile, output):

    if not tarfile.is_tarfile(infile):

        raise Exception("Input file %s is not a valid tarfile." % infile)

    xar_tool = bootstrap_toolchain("xar/xar")

    if not xar_tool:

        raise Exception("Could not find xar tool.")

    mkbom_tool = bootstrap_toolchain("mkbom/mkbom")

    if not mkbom_tool:

        raise Exception("Could not find mkbom tool.")

    # Note: CPIO isn't standard on all OS's

    cpio_tool = shutil.which("cpio")

    if not cpio_tool:

        raise Exception("Could not find cpio.")

    with mozfile.TemporaryDirectory() as tmpdir:

        with tarfile.open(infile) as tar:

            tar.extractall(path=tmpdir)

        app_list = list(Path(tmpdir).glob("*.app"))

        if len(app_list) != 1:

            raise Exception(

                "Input file should contain a single .app file. %s found."

                % len(app_list)

            )

        create_pkg(

            source_app=Path(app_list[0]),

            output_pkg=Path(output),

            mkbom_tool=Path(mkbom_tool),

            xar_tool=Path(xar_tool),

            cpio_tool=Path(cpio_tool),

        )

<?xml version="1.0" encoding="utf-8"?>

<installer-gui-script minSpecVersion="1">

    <pkg-ref id="${CFBundleIdentifier}">

        <bundle-version>

            <bundle CFBundleShortVersionString="${CFBundleShortVersionString}" CFBundleVersion="${CFBundleVersion}" id="${CFBundleIdentifier}" path="${app_name}.app"/>

        </bundle-version>

    </pkg-ref>

    <options customize="never" require-scripts="false" hostArchitectures="x86_64,arm64"/>

    <choices-outline>

        <line choice="default">

            <line choice="${CFBundleIdentifier}"/>

        </line>

    </choices-outline>

    <choice id="default"/>

    <choice id="${CFBundleIdentifier}" visible="false">

        <pkg-ref id="${CFBundleIdentifier}"/>

    </choice>

    <pkg-ref id="${CFBundleIdentifier}" version="${simple_version}" installKBytes="${installKBytes}">#${app_name_url_encoded}.pkg</pkg-ref>

</installer-gui-script>

 No newline at end of file

<?xml version="1.0" encoding="utf-8"?>

<pkg-info overwrite-permissions="true" relocatable="false" identifier="${CFBundleIdentifier}" postinstall-action="none" version="${simple_version}" format-version="2" generator-version="InstallCmds-681 (18F132)" install-location="/Applications" auth="root">

    <payload numberOfFiles="${numberOfFiles}" installKBytes="${installKBytes}"/>

    <bundle path="./${app_name}.app" id="${CFBundleIdentifier}" CFBundleShortVersionString="${CFBundleShortVersionString}" CFBundleVersion="${CFBundleVersion}"/>

    <bundle-version>

        <bundle id="${CFBundleIdentifier}"/>

    </bundle-version>

    <upgrade-bundle>

        <bundle id="${CFBundleIdentifier}"/>

    </upgrade-bundle>

    <update-bundle/>

    <atomic-update-bundle/>

    <strict-identifier>

        <bundle id="${CFBundleIdentifier}"/>

    </strict-identifier>

    <relocate>

        <bundle id="${CFBundleIdentifier}"/>

    </relocate>

</pkg-info>

 No newline at end of file

# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import lzma

import os

import plistlib

import struct

import subprocess

from pathlib import Path

from string import Template

from typing import List

from urllib.parse import quote

import mozfile

TEMPLATE_DIRECTORY = Path(__file__).parent / "apple_pkg"

RW_CHUNK_SIZE = 10240 * 1024  # 10MB read/write chunk sizes

def check_tools(*tools: List[Path]):

    """

    Check that each tool named in tools exists and is executable.

    Args:

        tools: List<str>, list of tools to check

    Raises:

        Exception: if tool not found from env, not a file, or not executable.

    """

    for tool in tools:

        if not tool.is_file():

            raise Exception('Required tool "%s" not found.' % (tool))

        if not os.access(tool, os.X_OK):

            raise Exception('Required tool "%s" is not executable' % (tool))

def get_apple_template(name: str) -> Template:

    """

    Given <name>, open file at <TEMPLATE_DIRECTORY>/<name>, read contents and

        return as a Template

    Args:

        name: str, Filename for the template

    Returns:

        Template, loaded from file

    """

    tmpl_path = TEMPLATE_DIRECTORY / name

    if not tmpl_path.is_file():

        raise Exception(f"Could not find template: {tmpl_path}")

    with tmpl_path.open("r") as tmpl:

        contents = tmpl.read()

    return Template(contents)

def save_text_file(content: str, destination: Path):

    """

    Saves a text file to <destination> with provided <content>

    Note: Overwrites contents

    Args:

        content: str, The desired contents of the file

        destination: Path, The file path

    """

    with destination.open("w") as out_fd:

        out_fd.write(content)

    print(f"Created text file at {destination}")

    print(f"Created text file size: {destination.stat().st_size} bytes")

def get_app_info_plist(app_path: Path) -> dict:

    """

    Retrieve most information from Info.plist file of an app.

    The Info.plist file should be located in ?.app/Contents/Info.plist

    Note: Ignores properties that are not <string> type

    Args:

        app_path: Path, the .app file/directory path

    Returns:

        dict, the dictionary of properties found in Info.plist

    """

    info_plist = app_path / "Contents/Info.plist"

    if not info_plist.is_file():

        raise Exception(f"Could not find Info.plist in {info_plist}")

    print(f"Reading app Info.plist from: {info_plist}")

    with info_plist.open("rb") as plist_fd:

        data = plistlib.load(plist_fd)

    return data

def create_payload(destination: Path, root_path: Path):

    """

    Creates a payload at <destination> based on <root_path>

    Args:

        destination: Path, the destination Path

        root_path: Path, the root directory Path

    """

    # Files to be cpio'd are root folder + contents

    file_list = ["./"] + get_relative_glob_list(root_path, "**/*")

    with mozfile.TemporaryDirectory() as tmp_dir:

        tmp_payload_path = Path(tmp_dir) / "Payload"

        print(f"Creating Payload with cpio from {root_path} to {tmp_payload_path}")

        print(f"Found {len(file_list)} files")

        with tmp_payload_path.open("wb") as tmp_payload:

            process = subprocess.run(

                [

                    "cpio",

                    "-o",  # copy-out mode

                    "--format",

                    "odc",  # old POSIX .1 portable format

                    "--owner",

                    "0:80",  # clean ownership

                ],

                stdout=tmp_payload,

                stderr=subprocess.PIPE,

                input="\n".join(file_list) + "\n",

                encoding="ascii",

                cwd=root_path,

            )

        # cpio outputs number of blocks to stderr

        print(f"[CPIO]: {process.stderr}")

        if process.returncode:

            raise Exception(f"CPIO error {process.returncode}")

        tmp_payload_size = tmp_payload_path.stat().st_size

        print(f"Uncompressed Payload size: {tmp_payload_size // 1024}kb")

        # Compress to xz

        lzcomp = lzma.LZMACompressor()

        xz_payload = Path(tmp_dir) / "Payload.xz"

        with tmp_payload_path.open("rb") as f_in, xz_payload.open("wb") as xz_f:

            while True:

                chunk = f_in.read(RW_CHUNK_SIZE)

                if not chunk:

                    break

                xz_f.write(lzcomp.compress(chunk))

            xz_f.write(lzcomp.flush())

        # Convert XZ file to pbzx Payload

        xz_payload_size = xz_payload.stat().st_size

        print(f"Compressed Payload size: {xz_payload_size // 1024}kb")

        with destination.open("wb") as f_out, xz_payload.open("rb") as xz_in:

            f_out.write(b"pbzx")

            f_out.write(struct.pack(">Q", tmp_payload_size))

            f_out.write(struct.pack(">Q", tmp_payload_size))  # yes, twice

            f_out.write(struct.pack(">Q", xz_payload_size))

            while True:

                chunk = xz_in.read(RW_CHUNK_SIZE)

                if not chunk:

                    break

                f_out.write(chunk)

        print(f"Compressed Payload file to {destination}")

        print(f"Compressed Payload size: {destination.stat().st_size // 1024}kb")

def create_bom(bom_path: Path, root_path: Path, mkbom_tool: Path):

    """

    Creates a Bill Of Materials file at <bom_path> based on <root_path>

    Args:

        bom_path: Path, destination Path for the BOM file

        root_path: Path, root directory Path

        mkbom_tool: Path, mkbom tool Path

    """

    print(f"Creating BOM file from {root_path} to {bom_path}")

    subprocess.check_call(

        [

            mkbom_tool,

            "-u",

            "0",

            "-g",

            "80",

            str(root_path),

            str(bom_path),

        ]

    )

    print(f"Created BOM File size: {bom_path.stat().st_size // 1024}kb")

def get_relative_glob_list(source: Path, glob: str) -> list[str]:

    """

    Given a source path, return a list of relative path based on glob

    Args:

        source: Path, source directory Path

        glob: str, unix style glob

    Returns:

        list[str], paths found in source directory

    """

    return [f"./{c.relative_to(source)}" for c in source.glob(glob)]

def xar_package_folder(source_path: Path, destination: Path, xar_tool: Path):

    """

    Create a pkg from <source_path> to <destination>

    The command is issued with <source_path> as cwd

    Args:

        source_path: Path, source absolute Path

        destination: Path, destination absolute Path

        xar_tool: Path, xar tool Path

    """

    if not source_path.is_absolute() or not destination.is_absolute():

        raise Exception("Source and destination should be absolute.")

    print(f"Creating pkg from {source_path} to {destination}")

    # Create a list of ./<file> - noting xar takes care of <file>/**

    file_list = get_relative_glob_list(source_path, "*")

    subprocess.check_call(

        [

            xar_tool,

            "--compression",

            "none",

            "-vcf",

            destination,

            *file_list,

        ],

        cwd=source_path,

    )

    print(f"Created PKG file to {destination}")

    print(f"Created PKG size: {destination.stat().st_size // 1024}kb")

def create_pkg(

    source_app: Path,

    output_pkg: Path,

    mkbom_tool: Path,

    xar_tool: Path,

    cpio_tool: Path,

):

    """

    Create a mac PKG installer from <source_app> to <output_pkg>

    Args:

        source_app: Path, source .app file/directory Path

        output_pkg: Path, destination .pkg file

        mkbom_tool: Path, mkbom tool Path

        xar_tool: Path, xar tool Path

        cpio: Path, cpio tool Path

    """

    check_tools(mkbom_tool, xar_tool, cpio_tool)

    app_name = source_app.name.rsplit(".", maxsplit=1)[0]

    with mozfile.TemporaryDirectory() as tmpdir:

        root_path = Path(tmpdir) / "darwin/root"

        flat_path = Path(tmpdir) / "darwin/flat"

        # Create required directories

        # TODO: Investigate Resources folder contents for other lproj?

        (flat_path / "Resources/en.lproj").mkdir(parents=True, exist_ok=True)

        (flat_path / f"{app_name}.pkg").mkdir(parents=True, exist_ok=True)

        root_path.mkdir(parents=True, exist_ok=True)

        # Copy files over

        subprocess.check_call(

            [

                "cp",

                "-R",

                str(source_app),

                str(root_path),

            ]

        )

        # Count all files (innards + itself)

        file_count = len(list(source_app.glob("**/*"))) + 1

        print(f"Calculated source files count: {file_count}")

        # Get package contents size

        package_size = sum(f.stat().st_size for f in source_app.glob("**/*")) // 1024

        print(f"Calculated source package size: {package_size}kb")

        app_info = get_app_info_plist(source_app)

        app_info["numberOfFiles"] = file_count

        app_info["installKBytes"] = package_size

        app_info["app_name"] = app_name

        app_info["app_name_url_encoded"] = quote(app_name)

        # This seems arbitrary, there might be another way of doing it,

        #   but Info.plist doesn't provide the simple version we need

        major_version = app_info["CFBundleShortVersionString"].split(".")[0]

        app_info["simple_version"] = f"{major_version}.0.0"

        pkg_info_tmpl = get_apple_template("PackageInfo.template")

        pkg_info = pkg_info_tmpl.substitute(app_info)

        save_text_file(pkg_info, flat_path / f"{app_name}.pkg/PackageInfo")

        distribution_tmp = get_apple_template("Distribution.template")

        distribution = distribution_tmp.substitute(app_info)

        save_text_file(distribution, flat_path / "Distribution")

        payload_path = flat_path / f"{app_name}.pkg/Payload"

        create_payload(payload_path, root_path)

        bom_path = flat_path / f"{app_name}.pkg/Bom"

        create_bom(bom_path, root_path, mkbom_tool)

        xar_package_folder(flat_path, output_pkg, xar_tool)