Commit 36759e60 authored by Emilio Cobos Álvarez's avatar Emilio Cobos Álvarez
Browse files

Bug 1540540 - Do not unconditionally re-extract stuff from mach bootstrap. r=glandium 

Keep a checksum to avoid re-extracting artifacts over and over.

In the future we can also check the checksum to avoid downloading the artifact
altogether, maybe.

Differential Revision: https://phabricator.services.mozilla.com/D25542

--HG--
extra : moz-landing-system : lando
parent cbd5883e

python/mozboot/mozboot/base.py

+2 −6
Original line number Diff line number Diff line
@@ -289,11 +289,7 @@ class BaseBootstrapper(object): 


    def install_toolchain_static_analysis(self, state_dir, checkout_root, toolchain_job):

        clang_tools_path = os.path.join(state_dir, 'clang-tools')

        import shutil

        if os.path.exists(clang_tools_path):

            shutil.rmtree(clang_tools_path)



        # Re-create the directory for clang_tools

        if not os.path.exists(clang_tools_path):

            os.mkdir(clang_tools_path)

        self.install_toolchain_artifact(clang_tools_path, checkout_root, toolchain_job)

python/mozbuild/mozbuild/action/tooltool.py

+32 −0
Original line number Diff line number Diff line
@@ -519,14 +519,39 @@ def clean_path(dirname): 
        shutil.rmtree(dirname)





CHECKSUM_SUFFIX = ".checksum"





def _cache_checksum_matches(base_file, checksum):

    try:

        with open(base_file + CHECKSUM_SUFFIX, "rb") as f:

            prev_checksum = f.read().strip()

            if prev_checksum == checksum:

                log.info("Cache matches, avoiding extracting in '%s'" % base_file)

                return True

            return False

    except IOError as e:

        return False





def _compute_cache_checksum(filename):

    with open(filename, "rb") as f:

        return digest_file(f, "sha256")





def unpack_file(filename, setup=None):

    """Untar `filename`, assuming it is uncompressed or compressed with bzip2,

    xz, gzip, or unzip a zip file. The file is assumed to contain a single

    directory with a name matching the base of the given filename.

    Xz support is handled by shelling out to 'tar'."""



    checksum = _compute_cache_checksum(filename)



    if tarfile.is_tarfile(filename):

        tar_file, zip_ext = os.path.splitext(filename)

        base_file, tar_ext = os.path.splitext(tar_file)

        if _cache_checksum_matches(base_file, checksum):

            return True

        clean_path(base_file)

        log.info('untarring "%s"' % filename)

        tar = tarfile.open(filename)
@@ -534,12 +559,16 @@ def unpack_file(filename, setup=None): 
        tar.close()

    elif filename.endswith('.tar.xz'):

        base_file = filename.replace('.tar.xz', '')

        if _cache_checksum_matches(base_file, checksum):

            return True

        clean_path(base_file)

        log.info('untarring "%s"' % filename)

        if not execute('tar -Jxf %s 2>&1' % filename):

            return False

    elif zipfile.is_zipfile(filename):

        base_file = filename.replace('.zip', '')

        if _cache_checksum_matches(base_file, checksum):

            return True

        clean_path(base_file)

        log.info('unzipping "%s"' % filename)

        z = zipfile.ZipFile(filename)
@@ -549,6 +578,9 @@ def unpack_file(filename, setup=None): 
        log.error("Unknown archive extension for filename '%s'" % filename)

        return False



    with open(base_file + CHECKSUM_SUFFIX, "wb") as f:

        f.write(checksum)



    if setup and not execute(os.path.join(base_file, setup)):

        return False

    return True