Commit 418e9cc7 authored by Julien Cristau's avatar Julien Cristau
Browse files

Bug 1821513 - update robustcheckout hg extension. r=releng-reviewers,bhearsum a=release 

Imported from version-control-tools revision 46c99d31ab14.

Differential Revision: https://phabricator.services.mozilla.com/D173040
parent c8ce2415

testing/mozharness/external_tools/robustcheckout.py

+41 −26
Original line number Diff line number Diff line
@@ -9,6 +9,8 @@ from a source repo using best practices to ensure optimal clone 
times and storage efficiency.

"""



from __future__ import absolute_import



import contextlib

import json

import os
@@ -39,7 +41,7 @@ from mercurial import ( 
# Causes worker to purge caches on process exit and for task to retry.

EXIT_PURGE_CACHE = 72



testedwith = b"4.5 4.6 4.7 4.8 4.9 5.0 5.1 5.2 5.3 5.4 5.5 5.6 5.7 5.8"

testedwith = b"4.5 4.6 4.7 4.8 4.9 5.0 5.1 5.2 5.3 5.4 5.5 5.6 5.7 5.8 5.9"

minimumhgversion = b"4.5"



cmdtable = {}
@@ -59,12 +61,8 @@ def getsparse(): 




def peerlookup(remote, v):

    # TRACKING hg46 4.6 added commandexecutor API.

    if util.safehasattr(remote, "commandexecutor"):

    with remote.commandexecutor() as e:

        return e.callcommand(b"lookup", {b"key": v}).result()

    else:

        return remote.lookup(v)





@command(
@@ -179,6 +177,8 @@ def robustcheckout( 
    # worker.backgroundclose only makes things faster if running anti-virus,

    # which our automation doesn't. Disable it.

    ui.setconfig(b"worker", b"backgroundclose", False)

    # Don't wait forever if the connection hangs

    ui.setconfig(b"http", b"timeout", 600)



    # By default the progress bar starts after 3s and updates every 0.1s. We

    # change this so it shows and updates every 1.0s.
@@ -509,6 +509,10 @@ def _docheckout( 
                        pycompat.bytestr(str(e.reason)),

                    )

                )

        elif isinstance(e, socket.timeout):

            ui.warn(b"socket timeout\n")

            handlenetworkfailure()

            return True

        else:

            ui.warn(

                b"unhandled exception during network operation; type: %s; "
@@ -529,7 +533,12 @@ def _docheckout( 
        rootnode = peerlookup(clonepeer, b"0")

    except error.RepoLookupError:

        raise error.Abort(b"unable to resolve root revision from clone " b"source")

    except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:

    except (

        error.Abort,

        ssl.SSLError,

        urllibcompat.urlerr.urlerror,

        socket.timeout,

    ) as e:

        if handlepullerror(e):

            return callself()

        raise
@@ -568,6 +577,11 @@ def _docheckout( 


    if storevfs.exists(b".hg/requires"):

        requires = set(storevfs.read(b".hg/requires").splitlines())

        # "share-safe" (enabled by default as of hg 6.1) moved most

        # requirements to a new file, so we need to look there as well to avoid

        # deleting and re-cloning each time

        if b"share-safe" in requires:

            requires |= set(storevfs.read(b".hg/store/requires").splitlines())

        # FUTURE when we require generaldelta, this is where we can check

        # for that.

        required = {b"dotencode", b"fncache"}
@@ -617,7 +631,12 @@ def _docheckout( 
                    shareopts=shareopts,

                    stream=True,

                )

        except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:

        except (

            error.Abort,

            ssl.SSLError,

            urllibcompat.urlerr.urlerror,

            socket.timeout,

        ) as e:

            if handlepullerror(e):

                return callself()

            raise
@@ -685,7 +704,12 @@ def _docheckout( 
                    pullop = exchange.pull(repo, remote, heads=pullrevs)

                    if not pullop.rheads:

                        raise error.Abort(b"unable to pull requested revision")

        except (error.Abort, ssl.SSLError, urllibcompat.urlerr.urlerror) as e:

        except (

            error.Abort,

            ssl.SSLError,

            urllibcompat.urlerr.urlerror,

            socket.timeout,

        ) as e:

            if handlepullerror(e):

                return callself()

            raise
@@ -721,14 +745,7 @@ def _docheckout( 
        try:

            old_sparse_fn = getattr(repo.dirstate, "_sparsematchfn", None)

            if old_sparse_fn is not None:

                # TRACKING hg50

                # Arguments passed to `matchmod.always` were unused and have been removed

                if util.versiontuple(n=2) >= (5, 0):

                repo.dirstate._sparsematchfn = lambda: matchmod.always()

                else:

                    repo.dirstate._sparsematchfn = lambda: matchmod.always(

                        repo.root, ""

                    )



            with timeit("purge", "purge"):

                if purge(
@@ -765,13 +782,9 @@ def _docheckout( 
                b"%s" % (sparse_profile, checkoutrevision)

            )



        # TRACKING hg48 - parseconfig takes `action` param

        if util.versiontuple(n=2) >= (4, 8):

        old_config = sparsemod.parseconfig(

            repo.ui, repo.vfs.tryread(b"sparse"), b"sparse"

        )

        else:

            old_config = sparsemod.parseconfig(repo.ui, repo.vfs.tryread(b"sparse"))



        old_includes, old_excludes, old_profiles = old_config
@@ -793,7 +806,9 @@ def _docheckout( 
            # one to change the sparse profile and another to update to the new

            # revision. This is not desired. But there's not a good API in

            # Mercurial to do this as one operation.

            with repo.wlock(), timeit("sparse_update_config", "sparse-update-config"):

            with repo.wlock(), repo.dirstate.parentchange(), timeit(

                "sparse_update_config", "sparse-update-config"

            ):

                # pylint --py3k: W1636

                fcounts = list(

                    map(