Commit 903b044a authored by Johan Lorenzo's avatar Johan Lorenzo
Browse files

Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions... 

Bug 1337825 - Schedule l10n repacks for fennec with specified locale revisions using an intree changesets file r=aki

MozReview-Commit-ID: 1DEIjFYAx74
parent 950258a3

taskcluster/docs/attributes.rst

+5 −0
Original line number Diff line number Diff line
@@ -129,6 +129,11 @@ all_locales 
For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the list

of relevant locales for the platform.



all_locales_with_changesets

===========================



Contains a dict of l10n changesets, mapped by locales (same as in ``all_locales``).



l10n_chunk

==========

For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk

taskcluster/taskgraph/transforms/l10n.py

+33 −22
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ Do transforms specific to l10n kind 
from __future__ import absolute_import, print_function, unicode_literals



import copy

import json



from mozbuild.chunkify import chunkify

from taskgraph.transforms.base import (
@@ -136,14 +137,24 @@ def _parse_locales_file(locales_file, platform=None): 
        If platform is unset matches all platforms.

    """

    locales = []



    with open(locales_file, mode='r') as f:

        if locales_file.endswith('json'):

        # Release process uses .json for locale files sometimes.

        raise NotImplementedError("Don't know how to parse a .json locales file")

            all_locales = json.load(f)

            # XXX Only single locales are fetched

            locales = {locale: data['revision'] for locale, data in all_locales.items() if 'android' in data['platforms']}

        else:

        with open(locales_file, mode='r') as lf:

            locales = lf.read().split()

            all_locales = f.read().split()

            # 'default' is the hg revision at the top of hg repo, in this context

            locales = {locale: 'default' for locale in all_locales}

    return locales



def _remove_ja_jp_mac_locale(locales):

    # ja-JP-mac is a mac-only locale, but there are no mac builds being repacked, so just omit it unconditionally

    return {

        locale: revision for locale, revision in locales.items() if locale != 'ja-JP-mac'

    }





@transforms.add

def setup_name(config, jobs):
@@ -232,14 +243,13 @@ def handle_keyed_by(config, jobs): 
@transforms.add

def all_locales_attribute(config, jobs):

    for job in jobs:

        locales = set(_parse_locales_file(job["locales-file"]))

        # ja-JP-mac is a mac-only locale, but there are no

        # mac builds being repacked, so just omit it unconditionally

        locales = locales - set(("ja-JP-mac", ))

        # Convert to mutable list.

        locales = list(sorted(locales))

        locales_with_changesets = _parse_locales_file(job["locales-file"])

        locales_with_changesets = _remove_ja_jp_mac_locale(locales_with_changesets)



        locales = sorted(locales_with_changesets.keys())

        attributes = job.setdefault('attributes', {})

        attributes["all_locales"] = locales

        attributes["all_locales_with_changesets"] = locales_with_changesets

        yield job
@@ -248,24 +258,25 @@ def chunk_locales(config, jobs): 
    """ Utilizes chunking for l10n stuff """

    for job in jobs:

        chunks = job.get('chunks')

        all_locales = job['attributes']['all_locales']

        locales_with_changesets = job['attributes']['all_locales_with_changesets']

        if chunks:

            if chunks > len(all_locales):

            if chunks > len(locales_with_changesets):

                # Reduce chunks down to the number of locales

                chunks = len(all_locales)

                chunks = len(locales_with_changesets)

            for this_chunk in range(1, chunks + 1):

                chunked = copy.deepcopy(job)

                chunked['name'] = chunked['name'].replace(

                    '/', '-{}/'.format(this_chunk), 1

                )

                chunked['mozharness']['options'] = chunked['mozharness'].get('options', [])

                my_locales = []

                my_locales = chunkify(all_locales, this_chunk, chunks)

                # chunkify doesn't work with dicts

                locales_with_changesets_as_list = locales_with_changesets.items()

                chunked_locales = chunkify(locales_with_changesets_as_list, this_chunk, chunks)

                chunked['mozharness']['options'].extend([

                    "locale={}".format(locale) for locale in my_locales

                    'locale={}:{}'.format(locale, changeset) for locale, changeset in chunked_locales

                ])

                chunked['attributes']['l10n_chunk'] = str(this_chunk)

                chunked['attributes']['chunk_locales'] = my_locales

                chunked['attributes']['chunk_locales'] = [locale for locale, _ in chunked_locales]  # strip revision



                # add the chunk number to the TH symbol

                group, symbol = split_symbol(
@@ -276,7 +287,7 @@ def chunk_locales(config, jobs): 
        else:

            job['mozharness']['options'] = job['mozharness'].get('options', [])

            job['mozharness']['options'].extend([

                "locale={}".format(locale) for locale in all_locales

                'locale={}:{}'.format(locale, changeset) for locale, changeset in locales_with_changesets.items()

            ])

            yield job