Commit 80e89a6e authored by Aki Sasaki's avatar Aki Sasaki
Browse files

bug 1423081 - reverse chunk deps. r=callek 

This allows us to funnel large numbers of tasks down to avoid hitting
MAX_DEPENDENCIES. I avoided using a morph here because we might break
certain cot assumptions.

MozReview-Commit-ID: BIILM9O6CI4

--HG--
extra : rebase_source : 48bd11e8b6f25887671aafec23b2a27aad98b9d1
extra : histedit_source : 7bd193e12043272ed4ea6059260ed7abfca4d1d1
parent 5397e10e

taskcluster/ci/post-balrog-dummy/kind.yml

0 → 100644
+57 −0
Original line number Diff line number Diff line 
# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.



loader: taskgraph.loader.transform:loader



transforms:

   - taskgraph.transforms.reverse_chunk_deps:transforms

   - taskgraph.transforms.task:transforms



kind-dependencies:

   - balrog



jobs:

   firefox:

      name: post-balrog-dummy

      description: Dummy task to deal with max_dependencies

      run-on-projects: []

      shipping-phase: promote

      shipping-product: firefox

      worker-type: aws-provisioner-v1/gecko-{level}-b-linux

      worker:

         implementation: docker-worker

         os: linux

         docker-image: "ubuntu:16.10"

         max-run-time: 600

         command:

            - /bin/bash

            - -c

            - echo "Dummy task"

      treeherder:

         symbol: Rel(BD)

         platform: linux/opt

         kind: test

         tier: 1



   devedition:

      name: post-balrog-dummy

      description: Dummy task to deal with max_dependencies

      run-on-projects: []

      shipping-phase: promote

      shipping-product: devedition

      worker-type: aws-provisioner-v1/gecko-{level}-b-linux

      worker:

         implementation: docker-worker

         os: linux

         docker-image: "ubuntu:16.10"

         max-run-time: 600

         command:

            - /bin/bash

            - -c

            - echo "Dummy task"

      treeherder:

         symbol: Rel(BD)

         platform: linux-devedition/opt

         kind: test

         tier: 1

taskcluster/ci/post-beetmover-dummy/kind.yml

0 → 100644
+92 −0
Original line number Diff line number Diff line 
# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.



loader: taskgraph.loader.transform:loader



transforms:

   - taskgraph.transforms.reverse_chunk_deps:transforms

   - taskgraph.transforms.task:transforms



kind-dependencies:

   - beetmover-checksums

   - beetmover-repackage



jobs:

   firefox-ship:

      name: post-beetmover-dummy

      description: Dummy task to deal with max_dependencies

      run-on-projects: []

      shipping-phase: ship

      shipping-product: firefox

      worker-type: aws-provisioner-v1/gecko-{level}-b-linux

      worker:

         implementation: docker-worker

         os: linux

         docker-image: "ubuntu:16.10"

         max-run-time: 600

         command:

            - /bin/bash

            - -c

            - echo "Dummy task"



   firefox-promote:

      name: post-beetmover-dummy

      description: Dummy task to deal with max_dependencies

      run-on-projects: []

      shipping-phase: promote

      shipping-product: firefox

      worker-type: aws-provisioner-v1/gecko-{level}-b-linux

      worker:

         implementation: docker-worker

         os: linux

         docker-image: "ubuntu:16.10"

         max-run-time: 600

         command:

            - /bin/bash

            - -c

            - echo "Dummy task"

      treeherder:

         symbol: Rel(BMD)

         platform: linux/opt

         kind: test

         tier: 1



   devedition-ship:

      name: post-beetmover-dummy

      description: Dummy task to deal with max_dependencies

      run-on-projects: []

      shipping-phase: ship

      shipping-product: devedition

      worker-type: aws-provisioner-v1/gecko-{level}-b-linux

      worker:

         implementation: docker-worker

         os: linux

         docker-image: "ubuntu:16.10"

         max-run-time: 600

         command:

            - /bin/bash

            - -c

            - echo "Dummy task"



   devedition-promote:

      name: post-beetmover-dummy

      description: Dummy task to deal with max_dependencies

      run-on-projects: []

      shipping-phase: promote

      shipping-product: devedition

      worker-type: aws-provisioner-v1/gecko-{level}-b-linux

      worker:

         implementation: docker-worker

         os: linux

         docker-image: "ubuntu:16.10"

         max-run-time: 600

         command:

            - /bin/bash

            - -c

            - echo "Dummy task"

      treeherder:

         symbol: Rel(BMD)

         platform: linux-devedition/opt

         kind: test

         tier: 1

taskcluster/taskgraph/transforms/reverse_chunk_deps.py

0 → 100644
+49 −0
Original line number Diff line number Diff line 
# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""

Adjust dependencies to not exceed MAX_DEPS

"""



from __future__ import absolute_import, print_function, unicode_literals

from copy import deepcopy



from taskgraph.transforms.base import TransformSequence

import taskgraph.transforms.release_deps as release_deps

from taskgraph.util.treeherder import split_symbol, join_symbol



transforms = TransformSequence()



# Max dependency limit per task.

# https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask

MAX_DEPS = 100





def yield_job(orig_job, deps, count):

    job = deepcopy(orig_job)

    job['dependencies'] = deps

    job['name'] = "{}-{}".format(orig_job['name'], count)

    if 'treeherder' in job:

        groupSymbol, symbol = split_symbol(job['treeherder']['symbol'])

        symbol += '-'

        symbol += str(count)

        job['treeherder']['symbol'] = join_symbol(groupSymbol, symbol)



    return job





@transforms.add

def add_dependencies(config, jobs):

    for job in release_deps.add_dependencies(config, jobs):

        count = 1

        deps = {}



        for dep_label in job['dependencies'].keys():

            deps[dep_label] = dep_label

            if len(deps) == MAX_DEPS:

                yield yield_job(job, deps, count)

                deps = {}

                count += 1

        if deps:

            yield yield_job(job, deps, count)

            count += 1