Commit a2365152 authored by Sebastian Hengst's avatar Sebastian Hengst
Browse files

Bug 1620610 - Remove coalescing for tasks. r=catlee 

Bug 1602446 disabled coalescing for tasks which was designed to reduce load on
the pools which ran those tasks by skipping some of them. After coalescing was
affecting almost no tasks, it started to coalesce more tasks in November 2019
(bug 1602446) up to a level for which too few tasks ran and test coverage was
impacted. Coalescing got disabled and the conclusion is that coalescing is
currently not needed.

Differential Revision: https://phabricator.services.mozilla.com/D65733

--HG--
extra : moz-landing-system : lando
parent 0af78314

taskcluster/ci/test/kind.yml

+0 −1
Original line number Diff line number Diff line
@@ -13,7 +13,6 @@ kind-dependencies: 
transforms:

    - taskgraph.transforms.tests:transforms

    - taskgraph.transforms.job:transforms

    - taskgraph.transforms.coalesce:transforms

    - taskgraph.transforms.task:transforms



# Each stanza in a file pointed to by 'jobs-from' describes a particular test

taskcluster/taskgraph/transforms/coalesce.py

deleted100644 → 0
+0 −40
Original line number Diff line number Diff line 
# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""

This transform enables coalescing for tasks with scm level > 1, by defining

values for the coalesce settings of the job.

"""



from __future__ import absolute_import



from taskgraph.transforms.base import TransformSequence

# Caller disabled - see below.

# from hashlib import sha256



transforms = TransformSequence()





@transforms.add

def enable_coalescing(config, jobs):

    """

    This transform enables coalescing for tasks with scm level > 1, setting the

    name to be equal to the the first 20 chars of the sha256 of the task name

    (label). The hashing is simply used to keep the name short, unique, and

    alphanumeric, since it is used in AMQP routing keys.



    Note, the coalescing keys themselves are not intended to be readable

    strings or embed information. The tasks they represent contain all relevant

    metadata.

    """

    for job in jobs:

        # TODO: Identify why coalescing became much more frequent (bug 1602446) or

        #       disable it for backfills, retriggers and manually requested tasks

        #       (bug 1397389).

        # if int(config.params['level']) > 1:

        #     job['coalesce'] = {

        #         'job-identifier': sha256(job["label"]).hexdigest()[:20],

        #         'age': 3600,

        #         'size': 5,

        #     }

        yield job

taskcluster/taskgraph/transforms/job/__init__.py

+0 −1
Original line number Diff line number Diff line
@@ -65,7 +65,6 @@ job_description_schema = Schema({ 
    Optional('run-on-projects'): task_description_schema['run-on-projects'],

    Optional('shipping-phase'): task_description_schema['shipping-phase'],

    Optional('shipping-product'): task_description_schema['shipping-product'],

    Optional('coalesce'): task_description_schema['coalesce'],

    Optional('always-target'): task_description_schema['always-target'],

    Exclusive('optimization', 'optimization'): task_description_schema['optimization'],

    Optional('needs-sccache'): task_description_schema['needs-sccache'],

taskcluster/taskgraph/transforms/openh264.py

+1 −8
Original line number Diff line number Diff line
@@ -15,14 +15,7 @@ transforms = TransformSequence() 
@transforms.add

def set_mh_options(config, jobs):

    """

    This transform enables coalescing for tasks with scm level > 1, setting the

    name to be equal to the the first 20 chars of the sha256 of the task name

    (label). The hashing is simply used to keep the name short, unique, and

    alphanumeric, since it is used in AMQP routing keys.



    Note, the coalescing keys themselves are not intended to be readable

    strings or embed information. The tasks they represent contain all relevant

    metadata.

    This transform sets the 'openh264_rev' attribute.

    """

    for job in jobs:

        repo = job.pop('repo')

taskcluster/taskgraph/transforms/task.py

+0 −55
Original line number Diff line number Diff line
@@ -192,27 +192,6 @@ task_description_schema = Schema({ 
        text_type

    ),



    # Coalescing provides the facility for tasks to be superseded by the same

    # task in a subsequent commit, if the current task backlog reaches an

    # explicit threshold. Both age and size thresholds need to be met in order

    # for coalescing to be triggered.

    Optional('coalesce'): {

        # A unique identifier per job (typically a hash of the job label) in

        # order to partition tasks into appropriate sets for coalescing. This

        # is combined with the project in order to generate a unique coalescing

        # key for the coalescing service.

        'job-identifier': text_type,



        # The minimum amount of time in seconds between two pending tasks with

        # the same coalescing key, before the coalescing service will return

        # tasks.

        'age': int,



        # The minimum number of backlogged tasks with the same coalescing key,

        # before the coalescing service will return tasks.

        'size': int,

    },



    # The `always-target` attribute will cause the task to be included in the

    # target_task_graph regardless of filtering. Tasks included in this manner

    # will be candidates for optimization even when `optimize_target_tasks` is
@@ -321,10 +300,6 @@ def get_branch_repo(config): 
    )]





COALESCE_KEY = '{project}.{job-identifier}'

SUPERSEDER_URL = 'https://coalesce.mozilla-releng.net/v1/list/{age}/{size}/{key}'





@memoize

def get_default_priority(graph_config, project):

    return evaluate_keyed_by(
@@ -364,24 +339,6 @@ def index_builder(name): 
    return wrap





def coalesce_key(config, task):

    return COALESCE_KEY.format(**{

               'project': config.params['project'],

               'job-identifier': task['coalesce']['job-identifier'],

           })





def superseder_url(config, task):

    key = coalesce_key(config, task)

    age = task['coalesce']['age']

    size = task['coalesce']['size']

    return SUPERSEDER_URL.format(

        age=age,

        size=size,

        key=key

    )





UNSUPPORTED_INDEX_PRODUCT_ERROR = """\

The gecko-v2 product {product} is not in the list of configured products in

`taskcluster/ci/config.yml'.
@@ -693,10 +650,6 @@ def build_docker_worker_payload(config, task, task_def): 
    if capabilities:

        payload['capabilities'] = capabilities



    # coalesce / superseding

    if 'coalesce' in task:

        payload['supersederUrl'] = superseder_url(config, task)



    check_caches_are_volumes(task)

    check_required_volumes(task)
@@ -895,10 +848,6 @@ def build_generic_worker_payload(config, task, task_def): 
    if features:

        task_def['payload']['features'] = features



    # coalesce / superseding

    if 'coalesce' in task:

        task_def['payload']['supersederUrl'] = superseder_url(config, task)





@payload_builder('scriptworker-signing', schema={

    # the maximum time to run, in seconds
@@ -1932,10 +1881,6 @@ def build_task(config, tasks): 
        if 'deadline-after' not in task:

            task['deadline-after'] = '1 day'



        if 'coalesce' in task:

            key = coalesce_key(config, task)

            routes.append('coalesce.v1.' + key)



        if 'priority' not in task:

            task['priority'] = get_default_priority(config.graph_config, config.params['project'])