Commit acd1f7b8 authored by Andrew Halberstadt's avatar Andrew Halberstadt
Browse files

Bug 1568277 - [taskgraph] Create optimize strategy aliases for the 'test' kind r=tomprince 

This allows test tasks to declare a static optimization name, which can then be
swapped in and out from the optimize code without needing to update the
transforms. This will make it easier to change optimization strategies or run
experimental ones.

Differential Revision: https://phabricator.services.mozilla.com/D40204

--HG--
extra : moz-landing-system : lando
parent c8af64de

taskcluster/taskgraph/optimize/__init__.py

+18 −2
Original line number Diff line number Diff line
@@ -75,7 +75,10 @@ def _get_optimizations(target_task_graph, strategies): 
        task = target_task_graph.tasks[label]

        if task.optimization:

            opt_by, arg = task.optimization.items()[0]

            return (opt_by, strategies[opt_by], arg)

            strategy = strategies[opt_by]

            if hasattr(strategy, 'description'):

                opt_by += " ({})".format(strategy.description)

            return (opt_by, strategy, arg)

        else:

            return ('never', strategies['never'], None)

    return optimizations
@@ -261,6 +264,7 @@ class Either(OptimizationStrategy): 
            raise TypeError("substrategies aren't registered: {}".format(

                ",  ".join(sorted(missing))))



        self.description = "-or-".join(substrategies)

        self.substrategies = [registry[sub] for sub in substrategies]

        self.split_args = kwargs.pop('split_args', None)

        if not self.split_args:
@@ -286,9 +290,21 @@ class Either(OptimizationStrategy): 
            lambda sub, arg: sub.should_replace_task(task, params, arg))





class Alias(Either):

    """Provides an alias to an existing strategy.



    This can be useful to swap strategies in and out without needing to modify

    the task transforms.

    """

    def __init__(self, strategy):

        super(Alias, self).__init__(strategy)





# Trigger registration in sibling modules.

import_sibling_modules()





# Register composite strategies.

register_strategy('skip-unless-schedules-or-seta', args=('skip-unless-schedules', 'seta'))(Either)
 
register_strategy('test', args=('skip-unless-schedules', 'seta'))(Either)

register_strategy('test-inclusive', args=('skip-unless-schedules',))(Alias)

register_strategy('test-try', args=('skip-unless-schedules',))(Alias)

taskcluster/taskgraph/test/test_optimize.py

+1 −0
Original line number Diff line number Diff line
@@ -70,6 +70,7 @@ class TestOptimize(unittest.TestCase): 
            ('t2', 't1', 'dep'))



    def assert_remove_tasks(self, graph, exp_removed, do_not_optimize=set()):

        optimize.registry = self.strategies

        got_removed = optimize.remove_tasks(

            target_task_graph=graph,

            optimizations=optimize._get_optimizations(graph, self.strategies),

taskcluster/taskgraph/transforms/tests.py

+6 −5
Original line number Diff line number Diff line
@@ -1416,15 +1416,16 @@ def make_job_description(config, tests): 
            schedules = [category, platform_family(test['build-platform'])]



        if test.get('when'):

            # This may still be used by comm-central.

            jobdesc['when'] = test['when']

        elif 'optimization' in test:

            jobdesc['optimization'] = test['optimization']

        elif not config.params.is_try() and category not in INCLUSIVE_COMPONENTS:

            # for non-try branches and non-inclusive suites, include SETA

            jobdesc['optimization'] = {'skip-unless-schedules-or-seta': schedules}

        elif config.params.is_try():

            jobdesc['optimization'] = {'test-try': schedules}

        elif category in INCLUSIVE_COMPONENTS:

            jobdesc['optimization'] = {'test-inclusive': schedules}

        else:

            # otherwise just use skip-unless-schedules

            jobdesc['optimization'] = {'skip-unless-schedules': schedules}

            jobdesc['optimization'] = {'test': schedules}



        run = jobdesc['run'] = {}

        run['using'] = 'mozharness-test'

taskcluster/taskgraph/util/schema.py

+4 −2
Original line number Diff line number Diff line
@@ -203,8 +203,10 @@ OptimizationSchema = voluptuous.Any( 
    {'skip-unless-changed': [basestring]},

    # skip this task if unless the change files' SCHEDULES contains any of these components

    {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)},

    # skip if SETA or skip-unless-schedules says to

    {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)},

    # optimize strategy aliases for the test kind

    {'test': list(schedules.ALL_COMPONENTS)},

    {'test-inclusive': list(schedules.ALL_COMPONENTS)},

    {'test-try': list(schedules.ALL_COMPONENTS)},

)



# shortcut for a string where task references are allowed