Commit 30fa8f8a authored by Andrew Halberstadt's avatar Andrew Halberstadt
Browse files

Bug 1513951 - [tryselect] Implement in-tree try_presets.yml file r=gbrown 

This creates a global preset file at:
tools/tryselect/try_presets.yml

Any presets defined here will be available for everyone to use.

Differential Revision: https://phabricator.services.mozilla.com/D21435

--HG--
extra : moz-landing-system : lando
parent b1f7c90d

tools/tryselect/cli.py

+8 −17
Original line number Diff line number Diff line
@@ -6,11 +6,9 @@ from __future__ import absolute_import, print_function, unicode_literals 


import os

import subprocess

import sys

import tempfile

from argparse import ArgumentParser



from .preset import presets

from .templates import all_templates
@@ -46,15 +44,17 @@ COMMON_ARGUMENT_GROUPS = { 
          'help': 'Load a saved selection.',

          }],

        [['--list-presets'],

         {'action': 'store_true',

          'dest': 'list_presets',

          'default': False,

         {'action': 'store_const',

          'dest': 'preset_action',

          'const': 'list',

          'default': None,

          'help': 'List available preset selections.',

          }],

        [['--edit-presets'],

         {'action': 'store_true',

          'dest': 'edit_presets',

          'default': False,

         {'action': 'store_const',

          'dest': 'preset_action',

          'const': 'edit',

          'default': None,

          'help': 'Edit the preset file.',

          }],

    ],
@@ -117,15 +117,6 @@ class BaseTryParser(ArgumentParser): 
            if '{msg}' not in args.message:

                args.message = '{}\n\n{}'.format(args.message, '{msg}')



        if 'preset' in self.common_groups:

            if args.list_presets:

                presets.list()

                sys.exit()



            if args.edit_presets:

                presets.edit()

                sys.exit()



    def parse_known_args(self, *args, **kwargs):

        args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs)

        self.validate(args)

tools/tryselect/mach_commands.py

+30 −8
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ from mach.decorators import ( 
    SettingsProvider,

    SubCommand,

)



from mozboot.util import get_state_dir

from mozbuild.base import BuildEnvironmentNotFoundException, MachCommandBase



CONFIG_ENVIRONMENT_NOT_FOUND = '''
@@ -70,18 +70,41 @@ class TrySelect(MachCommandBase): 
        self.subcommand = self._mach_context.handler.subcommand

        self.parser = self._mach_context.handler.parser



    def handle_presets(self, save, preset, **kwargs):

    def handle_presets(self, preset_action, save, preset, **kwargs):

        """Handle preset related arguments.



        This logic lives here so that the underlying selectors don't need

        special preset handling. They can all save and load presets the same

        way.

        """

        from tryselect.preset import presets, migrate_old_presets

        from tryselect.preset import MergedHandler, migrate_old_presets

        from tryselect.util.dicttools import merge



        # Create our handler using both local and in-tree presets. The first

        # path in this list will be treated as the 'user' file for the purposes

        # of saving and editing. All subsequent paths are 'read-only'. We check

        # an environment variable first for testing purposes.

        if os.environ.get('MACH_TRY_PRESET_PATHS'):

            preset_paths = os.environ['MACH_TRY_PRESET_PATHS'].split(os.pathsep)

        else:

            preset_paths = [

                os.path.join(get_state_dir(), 'try_presets.yml'),

                os.path.join(self.topsrcdir, 'tools', 'tryselect', 'try_presets.yml'),

            ]



        presets = MergedHandler(*preset_paths)

        user_presets = presets.handlers[0]



        # TODO: Remove after Jan 1, 2020.

        migrate_old_presets()

        migrate_old_presets(user_presets)



        if preset_action == 'list':

            presets.list()

            sys.exit()



        if preset_action == 'edit':

            user_presets.edit()

            sys.exit()



        default = self.parser.get_default

        if save:
@@ -89,19 +112,18 @@ class TrySelect(MachCommandBase): 


            # Only save non-default values for simplicity.

            kwargs = {k: v for k, v in kwargs.items() if v != default(k)}

            presets.save(save, selector=selector, **kwargs)

            user_presets.save(save, selector=selector, **kwargs)

            print('preset saved, run with: --preset={}'.format(save))

            sys.exit()



        if preset:

            if preset not in presets:

                # TODO: This should live in the parser's validation method, but

                # for now we want this check to run *after* preset migration.

                self.parser.error("preset '{}' does not exist".format(preset))



            name = preset

            preset = presets[name]

            selector = preset['selector']

            preset.pop('description', None)  # description isn't used by any selectors



            if not self.subcommand:

                self.subcommand = selector
@@ -129,7 +151,7 @@ class TrySelect(MachCommandBase): 
             category='ci',

             description='Push selected tasks to the try server',

             parser=generic_parser)

    def try_default(self, argv, **kwargs):

    def try_default(self, argv=None, **kwargs):

        """Push selected tests to the try server.



        The |mach try| command is a frontend for scheduling tasks to

tools/tryselect/preset.py

+45 −2
Original line number Diff line number Diff line
@@ -33,7 +33,12 @@ class PresetHandler(object): 
    def __getitem__(self, name):

        return self.presets[name]



    def __len__(self):

        return len(self.presets)



    def __str__(self):

        if not self.presets:

            return ''

        return yaml.safe_dump(self.presets, default_flow_style=False)



    def list(self):
@@ -56,12 +61,50 @@ class PresetHandler(object): 
            fh.write(str(self))





presets = PresetHandler(os.path.join(get_state_dir(), "try_presets.yml"))

class MergedHandler(object):

    def __init__(self, *paths):

        """Helper class for dealing with multiple preset files."""

        self.handlers = [PresetHandler(p) for p in paths]



    def __contains__(self, name):

        return any(name in handler for handler in self.handlers)



    def __getitem__(self, name):

        for handler in self.handlers:

            if name in handler:

                return handler[name]

        raise KeyError(name)



    def __len__(self):

        return sum(len(h) for h in self.handlers)



    def __str__(self):

        all_presets = {

            k: v

            for handler in self.handlers

            for k, v in handler.presets.items()

        }

        return yaml.safe_dump(all_presets, default_flow_style=False)



def migrate_old_presets():

    def list(self):

        if len(self) == 0:

            print("no presets found")

            return



        for handler in self.handlers:

            val = str(handler)

            if val:

                val = '\n  '.join([''] + val.splitlines() + [''])  # indent all lines by 2 spaces

                print("Presets from {}:".format(handler.path))

                print(val)





def migrate_old_presets(presets):

    """Move presets from the old `autotry.ini` format to the new

    `try_presets.yml` one.



    Args:

        presets (PresetHandler): Handler to migrate old presets into.

    """

    from .selectors.syntax import AutoTry, SyntaxParser

    old_preset_path = os.path.join(get_state_dir(), 'autotry.ini')

tools/tryselect/test/conftest.py

+18 −0
Original line number Diff line number Diff line
@@ -4,7 +4,10 @@ 


from __future__ import absolute_import, print_function, unicode_literals



import os



import pytest

import yaml

from mock import MagicMock

from moztest.resolve import TestResolver
@@ -40,3 +43,18 @@ def pytest_generate_tests(metafunc): 
        tests = list(load_tests())

        ids = ['{} {}'.format(t[0], ' '.join(t[1])).strip() for t in tests]

        metafunc.parametrize('template,args,expected', tests, ids=ids)



    elif all(fixture in metafunc.fixturenames for fixture in ('shared_name', 'shared_preset')):

        preset_path = os.path.join(push.build.topsrcdir, 'tools', 'tryselect', 'try_presets.yml')

        with open(preset_path, 'r') as fh:

            presets = yaml.safe_load(fh).items()



        ids = [p[0] for p in presets]



        # Mark fuzzy presets on Windows xfail due to fzf not being installed.

        if os.name == 'nt':

            for i, preset in enumerate(presets):

                if preset[1]['selector'] == 'fuzzy':

                    presets[i] = pytest.param(*preset, marks=pytest.mark.xfail)



        metafunc.parametrize('shared_name,shared_preset', presets, ids=ids)

tools/tryselect/test/python.ini

+1 −0
Original line number Diff line number Diff line
@@ -3,5 +3,6 @@ subsuite=try 
skip-if = python == 3



[test_again.py]

[test_presets.py]

[test_tasks.py]

[test_templates.py]