Commit 30fa8f8a authored by Andrew Halberstadt's avatar Andrew Halberstadt
Browse files

Bug 1513951 - [tryselect] Implement in-tree try_presets.yml file r=gbrown

This creates a global preset file at:
tools/tryselect/try_presets.yml

Any presets defined here will be available for everyone to use.

Differential Revision: https://phabricator.services.mozilla.com/D21435

--HG--
extra : moz-landing-system : lando
parent b1f7c90d
......@@ -6,11 +6,9 @@ from __future__ import absolute_import, print_function, unicode_literals
import os
import subprocess
import sys
import tempfile
from argparse import ArgumentParser
from .preset import presets
from .templates import all_templates
......@@ -46,15 +44,17 @@ COMMON_ARGUMENT_GROUPS = {
'help': 'Load a saved selection.',
}],
[['--list-presets'],
{'action': 'store_true',
'dest': 'list_presets',
'default': False,
{'action': 'store_const',
'dest': 'preset_action',
'const': 'list',
'default': None,
'help': 'List available preset selections.',
}],
[['--edit-presets'],
{'action': 'store_true',
'dest': 'edit_presets',
'default': False,
{'action': 'store_const',
'dest': 'preset_action',
'const': 'edit',
'default': None,
'help': 'Edit the preset file.',
}],
],
......@@ -117,15 +117,6 @@ class BaseTryParser(ArgumentParser):
if '{msg}' not in args.message:
args.message = '{}\n\n{}'.format(args.message, '{msg}')
if 'preset' in self.common_groups:
if args.list_presets:
presets.list()
sys.exit()
if args.edit_presets:
presets.edit()
sys.exit()
def parse_known_args(self, *args, **kwargs):
args, remainder = ArgumentParser.parse_known_args(self, *args, **kwargs)
self.validate(args)
......
......@@ -15,7 +15,7 @@ from mach.decorators import (
SettingsProvider,
SubCommand,
)
from mozboot.util import get_state_dir
from mozbuild.base import BuildEnvironmentNotFoundException, MachCommandBase
CONFIG_ENVIRONMENT_NOT_FOUND = '''
......@@ -70,18 +70,41 @@ class TrySelect(MachCommandBase):
self.subcommand = self._mach_context.handler.subcommand
self.parser = self._mach_context.handler.parser
def handle_presets(self, save, preset, **kwargs):
def handle_presets(self, preset_action, save, preset, **kwargs):
"""Handle preset related arguments.
This logic lives here so that the underlying selectors don't need
special preset handling. They can all save and load presets the same
way.
"""
from tryselect.preset import presets, migrate_old_presets
from tryselect.preset import MergedHandler, migrate_old_presets
from tryselect.util.dicttools import merge
# Create our handler using both local and in-tree presets. The first
# path in this list will be treated as the 'user' file for the purposes
# of saving and editing. All subsequent paths are 'read-only'. We check
# an environment variable first for testing purposes.
if os.environ.get('MACH_TRY_PRESET_PATHS'):
preset_paths = os.environ['MACH_TRY_PRESET_PATHS'].split(os.pathsep)
else:
preset_paths = [
os.path.join(get_state_dir(), 'try_presets.yml'),
os.path.join(self.topsrcdir, 'tools', 'tryselect', 'try_presets.yml'),
]
presets = MergedHandler(*preset_paths)
user_presets = presets.handlers[0]
# TODO: Remove after Jan 1, 2020.
migrate_old_presets()
migrate_old_presets(user_presets)
if preset_action == 'list':
presets.list()
sys.exit()
if preset_action == 'edit':
user_presets.edit()
sys.exit()
default = self.parser.get_default
if save:
......@@ -89,19 +112,18 @@ class TrySelect(MachCommandBase):
# Only save non-default values for simplicity.
kwargs = {k: v for k, v in kwargs.items() if v != default(k)}
presets.save(save, selector=selector, **kwargs)
user_presets.save(save, selector=selector, **kwargs)
print('preset saved, run with: --preset={}'.format(save))
sys.exit()
if preset:
if preset not in presets:
# TODO: This should live in the parser's validation method, but
# for now we want this check to run *after* preset migration.
self.parser.error("preset '{}' does not exist".format(preset))
name = preset
preset = presets[name]
selector = preset['selector']
preset.pop('description', None) # description isn't used by any selectors
if not self.subcommand:
self.subcommand = selector
......@@ -129,7 +151,7 @@ class TrySelect(MachCommandBase):
category='ci',
description='Push selected tasks to the try server',
parser=generic_parser)
def try_default(self, argv, **kwargs):
def try_default(self, argv=None, **kwargs):
"""Push selected tests to the try server.
The |mach try| command is a frontend for scheduling tasks to
......
......@@ -33,7 +33,12 @@ class PresetHandler(object):
def __getitem__(self, name):
return self.presets[name]
def __len__(self):
return len(self.presets)
def __str__(self):
if not self.presets:
return ''
return yaml.safe_dump(self.presets, default_flow_style=False)
def list(self):
......@@ -56,12 +61,50 @@ class PresetHandler(object):
fh.write(str(self))
presets = PresetHandler(os.path.join(get_state_dir(), "try_presets.yml"))
class MergedHandler(object):
def __init__(self, *paths):
"""Helper class for dealing with multiple preset files."""
self.handlers = [PresetHandler(p) for p in paths]
def __contains__(self, name):
return any(name in handler for handler in self.handlers)
def __getitem__(self, name):
for handler in self.handlers:
if name in handler:
return handler[name]
raise KeyError(name)
def __len__(self):
return sum(len(h) for h in self.handlers)
def __str__(self):
all_presets = {
k: v
for handler in self.handlers
for k, v in handler.presets.items()
}
return yaml.safe_dump(all_presets, default_flow_style=False)
def migrate_old_presets():
def list(self):
if len(self) == 0:
print("no presets found")
return
for handler in self.handlers:
val = str(handler)
if val:
val = '\n '.join([''] + val.splitlines() + ['']) # indent all lines by 2 spaces
print("Presets from {}:".format(handler.path))
print(val)
def migrate_old_presets(presets):
"""Move presets from the old `autotry.ini` format to the new
`try_presets.yml` one.
Args:
presets (PresetHandler): Handler to migrate old presets into.
"""
from .selectors.syntax import AutoTry, SyntaxParser
old_preset_path = os.path.join(get_state_dir(), 'autotry.ini')
......
......@@ -4,7 +4,10 @@
from __future__ import absolute_import, print_function, unicode_literals
import os
import pytest
import yaml
from mock import MagicMock
from moztest.resolve import TestResolver
......@@ -40,3 +43,18 @@ def pytest_generate_tests(metafunc):
tests = list(load_tests())
ids = ['{} {}'.format(t[0], ' '.join(t[1])).strip() for t in tests]
metafunc.parametrize('template,args,expected', tests, ids=ids)
elif all(fixture in metafunc.fixturenames for fixture in ('shared_name', 'shared_preset')):
preset_path = os.path.join(push.build.topsrcdir, 'tools', 'tryselect', 'try_presets.yml')
with open(preset_path, 'r') as fh:
presets = yaml.safe_load(fh).items()
ids = [p[0] for p in presets]
# Mark fuzzy presets on Windows xfail due to fzf not being installed.
if os.name == 'nt':
for i, preset in enumerate(presets):
if preset[1]['selector'] == 'fuzzy':
presets[i] = pytest.param(*preset, marks=pytest.mark.xfail)
metafunc.parametrize('shared_name,shared_preset', presets, ids=ids)
......@@ -3,5 +3,6 @@ subsuite=try
skip-if = python == 3
[test_again.py]
[test_presets.py]
[test_tasks.py]
[test_templates.py]
export topsrcdir=$TESTDIR/../../../
export MOZBUILD_STATE_PATH=$TMP/mozbuild
export MACH_TRY_PRESET_PATHS=$MOZBUILD_STATE_PATH/try_presets.yml
export MACHRC=$TMP/machrc
cat > $MACHRC << EOF
......
......@@ -19,18 +19,20 @@ Test preset with no subcommand
Pushed via `mach try syntax`
$ ./mach try $testargs --list-presets
foo:
no_artifact: true
platforms:
- linux
selector: syntax
tags:
- foo
talos:
- none
tests:
- mochitests
Presets from */mozbuild/try_presets.yml: (glob)
foo:
no_artifact: true
platforms:
- linux
selector: syntax
tags:
- foo
talos:
- none
tests:
- mochitests
$ unset EDITOR
$ ./mach try $testargs --edit-presets
error: must set the $EDITOR environment variable to use --edit-presets
......@@ -66,30 +68,32 @@ Test preset with syntax subcommand
Pushed via `mach try syntax`
$ ./mach try syntax $testargs --list-presets
bar:
no_artifact: true
platforms:
- win32
push: false
selector: syntax
tags:
- bar
talos:
- all
tests:
- none
foo:
no_artifact: true
platforms:
- linux
selector: syntax
tags:
- foo
talos:
- none
tests:
- mochitests
Presets from */mozbuild/try_presets.yml: (glob)
bar:
no_artifact: true
platforms:
- win32
push: false
selector: syntax
tags:
- bar
talos:
- all
tests:
- none
foo:
no_artifact: true
platforms:
- linux
selector: syntax
tags:
- foo
talos:
- none
tests:
- mochitests
$ ./mach try syntax $testargs --edit-presets
bar:
no_artifact: true
......@@ -177,36 +181,38 @@ Test preset with fuzzy subcommand
}
$ ./mach try fuzzy $testargs --list-presets
bar:
no_artifact: true
platforms:
- win32
push: false
selector: syntax
tags:
- bar
talos:
- all
tests:
- none
baz:
no_artifact: true
push: false
query:
- '''baz'
selector: fuzzy
foo:
no_artifact: true
platforms:
- linux
selector: syntax
tags:
- foo
talos:
- none
tests:
- mochitests
Presets from */mozbuild/try_presets.yml: (glob)
bar:
no_artifact: true
platforms:
- win32
push: false
selector: syntax
tags:
- bar
talos:
- all
tests:
- none
baz:
no_artifact: true
push: false
query:
- '''baz'
selector: fuzzy
foo:
no_artifact: true
platforms:
- linux
selector: syntax
tags:
- foo
talos:
- none
tests:
- mochitests
$ ./mach try fuzzy $testargs --edit-presets
bar:
no_artifact: true
......
......@@ -11,9 +11,22 @@ Test migration of syntax preset
$ ls $MOZBUILD_STATE_PATH/autotry.ini
*/mozbuild/autotry.ini (glob)
$ ./mach try syntax $testargs --list-presets
no presets found
$ ./mach try syntax $testargs --preset foo
migrating saved presets from '*/mozbuild/autotry.ini' to '*/mozbuild/try_presets.yml' (glob)
Presets from */mozbuild/try_presets.yml: (glob)
foo:
builds: o
platforms:
- all
selector: syntax
tags:
- bar
talos:
- none
tests:
- mochitest
$ ./mach try syntax $testargs --preset foo
Commit message:
try: -b o -p all -u mochitest -t none --tag bar
......@@ -21,20 +34,6 @@ Test migration of syntax preset
$ ls $MOZBUILD_STATE_PATH/autotry.ini
*/mozbuild/autotry.ini': No such file or directory (glob)
[2]
$ ./mach try syntax $testargs --list-presets
foo:
builds: o
platforms:
- all
selector: syntax
tags:
- bar
talos:
- none
tests:
- mochitest
Test migration of fuzzy preset
$ rm $MOZBUILD_STATE_PATH/try_presets.yml
......@@ -44,8 +43,7 @@ Test migration of syntax preset
> EOF
$ ls $MOZBUILD_STATE_PATH/autotry.ini
*/mozbuild/autotry.ini (glob)
$ ./mach try fuzzy $testargs --list-presets
no presets found
$ ./mach try fuzzy $testargs --preset foo
migrating saved presets from '*/mozbuild/autotry.ini' to '*/mozbuild/try_presets.yml' (glob)
Commit message:
......@@ -70,11 +68,13 @@ Test migration of syntax preset
*/mozbuild/autotry.ini': No such file or directory (glob)
[2]
$ ./mach try fuzzy $testargs --list-presets
foo:
query:
- '''foo | ''bar'
selector: fuzzy
Presets from */mozbuild/try_presets.yml: (glob)
foo:
query:
- '''foo | ''bar'
selector: fuzzy
Test unknown section prints message
......
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import mozunit
import pytest
@pytest.fixture(scope='session')
def run_mach():
import mach_bootstrap
from mach.config import ConfigSettings
from tryselect.tasks import build
mach = mach_bootstrap.bootstrap(build.topsrcdir)
def inner(args):
mach.settings = ConfigSettings()
return mach.run(args)
return inner
def test_shared_presets(run_mach, shared_name, shared_preset):
"""This test makes sure that we don't break any of the in-tree presets when
renaming/removing variables in any of the selectors.
"""
assert 'description' in shared_preset
assert 'selector' in shared_preset
selector = shared_preset['selector']
if selector == 'fuzzy':
assert 'query' in shared_preset
assert isinstance(shared_preset['query'], list)
# Run the preset and assert there were no exceptions.
assert run_mach(['try', '--no-push', '--preset', shared_name]) == 0
if __name__ == '__main__':
mozunit.main()
---
# Presets defined here will be available to all users. Run them with:
# $ mach try --preset <name>
#
# If editing this file, make sure to run:
# $ mach python-test tools/tryselect/test/test_presets.py
#
# Descriptions are required. Please keep this in alphabetical order.
sample-suites:
selector: fuzzy
description: >-
Runs one chunk of every test suite plus all suites that aren't chunked.
It is useful for testing infrastructure changes that can affect the
harnesses themselves but are unlikely to break specific tests.
query:
- ^test- -1$
- ^test- !1$ !2$ !3$ !4$ !5$ !6$ !7$ !8$ !9$ !0$
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment