Bug 1258497: Factor out some common code; r=gps

This code will also be used by taskgraph's legacy kind.

MozReview-Commit-ID: Gd4zZYEee2W

--HG--
extra : rebase_source : 41a76624322687d752c6d836c15700c92ed80cdb

testing/taskcluster/mach_commands.py

@@ -38,164 +38,6 @@ DEFAULT_JOB_PATH = os.path.join(
 # time after which a try build's results will expire
 TRY_EXPIRATION = "14 days"
 
-def merge_dicts(*dicts):
-    merged_dict = {}
-    for dictionary in dicts:
-        merged_dict.update(dictionary)
-    return merged_dict
-
-def gaia_info():
-    '''
-    Fetch details from in tree gaia.json (which links this version of
-    gecko->gaia) and construct the usual base/head/ref/rev pairing...
-    '''
-    gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
-
-    if gaia['git'] is None or \
-       gaia['git']['remote'] == '' or \
-       gaia['git']['git_revision'] == '' or \
-       gaia['git']['branch'] == '':
-
-       # Just use the hg params...
-       return {
-         'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
-         'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
-         'gaia_ref': gaia['revision'],
-         'gaia_rev': gaia['revision']
-       }
-
-    else:
-        # Use git
-        return {
-            'gaia_base_repository': gaia['git']['remote'],
-            'gaia_head_repository': gaia['git']['remote'],
-            'gaia_rev': gaia['git']['git_revision'],
-            'gaia_ref': gaia['git']['branch'],
-        }
-
-def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
-    """
-    Configure a build dependent task. This is shared between post-build and test tasks.
-
-    :param task_path: location to the task yaml
-    :param parameters: parameters to load the template
-    :param taskid: taskid of the dependent task
-    :param templates: reference to the template builder
-    :param build_treeherder_config: parent treeherder config
-    :return: the configured task
-    """
-    task = templates.load(task_path, parameters)
-    task['taskId'] = taskid
-
-    if 'requires' not in task:
-        task['requires'] = []
-
-    task['requires'].append(parameters['build_slugid'])
-
-    if 'treeherder' not in task['task']['extra']:
-        task['task']['extra']['treeherder'] = {}
-
-    # Copy over any treeherder configuration from the build so
-    # tests show up under the same platform...
-    treeherder_config = task['task']['extra']['treeherder']
-
-    treeherder_config['collection'] = \
-        build_treeherder_config.get('collection', {})
-
-    treeherder_config['build'] = \
-        build_treeherder_config.get('build', {})
-
-    treeherder_config['machine'] = \
-        build_treeherder_config.get('machine', {})
-
-    if 'routes' not in task['task']:
-        task['task']['routes'] = []
-
-    if 'scopes' not in task['task']:
-        task['task']['scopes'] = []
-
-    return task
-
-def set_interactive_task(task, interactive):
-    r"""Make the task interactive.
-
-    :param task: task definition.
-    :param interactive: True if the task should be interactive.
-    """
-    if not interactive:
-        return
-
-    payload = task["task"]["payload"]
-    if "features" not in payload:
-        payload["features"] = {}
-    payload["features"]["interactive"] = True
-
-def remove_caches_from_task(task):
-    r"""Remove all caches but tc-vcs from the task.
-
-    :param task: task definition.
-    """
-    whitelist = [
-        re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
-        re.compile("^tooltool-cache$"),
-    ]
-    try:
-        caches = task["task"]["payload"]["cache"]
-        for cache in caches.keys():
-            if not any(pat.match(cache) for pat in whitelist):
-                caches.pop(cache)
-    except KeyError:
-        pass
-
-def set_expiration(task, timestamp):
-    task_def = task['task']
-    task_def['expires'] = timestamp
-    if task_def.get('deadline', timestamp) > timestamp:
-        task_def['deadline'] = timestamp
-
-    try:
-        artifacts = task_def['payload']['artifacts']
-    except KeyError:
-        return
-
-    for artifact in artifacts.values():
-        artifact['expires'] = timestamp
-
-def query_vcs_info(repository, revision):
-    """Query the pushdate and pushid of a repository/revision.
-    This is intended to be used on hg.mozilla.org/mozilla-central and
-    similar. It may or may not work for other hg repositories.
-    """
-    if not repository or not revision:
-        sys.stderr.write('cannot query vcs info because vcs info not provided\n')
-        return None
-
-    VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
-
-    try:
-        import requests
-        url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
-                                                  revision)
-        sys.stderr.write("Querying version control for metadata: %s\n" % url)
-        contents = requests.get(url).json()
-
-        changesets = []
-        for c in contents['changesets']:
-            changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
-
-        pushid = contents['changesets'][-1]['pushid']
-        pushdate = contents['changesets'][-1]['pushdate'][0]
-
-        return VCSInfo(pushid, pushdate, changesets)
-
-    except Exception:
-        sys.stderr.write(
-            "Error querying VCS info for '%s' revision '%s'\n" % (
-                repository, revision,
-            )
-        )
-        return None
-
 @CommandProvider
 class DecisionTask(object):
     @Command('taskcluster-decision', category="ci",
@@ -219,6 +61,7 @@ class DecisionTask(object):
         help='email address of who owns this graph')
     @CommandArgument('task', help="Path to decision task to run.")
     def run_task(self, **params):
+        from taskcluster_graph.mach_util import gaia_info
         from taskcluster_graph.slugidjar import SlugidJar
         from taskcluster_graph.from_now import (
             json_time_from_now,
@@ -335,6 +178,14 @@ class Graph(object):
 
         from slugid import nice as slugid
 
+        from taskcluster_graph.mach_util import (
+            merge_dicts,
+            gaia_info,
+            configure_dependent_task,
+            set_interactive_task,
+            remove_caches_from_task,
+            query_vcs_info
+        )
         import taskcluster_graph.transform.routes as routes_transform
         import taskcluster_graph.transform.treeherder as treeherder_transform
         from taskcluster_graph.commit_parser import parse_commit
@@ -717,6 +568,11 @@ class CIBuild(object):
         dest="interactive",
         help="Run the task with the interactive feature enabled")
     def create_ci_build(self, **params):
+        from taskcluster_graph.mach_util import (
+            gaia_info,
+            set_interactive_task,
+            query_vcs_info
+        )
         from taskcluster_graph.templates import Templates
         from taskcluster_graph.image_builder import docker_image
         import taskcluster_graph.build_task

testing/taskcluster/taskcluster_graph/mach_util.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+from collections import defaultdict
+import os
+import json
+import copy
+import re
+import sys
+import time
+from collections import namedtuple
+
+ROOT = os.path.dirname(os.path.realpath(__file__))
+GECKO = os.path.realpath(os.path.join(ROOT, '..', '..', '..'))
+
+def merge_dicts(*dicts):
+    merged_dict = {}
+    for dictionary in dicts:
+        merged_dict.update(dictionary)
+    return merged_dict
+
+def gaia_info():
+    '''
+    Fetch details from in tree gaia.json (which links this version of
+    gecko->gaia) and construct the usual base/head/ref/rev pairing...
+    '''
+    gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
+
+    if gaia['git'] is None or \
+       gaia['git']['remote'] == '' or \
+       gaia['git']['git_revision'] == '' or \
+       gaia['git']['branch'] == '':
+
+       # Just use the hg params...
+       return {
+         'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
+         'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
+         'gaia_ref': gaia['revision'],
+         'gaia_rev': gaia['revision']
+       }
+
+    else:
+        # Use git
+        return {
+            'gaia_base_repository': gaia['git']['remote'],
+            'gaia_head_repository': gaia['git']['remote'],
+            'gaia_rev': gaia['git']['git_revision'],
+            'gaia_ref': gaia['git']['branch'],
+        }
+
+def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
+    """
+    Configure a build dependent task. This is shared between post-build and test tasks.
+
+    :param task_path: location to the task yaml
+    :param parameters: parameters to load the template
+    :param taskid: taskid of the dependent task
+    :param templates: reference to the template builder
+    :param build_treeherder_config: parent treeherder config
+    :return: the configured task
+    """
+    task = templates.load(task_path, parameters)
+    task['taskId'] = taskid
+
+    if 'requires' not in task:
+        task['requires'] = []
+
+    task['requires'].append(parameters['build_slugid'])
+
+    if 'treeherder' not in task['task']['extra']:
+        task['task']['extra']['treeherder'] = {}
+
+    # Copy over any treeherder configuration from the build so
+    # tests show up under the same platform...
+    treeherder_config = task['task']['extra']['treeherder']
+
+    treeherder_config['collection'] = \
+        build_treeherder_config.get('collection', {})
+
+    treeherder_config['build'] = \
+        build_treeherder_config.get('build', {})
+
+    treeherder_config['machine'] = \
+        build_treeherder_config.get('machine', {})
+
+    if 'routes' not in task['task']:
+        task['task']['routes'] = []
+
+    if 'scopes' not in task['task']:
+        task['task']['scopes'] = []
+
+    return task
+
+def set_interactive_task(task, interactive):
+    r"""Make the task interactive.
+
+    :param task: task definition.
+    :param interactive: True if the task should be interactive.
+    """
+    if not interactive:
+        return
+
+    payload = task["task"]["payload"]
+    if "features" not in payload:
+        payload["features"] = {}
+    payload["features"]["interactive"] = True
+
+def remove_caches_from_task(task):
+    r"""Remove all caches but tc-vcs from the task.
+
+    :param task: task definition.
+    """
+    whitelist = [
+        re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
+        re.compile("^tooltool-cache$"),
+    ]
+    try:
+        caches = task["task"]["payload"]["cache"]
+        for cache in caches.keys():
+            if not any(pat.match(cache) for pat in whitelist):
+                caches.pop(cache)
+    except KeyError:
+        pass
+
+def query_vcs_info(repository, revision):
+    """Query the pushdate and pushid of a repository/revision.
+    This is intended to be used on hg.mozilla.org/mozilla-central and
+    similar. It may or may not work for other hg repositories.
+    """
+    if not repository or not revision:
+        sys.stderr.write('cannot query vcs info because vcs info not provided\n')
+        return None
+
+    VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
+
+    try:
+        import requests
+        url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
+                                                  revision)
+        sys.stderr.write("Querying version control for metadata: %s\n" % url)
+        contents = requests.get(url).json()
+
+        changesets = []
+        for c in contents['changesets']:
+            changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
+
+        pushid = contents['changesets'][-1]['pushid']
+        pushdate = contents['changesets'][-1]['pushdate'][0]
+
+        return VCSInfo(pushid, pushdate, changesets)
+
+    except Exception:
+        sys.stderr.write(
+            "Error querying VCS info for '%s' revision '%s'\n" % (
+                repository, revision,
+            )
+        )
+        return None
+
+