Commit e0fcf8bf authored by Mike Hommey's avatar Mike Hommey
Browse files

Bug 1335651 - Move index_paths from DockerImageTask to the base Task class. r=dustin 

This does slightly change the behavior when artifacts expire, in that
if for some reason the artifact for the task that was found expired,
we don't try to get the artifact from a lower level task. In practice,
that shouldn't be a concern.

--HG--
extra : rebase_source : 8376c2cdec0b4608bce0b41a033d8ed74e7ee63f
parent 444ba0fc

taskcluster/taskgraph/task/base.py

+27 −3
Original line number Diff line number Diff line
@@ -5,6 +5,17 @@ 
from __future__ import absolute_import, print_function, unicode_literals



import abc

import json

import os

import urllib2





# if running in a task, prefer to use the taskcluster proxy (http://taskcluster/),

# otherwise hit the services directly

if os.environ.get('TASK_ID'):

    INDEX_URL = 'http://taskcluster/index/v1/task/{}'

else:

    INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'





class Task(object):
@@ -28,7 +39,7 @@ class Task(object): 
    """

    __metaclass__ = abc.ABCMeta



    def __init__(self, kind, label, attributes, task):

    def __init__(self, kind, label, attributes, task, index_paths=None):

        self.kind = kind

        self.label = label

        self.attributes = attributes
@@ -39,12 +50,15 @@ class Task(object): 


        self.attributes['kind'] = kind



        self.index_paths = index_paths or ()



    def __eq__(self, other):

        return self.kind == other.kind and \

            self.label == other.label and \

            self.attributes == other.attributes and \

            self.task == other.task and \

            self.task_id == other.task_id

            self.task_id == other.task_id and \

            self.index_paths == other.index_paths



    @classmethod

    @abc.abstractmethod
@@ -90,8 +104,18 @@ class Task(object): 
        dependencies on this task will isntead depend on that taskId.  It is an

        error to return no taskId for a task on which other tasks depend.



        The default never optimizes.

        The default optimizes when a taskId can be found for one of the index

        paths attached to the task.

        """

        for index_path in self.index_paths:

            try:

                url = INDEX_URL.format(index_path)

                existing_task = json.load(urllib2.urlopen(url))



                return True, existing_task['taskId']

            except urllib2.HTTPError:

                pass



        return False, None



    @classmethod

taskcluster/taskgraph/task/docker_image.py

+5 −20
Original line number Diff line number Diff line
@@ -5,7 +5,6 @@ 
from __future__ import absolute_import, print_function, unicode_literals



import logging

import json

import os

import urllib2
@@ -24,22 +23,12 @@ GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..')) 
# otherwise hit the services directly

if os.environ.get('TASK_ID'):

    ARTIFACT_URL = 'http://taskcluster/queue/v1/task/{}/artifacts/{}'

    INDEX_URL = 'http://taskcluster/index/v1/task/{}'

else:

    ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'

    INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'





class DockerImageTask(base.Task):



    def __init__(self, *args, **kwargs):

        self.index_paths = kwargs.pop('index_paths')

        super(DockerImageTask, self).__init__(*args, **kwargs)



    def __eq__(self, other):

        return super(DockerImageTask, self).__eq__(other) and \

               self.index_paths == other.index_paths



    @classmethod

    def load_tasks(cls, kind, path, config, params, loaded_tasks):

        parameters = {
@@ -99,22 +88,18 @@ class DockerImageTask(base.Task): 
        return []



    def optimize(self, params):

        for index_path in self.index_paths:

        optimized, taskId = super(DockerImageTask, self).optimize(params)

        if optimized and taskId:

            try:

                url = INDEX_URL.format(index_path)

                existing_task = json.load(urllib2.urlopen(url))

                # Only return the task ID if the artifact exists for the indexed

                # task.  Otherwise, continue on looking at each of the branches.  Method

                # continues trying other branches in case mozilla-central has an expired

                # artifact, but 'project' might not. Only return no task ID if all

                # branches have been tried

                # task.

                request = urllib2.Request(

                    ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar.zst'))

                    ARTIFACT_URL.format(taskId, 'public/image.tar.zst'))

                request.get_method = lambda: 'HEAD'

                urllib2.urlopen(request)



                # HEAD success on the artifact is enough

                return True, existing_task['taskId']

                return True, taskId

            except urllib2.HTTPError:

                pass