Bug 1568466 - part 3: Enable shipit graph (#7314)

taskcluster/ci/config.yml

@@ -26,7 +26,7 @@ taskgraph:
         mobile:
             name: "Fenix"
     cached-task-prefix: project.mobile.fenix
-    decision-parameters: 'fenix_taskgraph:get_decision_parameters'
+    decision-parameters: 'fenix_taskgraph.parameters:get_decision_parameters'
 
 workers:
     aliases:
@@ -71,3 +71,9 @@ workers:
 
 scriptworker:
     scope-prefix: project:mobile:fenix:releng
+
+
+release-promotion:
+    flavors:
+        ship:
+            target-tasks-method: release

taskcluster/fenix_taskgraph/__init__.py

@@ -4,17 +4,7 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import os
-import re
-
 from importlib import import_module
-from six import text_type
-from voluptuous import All, Any, Range, Required
-
-from taskgraph.parameters import extend_parameters_schema
-
-BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
-PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
 
 
 def register(graph_config):
@@ -22,52 +12,16 @@ def register(graph_config):
     Import all modules that are siblings of this one, triggering decorators in
     the process.
     """
-    _import_modules(["job", "worker_types", "routes", "target_tasks"])
-    extend_parameters_schema({
-        Required("pull_request_number"): Any(All(int, Range(min=1)), None),
-        Required("release_type"): text_type,
-        Required("release_version"): text_type,
-    })
+    _import_modules([
+        "job",
+        "parameters",
+        "release_promotion",
+        "routes",
+        "target_tasks",
+        "worker_types",
+    ])
 
 
 def _import_modules(modules):
     for module in modules:
         import_module(".{}".format(module), package=__name__)
-
-
-def get_decision_parameters(graph_config, parameters):
-    head_tag = parameters["head_tag"].decode("utf-8")
-    parameters["release_type"] = _resolve_release_type(head_tag)
-    parameters["release_version"] = read_version_file()
-    if head_tag:
-        parameters["release_version"] = head_tag[1:]
-
-    pr_number = os.environ.get("MOBILE_PULL_REQUEST_NUMBER", None)
-    parameters["pull_request_number"] = None if pr_number is None else int(pr_number)
-
-    if parameters["tasks_for"] == "github-release":
-        for param_name in ("release_type", "release_version"):
-            if not parameters[param_name]:
-                raise ValueError(
-                    'Cannot run github-release if "{}" is not defined. Got: {}'.format(
-                        param_name, parameters[param_name]
-                    )
-                )
-        parameters["target_tasks_method"] = "release"
-
-
-def read_version_file():
-    with open(os.path.join(os.path.dirname(__file__), '..', '..', 'version.txt')) as f:
-        return f.read().strip().decode('utf-8')
-
-
-def _resolve_release_type(head_tag):
-    if not head_tag:
-        return ""
-    elif BETA_SEMVER.match(head_tag):
-        return "beta"
-    elif PRODUCTION_SEMVER.match(head_tag):
-        return "production"
-    else:
-        raise ValueError('Github tag must be in semver format and prefixed with a "v", '
-                         'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')

taskcluster/fenix_taskgraph/parameters.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import re
+
+from six import text_type
+from taskgraph.parameters import extend_parameters_schema
+from voluptuous import All, Any, Optional, Range, Required
+
+
+BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
+PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
+
+
+extend_parameters_schema({
+    Required("pull_request_number"): Any(All(int, Range(min=1)), None),
+    Required("release_type"): text_type,
+    Optional("shipping_phase"): Any('build', 'ship', None),
+    Required("version"): text_type,
+})
+
+
+def get_decision_parameters(graph_config, parameters):
+    head_tag = parameters["head_tag"].decode("utf-8")
+    parameters["release_type"] = resolve_release_type(head_tag)
+    parameters["version"] = head_tag[1:] if head_tag else ""
+
+    pr_number = os.environ.get("MOBILE_PULL_REQUEST_NUMBER", None)
+    parameters["pull_request_number"] = None if pr_number is None else int(pr_number)
+
+    if parameters["tasks_for"] == "github-release":
+        for param_name in ("release_type", "version"):
+            if not parameters[param_name]:
+                raise ValueError(
+                    'Cannot run github-release if "{}" is not defined. Got: {}'.format(
+                        param_name, parameters[param_name]
+                    )
+                )
+        parameters["target_tasks_method"] = "release"
+
+
+def resolve_release_type(head_tag):
+    if not head_tag:
+        return ""
+    elif BETA_SEMVER.match(head_tag):
+        return "beta"
+    elif PRODUCTION_SEMVER.match(head_tag):
+        return "production"
+    else:
+        raise ValueError('Github tag must be in semver format and prefixed with a "v", '
+                         'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')

taskcluster/fenix_taskgraph/release_promotion.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.actions.registry import register_callback_action
+
+from taskgraph.util.taskcluster import get_artifact
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.decision import taskgraph_decision
+from taskgraph.parameters import Parameters
+from taskgraph.util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds
+
+from .parameters import resolve_release_type
+
+RELEASE_PROMOTION_PROJECTS = (
+    "https://github.com/JohanLorenzo/fenix",
+)
+
+
+def is_release_promotion_available(parameters):
+    return parameters['head_repository'] in RELEASE_PROMOTION_PROJECTS
+
+
+@register_callback_action(
+    name='release-promotion',
+    title='Ship Fenix',
+    symbol='${input.release_promotion_flavor}',
+    description="Ship Fenix",
+    generic=False,
+    order=500,
+    context=[],
+    available=is_release_promotion_available,
+    schema=lambda graph_config: {
+        'type': 'object',
+        'properties': {
+            'build_number': {
+                'type': 'integer',
+                'default': 1,
+                'minimum': 1,
+                'title': 'The release build number',
+                'description': ('The release build number. Starts at 1 per '
+                                'release version, and increments on rebuild.'),
+            },
+            'do_not_optimize': {
+                'type': 'array',
+                'description': ('Optional: a list of labels to avoid optimizing out '
+                                'of the graph (to force a rerun of, say, '
+                                'funsize docker-image tasks).'),
+                'items': {
+                    'type': 'string',
+                },
+            },
+            'revision': {
+                'type': 'string',
+                'title': 'Optional: revision to ship',
+                'description': ('Optional: the revision to ship.'),
+            },
+            'release_promotion_flavor': {
+                'type': 'string',
+                'description': 'The flavor of release promotion to perform.',
+                'default': 'build',
+                'enum': sorted(graph_config['release-promotion']['flavors'].keys()),
+            },
+            'rebuild_kinds': {
+                'type': 'array',
+                'description': ('Optional: an array of kinds to ignore from the previous '
+                                'graph(s).'),
+                'items': {
+                    'type': 'string',
+                },
+            },
+            'previous_graph_ids': {
+                'type': 'array',
+                'description': ('Optional: an array of taskIds of decision or action '
+                                'tasks from the previous graph(s) to use to populate '
+                                'our `previous_graph_kinds`.'),
+                'items': {
+                    'type': 'string',
+                },
+            },
+            'version': {
+                'type': 'string',
+                'description': ('Optional: override the version for release promotion. '
+                                "Occasionally we'll land a taskgraph fix in a later "
+                                'commit, but want to act on a build from a previous '
+                                'commit. If a version bump has landed in the meantime, '
+                                'relying on the in-tree version will break things.'),
+                'default': '',
+            },
+        },
+        "required": ['release_promotion_flavor', 'version', 'build_number'],
+    }
+)
+def release_promotion_action(parameters, graph_config, input, task_group_id, task_id):
+    release_promotion_flavor = input['release_promotion_flavor']
+    promotion_config = graph_config['release-promotion']['flavors'][release_promotion_flavor]
+
+    target_tasks_method = promotion_config['target-tasks-method'].format(
+        project=parameters['project']
+    )
+    rebuild_kinds = input.get('rebuild_kinds') or promotion_config.get('rebuild-kinds', [])
+    do_not_optimize = input.get('do_not_optimize') or promotion_config.get('do-not-optimize', [])
+
+    # make parameters read-write
+    parameters = dict(parameters)
+    # Build previous_graph_ids from ``previous_graph_ids`` or ``revision``.
+    previous_graph_ids = input.get('previous_graph_ids')
+    if not previous_graph_ids:
+        previous_graph_ids = [find_decision_task(parameters, graph_config)]
+
+    # Download parameters from the first decision task
+    parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
+    # Download and combine full task graphs from each of the previous_graph_ids.
+    # Sometimes previous relpro action tasks will add tasks, like partials,
+    # that didn't exist in the first full_task_graph, so combining them is
+    # important. The rightmost graph should take precedence in the case of
+    # conflicts.
+    combined_full_task_graph = {}
+    for graph_id in previous_graph_ids:
+        full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
+        combined_full_task_graph.update(full_task_graph)
+    _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
+    parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
+        combined_full_task_graph, previous_graph_ids, rebuild_kinds
+    )
+    parameters['do_not_optimize'] = do_not_optimize
+    parameters['target_tasks_method'] = target_tasks_method
+    parameters['build_number'] = int(input['build_number'])
+    # When doing staging releases on try, we still want to re-use tasks from
+    # previous graphs.
+    parameters['optimize_target_tasks'] = True
+    parameters['shipping_phase'] = input['release_promotion_flavor']
+
+    parameters['version'] = input['version'] if input.get('version') else read_version_file()
+    parameters['head_tag'] = 'v{}'.format(parameters['version'])
+    parameters['release_type'] = resolve_release_type(parameters['head_tag'])
+
+    parameters['pull_request_number'] = None
+
+    # make parameters read-only
+    parameters = Parameters(**parameters)
+
+    taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
+
+
+def read_version_file():
+    with open(os.path.join(os.path.dirname(__file__), '..', '..', 'version.txt')) as f:
+        return f.read().strip().decode('utf-8')

taskcluster/fenix_taskgraph/target_tasks.py

@@ -19,6 +19,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config):
 @_target_task('release')
 def target_tasks_default(full_task_graph, parameters, graph_config):
 
+    # TODO Use shipping-phase once we retire github-releases
     def filter(task, parameters):
         return task.attributes.get("release-type", "") == parameters["release_type"]
 

taskcluster/fenix_taskgraph/transforms/build.py

@@ -97,7 +97,7 @@ def add_release_version(config, tasks):
     for task in tasks:
         if task.pop("include-release-version", False):
             task["run"]["gradlew"].append(
-                '-PversionName={}'.format(config.params["release_version"])
+                '-PversionName={}'.format(config.params["version"])
             )
         yield task
 

taskcluster/fenix_taskgraph/transforms/mark_as_shipped.py

@@ -19,10 +19,10 @@ transforms = TransformSequence()
 def make_task_description(config, jobs):
     for job in jobs:
         product = "Fenix"
-        version = config.params['release_version'] or "{ver}"
+        version = config.params['version'] or "{ver}"
         job['worker']['release-name'] = '{product}-{version}-build{build_number}'.format(
             product=product,
             version=version,
             build_number=config.params.get('build_number', 1)
         )
-        yield job
\ No newline at end of file
+        yield job