Bug 1648785 - Let taskcluster upload APKs to Github releases (#12907)

taskcluster/ci/build/kind.yml

@@ -21,6 +21,7 @@ job-defaults:
     apk-artifact-template:
         type: file
         name: public/build/{abi}/target.apk
+        github-name: 'fenix-{version}-{abi}.apk'
         path: '/builds/worker/checkouts/src/app/build/outputs/apk/{gradle_build_type}/{fileName}'
     description: Build Fenix from source code.
     fetches:

taskcluster/ci/config.yml

@@ -43,6 +43,11 @@ workers:
             implementation: scriptworker-signing
             os: scriptworker
             worker-type: mobile-t-signing
+        github:
+            provisioner: scriptworker-k8s
+            implementation: scriptworker-github
+            os: scriptworker
+            worker-type: 'mobile-{level}-github'
         images:
             provisioner: 'mobile-{level}'
             implementation: docker-worker

taskcluster/ci/github-release/kind.yml

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: fenix_taskgraph.loader.multi_dep:loader
+
+transforms:
+    - fenix_taskgraph.transforms.multi_dep:transforms
+    - fenix_taskgraph.transforms.github_release:transforms
+    - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+    - signing
+
+primary-dependency: signing
+
+group-by: build-type
+
+only-for-build-types:
+    - beta
+    - release
+
+job-template:
+    description: Create/Update Github release and upload APKs
+    worker-type: github
+    worker:
+        action: release
+        github-project:
+            by-level:
+                '3': fenix
+                default: mock
+        is-prerelease:
+            by-build-type:
+                beta: true
+                release: false
+                default: true
+        release-name:
+            by-build-type:
+                beta: Firefox Beta {version}
+                release: Firefox {version}
+
+    treeherder:
+        job-symbol: gh-r
+        kind: build

taskcluster/fenix_taskgraph/transforms/browsertime.py

@@ -30,9 +30,10 @@ def add_variants(config, tasks):
         if build_type not in only_types:
             continue
 
-        for abi, apk_path in dep_task.attributes["apks"].items():
+        for abi, apk_metadata in dep_task.attributes["apks"].items():
             if abi not in only_abis:
                 continue
+            apk_path = apk_metadata["name"]
             for test in tests:
                 test = copy.deepcopy(test)
                 attributes = copy.deepcopy(dep_task.attributes)

taskcluster/fenix_taskgraph/transforms/build.py

@@ -121,7 +121,13 @@ def add_artifacts(config, tasks):
                         **apk
                     ),
                 })
-                apks[apk["abi"]] = apk_name
+                apks[apk["abi"]] = {
+                    "name": apk_name,
+                    "github-name": artifact_template["github-name"].format(
+                        version=config.params["version"],
+                        **apk
+                    )
+                }
 
         yield task
 

taskcluster/fenix_taskgraph/transforms/github_release.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply some defaults and minor modifications to the jobs defined in the github_release
+kind.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def resolve_keys(config, tasks):
+    for task in tasks:
+        for key in ("worker.github-project", "worker.is-prerelease", "worker.release-name"):
+            resolve_keyed_by(
+                task,
+                key,
+                item_name=task["name"],
+                **{
+                    'build-type': task["attributes"]["build-type"],
+                    'level': config.params["level"],
+                }
+            )
+        yield task
+
+
+@transforms.add
+def build_worker_definition(config, tasks):
+    for task in tasks:
+        worker_definition = {
+            "artifact-map": _build_artifact_map(task),
+            "git-tag": config.params["head_tag"].decode("utf-8"),
+            "git-revision": config.params["head_rev"].decode("utf-8"),
+            "release-name": task["worker"]["release-name"].format(version=config.params["version"]),
+        }
+
+        task["worker"].update(worker_definition)
+
+        yield task
+
+def _build_artifact_map(task):
+    artifact_map = []
+    github_names_per_path = {
+        apk_metadata["name"]: apk_metadata["github-name"]
+        for apk_metadata in task["attributes"]["apks"].values()
+    }
+
+    for upstream_artifact_metadata in task["worker"]["upstream-artifacts"]:
+        artifacts = {"paths": {}, "taskId": upstream_artifact_metadata["taskId"]}
+        for path in upstream_artifact_metadata["paths"]:
+            artifacts["paths"][path] = {
+                "destinations": [github_names_per_path[path]]
+            }
+
+        artifact_map.append(artifacts)
+
+    return artifact_map

taskcluster/fenix_taskgraph/transforms/multi_dep.py

@@ -66,7 +66,10 @@ def build_upstream_artifacts(config, tasks):
         }
 
         for dep in _get_all_deps(task).values():
-            paths = sorted(dep.attributes.get("apks", {}).values())
+            paths = sorted([
+                apk_metadata["name"]
+                for apk_metadata in dep.attributes.get("apks", {}).values()
+            ])
             if paths:
                 worker_definition["upstream-artifacts"].append({
                     "taskId": {"task-reference": "<{}>".format(dep.kind)},

taskcluster/fenix_taskgraph/transforms/raptor.py

@@ -30,9 +30,10 @@ def add_variants(config, tasks):
         if build_type not in only_types:
             continue
 
-        for abi, apk_path in dep_task.attributes["apks"].items():
+        for abi, apk_metadata in dep_task.attributes["apks"].items():
             if abi not in only_abis:
                 continue
+            apk_path = apk_metadata["name"]
             for test in tests:
                 test = copy.deepcopy(test)
                 attributes = copy.deepcopy(dep_task.attributes)

taskcluster/fenix_taskgraph/worker_types.py

@@ -109,7 +109,7 @@ def build_push_apk_payload(config, task, task_def):
         Required("release-name"): text_type,
     },
 )
-def build_push_apk_payload(config, task, task_def):
+def build_shipit_payload(config, task, task_def):
     worker = task["worker"]
 
     task_def["tags"]["worker-implementation"] = "scriptworker"
@@ -117,3 +117,43 @@ def build_push_apk_payload(config, task, task_def):
     task_def['payload'] = {
         'release_name': worker['release-name']
     }
+
+
+@payload_builder(
+    "scriptworker-github",
+    schema={
+        Required("upstream-artifacts"): [
+            {
+                Required("taskId"): taskref_or_string,
+                Required("taskType"): text_type,
+                Required("paths"): [text_type],
+            }
+        ],
+        Required("artifact-map"): [object],
+        Required("action"): text_type,
+        Required("git-tag"): text_type,
+        Required("git-revision"): text_type,
+        Required("github-project"): text_type,
+        Required("is-prerelease"): bool,
+        Required("release-name"): text_type,
+    },
+)
+def build_github_release_payload(config, task, task_def):
+    worker = task["worker"]
+
+    task_def["tags"]["worker-implementation"] = "scriptworker"
+
+    task_def["payload"] = {
+        "artifactMap": worker["artifact-map"],
+        "gitTag": worker["git-tag"],
+        "gitRevision": worker["git-revision"],
+        "isPrerelease": worker["is-prerelease"],
+        "releaseName": worker["release-name"],
+        "upstreamArtifacts": worker["upstream-artifacts"],
+    }
+
+    scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
+    task_def["scopes"].extend([
+        "{}:github:project:{}".format(scope_prefix, worker["github-project"]),
+        "{}:github:action:{}".format(scope_prefix, worker["action"]),
+    ])