Taskgraph skeleton

automation/taskcluster/decision_task.py

@@ -258,7 +258,4 @@ if __name__ == "__main__":
     else:
         raise Exception('Unsupported command "{}"'.format(command))
 
-    full_task_graph = schedule_task_graph(ordered_groups_of_tasks)
-
     populate_chain_of_trust_task_graph(full_task_graph)
-    populate_chain_of_trust_required_but_unused_files()

taskcluster/ci/config.yml

+---
+trust-domain: mobile
+treeherder:
+    group-names:
+        'I': 'Docker Image Builds'
+        'Rap': 'Raptor tests'
+        'Rap-P': 'Raptor power tests'
+
+task-priority: lowest
+
+taskgraph:
+    register: fenix_taskgraph:register
+    repositories:
+        mobile:
+            name: "Fenix"
+    cached-task-prefix: project.mobile.fenix
+
+workers:
+    aliases:
+        b-android:
+            provisioner: aws-provisioner-v1
+            implementation: docker-worker
+            os: linux
+            worker-type: 'mobile-{level}-b-ref-browser'
+        images:
+            provisioner: aws-provisioner-v1
+            implementation: docker-worker
+            os: linux
+            worker-type: 'mobile-{level}-images'
+        dep-signing:
+            provisioner: scriptworker-prov-v1
+            implementation: scriptworker-signing
+            os: scriptworker
+            worker-type: mobile-signing-dep-v1
+        signing:
+            provisioner: scriptworker-prov-v1
+            implementation: scriptworker-signing
+            os: scriptworker
+            worker-type:
+                by-level:
+                    "3": mobile-signing-v1
+                    default: mobile-signing-dep-v1
+        push-apk:
+            provisioner: scriptworker-prov-v1
+            implementation: scriptworker-pushapk
+            os: scriptworker
+            worker-type:
+                by-level:
+                    "3": mobile-pushapk-v1
+                    default: mobile-pushapk-dep-v1
+        t-bitbar.*:
+            provisioner: proj-autophone
+            implementation: generic-worker
+            os: linux-bitbar
+            worker-type: 'gecko-{alias}'
+
+scriptworker:
+    scope-prefix: project:mobile:fenix:releng

taskcluster/fenix_taskgraph/__init__.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from importlib import import_module
+
+
+def register(graph_config):
+    """
+    Import all modules that are siblings of this one, triggering decorators in
+    the process.
+    """
+    _import_modules(["job", "worker_types", "routes", "target"])
+
+
+def _import_modules(modules):
+    for module in modules:
+        import_module(".{}".format(module), package=__name__)

taskcluster/fenix_taskgraph/job.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
+from taskgraph.util import path
+from taskgraph.util.schema import Schema
+from voluptuous import Required, Optional
+from six import text_type
+
+from pipes import quote as shell_quote
+
+gradlew_schema = Schema(
+    {
+        Required("using"): "gradlew",
+        Required("gradlew"): [text_type],
+        Optional("post-gradlew"): [[text_type]],
+        # Base work directory used to set up the task.
+        Required("workdir"): text_type,
+        Optional("use-caches"): bool,
+        Optional("secrets"): [
+            {
+                Required("name"): text_type,
+                Required("path"): text_type,
+                Required("key"): text_type,
+                Optional("json"): bool,
+            }
+        ],
+    }
+)
+
+
+@run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
+def configure_gradlew(config, job, taskdesc):
+    run = job["run"]
+    worker = taskdesc["worker"] = job["worker"]
+
+    worker.setdefault("env", {}).update(
+        {"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")}
+    )
+
+    # defer to the run_task implementation
+    run["command"] = _extract_command(run)
+    secrets = run.pop("secrets", [])
+    scopes = taskdesc.setdefault("scopes", [])
+    scopes.extend(["secrets:get:{}".format(secret["name"]) for secret in secrets])
+
+    run["cwd"] = "{checkout}"
+    run["using"] = "run-task"
+    configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
+
+
+def _extract_command(run):
+    pre_gradle_commands = [["taskcluster/scripts/install-sdk.sh"]]
+    pre_gradle_commands += [
+        _generate_secret_command(secret) for secret in run.get("secrets", [])
+    ]
+
+    gradle_command = ["./gradlew"] + run.pop("gradlew")
+    post_gradle_commands = run.pop("post-gradlew", [])
+
+    commands = pre_gradle_commands + [gradle_command] + post_gradle_commands
+    shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in commands]
+    return " && ".join(shell_quoted_commands)
+
+
+def _generate_secret_command(secret):
+    secret_command = [
+        "taskcluster/scripts/get-secret.py",
+        "-s", secret["name"],
+        "-k", secret["key"],
+        "-f", secret["path"],
+    ]
+    if secret.get("json"):
+        secret_command.append("--json")
+
+    return secret_command

taskcluster/fenix_taskgraph/routes.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+
+from taskgraph.transforms.task import index_builder
+
+SIGNING_ROUTE_TEMPLATES = [
+    "index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.revision.{head_rev}",
+    "index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.latest",
+    "index.project.{trust-domain}.{project}.v3.{variant}.latest",
+]
+
+
+@index_builder("signing")
+def add_signing_indexes(config, task):
+    routes = task.setdefault("routes", [])
+
+    if config.params["level"] != "3":
+        return task
+
+    subs = config.params.copy()
+    subs["build_date"] = time.strftime(
+        "%Y.%m.%d", time.gmtime(config.params["build_date"])
+    )
+    subs["trust-domain"] = config.graph_config["trust-domain"]
+    subs["variant"] = task["attributes"]["build-type"]
+
+    for tpl in SIGNING_ROUTE_TEMPLATES:
+        routes.append(tpl.format(**subs))
+    return task

taskcluster/fenix_taskgraph/target.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.target_tasks import _target_task as target_task
+
+
+@target_task("nightly")
+def target_tasks_nightly(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required for a nightly build."""
+
+    def filter(task, parameters):
+        return task.attributes.get("nightly", False)
+
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]

taskcluster/fenix_taskgraph/worker_types.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from six import text_type
+
+from voluptuous import Required
+
+from taskgraph.util.schema import taskref_or_string
+from taskgraph.transforms.task import payload_builder
+
+
+@payload_builder(
+    "scriptworker-signing",
+    schema={
+        # the maximum time to run, in seconds
+        Required("max-run-time"): int,
+        Required("signing-type"): text_type,
+        # list of artifact URLs for the artifacts that should be signed
+        Required("upstream-artifacts"): [
+            {
+                # taskId of the task with the artifact
+                Required("taskId"): taskref_or_string,
+                # type of signing task (for CoT)
+                Required("taskType"): text_type,
+                # Paths to the artifacts to sign
+                Required("paths"): [text_type],
+                # Signing formats to use on each of the paths
+                Required("formats"): [text_type],
+            }
+        ],
+    },
+)
+def build_scriptworker_signing_payload(config, task, task_def):
+    worker = task["worker"]
+
+    task_def["tags"]["worker-implementation"] = "scriptworker"
+
+    task_def["payload"] = {
+        "maxRunTime": worker["max-run-time"],
+        "upstreamArtifacts": worker["upstream-artifacts"],
+    }
+
+    formats = set()
+    for artifacts in worker["upstream-artifacts"]:
+        formats.update(artifacts["formats"])
+
+    scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
+    task_def["scopes"].append(
+        "{}:signing:cert:{}".format(scope_prefix, worker["signing-type"])
+    )
+    task_def["scopes"].extend(
+        [
+            "{}:signing:format:{}".format(scope_prefix, format)
+            for format in sorted(formats)
+        ]
+    )
+
+
+@payload_builder(
+    "scriptworker-pushapk",
+    schema={
+        Required("upstream-artifacts"): [
+            {
+                Required("taskId"): taskref_or_string,
+                Required("taskType"): text_type,
+                Required("paths"): [text_type],
+            }
+        ],
+        Required("channel"): text_type,
+        Required("commit"): bool,
+        Required("product"): text_type,
+        Required("dep"): bool,
+    },
+)
+def build_push_apk_payload(config, task, task_def):
+    worker = task["worker"]
+
+    task_def["tags"]["worker-implementation"] = "scriptworker"
+
+    task_def["payload"] = {
+        "commit": worker["commit"],
+        "upstreamArtifacts": worker["upstream-artifacts"],
+        "channel": worker["channel"],
+    }
+
+    scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
+    task_def["scopes"].append(
+        "{}:googleplay:product:{}{}".format(
+            scope_prefix, worker["product"], ":dep" if worker["dep"] else ""
+        )
+    )

taskcluster/scripts/install-sdk.sh

+#!/bin/bash
+
+set +x
+
+ANDROID_SDK_VERSION=3859397
+
+curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
+unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip"
+yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses