Commit daa1693f authored by Johan Lorenzo's avatar Johan Lorenzo Committed by Sebastian Kaspari
Browse files

Taskgraph skeleton

parent 6710e460
This diff is collapsed.
......@@ -258,7 +258,4 @@ if __name__ == "__main__":
else:
raise Exception('Unsupported command "{}"'.format(command))
full_task_graph = schedule_task_graph(ordered_groups_of_tasks)
populate_chain_of_trust_task_graph(full_task_graph)
populate_chain_of_trust_required_but_unused_files()
---
trust-domain: mobile
treeherder:
group-names:
'I': 'Docker Image Builds'
'Rap': 'Raptor tests'
'Rap-P': 'Raptor power tests'
task-priority: lowest
taskgraph:
register: fenix_taskgraph:register
repositories:
mobile:
name: "Fenix"
cached-task-prefix: project.mobile.fenix
workers:
aliases:
b-android:
provisioner: aws-provisioner-v1
implementation: docker-worker
os: linux
worker-type: 'mobile-{level}-b-ref-browser'
images:
provisioner: aws-provisioner-v1
implementation: docker-worker
os: linux
worker-type: 'mobile-{level}-images'
dep-signing:
provisioner: scriptworker-prov-v1
implementation: scriptworker-signing
os: scriptworker
worker-type: mobile-signing-dep-v1
signing:
provisioner: scriptworker-prov-v1
implementation: scriptworker-signing
os: scriptworker
worker-type:
by-level:
"3": mobile-signing-v1
default: mobile-signing-dep-v1
push-apk:
provisioner: scriptworker-prov-v1
implementation: scriptworker-pushapk
os: scriptworker
worker-type:
by-level:
"3": mobile-pushapk-v1
default: mobile-pushapk-dep-v1
t-bitbar.*:
provisioner: proj-autophone
implementation: generic-worker
os: linux-bitbar
worker-type: 'gecko-{alias}'
scriptworker:
scope-prefix: project:mobile:fenix:releng
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from importlib import import_module
def register(graph_config):
"""
Import all modules that are siblings of this one, triggering decorators in
the process.
"""
_import_modules(["job", "worker_types", "routes", "target"])
def _import_modules(modules):
for module in modules:
import_module(".{}".format(module), package=__name__)
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
from taskgraph.util import path
from taskgraph.util.schema import Schema
from voluptuous import Required, Optional
from six import text_type
from pipes import quote as shell_quote
gradlew_schema = Schema(
{
Required("using"): "gradlew",
Required("gradlew"): [text_type],
Optional("post-gradlew"): [[text_type]],
# Base work directory used to set up the task.
Required("workdir"): text_type,
Optional("use-caches"): bool,
Optional("secrets"): [
{
Required("name"): text_type,
Required("path"): text_type,
Required("key"): text_type,
Optional("json"): bool,
}
],
}
)
@run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
def configure_gradlew(config, job, taskdesc):
run = job["run"]
worker = taskdesc["worker"] = job["worker"]
worker.setdefault("env", {}).update(
{"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")}
)
# defer to the run_task implementation
run["command"] = _extract_command(run)
secrets = run.pop("secrets", [])
scopes = taskdesc.setdefault("scopes", [])
scopes.extend(["secrets:get:{}".format(secret["name"]) for secret in secrets])
run["cwd"] = "{checkout}"
run["using"] = "run-task"
configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
def _extract_command(run):
pre_gradle_commands = [["taskcluster/scripts/install-sdk.sh"]]
pre_gradle_commands += [
_generate_secret_command(secret) for secret in run.get("secrets", [])
]
gradle_command = ["./gradlew"] + run.pop("gradlew")
post_gradle_commands = run.pop("post-gradlew", [])
commands = pre_gradle_commands + [gradle_command] + post_gradle_commands
shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in commands]
return " && ".join(shell_quoted_commands)
def _generate_secret_command(secret):
secret_command = [
"taskcluster/scripts/get-secret.py",
"-s", secret["name"],
"-k", secret["key"],
"-f", secret["path"],
]
if secret.get("json"):
secret_command.append("--json")
return secret_command
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import time
from taskgraph.transforms.task import index_builder
SIGNING_ROUTE_TEMPLATES = [
"index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.revision.{head_rev}",
"index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.latest",
"index.project.{trust-domain}.{project}.v3.{variant}.latest",
]
@index_builder("signing")
def add_signing_indexes(config, task):
routes = task.setdefault("routes", [])
if config.params["level"] != "3":
return task
subs = config.params.copy()
subs["build_date"] = time.strftime(
"%Y.%m.%d", time.gmtime(config.params["build_date"])
)
subs["trust-domain"] = config.graph_config["trust-domain"]
subs["variant"] = task["attributes"]["build-type"]
for tpl in SIGNING_ROUTE_TEMPLATES:
routes.append(tpl.format(**subs))
return task
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.target_tasks import _target_task as target_task
@target_task("nightly")
def target_tasks_nightly(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build."""
def filter(task, parameters):
return task.attributes.get("nightly", False)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from six import text_type
from voluptuous import Required
from taskgraph.util.schema import taskref_or_string
from taskgraph.transforms.task import payload_builder
@payload_builder(
"scriptworker-signing",
schema={
# the maximum time to run, in seconds
Required("max-run-time"): int,
Required("signing-type"): text_type,
# list of artifact URLs for the artifacts that should be signed
Required("upstream-artifacts"): [
{
# taskId of the task with the artifact
Required("taskId"): taskref_or_string,
# type of signing task (for CoT)
Required("taskType"): text_type,
# Paths to the artifacts to sign
Required("paths"): [text_type],
# Signing formats to use on each of the paths
Required("formats"): [text_type],
}
],
},
)
def build_scriptworker_signing_payload(config, task, task_def):
worker = task["worker"]
task_def["tags"]["worker-implementation"] = "scriptworker"
task_def["payload"] = {
"maxRunTime": worker["max-run-time"],
"upstreamArtifacts": worker["upstream-artifacts"],
}
formats = set()
for artifacts in worker["upstream-artifacts"]:
formats.update(artifacts["formats"])
scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
task_def["scopes"].append(
"{}:signing:cert:{}".format(scope_prefix, worker["signing-type"])
)
task_def["scopes"].extend(
[
"{}:signing:format:{}".format(scope_prefix, format)
for format in sorted(formats)
]
)
@payload_builder(
"scriptworker-pushapk",
schema={
Required("upstream-artifacts"): [
{
Required("taskId"): taskref_or_string,
Required("taskType"): text_type,
Required("paths"): [text_type],
}
],
Required("channel"): text_type,
Required("commit"): bool,
Required("product"): text_type,
Required("dep"): bool,
},
)
def build_push_apk_payload(config, task, task_def):
worker = task["worker"]
task_def["tags"]["worker-implementation"] = "scriptworker"
task_def["payload"] = {
"commit": worker["commit"],
"upstreamArtifacts": worker["upstream-artifacts"],
"channel": worker["channel"],
}
scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
task_def["scopes"].append(
"{}:googleplay:product:{}{}".format(
scope_prefix, worker["product"], ":dep" if worker["dep"] else ""
)
)
#!/bin/bash
set +x
ANDROID_SDK_VERSION=3859397
curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip"
yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment