Unverified Commit 6a8f7224 authored by Johan Lorenzo's avatar Johan Lorenzo Committed by GitHub
Browse files

Bug 1580778 - Migrate build tasks to taskgraph and retire the old decision task (#5488)

parent 7c55aff1
......@@ -8,7 +8,6 @@ jobs:
job:
type: decision-task
treeherder-symbol: Nd
# TODO change target method once first tasks are migrated off "old-decision"
target-tasks-method: nightly
when: [{hour: 6, minute: 0}]
when: [{hour: 18, minute: 0}]
......@@ -16,6 +15,5 @@ jobs:
job:
type: decision-task
treeherder-symbol: raptor-D
# TODO change target method once first tasks are migrated off "old-decision"
target-tasks-method: raptor
when: [{hour: 1, minute: 0}]
......@@ -22,6 +22,7 @@ out/
# Gradle files
.gradle/
build/
!taskcluster/ci/build
# Local configuration file (sdk path, etc)
local.properties
......
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Decision task for nightly releases.
"""
from __future__ import print_function
import argparse
import datetime
import os
import re
import taskcluster
from lib.gradle import get_variant
from lib.tasks import (
fetch_mozharness_task_id,
schedule_task_graph,
TaskBuilder,
)
from lib.chain_of_trust import (
populate_chain_of_trust_task_graph,
populate_chain_of_trust_required_but_unused_files
)
def pr(builder):
tasks = []
variant = get_variant('debug', 'geckoNightly')
tasks.append(builder.craft_assemble_pr_task(variant))
tasks.append(builder.craft_test_pr_task(variant))
for task in tasks:
task['attributes']['code-review'] = True
return tasks
def push(builder):
# We want the same tasks on pushes than on PRs, for now.
return pr(builder)
def raptor(builder, is_staging):
mozharness_task_id = fetch_mozharness_task_id()
gecko_revision = taskcluster.Queue({
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
}).task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('forPerformanceTest', 'geckoNightly')
build_task = builder.craft_assemble_raptor_task(variant)
# Signing and Raptor tasks are generated in taskgraph
return [build_task]
def release(builder, channel, engine, is_staging, version_name):
variant = get_variant('fenix' + channel.capitalize(), engine)
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
# The signing push-apk tasks are generated by taskgraph
return [build_task]
def release_as_fennec(builder, is_staging, version_name):
variant = get_variant('fennecProduction', 'geckoBeta')
channel = 'fennec-production'
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
# The signing task is generated by taskgraph
return [build_task]
def nightly_to_production_app(builder, is_staging, version_name):
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
# keep updating this "backwards-compatible" nightly for a while yet
variant = get_variant('fenixNightlyLegacy', 'geckoNightly')
taskcluster_apk_paths = variant.upstream_artifacts()
build_task = builder.craft_assemble_release_task(
variant, 'nightly-legacy', is_staging, version_name)
# Nimbledroid, signing and push-apk tasks are generated by taskgraph
return [build_task]
import json
def populate_chain_of_trust_required_but_unused_files():
# These files are needed to keep chainOfTrust happy. However, they have no
# need for android-components, at the moment. For more details, see:
# https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585
for file_names in ('actions.json', 'parameters.yml'):
with open(file_names, 'w') as f:
json.dump({}, f) # Yaml is a super-set of JSON.
def populate_chain_of_trust_task_graph(full_task_graph):
# full_task_graph must follow the format:
# {
# task_id: full_task_definition
# }
with open('task-graph.json', 'w') as f:
json.dump(full_task_graph, f)
......@@ -2,360 +2,8 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function
import arrow
import datetime
import json
import os
import taskcluster
from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
DEFAULT_EXPIRES_IN = '1 year'
DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
_OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix'
_DEFAULT_TASK_URL = 'https://queue.taskcluster.net/v1/task'
GOOGLE_APPLICATION_CREDENTIALS = '.firebase_token.json'
# Bug 1558456 - Stop tracking youtube-playback-test on motoG5 for >1080p cases
ARM_RAPTOR_URL_PARAMS = [
"exclude=1,2,9,10,17,18,21,22,26,28,30,32,39,40,47,"
"48,55,56,63,64,71,72,79,80,83,84,89,90,95,96",
]
class TaskBuilder(object):
def __init__(
self,
task_id,
repo_url,
git_ref,
short_head_branch, commit, owner, source, scheduler_id, date_string,
tasks_priority='lowest',
trust_level=1
):
self.task_id = task_id
self.repo_url = repo_url
self.git_ref = git_ref
self.short_head_branch = short_head_branch
self.commit = commit
self.owner = owner
self.source = source
self.scheduler_id = scheduler_id
self.trust_level = trust_level
self.tasks_priority = tasks_priority
self.date = arrow.get(date_string, 'YYYYMMDDHHmmss')
self.trust_level = trust_level
def craft_assemble_release_task(self, variant, channel, is_staging, version_name):
if is_staging:
secret_index = 'garbage/staging/project/mobile/fenix'
else:
secret_index = 'project/mobile/fenix/{}'.format(channel)
pre_gradle_commands = (
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
secret_index, key, target_file
)
for key, target_file in (
('sentry_dsn', '.sentry_token'),
('leanplum', '.leanplum_token'),
('adjust', '.adjust_token'),
('digital_asset_links', '.digital_asset_links_token'),
('firebase', 'app/src/{}/res/values/firebase.xml'.format(variant.build_type)),
)
)
capitalized_build_type = upper_case_first_letter(variant.build_type)
gradle_commands = (
'./gradlew --no-daemon -PversionName="{}" clean test assemble{}'.format(
version_name, capitalized_build_type),
)
command = ' && '.join(
cmd
for commands in (pre_gradle_commands, gradle_commands)
for cmd in commands
if cmd
)
routes = [] if is_staging else [
"notify.email.fenix-eng-notifications@mozilla.com.on-failed"
]
return self._craft_build_ish_task(
name='Build {} task'.format(capitalized_build_type),
description='Build Fenix {} from source code'.format(capitalized_build_type),
command=command,
scopes=[
"secrets:get:{}".format(secret_index)
],
artifacts=variant.artifacts(),
routes=routes,
treeherder={
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'collection': {
'opt': True,
},
'symbol': '{}-A'.format(variant.build_type),
'tier': 1,
},
attributes={
'apks': variant.upstream_artifacts_per_abi,
}
)
def craft_assemble_raptor_task(self, variant):
command = ' && '.join((
'echo "https://fake@sentry.prod.mozaws.net/368" > .sentry_token',
'echo "--" > .adjust_token',
'echo "-:-" > .leanplum_token',
'touch .digital_asset_links_token',
'./gradlew --no-daemon clean assemble{}'.format(variant.name),
))
return self._craft_build_ish_task(
name='assemble: {}'.format(variant.name),
description='Building and testing variant {}'.format(variant.name),
command=command,
artifacts=variant.artifacts(),
treeherder={
'groupSymbol': variant.build_type,
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'symbol': 'A',
'tier': 1,
},
attributes={
'build-type': 'raptor',
'apks': variant.upstream_artifacts_per_abi,
},
)
def craft_assemble_pr_task(self, variant):
return self._craft_clean_gradle_task(
name='assemble: {}'.format(variant.name),
description='Building and testing variant {}'.format(variant.name),
gradle_task='assemble{}'.format(variant.name),
artifacts=variant.artifacts(),
treeherder={
'groupSymbol': variant.build_type,
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'symbol': 'A',
'tier': 1,
},
)
def craft_test_pr_task(self, variant):
# upload coverage only once, if the variant is arm64
secret_index = 'project/mobile/fenix/public-tokens'
pre_gradle_commands = (
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
secret_index, key, target_file
)
for key, target_file in (
('codecov', '.cc_token'),
)
)
gradle_commands = (
'./gradlew --no-daemon clean -Pcoverage jacocoGeckoNightlyDebugTestReport',
)
post_gradle_commands = (
'automation/taskcluster/upload_coverage_report.sh',
)
command = ' && '.join(
cmd
for commands in (pre_gradle_commands, gradle_commands, post_gradle_commands)
for cmd in commands
if cmd
)
return self._craft_build_ish_task(
name='test: {}'.format(variant.name),
description='Building and testing variant {}'.format(variant.name),
command=command,
treeherder={
'groupSymbol': variant.build_type,
'jobKind': 'test',
'machine': {
'platform': 'android-all',
},
'symbol': 'T',
'tier': 1,
},
scopes=[
'secrets:get:{}'.format(secret_index)
]
)
def _craft_clean_gradle_task(
self, name, description, gradle_task, artifacts=None, routes=None, treeherder=None, scopes=None
):
return self._craft_build_ish_task(
name=name,
description=description,
command='./gradlew --no-daemon clean {}'.format(gradle_task),
artifacts=artifacts,
routes=routes,
treeherder=treeherder,
scopes=scopes,
)
def _craft_build_ish_task(
self, name, description, command, dependencies=None, artifacts=None,
routes=None, treeherder=None, env_vars=None, scopes=None, attributes=None
):
artifacts = {} if artifacts is None else artifacts
scopes = [] if scopes is None else scopes
env_vars = {} if env_vars is None else env_vars
checkout_command = ' && '.join([
"export TERM=dumb",
"git fetch {} {}".format(self.repo_url, self.git_ref),
"git config advice.detachedHead false",
"git checkout FETCH_HEAD",
])
command = '{} && {}'.format(checkout_command, command)
features = {}
if artifacts:
features['chainOfTrust'] = True
if any(scope.startswith('secrets:') for scope in scopes):
features['taskclusterProxy'] = True
payload = {
"features": features,
"env": env_vars,
"maxRunTime": 7200,
"image": "mozillamobile/fenix:1.4",
"command": [
"/bin/bash",
"--login",
"-cx",
# Some tasks like nimbledroid do have tasks references
{'task-reference': command},
],
"artifacts": artifacts,
}
return self._craft_default_task_definition(
worker_type='mobile-{}-b-fenix'.format(self.trust_level),
provisioner_id='aws-provisioner-v1',
name=name,
description=description,
payload=payload,
dependencies=dependencies,
routes=routes,
scopes=scopes,
treeherder=treeherder,
attributes=attributes,
)
def _craft_default_task_definition(
self,
worker_type,
provisioner_id,
name,
description,
payload,
dependencies=None,
routes=None,
scopes=None,
treeherder=None,
notify=None,
attributes=None
):
dependencies = {} if dependencies is None else dependencies
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
treeherder = {} if treeherder is None else treeherder
attributes = {} if attributes is None else attributes
created = datetime.datetime.now()
deadline = taskcluster.fromNow('1 day')
expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN)
routes.append('checks')
if self.trust_level == 3:
routes.append('tc-treeherder.v2.fenix.{}'.format(self.commit))
extra = {
"treeherder": treeherder,
}
if notify:
extra['notify'] = notify
return {
"attributes": attributes,
"dependencies": dependencies,
"label": name,
"task": {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"requires": "all-completed",
"routes": routes,
"scopes": scopes,
"payload": payload,
"extra": extra,
"metadata": {
"name": "Fenix - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
},
},
}
def schedule_task(queue, taskId, task):
print("TASK", taskId)
print(json.dumps(task, indent=4, separators=(',', ': ')))
result = queue.createTask(taskId, task)
print("RESULT", taskId)
print(json.dumps(result))
def schedule_task_graph(ordered_groups_of_tasks):
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
full_task_graph = {}
# TODO: Switch to async python to speed up submission
for group_of_tasks in ordered_groups_of_tasks:
for task_id, task_definition in group_of_tasks.items():
schedule_task(queue, task_id, task_definition)
full_task_graph[task_id] = {
# Some values of the task definition are automatically filled. Querying the task
# allows to have the full definition. This is needed to make Chain of Trust happy
'task': queue.task(task_id),
}
return full_task_graph
def fetch_mozharness_task_id():
# We now want to use the latest available raptor
raptor_index = 'gecko.v2.mozilla-central.nightly.latest.mobile.android-x86_64-opt'
return taskcluster.Index({
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
}).findTask(raptor_index)['taskId']
import re
def convert_camel_case_into_kebab_case(string):
# Inspired from https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case # noqa: E501
first_pass = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', string)
return re.sub('([a-z0-9])([A-Z])', r'\1-\2', first_pass).lower()
def lower_case_first_letter(string):
return '{}{}'.format(string[0].lower(), string[1:])
def upper_case_first_letter(string):
return string[0].upper() + string[1:]
import taskcluster
class VariantApk:
def __init__(self, build_type, abi, engine, file_name):
self.abi = abi
self.taskcluster_path = u'public/build/{}/{}/target.apk'.format(abi, engine)
self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name)
class Variant:
def __init__(self, name, build_type, apks):
self.name = name
self.build_type = build_type
self._apks = apks
def get_apk(self, abi):
return [apk for apk in self._apks if apk.abi == abi][0]
def artifacts(self):
return {
apk.taskcluster_path: {
'type': 'file',
'path': apk.absolute_path,
'expires': taskcluster.stringDate(taskcluster.fromNow('1 year')),
} for apk in self._apks
}
def upstream_artifacts(self):
return [apk.taskcluster_path for apk in self._apks]
@property
def upstream_artifacts_per_abi(self):
return {apk.abi: apk.taskcluster_path for apk in self._apks}
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import argparse
import datetime
import jsone
import os
import slugid
import taskcluster
import yaml
from git import Repo
from lib.tasks import schedule_task
ROOT = os.path.join(os.path.dirname(__file__), '../..')
class InvalidGithubRepositoryError(Exception):
pass
def calculate_git_references(root):
repo = Repo(root)
remote = repo.remote()
branch = repo.head.reference
if not remote.url.startswith('https://github.com'):
raise InvalidGithubRepositoryError('expected remote to be a GitHub repository (accessed via HTTPs)')
html_url = remote.url[:-4] if remote.url.endswith('.git') else remote.url
return html_url, str(branch), str(branch.commit)
def make_decision_task(params):
"""Generate a basic decision task, based on the root .taskcluster.yml"""
with open(os.path.join(ROOT, '.taskcluster.yml'), 'rb') as f:
taskcluster_yml = yaml.safe_load(f)
slugids = {}
def as_slugid(name):
if name not in slugids:
slugids[name] = slugid.nice()
return slugids[name]
repository_parts = params['html_url'].split('/')
repository_full_name = '/'.join((repository_parts[-2], repository_parts[-1]))
# provide a similar JSON-e context to what taskcluster-github provides
context = {
'tasks_for': 'cron',
'cron': {
'task_id': params['cron_task_id'],
'name': params['name'],
},
'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z',