Unverified Commit 44432d28 authored by Johan Lorenzo's avatar Johan Lorenzo Committed by GitHub
Browse files

Bug 1622339 - Move android-sdk installation into a toolchain task (#9394)

parent 24596fd3
...@@ -8,7 +8,7 @@ tasks: ...@@ -8,7 +8,7 @@ tasks:
- $let: - $let:
taskgraph: taskgraph:
branch: taskgraph branch: taskgraph
revision: a9a010487bf10fca6dc34ba53f40d34a8ab51fc0 revision: f949fb56d96b9509b071ba55853f2d60e8dbbf57
trustDomain: mobile trustDomain: mobile
in: in:
$let: $let:
...@@ -255,13 +255,13 @@ tasks: ...@@ -255,13 +255,13 @@ tasks:
$if: 'tasks_for == "action"' $if: 'tasks_for == "action"'
then: > then: >
PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
taskcluster/scripts/install-sdk.sh && taskcluster/scripts/decision-install-sdk.sh &&
ln -s /builds/worker/artifacts artifacts && ln -s /builds/worker/artifacts artifacts &&
~/.local/bin/taskgraph action-callback ~/.local/bin/taskgraph action-callback
else: > else: >
PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml && PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
taskcluster/scripts/install-sdk.sh && taskcluster/scripts/decision-install-sdk.sh &&
ln -s /builds/worker/artifacts artifacts && ln -s /builds/worker/artifacts artifacts &&
~/.local/bin/taskgraph decision ~/.local/bin/taskgraph decision
--pushlog-id='0' --pushlog-id='0'
......
...@@ -9,8 +9,27 @@ transforms: ...@@ -9,8 +9,27 @@ transforms:
- taskgraph.transforms.job:transforms - taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms - taskgraph.transforms.task:transforms
kind-dependencies:
- toolchain
job-defaults: job-defaults:
# Builds generate multiple APKs with different ABIs. For each APK described
# by `gradlew printVariant`, an artifact will be generated. `variant` and
# the per-apk config from `printVariant` can be used as substitutions in
# `name` and `path`.
apk-artifact-template:
type: file
name: public/build/{abi}/{geckoview_engine}/target.apk
path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
description: Build Fenix from source code. description: Build Fenix from source code.
fetches:
toolchain:
- android-sdk-linux
run:
using: gradlew
use-caches: false
run-on-tasks-for: []
treeherder: treeherder:
kind: build kind: build
symbol: B symbol: B
...@@ -21,18 +40,7 @@ job-defaults: ...@@ -21,18 +40,7 @@ job-defaults:
docker-image: {in-tree: base} docker-image: {in-tree: base}
max-run-time: 7200 max-run-time: 7200
chain-of-trust: true chain-of-trust: true
run:
using: gradlew
use-caches: false
run-on-tasks-for: []
# Builds generate multiple APKs with different ABIs. For each APK described
# by `gradlew printVariant`, an artifact will be generated. `variant` and
# the per-apk config from `printVariant` can be used as substitutions in
# `name` and `path`.
apk-artifact-template:
type: file
name: public/build/{abi}/{geckoview_engine}/target.apk
path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
jobs: jobs:
debug: debug:
......
...@@ -4,8 +4,10 @@ treeherder: ...@@ -4,8 +4,10 @@ treeherder:
group-names: group-names:
'beta': 'Beta-related tasks' 'beta': 'Beta-related tasks'
'betaFennec': 'Beta-related tasks with same APK configuration as Fennec' 'betaFennec': 'Beta-related tasks with same APK configuration as Fennec'
'Btime': 'Raptor-Browsertime tests'
'bump': 'Bump dependencies' 'bump': 'Bump dependencies'
'debug': 'Builds made for testing' 'debug': 'Builds made for testing'
'Fetch': 'Fetch and store content'
'forPerformanceTest': 'Builds made for Raptor and other performance tests' 'forPerformanceTest': 'Builds made for Raptor and other performance tests'
'I': 'Docker Image Builds' 'I': 'Docker Image Builds'
'nightly': 'Nightly-related tasks' 'nightly': 'Nightly-related tasks'
...@@ -14,7 +16,7 @@ treeherder: ...@@ -14,7 +16,7 @@ treeherder:
'productionFennec': 'Production-related tasks with same APK configuration as Fennec' 'productionFennec': 'Production-related tasks with same APK configuration as Fennec'
'Rap': 'Raptor tests' 'Rap': 'Raptor tests'
'Rap-P': 'Raptor power tests' 'Rap-P': 'Raptor power tests'
'Btime': 'Raptor-Browsertime tests' 'TL': 'Toolchain builds for Linux 64-bits'
task-priority: highest task-priority: highest
......
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
transforms:
- taskgraph.transforms.fetch:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
docker-image: {in-tree: base}
jobs:
android-sdk-3859397:
description: Android SDK
fetch:
type: static-url
url: https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip
artifact-name: sdk-tools-linux.zip
sha256: 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0
size: 136964098
artifact-prefix: mobile/android-sdk
fetch-alias: android-sdk
...@@ -9,14 +9,17 @@ transforms: ...@@ -9,14 +9,17 @@ transforms:
- taskgraph.transforms.job:transforms - taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms - taskgraph.transforms.task:transforms
kind-dependencies:
- toolchain
job-defaults: job-defaults:
attributes: attributes:
code-review: true code-review: true
retrigger: true retrigger: true
worker-type: b-android fetches:
worker: toolchain:
docker-image: {in-tree: base} - android-sdk-linux
max-run-time: 7200
run: run:
use-caches: false use-caches: false
run-on-tasks-for: [github-pull-request, github-push] run-on-tasks-for: [github-pull-request, github-push]
...@@ -24,6 +27,10 @@ job-defaults: ...@@ -24,6 +27,10 @@ job-defaults:
kind: test kind: test
platform: 'lint/opt' platform: 'lint/opt'
tier: 1 tier: 1
worker-type: b-android
worker:
docker-image: {in-tree: base}
max-run-time: 7200
jobs: jobs:
compare-locales: compare-locales:
......
...@@ -9,11 +9,18 @@ transforms: ...@@ -9,11 +9,18 @@ transforms:
- taskgraph.transforms.job:transforms - taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms - taskgraph.transforms.task:transforms
kind-dependencies:
- toolchain
job-defaults: job-defaults:
description: Test Fenix
include-pull-request-number: false
attributes: attributes:
retrigger: true retrigger: true
description: Test Fenix
fetches:
toolchain:
- android-sdk-linux
include-pull-request-number: false
run: run:
using: gradlew using: gradlew
use-caches: false use-caches: false
...@@ -31,10 +38,6 @@ jobs: ...@@ -31,10 +38,6 @@ jobs:
build-type: debug build-type: debug
geckoview-engine: geckoNightly geckoview-engine: geckoNightly
code-review: true code-review: true
treeherder:
platform: 'android-all/opt'
symbol: debug(T)
tier: 1
run: run:
gradlew: ['clean', '-Pcoverage', 'jacocoGeckoNightlyDebugTestReport'] gradlew: ['clean', '-Pcoverage', 'jacocoGeckoNightlyDebugTestReport']
post-gradlew: post-gradlew:
...@@ -43,6 +46,10 @@ jobs: ...@@ -43,6 +46,10 @@ jobs:
- name: project/mobile/fenix/public-tokens - name: project/mobile/fenix/public-tokens
key: codecov key: codecov
path: .cc_token path: .cc_token
treeherder:
platform: 'android-all/opt'
symbol: debug(T)
tier: 1
worker: worker:
artifacts: artifacts:
- name: public/reports/index.html - name: public/reports/index.html
......
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
job-defaults:
run:
using: toolchain-script
treeherder:
kind: build
platform: toolchains/opt
tier: 1
worker-type: b-android
worker:
docker-image: {in-tree: base}
max-run-time: 1800
linux64-android-sdk-linux-repack:
attributes:
artifact_prefix: mobile/android-sdk
description: "Android SDK (Linux) repack toolchain build"
fetches:
fetch:
- android-sdk
run:
script: repack-android-sdk-linux.sh
resources: []
toolchain-artifact: mobile/android-sdk/android-sdk-linux.tar.xz
toolchain-alias: android-sdk-linux
treeherder:
symbol: TL(android-sdk-linux)
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
job-defaults:
run:
using: index-search
run-on-projects: []
worker-type: always-optimized
browsertime:
attributes:
toolchain-artifact: public/build/browsertime.tar.bz2
description: "Browsertime toolchain"
run:
index-search:
- gecko.cache.level-3.toolchains.v3.browsertime.latest
linux64-ffmpeg-4.1.4:
attributes:
toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz'
description: "FFMPEG fetch"
run:
index-search:
- gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest
linux64-geckodriver:
attributes:
toolchain-artifact: public/build/geckodriver.tar.xz
description: "Geckodriver toolchain"
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest
linux64-minidump-stackwalk:
attributes:
toolchain-artifact: public/build/minidump_stackwalk.tar.xz
description: "minidump-stackwalk toolchain"
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest
linux64-node:
attributes:
toolchain-artifact: public/build/node.tar.xz
description: "Node.js toolchain"
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-node-10.latest
visual-metrics:
attributes:
toolchain-artifact: public/visualmetrics.py
description: "Browsertime visual metrics analysis script"
run:
index-search:
- gecko.cache.level-3.content.v1.visual-metrics.latest
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
--- ---
loader: taskgraph.loader.transform:loader loader: taskgraph.loader.transform:loader
kind-dependencies:
- fetch
transforms: transforms:
- taskgraph.transforms.job:transforms - taskgraph.transforms.job:transforms
- taskgraph.transforms.cached_tasks:transforms
- taskgraph.transforms.task:transforms - taskgraph.transforms.task:transforms
job-defaults:
worker-type: always-optimized
run:
using: index-search
jobs: jobs-from:
browsertime: - android.yml
description: "Browsertime toolchain" - gecko-derived.yml
attributes:
toolchain-artifact: public/build/browsertime.tar.bz2
run:
index-search:
- gecko.cache.level-3.toolchains.v3.browsertime.latest
linux64-ffmpeg-4.1.4:
description: "FFMPEG fetch"
attributes:
toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz'
run:
index-search:
- gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest
linux64-geckodriver:
description: "Geckodriver toolchain"
attributes:
toolchain-artifact: public/build/geckodriver.tar.xz
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest
linux64-minidump-stackwalk:
description: "minidump_stackwalk toolchain"
attributes:
toolchain-artifact: public/build/minidump_stackwalk.tar.xz
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest
linux64-node:
description: "Node.js toolchain"
attributes:
toolchain-artifact: public/build/node.tar.xz
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-node-10.latest
visual-metrics:
description: "Browsertime visual metrics analsyis script"
attributes:
toolchain-artifact: public/visualmetrics.py
run:
index-search:
- gecko.cache.level-3.content.v1.visual-metrics.latest
...@@ -19,14 +19,9 @@ WORKDIR /builds/worker/ ...@@ -19,14 +19,9 @@ WORKDIR /builds/worker/
#-- Configuration ----------------------------------------------------------------------------------------------------- #-- Configuration -----------------------------------------------------------------------------------------------------
#---------------------------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------------------------------------------------
ENV ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' \ ENV GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
ANDROID_SDK_VERSION='3859397' \
ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \
CURL='curl --location --retry 5' \
GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
LANG='en_US.UTF-8' \ LANG='en_US.UTF-8' \
TERM='dumb' \ TERM='dumb'
SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
#---------------------------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------------------------------------------------
#-- System ------------------------------------------------------------------------------------------------------------ #-- System ------------------------------------------------------------------------------------------------------------
...@@ -54,14 +49,6 @@ RUN pip install taskcluster ...@@ -54,14 +49,6 @@ RUN pip install taskcluster
RUN locale-gen "$LANG" RUN locale-gen "$LANG"
RUN $CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \
&& echo "$ANDROID_SDK_SHA256 $SDK_ZIP_LOCATION" | sha256sum --check \
&& unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \
&& rm "$SDK_ZIP_LOCATION" \
&& yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \
&& chown -R worker:worker "$ANDROID_SDK_ROOT"
# %include-run-task # %include-run-task
ENV SHELL=/bin/bash \ ENV SHELL=/bin/bash \
......
...@@ -5,7 +5,8 @@ MAINTAINER Johan Lorenzo <jlorenzo+tc@mozilla.com> ...@@ -5,7 +5,8 @@ MAINTAINER Johan Lorenzo <jlorenzo+tc@mozilla.com>
USER worker:worker USER worker:worker
ENV HUB_ARCHIVE='hub.tgz' \ ENV CURL='curl --location --retry 5' \
HUB_ARCHIVE='hub.tgz' \
HUB_ROOT='/builds/worker/hub' \ HUB_ROOT='/builds/worker/hub' \
HUB_SHA256='734733c9d807715a4ec26ccce0f9987bd19f1c3f84dd35e56451711766930ef0' \ HUB_SHA256='734733c9d807715a4ec26ccce0f9987bd19f1c3f84dd35e56451711766930ef0' \
HUB_VERSION='2.14.1' HUB_VERSION='2.14.1'
......
...@@ -72,9 +72,11 @@ def configure_gradlew(config, job, taskdesc): ...@@ -72,9 +72,11 @@ def configure_gradlew(config, job, taskdesc):
run = job["run"] run = job["run"]
worker = taskdesc["worker"] = job["worker"] worker = taskdesc["worker"] = job["worker"]
worker.setdefault("env", {}).update( worker.setdefault("env", {}).update({
{"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")} "ANDROID_SDK_ROOT": path.join(
) run["workdir"], worker["env"]["MOZ_FETCHES_DIR"], "android-sdk-linux"
)
})
run["command"] = _extract_gradlew_command(run) run["command"] = _extract_gradlew_command(run)
_inject_secrets_scopes(run, taskdesc) _inject_secrets_scopes(run, taskdesc)
......
#!/bin/bash
set -ex
export CURL='curl --location --retry 5'
ANDROID_SDK_VERSION='3859397'
ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
$CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
echo "$ANDROID_SDK_SHA256 $SDK_ZIP_LOCATION" | sha256sum --check
unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION"
rm "$SDK_ZIP_LOCATION"
yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses
#!/bin/bash
set +x
ANDROID_SDK_VERSION=3859397
curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip"
yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses
#!/bin/bash
export ANDROID_SDK_ROOT=$MOZ_FETCHES_DIR
yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses
# It's nice to have the build logs include the state of the world upon completion.
"${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --list
tar cf - -C "$ANDROID_SDK_ROOT" . --transform 's,^\./,android-sdk-linux/,' | xz > "$UPLOAD_DIR/android-sdk-linux.tar.xz"
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment