Merge #6449

6449: Bug 1622339 - Move android-sdk installation into a toolchain task r=tomprince a=JohanLorenzo
Co-authored-by: Johan Lorenzo <jlorenzo@mozilla.com>

.taskcluster.yml

@@ -7,7 +7,7 @@ tasks:
     - $let:
           taskgraph:
               branch: taskgraph
-              revision: a9a010487bf10fca6dc34ba53f40d34a8ab51fc0
+              revision: f949fb56d96b9509b071ba55853f2d60e8dbbf57
           trustDomain: mobile
       in:
           $let:
@@ -249,7 +249,7 @@ tasks:
                                     else: >
                                         PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
                                         PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
-                                        taskcluster/scripts/install-sdk.sh &&
+                                        taskcluster/scripts/decision-install-sdk.sh &&
                                         ln -s /builds/worker/artifacts artifacts &&
                                         ~/.local/bin/taskgraph decision
                                         --pushlog-id='0'

taskcluster/ac_taskgraph/job.py

@@ -38,9 +38,11 @@ def configure_gradlew(config, job, taskdesc):
     run = job["run"]
     worker = taskdesc["worker"] = job["worker"]
 
-    worker.setdefault("env", {}).update(
-        {"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")}
-    )
+    worker.setdefault("env", {}).update({
+        "ANDROID_SDK_ROOT": path.join(
+            run["workdir"], worker["env"]["MOZ_FETCHES_DIR"], "android-sdk-linux"
+        )
+    })
 
     # defer to the run_task implementation
     run["command"] = _extract_command(run)

taskcluster/ci/build-samples-browser/kind.yml

@@ -10,6 +10,8 @@ transforms:
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
+kind-dependencies:
+    - toolchain
 
 
 job-defaults:
@@ -18,6 +20,9 @@ job-defaults:
         code-review: true
         component: samples-browser
     description: Execute Gradle tasks for component "{component}"
+    fetches:
+        toolchain:
+            - android-sdk-linux
     include-coverage: true
     run:
         using: gradlew

taskcluster/ci/build/kind.yml

@@ -13,17 +13,15 @@ transforms:
 not-for-components:
     - samples-browser   # Their configs are too different, so they're handled in a specific kind
 
+kind-dependencies:
+    - toolchain
+
+
 job-defaults:
     artifact-template:
         type: file
         name: public/build/{artifact_file_name}
         path: '/builds/worker/checkouts/src/{component_path}/build/maven/org/mozilla/components/{component}/{version_with_snapshot}/{artifact_file_name}'
-    expose-artifacts:
-        by-build-type:
-            release: true
-            snapshot: true
-            nightly: true
-            default: false
     attributes:
         code-review:
             by-build-type:
@@ -32,6 +30,15 @@ job-defaults:
                 nightly: false
                 default: true
     description: Execute Gradle tasks for component "{component}"
+    expose-artifacts:
+        by-build-type:
+            release: true
+            snapshot: true
+            nightly: true
+            default: false
+    fetches:
+        toolchain:
+            - android-sdk-linux
     include-coverage:
         by-build-type:
             release: false

taskcluster/ci/config.yml

@@ -2,7 +2,10 @@
 trust-domain: mobile
 treeherder:
     group-names:
+        Fetch: Fetch and store content
         I: Docker Image Builds
+        TL: Toolchain builds for Linux 64-bits
+
         # Groups are generated off buildconfig.yml, except these ones:
         samples-browser-beta: samples-browser-beta
         samples-browser-release: samples-browser-release

taskcluster/ci/fetch/kind.yml

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.fetch:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    docker-image: {in-tree: base}
+
+jobs:
+    android-sdk-3859397:
+        description: Android SDK
+        fetch:
+            type: static-url
+            url: https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip
+            artifact-name: sdk-tools-linux.zip
+            sha256: 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0
+            size: 136964098
+        artifact-prefix: mobile/android-sdk
+        fetch-alias: android-sdk

taskcluster/ci/lint/kind.yml

@@ -9,13 +9,16 @@ transforms:
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
+kind-dependencies:
+    - toolchain
+
+
 job-defaults:
     attributes:
         code-review: true
-    worker-type: b-android
-    worker:
-        docker-image: {in-tree: base}
-        max-run-time: 7200
+    fetches:
+        toolchain:
+            - android-sdk-linux
     run:
         use-caches: false
     run-on-tasks-for: [github-pull-request, github-push]
@@ -23,6 +26,10 @@ job-defaults:
         kind: test
         platform: 'lint/opt'
         tier: 1
+    worker-type: b-android
+    worker:
+        docker-image: {in-tree: base}
+        max-run-time: 7200
 
 jobs:
     compare-locales:

taskcluster/ci/test/kind.yml

@@ -8,16 +8,23 @@ transforms:
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
+kind-dependencies:
+    - toolchain
+
+
 job-defaults:
     attributes:
         build-type: debug
         code-review: true
+    fetches:
+        toolchain:
+            - android-sdk-linux
     run:
         # TODO Generate APKs in a build task instead
         gradlew: ['clean', 'assembleGeckoNightly', 'assembleAndroidTest', 'assembleDebug']
         secrets:
             - name: project/mobile/android-components/firebase
-              key: firebaseToken                  
+              key: firebaseToken
               path: .firebase_token.json
               json: true
         using: gradlew
@@ -39,6 +46,7 @@ job-defaults:
             GOOGLE_PROJECT: moz-android-components-230120
         max-run-time: 7200
 
+
 jobs:
     unit-browser-engine-gecko-nightly:
         description: 'Run unit tests on device for browser component'

taskcluster/ci/toolchain/android.yml

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    run:
+        using: toolchain-script
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        tier: 1
+    worker-type: b-android
+    worker:
+        docker-image: {in-tree: base}
+        max-run-time: 1800
+
+
+linux64-android-sdk-linux-repack:
+    attributes:
+        artifact_prefix: mobile/android-sdk
+    description: "Android SDK (Linux) repack toolchain build"
+    fetches:
+        fetch:
+            - android-sdk
+    run:
+        script: repack-android-sdk-linux.sh
+        resources: []
+        toolchain-artifact: mobile/android-sdk/android-sdk-linux.tar.xz
+        toolchain-alias: android-sdk-linux
+    treeherder:
+        symbol: TL(android-sdk-linux)

taskcluster/ci/toolchain/kind.yml

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - fetch
+
+transforms:
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+
+jobs-from:
+    - android.yml

taskcluster/docker/base/Dockerfile

@@ -19,13 +19,9 @@ WORKDIR /builds/worker/
 #-- Configuration -----------------------------------------------------------------------------------------------------
 #----------------------------------------------------------------------------------------------------------------------
 
-ENV ANDROID_SDK_VERSION='3859397' \
-    ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \
-    CURL='curl --location --retry 5' \
-    GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
+ENV GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
     LANG='en_US.UTF-8' \
-    TERM='dumb' \
-    SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
+    TERM='dumb'
 
 #----------------------------------------------------------------------------------------------------------------------
 #-- System ------------------------------------------------------------------------------------------------------------
@@ -52,12 +48,6 @@ RUN pip install taskcluster
 
 RUN locale-gen "$LANG"
 
-RUN $CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \
-    && unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \
-    && rm "$SDK_ZIP_LOCATION" \
-    && yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \
-    && chown -R worker:worker "$ANDROID_SDK_ROOT"
-
 
 # %include-run-task
 

taskcluster/docker/ui-tests/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get install -y jq \
 
 USER worker:worker
 
+ENV CURL 'curl --location --retry 5'
 ENV GOOGLE_SDK_DOWNLOAD ./gcloud.tar.gz
 ENV GOOGLE_SDK_VERSION 233
 

taskcluster/scripts/decision-install-sdk.sh

+#!/bin/bash
+
+set -ex
+
+export CURL='curl --location --retry 5'
+
+ANDROID_SDK_VERSION='3859397'
+ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
+SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
+
+$CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
+echo "$ANDROID_SDK_SHA256  $SDK_ZIP_LOCATION" | sha256sum --check
+unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION"
+rm "$SDK_ZIP_LOCATION"
+yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses

taskcluster/scripts/install-sdk.sh

-#!/bin/bash
-
-set +x
-
-ANDROID_SDK_VERSION=3859397
-
-curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
-unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip"
-yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses

taskcluster/scripts/toolchain/repack-android-sdk-linux.sh

+#!/bin/bash
+
+export ANDROID_SDK_ROOT=$MOZ_FETCHES_DIR
+
+yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses
+
+# It's nice to have the build logs include the state of the world upon completion.
+"${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --list
+
+tar cf - -C "$ANDROID_SDK_ROOT" . --transform 's,^\./,android-sdk-linux/,' | xz > "$UPLOAD_DIR/android-sdk-linux.tar.xz"