Unverified Commit 714c2c84 authored by Johan Lorenzo's avatar Johan Lorenzo Committed by GitHub
Browse files

Bug 1625126 - Cache external dependencies in a single task and let gradle tasks use it (#10316)

parent 7cc48166
......@@ -4,6 +4,7 @@ buildscript {
// This logic is duplicated in the allprojects block: I don't know how to fix that.
repositories {
maven {
name "Mozilla Nightly"
url "https://nightly.maven.mozilla.org/maven2"
content {
// Improve performance: only check moz maven for mozilla deps.
......@@ -11,29 +12,45 @@ buildscript {
}
}
maven {
name "Mozilla"
url "https://maven.mozilla.org/maven2"
content {
// Improve performance: only check moz maven for mozilla deps.
includeGroupByRegex RepoMatching.mozilla
}
}
google() {
content {
// Improve performance: only check google maven for mozilla deps.
includeGroupByRegex RepoMatching.androidx
includeGroupByRegex RepoMatching.comGoogleAndroid
includeGroupByRegex RepoMatching.comGoogleFirebase
includeGroupByRegex RepoMatching.comAndroid
if (project.hasProperty("googleRepo")) {
maven {
name "Google"
url project.property("googleRepo")
}
} else {
google() {
content {
// Improve performance: only check google maven for mozilla deps.
includeGroupByRegex RepoMatching.androidx
includeGroupByRegex RepoMatching.comGoogleAndroid
includeGroupByRegex RepoMatching.comGoogleFirebase
includeGroupByRegex RepoMatching.comAndroid
}
}
}
jcenter() {
content {
// Improve security: don't search deps with known repos.
excludeGroupByRegex RepoMatching.mozilla
excludeGroupByRegex RepoMatching.androidx
excludeGroupByRegex RepoMatching.comGoogleAndroid
excludeGroupByRegex RepoMatching.comGoogleFirebase
excludeGroupByRegex RepoMatching.comAndroid
if (project.hasProperty("jcenterRepo")) {
maven {
name "BintrayJCenter"
url project.property("jcenterRepo")
}
} else {
jcenter() {
content {
// Improve security: don't search deps with known repos.
excludeGroupByRegex RepoMatching.mozilla
excludeGroupByRegex RepoMatching.androidx
excludeGroupByRegex RepoMatching.comGoogleAndroid
excludeGroupByRegex RepoMatching.comGoogleFirebase
excludeGroupByRegex RepoMatching.comAndroid
}
}
}
}
......@@ -60,6 +77,7 @@ allprojects {
// This logic is duplicated in the buildscript block: I don't know how to fix that.
repositories {
maven {
name "Mozilla Nightly"
url "https://nightly.maven.mozilla.org/maven2"
content {
// Improve performance: only check moz maven for mozilla deps.
......@@ -67,29 +85,44 @@ allprojects {
}
}
maven {
name "Mozilla"
url "https://maven.mozilla.org/maven2"
content {
// Improve performance: only check moz maven for mozilla deps.
includeGroupByRegex RepoMatching.mozilla
}
}
google() {
content {
// Improve performance: only check google maven for google deps.
includeGroupByRegex RepoMatching.androidx
includeGroupByRegex RepoMatching.comGoogleAndroid
includeGroupByRegex RepoMatching.comGoogleFirebase
includeGroupByRegex RepoMatching.comAndroid
if (project.hasProperty("googleRepo")) {
maven {
name "Google"
url project.property("googleRepo")
}
} else {
google() {
content {
// Improve performance: only check google maven for mozilla deps.
includeGroupByRegex RepoMatching.androidx
includeGroupByRegex RepoMatching.comGoogleAndroid
includeGroupByRegex RepoMatching.comGoogleFirebase
includeGroupByRegex RepoMatching.comAndroid
}
}
}
jcenter() {
content {
// Improve security: don't search deps with known repos.
excludeGroupByRegex RepoMatching.mozilla
excludeGroupByRegex RepoMatching.androidx
excludeGroupByRegex RepoMatching.comGoogleAndroid
excludeGroupByRegex RepoMatching.comGoogleFirebase
excludeGroupByRegex RepoMatching.comAndroid
if (project.hasProperty("jcenterRepo")) {
maven {
name "BintrayJCenter"
url project.property("jcenterRepo")
}
} else {
jcenter() {
content {
// Improve security: don't search deps with known repos.
excludeGroupByRegex RepoMatching.mozilla
excludeGroupByRegex RepoMatching.androidx
excludeGroupByRegex RepoMatching.comGoogleAndroid
excludeGroupByRegex RepoMatching.comGoogleFirebase
excludeGroupByRegex RepoMatching.comAndroid
}
}
}
}
......@@ -144,3 +177,10 @@ tasks.withType(io.gitlab.arturbosch.detekt.Detekt.class) {
exclude("**/test/**")
exclude("**/tmp/**")
}
task listRepositories {
doLast {
println "Repositories:"
project.repositories.each { println "Name: " + it.name + "; url: " + it.url }
}
}
......@@ -26,6 +26,7 @@ job-defaults:
fetches:
toolchain:
- android-sdk-linux
- android-gradle-dependencies
run:
using: gradlew
use-caches: false
......
......@@ -34,7 +34,12 @@ workers:
provisioner: 'mobile-{level}'
implementation: docker-worker
os: linux
worker-type: 'b-linux'
worker-type: b-linux
b-android-large:
provisioner: 'mobile-{level}'
implementation: docker-worker
os: linux
worker-type: b-linux-large
dep-signing:
provisioner: scriptworker-k8s
implementation: scriptworker-signing
......
......@@ -11,6 +11,9 @@ transforms:
- taskgraph.transforms.task:transforms
jobs:
android-build:
symbol: I(agb)
parent: base
base:
symbol: I(base)
bump:
......
......@@ -20,6 +20,7 @@ job-defaults:
fetches:
toolchain:
- android-sdk-linux
- android-gradle-dependencies
run:
use-caches: false
run-on-tasks-for: [github-pull-request, github-push]
......
......@@ -20,6 +20,7 @@ job-defaults:
fetches:
toolchain:
- android-sdk-linux
- android-gradle-dependencies
include-pull-request-number: false
run:
using: gradlew
......
......@@ -9,10 +9,6 @@ job-defaults:
kind: build
platform: toolchains/opt
tier: 1
worker-type: b-android
worker:
docker-image: {in-tree: base}
max-run-time: 1800
linux64-android-sdk-linux-repack:
......@@ -29,3 +25,35 @@ linux64-android-sdk-linux-repack:
toolchain-alias: android-sdk-linux
treeherder:
symbol: TL(android-sdk-linux)
worker:
docker-image: {in-tree: base}
max-run-time: 1800
worker-type: b-android
linux64-android-gradle-dependencies:
description: "Android Gradle dependencies toolchain task"
fetches:
toolchain:
# Aliases aren't allowed for toolchains depending on toolchains.
- linux64-android-sdk-linux-repack
run:
script: android-gradle-dependencies.sh
sparse-profile: null
resources:
- taskcluster/scripts/toolchain/android-gradle-dependencies.sh
- taskcluster/scripts/toolchain/android-gradle-dependencies/**
- buildSrc/src/main/java/Dependencies.kt
# We don't follow 'buildSrc/src/main/java/AndroidComponents.kt' because we only cache
# dependencies outside of the ones hosted on maven.mozilla.org.
toolchain-artifact: public/build/android-gradle-dependencies.tar.xz
toolchain-alias: android-gradle-dependencies
treeherder:
symbol: TL(gradle-dependencies)
worker:
docker-image: {in-tree: android-build}
env:
# TODO do no hardcode
ANDROID_SDK_ROOT: /builds/worker/fetches/android-sdk-linux
max-run-time: 14400
worker-type: b-android-large
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# %ARG DOCKER_IMAGE_PARENT
FROM $DOCKER_IMAGE_PARENT
MAINTAINER Johan Lorenzo <jlorenzo+tc@mozilla.com>
VOLUME /builds/worker/checkouts
# Install Sonatype Nexus. Cribbed directly from
# https://github.com/sonatype/docker-nexus/blob/fffd2c61b2368292040910c055cf690c8e76a272/oss/Dockerfile.
ENV NEXUS_ARCHIVE='nexus-bundle.tar.gz' \
NEXUS_ROOT='/opt/sonatype/nexus' \
NEXUS_SHA1SUM=1a9aaad8414baffe0a2fd46eed1f41b85f4049e6 \
NEXUS_VERSION=2.12.0-01 \
NEXUS_WORK=/builds/worker/workspace/nexus
RUN mkdir -p "$NEXUS_ROOT" \
&& chown -R worker:worker "$NEXUS_ROOT"
USER worker:worker
RUN $CURL --output "$NEXUS_ARCHIVE" "https://download.sonatype.com/nexus/oss/nexus-${NEXUS_VERSION}-bundle.tar.gz" \
&& echo "$NEXUS_SHA1SUM $NEXUS_ARCHIVE" | sha1sum --check \
&& tar xzvf "$NEXUS_ARCHIVE" --strip-components=1 --directory="$NEXUS_ROOT" \
&& rm "$NEXUS_ARCHIVE"
# run-task expects to run as root
USER root
......@@ -19,7 +19,8 @@ WORKDIR /builds/worker/
#-- Configuration -----------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------------------
ENV GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
ENV CURL='curl --location --retry 5' \
GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
LANG='en_US.UTF-8' \
TERM='dumb'
......
......@@ -5,8 +5,7 @@ MAINTAINER Johan Lorenzo <jlorenzo+tc@mozilla.com>
USER worker:worker
ENV CURL='curl --location --retry 5' \
HUB_ARCHIVE='hub.tgz' \
ENV HUB_ARCHIVE='hub.tgz' \
HUB_ROOT='/builds/worker/hub' \
HUB_SHA256='734733c9d807715a4ec26ccce0f9987bd19f1c3f84dd35e56451711766930ef0' \
HUB_VERSION='2.14.1'
......
......@@ -72,19 +72,20 @@ def configure_gradlew(config, job, taskdesc):
run = job["run"]
worker = taskdesc["worker"] = job["worker"]
fetches_dir = path.join(run["workdir"], worker["env"]["MOZ_FETCHES_DIR"])
worker.setdefault("env", {}).update({
"ANDROID_SDK_ROOT": path.join(
run["workdir"], worker["env"]["MOZ_FETCHES_DIR"], "android-sdk-linux"
fetches_dir, "android-sdk-linux"
)
})
run["command"] = _extract_gradlew_command(run)
run["command"] = _extract_gradlew_command(run, fetches_dir)
_inject_secrets_scopes(run, taskdesc)
_set_run_task_attributes(job)
configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
def _extract_gradlew_command(run):
def _extract_gradlew_command(run, fetches_dir):
pre_gradle_commands = run.pop("pre-gradlew", [])
pre_gradle_commands += [
_generate_dummy_secret_command(secret) for secret in run.pop("dummy-secrets", [])
......@@ -93,7 +94,14 @@ def _extract_gradlew_command(run):
_generate_secret_command(secret) for secret in run.get("secrets", [])
]
gradle_command = ["./gradlew"] + run.pop("gradlew")
maven_dependencies_dir = path.join(fetches_dir, "android-gradle-dependencies")
gradle_repos_args = [
"-P{repo_name}Repo=file://{dir}/{repo_name}".format(
dir=maven_dependencies_dir, repo_name=repo_name
)
for repo_name in ("google", "jcenter")
]
gradle_command = ["./gradlew"] + gradle_repos_args + ["listRepositories"] + run.pop("gradlew")
post_gradle_commands = run.pop("post-gradlew", [])
commands = pre_gradle_commands + [gradle_command] + post_gradle_commands
......
#!/bin/bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
set -ex
function get_abs_path {
local file_path="$1"
echo "$( cd "$(dirname "$file_path")" >/dev/null 2>&1 ; pwd -P )"
}
CURRENT_DIR="$(get_abs_path $0)"
PROJECT_DIR="$(get_abs_path $CURRENT_DIR/../../../..)"
pushd $PROJECT_DIR
. taskcluster/scripts/toolchain/android-gradle-dependencies/before.sh
NEXUS_PREFIX='http://localhost:8081/nexus/content/repositories'
GRADLE_ARGS="--parallel -PgoogleRepo=$NEXUS_PREFIX/google/ -PjcenterRepo=$NEXUS_PREFIX/jcenter/"
# We build everything to be sure to fetch all dependencies
./gradlew $GRADLE_ARGS assemble assembleAndroidTest ktlint detekt
# Some tests may be flaky, although they still download dependencies. So we let the following
# command fail, if needed.
set +e; ./gradlew $GRADLE_ARGS -Pcoverage test; set -e
# ./gradlew lint is missing because of https://github.com/mozilla-mobile/fenix/issues/10439. So far,
# we're lucky and the dependencies it fetches are found elsewhere.
. taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh
popd
#!/bin/bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This is copy of
# https://searchfox.org/mozilla-central/rev/2cd2d511c0d94a34fb7fa3b746f54170ee759e35/taskcluster/scripts/misc/android-gradle-dependencies/after.sh.
# gradle-plugins was removed because it's not used in this project.
set -x -e
echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
set -v
# Package everything up.
pushd $WORKSPACE
mkdir -p android-gradle-dependencies /builds/worker/artifacts
cp -R ${NEXUS_WORK}/storage/jcenter android-gradle-dependencies
cp -R ${NEXUS_WORK}/storage/google android-gradle-dependencies
tar cf - android-gradle-dependencies | xz > /builds/worker/artifacts/android-gradle-dependencies.tar.xz
popd
#!/bin/bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This is a copy of
# https://searchfox.org/mozilla-central/rev/2cd2d511c0d94a34fb7fa3b746f54170ee759e35/taskcluster/scripts/misc/android-gradle-dependencies/before.sh.
# `misc` was renamed into `toolchain` and `/builds/worker/workspace/build` was changed into
# `/builds/worker/checkouts/`
set -x -e
echo "running as" $(id)
: WORKSPACE ${WORKSPACE:=/builds/worker/workspace}
set -v
mkdir -p ${NEXUS_WORK}/conf
cp /builds/worker/checkouts/src/taskcluster/scripts/toolchain/android-gradle-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml
RUN_AS_USER=worker /opt/sonatype/nexus/bin/nexus restart
# Wait "a while" for Nexus to actually start. Don't fail if this fails.
wget --quiet --retry-connrefused --waitretry=2 --tries=100 \
http://localhost:8081/nexus/service/local/status || true
rm -rf status
# It's helpful when debugging to see the "latest state".
curl http://localhost:8081/nexus/service/local/status || true
# Verify Nexus has actually started. Fail if this fails.
curl --fail --silent --location http://localhost:8081/nexus/service/local/status | grep '<state>STARTED</state>'
# It's helpful when debugging to see the repository configurations.
curl http://localhost:8081/nexus/service/local/repositories || true
<?xml version="1.0" encoding="UTF-8"?>
<!-- This Source Code Form is subject to the terms of the Mozilla Public
- License, v. 2.0. If a copy of the MPL was not distributed with this
- file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
<nexusConfiguration>
<version>2.8.0</version>
<nexusVersion>2.12.0-01</nexusVersion>
<globalConnectionSettings>
<connectionTimeout>20000</connectionTimeout>
<retrievalRetryCount>3</retrievalRetryCount>
<queryString></queryString>
</globalConnectionSettings>
<restApi>
<uiTimeout>60000</uiTimeout>
</restApi>
<httpProxy>
<enabled>true</enabled>
<port>8082</port>
<proxyPolicy>strict</proxyPolicy>
</httpProxy>
<routing>
<resolveLinks>true</resolveLinks>
</routing>
<repositories>
<repository>
<id>jcenter</id>
<name>jcenter</name>
<providerRole>org.sonatype.nexus.proxy.repository.Repository</providerRole>
<providerHint>maven2</providerHint>
<localStatus>IN_SERVICE</localStatus>
<notFoundCacheActive>true</notFoundCacheActive>
<notFoundCacheTTL>1440</notFoundCacheTTL>
<userManaged>true</userManaged>
<exposed>true</exposed>
<browseable>true</browseable>
<writePolicy>READ_ONLY</writePolicy>
<indexable>true</indexable>
<searchable>true</searchable>
<localStorage>
<provider>file</provider>
</localStorage>
<remoteStorage>
<url>https://jcenter.bintray.com/</url>
</remoteStorage>
<externalConfiguration>
<repositoryPolicy>RELEASE</repositoryPolicy>
<checksumPolicy>STRICT</checksumPolicy>
<fileTypeValidation>true</fileTypeValidation>
<downloadRemoteIndex>false</downloadRemoteIndex>
<artifactMaxAge>-1</artifactMaxAge>
<metadataMaxAge>1440</metadataMaxAge>
<itemMaxAge>1440</itemMaxAge>
<autoBlockActive>true</autoBlockActive>
</externalConfiguration>
</repository>
<repository>
<id>gradle-plugins</id>
<name>Gradle Plugins</name>
<providerRole>org.sonatype.nexus.proxy.repository.Repository</providerRole>
<providerHint>maven2</providerHint>
<localStatus>IN_SERVICE</localStatus>
<notFoundCacheActive>true</notFoundCacheActive>
<notFoundCacheTTL>1440</notFoundCacheTTL>
<userManaged>true</userManaged>
<exposed>true</exposed>
<browseable>true</browseable>
<writePolicy>READ_ONLY</writePolicy>
<indexable>true</indexable>
<searchable>true</searchable>
<localStorage>
<provider>file</provider>
</localStorage>
<remoteStorage>
<url>https://plugins.gradle.org/m2/</url>
</remoteStorage>
<externalConfiguration>
<repositoryPolicy>RELEASE</repositoryPolicy>
<checksumPolicy>STRICT</checksumPolicy>
<fileTypeValidation>true</fileTypeValidation>
<downloadRemoteIndex>false</downloadRemoteIndex>
<artifactMaxAge>-1</artifactMaxAge>
<metadataMaxAge>1440</metadataMaxAge>
<itemMaxAge>1440</itemMaxAge>
<autoBlockActive>true</autoBlockActive>
</externalConfiguration>
</repository>
<repository>
<id>google</id>
<name>google</name>
<providerRole>org.sonatype.nexus.proxy.repository.Repository</providerRole>
<providerHint>maven2</providerHint>
<localStatus>IN_SERVICE</localStatus>
<notFoundCacheActive>true</notFoundCacheActive>
<notFoundCacheTTL>1440</notFoundCacheTTL>
<userManaged>true</userManaged>
<exposed>true</exposed>
<browseable>true</browseable>
<writePolicy>READ_ONLY</writePolicy>
<indexable>true</indexable>
<searchable>true</searchable>
<localStorage>
<provider>file</provider>
</localStorage>
<remoteStorage>
<url>https://maven.google.com/</url>
</remoteStorage>
<externalConfiguration>
<repositoryPolicy>RELEASE</repositoryPolicy>
<!-- Google doesn't publish checksums. Why, Google, why? -->
<checksumPolicy>STRICT_IF_EXISTS</checksumPolicy>
<fileTypeValidation>true</fileTypeValidation>
<downloadRemoteIndex>false</downloadRemoteIndex>
<artifactMaxAge>-1</artifactMaxAge>
<metadataMaxAge>1440</metadataMaxAge>
<itemMaxAge>1440</itemMaxAge>
<autoBlockActive>true</autoBlockActive>
</externalConfiguration>
</repository>
<repository>
<id>central</id>
<name>Central</name>
<providerRole>org.sonatype.nexus.proxy.repository.Repository</providerRole>
<providerHint>maven2</providerHint>
<localStatus>IN_SERVICE</localStatus>
<notFoundCacheActive>true</notFoundCacheActive>
<notFoundCacheTTL>1440</notFoundCacheTTL>
<userManaged>true</userManaged>
<exposed>true</exposed>
<browseable>true</browseable>
<writePolicy>READ_ONLY</writePolicy>
<indexable>true</indexable>
<searchable>true</searchable>
<localStorage>
<provider>file</provider>
</localStorage>
<remoteStorage>
<url>https://repo1.maven.org/maven2/</url>
</remoteStorage>
<externalConfiguration>
<proxyMode>ALLOW</proxyMode>
<artifactMaxAge>-1</artifactMaxAge>
<itemMaxAge>1440</itemMaxAge>
<cleanseRepositoryMetadata>false</cleanseRepositoryMetadata>
<downloadRemoteIndex>false</downloadRemoteIndex>
<checksumPolicy>WARN</checksumPolicy>
<repositoryPolicy>RELEASE</repositoryPolicy>
</externalConfiguration>
</repository>
<repository>
<id>apache-snapshots</id>
<name>Apache Snapshots</name>
<providerRole>org.sonatype.nexus.proxy.repository.Repository</providerRole>
<providerHint>maven2</providerHint>
<localStatus>IN_SERVICE</localStatus>
<notFoundCacheActive>true</notFoundCacheActive>
<notFoundCacheTTL>1440</notFoundCacheTTL>
<userManaged>true</userManaged>
<exposed>true</exposed>
<browseable>true</browseable>
<writePolicy>READ_ONLY</writePolicy>
<indexable>true</indexable>
<searchable>true</searchable>
<localStorage>