Bug 34108: Add scripts to check for needed toolchain updates

Checking toolchain updates can be done with the following commands:

  make list_toolchain_updates-geckoview
  make list_toolchain_updates-application-services
  make list_toolchain_updates-android-components
  make list_toolchain_updates-fenix

Makefile

@@ -201,6 +201,18 @@ dmg2mar-alpha: submodule-update
 	tools/update-responses/download_missing_versions alpha
 	CHECK_CODESIGNATURE_EXISTS=1 MAR_SKIP_EXISTING=1 tools/update-responses/gen_incrementals alpha
 
+list_toolchain_updates-fenix: submodule-update
+	$(rbm) build fenix --step list_toolchain_updates --target nightly --target torbrowser-android-armv7
+
+list_toolchain_updates-android-components: submodule-update
+	$(rbm) build android-components --step list_toolchain_updates --target nightly --target torbrowser-android-armv7
+
+list_toolchain_updates-application-services: submodule-update
+	$(rbm) build application-services --step list_toolchain_updates --target nightly --target torbrowser-android-armv7
+
+list_toolchain_updates-geckoview: submodule-update
+	$(rbm) build geckoview --step list_toolchain_updates --target nightly --target torbrowser-android-armv7
+
 submodule-update:
 	git submodule update --init
 

projects/android-components/config

@@ -50,3 +50,23 @@ input_files:
   - URL: https://people.torproject.org/~gk/mirrors/sources/glean-parser-[% c('var/glean_parser') %].tar.bz2
     sha256sum: 19dbdd4958022a1a638e0217489ab722fe7d4f588f1978a4ae162f93e75694c2
     enable: '[% !c("var/fetch_gradle_dependencies") %]'
+
+steps:
+  list_toolchain_updates:
+    var:
+      container:
+        use_container: 0
+      get_glean_version: |
+        #!/bin/bash
+        read -d '' p << 'EOF' || true
+        if (m/const\\sval\\smozilla_glean\\s=\\s"([^"]+)"/) {
+          print $1;
+          exit;
+        }
+        EOF
+        perl -ne "$p" < buildSrc/src/main/java/Dependencies.kt
+      glean_version: '[% exec(c("var/get_glean_version")) %]'
+    input_files:
+      - name: glean
+        project: glean
+        pkg_type: src

projects/android-components/list_toolchain_updates_checks

+#!/bin/bash
+
+# compileSdkVersion
+read -d '' p << 'EOF' || true
+if (m/compileSdkVersion:\\s+(.*)$/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat .config.yml | perl -ne "$p")
+current=30
+check_update_needed compileSdkVersion "$needed" "$current"
+
+
+# targetSdkVersion
+read -d '' p << 'EOF' || true
+if (m/targetSdkVersion:\\s+(.*)$/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat .config.yml | perl -ne "$p")
+current=30
+check_update_needed targetSdkVersion "$needed" "$current"
+
+
+# minSdkVersion
+read -d '' p << 'EOF' || true
+if (m/minSdkVersion:\\s+(.*)$/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat .config.yml | perl -ne "$p")
+current=21
+check_update_needed minSdkVersion "$needed" "$current"
+
+
+# application-services
+read -d '' p << 'EOF' || true
+if (m/const\\sval\\smozilla_appservices\\s=\\s"([^"]+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat buildSrc/src/main/java/Dependencies.kt | perl -ne "$p")
+current='[% pc("application-services", "version") %]'
+check_update_needed application-services "$needed" "$current"
+
+
+# glean_parser
+read -d '' p << 'EOF' || true
+if (m/^\\s*"glean_parser==([^"]+)",/) {
+  print $1;
+  exit;
+}
+EOF
+tar xf $rootdir/[% c('input_files_by_name/glean') %]
+needed=$(cat glean-[% c("var/glean_version") %]/glean-core/python/setup.py | perl -ne "$p")
+current='[% c("var/glean_parser") %]'
+check_update_needed glean_parser "$needed" "$current"
+
+
+# gradle
+read -d '' p << 'EOF' || true
+if (m|distributionUrl=https\\\\://services.gradle.org/distributions/gradle-(.*)-all.zip|) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat gradle/wrapper/gradle-wrapper.properties | perl -ne "$p")
+current='[% c("var/gradle_version") %]'
+check_update_needed gradle "$needed" "$current"
+
+
+# android-gradle-plugin
+read -d '' p << 'EOF' || true
+if (m/const\\s+val\\s+android_gradle_plugin\\s+=\\s+"([^"]+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat buildSrc/src/main/java/Dependencies.kt | perl -ne "$p")
+current='4.0.1'
+check_update_needed android-gradle-plugin "$needed" "$current"

projects/android-toolchain/config

@@ -31,6 +31,7 @@ var:
   android_platform_revision_new: 03
   android_ndk_version: 21
   android_ndk_revision: d
+  sdk_tools_version: 4333796
 input_files:
   - project: container-image
   - URL: '[% c("var/google_repo") %]/build-tools_r[% c("version") %]-linux.zip'
@@ -54,7 +55,7 @@ input_files:
   - URL: '[% c("var/google_repo") %]/platform-tools_r[% c("version") %]-linux.zip'
     name: platform_tools
     sha256sum: 633b6dfa245f5dc58d15da9ead655bcd14de5784196ec0f0dc7e37a5acb61be4
-  - URL: '[% c("var/google_repo") %]/sdk-tools-linux-4333796.zip'
+  - URL: '[% c("var/google_repo") %]/sdk-tools-linux-[% c("var/sdk_tools_version") %].zip'
     name: android_sdk_tools
     sha256sum: 92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9
   - URL: '[% c("var/google_repo") %]/android-ndk-r[% c("var/android_ndk_version") %][% c("var/android_ndk_revision") %]-linux-x86_64.zip'

projects/application-services/config

@@ -81,3 +81,10 @@ input_files:
   - filename: viaduct-workaround.patch
   - filename: viaduct-workaround2.patch
   - filename: update-cargo-lock.patch
+
+steps:
+  list_toolchain_updates:
+    input_files: []
+    var:
+      container:
+        use_container: 0

projects/application-services/list_toolchain_updates_checks

+#!/bin/bash
+
+# ndkVersion
+read -d '' p << 'EOF' || true
+if (m/^\\s*ndkVersion:\\s"([^"]*)",/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build.gradle | perl -ne "$p")
+current='21.3.6528147'
+check_update_needed ndkVersion "$needed" "$current"
+
+
+# compileSdkVersion
+read -d '' p << 'EOF' || true
+if (m/^\\s*compileSdkVersion:\\s([^"]*),/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build.gradle | perl -ne "$p")
+current=29
+check_update_needed compileSdkVersion "$needed" "$current"
+
+
+# minSdkVersion
+read -d '' p << 'EOF' || true
+if (m/^\\s*minSdkVersion:\\s([^"]*),/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build.gradle | perl -ne "$p")
+current=21
+check_update_needed minSdkVersion "$needed" "$current"
+
+
+# gradle
+read -d '' p << 'EOF' || true
+if (m|distributionUrl=https\\\\://services.gradle.org/distributions/gradle-(.*)-.*.zip|) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat gradle/wrapper/gradle-wrapper.properties | perl -ne "$p")
+current='[% c("var/gradle_version") %]'
+check_update_needed gradle "$needed" "$current"
+
+
+# nss-nspr
+read -d '' p << 'EOF' || true
+if (m/NSS_ARCHIVE="nss-(.*-with-nspr-.*)\\.tar\\.gz"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat libs/build-all.sh | perl -ne "$p")
+current='[% pc("nss", "version") %]-with-nspr-[% pc("nss", "nspr_version") %]'
+check_update_needed nss-nspr "$needed" "$current"
+
+
+# sqlcipher
+read -d '' p << 'EOF' || true
+if (m/SQLCIPHER_VERSION="([^"]+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat libs/build-all.sh | perl -ne "$p")
+current='[% pc("sqlcipher", "version") %]'
+check_update_needed sqlcipher "$needed" "$current"
+
+
+# android-gradle-plugin
+read -d '' p << 'EOF' || true
+if (m/^\\s*ext\\.android_gradle_plugin_version\\s=\\s'([^']*)'/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build.gradle | perl -ne "$p")
+current='4.0.1'
+check_update_needed android-gradle-plugin "$needed" "$current"
+
+
+# android-components
+read -d '' p << 'EOF' || true
+if (m/^\\s*ext\\.android_components_version\\s=\\s'([^']*)'/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build.gradle | perl -ne "$p")
+current=71.0.0
+check_update_needed android-components "$needed" "$current"
+

projects/common/list_toolchain_updates

+#!/usr/bin/bash
+
+set -e
+
+rootdir="$(pwd)"
+no_updates="$rootdir/no_updates.txt"
+updates="$rootdir/updates.txt"
+
+tar -xf [% project %]-[% c('version') %].tar.gz
+cd [% project %]-[% c('version') %]
+
+function check_update_needed() {
+  name="$1"
+  v1="$2"
+  v2="$3"
+  if test "$v1" = "$v2"
+  then
+    echo "* $name ($v1)" >> "$no_updates"
+  else
+    echo "* $name needs to be updated to $v1 (currently at $v2)" >> "$updates"
+  fi
+}
+
+[% INCLUDE list_toolchain_updates_checks %]
+
+echo '### Component: [% project %] ([% c("git_hash") %])'
+
+if test -f "$updates"
+then
+  echo "The following components need to be updated:"
+  cat "$updates"
+  echo
+fi
+if test -f "$no_updates"
+then
+  echo "The following components don't need to be updated:"
+  cat "$no_updates"
+fi

projects/fenix/config

@@ -61,3 +61,10 @@ input_files:
     name: tor-android-service
   - project: tor-onion-proxy-library
     name: topl
+
+steps:
+  list_toolchain_updates:
+    input_files: []
+    var:
+      container:
+        use_container: 0

projects/fenix/list_toolchain_updates_checks

+#!/bin/bash
+
+# android-components
+read -d '' p << 'EOF' || true
+if (m/const\\s+val\\s+VERSION\\s+=\\s+"([^"]+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat buildSrc/src/main/java/AndroidComponents.kt | perl -ne "$p")
+current='[% pc("android-components", "var/android_components_version") %]'
+check_update_needed android-components "$needed" "$current"
+
+
+# targetSdkVersion
+read -d '' p << 'EOF' || true
+if (m/const\\s+val\\s+targetSdkVersion\\s+=\\s+([^"]+)/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat buildSrc/src/main/java/Config.kt | perl -ne "$p")
+current=29
+check_update_needed targetSdkVersion "$needed" "$current"
+
+
+# compileSdkVersion
+read -d '' p << 'EOF' || true
+if (m/const\\s+val\\s+compileSdkVersion\\s+=\\s+([^"]+)/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat buildSrc/src/main/java/Config.kt | perl -ne "$p")
+current=29
+check_update_needed compileSdkVersion "$needed" "$current"
+
+
+# minSdkVersion
+read -d '' p << 'EOF' || true
+if (m/const\\s+val\\s+minSdkVersion\\s+=\\s+([^"]+)/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat buildSrc/src/main/java/Config.kt | perl -ne "$p")
+current=21
+check_update_needed compileSdkVersion "$needed" "$current"
+
+
+# gradle
+read -d '' p << 'EOF' || true
+if (m|distributionUrl=https\\\\://services.gradle.org/distributions/gradle-(.*)-all.zip|) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat gradle/wrapper/gradle-wrapper.properties | perl -ne "$p")
+current='[% c("var/gradle_version") %]'
+check_update_needed gradle "$needed" "$current"
+
+
+# android-gradle-plugin
+read -d '' p << 'EOF' || true
+if (m/const\\s+val\\s+android_gradle_plugin\\s+=\\s+"([^"]+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat buildSrc/src/main/java/Dependencies.kt | perl -ne "$p")
+current='4.0.1'
+check_update_needed android-gradle-plugin "$needed" "$current"
+

projects/geckoview/config

@@ -137,6 +137,11 @@ steps:
         target_prepend:
           - torbrowser-android-x86_64
 
+  list_toolchain_updates:
+    input_files: []
+    var:
+      container:
+        use_container: 0
 
 targets:
   nightly:

projects/geckoview/list_toolchain_updates_checks

+#!/bin/bash
+
+# ndk version
+read -d '' p << 'EOF' || true
+if (m/^\\s*NDK_VERSION\\s*=\\s*"(.+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat python/mozboot/mozboot/android.py | perl -ne "$p")
+current='r[% pc("android-toolchain", "var/android_ndk_version") %][% pc("android-toolchain", "var/android_ndk_revision") %]'
+check_update_needed ndk_version "$needed" "$current"
+
+
+# rust
+read -d '' p << 'EOF' || true
+my $d = YAML::XS::LoadFile('taskcluster/ci/toolchain/rust.yml');
+foreach my $t (keys %$d) {
+  if ($d->{$t}{run}{'toolchain-alias'} eq 'linux64-rust-android') {
+    my $channel;
+    foreach my $arg (@{$d->{$t}{run}{arguments}}) {
+      if ($arg eq '--channel') {
+        $channel = 1;
+        next;
+      }
+      if ($channel) {
+        print $arg;
+        exit;
+      }
+    }
+  }
+}
+EOF
+needed=$(perl -MYAML::XS -e "$p")
+current='[% pc("rust", "version") %]'
+check_update_needed rust "$needed" "$current"
+
+
+# build_tools
+read -d '' p << 'EOF' || true
+if (m/build_tools_version\\s*=\\s*"([^"]+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build/moz.configure/android-sdk.configure | perl -ne "$p")
+current='[% pc("android-toolchain", "version") %]'
+check_update_needed build_tools "$needed" "$current"
+
+
+# target_sdk
+read -d '' p << 'EOF' || true
+if (m/target_sdk_version\\s*=\\s*"(.+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build/moz.configure/android-sdk.configure | perl -ne "$p")
+current='[% pc("android-toolchain", "var/android_api_level") %]'
+check_update_needed target_sdk "$needed" "$current"
+
+
+# sdk-tools
+read -d '' p << 'EOF' || true
+if (m|https://dl\\.google\\.com/android/repository/sdk-tools-.+-([^-]+).zip|) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat python/mozboot/mozboot/android.py | perl -ne "$p")
+current='[% pc("android-toolchain", "var/sdk_tools_version") %]'
+check_update_needed sdk-tools "$needed" "$current"
+
+
+# min-android
+read -d '' p << 'EOF' || true
+use Path::Tiny;
+use Digest::SHA qw(sha256_hex);
+my $f;
+my $min_indent;
+foreach (path('build/moz.configure/android-ndk.configure')->lines_utf8) {
+  if ($_ eq "def min_android_version(target):\\n") {
+    $f = $_;
+    next;
+  } else {
+    next unless $f;
+  }
+  m/^(\\s*)/;
+  my $indent = length $1;
+  $min_indent = $indent unless $min_indent;
+  last if $indent < $min_indent;
+  $f .= $_;
+}
+print substr(sha256_hex($f), 0, 10);
+EOF
+needed=$(perl -e "$p")
+# We can't easily parse the min_android_version function.
+# Instead we get a checksum of the function, and manually check it when
+# it was updated.
+# Current value of min_android_version is:
+#  21 on aarch64, x86_64
+#  16 on other archs
+current=303de6de36
+check_update_needed min-android "$needed" "$current"
+
+
+# min_sdk
+read -d '' p << 'EOF' || true
+if (m/^\\s*MOZ_ANDROID_MIN_SDK_VERSION\\s*=\\s*([^\\s]+)/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat mobile/android/confvars.sh | perl -ne "$p")
+current=16
+check_update_needed min_sdk "$needed" "$current"
+
+
+# gradle
+read -d '' p << 'EOF' || true
+if (m|distributionUrl=https\\\\://services.gradle.org/distributions/gradle-(.*)-all.zip|) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat gradle/wrapper/gradle-wrapper.properties | perl -ne "$p")
+current='[% c("var/gradle_version") %]'
+check_update_needed gradle "$needed" "$current"
+
+
+# cbindgen
+read -d '' p << 'EOF' || true
+if (m/^\\s*cbindgen_min_version\\s*=\\s*Version\\("([^"]+)"\\)/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build/moz.configure/bindgen.configure | perl -ne "$p")
+current='[% pc("cbindgen", "version") %]'
+check_update_needed cbindgen "$needed" "$current"
+
+
+# nasm
+read -d '' p << 'EOF' || true
+if (m/^\\s*MODERN_NASM_VERSION\\s*=\\s*LooseVersion\\("([^"]+)"\\)/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat python/mozboot/mozboot/base.py | perl -ne "$p")
+current='2.14'
+check_update_needed nasm "$needed" "$current"
+
+
+# clang
+read -d '' p << 'EOF' || true
+my $d = YAML::XS::LoadFile('taskcluster/ci/toolchain/clang.yml');
+my $clang_toolchain;
+foreach my $t (keys %$d) {
+  if ($d->{$t}{run}{'toolchain-alias'} eq 'linux64-clang-android-cross') {
+    foreach my $fetch (@{$d->{$t}{fetches}{fetch}}) {
+      $clang_toolchain = $fetch if $fetch =~ m/^clang-.*/;
+    }
+    last;
+  }
+}
+
+if (!$clang_toolchain) {
+  print STDERR "Error: could not find clang toolchain";
+  exit 1;
+}
+
+my $fetch = YAML::XS::LoadFile('taskcluster/ci/fetch/toolchains.yml');
+print $fetch->{$clang_toolchain}{fetch}{revision};
+EOF
+needed=$(perl -MYAML::XS -e "$p")
+current='[% pc("llvm-project", "git_hash") %]'
+check_update_needed clang "$needed" "$current"
+
+
+# node
+read -d '' p << 'EOF' || true
+if (m/^\\s*NODE_MIN_VERSION\\s*=\\s*StrictVersion\\("([^"]+)"\\)/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat python/mozbuild/mozbuild/nodeutil.py | perl -ne "$p")
+current='[% pc("node", "version") %]'
+check_update_needed node "$needed" "$current"
+
+
+# python
+read -d '' p << 'EOF' || true
+if (m/find_python3_executable\\(min_version\\s*=\\s*"([^"]+)"/) {
+  print $1;
+  exit;
+}
+EOF
+needed=$(cat build/moz.configure/init.configure | perl -ne "$p")
+current=3.6.0
+check_update_needed python "$needed" "$current"

projects/glean/config

+# vim: filetype=yaml sw=2
+git_hash: 'v[% c("version") %]'
+git_url:  https://github.com/mozilla/glean.git
+filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz'
+version: '[% pc(c("origin_project"), "var/glean_version", { step => "list_toolchain_updates" }) %]'
+
+src: |
+  #!/bin/bash
+  mv -f [% project %]-[% c('version') %].tar.gz [% dest_dir %]/[% c('filename') %]

projects/llvm-project/config

 # vim: filetype=yaml sw=2
 version: 11.0.1
-git_hash: 43ff75f2c3feef64f9d73328230d34dac8832a9
+git_hash: 43ff75f2c3feef64f9d73328230d34dac8832a91
 git_url: https://github.com/llvm/llvm-project
 filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz'

rbm.conf

@@ -15,6 +15,9 @@ steps:
       set -e
       mkdir -p '[% dest_dir %]'
       mv -vf '[% project %]-[% c("version") %].tar.xz' '[% dest_dir %]/[% c("filename") %]'
+  list_toolchain_updates:
+    build_log: '-'
+    list_toolchain_updates: '[% INCLUDE list_toolchain_updates %]'
 
 # buildconf contains build options that the user can change in rbm.local.conf
 # When adding a new option to buildconf, a default value should be defined