Bug 41089: Add tor-browser build scripts + Makefile to tor-browser

.gitignore

@@ -351,3 +351,6 @@ mobile/android/annotations/bin/
 
 # Ignore generated log files under media/libvpx
 media/libvpx/config/**/config.log
+
+# Ignore binary base of Tor Browser
+.binaries

tools/geckoview/.gitignore

+android-env.sh

tools/geckoview/Makefile

+.DEFAULT_GOAL := all
+
+# one of armv7 aarch64 x86 x86_64
+ARCH=aarch64
+
+ARCHS=$(ARCH)
+OS="${shell uname}"
+
+# https://stackoverflow.com/questions/18136918/how-to-get-current-relative-directory-of-your-makefile
+mkfile_path := "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))"
+
+DEV_ROOT = "$(mkfile_path)/../.."
+
+clobber: env
+	./clobber.sh $(DEV_ROOT) $(ARCH)
+
+config: env
+	./config.sh $(DEV_ROOT) $(ARCH)
+
+build: env
+	./build.sh $(DEV_ROOT) $(ARCH)
+
+fataar: env
+	bash -c "source android-env.sh && ./fataar.py $(DEV_ROOT) $(ARCHS)"
+
+package: env
+	./package-gradle.sh $(DEV_ROOT) $(ARCH)
+
+show:
+	ls -lh ~/.m2/repository/org/mozilla/geckoview/geckoview-default-omni/
+	ls -lh ~/.m2/repository/org/mozilla/geckoview/geckoview-default-omni/$(shell ls ~/.m2/repository/org/mozilla/geckoview/geckoview-default-omni | tail -n 2 | head -n 1)
+
+env:
+	test -e android-env.sh || { echo "copy android-env-...-template.sh to android-env.sh and edit appropriatly"; exit 1; }
+
+all: env build package
+
+jslint:
+	./jslint.sh $(DEV_ROOT) $(JS)
+
+clean:
+	rm -rf $(BUILD_OUTPUT)
+

tools/geckoview/android-env-linux-template.sh

+export MOZ_BUILD_DATE=20230710165010 # This should match the data in [firefox-android](https://gitlab.torproject.org/tpo/applications/firefox-android)/android-components/plugins/dependencies/src/main/java/Gecko.kt ~ln 12's def of the variable *version*, the date component
+export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+export ANDROID_HOME=$HOME/.mozbuild/android-sdk-linux/ # or $HOME/Android/Sdk/ # Or .../android-toolchain/android-sdk-linux if you extract android-toolchain from tor-browser-build
+export ANDROID_NDK_HOME=$ANDROID_HOME/ndk/23.2.8568313/ # for 115esr
+export GRADLE_HOME=/FULL/PATH/TO/tor-browser-build/out/gradle/gradle-7.5.1 # Or the version that we currently use
+export LOCAL_DEV_BUILD=1
+export PATH=/FULL/PATH/TO/tor-browser-build/out/clang/clang-16.x.y-arm/bin/:$PATH # prepend our newly built and assembled clang to the path so it gets used to build geckoview

tools/geckoview/android-env-macos-template.sh

+export MOZ_BUILD_DATE=20230710165010 # This should match the data in [firefox-android](https://gitlab.torproject.org/tpo/applications/firefox-android)/android-components/plugins/dependencies/src/main/java/Gecko.kt ~ln 12's def of the variable *version*, the date component
+export JAVA_HOME=/opt/homebrew/opt/openjdk@11/libexec/openjdk.jdk/Contents/Home/ # for arm64. Or JAVA_HOME=/usr/local/opt/openjdk@11/libexec/openjdk.jdk/Contents/Home/ for x86_64.
+export ANDROID_HOME=$HOME/Library/Android/sdk # or $HOME/.mozbuild/android-sdk-macosx/
+export ANDROID_NDK_HOME=$ANDROID_HOME/ndk/23.2.8568313 # will need to download NDK 23.2.8568313 via android studio
+export GRADLE_HOME=/opt/homebrew/Cellar/gradle@7/7.6.4 # for arm64 or /usr/local/Cellar/gradle@7/7.6.4 for x86_64. Make sure the version is up to date
+export LOCAL_DEV_BUILD=1
+export PATH=$ANDROID_HOME/ndk/25.2.9519653/toolchains/llvm/prebuilt/darwin-x86_64/bin/:$PATH # prepend android studios latest ndk to the path so it's clang gets used to build geckoview. NDK 25.2.9519653 uses clang 14.0.7, ideally we'd use clang 16 (to be the same as Linux) but that's not an option yet for android studio. NDK 26.1.10909125 uses clang 17.0.2, which we should evaluate with the esr128 migration

tools/geckoview/build.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+ARCH=$2
+
+source android-env.sh
+
+cd $DEV_ROOT
+MOZCONFIG=mozconfig-android-$ARCH  ./mach build

tools/geckoview/clobber.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+ARCH=$2
+
+source android-env.sh
+
+cd $DEV_ROOT
+MOZCONFIG=mozconfig-android-$ARCH ./mach clobber
+MOZCONFIG=mozconfig-android-all ./mach clobber

tools/geckoview/config.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+ARCH=$2
+
+source android-env.sh
+
+cd $DEV_ROOT
+MOZCONFIG=mozconfig-android-$ARCH  ./mach configure  --without-wasm-sandboxed-libraries

tools/geckoview/fataar.py

+#!/usr/bin/env python3
+import os
+import re
+import subprocess
+import sys
+
+dev_root = sys.argv[1]
+archs_in = re.split("\\s+|,", sys.argv[2]) if len(sys.argv) >= 3 else []
+archs_out = []
+env = dict(os.environ)
+
+env["MOZCONFIG"] = "mozconfig-android-all"
+if "armv7" in archs_in:
+    env["MOZ_ANDROID_FAT_AAR_ARMEABI_V7A"] = (
+        dev_root
+        + "/obj-arm-linux-androideabi/gradle/build/mobile/android/geckoview/outputs/aar/geckoview-withGeckoBinaries-debug.aar"
+    )
+    archs_out.append("armeabi-v7a")
+if "aarch64" in archs_in:
+    env["MOZ_ANDROID_FAT_AAR_ARM64_V8A"] = (
+        dev_root
+        + "/obj-aarch64-linux-android/gradle/build/mobile/android/geckoview/outputs/aar/geckoview-withGeckoBinaries-debug.aar"
+    )
+    archs_out.append("arm64-v8a")
+if "x86" in archs_in or "i686" in archs_in:
+    env["MOZ_ANDROID_FAT_AAR_X86"] = (
+        dev_root
+        + "/obj-i386-linux-android/gradle/build/mobile/android/geckoview/outputs/aar/geckoview-withGeckoBinaries-debug.aar"
+    )
+    archs_out.append("x86")
+if "x86_64" in archs_in or "x86-64" in archs_in:
+    env["MOZ_ANDROID_FAT_AAR_X86_64"] = (
+        dev_root
+        + "/obj-x86_64-linux-android/gradle/build/mobile/android/geckoview/outputs/aar/geckoview-withGeckoBinaries-debug.aar"
+    )
+    archs_out.append("x86_64")
+env["MOZ_ANDROID_FAT_AAR_ARCHITECTURES"] = ",".join(archs_out)
+
+if not archs_out:
+    print(
+        "The architectures have not specified or are not valid.",
+        file=sys.stderr,
+    )
+    print('Usage: make fat-aar ARCHS="$archs"', file=sys.stderr)
+    print(
+        "Valid architectures are armv7 aarch64 x86 x86_64, and must be separated with a space.",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+subprocess.run(["./mach", "configure"], cwd=dev_root, env=env, check=True)
+subprocess.run(["./mach", "build"], cwd=dev_root, env=env, check=True)

tools/geckoview/jslint.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+JS_FILE=$2
+
+source android-env.sh
+
+cd $DEV_ROOT
+./mach lint -l eslint --fix $JS_FILE

tools/geckoview/package-gradle.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+ARCH=$2
+
+source android-env.sh
+
+env ARCHS=$ARCH make fataar
+
+cd $DEV_ROOT
+MOZCONFIG=mozconfig-android-$ARCH ./mach build binaries
+MOZCONFIG=mozconfig-android-$ARCH ./mach gradle geckoview:publishWithGeckoBinariesDebugPublicationToMavenRepository
+MOZCONFIG=mozconfig-android-all ./mach gradle geckoview:publishWithGeckoBinariesDebugPublicationToMavenLocal exoplayer2:publishDebugPublicationToMavenLocal
+
+

tools/torbrowser/Makefile

+.DEFAULT_GOAL := all
+
+# https://stackoverflow.com/questions/18136918/how-to-get-current-relative-directory-of-your-makefile
+mkfile_path := "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))"
+
+DEV_ROOT = "$(mkfile_path)/../.."
+BINARIES = "$(DEV_ROOT)/.binaries"
+ARCHITECTURE = "$(shell uname -m)"
+
+# Correct the architecture naming for ARM to match what mozilla has
+ifeq ($(ARCHITECTURE), "arm64")
+  ARCHITECTURE = "aarch64"
+endif
+
+# Define build output path based on the platform.
+ifeq ("$(shell uname)", "Darwin")
+  BUILD_OUTPUT = "$(DEV_ROOT)/obj-$(ARCHITECTURE)-apple-darwin$(shell uname -r)"
+else
+  BUILD_OUTPUT = "$(DEV_ROOT)/obj-$(ARCHITECTURE)-pc-linux-gnu"
+endif
+
+# Define the run command based on the platform.
+ifeq ("$(shell uname)", "Darwin")
+  RUN_CMD := cd "$(BINARIES)/Tor Browser.app/Contents/MacOS/" && ./firefox --purgecaches
+else
+  RUN_CMD := "$(BINARIES)/dev/Browser/start-tor-browser" -v --purgecaches $(ARGS)
+endif
+
+config:
+	./config.sh $(DEV_ROOT)
+
+ide-vscode:
+	./ide.sh vscode $(DEV_ROOT)
+
+ide-eclipse:
+	./ide.sh eclipse $(DEV_ROOT)
+
+ide-visualstudio:
+	./ide.sh visualstudio $(DEV_ROOT)
+
+fetch:
+	./fetch.sh $(BINARIES)
+
+build:
+	./build.sh $(DEV_ROOT)
+
+deploy:
+	./deploy.sh $(BINARIES) $(BUILD_OUTPUT)
+
+all: build deploy
+
+run:
+	$(RUN_CMD)
+
+jslint:
+	./jslint.sh $(DEV_ROOT) $(JS)
+
+clobber:
+	./clobber.sh $(DEV_ROOT)
+
+clean:
+	rm -rf $(BUILD_OUTPUT)
+

tools/torbrowser/browser-self-sign-macos.sh

+#!/bin/bash
+
+CERTNAME=my-codesign-cert-tor
+BROWSERPATH=.
+
+if [ $# -ge 1 ]
+then
+  BROWSERPATH=$1
+fi
+
+
+security find-certificate -c $CERTNAME > /dev/null
+
+if [ $? -ne 0 ]
+then
+  echo ""
+  echo "ERROR: Self Signing Certificate not found, please create:"
+  echo "  1. In the Keychain Access app on your Mac, choose Keychain Access > Certificate Assistant > Create a Certificate."
+  echo "  2. Enter the name '$CERTNAME' for the certificate"
+  echo "  3. Choose an identity type:  Self Signed Root"
+  echo "  4. Certificate Type > Code Signing"
+  echo "  5. Check 'Let me override defaults' & click Continue."
+  echo "  6. Enter a unique Serial Number. (123 is fine)"
+  echo "  7. Enter a big Validity Period (days), like 3560 & click Continue."
+  echo "  8. Fill in your personal information & click Continue."
+  echo "  9. Accept defaults for the rest of the dialog boxes. (Continue several times)"
+  echo "  10. Certificate Created! Click Done."
+  echo ""
+  echo "For additional help see:"
+  echo "  https://support.apple.com/en-ca/guide/keychain-access/kyca8916/mac"
+  echo "  https://stackoverflow.com/questions/58356844/what-are-the-ways-or-technologies-to-sign-an-executable-application-file-in-mac"
+  
+  echo ""
+  read -n 1 -r -s -p $'Press enter to launch "Keychain Access"...\n'
+  open /System/Applications/Utilities/Keychain\ Access.app
+
+  exit -1
+fi
+
+echo "Found $CERTNAME, looking for browser to sign..."
+
+if [ ! -f "$BROWSERPATH/XUL" ]
+then
+  TESTPATH="$BROWSERPATH/Contents/MacOS"
+  if [ -f "$TESTPATH/XUL" ]
+  then
+      BROWSERPATH=$TESTPATH
+  else
+    echo "Error: browser files not detected in $BROWSERPATH!"
+    echo "  This script needs to be run in the 'Contents/MacOS' directory of a SomeBrowser.app directory"
+    exit -1
+  fi
+fi
+
+echo "Mozilla based browser found, signing..."
+echo '  Will be asked for password to certificate for all the things that need to be signed. Click "Always Allow" to automate'
+
+cd "$BROWSERPATH"
+
+codesign -s $CERTNAME *.dylib
+codesign -s $CERTNAME plugin-container.app
+
+if [ -d Tor ]
+then
+  codesign -s $CERTNAME Tor/PluggableTransports/*
+  codesign -s $CERTNAME Tor/libevent-2.1.7.dylib
+  if [ -f Tor/tor.real ]
+  then
+    codesign -s $CERTNAME Tor/tor.real
+  fi
+  if [ -f Tor/tor ]
+  then
+    codesign -s $CERTNAME Tor/tor
+  fi
+fi
+
+codesign -s $CERTNAME XUL
+
+if [ -d updater.app ]
+then
+  codesign -s $CERTNAME updater.app
+fi
+
+# mullvadbrowser
+if [ -f mullvadbrowser ]
+then
+  codesign -s $CERTNAME mullvadbrowser
+fi
+
+# BB or TB
+if [ -f firefox ]
+then
+  codesign -s $CERTNAME firefox
+fi
+
+echo ""
+echo "Browser signing step done!"
+echo ""
+
+echo "App still needs one more override to be easily opened with double click in Finder"
+echo "Alternatively you can right click it, select 'Open' and then select 'Open' from the override popup"
+echo "Or to enable it to be double clicked to open perform the following"
+echo ""
+echo "Double click the app and select either 'Ok' or 'Cancel' from the warning popup depending on which you get (Do Not 'Move to Trash')"
+echo 'Go to Preferences -> Security & Privacy and click on padlock to allow changes. '
+echo '  Then in "Allow appications downloaded from" select either:'
+echo '    - App Store and identified developers'
+echo '    - Anywhere'
+echo '  Below that may be a notice about your specific app saying it was blocked because it was not from an identified developer. Click "Open Anyways" and "Open"'
+

tools/torbrowser/build.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+
+cd $DEV_ROOT
+./mach build
+
+if [ -z "$LOCALES" ]; then
+  ./mach build stage-package
+else
+  export MOZ_CHROME_MULTILOCALE=$LOCALES
+  # No quotes on purpose
+  ./mach package-multi-locale --locales en-US $MOZ_CHROME_MULTILOCALE
+  AB_CD=multi ./mach build stage-package
+fi

tools/torbrowser/clobber.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+
+cd $DEV_ROOT
+./mach clobber

tools/torbrowser/config.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+
+cd $DEV_ROOT
+./mach configure

tools/torbrowser/deploy.sh

+#!/bin/bash
+set -e
+
+BINARIES="$1"
+BUILD_OUTPUT="$2"
+SCRIPT_DIR="$(realpath "$(dirname "$0")")"
+
+RESDIR="$BUILD_OUTPUT/dist/firefox"
+if [ "$(uname)" = "Darwin" ]; then
+    RESDIR="$RESDIR/Tor Browser.app/Contents/Resources"
+fi
+
+# Repackage the manual
+# rm -rf $BUILD_OUTPUT/_omni
+# mkdir $BUILD_OUTPUT/_omni
+# unzip $BINARIES/dev/Browser/browser/omni.ja -d $BUILD_OUTPUT/_omni
+# cd $BUILD_OUTPUT/_omni && zip -Xmr $RESDIR/browser/omni.ja chrome/browser/content/browser/manual
+# rm -rf $BUILD_OUTPUT/_omni
+
+if [ "$(uname)" = "Darwin" ]; then
+
+    # copy binaries
+    cp -r "$BUILD_OUTPUT/dist/firefox/"*.app/Contents/* "$BINARIES/Tor Browser.app/Contents/"
+    rm -rf "$BINARIES/TorBrowser-Data/Browser/Caches/*.default/startupCache"
+
+    # Self sign the Binaries
+    cd "$BINARIES/Tor Browser.app/Contents/MacOS"
+    "$SCRIPT_DIR/browser-self-sign-macos.sh"
+
+else
+
+    # backup the startup script
+    mv "$BINARIES/dev/Browser/firefox" "$BINARIES/dev/Browser/firefox.bak"
+
+    # copy binaries
+    cp -r "$RESDIR/"* "$BINARIES/dev/Browser"
+    rm -rf "$BINARIES/dev/Browser/TorBrowser/Data/Browser/profile.default/startupCache"
+
+    # shuffle firefox bin around and restore script to match a real deployment
+    mv "$BINARIES/dev/Browser/firefox" "$BINARIES/dev/Browser/firefox.real"
+    mv "$BINARIES/dev/Browser/firefox.bak" "$BINARIES/dev/Browser/firefox"
+
+fi

tools/torbrowser/fetch.sh

+#!/bin/sh
+set -e
+
+BINARIES_DIR="$1"
+
+# download the current downloads.json
+wget https://aus1.torproject.org/torbrowser/update_3/alpha/downloads.json
+# get url for latest alpha linux package
+TOR_BROWSER_VERSION=$(grep -Eo "\"version\":\"[0-9.a]+\"" downloads.json | grep -Eo "[0-9.a]+")
+if [ "$(uname)" = "Darwin" ]; then
+    TOR_BROWSER_PACKAGE="tor-browser-macos-${TOR_BROWSER_VERSION}.dmg"
+  else
+    TOR_BROWSER_PACKAGE="tor-browser-linux-x86_64-${TOR_BROWSER_VERSION}.tar.xz"
+fi
+TOR_BROWSER_PACKAGE_URL="https://dist.torproject.org/torbrowser/${TOR_BROWSER_VERSION}/${TOR_BROWSER_PACKAGE}"
+
+# remove download manifest
+rm downloads.json
+
+# clear out previous tor-browser and previous package
+rm -rf "${BINARIES_DIR}"
+rm -f "${TOR_BROWSER_PACKAGE}"
+
+# download
+wget "${TOR_BROWSER_PACKAGE_URL}"
+mkdir -p "${BINARIES_DIR}"
+
+# and extract
+if [ "$(uname)" = "Darwin" ]
+  then
+    hdiutil attach "${TOR_BROWSER_PACKAGE}"
+    cp -R "/Volumes/Tor Browser Alpha/Tor Browser Alpha.app" "${BINARIES_DIR}/Tor Browser.app"
+    hdiutil detach "/Volumes/Tor Browser Alpha"
+  else
+    tar -xf "${TOR_BROWSER_PACKAGE}" -C "${BINARIES_DIR}"
+    mv "${BINARIES_DIR}/tor-browser" "${BINARIES_DIR}/dev"
+fi
+
+# Final cleanup
+rm -f "${TOR_BROWSER_PACKAGE}"

tools/torbrowser/ide.sh

+#!/bin/bash
+set -e
+IDE=$1
+DEV_ROOT=$2
+
+cd $DEV_ROOT
+./mach ide $IDE

tools/torbrowser/jslint.sh

+#!/bin/bash
+set -e
+DEV_ROOT=$1
+JS_FILE=$2
+
+cd $DEV_ROOT
+./mach lint -l eslint --fix $JS_FILE