Skip to content
Snippets Groups Projects
Verified Commit 6c8e142b authored by Pier Angelo Vendrame's avatar Pier Angelo Vendrame :jack_o_lantern:
Browse files

Initial commit.

Added a few scripts I've been accumulating for a while now.
parents
No related branches found
No related tags found
No related merge requests found
# Lazy scripts
My daily work as a Tor Browser developer involves several tasks that can be
automated with some scripts to save a lot of time.
I have several ones now and I think they could be useful also to other members
of the team, therefore I decided to create this repository.
They are not production ready, sometimes they have been used just a couple of
times, and then I have fixed their outcome without further improving the script.
Use at your own risk!
#!/usr/bin/env python3
# Check if the hashes of the local build match with the hashes of
# someone who built on the build servers.
# This script should be run from the root of a tor-browser-build clone.
from pathlib import Path
import sys
import requests
version = sys.argv[1]
channel = "alpha" if "a" in version else "release"
def download_hashes(browser, incrementals):
builder = sys.argv[2]
template = "https://tb-build-0{}.torproject.org/~{}/builds/{}browser/{}/unsigned/{}/sha256sums-unsigned-build{}.txt"
args = [2, builder, browser, channel, version, ".incrementals" if incrementals else ""]
r = requests.get(template.format(*args))
if r.status_code == 200:
return r.text
args[0] = 3
r = requests.get(template.format(*args))
if r.status_code == 200:
return r.text
def check_browser(browser, name):
output = repo / f"{browser}browser" / channel / "unsigned" / version
if output.exists():
build = output / "sha256sums-unsigned-build.txt"
build_remote = download_hashes(browser, False)
if not build_remote:
print(f"Failed to download hashes for {name}!")
return
with build.open() as f:
if f.read() == build_remote:
print(f"{name} matches")
else:
print(f"{name} doesn't match!")
incr = output / "sha256sums-unsigned-build.incrementals.txt"
if incr.exists():
incr_remote = download_hashes(browser, True)
if not incr_remote:
print(f"Failed to download hashes for {name} (incrementals)!")
return
with incr.open() as f:
if f.read() == incr_remote:
print(f"{name} incrementals match")
else:
print(f"{name} incrementals don't match")
if __name__ == "__main__":
check_browser("tor", "TBB")
check_browser("mullvad", "MB")
#!/usr/bin/env python3
# This script should be run in a container after an online build of
# something that uses Gradle (GeckoView, Android Components, Fenix,
# etc...).
# The idea is to let Gradle work normally, and then we copy all the
# file it downloaded in /var/tmp/dist/android-toolchain/gradle/caches
# (which this script assumes it has been renamed to caches___online)
# to a directory in a format that looks like the directories we create
# with the downloaded artifacts.
# Then we run an offline build using that directory as a source to make
# sure we have not missed anything in the process.
from collections import namedtuple
from pathlib import Path
from shutil import copyfile
Artifact = namedtuple(
"Artifact", ["path", "group_id", "artifact_id", "version", "sha1", "file"]
)
dest = Path("/var/tmp/dist/gradle-dependencies")
old_dir = Path(
"/var/tmp/dist/android-toolchain/gradle/caches___online/modules-2/files-2.1/"
)
artifacts = []
for path in old_dir.rglob("*"):
# We assume we need all the files, so we don't either check the
# suffix.
if not path.is_file():
continue
rel = str(path.relative_to(old_dir))
artifacts.append(Artifact(*([path] + rel.split("/"))))
for a in artifacts:
dest_dir = dest / a.group_id.replace(".", "/") / a.artifact_id / a.version
dest_dir.mkdir(parents=True, exist_ok=True)
copyfile(a.path, dest_dir / a.file)
#!/usr/bin/env python3
# Parse the output of `git log --oneline` to find which commits are
# associated to confidential Bugzilla tickets and therefore are
# possible backport candidates.
import re
import requests
# TODO: Invoke `git log` directly.
with open("bugs.txt") as f:
for line in f.readlines():
m = re.match("^[0-9a-f]+\sBug ([0-9]+)", line)
if not m:
continue
num = m.group(1)
url = f"https://bugzilla.mozilla.org/show_bug.cgi?id={num}"
r = requests.get(url)
if r.text.find("<title>Access Denied</title>") != -1:
print(url)
#!/usr/bin/env python3
# This script is intended to be run inside a container, after we have
# updated /var/tmp/gradle-dependencies (e.g., with
# extract-gradle-artifacts.py) to create a list that looks like the
# gradle-depdencies-list.txt files we include in tor-browser-build.
from hashlib import sha256
from pathlib import Path
dest = Path("/var/tmp/dist/gradle-dependencies")
for path in dest.rglob("*"):
if not path.is_file():
continue
with path.open("rb") as f:
h = sha256(f.read()).hexdigest()
rel = str(path.relative_to(dest))
print(f"{h} | $HOST/{rel}")
#!/usr/bin/env python3
# This file takes two lists of Gradle dependencies: one that used to
# work (old-list.txt), and another one that has been created with
# list-gradle-artifacts.py (new.txt).
# It tries to match new artifacts against the old ones, if a repository
# hosted an old version, it is likely that it is hosting also a newer
# one. If we do not find a certain artifact we try to look for it on a
# list of known Maven repositories, one by one until we match the
# needed file.
from hashlib import sha256
from urllib.parse import urlparse, urlunparse
import requests
def try_dl(url, expected_sha):
r = requests.get(url)
if r.status_code != 200:
return False
sha = sha256(r.content).hexdigest()
if sha != expected_sha:
print(f"{url} found, but different sha downloaded")
return False
return True
old_list = []
with open("old-list.txt") as f:
old_list = [li.strip().split(" | ") for li in list(f.readlines())[3:]]
with open("new-list.txt") as f:
new_list = [li.strip().split(" | ") for li in f.readlines()]
available = {}
known_artifacts = {}
for sha, url in old_list:
u = urlparse(url)
if u.path.startswith("/maven2/"):
path = u.path[8:]
elif u.path.startswith("/m2/"):
path = u.path[4:]
else:
path = u.path[1:]
available[path] = (sha, url)
u = u._replace(path=u.path.rsplit("/", 2)[0])
known_artifacts[path.rsplit("/", 2)[0]] = urlunparse(u)
matched = []
missed = []
for sha, path in new_list:
path = path[6:]
if path in available and available[path][0] == sha:
matched.append((sha, available[path][1]))
elif path in available and available[path][0] != sha:
print(f"Mismatching sha {path}")
else:
missed.append((sha, path))
repos = [
"https://repo1.maven.org/maven2/",
"https://maven.google.com/",
"https://plugins.gradle.org/m2/",
"https://repo.maven.apache.org/maven2/",
]
for sha, path in missed:
prefix, version, file = path.rsplit("/", 2)
if prefix in known_artifacts:
maybe_url = f"{known_artifacts[prefix]}/{version}/{file}"
if try_dl(maybe_url, sha):
matched.append((sha, maybe_url))
continue
found = False
for repo in repos:
maybe_url = repo + path
if try_dl(maybe_url, sha):
matched.append((sha, maybe_url))
found = True
break
if not found:
print(f"Could not find {path}")
matched = sorted(matched, key=lambda a: a[1])
with open("output.txt", "w") as f:
f.write("sha256sum | url\n")
for a in matched:
f.write(f"{a[0]} | {a[1]}\n")
# This is the mozconfig I use for my Linux dev builds.
# Sometimes I enable the commented debug options.
. $topsrcdir/mozconfig-linux-x86_64-dev
CBINDGEN=/home/piero/Tor/toolchains/cbindgen-0.24.3/cbindgen
# ac_add_options --enable-debug
# ac_add_options --enable-debug-js-modules
# ac_add_options --disable-optimize
# ac_add_options --disable-rust-simd
ac_add_options --enable-tests
# To build without our mozconfigs
# ac_add_options --without-wasm-sandboxed-libraries
# This file is intended to be sourced in a fish shell before building Tor
# Browser for Android.
# It is not generic at all, and it does not intend to be.
# However, it might help other people to create their version of a similar
# script.
# Get it from android-components/buildSrc/src/main/java/Gecko.kt
# This is for Firefox 99.0b3
#set -gx MOZ_BUILD_DATE 20220315185755
# This is for Firefox 102
# set -gx MOZ_BUILD_DATE 20220705093820
# This is for Firefox 115
set -gx MOZ_BUILD_DATE 20230710165010
set TOOLCHAINS "$HOME/Tor/toolchains"
set -gx GRADLE_HOME "$TOOLCHAINS/gradle-7.5.1/"
set -gx JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64
set -gx ANDROID_HOME $TOOLCHAINS/android-sdk-linux
set -gx ANDROID_NDK_HOME $ANDROID_HOME/ndk/android-ndk-r23c
set -gx PATH "$JAVA_HOME/bin/" "$TOOLCHAINS/clang-16.0.4/bin" "$TOOLCHAINS/cbindgen-0.24.3" "$GRADLE_HOME/bin" $PATH
# set -gx WASI_SYSROOT "/media/lssd/Tor/wasi-sysroot/"
set -gx LOCAL_DEV_BUILD 1
#!/usr/bin/env python3
# This is a hack to get faster Tor Browser/Mullvad Browser builds.
# It's a work in progress and doesn't build GeckoView/Firefox Android yet.
# However, I would like to hack RBM to output some machine-readable data of
# what it will do, and use it to build a better scheduler.
# This script should be run on the root of a tor-browser-build clone.
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import psutil
def run_rbm(args, targets=[], capture=False):
args = ["rbm/rbm"] + args
for t in targets:
args += ["--target", t]
proc = subprocess.run(
args, capture_output=capture, env={"RBM_NO_DEBUG": "1"}
)
if capture:
return proc.stdout.decode("utf-8")
def get_conf(project, key, targets, step=None):
args = ["showconf", project, key]
if not step is None:
args += ["--step", step]
return run_rbm(args, targets, True).strip()
def should_build(project, targets, step=None):
filename = get_conf(project, "filename", targets, step)
return not (Path("out") / project / filename).exists()
def build_if_needed(project, targets, step=None):
if not should_build(project, targets, step):
print(f"No need to build {project} ({', '.join(targets)})")
return
print(f"Building {project} ({', '.join(targets)})")
args = ["build", project]
if step is not None:
args += ["--step", step]
run_rbm(args, targets)
def threaded_build(project, targets, step=None):
t = threading.Thread(target=build_if_needed, args=(project, targets, step))
t.start()
return t
def _desktop_helper(targets, ff_thread):
ff_thread.join()
build_if_needed("browser", targets)
def build_firefox(browser, platform, channel):
targets = [f"{browser}-{platform}", channel]
project = "geckoview" is_android else "firefox"
if not should_build(project, targets):
if is_android:
print(f"GeckoView {targets} already up to date")
return
else:
print(
f"Firefox {targets} already up to date, (maybe) building browser"
)
return threaded_build("browser", targets)
t = threaded_build(project, targets)
full_for = 0
while full_for < 120:
while psutil.cpu_percent() < 90.0:
time.sleep(1)
max_start = time.time()
while psutil.cpu_percent() >= 90.0:
full_for = time.time() - max_start
time.sleep(1)
print("Firefox is probably linking, starting the next platform")
if is_android:
return t
t2 = threading.Thread(target=_desktop_helper, args=(targets, t))
t2.start()
return t2
if __name__ == "__main__":
channel = sys.argv[1]
subprocess.run(["make", "submodule-update"])
mullvad = len(sys.argv) >= 3 and sys.argv[2] == "mullvad"
if mullvad:
threads = []
desktop = [
"linux-x86_64",
"windows-x86_64",
"macos-x86_64",
"macos-aarch64",
]
else:
threads = [
threaded_build("manual", ["torbrowser-linux-x86_64", channel]),
threaded_build(
"firefox", ["torbrowser-macos", channel], "src-tarballs"
),
]
desktop = [
"linux-x86_64",
"linux-i686",
"windows-x86_64",
"windows-i686",
"macos-x86_64",
"macos-aarch64",
]
if not mullvad:
for p in desktop:
build_if_needed("tor-expert-bundle", ["torbrowser-" + p, channel])
build_if_needed(
"tor-android-service", ["torbrowser-android-armv7", channel]
)
for p in desktop:
t = build_desktop(
"mullvadbrowser" if mullvad else "torbrowser", p, channel
)
if not t is None:
threads.append(t)
for t in threads:
t.join()
#!/bin/bash
# Repack an APK after updating some files. Then align the resulting file and
# sign it again with the QA key.
# This script exists to update JavaScript files on an APK without having to go
# through Firefox's build system and gradle. It takes a few seconds instead of
# at least 1-1.5 minutes.
# It should be run in a directory prepared with the prepare function.
# TODO: Read arguments and expose a way to call prepare
set -e
# TODO: Generalize!
tools="$ANDROID_HOME/build-tools/31.0.0"
apksigner="$tools/apksigner"
zipalign="$tools/zipalign"
key=~/Tor/tor-browser-build/projects/browser/android-qa.keystore
prepare() {
apk="$1"
mkdir apk
cd apk
7z x "$apk"
mkdir ../omni
cd ../omni
7z x ../apk/assets/omni.ja
}
repack() {
rm -f repacked.apk aligned.apk apk/assets/omni.ja
cd omni
7z -tzip a ../apk/assets/omni.ja .
cd ../apk
# Some files cannot be compressed on R+.
# Also, we want to be faaast, we don't care of size, so just copy
7z -tzip a -mm=copy ../repacked.apk .
cd ..
"$zipalign" -p 4 repacked.apk aligned.apk
"$apksigner" sign \
--ks "$key" \
--in aligned.apk \
--out signed.apk \
--ks-key-alias androidqakey \
--key-pass pass:android \
--ks-pass pass:android
}
repack
# I've repacked an APK because I want to install it :P
# This assumes only one emulator is running an no actual Android device with ADB
# enabled is connected to your computer.
adb install signed.apk
#!/bin/bash
# This script goes through the process of building GeckoView for x86_64 (to use
# in the emulator), publish it to the local Maven repository (~/.m2) and then
# use it to build Fenix.
# Finally, it signs the build with our QA keys and install it on the emulator.
# This script is intended to be run from a directory that includes a
# tor-browser.git cloned into a tor-browser directory, and a firefox-android.git
# cloned into a firefox-android directory at the same level.
# It also assumes any change to $PATH has been made.
# Also, the script works for incremental directories, so any clobber or
# configure needs to be done before running it.
set -e
cd "$(dirname "$0")"
if [[ -z "$MOZ_BUILD_DATE" ]]; then
# The magic file is pierenvironment.fish.
echo "Have you sourced your magic file?"
exit 1
fi
pushd tor-browser
# Only emulator now :P
# MOZCONFIG=mozconfig-android-aarch64 ./mach build
MOZCONFIG=mozconfig-android-x86_64 ./mach build
pushd tools/torbrowser
# make fat-aar ARCHS="aarch64 x86_64"
make fat-aar ARCHS="x86_64"
popd
MOZCONFIG=mozconfig-android-all ./mach gradle \
geckoview:publishWithGeckoBinariesDebugPublicationToMavenLocal \
exoplayer2:publishDebugPublicationToMavenLocal
popd
pushd firefox-android/fenix
gradle --no-daemon -Dorg.gradle.jvmargs=-Xmx20g -PdisableOptimization assembleNightly
tools/tba-sign-devbuilds.sh
# Assumption: only one emulator is running and no real Android device with
# debugging enabled is connected.
adb install app/build/outputs/apk/fenix/nightly/app-fenix-x86_64-nightly-signed.apk
popd
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment