Changed update_files to upload releases
We were running out of space in both of our github and gitlab repositories due to storage limits. This change switches to using the github REST API to upload torbrowser binaries as releases. Releases are not subject to the space limits of repositories and, even better, do not count against our bandwidth limits when users download them.
| #!/bin/bash | ||
| # | ||
| # This file is part of GetTor, a Tor Browser distribution system. | ||
| # | ||
| # :authors: hiro <hiro@torproject.org> | ||
| # see also AUTHORS file | ||
| # | ||
| # :copyright: (c) 2008-2019, The Tor Project, Inc. | ||
| # | ||
| # :license: This is Free Software. See LICENSE for license information. | ||
| cd ~/releases | ||
| git checkout releases | ||
| rm -f torbrowser-* TorBrowser-* tor-browser-* | ||
| for row in $( | ||
| curl -s 'https://aus1.torproject.org/torbrowser/update_3/release/downloads.json' | | ||
| jq -r '.downloads' | ||
| ); do | ||
| r=$( | ||
| echo ${row} | | ||
| egrep -o 'https?://[^ ]+' | | ||
| tr -d '",' | ||
| ); | ||
| if [[ $r = *[!\ ]* ]]; then | ||
| wget $r | ||
| git fetch --all | ||
| git checkout releases | ||
| git add . | ||
| git commit -m '[dist ci] commit from CI runner - update with new torbrowser downloads' | ||
| diffs=$(git diff origin/releases) | ||
| if [ -z "$diffs" ]; then | ||
| echo "No new releases" | ||
| else | ||
| git push -f --follow-tags origin releases | ||
| fi | ||
| git checkout torbrowser-releases | ||
| git merge releases | ||
| diffs=$(git diff github/torbrowser-releases) | ||
| if [ -z "$diffs" ]; then | ||
| echo "No new releases" | ||
| else | ||
| git push -f --follow-tags github torbrowser-releases | ||
| fi | ||
| fi; | ||
| done | ||
| #!/usr/bin/env python3 | ||
| from github import Github | ||
|
||
| import sys | ||
| import json | ||
| import urllib | ||
| import subprocess | ||
| REPO_NAME = "TheTorProject/gettorbrowser" | ||
| def delete_old_releases(repo): | ||
| for release in repo.get_releases(): | ||
| release.delete_release() | ||
| #Download list of tor browser releases and upload them to github | ||
| def upload_files(release): | ||
| url = urllib.request.urlopen("https://aus1.torproject.org/torbrowser/update_3/release/downloads.json") | ||
| data = json.loads(url.read().decode()) | ||
| for arch in data['downloads']: | ||
| for locale in data['downloads'][arch]: | ||
| for asset in data['downloads'][arch][locale]: | ||
| url = data['downloads'][arch][locale][asset] | ||
| filename = url.split('/')[-1] | ||
| print("Downloading " + filename) | ||
| subprocess.check_call(["/usr/bin/wget", url]) | ||
| release.upload_asset(filename) | ||
| def main(token): | ||
| #Initialize a new release | ||
| g = Github(token) | ||
| repo = g.get_repo(REPO_NAME) | ||
| delete_old_releases(repo) | ||
| #Create a new release | ||
| release = repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.") | ||
| upload_files(release) | ||
| if __name__ == "__main__": | ||
| if len(sys.argv) != 2: | ||
| print("Usage: {} AUTH_TOKEN".format(sys.argv[0]), file=sys.stderr) | ||
|
||
| print("\nAUTH_TOKEN should be an authentication token for a user" | ||
| "with access to the gettor repository.", file=sys.stderr) | ||
| sys.exit(1) | ||
| token = sys.argv[1] | ||
| sys.exit(main(token)) | ||
