Commit 678bf6c6 authored by Donncha O'Cearbhaill's avatar Donncha O'Cearbhaill
Browse files

Rewrite bundles2github to use releases

parent d8a9d30d
......@@ -11,6 +11,7 @@
# :license: This is Free Software. See LICENSE for license information.
import os
import re
import hashlib
"""Common utilities for GetTor modules."""
......@@ -19,6 +20,10 @@ import hashlib
LOGGING_FORMAT = "[%(levelname)s] %(asctime)s; %(message)s"
DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S
windows_regex = '^torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe$'
linux_regex = '^tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz$'
osx_regex = '^TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg$'
def get_logging_format():
"""Get the logging format.
......@@ -49,7 +54,7 @@ def get_sha256(string):
return str(hashlib.sha256(string).hexdigest())
def get_bundle_info(file, osys):
def get_bundle_info(filename, osys=None):
"""Get the os, arch and lc from a bundle string.
:param: file (string) the name of the file.
......@@ -60,63 +65,35 @@ def get_bundle_info(file, osys):
:return: (list) the os, arch and lc.
if(osys == 'windows'):
m =
if m:
lc =
return 'windows', '32/64', lc
raise ValueError("Invalid bundle format %s" % file)
elif(osys == 'linux'):
m =
if m:
arch =
lc =
return 'linux', arch, lc
raise ValueError("Invalid bundle format %s" % file)
elif(osys == 'osx'):
m =
if m:
os = 'osx'
arch =
lc =
return 'osx', arch, lc
raise ValueError("Invalid bundle format %s" % file)
def valid_format(file, osys):
m_windows =, filename)
m_linux =, filename)
m_osx =, filename)
if m_windows:
return 'windows', '32/64',
elif m_linux:
return 'linux',,
elif m_osx:
return 'osx',,
raise ValueError("Invalid bundle format %s" % file)
def valid_format(filename, osys=None):
"""Check for valid bundle format
Check if the given file has a valid bundle format
(e.g. tor-browser-linux32-3.6.2_es-ES.tar.xz)
:param: file (string) the name of the file.
:param: osys (string) the OS.
:return: (boolean) true if the bundle format is valid, false otherwise.
if(osys == 'windows'):
m =
elif(osys == 'linux'):
m =
elif(osys == 'osx'):
m =
if m:
m_windows =, filename)
m_linux =, filename)
m_osx =, filename)
if any((m_windows, m_linux, m_osx)):
return True
return False
......@@ -139,3 +116,16 @@ def get_file_sha256(file):
buf =
return hasher.hexdigest()
def find_files_to_upload(upload_dir):
Find the files which are named correctly and have a .asc file
files = []
for name in os.listdir(upload_dir):
asc_file = os.path.join(upload_dir, "{}.asc".format(name))
if valid_format(name) and os.path.isfile(asc_file):
files.extend([name, "{}.asc".format(name)])
return files
......@@ -13,24 +13,51 @@
import os
import re
import sh
import sys
import time
import shutil
import hashlib
import argparse
import ConfigParser
from import github
import github3
import gnupg
import gettor.core
from gettor.utils import get_bundle_info, get_file_sha256
from gettor.utils import (get_bundle_info, get_file_sha256,
import urllib3
# Actually verify Github's cert!
def upload_new_release(github_repo, version, upload_dir):
Returns a Release object
# Create a new release of this TBB
release = target_repo.create_release(
name='Tor Browser Bundle {}'.format(version),
for filename in find_files_to_upload(upload_dir):
# Upload each file for this release
file_path = os.path.join(upload_dir, filename)
print("Uploading file {}".format(filename))
filename, open(file_path, 'rb'))
return release
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Utility to upload Tor Browser to Github.'
# with this we only get the links of files already uploaded
# useful when somethings fail after uploading
......@@ -45,24 +72,33 @@ if __name__ == '__main__':
args = parser.parse_args()
config = ConfigParser.ConfigParser()'github-local.cfg')
# this script should be called after fetching the latest Tor Browser,
# and specifying the latest version
version = args.version
if args.version:
version = args.version
tbb_version_config = ConfigParser.ConfigParser()'latest_torbrowser.cfg')
version = tbb_version_config.get('version', 'current')
# the token allow us to run this script without GitHub user/pass
gh_token = ''
github_access_token = config.get('app', 'access_token')
# path to the fingerprint that signed the packages
tb_key = os.path.abspath('torbrowser-key.asc')
tb_key = os.path.abspath('tbb-key-torbrowserteam.asc')
# path to the latest version of Tor Browser
tb_path = os.path.abspath('upload/latest')
tb_path = os.path.abspath('latest')
# path to the repository where we upload Tor Browser
repo_path = os.path.abspath('gettorbrowser')
# user and repository where we upload Tor Browser
github_user = config.get('app', 'user')
github_repo = config.get('app', 'repo')
# wait time between pushing the files to GH and asking for its links
wait_time = 10
gh = github3.login(token=github_access_token)
target_repo = gh.repository(github_user, github_repo)
# import key fingerprint
gpg = gnupg.GPG()
......@@ -74,86 +110,52 @@ if __name__ == '__main__':
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))
# we should have previously created a repository on GitHub where we
# want to push the files using an SSH key (to avoid using user/pass)
remote = 'origin'
branch = 'master'
user = 'TheTorProject'
repo = 'gettorbrowser'
raw_content = '' %\
(user, repo, branch)
# steps:
# 1) copy folder with latest version of Tor Browser
# 2) add files via sh.git
# 3) make a commit for the new version
# 4) push the changes
if not args.links:
os.path.abspath('%s/%s' % (repo_path, version))
git = sh.git.bake(_cwd=repo_path)
git.add('%s' % version)
# it takes a while to process the recently pushed files
print "Wait a few seconds before asking for the links to Github..."
print "Trying to get the links"
gh = github.GitHub(gh_token, None)
repocontent = gh.repo(
).contents().get('%s' % version)
core = gettor.core.Core(
# erase old links, if any
# Find any published releases with this version number
for release in target_repo.iter_releases():
if release.tag_name == 'v{}'.format(version) and not release.draft:
print("Found an existing published release with this version. "
"Not uploading again unless you delete the published "
"release '{}'.".format(release.tag_name))
release = None
if args.links or release:
# Only generating link file, should use previously published release
if not release:
print("Error occured! Could not find a published release for "
"version {}".format(version))
# Remove any drafts to clean broken uploads
print('Uploading release, please wait, this might take a while!')
# Upload the latest browser bundles to a new release
release = upload_new_release(target_repo, version, tb_path)
# Upload success, publish the release
# Create the links file for this release
core = gettor.core.Core(os.path.abspath('../core.cfg'))
# Erase old links if any and create a new empty one
core.create_links_file('GitHub', readable_fp)
for file in repocontent:
# e.g.
m ='%s.*\/(.*)' % raw_content, file[u'download_url'])
if m:
filename =
# get bundle info according to its OS
if re.match('.*\.exe$', filename):
osys, arch, lc = get_bundle_info(filename, 'windows')
filename_asc = filename.replace('exe', 'exe.asc')
elif re.match('.*\.dmg$', filename):
osys, arch, lc = get_bundle_info(filename, 'osx')
filename_asc = filename.replace('dmg', 'dmg.asc')
elif re.match('.*\.tar.xz$', filename):
osys, arch, lc = get_bundle_info(filename, 'linux')
filename_asc = filename.replace('tar.xz', 'tar.xz.asc')
# don't care about other files (asc or txt)
sha256 = get_file_sha256(
'%s/%s/%s' % (repo, version, filename)
# since the url is easy to construct and it doesn't involve any
# kind of unique hash or identifier, we get the link for the
# asc signature just by adding '.asc'
link_asc = file[u'download_url'].replace(filename, filename_asc)
link = "%s$%s$%s$" % (file[u'download_url'], link_asc, sha256)
print "Adding %s" % file[u'download_url']
core.add_link('GitHub', osys, lc, link)
print("Creating releases")
for asset in release.assets:
filename = asset.download_url.split('/')[-1]
osys, arch, lc = get_bundle_info(asset.download_url)
sha256 = get_file_sha256(
os.path.abspath(os.path.join(tb_path, filename))
link = "{}${}${}$".format(
print("Adding {}".format(asset.download_url))
core.add_link('GitHub', osys, lc, link)
print "Github links updated!"
upload_dir: latest
tbb_key: torbrowser-key.asc
access_token: suchtoken
user: username
repo: gettor-front
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment