Skip to content

GitLab

Commit 64510417 authored by Israel Leiva's avatar Israel Leiva
Browse files

Merge pull request #14 from DonnchaC/github-new 

Host TBB bundles on Github Releases
parents 41031394 cfde8707
Hide whitespace changes
Inline Side-by-side
gettor/utils.py
View file @ 64510417
......@@ -11,6 +11,7 @@
# :license: This is Free Software. See LICENSE for license information.
import os
import re
import hashlib
"""Common utilities for GetTor modules."""
......@@ -19,6 +20,10 @@ import hashlib
LOGGING_FORMAT = "[%(levelname)s] %(asctime)s; %(message)s"
DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S
windows_regex = '^torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe$'
linux_regex = '^tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz$'
osx_regex = '^TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg$'
def get_logging_format():
"""Get the logging format.
......@@ -49,7 +54,7 @@ def get_sha256(string):
return str(hashlib.sha256(string).hexdigest())
def get_bundle_info(file, osys):
def get_bundle_info(filename, osys=None):
"""Get the os, arch and lc from a bundle string.
:param: file (string) the name of the file.
......@@ -60,63 +65,35 @@ def get_bundle_info(file, osys):
:return: (list) the os, arch and lc.
"""
if(osys == 'windows'):
m = re.search(
'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe',
file)
if m:
lc = m.group(1)
return 'windows', '32/64', lc
else:
raise ValueError("Invalid bundle format %s" % file)
elif(osys == 'linux'):
m = re.search(
'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
file)
if m:
arch = m.group(1)
lc = m.group(2)
return 'linux', arch, lc
else:
raise ValueError("Invalid bundle format %s" % file)
elif(osys == 'osx'):
m = re.search(
'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg',
file)
if m:
os = 'osx'
arch = m.group(1)
lc = m.group(2)
return 'osx', arch, lc
else:
raise ValueError("Invalid bundle format %s" % file)
def valid_format(file, osys):
m_windows = re.search(windows_regex, filename)
m_linux = re.search(linux_regex, filename)
m_osx = re.search(osx_regex, filename)
if m_windows:
return 'windows', '32/64', m_windows.group(1)
elif m_linux:
return 'linux', m_linux.group(1), m_linux.group(2)
elif m_osx:
return 'osx', m_osx.group(1), m_osx.group(2)
else:
raise ValueError("Invalid bundle format %s" % file)
def valid_format(filename, osys=None):
"""Check for valid bundle format
Check if the given file has a valid bundle format
(e.g. tor-browser-linux32-3.6.2_es-ES.tar.xz)
:param: file (string) the name of the file.
:param: osys (string) the OS.
:return: (boolean) true if the bundle format is valid, false otherwise.
"""
if(osys == 'windows'):
m = re.search(
'torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe',
file)
elif(osys == 'linux'):
m = re.search(
'tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
file)
elif(osys == 'osx'):
m = re.search(
'TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg',
file)
if m:
m_windows = re.search(windows_regex, filename)
m_linux = re.search(linux_regex, filename)
m_osx = re.search(osx_regex, filename)
if any((m_windows, m_linux, m_osx)):
return True
else:
return False
......@@ -139,3 +116,16 @@ def get_file_sha256(file):
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
return hasher.hexdigest()
def find_files_to_upload(upload_dir):
"""
Find the files which are named correctly and have a .asc file
"""
files = []
for name in os.listdir(upload_dir):
asc_file = os.path.join(upload_dir, "{}.asc".format(name))
if valid_format(name) and os.path.isfile(asc_file):
files.extend([name, "{}.asc".format(name)])
return files
upload/bundles2github.py
View file @ 64510417
......@@ -11,26 +11,55 @@
# :license: This is Free Software. See LICENSE for license information.
#
# Use pyopenssl to verify TLS certifcates
try:
import urllib3.contrib.pyopenssl
urllib3.contrib.pyopenssl.inject_into_urllib3()
except ImportError:
pass
import os
import re
import sh
import sys
import time
import shutil
import hashlib
import argparse
from libsaas.services import github
import ConfigParser
import gnupg
import github3
import gettor.core
from gettor.utils import get_bundle_info, get_file_sha256
from gettor.utils import (get_bundle_info, get_file_sha256,
find_files_to_upload)
def upload_new_release(github_repo, version, upload_dir):
"""
Returns a Release object
"""
# Create a new release of this TBB
release = target_repo.create_release(
'v{}'.format(version),
target_commitish="master",
name='Tor Browser Bundle {}'.format(version),
body='',
draft=True,
)
for filename in find_files_to_upload(upload_dir):
# Upload each file for this release
file_path = os.path.join(upload_dir, filename)
print("Uploading file {}".format(filename))
release.upload_asset('application/zip',
filename, open(file_path, 'rb'))
return release
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Utility to upload Tor Browser to Github.'
)
# with this we only get the links of files already uploaded
# useful when somethings fail after uploading
parser.add_argument(
......@@ -45,24 +74,33 @@ if __name__ == '__main__':
args = parser.parse_args()
config = ConfigParser.ConfigParser()
config.read('github-local.cfg')
# this script should be called after fetching the latest Tor Browser,
# and specifying the latest version
version = args.version
if args.version:
version = args.version
else:
tbb_version_config = ConfigParser.ConfigParser()
tbb_version_config.read('latest_torbrowser.cfg')
version = tbb_version_config.get('version', 'current')
# the token allow us to run this script without GitHub user/pass
gh_token = ''
github_access_token = config.get('app', 'access_token')
# path to the fingerprint that signed the packages
tb_key = os.path.abspath('torbrowser-key.asc')
tb_key = os.path.abspath('tbb-key-torbrowserteam.asc')
# path to the latest version of Tor Browser
tb_path = os.path.abspath('upload/latest')
tb_path = os.path.abspath('latest')
# path to the repository where we upload Tor Browser
repo_path = os.path.abspath('gettorbrowser')
# user and repository where we upload Tor Browser
github_user = config.get('app', 'user')
github_repo = config.get('app', 'repo')
# wait time between pushing the files to GH and asking for its links
wait_time = 10
gh = github3.login(token=github_access_token)
target_repo = gh.repository(github_user, github_repo)
# import key fingerprint
gpg = gnupg.GPG()
......@@ -74,86 +112,52 @@ if __name__ == '__main__':
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))
# we should have previously created a repository on GitHub where we
# want to push the files using an SSH key (to avoid using user/pass)
remote = 'origin'
branch = 'master'
user = 'TheTorProject'
repo = 'gettorbrowser'
raw_content = 'https://raw.githubusercontent.com/%s/%s/%s/' %\
(user, repo, branch)
# steps:
# 1) copy folder with latest version of Tor Browser
# 2) add files via sh.git
# 3) make a commit for the new version
# 4) push the changes
if not args.links:
shutil.copytree(
tb_path,
os.path.abspath('%s/%s' % (repo_path, version))
)
git = sh.git.bake(_cwd=repo_path)
git.add('%s' % version)
git.commit(m=version)
git.push()
# it takes a while to process the recently pushed files
print "Wait a few seconds before asking for the links to Github..."
time.sleep(wait_time)
# Find any published releases with this version number
for release in target_repo.iter_releases():
if release.tag_name == 'v{}'.format(version) and not release.draft:
print("Found an existing published release with this version. "
"Not uploading again unless you delete the published "
"release '{}'.".format(release.tag_name))
break
else:
release = None
if args.links or release:
# Only generating link file, should use previously published release
if not release:
print("Error occured! Could not find a published release for "
"version {}".format(version))
sys.exit(1)
else:
# Remove any drafts to clean broken uploads
print('Uploading release, please wait, this might take a while!')
# Upload the latest browser bundles to a new release
release = upload_new_release(target_repo, version, tb_path)
# Upload success, publish the release
release.edit(draft=False)
# Create the links file for this release
core = gettor.core.Core(os.path.abspath('../core.cfg'))
# Erase old links if any and create a new empty one
core.create_links_file('GitHub', readable_fp)
print "Trying to get the links"
gh = github.GitHub(gh_token, None)
repocontent = gh.repo(
user,
repo
).contents().get('%s' % version)
print("Creating links file")
for asset in release.assets:
url = asset.browser_download_url
if url.endswith('.asc'):
continue
core = gettor.core.Core(
os.path.abspath('core.cfg')
)
osys, arch, lc = get_bundle_info(asset.name)
sha256 = get_file_sha256(
os.path.abspath(os.path.join(tb_path, asset.name))
)
# erase old links, if any
core.create_links_file('GitHub', readable_fp)
link = "{}${}${}$".format(url, url + ".asc", sha256)
for file in repocontent:
# e.g. https://raw.githubusercontent.com/gettorbrowser/dl/master/4.0.7/TorBrowser-4.0.4-osx32_en-US.dmg
m = re.search('%s.*\/(.*)' % raw_content, file[u'download_url'])
if m:
filename = m.group(1)
# get bundle info according to its OS
if re.match('.*\.exe$', filename):
osys, arch, lc = get_bundle_info(filename, 'windows')
filename_asc = filename.replace('exe', 'exe.asc')
elif re.match('.*\.dmg$', filename):
osys, arch, lc = get_bundle_info(filename, 'osx')
filename_asc = filename.replace('dmg', 'dmg.asc')
elif re.match('.*\.tar.xz$', filename):
osys, arch, lc = get_bundle_info(filename, 'linux')
filename_asc = filename.replace('tar.xz', 'tar.xz.asc')
else:
# don't care about other files (asc or txt)
continue
sha256 = get_file_sha256(
os.path.abspath(
'%s/%s/%s' % (repo, version, filename)
)
)
# since the url is easy to construct and it doesn't involve any
# kind of unique hash or identifier, we get the link for the
# asc signature just by adding '.asc'
link_asc = file[u'download_url'].replace(filename, filename_asc)
link = "%s$%s$%s$" % (file[u'download_url'], link_asc, sha256)
print "Adding %s" % file[u'download_url']
core.add_link('GitHub', osys, lc, link)
print("Adding {}".format(url))
core.add_link('GitHub', osys, lc, link)
print "Github links updated!"
upload/github.cfg 0 → 100644
View file @ 64510417
[general]
upload_dir: latest
tbb_key: torbrowser-key.asc
[app]
access_token: suchtoken
user: username
repo: gettor-front
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment