Skip to content

GitLab

G
gettor

GitLab is used only for code review, issue tracking and project management. Canonical locations for source code are still https://gitweb.torproject.org/ https://git.torproject.org/ and git-rw.torproject.org.

Commit 880912bb authored by Cecylia Bocovich's avatar Cecylia Bocovich
Browse files
Options

Remove all old upload scripts 

These have been replaced by newer scripts located in the top-level
scripts/ directory.
parent 63559201
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 962 deletions
upload/bundles2drive.py deleted 100644 → 0
View file @ 63559201
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: poly <poly@darkdepths.net>
# Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Poly
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import re
import os
import gnupg
import hashlib
import logging
import argparse
import ConfigParser
import gettor.core
from gettor.utils import get_bundle_info, get_file_sha256, valid_format
# import google drive libs
import httplib2
from apiclient.discovery import build
from apiclient.http import MediaFileUpload
from apiclient import errors
from oauth2client.client import FlowExchangeError
from oauth2client.client import OAuth2WebServerFlow
from oauth2client.client import Credentials
def upload_files(client, basedir):
"""Upload files to Google Drive.
Looks for tor browser files inside basedir.
:param: basedir (string) path of the folder with the files to be
uploaded.
:param: client (object) Google Drive object.
:raise: UploadError if something goes wrong while uploading the
files to Google Drive. All files are uploaded to '/'.
:return: (dict) the names of the uploaded files as the keys,
and file id as the value
"""
files = []
for name in os.listdir(basedir):
path = os.path.abspath(os.path.join(basedir, name))
if os.path.isfile(path) and valid_format(name, 'linux'):
files.append(name)
for name in os.listdir(basedir):
path = os.path.abspath(os.path.join(basedir, name))
if os.path.isfile(path) and valid_format(name, 'windows'):
files.append(name)
for name in os.listdir(basedir):
path = os.path.abspath(os.path.join(basedir, name))
if os.path.isfile(path) and valid_format(name, 'osx'):
files.append(name)
# dictionary to store file names and IDs
files_dict = dict()
for file in files:
asc = "%s.asc" % file
abs_file = os.path.abspath(os.path.join(basedir, file))
abs_asc = os.path.abspath(os.path.join(basedir, asc))
if not os.path.isfile(abs_asc):
# there are some .mar files that don't have .asc, don't upload it
continue
# upload tor browser installer
file_body = MediaFileUpload(
abs_file,
mimetype="application/octet-stream",
resumable=True
)
body = {
'title': file
}
print "Uploading '%s'..." % file
try:
file_data = drive_service.files().insert(
body=body,
media_body=file_body
).execute()
except errors.HttpError, e:
print str(e)
# upload signature
asc_body = MediaFileUpload(abs_asc, resumable=True)
asc_head = {
'title': "%s.asc" % file
}
print "Uploading '%s'..." % asc
try:
asc_data = drive_service.files().insert(
body=asc_head,
media_body=asc_body
).execute()
except errors.HttpError, e:
print str(e)
# add filenames and file id to dict
files_dict[file] = file_data['id']
files_dict[asc] = asc_data['id']
return files_dict
def share_file(service, file_id):
"""Make files public
For a given file-id, sets role 'reader' to 'anyone'. Returns public
link to file.
:param: file_id (string)
:return: (string) url to shared file
"""
permission = {
'type': "anyone",
'role': "reader",
'withLink': True
}
try:
service.permissions().insert(
fileId=file_id,
body=permission
).execute()
except errors.HttpError, error:
print('An error occured while sharing: %s' % file_id)
try:
file = service.files().get(fileId=file_id).execute()
except errors.HttpError, error:
print('Error occured while fetch public link for file: %s' % file_id)
print "Uploaded %s to %s" % (file['title'], file['webContentLink'])
return file['webContentLink']
def get_files_links(service, v):
"""Print links of uploaded files.
:param: service (object): Goolge Drive service object.
:param: v (string): Version of Tor Browser to look for.
"""
windows_re = 'torbrowser-install-%s_\w\w(-\w\w)?\.exe(\.asc)?' % v
linux_re = 'tor-browser-linux\d\d-%s_(\w\w)(-\w\w)?\.tar\.xz(\.asc)?' % v
osx_re = 'TorBrowser-%s-osx\d\d_(\w\w)(-\w\w)?\.dmg(\.asc)?' % v
# dictionary to store file names and IDs
files_dict = dict()
print "Trying to fetch links of uploaded files..."
links = service.files().list().execute()
items = links.get('items', [])
if not items:
raise ValueError('No files found.')
else:
for item in items:
if re.search(windows_re, item['title']):
files_dict[item['title']] = item['id']
elif re.search(linux_re, item['title']):
files_dict[item['title']] = item['id']
elif re.search(osx_re, item['title']):
files_dict[item['title']] = item['id']
return files_dict
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Utility to upload Tor Browser to Google Drive.'
)
# if no LC specified, download all
parser.add_argument(
'-l', '--links', default=None,
help='Create links file with files already uploaded and '\
'matching the specified version. '
)
args = parser.parse_args()
config = ConfigParser.ConfigParser()
config.read('drive.cfg')
client_id = config.get('app', 'client-id')
app_secret = config.get('app', 'secret')
refresh_token = config.get('app', 'refresh_token')
upload_dir = config.get('general', 'upload_dir')
# important: this key must be the one that signed the packages
tbb_key = config.get('general', 'tbb_key')
# requests full access to drive account
OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive'
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
print "Authenticating..."
flow = OAuth2WebServerFlow(
client_id,
app_secret,
OAUTH_SCOPE,
redirect_uri=REDIRECT_URI
)
# If no valid token found, need to prompt user.
# this should only occur once
if not refresh_token:
flow.params['access_type'] = 'offline'
flow.params['approval_prompt'] = 'force'
authorize_url = flow.step1_get_authorize_url()
print 'Go to the following link in your browser: ' + authorize_url
code = raw_input('Enter verification code: ').strip()
try:
credentials = flow.step2_exchange(code)
except FlowExchangeError as e:
print str(e)
# oauth2 credentials instance must be stored as json string
config.set('app', 'refresh_token', credentials.to_json())
with open('drive.cfg', 'wb') as configfile:
config.write(configfile)
else:
# we already have a valid token
credentials = Credentials.new_from_json(refresh_token)
# authenticate with oauth2
http = httplib2.Http()
http = credentials.authorize(http)
# initialize drive instance
drive_service = build('drive', 'v2', http=http)
# import key fingerprint
gpg = gnupg.GPG()
key_data = open(tbb_key).read()
import_result = gpg.import_keys(key_data)
fp = import_result.results[0]['fingerprint']
# make groups of four characters to make fingerprint more readable
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))
try:
# helpful when something fails but files are uploaded.
if args.links:
uploaded_files = get_files_links(drive_service, args.links)
if not uploaded_files:
raise ValueError("There are no files for that version")
else:
uploaded_files = upload_files(drive_service, upload_dir)
# use default config
core = gettor.core.Core('/home/gettor/core.cfg')
# erase old links
core.create_links_file('Drive', readable)
# recognize file OS by its extension
p1 = re.compile('.*\.tar.xz$')
p2 = re.compile('.*\.exe$')
p3 = re.compile('.*\.dmg$')
p4 = re.compile('.*\.asc$')
for file in uploaded_files.keys():
# only run for tor browser installers
if p4.match(file):
continue
asc = "%s.asc" % file
abs_file = os.path.abspath(os.path.join(upload_dir, file))
abs_asc = os.path.abspath(os.path.join(upload_dir, asc))
sha_file = get_file_sha256(abs_file)
# build links
link_file = share_file(
drive_service,
uploaded_files[file]
)
link_asc = share_file(
drive_service,
uploaded_files["%s.asc" % file]
)
if p1.match(file):
osys, arch, lc = get_bundle_info(file, 'linux')
elif p2.match(file):
osys, arch, lc = get_bundle_info(file, 'windows')
elif p3.match(file):
osys, arch, lc = get_bundle_info(file, 'osx')
link = "%s$%s$%s$" % (link_file, link_asc, sha_file)
# note that you should only upload bundles for supported locales
core.add_link('Drive', osys, lc, link)
except (ValueError, RuntimeError) as e:
print str(e)
upload/bundles2dropbox.py deleted 100644 → 0
View file @ 63559201
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import re
import os
import gnupg
import hashlib
import ConfigParser
import dropbox
import gettor.core
from gettor.utils import get_bundle_info, get_file_sha256, valid_format
def upload_files(basedir, client):
"""Upload files to Dropbox.
Looks for files ending with 'tar.xz' inside basedir.
:param: basedir (string) path of the folder with the files to be
uploaded.
:param: client (object) DropboxClient object.
:raise: ValueError if the .xz file doesn't have an .asc file.
:raise: UploadError if something goes wrong while uploading the
files to Dropbox. All files are uploaded to '/'.
:return: (list) the names of the uploaded files.
"""
files = []
for name in os.listdir(basedir):
path = os.path.abspath(os.path.join(basedir, name))
if os.path.isfile(path) and valid_format(name, 'linux'):
files.append(name)
for name in os.listdir(basedir):
path = os.path.abspath(os.path.join(basedir, name))
if os.path.isfile(path) and valid_format(name, 'windows'):
files.append(name)
for name in os.listdir(basedir):
path = os.path.abspath(os.path.join(basedir, name))
if os.path.isfile(path) and valid_format(name, 'osx'):
files.append(name)
for file in files:
asc = "%s.asc" % file
abs_file = os.path.abspath(os.path.join(basedir, file))
abs_asc = os.path.abspath(os.path.join(basedir, asc))
if not os.path.isfile(abs_asc):
# there are some .mar files that don't have .asc, don't upload it
continue
# chunk upload for big files
to_upload = open(abs_file, 'rb')
size = os.path.getsize(abs_file)
uploader = client.get_chunked_uploader(to_upload, size)
while uploader.offset < size:
try:
upload = uploader.upload_chunked()
except dropbox.rest.ErrorResponse, e:
print("An error ocurred while uploading %s: %s" % abs_file, e)
uploader.finish(file)
print "Uploading %s" % file
# this should be small, upload it simple
to_upload_asc = open(abs_asc, 'rb')
response = client.put_file(asc, to_upload_asc)
print "Uploading %s" % asc
return files
if __name__ == '__main__':
config = ConfigParser.ConfigParser()
config.read('dropbox.cfg')
app_key = config.get('app', 'key')
app_secret = config.get('app', 'secret')
access_token = config.get('app', 'access_token')
upload_dir = config.get('general', 'upload_dir')
# important: this key must be the one that signed the packages
tbb_key = config.get('general', 'tbb_key')
client = dropbox.client.DropboxClient(access_token)
# import key fingerprint
gpg = gnupg.GPG()
key_data = open(tbb_key).read()
import_result = gpg.import_keys(key_data)
fp = import_result.results[0]['fingerprint']
# make groups of four characters to make fingerprint more readable
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))
try:
uploaded_files = upload_files(upload_dir, client)
# use default config
core = gettor.core.Core('/home/gettor/core.cfg')
# erase old links
core.create_links_file('Dropbox', readable)
# recognize file OS by its extension
p1 = re.compile('.*\.tar.xz$')
p2 = re.compile('.*\.exe$')
p3 = re.compile('.*\.dmg$')
for file in uploaded_files:
# build file names
asc = "%s.asc" % file
abs_file = os.path.abspath(os.path.join(upload_dir, file))
abs_asc = os.path.abspath(os.path.join(upload_dir, asc))
sha_file = get_file_sha256(abs_file)
# build links
link_file = client.share(file, short_url=False)
# if someone finds how to do this with the API, please tell me!
link_file[u'url'] = link_file[u'url'].replace('?dl=0', '?dl=1')
link_asc = client.share(asc, short_url=False)
link_asc[u'url'] = link_asc[u'url'].replace('?dl=0', '?dl=1')
if p1.match(file):
osys, arch, lc = get_bundle_info(file, 'linux')
elif p2.match(file):
osys, arch, lc = get_bundle_info(file, 'windows')
elif p3.match(file):
osys, arch, lc = get_bundle_info(file, 'osx')
link = "%s$%s$%s$" % (link_file[u'url'], link_asc[u'url'], sha_file)
# note that you should only upload bundles for supported locales
core.add_link('Dropbox', osys, lc, link)
except (ValueError, RuntimeError) as e:
print str(e)
except dropbox.rest.ErrorResponse as e:
print str(e)
upload/bundles2github.py deleted 100644 → 0
View file @ 63559201
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2015, The Tor Project, Inc.
# (c) 2015, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
#
# Use pyopenssl to verify TLS certifcates
try:
import urllib3.contrib.pyopenssl
urllib3.contrib.pyopenssl.inject_into_urllib3()
except ImportError:
pass
import os
import sys
import argparse
import ConfigParser
import gnupg
import github3
import gettor.core
from gettor.utils import (get_bundle_info, get_file_sha256,
find_files_to_upload)
def upload_new_release(github_repo, version, upload_dir):
"""
Returns a Release object
"""
# Create a new release of this TBB
release = target_repo.create_release(
'v{}'.format(version),
target_commitish="master",
name='Tor Browser {}'.format(version),
body='',
draft=True,
)
for filename in find_files_to_upload(upload_dir):
# Upload each file for this release
file_path = os.path.join(upload_dir, filename)
print("Uploading file {}".format(filename))
release.upload_asset('application/zip',
filename, open(file_path, 'rb'))
return release
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Utility to upload Tor Browser to Github.'
)
# with this we only get the links of files already uploaded
# useful when somethings fail after uploading
parser.add_argument(
'-l', '--links', default=None,
help='Create links file with files already uploaded.'
)
args = parser.parse_args()
config = ConfigParser.ConfigParser()
config.read('github.cfg')
tbb_version_path = config.get('general', 'version_cfg_path')
tbb_version_config = ConfigParser.ConfigParser()
tbb_version_config.read(tbb_version_path)
version = tbb_version_config.get('version', 'current')
# the token allow us to run this script without GitHub user/pass
github_access_token = config.get('app', 'access_token')
# path to the fingerprint that signed the packages
tb_key = config.get('general', 'tbb_key_path')
# path to the latest version of Tor Browser
tb_path = config.get('general', 'latest_path')
# path to gettor code configuration
core_path = config.get('general', 'core_path')
# user and repository where we upload Tor Browser
github_user = config.get('app', 'user')
github_repo = config.get('app', 'repo')
gh = github3.login(token=github_access_token)
target_repo = gh.repository(github_user, github_repo)
# import key fingerprint
gpg = gnupg.GPG()
key_data = open(tb_key).read()
import_result = gpg.import_keys(key_data)
fp = import_result.results[0]['fingerprint']
# make groups of four characters to make fingerprint more readable
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))
# Find any published releases with this version number
for release in target_repo.releases():
if release.tag_name == 'v{}'.format(version) and not release.draft:
print("Found an existing published release with this version. "
"Not uploading again unless you delete the published "
"release '{}'.".format(release.tag_name))
break
else:
release = None
if args.links or release:
# Only generating link file, should use previously published release
if not release:
print("Error occured! Could not find a published release for "
"version {}".format(version))
sys.exit(1)
else:
# Remove any drafts to clean broken uploads
print('Uploading release, please wait, this might take a while!')
# Upload the latest browser bundles to a new release
release = upload_new_release(target_repo, version, tb_path)
# Upload success, publish the release
release.edit(draft=False)
# Create the links file for this release
core = gettor.core.Core(core_path)
# Erase old links if any and create a new empty one
core.create_links_file('GitHub', readable_fp)
print("Creating links file")
for asset in release.assets():
url = asset.browser_download_url
if url.endswith('.asc'):
continue
osys, arch, lc = get_bundle_info(asset.name)
sha256 = get_file_sha256(
os.path.abspath(os.path.join(tb_path, asset.name))
)
link = "{}${}${}$".format(url, url + ".asc", sha256)
print("Adding {}".format(url))
core.add_link('GitHub', osys, lc, link)
print "Github links updated!"
upload/drive.cfg deleted 100644 → 0
View file @ 63559201
[general]
upload_dir = latest
tbb_key = torbrowser-key.asc
[app]
client-id =
secret =
refresh_token =
upload/dropbox.cfg deleted 100644 → 0
View file @ 63559201
[general]
upload_dir: latest
tbb_key: torbrowser-key.asc
[app]
key: suchkey
secret: suchsecret
access_token: suchtoken
upload/fetch_latest_torbrowser.py deleted 100644 → 0
View file @ 63559201
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@torproject.org>
#
# :copyright: (c) 2015, The Tor Project, Inc.
# (c) 2015, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
#
import os
import urllib2
import json
import argparse
import ConfigParser
import shutil
# this path should be relative to this script (or absolute)
UPLOAD_SCRIPTS = {
'dropbox': 'bundles2dropbox.py',
'drive': 'bundles2drive.py'
}
# "regex" for filtering downloads in wget
OS_RE = {
'windows': '%s.exe,%s.exe.asc',
'linux': '%s.tar.xz,%s.tar.xz.asc',
'osx': '%s.dmg,%s.dmg.asc',
}
def main():
"""Script to fetch the latest Tor Browser.
Fetch the latest version of Tor Browser and upload it to the supported
providers (e.g. Dropbox). Ideally, this script should be executed with
a cron in order to automate the updating of the files served by GetTor