Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Cecylia Bocovich
gettor
Commits
880912bb
Commit
880912bb
authored
Apr 13, 2020
by
Cecylia Bocovich
Browse files
Remove all old upload scripts
These have been replaced by newer scripts located in the top-level scripts/ directory.
parent
63559201
Changes
11
Hide whitespace changes
Inline
Side-by-side
upload/bundles2drive.py
deleted
100644 → 0
View file @
63559201
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: poly <poly@darkdepths.net>
# Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Poly
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import
re
import
os
import
gnupg
import
hashlib
import
logging
import
argparse
import
ConfigParser
import
gettor.core
from
gettor.utils
import
get_bundle_info
,
get_file_sha256
,
valid_format
# import google drive libs
import
httplib2
from
apiclient.discovery
import
build
from
apiclient.http
import
MediaFileUpload
from
apiclient
import
errors
from
oauth2client.client
import
FlowExchangeError
from
oauth2client.client
import
OAuth2WebServerFlow
from
oauth2client.client
import
Credentials
def
upload_files
(
client
,
basedir
):
"""Upload files to Google Drive.
Looks for tor browser files inside basedir.
:param: basedir (string) path of the folder with the files to be
uploaded.
:param: client (object) Google Drive object.
:raise: UploadError if something goes wrong while uploading the
files to Google Drive. All files are uploaded to '/'.
:return: (dict) the names of the uploaded files as the keys,
and file id as the value
"""
files
=
[]
for
name
in
os
.
listdir
(
basedir
):
path
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
name
))
if
os
.
path
.
isfile
(
path
)
and
valid_format
(
name
,
'linux'
):
files
.
append
(
name
)
for
name
in
os
.
listdir
(
basedir
):
path
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
name
))
if
os
.
path
.
isfile
(
path
)
and
valid_format
(
name
,
'windows'
):
files
.
append
(
name
)
for
name
in
os
.
listdir
(
basedir
):
path
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
name
))
if
os
.
path
.
isfile
(
path
)
and
valid_format
(
name
,
'osx'
):
files
.
append
(
name
)
# dictionary to store file names and IDs
files_dict
=
dict
()
for
file
in
files
:
asc
=
"%s.asc"
%
file
abs_file
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
file
))
abs_asc
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
asc
))
if
not
os
.
path
.
isfile
(
abs_asc
):
# there are some .mar files that don't have .asc, don't upload it
continue
# upload tor browser installer
file_body
=
MediaFileUpload
(
abs_file
,
mimetype
=
"application/octet-stream"
,
resumable
=
True
)
body
=
{
'title'
:
file
}
print
"Uploading '%s'..."
%
file
try
:
file_data
=
drive_service
.
files
().
insert
(
body
=
body
,
media_body
=
file_body
).
execute
()
except
errors
.
HttpError
,
e
:
print
str
(
e
)
# upload signature
asc_body
=
MediaFileUpload
(
abs_asc
,
resumable
=
True
)
asc_head
=
{
'title'
:
"%s.asc"
%
file
}
print
"Uploading '%s'..."
%
asc
try
:
asc_data
=
drive_service
.
files
().
insert
(
body
=
asc_head
,
media_body
=
asc_body
).
execute
()
except
errors
.
HttpError
,
e
:
print
str
(
e
)
# add filenames and file id to dict
files_dict
[
file
]
=
file_data
[
'id'
]
files_dict
[
asc
]
=
asc_data
[
'id'
]
return
files_dict
def
share_file
(
service
,
file_id
):
"""Make files public
For a given file-id, sets role 'reader' to 'anyone'. Returns public
link to file.
:param: file_id (string)
:return: (string) url to shared file
"""
permission
=
{
'type'
:
"anyone"
,
'role'
:
"reader"
,
'withLink'
:
True
}
try
:
service
.
permissions
().
insert
(
fileId
=
file_id
,
body
=
permission
).
execute
()
except
errors
.
HttpError
,
error
:
print
(
'An error occured while sharing: %s'
%
file_id
)
try
:
file
=
service
.
files
().
get
(
fileId
=
file_id
).
execute
()
except
errors
.
HttpError
,
error
:
print
(
'Error occured while fetch public link for file: %s'
%
file_id
)
print
"Uploaded %s to %s"
%
(
file
[
'title'
],
file
[
'webContentLink'
])
return
file
[
'webContentLink'
]
def
get_files_links
(
service
,
v
):
"""Print links of uploaded files.
:param: service (object): Goolge Drive service object.
:param: v (string): Version of Tor Browser to look for.
"""
windows_re
=
'torbrowser-install-%s_\w\w(-\w\w)?\.exe(\.asc)?'
%
v
linux_re
=
'tor-browser-linux\d\d-%s_(\w\w)(-\w\w)?\.tar\.xz(\.asc)?'
%
v
osx_re
=
'TorBrowser-%s-osx\d\d_(\w\w)(-\w\w)?\.dmg(\.asc)?'
%
v
# dictionary to store file names and IDs
files_dict
=
dict
()
print
"Trying to fetch links of uploaded files..."
links
=
service
.
files
().
list
().
execute
()
items
=
links
.
get
(
'items'
,
[])
if
not
items
:
raise
ValueError
(
'No files found.'
)
else
:
for
item
in
items
:
if
re
.
search
(
windows_re
,
item
[
'title'
]):
files_dict
[
item
[
'title'
]]
=
item
[
'id'
]
elif
re
.
search
(
linux_re
,
item
[
'title'
]):
files_dict
[
item
[
'title'
]]
=
item
[
'id'
]
elif
re
.
search
(
osx_re
,
item
[
'title'
]):
files_dict
[
item
[
'title'
]]
=
item
[
'id'
]
return
files_dict
if
__name__
==
'__main__'
:
parser
=
argparse
.
ArgumentParser
(
description
=
'Utility to upload Tor Browser to Google Drive.'
)
# if no LC specified, download all
parser
.
add_argument
(
'-l'
,
'--links'
,
default
=
None
,
help
=
'Create links file with files already uploaded and '
\
'matching the specified version. '
)
args
=
parser
.
parse_args
()
config
=
ConfigParser
.
ConfigParser
()
config
.
read
(
'drive.cfg'
)
client_id
=
config
.
get
(
'app'
,
'client-id'
)
app_secret
=
config
.
get
(
'app'
,
'secret'
)
refresh_token
=
config
.
get
(
'app'
,
'refresh_token'
)
upload_dir
=
config
.
get
(
'general'
,
'upload_dir'
)
# important: this key must be the one that signed the packages
tbb_key
=
config
.
get
(
'general'
,
'tbb_key'
)
# requests full access to drive account
OAUTH_SCOPE
=
'https://www.googleapis.com/auth/drive'
REDIRECT_URI
=
'urn:ietf:wg:oauth:2.0:oob'
print
"Authenticating..."
flow
=
OAuth2WebServerFlow
(
client_id
,
app_secret
,
OAUTH_SCOPE
,
redirect_uri
=
REDIRECT_URI
)
# If no valid token found, need to prompt user.
# this should only occur once
if
not
refresh_token
:
flow
.
params
[
'access_type'
]
=
'offline'
flow
.
params
[
'approval_prompt'
]
=
'force'
authorize_url
=
flow
.
step1_get_authorize_url
()
print
'Go to the following link in your browser: '
+
authorize_url
code
=
raw_input
(
'Enter verification code: '
).
strip
()
try
:
credentials
=
flow
.
step2_exchange
(
code
)
except
FlowExchangeError
as
e
:
print
str
(
e
)
# oauth2 credentials instance must be stored as json string
config
.
set
(
'app'
,
'refresh_token'
,
credentials
.
to_json
())
with
open
(
'drive.cfg'
,
'wb'
)
as
configfile
:
config
.
write
(
configfile
)
else
:
# we already have a valid token
credentials
=
Credentials
.
new_from_json
(
refresh_token
)
# authenticate with oauth2
http
=
httplib2
.
Http
()
http
=
credentials
.
authorize
(
http
)
# initialize drive instance
drive_service
=
build
(
'drive'
,
'v2'
,
http
=
http
)
# import key fingerprint
gpg
=
gnupg
.
GPG
()
key_data
=
open
(
tbb_key
).
read
()
import_result
=
gpg
.
import_keys
(
key_data
)
fp
=
import_result
.
results
[
0
][
'fingerprint'
]
# make groups of four characters to make fingerprint more readable
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable
=
' '
.
join
(
fp
[
i
:
i
+
4
]
for
i
in
xrange
(
0
,
len
(
fp
),
4
))
try
:
# helpful when something fails but files are uploaded.
if
args
.
links
:
uploaded_files
=
get_files_links
(
drive_service
,
args
.
links
)
if
not
uploaded_files
:
raise
ValueError
(
"There are no files for that version"
)
else
:
uploaded_files
=
upload_files
(
drive_service
,
upload_dir
)
# use default config
core
=
gettor
.
core
.
Core
(
'/home/gettor/core.cfg'
)
# erase old links
core
.
create_links_file
(
'Drive'
,
readable
)
# recognize file OS by its extension
p1
=
re
.
compile
(
'.*\.tar.xz$'
)
p2
=
re
.
compile
(
'.*\.exe$'
)
p3
=
re
.
compile
(
'.*\.dmg$'
)
p4
=
re
.
compile
(
'.*\.asc$'
)
for
file
in
uploaded_files
.
keys
():
# only run for tor browser installers
if
p4
.
match
(
file
):
continue
asc
=
"%s.asc"
%
file
abs_file
=
os
.
path
.
abspath
(
os
.
path
.
join
(
upload_dir
,
file
))
abs_asc
=
os
.
path
.
abspath
(
os
.
path
.
join
(
upload_dir
,
asc
))
sha_file
=
get_file_sha256
(
abs_file
)
# build links
link_file
=
share_file
(
drive_service
,
uploaded_files
[
file
]
)
link_asc
=
share_file
(
drive_service
,
uploaded_files
[
"%s.asc"
%
file
]
)
if
p1
.
match
(
file
):
osys
,
arch
,
lc
=
get_bundle_info
(
file
,
'linux'
)
elif
p2
.
match
(
file
):
osys
,
arch
,
lc
=
get_bundle_info
(
file
,
'windows'
)
elif
p3
.
match
(
file
):
osys
,
arch
,
lc
=
get_bundle_info
(
file
,
'osx'
)
link
=
"%s$%s$%s$"
%
(
link_file
,
link_asc
,
sha_file
)
# note that you should only upload bundles for supported locales
core
.
add_link
(
'Drive'
,
osys
,
lc
,
link
)
except
(
ValueError
,
RuntimeError
)
as
e
:
print
str
(
e
)
upload/bundles2dropbox.py
deleted
100644 → 0
View file @
63559201
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import
re
import
os
import
gnupg
import
hashlib
import
ConfigParser
import
dropbox
import
gettor.core
from
gettor.utils
import
get_bundle_info
,
get_file_sha256
,
valid_format
def
upload_files
(
basedir
,
client
):
"""Upload files to Dropbox.
Looks for files ending with 'tar.xz' inside basedir.
:param: basedir (string) path of the folder with the files to be
uploaded.
:param: client (object) DropboxClient object.
:raise: ValueError if the .xz file doesn't have an .asc file.
:raise: UploadError if something goes wrong while uploading the
files to Dropbox. All files are uploaded to '/'.
:return: (list) the names of the uploaded files.
"""
files
=
[]
for
name
in
os
.
listdir
(
basedir
):
path
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
name
))
if
os
.
path
.
isfile
(
path
)
and
valid_format
(
name
,
'linux'
):
files
.
append
(
name
)
for
name
in
os
.
listdir
(
basedir
):
path
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
name
))
if
os
.
path
.
isfile
(
path
)
and
valid_format
(
name
,
'windows'
):
files
.
append
(
name
)
for
name
in
os
.
listdir
(
basedir
):
path
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
name
))
if
os
.
path
.
isfile
(
path
)
and
valid_format
(
name
,
'osx'
):
files
.
append
(
name
)
for
file
in
files
:
asc
=
"%s.asc"
%
file
abs_file
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
file
))
abs_asc
=
os
.
path
.
abspath
(
os
.
path
.
join
(
basedir
,
asc
))
if
not
os
.
path
.
isfile
(
abs_asc
):
# there are some .mar files that don't have .asc, don't upload it
continue
# chunk upload for big files
to_upload
=
open
(
abs_file
,
'rb'
)
size
=
os
.
path
.
getsize
(
abs_file
)
uploader
=
client
.
get_chunked_uploader
(
to_upload
,
size
)
while
uploader
.
offset
<
size
:
try
:
upload
=
uploader
.
upload_chunked
()
except
dropbox
.
rest
.
ErrorResponse
,
e
:
print
(
"An error ocurred while uploading %s: %s"
%
abs_file
,
e
)
uploader
.
finish
(
file
)
print
"Uploading %s"
%
file
# this should be small, upload it simple
to_upload_asc
=
open
(
abs_asc
,
'rb'
)
response
=
client
.
put_file
(
asc
,
to_upload_asc
)
print
"Uploading %s"
%
asc
return
files
if
__name__
==
'__main__'
:
config
=
ConfigParser
.
ConfigParser
()
config
.
read
(
'dropbox.cfg'
)
app_key
=
config
.
get
(
'app'
,
'key'
)
app_secret
=
config
.
get
(
'app'
,
'secret'
)
access_token
=
config
.
get
(
'app'
,
'access_token'
)
upload_dir
=
config
.
get
(
'general'
,
'upload_dir'
)
# important: this key must be the one that signed the packages
tbb_key
=
config
.
get
(
'general'
,
'tbb_key'
)
client
=
dropbox
.
client
.
DropboxClient
(
access_token
)
# import key fingerprint
gpg
=
gnupg
.
GPG
()
key_data
=
open
(
tbb_key
).
read
()
import_result
=
gpg
.
import_keys
(
key_data
)
fp
=
import_result
.
results
[
0
][
'fingerprint'
]
# make groups of four characters to make fingerprint more readable
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable
=
' '
.
join
(
fp
[
i
:
i
+
4
]
for
i
in
xrange
(
0
,
len
(
fp
),
4
))
try
:
uploaded_files
=
upload_files
(
upload_dir
,
client
)
# use default config
core
=
gettor
.
core
.
Core
(
'/home/gettor/core.cfg'
)
# erase old links
core
.
create_links_file
(
'Dropbox'
,
readable
)
# recognize file OS by its extension
p1
=
re
.
compile
(
'.*\.tar.xz$'
)
p2
=
re
.
compile
(
'.*\.exe$'
)
p3
=
re
.
compile
(
'.*\.dmg$'
)
for
file
in
uploaded_files
:
# build file names
asc
=
"%s.asc"
%
file
abs_file
=
os
.
path
.
abspath
(
os
.
path
.
join
(
upload_dir
,
file
))
abs_asc
=
os
.
path
.
abspath
(
os
.
path
.
join
(
upload_dir
,
asc
))
sha_file
=
get_file_sha256
(
abs_file
)
# build links
link_file
=
client
.
share
(
file
,
short_url
=
False
)
# if someone finds how to do this with the API, please tell me!
link_file
[
u
'url'
]
=
link_file
[
u
'url'
].
replace
(
'?dl=0'
,
'?dl=1'
)
link_asc
=
client
.
share
(
asc
,
short_url
=
False
)
link_asc
[
u
'url'
]
=
link_asc
[
u
'url'
].
replace
(
'?dl=0'
,
'?dl=1'
)
if
p1
.
match
(
file
):
osys
,
arch
,
lc
=
get_bundle_info
(
file
,
'linux'
)
elif
p2
.
match
(
file
):
osys
,
arch
,
lc
=
get_bundle_info
(
file
,
'windows'
)
elif
p3
.
match
(
file
):
osys
,
arch
,
lc
=
get_bundle_info
(
file
,
'osx'
)
link
=
"%s$%s$%s$"
%
(
link_file
[
u
'url'
],
link_asc
[
u
'url'
],
sha_file
)
# note that you should only upload bundles for supported locales
core
.
add_link
(
'Dropbox'
,
osys
,
lc
,
link
)
except
(
ValueError
,
RuntimeError
)
as
e
:
print
str
(
e
)
except
dropbox
.
rest
.
ErrorResponse
as
e
:
print
str
(
e
)
upload/bundles2github.py
deleted
100644 → 0
View file @
63559201
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2015, The Tor Project, Inc.
# (c) 2015, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
#
# Use pyopenssl to verify TLS certifcates
try
:
import
urllib3.contrib.pyopenssl
urllib3
.
contrib
.
pyopenssl
.
inject_into_urllib3
()
except
ImportError
:
pass
import
os
import
sys
import
argparse
import
ConfigParser
import
gnupg
import
github3
import
gettor.core
from
gettor.utils
import
(
get_bundle_info
,
get_file_sha256
,
find_files_to_upload
)
def
upload_new_release
(
github_repo
,
version
,
upload_dir
):
"""
Returns a Release object
"""
# Create a new release of this TBB
release
=
target_repo
.
create_release
(
'v{}'
.
format
(
version
),
target_commitish
=
"master"
,
name
=
'Tor Browser {}'
.
format
(
version
),
body
=
''
,
draft
=
True
,
)
for
filename
in
find_files_to_upload
(
upload_dir
):
# Upload each file for this release
file_path
=
os
.
path
.
join
(
upload_dir
,
filename
)
print
(
"Uploading file {}"
.
format
(
filename
))
release
.
upload_asset
(
'application/zip'
,
filename
,
open
(
file_path
,
'rb'
))
return
release
if
__name__
==
'__main__'
:
parser
=
argparse
.
ArgumentParser
(
description
=
'Utility to upload Tor Browser to Github.'
)
# with this we only get the links of files already uploaded
# useful when somethings fail after uploading
parser
.
add_argument
(
'-l'
,
'--links'
,
default
=
None
,
help
=
'Create links file with files already uploaded.'
)
args
=
parser
.
parse_args
()
config
=
ConfigParser
.
ConfigParser
()
config
.
read
(
'github.cfg'
)
tbb_version_path
=
config
.
get
(
'general'
,
'version_cfg_path'
)
tbb_version_config
=
ConfigParser
.
ConfigParser
()
tbb_version_config
.
read
(
tbb_version_path
)
version
=
tbb_version_config
.
get
(
'version'
,
'current'
)
# the token allow us to run this script without GitHub user/pass
github_access_token
=
config
.
get
(
'app'
,
'access_token'
)
# path to the fingerprint that signed the packages
tb_key
=
config
.
get
(
'general'
,
'tbb_key_path'
)
# path to the latest version of Tor Browser
tb_path
=
config
.
get
(
'general'
,
'latest_path'
)
# path to gettor code configuration
core_path
=
config
.
get
(
'general'
,
'core_path'
)
# user and repository where we upload Tor Browser
github_user
=
config
.
get
(
'app'
,
'user'
)
github_repo
=
config
.
get
(
'app'
,
'repo'
)
gh
=
github3
.
login
(
token
=
github_access_token
)
target_repo
=
gh
.
repository
(
github_user
,
github_repo
)
# import key fingerprint
gpg
=
gnupg
.
GPG
()
key_data
=
open
(
tb_key
).
read
()
import_result
=
gpg
.
import_keys
(
key_data
)
fp
=
import_result
.
results
[
0
][
'fingerprint'
]
# make groups of four characters to make fingerprint more readable
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable_fp
=
' '
.
join
(
fp
[
i
:
i
+
4
]
for
i
in
xrange
(
0
,
len
(
fp
),
4
))
# Find any published releases with this version number
for
release
in
target_repo
.
releases
():
if
release
.
tag_name
==
'v{}'
.
format
(
version
)
and
not
release
.
draft
:
print
(
"Found an existing published release with this version. "
"Not uploading again unless you delete the published "
"release '{}'."
.
format
(
release
.
tag_name
))
break
else
:
release
=
None
if
args
.
links
or
release
:
# Only generating link file, should use previously published release
if
not
release
:
print
(
"Error occured! Could not find a published release for "
"version {}"
.
format
(
version
))
sys
.
exit
(
1
)
else
:
# Remove any drafts to clean broken uploads
print
(
'Uploading release, please wait, this might take a while!'
)
# Upload the latest browser bundles to a new release
release
=
upload_new_release
(
target_repo
,
version
,
tb_path
)
# Upload success, publish the release
release
.
edit
(
draft
=
False
)
# Create the links file for this release
core
=
gettor
.
core
.
Core
(
core_path
)
# Erase old links if any and create a new empty one
core
.
create_links_file
(
'GitHub'
,
readable_fp
)
print
(
"Creating links file"
)
for
asset
in
release
.
assets
():
url
=
asset
.
browser_download_url
if
url
.
endswith
(
'.asc'
):
continue
osys
,
arch
,
lc
=
get_bundle_info
(
asset
.
name
)
sha256
=
get_file_sha256
(
os
.
path
.
abspath
(
os
.
path
.
join
(
tb_path
,
asset
.
name
))
)