bundles2dropbox.py 7.77 KB
Newer Older
1
2
# -*- coding: utf-8 -*-
#
3
# This file is part of GetTor, a Tor Browser distribution system.
4
#
5
6
7
8
9
10
11
12
# :authors: Israel Leiva <ilv@riseup.net>
#           see also AUTHORS file
#
# :copyright:   (c) 2008-2014, The Tor Project, Inc.
#               (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.

13
14
import re
import os
15
import gnupg
16
import hashlib
17
18
import ConfigParser

19
import dropbox
20
import gettor.core
21

22
def valid_format(file, osys):
23
    """Check for valid bundle format
24

25
26
    Check if the given file has a valid bundle format
    (e.g. tor-browser-linux32-3.6.2_es-ES.tar.xz)
27

28
    :param: file (string) the name of the file.
29
    :param: osys (string) the OS.
30
31
32
33

    :return: (boolean) true if the bundle format is valid, false otherwise.

    """
34
    if(osys == 'windows'):
35
36
37
        m = re.search(
            'torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe',
            file)
38
39
40
41
42
    elif(osys == 'linux'):
        m = re.search(
            'tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
            file)
    elif(osys == 'osx'):
43
        m = re.search(
44
            'TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg',     
45
            file)
46
47
48
49
50
51
    if m:
        return True
    else:
        return False


52
def get_bundle_info(file, osys):
53
    """Get the os, arch and lc from a bundle string.
54

55
    :param: file (string) the name of the file.
56
    :param: osys (string) the OS.
57
58
59
60

    :raise: ValueError if the bundle doesn't have a valid bundle format.

    :return: (list) the os, arch and lc.
61
62

    """
63
    if(osys == 'windows'):
64
65
66
67
68
        m = re.search(
            'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe',
            file)
        if m:
            lc = m.group(1)
69
            return 'windows', '32/64', lc
70
71
        else:
            raise ValueError("Invalid bundle format %s" % file)
72
73
74
75
76
77
78
79
80
81
82
    elif(osys == 'linux'):
        m = re.search(
            'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
            file)
        if m:
            arch = m.group(1)
            lc = m.group(2)
            return 'linux', arch, lc
        else:
            raise ValueError("Invalid bundle format %s" % file)
    elif(osys == 'osx'):
83
        m = re.search(
84
            'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg',
85
86
            file)
        if m:
87
88
89
90
            os = 'osx'
            arch = m.group(1)
            lc = m.group(2)
            return 'osx', arch, lc
91
92
        else:
            raise ValueError("Invalid bundle format %s" % file)
93

94

95
def get_file_sha256(file):
96
97
98
99
100
101
102
103
    """Get the sha256 of a file.

    :param: file (string) the path of the file.

    :return: (string) the sha256 hash.

    """
    # as seen on the internetz
104
    BLOCKSIZE = 65536
105
    hasher = hashlib.sha256()
106
107
108
109
110
111
    with open(file, 'rb') as afile:
        buf = afile.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(BLOCKSIZE)
    return hasher.hexdigest()
112

113

114
def upload_files(basedir, client):
115
    """Upload files to Dropbox.
116

117
    Looks for files ending with 'tar.xz' inside basedir.
118

119
120
121
122
123
124
125
126
127
    :param: basedir (string) path of the folder with the files to be
            uploaded.
    :param: client (object) DropboxClient object.

    :raise: ValueError if the .xz file doesn't have an .asc file.
    :raise: UploadError if something goes wrong while uploading the
            files to Dropbox. All files are uploaded to '/'.

    :return: (list) the names of the uploaded files.
128
129
130
131

    """
    files = []

132
    p = re.compile('.*\.tar.xz$')
133
134
    for name in os.listdir(basedir):
        path = os.path.abspath(os.path.join(basedir, name))
135
136
        if os.path.isfile(path) and p.match(path)\
        and valid_format(name, 'linux'):
137
138
            files.append(name)

139
140
141
142
    p = re.compile('.*\.exe$')
    for name in os.listdir(basedir):
        path = os.path.abspath(os.path.join(basedir, name))
        if os.path.isfile(path) and p.match(path)\
143
144
145
146
147
148
149
150
        and valid_format(name, 'windows'):
            files.append(name)

    p = re.compile('.*\.dmg$')
    for name in os.listdir(basedir):
        path = os.path.abspath(os.path.join(basedir, name))
        if os.path.isfile(path) and p.match(path)\
        and valid_format(name, 'osx'):
151
152
            files.append(name)

153
    for file in files:
154
        asc = "%s.asc" % file
155
156
157
158
        abs_file = os.path.abspath(os.path.join(basedir, file))
        abs_asc = os.path.abspath(os.path.join(basedir, asc))

        if not os.path.isfile(abs_asc):
159
160
            # there are some .mar files that don't have .asc, don't upload it
            continue
161

162
        # chunk upload for big files
163
164
165
166
167
168
        to_upload = open(abs_file, 'rb')
        size = os.path.getsize(abs_file)
        uploader = client.get_chunked_uploader(to_upload, size)
        while uploader.offset < size:
            try:
                upload = uploader.upload_chunked()
Sukhbir Singh's avatar
Sukhbir Singh committed
169
            except dropbox.rest.ErrorResponse, e:
170
                print("An error ocurred while uploading %s: %s" % abs_file, e)
171
        uploader.finish(file)
172
        print "Uploading %s" % file
173

174
        # this should be small, upload it simple
175
176
        to_upload_asc = open(abs_asc, 'rb')
        response = client.put_file(asc, to_upload_asc)
177
        print "Uploading %s" % asc
178
179
180

    return files

181
if __name__ == '__main__':
182
183
184
185
186
187
188
    config = ConfigParser.ConfigParser()
    config.read('dropbox.cfg')

    app_key = config.get('app', 'key')
    app_secret = config.get('app', 'secret')
    access_token = config.get('app', 'access_token')
    upload_dir = config.get('general', 'upload_dir')
189

190
191
    # important: this key must be the one that signed the packages
    tbb_key = config.get('general', 'tbb_key')
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207

    client = dropbox.client.DropboxClient(access_token)

    # import key fingerprint
    gpg = gnupg.GPG()
    key_data = open(tbb_key).read()
    import_result = gpg.import_keys(key_data)
    fp = import_result.results[0]['fingerprint']

    # make groups of four characters to make fingerprint more readable
    # e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
    readable = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))

    try:
        uploaded_files = upload_files(upload_dir, client)
        # use default config
208
209
        core = gettor.core.Core('/home/gettor/core.cfg')

210
211
212
        # erase old links
        core.create_links_file('Dropbox', readable)

213
214
215
216
        # recognize file OS by its extension
        p1 = re.compile('.*\.tar.xz$')
        p2 = re.compile('.*\.exe$')
        p3 = re.compile('.*\.dmg$')
217

218
        for file in uploaded_files:
219
            # build file names
220
            asc = "%s.asc" % file
221
222
            abs_file = os.path.abspath(os.path.join(upload_dir, file))
            abs_asc = os.path.abspath(os.path.join(upload_dir, asc))
223

224
            sha_file = get_file_sha256(abs_file)
225

226
            # build links
227
            link_file = client.share(file, short_url=False)
228
            # if someone finds how to do this with the API, please tell me!
229
            link_file[u'url'] = link_file[u'url'].replace('?dl=0', '?dl=1')
230
            link_asc = client.share(asc, short_url=False)
231
            link_asc[u'url'] = link_asc[u'url'].replace('?dl=0', '?dl=1')
232
233
234
235
236
237
            if p1.match(file):
                osys, arch, lc = get_bundle_info(file, 'linux')
            elif p2.match(file):
                osys, arch, lc = get_bundle_info(file, 'windows')
            elif p3.match(file):
                osys, arch, lc = get_bundle_info(file, 'osx')
238
239
240
241
242
243

            link = "Package (%s-bit): %s\nASC signature (%s-bit): %s\n"\
                   "Package SHA256 checksum (%s-bit): %s\n" %\
                   (arch, link_file[u'url'], arch, link_asc[u'url'],
                    arch, sha_file)

244
            # note that you should only upload bundles for supported locales
245
            core.add_link('Dropbox', osys, lc, link)
246
247
248
249
    except (ValueError, RuntimeError) as e:
        print str(e)
    except dropbox.rest.ErrorResponse as e:
        print str(e)