update_git 5.14 KB
Newer Older
1
2
#!/usr/bin/env python3

3
4
5
# Dependencies:
#  pip3 install PyGithub
#  pip3 install python-gitlab
6

7
8
9
# Before running, place the authentication token for a user with push access to
# the Github repository in an environment variable 'GITHUB_AUTH' and a token to
# access the GitLab repository in an environment variable 'GITLAB_AUTH'
10
11

from github import Github
12
from gitlab import Gitlab
13
14
15
16
17
18

import sys
import json
import urllib
import subprocess
import os
19
import base64
20

Cecylia Bocovich's avatar
Cecylia Bocovich committed
21
REPO_NAME = "torproject/torbrowser-releases"
22
23
24
25
26

failure = False

failed_uploads = []

27
28
29
30
31
32
33
34
35
36
37
class GitlabRemote:
    def __init__(self, token):
        self.gl = Gitlab("https://gitlab.com", private_token=token)

    def delete_old_projects(self):
        projects = self.gl.projects.list(owned=True)
        for project in projects:
            project.delete()

    def create_projects(self):
        # Create one project for each platform
38
39
40
41
42
43
        windows_repo = self.gl.projects.create({'name': 'torbrowser-windows',
            'visibility': 'public'})
        mac_repo = self.gl.projects.create({'name': 'torbrowser-osx',
            'visibility': 'public'})
        linux_repo = self.gl.projects.create({'name': 'torbrowser-linux',
            'visibility': 'public'})
44
45
46
47
48
49
50
51
52

        self.repositories = {
            "linux32" : linux_repo,
            "linux64" : linux_repo,
            "win32" : windows_repo,
            "win64" : windows_repo,
            "osx64" : mac_repo
        }

53
54
55
56
57
58
59
60
61
62
63
64

    def upload(self, filename, arch):
        with open(filename, 'rb') as f:
            contents = f.read()
        data = {
            'branch': 'master',
            'commit_message': 'GetTor automated commit - update with new Tor Browser downloads',
            'actions': [
                {
                    # Binary files need to be base64 encoded
                    'action': 'create',
                    'file_path': filename,
65
                    'content': base64.b64encode(contents).decode("ascii"),
66
67
68
69
70
                    'encoding': 'base64',
                }
            ]
        }

71
        self.repositories[arch].commits.create(data)
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

class GithubRemote:
    def __init__(self, token):
        g = Github(token)
        self.repo = g.get_repo(REPO_NAME)

    def delete_old_releases(self):
        for release in self.repo.get_releases():
            for asset in release.get_assets():
                asset.delete_asset()
            release.delete_release()

    def create_release(self):
        self.release = self.repo.create_git_release("torbrowser-release", "Tor Browser releases", "These releases were uploaded to be distributed with gettor.")

    def upload(self, filename, arch):
        self.release.upload_asset(filename)
89

90
#Download list of tor browser releases and upload them to remote repositories
91
92
def upload_files(remotes):
    if len(remotes) == 0:
93
        print("Error: No remotes to update", file=sys.stderr)
94
95
        return 1

96
97
98
99
100
101
102
103
104
    url = urllib.request.urlopen("https://aus1.torproject.org/torbrowser/update_3/release/downloads.json")
    data = json.loads(url.read().decode())
    for arch in data['downloads']:
        for locale in data['downloads'][arch]:
            for asset in data['downloads'][arch][locale]:
                url = data['downloads'][arch][locale][asset]
                filename = url.split('/')[-1]
                try:
                    subprocess.check_call(["/usr/bin/wget", "--quiet", url])
105

106
                except:
107
108
                    print("Error: failed to fetch "+url+". Will retry later.",
                            file=sys.stderr)
109
                    failed_uploads.append(url)
110
111
112
113
114
                    continue

                for remote in remotes:
                    remote.upload(filename, arch)
                os.remove(filename)
115
116
117
    
    failure = False

118
119
120
121
122
    #Retry failed uploads
    for url in failed_uploads:
        filename = url.split('/')[-1]
        try:
            subprocess.check_call(["/usr/bin/wget", "--quiet", url])
123
124
            for remote in remotes:
                remote.upload(filename, arch)
125
126
            os.remove(filename)
        except:
127
128
            print("Error: failed to update "+url+". Please upload this file manually.",
                    file=sys.stderr)
129
130
131
            failure = True

    if failure:
132
        return 1
133
134


135
136
def main():
    remotes = []
137
    if 'GITHUB_AUTH' not in os.environ:
138
139
        print("WARNING: No Github authentication token given", file=sys.stderr)
        print("The authentication token for github should be placed in the environment"
140
                "variable 'GITHUB_AUTH'", file=sys.stderr)
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    else:
        github = GithubRemote(os.environ['GITHUB_AUTH'])
        github.delete_old_releases()
        github.create_release()
        remotes.append(github)

    if 'GITLAB_AUTH' not in os.environ:
        print("WARNING: No GitLab authenticatin token given", file=sys.stderr)
        print("The authentication token for gitlab should be placed in the environment"
                "variable 'GITLAB_AUTH'", file=sys.stderr)
    else:
        gitlab = GitlabRemote(os.environ['GITLAB_AUTH'])
        gitlab.delete_old_projects()
        gitlab.create_projects()
        remotes.append(gitlab)

    return upload_files(remotes)

if __name__ == "__main__":
    sys.exit(main())