Merge branch 'develop'

gettor/core.py

@@ -253,43 +253,49 @@ class Core(object):
                     t32, t64 = [t for t in providers[pname].split(",") if t]
                     
                     link, signature, chs32 = [l for l in t32.split("$") if l]
+                    link = "%s: %s" % (pname, link)
                     links32[link] = signature
 
                     link, signature, chs64 = [l for l in t64.split("$") if l]
+                    link = "%s: %s" % (pname, link.lstrip())
                     links64[link] = signature
                     
                 else:
                     link, signature, chs32 = [l for l in providers[pname].split("$") if l]
+                    link = "%s: %s" % (pname, link)
                     links32[link] = signature
                     
                 #providers[pname] = providers[pname].replace(",", "")
                 #providers[pname] = providers[pname].replace("$", "\n\n")
 
+                ### We will improve and add the verification section soon ###
                 # all packages are signed with same key
                 # (Tor Browser developers)
-                fingerprint = config.get('key', 'fingerprint')
+                # fingerprint = config.get('key', 'fingerprint')
                 # for now, english messages only
-                fingerprint_msg = self._get_msg('fingerprint', 'en')
-                fingerprint_msg = fingerprint_msg % fingerprint
+                # fingerprint_msg = self._get_msg('fingerprint', 'en')
+                # fingerprint_msg = fingerprint_msg % fingerprint
             except ConfigParser.Error as e:
                 raise InternalError("%s" % str(e))
 
         # create the final links list with all providers
         all_links = []
 
-        msg = "Tor Browser %s-bit:" % arch
+        msg = "Tor Browser %s-bit:\n" % arch
         for link in links32:
             msg = "%s\n%s" % (msg, link)
             
         all_links.append(msg)
         
         if osys == 'linux':
-            msg = "\n\n\nTor Browser 64-bit:"
+            msg = "\n\n\nTor Browser 64-bit:\n"
             for link in links64:
                 msg = "%s\n%s" % (msg, link)
         
             all_links.append(msg)
         
+        ### We will improve and add the verification section soon ###
+        """
         msg = "\n\n\nTor Browser's signature %s-bit:" %\
               arch
         for link in links32:
@@ -312,6 +318,9 @@ class Core(object):
             msg = "SHA256 of Tor Browser 64-bit (advanced): %s\n" % chs64
             all_links.append(msg)
 
+        """
+        ### end verification ###
+
         """
         for key in providers.keys():
             # get more friendly description of the provider
@@ -328,8 +337,9 @@ class Core(object):
                 raise InternalError("%s" % str(e))
         """
 
+        ### We will improve and add the verification section soon ###
         # add fingerprint after the links
-        all_links.append(fingerprint_msg)
+        # all_links.append(fingerprint_msg)
 
         if all_links:
             return "".join(all_links)

gettor/http.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of GetTor.
+#
+# :authors: Israel Leiva <ilv@torproject.org>
+#           see also AUTHORS file
+#
+# :copyright:   (c) 2008-2015, The Tor Project, Inc.
+#               (c) 2015, Israel Leiva
+#
+# :license: This is Free Software. See LICENSE for license information.
+
+import os
+import re
+import json
+import codecs
+import urllib2
+import ConfigParser
+
+from time import gmtime, strftime
+
+import core
+import utils
+
+"""GetTor RESTful API"""
+
+# currently supported locales for Tor Browser
+LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl',
+      'pt-PT', 'ru', 'tr', 'vi', 'zh-CN']
+
+# https://gitweb.tpo/tor-browser-spec.git/tree/processes/VersionNumbers
+# does not say anything about operating systems, so it's possible the
+# notation might change in the future. We should always use the same three
+# strings though: linux, windows, osx.
+OS = {
+    'Linux': 'linux',
+    'Windows': 'windows',
+    'MacOS': 'osx'
+}
+
+# based on
+# https://gitweb.tpo.org/tor-browser-spec.git/tree/processes/VersionNumbers
+# except for the first one, which is based on current RecommendedTBBVersions
+RE = {
+    'os': '(.*)-(\w+)',
+    'alpha': '\d\.\d(\.\d)*a\d+',
+    'beta': '\d\.\d(\.\d)*b\d+',
+    'stable': '\d\.\d(\.\d)*'
+}
+
+# strings to build names of packages depending on OS.
+PKG = {
+    'windows': 'torbrowser-install-%s_%s.exe',
+    'linux': 'tor-browser-linux%s-%s_%s.tar.xz',
+    'osx': 'TorBrowser-%s-osx64_%s.dmg'
+}
+
+# bin and asc are used to build the download links for each version, os and lc
+URL = {
+    'version': 'https://www.torproject.org/projects/torbrowser/RecommendedTBBVersions',
+    'bin': 'https://www.torproject.org/dist/torbrowser/%s/%s',
+    'asc': 'https://www.torproject.org/dist/torbrowser/%s/%s.asc'
+}
+
+
+
+class ConfigError(Exception):
+    pass
+
+
+class InternalError(Exception):
+    pass
+
+
+class HTTP(object):
+    """ Provide useful resources via RESTful API. """
+    def __init__(self, cfg=None):
+        """ Create new object by reading a configuration file.
+
+        :param: cfg (string) path of the configuration file.
+
+        """
+        default_cfg = 'http.cfg'
+        config = ConfigParser.ConfigParser()
+
+        if cfg is None or not os.path.isfile(cfg):
+            cfg = default_cfg
+
+        try:
+            with open(cfg) as f:
+                config.readfp(f)
+        except IOError:
+            raise ConfigError("File %s not found!" % cfg)
+
+        try:
+            # path to static tree of API
+            self.tree = config.get('general', 'tree')
+            # server that provides the RESTful API
+            self.server = config.get('general', 'url')
+            # path to the links files
+            self.links_path = config.get('general', 'links')
+            # path to mirrors in json
+            self.mirrors_path = config.get('general', 'mirrors')
+
+            # we will ask gettor.core for the links
+            core_cfg = config.get('general', 'core')
+            self.core = core.Core(core_cfg)
+
+        except ConfigParser.Error as e:
+            raise ConfigError("Configuration error: %s" % str(e))
+        except core.ConfigError as e:
+            raise InternalError("HTTP error: %s" % str(e))
+
+    def _is_json(self, my_json):
+        """ Check if json generated is valid.
+
+        :param: my_json (string) data to ve verified.
+
+        :return: (bool) true if data is json-valid, false otherwise.
+
+        """
+        try:
+            json_object = json.loads(my_json)
+        except ValueError, e:
+            return False
+        return True
+    
+    def _write_json(self, path, content):
+        """
+        """
+        try:
+            with codecs.open(
+                path,
+                "w",
+                "utf-8"
+            ) as jsonfile:
+                # Make pretty json
+                json.dump(
+                    content,
+                    jsonfile,
+                    sort_keys=True,
+                    indent=4,
+                    separators=(',', ': '),
+                    encoding="utf-8",
+                )
+        except IOError as e:
+            #logging.error("Couldn't write json: %s" % str(e))
+            print "Error building %s: %s" % (path, str(e))
+        print "%s built" % path
+
+    def _get_provider_name(self, p):
+        """ Return simplified version of provider's name.
+
+        :param: p (string) provider's name.
+
+        :return: (string) provider's name in lowercase and without spaces.
+
+        """
+        p = p.replace(' ', '-')
+        return p.lower()
+
+    def _add_links(self, lv, release, version, os):
+        """ Add link for all locales in LC depending on given OS.
+
+        :param: lv (dict) latest version data structure.
+        :param: release (string) release to which add the links.
+        :param: version (string) version obtained from tpo.
+        :param: os (string) operating system.
+
+        """
+        for lc in LC:
+            if os == 'linux':
+                pkg32 = PKG['linux'] % ('32', version, lc)
+                link_bin32 = URL['bin'] % (version, pkg32)
+                link_asc32 = URL['asc'] % (version, pkg32)
+
+                pkg64 = PKG['linux'] % ('64', version, lc)
+                link_bin64 = URL['bin'] % (version, pkg64)
+                link_asc64 = URL['asc'] % (version, pkg64)
+
+                lv[release]['downloads'][os][lc] = {
+                    'binary32': link_bin32,
+                    'signature32': link_asc32,
+                    'binary64': link_bin64,
+                    'signature64': link_asc64,
+                }
+            else:
+                if os == 'windows':
+                    pkg = PKG['windows'] % (version, lc)
+
+                elif os == 'osx':
+                    pkg = PKG['osx'] % (version, lc)
+
+                else:
+                    continue
+
+                link_bin = URL['bin'] % (version, pkg)
+                link_asc = URL['asc'] % (version, pkg)
+                lv[release]['downloads'][os][lc] = {
+                    'binary': link_bin,
+                    'signature': link_asc
+                }
+
+    def _load_latest_version(self):
+        """ Load latest version data. """
+        response = urllib2.urlopen(URL['version'])
+        json_response = json.load(response)
+
+        lv = {
+            'stable': {
+                'latest_version': '',
+                'downloads': {}
+            },
+            'alpha': {
+                'latest_version': '',
+                'downloads': {}
+            },
+            'beta': {
+                'latest_version': '',
+                'downloads': {}
+            }
+        }
+
+        self.releases = {
+            'alpha': '%s/latest/alpha' % self.server,
+            'beta': '%s/latest/beta' % self.server,
+            'stable': '%s/latest/stable' % self.server,
+            'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime())
+        }
+
+        # one iteration to find the latest version for each release
+        for v in json_response:
+            # latest version for each release
+            if not re.match(RE['os'], v):
+                if re.match(RE['alpha'], v):
+                    if v > lv['alpha']['latest_version']:
+                        # we'll use the latest one
+                        lv['alpha']['latest_version'] = v
+
+                elif re.match(RE['beta'], v):
+                    if v > lv['beta']['latest_version']:
+                        # we'll use the latest one
+                        lv['beta']['latest_version'] = v
+
+                elif re.match(RE['stable'], v):
+                    if v > lv['stable']['latest_version']:
+                        # we'll use the latest one
+                        lv['stable']['latest_version'] = v
+
+        latest_alpha = lv['alpha']['latest_version']
+        latest_beta = lv['beta']['latest_version']
+        latest_stable = lv['stable']['latest_version']
+
+        # another iteration to add the links
+        for v in json_response:
+            # based on current RecommendedTBBVersions scheme
+            # for each release and for each os we build links for all locales
+            if re.match(RE['os'], v):
+                m = re.match(RE['os'], v)
+                version = m.group(1)
+                osys = m.group(2)
+
+                if osys in OS:
+                    if latest_alpha and version == latest_alpha \
+                            and re.match(RE['alpha'], version):
+                        lv['alpha']['downloads'][OS[osys]] = {}
+                        self._add_links(lv, 'alpha', version, OS[osys])
+
+                    elif latest_beta and version == latest_beta \
+                            and re.match(RE['beta'], version):
+                        lv['beta']['downloads'][OS[osys]] = {}
+                        self._add_links(lv, 'beta', version, OS[osys])
+
+                    elif latest_stable and version == latest_stable \
+                            and re.match(RE['stable'], version):
+                        lv['stable']['downloads'][OS[osys]] = {}
+                        self._add_links(lv, 'stable', version, OS[osys])
+
+        lv['updated_at'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
+        self.lv = lv
+
+    def _load_links(self):
+        """ Load links and providers data. """
+        links_files = []
+
+        # look for files ending with .links in links_path
+        p = re.compile('.*\.links$')
+        for name in os.listdir(self.links_path):
+            path = os.path.abspath(os.path.join(self.links_path, name))
+            if os.path.isfile(path) and p.match(path):
+                links_files.append(path)
+
+        links = {}
+        providers = {}
+        supported_os = self.core.get_supported_os()
+        supported_lc = self.core.get_supported_lc()
+
+        for name in links_files:
+            config = ConfigParser.ConfigParser()
+            try:
+                with open(name) as f:
+                    config.readfp(f)
+            except IOError:
+                raise InternalError("File %s not found!" % name)
+
+            try:
+                pname = config.get('provider', 'name')
+                pname = self._get_provider_name(pname)
+
+                # build providers dict
+                providers[pname] = '%s/providers/%s' % (self.server, pname)
+                providers['updated_at'] = strftime(
+                    "%Y-%m-%d %H:%M:%S", gmtime()
+                )
+
+                self.providers = providers
+                links[pname] = {}
+
+                # build links data.
+                for osys in supported_os:
+                    links[pname][osys] = {}
+                    for lc in supported_lc:
+                        links[pname][osys][lc] = {}
+
+                for osys in supported_os:
+                    for lc in supported_lc:
+                        l_str = config.get(osys, lc)
+
+                        # linux has 32 and 64 bit packages
+                        if osys == 'linux':
+                            l32_str, l64_str = l_str.split(',')
+
+                            link32, sig32, sha32 = [
+                                l for l in l32_str.split("$") if l
+                            ]
+
+                            link64, sig64, sha64 = [
+                                l for l in l64_str.split("$") if l
+                            ]
+                            link64 = link64.lstrip()
+
+                            links[pname][osys][lc]['binary32'] = link32
+                            links[pname][osys][lc]['signature32'] = sig32
+                            links[pname][osys][lc]['sha256-32'] = sha32
+                            links[pname][osys][lc]['binary64'] = link64
+                            links[pname][osys][lc]['signature64'] = sig64
+                            links[pname][osys][lc]['sha256-64'] = sha64
+
+                        else:
+                            link, sig, sha = [l for l in l_str.split("$") if l]
+                            links[pname][osys][lc]['binary'] = link
+                            links[pname][osys][lc]['signature'] = sig
+                            links[pname][osys][lc]['sha256'] = sha
+
+            except ConfigParser.Error as e:
+                raise InternalError("%s" % str(e))
+
+        links['updated_at'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
+        self.links = links
+
+    def _load_mirrors(self):
+        """ Load mirrors data. """
+        mirrors = []
+
+        # json of mirrors should be obtained from get_mirrors.py
+        json_data = open(self.mirrors_path).read()
+        mirrors = json.loads(json_data)
+
+        self.mirrors = mirrors
+
+    def _load_resources(self):
+        """ Load available resources data. """
+
+        self.resources = {
+            'providers': '%s/providers' % self.server,
+            'mirrors': '%s/mirrors' % self.server,
+            'latest_version': '%s/latest' % self.server,
+            'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime())
+        }
+
+    def load_data(self):
+        """ Load all data.
+
+        Since data is not frequently updated, we load all data before
+        running the RESTful API. Every time the links/mirrors/version
+        data is updated we should restart the API.
+
+        """
+        self._load_links()
+        self._load_mirrors()
+        self._load_resources()
+        self._load_latest_version()
+
+    def build(self):
+        """ Build RESTful API. """
+        
+        print "Building API"
+        
+        # resources
+        self._write_json(
+            os.path.join(self.tree, 'api'),
+            self.resources
+        )
+
+        api_path = os.path.join(self.tree, 'api-content')
+        if not os.path.isdir(api_path):
+            os.mkdir(api_path)
+        
+        # providers
+        self._write_json(
+            os.path.join(api_path, 'providers'),
+            self.providers
+        )
+
+        providers_path = os.path.join(api_path, 'providers-content')
+        if not os.path.isdir(providers_path):
+            os.mkdir(providers_path)
+
+        for provider in self.links:
+            if provider == 'updated_at':
+                continue
+
+            self._write_json(
+                os.path.join(providers_path, provider),
+                self.links[provider]
+            )
+
+            provider_path = os.path.join(
+                providers_path,
+                "%s-content" % provider
+            )
+
+            if not os.path.isdir(provider_path):
+                os.mkdir(provider_path)
+            
+            for osys in self.links[provider]:
+                self._write_json(
+                    os.path.join(provider_path, osys),
+                    self.links[provider][osys]
+                )
+
+                provider_os_path = os.path.join(
+                    provider_path, "%s-content" % osys
+                )            
+
+                if not os.path.isdir(provider_os_path):
+                    os.mkdir(provider_os_path)
+                
+                for lc in self.links[provider][osys]:
+                    self._write_json(
+                        os.path.join(provider_os_path, lc),
+                        self.links[provider][osys][lc]
+                    )
+
+        # latest version
+        self._write_json(
+            os.path.join(api_path, 'latest'),
+            self.lv
+        )
+        
+        lv_path = os.path.join(api_path, 'latest-content')
+        if not os.path.isdir(lv_path):
+            os.mkdir(lv_path)
+
+        for release in self.lv:
+            if release == 'updated_at':
+                continue
+
+            self._write_json(
+                os.path.join(lv_path, release),
+                self.lv[release]
+            )
+            
+            release_path = os.path.join(
+                lv_path,
+                "%s-content" % release
+            )
+
+            if not os.path.isdir(release_path):
+                os.mkdir(release_path)
+            
+            for osys in self.lv[release]['downloads']:
+                self._write_json(
+                    os.path.join(release_path, osys),
+                    self.lv[release]['downloads'][osys]
+                )
+
+                release_os_path = os.path.join(
+                    release_path,
+                    "%s-content" % osys
+                )
+
+                if not os.path.isdir(release_os_path):
+                    os.mkdir(release_os_path)
+                
+                for lc in self.lv[release]['downloads'][osys]:
+                    self._write_json(
+                        os.path.join(release_os_path, lc),
+                        self.lv[release]['downloads'][osys][lc]
+                    )
+
+        # mirrors
+        self._write_json(
+            os.path.join(api_path, 'mirrors'),
+            self.mirrors
+        )

gettor/smtp.py

@@ -33,6 +33,12 @@ import blacklist
 
 """SMTP module for processing email requests."""
 
+OS = {
+    'osx': 'Mac OS X',
+    'linux': 'Linux',
+    'windows': 'Windows'
+}
+
 
 class ConfigError(Exception):
     pass
@@ -368,7 +374,7 @@ class SMTP(object):
         try:
             links_subject = self._get_msg('links_subject', 'en')
             links_msg = self._get_msg('links_msg', 'en')
-            links_msg = links_msg % (os, lc, links)
+            links_msg = links_msg % (os, links)
 
             self._send_email(
                 from_addr,

lang/smtp/i18n/en/LC_MESSAGES/en.po

@@ -22,42 +22,28 @@ msgstr "The locale you requested '%s' is not supported."
 
 #: Links message
 msgid "links_msg"
-msgstr "Hello there! this is the 'GetTor' robot.\n\
+msgstr "You requested Tor Browser for %s.\n\
 \n\
-Below are the links for your request (Tor Browser for %s, %s package):\n\
+You will need only one of the links below to download the bundle. If a link\n\
+does not work for you, try the next one.\n\
 \n\
 %s\n\
 \n\
-===========================================================================\n\
-Still need help? If you have any questions, trouble connecting to Tor\n\
-network, or need to talk to a human, please contact our support team at:\n\
 \n\
-    help@rt.torproject.org\n\
-\n\
-We are ready to answer your queries in English, Farsi, Chinese, Arabic,\n\
-French and Spanish."
+--\n\
+GetTor"