Commit 143129d8 authored by ilv's avatar ilv
Browse files

Progress for status update August 1st. New code structure and XMPP module.

parent 89241e80
#!/usr/bin/python
#
# Dummy script to test GetTore's Core module progress
# Dummy script to test GetTore's Core module
#
import gettor
import gettor.core
try:
core = gettor.Core('gettor.cfg')
core = gettor.core.Core()
links = core.get_links('dummy service', 'linux', 'es')
print links
except ValueError as e:
print "Value error: " + str(e)
except RuntimeError as e:
except gettor.core.ConfigurationError as e:
print "Misconfiguration: " + str(e)
except gettor.core.UnsupportedOSError as e:
print "Unsupported OS: " + str(e)
except gettor.core.UnsupportedLocaleError as e:
print "Unsupported Locale: " + str(e)
except gettor.core.InternalError as e:
print "Internal error: " + str(e)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
import re
import os
import gnupg
import hashlib
import dropbox
import gettor
import gettor.core
def valid_bundle_format(file):
......@@ -24,12 +28,12 @@ def valid_bundle_format(file):
def get_bundle_info(file):
"""
Get the operating system and locale from a bundle string.
"""Get the operating system and locale from a bundle string.
it raises a ValueError if the bundle doesn't have a valid format
(although you should probably call valid_bundle_format first).
It returns the pair of strings operating system, locale.
it raises a ValueError if the bundle doesn't have a valid format
(although you should probably call valid_bundle_format first).
It returns the pair of strings operating system, locale.
"""
m = re.search(
'tor-browser-(\w+)\d\d-\d\.\d\.\d_(\w\w)-\w+\.tar\.xz',
......@@ -41,17 +45,33 @@ def get_bundle_info(file):
else:
raise ValueError("Bundle invalid format %s" % file)
def get_file_sha1(file):
"""Get the sha1 of a file.
Desc.
"""
# as seen on the internet
BLOCKSIZE = 65536
hasher = hashlib.sha1()
with open(file, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
return hasher.hexdigest()
def upload_files(basedir, client):
"""
Upload files from 'basedir' to Dropbox.
"""Upload files from 'basedir' to Dropbox.
It looks for files ending with 'tar.xz' inside 'basedir'. It
raises ValueError in case the given file doesn't have a .asc file.
It raises UploadError if something goes wrong while uploading the
files to Dropbox. All files are uploaded to '/'.
It looks for files ending with 'tar.xz' inside 'basedir'. It
raises ValueError in case the given file doesn't have a .asc file.
It raises UploadError if something goes wrong while uploading the
files to Dropbox. All files are uploaded to '/'.
Returns a list with the names of the uploaded files.
Returns a list with the names of the uploaded files.
"""
files = []
......@@ -87,46 +107,53 @@ def upload_files(basedir, client):
return files
# Test app for now
# TO DO: use config file
app_key = ''
app_secret = ''
access_token = ''
upload_dir = 'upload/'
tbb_key = 'tbb-key.asc'
client = dropbox.client.DropboxClient(access_token)
# Import key that signed the packages and get fingerprint
gpg = gnupg.GPG()
key_data = open(tbb_key).read()
import_result = gpg.import_keys(key_data)
fingerprint = import_result.results[0]['fingerprint']
# Make groups of four characters to make fingerprint more readable
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable = ' '.join(fingerprint[i:i+4] for i in xrange(0, len(fingerprint), 4))
try:
uploaded_files = upload_files(upload_dir, client)
core = gettor.Core('gettor.cfg')
# This erases the old links file
core.create_links_file('Dropbox', readable)
for file in uploaded_files:
# build file names
asc = file + '.asc'
abs_file = os.path.abspath(os.path.join(upload_dir, file))
abs_asc = os.path.abspath(os.path.join(upload_dir, asc))
# build links
link_file = client.share(file)
link_asc = client.share(asc)
link = link_file[u'url'] + ' ' + link_asc[u'url']
if __name__ == '__main__':
# to-do: use config file
app_key = ''
app_secret = ''
access_token = ''
upload_dir = 'upload/'
# important: this must be the key that signed the packages
tbb_key = 'tbb-key.asc'
client = dropbox.client.DropboxClient(access_token)
# import key fingerprint
gpg = gnupg.GPG()
key_data = open(tbb_key).read()
import_result = gpg.import_keys(key_data)
fp = import_result.results[0]['fingerprint']
# make groups of four characters to make fingerprint more readable
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))
try:
uploaded_files = upload_files(upload_dir, client)
# use default config
core = gettor.core.Core()
# add links
operating_system, locale = get_bundle_info(file)
core.add_link('Dropbox', operating_system, locale, link)
except (ValueError, RuntimeError) as e:
print str(e)
except dropbox.rest.ErrorResponse as e:
print str(e)
# erase old links
core.create_links_file('Dropbox', readable)
for file in uploaded_files:
# build file names
asc = file + '.asc'
abs_file = os.path.abspath(os.path.join(upload_dir, file))
abs_asc = os.path.abspath(os.path.join(upload_dir, asc))
sha1_file = get_file_sha1(abs_file)
# build links
link_file = client.share(file)
link_asc = client.share(asc)
link = link_file[u'url'] + ' ' + link_asc[u'url'] + ' ' + sha1_file
# add links
operating_system, locale = get_bundle_info(file)
core.add_link('Dropbox', operating_system, locale, link)
except (ValueError, RuntimeError) as e:
print str(e)
except dropbox.rest.ErrorResponse as e:
print str(e)
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
import os
import re
import inspect
import logging
import tempfile
import ConfigParser
import utils
"""Core module for getting links from providers."""
class ConfigurationError(Exception):
pass
class UnsupportedOSError(Exception):
pass
class UnsupportedLocaleError(Exception):
pass
class LinkFormatError(Exception):
pass
class LinkFileError(Exception):
pass
class InternalError(Exception):
pass
class Core(object):
"""Get links from providers and deliver them to other modules.
Public methods:
get_links(): Get the links for the OS and locale requested.
create_links_file(): Create a file to store links of a provider.
add_link(): Add a link to a links file of a provider.
get_supported_os(): Get a list of supported operating systems.
get_supported_locale(): Get a list of supported locales.
Exceptions:
UnsupportedOSError: Request for an unsupported operating system.
UnsupportedLocaleError: Request for an unsupported locale.
ConfigurationError: Something's misconfigured.
LinkFormatError: The link added doesn't seem legit.
LinkFileError: Error related to the links file of a provider.
InternalError: Something went wrong internally.
"""
def __init__(self, cfg=None):
"""Create a new core object by reading a configuration file.
Raises: ConfigurationError if the configuration file doesn't exists
or if something goes wrong while reading options from it.
Params: cfg - path of the configuration file.
"""
# Define a set of default values
DEFAULT_CONFIG_FILE = 'core.cfg'
logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
logger = logging.getLogger(__name__)
config = ConfigParser.ConfigParser()
if cfg is None or not os.path.isfile(cfg):
cfg = DEFAULT_CONFIG_FILE
logger.info("Using default configuration")
logger.info("Reading configuration file %s" % cfg)
config.read(cfg)
try:
self.basedir = config.get('general', 'basedir')
except ConfigParser.Error as e:
logger.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.linksdir = config.get('links', 'dir')
self.linksdir = os.path.join(self.basedir, self.linksdir)
except ConfigParser.Error as e:
logger.warning("Couldn't read 'links' from 'dir' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.supported_locales = config.get('links', 'locales')
except ConfigParser.Error as e:
logger.warning("Couldn't read 'locales' from 'links' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.supported_os = config.get('links', 'os')
except ConfigParser.Error as e:
logger.warning("Couldn't read 'os' from 'links' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.loglevel = config.get('log', 'level')
except ConfigParser.Error as e:
logger.warning("Couldn't read 'level' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.logdir = config.get('log', 'dir')
self.logdir = os.path.join(self.basedir, self.logdir)
except ConfigParser.Error as e:
logger.warning("Couldn't read 'dir' from 'log' %s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
# Keep log levels separated
self.logger = utils.filter_logging(logger, self.logdir, self.loglevel)
# self.logger.setLevel(logging.getLevelName(self.loglevel))
self.logger.info('Redirecting logging to %s' % self.logdir)
# Stop logging on stdout from now on
self.logger.propagate = False
self.logger.debug("New core object created")
def get_links(self, service, operating_system, locale):
"""Get links for OS in locale.
This method should be called from the services modules of
GetTor (e.g. SMTP). To make it easy we let the module calling us
specify the name of the service (for stats purpose).
Raises: UnsupportedOSError: if the operating system is not supported.
UnsupportedLocaleError: if the locale is not supported.
InternalError: if something goes wrong while internally.
Params: service - the name of the service trying to get the links.
operating_system - the name of the operating system.
locale - two-character string representing the locale.
Returns: String with links.
"""
# Which module called us and what was asking for?
self.logger.info("%s did a request for %s, %s." %
(service, operating_system, locale))
if locale not in self.supported_locales:
self.logger.warning("Request for unsupported locale: %s" % locale)
raise UnsupportedLocaleError("Locale %s not supported at the "
"moment" % locale)
if operating_system not in self.supported_os:
self.logger.warning("Request for unsupported operating system: %s"
% operating_system)
raise UnsupportedOSError("Operating system %s not supported at the"
"moment" % operating_system)
# This could change in the future, let's leave it isolated.
links = self._get_links(operating_system, locale)
if links is None:
self.logger.error("Couldn't get the links", exc_info=True)
raise InternalError("Something went wrong internally. See logs for"
" detailed info.")
self.logger.info("Returning the links")
return links
def _get_links(self, operating_system, locale):
"""Internal method to get the links.
Looks for the links inside each provider file. This should only be
called from get_links() method.
Returns: String with the links on success.
None on failure.
Params: operating_system - name of the operating system
locale: two-character string representing the locale.
"""
# Read the links files using ConfigParser
# See the README for more details on the format used
links = []
# Look for files ending with .links
p = re.compile('.*\.links$')
for name in os.listdir(self.linksdir):
path = os.path.abspath(os.path.join(self.linksdir, name))
if os.path.isfile(path) and p.match(path):
links.append(path)
# Let's create a dictionary linking each provider with the links
# found for operating_system and locale. This way makes it easy
# to check if no links were found
providers = {}
self.logger.info("Reading links from providers directory")
for name in links:
self.logger.debug("Reading %s" % name)
# We're reading files listed on linksdir, so they must exist!
config = ConfigParser.ConfigParser()
config.read(name)
try:
pname = config.get('provider', 'name')
except ConfigParser.Error as e:
self.logger.warning("Couldn't get 'name' from 'provider' (%s)"
% name)
raise InternalError("Error while reading %s links file. See "
"log file" % name)
self.logger.debug("Checking if %s has links for %s in %s" %
(pname, operating_system, locale))
try:
providers[pname] = config.get(operating_system, locale)
except ConfigParser.Error as e:
self.logger.warning("Couldn't get %s from %s (%s)" %
(locale, operating_system, name))
raise InternalError("Error while reading %s links file. See "
"log file" % name)
# Each provider must have a fingerprint of the key used to
# sign the uploaded packages
try:
self.logger.debug("Trying to get fingerprint from %s", pname)
fingerprint = config.get('key', 'fingerprint')
providers[pname] = providers[pname] + "\nFingerprint: "
providers[pname] = providers[pname] + fingerprint
self.logger.debug("Fingerprint added %s", fingerprint)
except ConfigParser.Error as e:
self.logger.warning("Couldn't get 'fingerprint' from 'key' "
"(%s)" % name)
raise InternalError("Error while reading %s links file. See "
"log file" % name)
# Create the final links list with all providers
all_links = []
self.logger.debug("Joining all links found for %s in %s" %
(operating_system, locale))
for key in providers.keys():
all_links.append(
"\n%s\n%s\n" % (key, ''.join(providers[key]))
)
if all_links:
return "".join(all_links)
else:
self.logger.warning("Trying to get supported os and locales, but"
" no links were found")
return None
def get_supported_os(self):
"""Public method to get the list of supported operating systems.
Returns: List of strings.
"""
return self.supported_os.split(',')
def get_supported_locales(self):
"""Public method to get the list of supported locales.
Returns: List of strings.
"""
return self.supported_locales.split(',')
def create_links_file(self, provider, fingerprint):
"""Public method to create a links file for a provider.
This should be used by all providers since it writes the links
file with the proper format. It backs up the old links file
(if exists) and creates a new one.
Params: provider - provider's name (links file will use this
name in lower case).
fingerprint: fingerprint of the key that signed the packages
to be uploaded to the provider.
"""
linksfile = os.path.join(self.linksdir, provider.lower() + '.links')
linksfile_backup = ""
self.logger.info("Request to create new %s" % linksfile)
if os.path.isfile(linksfile):
# Backup the old file in case something fails
linksfile_backup = linksfile + '.backup'
self.logger.info("Backing up %s to %s"
% (linksfile, linksfile_backup))
os.rename(linksfile, linksfile_backup)
try:
# This creates an empty links file (with no links)
content = ConfigParser.RawConfigParser()
content.add_section('provider')
content.set('provider', 'name', provider)
content.add_section('key')
content.set('key', 'fingerprint', fingerprint)
content.add_section('linux')
content.add_section('windows')
content.add_section('osx')
with open(linksfile, 'w+') as f:
content.write(f)
self.logger.info("New %s created" % linksfile)
except Exception as e:
if linksfile_backup:
os.rename(linksfile_backup, linksfile)
raise LinkFileError("Error while trying to create new links file.")
def add_link(self, provider, operating_system, locale, link):
"""Public method to add a link to a provider's links file.
Use ConfigParser to add a link into the operating_system
section, under the locale option. It checks for valid format;
the provider's script should use the right format (see design).
Raises: UnsupportedOSError: if the operating system is not supported.
UnsupportedLocaleError: if the locale is not supported.
LinkFileError: if there is no links file for the provider.
LinkFormatError: if the link format doesn't seem legit.
InternalError: if the links file doesn't have a section for the
OS requested. This *shouldn't* happen because
it means the file wasn't created correctly.
Params: provider - name of the provider.
operating_system - name of the operating system.
locale - two-character string representing the locale.
link - string to be added. The format should be as follows:
https://pkg_url https://asc_url
where pkg_url is the url for the bundle and asc_url is the
url for the asc of the bundle.
"""
linksfile = os.path.join(self.linksdir, provider.lower() + '.links')
# Don't try to add unsupported stuff
if locale not in self.supported_locales:
self.logger.warning("Trying to add link for unsupported locale: %s"
% locale)
raise UnsupportedLocaleError("Locale %s not supported at the "
"moment" % locale)
if operating_system not in self.supported_os:
self.logger.warning("Trying to add link for unsupported operating "
"system: %s" % operating_system)
raise UnsupportedOSError("Operating system %s not supported at the"
" moment" % operating_system)
# Check if the link has a legit format
# e.g. https://db.tt/JjfUTb04 https://db.tt/MEfUTb04
p = re.compile('^https://.+\shttps://.+$')
if not p.match(link):
self.logger.warning("Trying to add an invalid link: %s"
% link)
raise LinkFormatError("Link '%s' doesn't seem to have a valid "
"format" % link)
if os.path.isfile(linksfile):
content = ConfigParser.RawConfigParser()
content.readfp(open(linksfile))
# Check if exists and entry for locale; if not, create it
try:
links = content.get(operating_system, locale)
links = links + ",\n" + link
content.set(operating_system, locale, links)
with open(linksfile, 'w') as f:
content.write(f)
self.logger.info("Link %s added to %s %s in %s"
% (link, operating_system, locale, provider))
except ConfigParser.NoOptionError:
content.set(operating_system, locale, link)
with open(linksfile, 'w') as f:
content.write(f)
self.logger.info("Link %s added to %s-%s in %s"
% (link, operating_system, locale, provider))
except ConfigParser.NoSectionError:
# This shouldn't happen, but just in case
self.logger.error("Unknown section %s in links file")
raise InternalError("Unknown %s section in links file"
% operating_system)
else:
raise LinkFileError("There is no links file for %s" % provider)
This diff is collapsed.
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
import os
import logging
"""Common utilities for GetTor modules."""
class SingleLevelFilter(logging.Filter):
"""Filter logging levels to create separated logs.
Public methods:
filter(): fitler logging levels.
"""
def __init__(self, passlevel, reject):
"""Create a new object with level to be filtered.