Skip to content
GitLab
Commit bff816b0 authored by Hiro's avatar Hiro 🏄
Browse files

Import ilv PR introduce major refactoring and restructure to use Twisted 

Update code and repository structure
Lay foundation to add testing and testing coverage
parent ba190034
Expand all Hide whitespace changes
Inline Side-by-side
.coveragerc 0 → 100644
View file @ bff816b0
[run]
source = gettor
branch = True
#parallel = True
timid = False
[report]
omit =
*/_langs*
*/_version*
*/__init__*
*/sitecustomize*
*/test/*
# Regexes for lines to exclude from report generation:
exclude_lines =
pragma: no cover
# don't complain if the code doesn't hit unimplemented sections:
raise NotImplementedError
pass
# don't complain if non-runnable or debuging code isn't run:
if 0:
if False:
if self[.verbosity.]
if options[.verbosity.]
def __repr__
if __name__ == .__main__.:
except Exception as impossible:
# Ignore source code which cannot be found:
ignore_errors = True
# Exit with status code 2 if under this percentage is covered:
fail_under = 80
[html]
directory = doc/coverage-html
.gitignore
View file @ bff816b0
venv
__pycache__
*.pyc
log
gettor.db
AUTHORS
View file @ bff816b0
Current maintainer/core developers:
hiro <hiro@torproject.org>
Israel Leiva <ilv@torproject.org> 4096R/540BFC0E
Past core developers:
......
Makefile
View file @ bff816b0
.PHONY: install test
.DEFAULT: install test
TRIAL:=$(shell which trial)
VERSION:=$(shell git describe)
define PYTHON_WHICH
import platform
import sys
sys.stdout.write(platform.python_implementation())
endef
PYTHON_IMPLEMENTATION:=$(shell python3 -c '$(PYTHON_WHICH)')
test:
python3 setup.py test
coverage-test:
ifeq ($(PYTHON_IMPLEMENTATION),PyPy)
@echo "Detected PyPy... not running coverage."
python setup.py test
else
coverage run --rcfile=".coveragerc" $(TRIAL) ./test/test_*.py
coverage report --rcfile=".coveragerc"
endif
coverage-html:
coverage html --rcfile=".coveragerc"
coverage: coverage-test coverage-html
tags:
find ./gettor -type f -name "*.py" -print | xargs etags
TAGS 0 → 100644
View file @ bff816b0
This diff is collapsed.
bin/gettor_service 0 → 100755
View file @ bff816b0
#!/bin/bash
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014-2018, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
source venv/bin/activate
case "$1" in
start)
twistd3 --python=scripts/gettor --logfile=log/gettor.log --pidfile=gettor.pid
;;
stop)
kill -INT `cat gettor.pid`
;;
restart)
$0 stop
sleep 2;
$0 start
;;
status)
if [ -e gettor.pid ]; then
echo gettor is running with pid=`cat gettor.pid`
else
echo gettor is NOT running
exit 1
fi
;;
*)
echo "Usage: $0 {start|stop|status|restart}"
esac
exit 0
config/blacklist.cfg deleted 100644 → 0
View file @ ba190034
[general]
db: /path/to/gettor.db
[log]
level: DEBUG
dir: /path/to/log
config/core.cfg deleted 100644 → 0
View file @ ba190034
[general]
basedir: /path/to/gettor
db: gettor.db
[links]
dir: /path/to/providers/
os: linux,windows,osx
locales: es,en
[log]
dir: /path/to/log/
level: DEBUG
config/smtp.cfg deleted 100644 → 0
View file @ ba190034
[general]
basedir: /path/to/gettor/smtp
mirrors: /path/to/mirrors
our_domain: torproject.org
core_cfg: /path/to/core.cfg
[blacklist]
cfg: /path/to/blacklist.cfg
max_requests: 3
wait_time: 20
[i18n]
dir: /path/to/i18n/
[log]
level: DEBUG
dir: /path/to/log/
config/xmpp.cfg deleted 100644 → 0
View file @ ba190034
[account]
user: account@domain
password:
[general]
basedir: /path/to/gettor/xmpp/
core_cfg: /path/to/core.cfg
max_words: 10
db: /path/to/gettor.db
[blacklist]
cfg: /path/to/blacklist.cfg
max_requests: 3
wait_time: 20
[i18n]
dir: /path/to/i18n/
[log]
level: DEBUG
dir: /path/to/log/
dev_scripts/create_db.py deleted 100644 → 0
View file @ ba190034
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import sqlite3
import argparse
def main():
"""Create/delete GetTor database for managing stats and blacklisting.
Database file (.db) must be empty. If it doesn't exist, it will be
created. See argparse usage for more details.
"""
parser = argparse.ArgumentParser(description='Utility for GetTor'
' database')
parser.add_argument('-c', '--create', default=None,
metavar='path_to_database.db',
help='create database')
parser.add_argument('-d', '--delete', default=None,
metavar='path_to_database.db',
help='delete database')
args = parser.parse_args()
if args.create:
con = sqlite3.connect(args.create)
with con:
cur = con.cursor()
# table for handling users (i.e. blacklist)
cur.execute(
"CREATE TABLE users(id TEXT, service TEXT, times INT,"
"blocked INT, last_request TEXT)"
)
cur.execute(
"CREATE TABLE requests(date TEXT, request TEXT, os TEXT,"
" locale TEXT, channel TEXT, PRIMARY KEY (date, channel))"
)
print "Database %s created" % os.path.abspath(args.create)
elif args.delete:
os.remove(os.path.abspath(args.delete))
print "Database %s deleted" % os.path.abspath(args.delete)
else:
print "See --help for details on usage."
if __name__ == "__main__":
main()
gettor.conf.json 0 → 100644
View file @ bff816b0
{
"platforms": ["linux", "osx", "windows"],
"dbname": "gettor.db",
"email_parser_logfile": "email_parser.log",
"email_requests_limit": 5,
"sendmail_interval": 10,
"sendmail_addr": "email@addr",
"sendmail_host": "host",
"sendmail_port": 587
}
gettor/__init__.py
View file @ bff816b0
# yes it's empty, of such a fullness
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
from .utils import strings
__version__ = strings.get_version()
__locales__ = strings.get_locales()
gettor/core.py deleted 100644 → 0
View file @ ba190034
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import re
import logging
import gettext
import tempfile
import ConfigParser
import db
import utils
"""Core module for getting links from providers."""
class ConfigError(Exception):
pass
class NotSupportedError(Exception):
pass
class LinkFormatError(Exception):
pass
class LinkFileError(Exception):
pass
class InternalError(Exception):
pass
class Core(object):
"""Get links from providers and deliver them to other modules.
Public methods:
get_links(): Get the links for the OS and locale requested.
create_links_file(): Create a file to store links of a provider.
add_link(): Add a link to a links file of a provider.
get_supported_os(): Get a list of supported operating systems.
get_supported_lc(): Get a list of supported locales.
Exceptions:
UnsupportedOSError: OS and/or locale not supported.
ConfigError: Something's misconfigured.
LinkFormatError: The link added doesn't seem legit.
LinkFileError: Error related to the links file of a provider.
InternalError: Something went wrong internally.
"""
def __init__(self, cfg=None):
"""Create a new core object by reading a configuration file.
:param: cfg (string) the path of the configuration file.
:raise: ConfigurationError if the configuration file doesn't exists
or if something goes wrong while reading options from it.
"""
default_cfg = 'core.cfg'
config = ConfigParser.ConfigParser()
if cfg is None or not os.path.isfile(cfg):
cfg = default_cfg
try:
with open(cfg) as f:
config.readfp(f)
except IOError:
raise ConfigError("File %s not found!" % cfg)
try:
self.supported_lc = config.get('links', 'locales')
self.supported_os = config.get('links', 'os')
basedir = config.get('general', 'basedir')
self.linksdir = config.get('links', 'dir')
self.linksdir = os.path.join(basedir, self.linksdir)
self.i18ndir = config.get('i18n', 'dir')
loglevel = config.get('log', 'level')
logdir = config.get('log', 'dir')
logfile = os.path.join(logdir, 'core.log')
dbname = config.get('general', 'db')
dbname = os.path.join(basedir, dbname)
self.db = db.DB(dbname)
except ConfigParser.Error as e:
raise ConfigError("Configuration error: %s" % str(e))
except db.Exception as e:
raise InternalError("%s" % e)
# logging
log = logging.getLogger(__name__)
logging_format = utils.get_logging_format()
date_format = utils.get_date_format()
formatter = logging.Formatter(logging_format, date_format)
log.info('Redirecting CORE logging to %s' % logfile)
logfileh = logging.FileHandler(logfile, mode='a+')
logfileh.setFormatter(formatter)
logfileh.setLevel(logging.getLevelName(loglevel))
log.addHandler(logfileh)
# stop logging on stdout from now on
log.propagate = False
self.log = log
def _get_msg(self, msgid, lc):
"""Get message identified by msgid in a specific locale.
:param: msgid (string) the identifier of a string.
:param: lc (string) the locale.
:return: (string) the message from the .po file.
"""
# obtain the content in the proper language
try:
t = gettext.translation(lc, self.i18ndir, languages=[lc])
_ = t.ugettext
msgstr = _(msgid)
return msgstr
except IOError as e:
raise ConfigError("%s" % str(e))
def get_links(self, service, os, lc):
"""Get links for OS in locale.
This method should be called from the services modules of
GetTor (e.g. SMTP). To make it easy we let the module calling us
specify the name of the service (for stats purpose).
:param: service (string) the service trying to get the links.
:param: os (string) the operating system.
:param: lc (string) tthe locale.
:raise: InternalError if something goes wrong while internally.
:return: (string) the links.
"""
# english and windows by default
if lc not in self.supported_lc:
self.log.debug("Request for locale not supported. Default to en")
lc = 'en'
if os not in self.supported_os:
self.log.debug("Request for OS not supported. Default to windows")
os = 'windows'
# this could change in the future, let's leave it isolated.
self.log.debug("Trying to get the links...")
try:
links = self._get_links(os, lc)
self.log.debug("OK")
except InternalError as e:
self.log.debug("FAILED")
raise InternalError("%s" % str(e))
if links is None:
self.log.debug("No links found")
raise InternalError("No links. Something is wrong.")
return links
def _get_links(self, osys, lc):
"""Internal method to get the links.
Looks for the links inside each provider file. This should only be
called from get_links() method.
:param: osys (string) the operating system.
:param: lc (string) the locale.
:return: (string/None) links on success, None otherwise.
"""
# read the links files using ConfigParser
# see the README for more details on the format used
links_files = []
links32 = {}
links64 = {}
# for the message to be sent
if osys == 'windows':
arch = '32/64'
elif osys == 'osx':
arch = '64'
else:
arch = '32'
# look for files ending with .links
p = re.compile('.*\.links$')
for name in os.listdir(self.linksdir):
path = os.path.abspath(os.path.join(self.linksdir, name))
if os.path.isfile(path) and p.match(path):
links_files.append(path)
# let's create a dictionary linking each provider with the links
# found for os and lc. This way makes it easy to check if no
# links were found
providers = {}
# separator
spt = '=' * 72
# reading links from providers directory
for name in links_files:
# we're reading files listed on linksdir, so they must exist!
config = ConfigParser.ConfigParser()
# but just in case they don't
try:
with open(name) as f:
config.readfp(f)
except IOError:
raise InternalError("File %s not found!" % name)
try:
pname = config.get('provider', 'name')
# check if current provider pname has links for os in lc
providers[pname] = config.get(osys, lc)
except ConfigParser.Error as e:
# we should at least have the english locale available
self.log.error("Request for %s, returning 'en' instead" % lc)
providers[pname] = config.get(osys, 'en')
try:
#test = providers[pname].split("$")
#self.log.debug(test)
if osys == 'linux':
t32, t64 = [t for t in providers[pname].split(",") if t]
link, signature, chs32 = [l for l in t32.split("$") if l]
link = " %s: %s" % (pname, link)
links32[link] = signature
link, signature, chs64 = [l for l in t64.split("$") if l]
link = " %s: %s" % (pname, link.lstrip())
links64[link] = signature
else:
link, signature, chs32 = [l for l in providers[pname].split("$") if l]
link = " %s: %s" % (pname, link)
links32[link] = signature
#providers[pname] = providers[pname].replace(",", "")
#providers[pname] = providers[pname].replace("$", "\n\n")
### We will improve and add the verification section soon ###
# all packages are signed with same key
# (Tor Browser developers)
# fingerprint = config.get('key', 'fingerprint')
# for now, english messages only
# fingerprint_msg = self._get_msg('fingerprint', 'en')
# fingerprint_msg = fingerprint_msg % fingerprint
except ConfigParser.Error as e:
raise InternalError("%s" % str(e))
# create the final links list with all providers
all_links = []
msg = "Tor Browser %s-bit:\n" % arch
for link in links32:
msg = "%s\n%s" % (msg, link)
all_links.append(msg)
if osys == 'linux':
msg = "\n\n\nTor Browser 64-bit:\n"
for link in links64:
msg = "%s\n%s" % (msg, link)
all_links.append(msg)
### We will improve and add the verification section soon ###
"""
msg = "\n\n\nTor Browser's signature %s-bit:" %\
arch
for link in links32:
msg = "%s\n%s" % (msg, links32[link])
all_links.append(msg)
if osys == 'linux':
msg = "\n\n\nTor Browser's signature 64-bit:"
for link in links64:
msg = "%s%s" % (msg, links64[link])
all_links.append(msg)
msg = "\n\n\nSHA256 of Tor Browser %s-bit (advanced): %s\n" %\
(arch, chs32)
all_links.append(msg)
if osys == 'linux':
msg = "SHA256 of Tor Browser 64-bit (advanced): %s\n" % chs64
all_links.append(msg)
"""
### end verification ###
"""
for key in providers.keys():
# get more friendly description of the provider
try:
# for now, english messages only
provider_desc = self._get_msg('provider_desc', 'en')
provider_desc = provider_desc % key
all_links.append(
"%s\n%s\n\n%s%s\n\n\n" %
(provider_desc, spt, ''.join(providers[key]), spt)
)
except ConfigError as e:
raise InternalError("%s" % str(e))
"""
### We will improve and add the verification section soon ###
# add fingerprint after the links
# all_links.append(fingerprint_msg)
if all_links:
return "".join(all_links)
else:
# we're trying to get supported os an lc
# but there aren't any links!
return None
def get_supported_os(self):
"""Public method to get the list of supported operating systems.
:return: (list) the supported operating systems.
"""
return self.supported_os.split(',')