Commit 02435673 authored by ilv's avatar ilv
Browse files

Lots of changes: added blacklist and stats via sqlite database, cleaned the...

Lots of changes: added blacklist and stats via sqlite database, cleaned the code (including comments), and minor changes on SMTP module. Both SMTP and XMPP tested. This is the code that will be submitted to melange.
parent 9e716476
gettor is distributed under this license:
Copyright (c) 2008-2014, The Tor Project, Inc.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the names of the copyright owners nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
[general]
db: /path/to/gettor.db
[log]
level: DEBUG
dir: /path/to/log
[general]
basedir: ./
basedir: /path/to/gettor
db: gettor.db
[links]
dir: providers/
os: linux, windows, osx
locales: es, en
os: linux,windows,osx
locales: es,en
[log]
dir: log/
......
......@@ -2,6 +2,14 @@
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import re
import os
import gnupg
......@@ -46,11 +54,7 @@ def get_bundle_info(file):
raise ValueError("Bundle invalid format %s" % file)
def get_file_sha256(file):
"""Get the sha256 of a file.
Desc.
"""
"""Get the sha256 of a file."""
# as seen on the internet
BLOCKSIZE = 65536
......
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import time
import logging
import sqlite3
import datetime
import ConfigParser
import db
import utils
"""Blacklist module for managing blacklisting of users."""
class BlacklistError(Exception):
pass
class Blacklist(object):
"""Manage blacklisting of users.
Public methods:
is_blacklisted(): Check if someone is blacklisted.
Exceptions:
ConfigurationError: Bad configuration.
BlacklistError: User is blacklisted.
"""
def __init__(self, cfg=None):
"""Create new object by reading a configuration file.
:param: cfg (string) path of the configuration file.
"""
# define a set of default values
DEFAULT_CONFIG_FILE = 'blacklist.cfg'
logging.basicConfig(format='[%(levelname)s] %(asctime)s - %(message)s',
datefmt="%Y-%m-%d %H:%M:%S")
log = logging.getLogger(__name__)
config = ConfigParser.ConfigParser()
if cfg is None or not os.path.isfile(cfg):
cfg = DEFAULT_CONFIG_FILE
log.info("Using default configuration")
log.info("Reading configuration file %s" % cfg)
config.read(cfg)
try:
dbname = config.get('general', 'db')
self.db = db.DB(dbname)
except ConfigParser.Error as e:
log.warning("Couldn't read 'db' from 'general' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.logdir = config.get('log', 'dir')
except ConfigParser.Error as e:
log.warning("Couldn't read 'dir' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.loglevel = config.get('log', 'level')
except ConfigParser.Error as e:
log.warning("Couldn't read 'level' from 'log' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
# keep log levels separated
self.log = utils.filter_logging(log, self.logdir, self.loglevel)
self.log.setLevel(logging.getLevelName(self.loglevel))
log.debug('Redirecting logging to %s' % self.logdir)
# stop logging on stdout from now on
log.propagate = False
self.log.debug("New blacklist object created")
def is_blacklisted(self, user, service, max_req, wait_time):
"""Check if a user is blacklisted.
The user is blacklisted if:
a) The 'blocked' field is set to one, meaning that is permanently
blacklisted.
b) Does too many requests on a short period of time. For now, a user
that makes more than 'max_req' requests should wait 'wait_time'
minutes to make a new request.
:param: user (string) the hashed user.
:param: service (string) the service the user is making a request to.
:param: max_req (int) maximum number of requests a user can make
in a row.
:param: wait_time (int) amount of time the user must wait before
making requests again after 'max_req' requests is reached.
For now this is considered in minutes.
:raise: BlacklistError if the user is blacklisted
"""
r = self.db.get_user(user, service)
if r:
# permanently blacklisted
if r['blocked']:
self.log.info("Request from blocked user %s" % user)
self.db.update_user(user, service, r['times']+1, 1)
raise BlacklistError("Blocked user")
# don't be greedy
elif r['times'] >= max_req:
last = datetime.datetime.fromtimestamp(float(
r['last_request']))
next = last + datetime.timedelta(minutes=wait_time)
if datetime.datetime.now() < next:
self.log.info("Too many requests from user %s" % user)
self.db.update_user(user, service, r['times']+1, 0)
raise BlacklistError("Too many requests")
else:
# fresh user again!
self.log.debug("Request after wait time, cleaning up for"
" %s" % user)
self.db.update_user(user, service, 1, 0)
else:
self.log.debug("Adding up a request for %s" % user)
self.db.update_user(user, service, r['times']+1, 0)
else:
self.log.debug("New request for %s" % user)
self.db.add_user(user, service, 0)
......@@ -2,6 +2,13 @@
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import re
......@@ -10,6 +17,7 @@ import logging
import tempfile
import ConfigParser
import db
import utils
"""Core module for getting links from providers."""
......@@ -48,7 +56,7 @@ class Core(object):
create_links_file(): Create a file to store links of a provider.
add_link(): Add a link to a links file of a provider.
get_supported_os(): Get a list of supported operating systems.
get_supported_locale(): Get a list of supported locales.
get_supported_lc(): Get a list of supported locales.
Exceptions:
......@@ -91,6 +99,14 @@ class Core(object):
logger.warning("Couldn't read 'basedir' from 'general' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
dbname = config.get('general', 'db')
dbname = os.path.join(self.basedir, dbname)
self.db = db.DB(dbname)
except ConfigParser.Error as e:
logger.warning("Couldn't read 'db' from 'general' (%s)" % cfg)
raise ConfigurationError("Error with conf. See log file.")
try:
self.linksdir = config.get('links', 'dir')
self.linksdir = os.path.join(self.basedir, self.linksdir)
......@@ -273,7 +289,7 @@ class Core(object):
"""
return self.supported_os.split(',')
def get_supported_locales(self):
def get_supported_lc(self):
"""Public method to get the list of supported locales.
Returns: List of strings.
......@@ -400,3 +416,20 @@ class Core(object):
% operating_system)
else:
raise LinkFileError("There is no links file for %s" % provider)
def add_request_to_db(self, service, type, os, lc, pt, status, logfile):
"""Add request to database.
This is for keeping stats about what is the most, less requested
and stuff like that. Hopefully it will help to improve user experience.
:param: type (string) the type of the request.
:param: os (string) the operating system.
:param: lc (string) the locale.
:param: pt (bool) true if the user asked about pt, false otherwise.
:param: status (string) short text describing the status.
:param: logfile (string) path of the logfile of the email in case
something went really wrong (address blacklisted/malformed).
"""
self.db.add_request(service, type, os, lc, pt, status, logfile)
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import time
import sqlite3
import datetime
"""DB interface for comunicating with sqlite3"""
class DB(object):
"""
Public methods:
add_request(): add a request to the database (requests table).
get_user(): get user info from the database (users table).
add_user(): add a user to the database (users table).
update_user(): update a user on the database (users table).
"""
def __init__(self, dbname):
"""Create a new db object.
:param: dbname (string) the path of the database.
"""
self.con = sqlite3.connect(dbname)
self.con.row_factory = sqlite3.Row
def add_request(self, service, type, os, lc, pt, status, logfile):
"""Add a request to the database.
This is for keeping stats about what is the most, less requested
and stuff like that. Hopefully it will help to improve user experience.
:param: type (string) the type of the request.
:param: os (string) the operating system.
:param: lc (string) the locale.
:param: pt (bool) true if the user asked about pt, false otherwise.
:param: status (string) short text describing the status.
:param: logfile (string) path of the logfile of the email in case
something went really wrong (address blacklisted/malformed).
"""
now = datetime.datetime.now()
with self.con:
cur = self.con.cursor()
cur.execute("INSERT INTO requests VALUES(?,?,?,?,?,?,?,?,?,?)",
(service, type, os, lc, pt, now.year, now.month,
now.day, status, logfile))
def get_user(self, user, service):
"""Get user info from the database.
:param: user (string) unique (hashed) string that represents the user.
:param: service (string) the service related to the user (e.g. SMTP).
:return: (dict) the user information, with fields as indexes
(e.g. row['user']).
"""
with self.con:
cur = self.con.cursor()
cur.execute("SELECT * FROM users WHERE id =? AND service =?",
(user, service))
row = cur.fetchone()
return row
def add_user(self, user, service, blocked):
"""Add a user to the database.
We add a user with one 'times' and the current time as 'last_request'
by default.
:param: user (string) unique (hashed) string that represents the user.
:param: service (string) the service related to the user (e.g. SMTP).
:param: blocked (int) one if user is blocked, zero otherwise.
"""
with self.con:
cur = self.con.cursor()
cur.execute("INSERT INTO users VALUES(?,?,?,?,?)",
(user, service, 1, blocked, str(time.time())))
def update_user(self, user, service, times, blocked):
"""Update a user on the database.
We update the user info with the current time as 'last_request'.
:param: user (string) unique (hashed) string that represents the user.
:param: service (string) the service related to the user (e.g. SMTP).
:param: times (int) the number of requests the user has made.
:param: blocked (int) one if user is blocked, zero otherwise.
"""
with self.con:
cur = self.con.cursor()
cur.execute("UPDATE users SET times =?, blocked =?,"
" last_request =? WHERE id =? AND service =?",
(times, blocked, str(time.time()), user, service))
This diff is collapsed.
......@@ -2,9 +2,17 @@
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import logging
import hashlib
"""Common utilities for GetTor modules."""
......@@ -23,7 +31,7 @@ class SingleLevelFilter(logging.Filter):
If reject value is false, all but the passlevel will be filtered.
Useful for logging in separated files.
Params: passlevel - name of a logging level.
:param: passlevel (string) the name of a logging level.
"""
......@@ -40,11 +48,11 @@ class SingleLevelFilter(logging.Filter):
def filter_logging(logger, dir, level):
"""Create separated files for each level of logging.
Params: logger - a logging object.
dir - directory to put the log files.
level - the level of logging for the all.log file.
:param: logger (object) a logging object.
:param: dir (string) directory to put the log files.
:param: level (string) the level of logging for the all.log file.
Returns: logger object.
:return: (object) a logging object.
"""
# Keep a good format
......@@ -85,3 +93,12 @@ def filter_logging(logger, dir, level):
return logger
def get_sha256(string):
"""Get sha256 of a string.
:param: (string) the string to be hashed.
:return: (string) the sha256 of string.
"""
return str(hashlib.sha256(string).hexdigest())
This diff is collapsed.
......@@ -35,11 +35,6 @@ Tor Browser Bundle:\n\
===\n\
%s\n\
===\n\
Pluggable Transport Bundle:\n\
===\n\
%s\n\
Tip: If you are in Iran, China, or under heavy censorship, you need the\n\
Pluggable Transport Bundle.\n\
\n\
===\n\
Support:\n\
......@@ -64,11 +59,7 @@ Tor Browser Bundle:\n\
===\n\
%s\n\
===\n\
Pluggable Transport Bundle:\n\
===\n\
%s\n\
Tip: If you are in Iran, China, or under heavy censorship, you need the\n\
Pluggable Transport Bundle.\n\
Info about pluggable transports.\n\
\n\
===\n\
Support:\n\
......
......@@ -31,15 +31,8 @@ msgstr """Gracias por tu petición para %s-%s.\n\
Aquí están los links de descarga:\n\
\n\
===\n\
Tor Browser Bundle:\n\
===\n\
%s\n\
===\n\
Pluggable Transport Bundle:\n\
===\n\
%s\n\
Tip: Si estás en Iran, China, o bajo fuerte censura, necesitas el\n\
Pluggable Transport Bundle.\n\
\n\
===\n\
Soporte:\n\
......@@ -61,15 +54,9 @@ msgstr """Gracias por tu petición para %s-%s.\n\
Aquí están los links de descarga:\n\
\n\
===\n\
Tor Browser Bundle:\n\
===\n\
%s\n\
===\n\
Pluggable Transport Bundle:\n\
===\n\
%s\n\
Tip: Si estás en Iran, China, o bajo fuerte censura, necesitas el\n\
Pluggable Transport Bundle.\n\
Tip: Información sobre pluggable transports.\n\
\n\
===\n\
Soporte:\n\
......
......@@ -9,6 +9,9 @@ en = https://db.tt/ZAzgGm4Q https://db.tt/dUI80d2K 98ea6e4f216f2fb4b69fff9b3a448
es = https://db.tt/jp3Ytnzh https://db.tt/MDfUTb04 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
[windows]
en = https://db.tt/ZAzgGm4Q https://db.tt/dUI80d2K 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
es = https://db.tt/jp3Ytnzh https://db.tt/MDfUTb04 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
[osx]
en = https://db.tt/ZAzgGm4Q https://db.tt/dUI80d2K 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
es = https://db.tt/jp3Ytnzh https://db.tt/MDfUTb04 98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser Bundle distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import sys
import time
import sqlite3
import argparse
def main():
"""Script for managing blacklisting of users.
See argparse usage for more details.
"""
parser = argparse.ArgumentParser(description='Utility for GetTor'
' blacklisting')
parser.add_argument('database', metavar='database.db', type=str,
help='the database file')
parser.add_argument('-u', '--user', default=None,
help='filter by user hash')
parser.add_argument('-s', '--service', default=None,
help='filter by service')
parser.add_argument('-b', '--blocked', default=None,
help='filter by blocked users')
parser.add_argument('-a', '--add', default=None, nargs=3,
metavar=('USER', 'SERVICE', 'BLOCKED'),
help='add user')
parser.add_argument('-c', '--clean', default=None, const='c', nargs='?',
metavar='user hash',
help='clean table (delete expired blacklistings)')
parser.add_argument('-r', '--requests', default=None,
help='number of requests; everyone with number of'
' requests greather than this will be cleaned up')
args = parser.parse_args()
query = ''
con = sqlite3.connect(args.database)
if args.add:
# add new entry, useful for adding users permanently blocked
query = "INSERT INTO users VALUES('%s', '%s', 1, %s, %s)"\
% (args.add[0], args.add[1], args.add[2], time.time())
with con:
cur = con.cursor()
cur.execute(query)
print "Query execute successfully"
elif args.clean:
if args.clean == 'c':
if args.requests:
# delete by number of times
query = "DELETE FROM users WHERE times > %s" % args.requests
with con:
cur = con.cursor()
cur.execute(query)
print "Query executed successfully."
else:
sys.exit("Number of requests missing. See --help.")
else:
# delete by id
query = "DELETE FROM users WHERE id='%s'" % args.clean
with con:
cur = con.cursor()
cur.execute(query)
print "Query execute succcessfully."
else:
query = "SELECT * FROM users"
has_where = False
# filter
if args.service:
query = "%s %s" % (query, "WHERE service='%s'" % args.service)
has_where = True
if args.user:
if has_where:
query = "%s %s" % (query, "AND id='%s'" % args.user)