Commit 0fc5f539 authored by aagbsn's avatar aagbsn
Browse files

1608 - don't give out known blocked bridges

This set of changes allows bridgedb to filter responses
by country; if a bridge is known to be blocked in a country
it can be filtered from the set of returned bridges.
parent 75b386c8
......@@ -11,6 +11,9 @@ To set up:
- You can make a self-signed certificate with
openssl req -x509 -new -nodes > cert
- Set up PYTHONPATH to include "~/lib/python2.4/site-packages/".
- To install Maxmind GeoIP
- Debian: apt-get install python-geoip
- Others: http://www.maxmind.com/app/python
To re-generate and update the i18n files (in case translated strings
have changed in BridgeDB):
......@@ -47,5 +50,18 @@ To use with email:
- Any mail sent to the email port with a subject or a single line _exactly_
equal to "get bridges" will get answered, assuming the domain is okay.
To indicate which bridges are blocked:
- Uncomment or add COUNTRY_BLOCK_FILE to your bridgedb.conf
- The syntax of the COUNTRY_BLOCK_FILE, 1 entry per line:
fingerprint <bridge fingerprint> country-code <country code>
- If this file is present, bridgedb will filter blocked bridges from responses
- For GeoIP support make sure to install Maxmind GeoIP
To update the SQL schema:
- CREATE TABLE BlockedBridges ( id INTEGER PRIMARY KEY NOT NULL, hex_key, blocking_country);
- CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key);
- REPLACE INTO Config VALUES ( 'schema-version', 2 );
Support
- Send your questions to nickm.
......@@ -21,6 +21,9 @@ BRIDGE_FILES = [ "./bridge-descriptors" ]
# current bridges are Running.
STATUS_FILE = "networkstatus-bridges"
# Either a file that contains blocked bridges list or None
#COUNTRY_BLOCK_FILE = "./blocked-bridges"
# Only consider routers whose purpose matches this string.
BRIDGE_PURPOSE = "bridge"
# File to store persistent info in.
......
......@@ -104,6 +104,7 @@ class Bridge:
## fingerprint -- The bridge's identity digest, in lowercase hex, with
## no spaces.
## running,stable -- DOCDOC
## blockingCountries -- list of country codes blocking this bridge
def __init__(self, nickname, ip, orport, fingerprint=None, id_digest=None):
"""Create a new Bridge. One of fingerprint and id_digest must be
set."""
......@@ -111,6 +112,7 @@ class Bridge:
self.ip = ip
self.orport = orport
self.running = self.stable = None
self.blockingCountries = None
if id_digest is not None:
assert fingerprint is None
if len(id_digest) != DIGEST_LEN:
......@@ -151,6 +153,15 @@ class Bridge:
if stable is not None:
self.stable = stable
def setBlockingCountries(self, blockingCountries):
if blockingCountries is not None:
self.blockingCountries = blockingCountries
def isBlocked(self, countryCode):
if self.blockingCountries is not None and countryCode is not None:
if countryCode in self.blockingCountries:
return True
return False
def parseDescFile(f, bridge_purpose='bridge'):
"""Generator. Parses a cached-descriptors file 'f', and yields a Bridge
......@@ -200,10 +211,24 @@ def parseStatusFile(f):
flags = line.split()
yield ID, ("Running" in flags), ("Stable" in flags)
def parseCountryBlockFile(f):
"""Generator. Parses a blocked-bridges file 'f', and yields a
fingerprint, countryCode tuple for every entry"""
fingerprint = countryCode = None
for line in f:
line = line.strip()
m = re.match(r"fingerprint\s+(?P<fingerprint>\w+?)\s+country-code\s+(?P<countryCode>\w+)$", line)
try:
fingerprint = m.group('fingerprint').lower()
countryCode = m.group('countryCode').lower()
yield fingerprint, countryCode
except AttributeError, IndexError:
logging.warn("Unparseable line in blocked-bridges file: %s", line)
class BridgeHolder:
"""Abstract base class for all classes that hold bridges."""
def insert(self, bridge):
raise NotImplemented()
raise NotImplementedError
def clear(self):
pass
......@@ -326,7 +351,7 @@ class BridgeRing(BridgeHolder):
assert len(r) == N
return r
def getBridges(self, pos, N=1):
def getBridges(self, pos, N=1, countryCode=None):
"""Return the N bridges appearing in the ring after position pos"""
forced = []
for _,_,count,subring in self.subrings:
......@@ -523,3 +548,40 @@ class BridgeSplitter(BridgeHolder):
def dumpAssignments(self, f, description=""):
for name,ring in self.ringsByName.iteritems():
ring.dumpAssignments(f, "%s %s" % (description, name))
class BridgeBlock:
"""Base class that abstracts bridge blocking"""
def __init__(self):
pass
def insert(self, fingerprint, blockingRule):
raise NotImplementedError
def clear(self):
pass
def assignmentsArePersistent(self):
return True
class CountryBlock(BridgeBlock):
"""Countrywide bridge blocking"""
def __init__(self):
self.db = bridgedb.Storage.getDB()
def clear(self):
assert self.db
self.db.cleanBridgeBlocks()
self.db.commit()
def insert(self, fingerprint, blockingRule):
""" insert a country based blocking rule """
assert self.db
countryCode = blockingRule
self.db.addBridgeBlock(fingerprint, countryCode)
self.db.commit()
def getBlockingCountries(self, fingerprint):
""" returns a list of country codes where this fingerprint is blocked"""
assert self.db
if fingerprint is not None:
return self.db.getBlockingCountries(fingerprint)
......@@ -75,7 +75,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder):
"""Assign a bridge to this distributor."""
self.splitter.insert(bridge)
def getBridgesForIP(self, ip, epoch, N=1):
def getBridgesForIP(self, ip, epoch, N=1, countryCode=None):
"""Return a list of bridges to give to a user.
ip -- the user's IP address, as a dotted quad.
epoch -- the time period when we got this request. This can
......@@ -94,7 +94,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder):
if category.contains(ip):
logging.info("category<%s>%s"%(epoch,area))
pos = self.areaOrderHmac("category<%s>%s"%(epoch,area))
return ring.getBridges(pos, N)
return ring.getBridges(pos, N, countryCode)
# Which bridge cluster should we look at?
h = int( self.areaClusterHmac(area)[:8], 16)
......@@ -240,7 +240,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
"""Assign a bridge to this distributor."""
self.ring.insert(bridge)
def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None):
def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, countryCode=None):
"""Return a list of bridges to give to a user.
emailaddress -- the user's email address, as given in a from line.
epoch -- the time period when we got this request. This can
......@@ -266,7 +266,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
raise TooSoonEmail("Too many emails; wait till later", emailaddress)
pos = self.emailHmac("<%s>%s" % (epoch, emailaddress))
result = self.ring.getBridges(pos, N)
result = self.ring.getBridges(pos, N, countryCode)
db.setEmailTime(emailaddress, now)
db.commit()
......
......@@ -49,5 +49,7 @@ bridge addresses."""),
# BRIDGEDB_TEXT[7]
_("""(no bridges currently available)"""),
# BRIDGEDB_TEXT[8]
_("""(e-mail requests not currently supported)""")
_("""(e-mail requests not currently supported)"""),
# BRIDGEDB_TEXT[9]
_("""(Might be blocked)""")
]
......@@ -146,15 +146,24 @@ def load(cfg, splitter, clear=False):
"""Read all the bridge files from cfg, and pass them into a splitter
object.
"""
countryblock = Bridges.CountryBlock()
if clear:
logging.info("Clearing old bridges")
splitter.clear()
logging.info("Clearing old blocked bridges")
countryblock.clear()
logging.info("Loading bridges")
status = {}
if hasattr(cfg, "STATUS_FILE"):
f = open(cfg.STATUS_FILE, 'r')
for ID, running, stable in Bridges.parseStatusFile(f):
status[ID] = running, stable
f.close()
if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
f = open(cfg.COUNTRY_BLOCK_FILE, 'r')
for fingerprint, countryCode in Bridges.parseCountryBlockFile(f):
countryblock.insert(fingerprint, countryCode)
f.close()
for fname in cfg.BRIDGE_FILES:
f = open(fname, 'r')
for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
......@@ -162,6 +171,8 @@ def load(cfg, splitter, clear=False):
if s is not None:
running, stable = s
bridge.setStatus(running=running, stable=stable)
bridge.setBlockingCountries(
countryblock.getBlockingCountries(bridge.fingerprint))
splitter.insert(bridge)
f.close()
......
......@@ -26,6 +26,16 @@ import twisted.mail.smtp
import bridgedb.Dist
import bridgedb.I18n as I18n
try:
import GeoIP
# GeoIP data object: choose database here
# This is the same geoip implementation that pytorctl uses
geoip = GeoIP.new(GeoIP.GEOIP_STANDARD)
logging.info("GeoIP database loaded")
except:
geoip = None
logging.warn("GeoIP database not found")
class WebResource(twisted.web.resource.Resource):
"""This resource is used by Twisted Web to give a web page with some
bridges in response to a request."""
......@@ -51,6 +61,7 @@ class WebResource(twisted.web.resource.Resource):
interval = self.schedule.getInterval(time.time())
bridges = ( )
ip = None
countryCode = None
if self.useForwardedHeader:
h = request.getHeader("X-Forwarded-For")
if h:
......@@ -61,26 +72,29 @@ class WebResource(twisted.web.resource.Resource):
else:
ip = request.getClientIP()
# See if we did get a request for a certain locale, otherwise fall back
# to 'en':
# Try evaluating the path /foo first, then check if we got a ?lang=foo
default_lang = lang = "en"
if len(request.path) > 1:
lang = request.path[1:]
if lang == default_lang:
lang = request.args.get("lang", [default_lang])
lang = lang[0]
t = I18n.getLang(lang)
if geoip:
countryCode = geoip.country_code_by_addr(ip)
# allow client to specify a country
forcecc = getCCFromRequest(request)
if forcecc != None:
countryCode = forcecc
# get locale
t = getLocaleFromRequest(request)
format = request.args.get("format", None)
if format and len(format): format = format[0] # choose the first arg
if ip:
bridges = self.distributor.getBridgesForIP(ip, interval,
self.nBridgesToGive)
self.nBridgesToGive,
countryCode)
if bridges:
answer = "".join("%s\n" % b.getConfigLine(self.includeFingerprints)
for b in bridges)
answer = "".join("%s %s\n" % (
b.getConfigLine(self.includeFingerprints),
(I18n.BRIDGEDB_TEXT[9] if b.isBlocked(countryCode) else "")
) for b in bridges)
else:
answer = t.gettext(I18n.BRIDGEDB_TEXT[7])
......@@ -253,7 +267,8 @@ def getMailResponse(lines, ctx):
try:
interval = ctx.schedule.getInterval(time.time())
bridges = ctx.distributor.getBridgesForEmail(clientAddr,
interval, ctx.N)
interval, ctx.N,
countryCode=None)
except bridgedb.Dist.BadEmail, e:
logging.info("Got a mail from a bad email address %r: %s.",
clientAddr, e)
......@@ -329,6 +344,7 @@ def getLocaleFromPlusAddr(address):
return replyLocale
class MailContext:
"""Helper object that holds information used by email subsystem."""
def __init__(self, cfg, dist, sched):
......@@ -451,3 +467,20 @@ def runServers():
"""Start all the servers that we've configured. Exits when they do."""
reactor.run()
def getLocaleFromRequest(request):
# See if we did get a request for a certain locale, otherwise fall back
# to 'en':
# Try evaluating the path /foo first, then check if we got a ?lang=foo
default_lang = lang = "en"
if len(request.path) > 1:
lang = request.path[1:]
if lang == default_lang:
lang = request.args.get("lang", [default_lang])
lang = lang[0]
return I18n.getLang(lang)
def getCCFromRequest(request):
path = re.sub(r'[^a-zA-Z]', '', request.path)
if len(path) == 2:
return path.lower()
return None
......@@ -111,7 +111,7 @@ class SqliteDict:
# Here is the SQL schema.
SCHEMA1_SCRIPT = """
SCHEMA2_SCRIPT = """
CREATE TABLE Config (
key PRIMARY KEY NOT NULL,
value
......@@ -136,7 +136,15 @@ SCHEMA1_SCRIPT = """
CREATE INDEX EmailedBridgesWhenMailed on EmailedBridges ( email );
INSERT INTO Config VALUES ( 'schema-version', 1 );
CREATE TABLE BlockedBridges (
id INTEGER PRIMARY KEY NOT NULL,
hex_key,
blocking_country
);
CREATE INDEX BlockedBridgesBlockingCountry on BlockedBridges(hex_key);
INSERT INTO Config VALUES ( 'schema-version', 2 );
"""
class BridgeData:
......@@ -267,6 +275,52 @@ class Database:
cur.execute("UPDATE Bridges SET distributor = ? WHERE hex_key = ?",
(distributor, hex_key))
def addBridgeBlock(self, fingerprint, countryCode):
cur = self._cur
cur.execute("INSERT OR REPLACE INTO BlockedBridges "
"(hex_key,blocking_country) VALUES (?,?)",
(fingerprint, countryCode))
def delBridgeBlock(self, fingerprint, countryCode):
cur = self._cur
cur.execute("DELETE FROM BlockedBridges WHERE hex_key = ? "
"AND blocking_country = ?", (fingerprint, countryCode))
def cleanBridgeBlocks(self):
cur = self._cur
cur.execute("DELETE FROM BlockedBridges")
def getBlockingCountries(self, fingerprint):
cur = self._cur
cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE hex_key = ? ",
(fingerprint,))
v = cur.fetchall()
if v is None:
return None
# return list of country-codes
return [ str(result[1]) for (result) in v ]
def getBlockedBridges(self, countryCode):
cur = self._cur
cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE blocking_country = ? ",
(countryCode,))
v = cur.fetchall()
if v is None:
return None
# return list of fingerprints
return [ str(result[0]) for (result) in v ]
def isBlocked(self, fingerprint, countryCode):
cur = self._cur
cur.execute("SELECT hex_key, blocking_country FROM BlockedBridges WHERE "
"hex_key = ? AND blocking_country = ?",
(fingerprint, countryCode))
v = cur.fetchone()
if v is None:
return False
return True
def openDatabase(sqlite_file):
conn = sqlite3.Connection(sqlite_file)
cur = conn.cursor()
......@@ -274,11 +328,11 @@ def openDatabase(sqlite_file):
try:
cur.execute("SELECT value FROM Config WHERE key = 'schema-version'")
val, = cur.fetchone()
if val != 1:
if val != 2:
logging.warn("Unknown schema version %s in database.", val)
except sqlite3.OperationalError:
logging.warn("No Config table found in DB; creating tables")
cur.executescript(SCHEMA1_SCRIPT)
cur.executescript(SCHEMA2_SCRIPT)
conn.commit()
finally:
cur.close()
......@@ -292,7 +346,7 @@ def openOrConvertDatabase(sqlite_file, db_file):
conn = sqlite3.Connection(sqlite_file)
cur = conn.cursor()
cur.executescript(SCHEMA1_SCRIPT)
cur.executescript(SCHEMA2_SCRIPT)
conn.commit()
import anydbm
......
......@@ -233,6 +233,24 @@ class SQLStorageTests(unittest.TestCase):
cur.execute("SELECT * FROM EmailedBridges")
self.assertEquals(len(cur.fetchall()), 1)
db.addBridgeBlock(b2.fingerprint, 'us')
self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), True)
db.delBridgeBlock(b2.fingerprint, 'us')
self.assertEquals(db.isBlocked(b2.fingerprint, 'us'), False)
db.addBridgeBlock(b2.fingerprint, 'uk')
db.addBridgeBlock(b3.fingerprint, 'uk')
self.assertEquals(set([b2.fingerprint, b3.fingerprint]),
set(db.getBlockedBridges('uk')))
db.addBridgeBlock(b2.fingerprint, 'cn')
db.addBridgeBlock(b2.fingerprint, 'de')
db.addBridgeBlock(b2.fingerprint, 'jp')
db.addBridgeBlock(b2.fingerprint, 'se')
db.addBridgeBlock(b2.fingerprint, 'kr')
self.assertEquals(set(db.getBlockingCountries(b2.fingerprint)),
set(['uk', 'cn', 'de', 'jp', 'se', 'kr']))
def testSuite():
suite = unittest.TestSuite()
loader = unittest.TestLoader()
......
......@@ -13,11 +13,11 @@ KNOWN_INTERVALS = [ "hour", "day", "week", "month" ]
class Schedule:
def intervalStart(self, when):
raise NotImplemented()
raise NotImplementedError
def getInterval(self, when):
raise NotImplemented()
raise NotImplementedError
def nextIntervalStarts(self, when):
raise NotImplemented()
raise NotImplementedError
class IntervalSchedule(Schedule):
"""An IntervalSchedule splits time into somewhat natural periods,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment