Commit ac136fe5 authored by Nick Mathewson's avatar Nick Mathewson 👉
Browse files

Implement code to force handing out a minimum number of ORs with a given...

Implement code to force handing out a minimum number of ORs with a given ORPort from a given ring, if possible (will not give good results until the number of such bridges is over the number of rings).  Also, initial support for IP categories, usable for proxy selection.

svn:r15137
parent e0329cff
......@@ -33,6 +33,10 @@ MASTER_KEY_FILE = "./secret_key"
# How many clusters do we group IPs in when distributing bridges based on IP?
N_IP_CLUSTERS = 4
# If possible, always give a certain number of answers with a given ORPort.
# This is a list of (port,minimum) tuples.
FORCE_PORTS = [ (443, 1) ]
#==========
# Options related to HTTPS
......
......@@ -204,6 +204,18 @@ class BridgeHolder:
def assignmentsArePersistent(self):
return True
class BridgeRingParameters:
"""DOCDOC"""
def __init__(self, needPorts=()):
"""DOCDOC takes list of port, count"""
for port,count in needPorts:
if not (1 <= port <= 65535):
raise TypeError("Port %s out of range."%port)
if count <= 0:
raise TypeError("Count %s out of range."%count)
self.needPorts = needPorts[:]
class BridgeRing(BridgeHolder):
"""Arranges bridges in a ring based on an hmac function."""
## Fields:
......@@ -212,14 +224,30 @@ class BridgeRing(BridgeHolder):
## isSorted: true iff sortedKeys is currently sorted.
## sortedKeys: a list of all the hmacs, in order.
## name: a string to represent this ring in the logs.
def __init__(self, key):
def __init__(self, key, answerParameters=None):
"""Create a new BridgeRing, using key as its hmac key."""
self.bridges = {}
self.bridgesByID = {}
self.hmac = get_hmac_fn(key, hex=False)
self.isSorted = False
self.sortedKeys = []
self.name = "Ring"
if answerParameters is None:
answerParameters = BridgeRingParameters()
self.answerParameters = answerParameters
self.portSubrings = [] #DOCDOC
for port,count in self.answerParameters.needPorts:
#note that we really need to use the same key here, so that
# the mapping is in the same order for all subrings.
self.portSubrings.append( (port,count,BridgeRing(key,None)) )
self.setName("Ring")
def setName(self, name):
"""DOCDOC"""
self.name = name
for port,_,subring in self.portSubrings:
subring.setName("%s (port-%s subring)"%(name, port))
def __len__(self):
return len(self.bridges)
......@@ -227,13 +255,17 @@ class BridgeRing(BridgeHolder):
def insert(self, bridge):
"""Add a bridge to the ring. If the bridge is already there,
replace the old one."""
for port,_,subring in self.portSubrings:
if port == bridge.orport:
subring.insert(bridge)
ident = bridge.getID()
pos = self.hmac(ident)
if not self.bridges.has_key(pos):
self.sortedKeys.append(pos)
self.isSorted = False
self.bridges[pos] = bridge
self.bridgesByID[id] = bridge
self.bridgesByID[ident] = bridge
logging.debug("Adding %s to %s", bridge.getConfigLine(), self.name)
def _sort(self):
......@@ -260,15 +292,29 @@ class BridgeRing(BridgeHolder):
def getBridges(self, pos, N=1):
"""Return the N bridges appearing in the ring after position pos"""
keys = self._getBridgeKeysAt(pos, N)
forced = []
for _,count,subring in self.portSubrings:
if len(subring) < count:
count = len.subring
forced.extend(subring._getBridgeKeysAt(pos, count))
keys = forced[:]
for k in self._getBridgeKeysAt(pos, N):
if k not in forced:
keys.append(k)
keys = keys[:N]
keys.sort()
return [ self.bridges[k] for k in keys ]
def getBridgeByID(self, fp):
"""Return the bridge whose identity digest is fp, or None if no such
bridge exists."""
return self.bridgesByID.get(fp)
for _,_,subring in self.portSubrings:
b = subring.getBridgeByID(fp)
if b is not None:
return b
return self.bridgesByID.get(fp)
class LogDB:
"""Wraps a database object and records all modifications to a
......
......@@ -33,17 +33,31 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder):
## rings of this distributor.
## areaOrderHmac -- an hmac function used to order areas within rings.
## areaClusterHmac -- an hmac function used to assign areas to rings.
def __init__(self, areaMapper, nClusters, key):
def __init__(self, areaMapper, nClusters, key, ipCategories=(),
answerParameters=None):
self.areaMapper = areaMapper
self.rings = []
self.categoryRings = [] #DOCDDOC
self.categories = [] #DOCDOC
for n in xrange(nClusters):
key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n)
self.rings.append( bridgedb.Bridges.BridgeRing(key1) )
self.rings[-1].name = "IP ring %s"%len(self.rings)
self.rings.append( bridgedb.Bridges.BridgeRing(key1,
answerParameters) )
self.rings[-1].setName("IP ring %s"%len(self.rings))
n = nClusters
for c in ipCategories:
key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n)
ring = bridgedb.Bridges.BridgeRing(key1, answerParameters)
self.categoryRings.append( ring )
self.categoryRings[-1].setName(
"IP category ring %s"%len(self.categoryRings))
self.categories.append( (c, ring) )
n += 1
key2 = bridgedb.Bridges.get_hmac(key, "Assign-Bridges-To-Rings")
self.splitter = bridgedb.Bridges.FixedBridgeSplitter(key2, self.rings)
self.splitter = bridgedb.Bridges.FixedBridgeSplitter(key2,
self.rings+self.categoryRings)
key3 = bridgedb.Bridges.get_hmac(key, "Order-Areas-In-Rings")
self.areaOrderHmac = bridgedb.Bridges.get_hmac_fn(key3, hex=False)
......@@ -67,6 +81,11 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder):
area = self.areaMapper(ip)
for category, ring in self.categories:
if category.contains(ip):
pos = self.areaOrderHmac("category<%s>%s"%(epoch,area))
return ring.getBridges(pos, N)
# Which bridge cluster should we look at?
h = int( self.areaClusterHmac(area)[:8], 16)
clusterNum = h % len(self.rings)
......@@ -183,12 +202,13 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
## store -- a database object to remember what we've given to whom.
## domainmap -- a map from lowercase domains that we support mail from
## to their canonical forms.
def __init__(self, key, store, domainmap, domainrules):
def __init__(self, key, store, domainmap, domainrules,
answerParameters=None):
key1 = bridgedb.Bridges.get_hmac(key, "Map-Addresses-To-Ring")
self.emailHmac = bridgedb.Bridges.get_hmac_fn(key1, hex=False)
key2 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring")
self.ring = bridgedb.Bridges.BridgeRing(key2)
self.ring = bridgedb.Bridges.BridgeRing(key2, answerParameters)
self.ring.name = "email ring"
# XXXX clear the store when the period rolls over!
self.store = store
......@@ -199,7 +219,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
"""Assign a bridge to this distributor."""
self.ring.insert(bridge)
def getBridgesForEmail(self, emailaddress, epoch, N=1):
def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None):
"""Return a list of bridges to give to a user.
emailaddress -- the user's email address, as given in a from line.
epoch -- the time period when we got this request. This can
......@@ -226,7 +246,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
return result
pos = self.emailHmac("<%s>%s" % (epoch, emailaddress))
result = self.ring.getBridges(pos, N)
result = self.ring.getBridges(pos, N, parameters)
memo = "".join(b.getID() for b in result)
self.store[emailaddress] = memo
return result
......
......@@ -43,6 +43,8 @@ CONFIG = Conf(
N_IP_CLUSTERS = 4,
MASTER_KEY_FILE = "./secret_key",
REQUIRE_ORPORTS = [(443, 1)],
HTTPS_DIST = True,
HTTPS_SHARE=10,
HTTPS_BIND_IP=None,
......@@ -181,13 +183,18 @@ def startup(cfg):
splitter = Bridges.BridgeSplitter(Bridges.get_hmac(key, "Splitter-Key"),
Bridges.PrefixStore(store, "sp|"))
# Create ring parameters.
forcePorts = getattr(cfg, "FORCE_PORTS")
ringParams=Bridges.BridgeRingParameters(forcePorts=forcePorts)
emailDistributor = ipDistributor = None
# As appropriate, create an IP-based distributor.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
ipDistributor = Dist.IPBasedDistributor(
Dist.uniformMap,
cfg.N_IP_CLUSTERS,
Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"))
Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"),
answerParameters=ringParams)
splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE)
webSchedule = Time.IntervalSchedule("day", 2)
......@@ -199,7 +206,8 @@ def startup(cfg):
Bridges.get_hmac(key, "Email-Dist-Key"),
Bridges.PrefixStore(store, "em|"),
cfg.EMAIL_DOMAIN_MAP.copy(),
cfg.EMAIL_DOMAIN_RULES.copy())
cfg.EMAIL_DOMAIN_RULES.copy(),
answerParameters=ringParams)
splitter.addRing(emailDistributor, "email", cfg.EMAIL_SHARE)
emailSchedule = Time.IntervalSchedule("day", 1)
......
......@@ -5,6 +5,7 @@
import doctest
import unittest
import warnings
import random
import bridgedb.Bridges
import bridgedb.Main
......@@ -18,11 +19,70 @@ class TestCase0(unittest.TestCase):
def testFooIsFooish(self):
self.assert_(True)
def randomIP():
return ".".join([str(random.randrange(1,256)) for _ in xrange(4)])
def fakeBridge(orport=8080):
nn = "bridge-%s"%random.randrange(0,1000000)
ip = randomIP()
fp = "".join([random.choice("0123456789ABCDEF") for _ in xrange(40)])
return bridgedb.Bridges.Bridge(nn,ip,orport,fingerprint=fp)
class RhymesWith255Category:
def contains(self, ip):
return ip.endswith(".255")
class IPBridgeDistTests(unittest.TestCase):
def dumbAreaMapper(self, ip):
return ip
def testBasicDist(self):
d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo")
for _ in xrange(256):
d.insert(fakeBridge())
n = d.getBridgesForIP("1.2.3.4", "x", 2)
n2 = d.getBridgesForIP("1.2.3.4", "x", 2)
self.assertEquals(n, n2)
def testDistWithCategories(self):
d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo",
[RhymesWith255Category()])
assert len(d.categoryRings) == 1
rhymesWith255Ring = d.categoryRings[0]
for _ in xrange(256):
d.insert(fakeBridge())
# Make sure this IP doesn't get any rhymes-with-255 bridges
n = d.getBridgesForIP("1.2.3.4", "x", 10)
for b in n:
self.assertFalse(b.getID() in rhymesWith255Ring.bridgesByID)
# Make sure these IPs all get rhymes-with-255 bridges
for ip in ("6.7.8.255", "10.10.10.255"):
n = d.getBridgesForIP("1.2.3.255", "xyz", 10)
for b in n:
self.assertTrue(b.getID() in rhymesWith255Ring.bridgesByID)
def testDistWithPortRestrictions(self):
param = bridgedb.Bridges.BridgeRingParameters(needPorts=[(443, 1)])
d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Baz",
answerParameters=param)
for _ in xrange(32):
d.insert(fakeBridge(443))
for _ in range(256):
d.insert(fakeBridge())
for _ in xrange(16):
i = randomIP()
n = d.getBridgesForIP(i, "x", 2)
count = 0
for b in n:
if b.orport == 443:
count += 1
self.assertTrue(count >= 1)
def testSuite():
suite = unittest.TestSuite()
loader = unittest.TestLoader()
for klass in [ TestCase0 ]:
for klass in [ TestCase0, IPBridgeDistTests ]:
suite.addTest(loader.loadTestsFromTestCase(klass))
for module in [ bridgedb.Bridges,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment