Commit 5b7d898d authored by Nick Mathewson's avatar Nick Mathewson 👉
Browse files

r17150@catbus: nickm | 2007-12-14 15:32:46 -0500

 Fix bugs to the point where everything in bridgedb works; check in an initial configuration file.


svn:r12812
parent aea0b891
......@@ -2,6 +2,10 @@
For dec:
- reload bridges on sighup
- write a README
- make the configuration logic a little more sensible.
- proper logging
- check that incoming IP of email is sane.
- check more email headers for sanity
Later:
- document stuff better
......@@ -13,4 +17,3 @@ Later:
- decent template for web interface
- decent template for mail interface
- implement 'help' command
\ No newline at end of file
#==========
# General-purpose options.
# We chdir to this directory when we start; all files with relative
# pathnames are created under this directory
RUN_IN_DIR = "~/run/"
# Files from which we read descriptors on start and SIGHUP
# XXXX SIGHUP not implemented.
BRIDGE_FILES = [ "./cached-descriptors", "./cached-descriptors.new" ]
# Only consider routers whose purpose matches this string.
BRIDGE_PURPOSE = "bridge"
# File to store persistent info in.
DB_FILE = "./bridgedist.db"
# File to log changes to persistent info in. For debugging and bugfixing.
DB_LOG_FILE = "./bridgedist.log"
# File in which we store our secret HMAC root key.
MASTER_KEY_FILE = "./secret_key"
# How many clusters do we group IPs in when distributing bridges based on IP?
N_IP_CLUSTERS = 4
#==========
# Options related to HTTPS
# True if we are enableing distribution via HTTP or HTTPS; False otherwise.
HTTPS_DIST = True
# What proportion of bridges do we allocate to HTTP distribution? See
# EMAIL_SHARE and RESERVED_SHARE.
HTTPS_SHARE=10
# An IP address (form "1.2.3.4") where we listen for HTTPS connections.
# "None" to listen on the default interface.
HTTPS_BIND_IP=None
# Port to listen on for incoming HTTPS connections
HTTPS_PORT=6789
# Certificate file
HTTPS_CERT_FILE="cert"
# Private key file.
HTTPS_KEY_FILE="privkey.pem"
# IP and port to listen on for unencrypted HTTP connections. Debugging only.
HTTP_UNENCRYPTED_BIND_IP=None
HTTP_UNENCRYPTED_PORT=None
# How many bridges do we give back in an answer?
HTTPS_N_BRIDGES_PER_ANSWER=2
#==========
# Options related to Email
# True if we are enableing distribution via Email; false otherwise.
EMAIL_DIST = True
# What proportion of bridges do we allocate to Email distribution? See
# HTTPS_SHARE and RESERVED_SHARE.
EMAIL_SHARE=10
# Canonical versions of domains that we will reply to.
EMAIL_DOMAINS = [ "gmail.com", "yahoo.com" ]
# Map from unofficial domain to canonical domain.
EMAIL_DOMAIN_MAP = { "mail.google.com" : "gmail.com",
"googlemail.com" : "gmail.com",
}
# If there are any IPS in this list, only allow incoming connections from
# those IPs.
EMAIL_RESTRICT_IPS=[]
# IP and port to listen on for email connections. Debugging only.
EMAIL_BIND_IP=None
EMAIL_PORT=6725
# How many bridges to we give back in an answer?
EMAIL_N_BRIDGES_PER_ANSWER=2
#==========
# Options related to unallocated bridges.
# We split bridges into a group accessible by HTTPS queries; a group
# accessible by email queries; and a group that we don't assign to any
# query mechanism. Once a bridge is assigned to either of the first
# two groups, it stays there persistently. The bridges are allocated
# to these groups in an proportion of
# HTTPS_SHARE : EMAIL_SHARE : RESERVED_SHARE
RESERVED_SHARE=2
......@@ -50,7 +50,7 @@ def is_valid_fingerprint(fp):
if len(fp) != HEX_FP_LEN:
return False
try:
toHex(fp)
fromHex(fp)
except TypeError:
return False
else:
......@@ -89,8 +89,9 @@ class Bridge:
raise TypeError("Bridge with invalid ID")
self.fingerprint = toHex(id_digest)
elif fingerprint is not None:
if not is_valid_fingerprint(fromHex(fingerprint)):
raise TypeError("Bridge with invalid fingerprint")
if not is_valid_fingerprint(fingerprint):
raise TypeError("Bridge with invalid fingerprint (%r)"%
fingerprint)
self.fingerprint = fingerprint.lower()
else:
raise TypeError("Bridge with no ID")
......@@ -153,6 +154,9 @@ class BridgeRing(BridgeHolder):
self.isSorted = False
self.sortedKeys = []
def __len__(self):
return len(self.bridgesByID)
def insert(self, bridge):
id = bridge.getID()
pos = self.hmac(id)
......@@ -243,9 +247,9 @@ class PrefixStore:
n = len(self._p)
return [ k[n:] for k in self._d.keys() if k.startswith(self._p) ]
def FixedBridgeSplitter(BridgeHolder):
class FixedBridgeSplitter(BridgeHolder):
def __init__(self, key, rings):
self.hmac = get_mac_fn(key, hex=True)
self.hmac = get_hmac_fn(key, hex=True)
self.rings = rings[:]
for r in self.rings:
assert(isinstance(r, BridgeHolder))
......@@ -255,7 +259,14 @@ def FixedBridgeSplitter(BridgeHolder):
digest = self.hmac(bridge.getID())
pos = long( digest[:8], 16 )
which = pos % len(self.rings)
self.ring[which].insert(bridge)
self.rings[which].insert(bridge)
def __len__(self):
n = 0
for r in self.rings:
n += len(r)
return n
class UnallocatedHolder(BridgeHolder):
def insert(self, bridge):
......@@ -278,9 +289,9 @@ class BridgeTracker:
# The first-seen time only gets updated if it wasn't already set.
self.firstSeenStore.setdefault(bridgeID, now)
def BridgeSplitter(BridgeHolder):
class BridgeSplitter(BridgeHolder):
def __init__(self, key, store):
self.hmac = hmac.new(key, digestmod=DIGESTMOD)
self.hmac = get_hmac_fn(key, hex=True)
self.store = store
self.ringsByName = {}
self.totalP = 0
......@@ -288,6 +299,12 @@ def BridgeSplitter(BridgeHolder):
self.rings = []
self.statsHolders = []
def __len__(self):
n = 0
for r in self.rings:
n += len(r)
return n
def addRing(self, ring, ringname, p=1):
assert isinstance(ring, BridgeHolder)
self.ringsByName[ringname] = ring
......@@ -310,7 +327,7 @@ def BridgeSplitter(BridgeHolder):
else:
pos = self.hmac(bridgeID)
n = int(pos[:8], 16) % self.totalP
pos = bisect.bisect_right(self.pValues, p) - 1
pos = bisect.bisect_right(self.pValues, n) - 1
assert 0 <= pos < len(self.rings)
ringname = self.rings[pos]
ring = self.ringsByName.get(ringname)
......
......@@ -29,10 +29,10 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder):
self.splitter = bridgedb.Bridges.FixedBridgeSplitter(key2, self.rings)
key3 = bridgedb.Bridges.get_hmac(key, "Order-Areas-In-Rings")
self.areaOrderHmac = bridgedb.Bridges.get_hmac_fn(key3, hex=True)
self.areaOrderHmac = bridgedb.Bridges.get_hmac_fn(key3, hex=False)
key4 = bridgedb.Bridges.get_hmac(key, "Assign-Areas-To-Rings")
self.areaClusterHmac = bridgedb.Bridges.get_hmac_fun(key4, hex=True)
self.areaClusterHmac = bridgedb.Bridges.get_hmac_fn(key4, hex=True)
def insert(self, bridge):
self.splitter.insert(bridge)
......@@ -125,10 +125,10 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
def __init__(self, key, store, domainmap):
key1 = bridgedb.Bridges.get_hmac(key, "Map-Addresses-To-Ring")
self.emailHmac = bridgedb.Bridges.get_hmac_fn(key1, hex=1)
self.emailHmac = bridgedb.Bridges.get_hmac_fn(key1, hex=False)
key2 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring")
self.ring = bridgedb.Bridges.BrigeRing(key2)
self.ring = bridgedb.Bridges.BridgeRing(key2)
self.store = store
self.domainmap = domainmap
......@@ -136,12 +136,12 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
self.ring.insert(bridge)
def getBridgesForEmail(self, emailaddress, epoch, N=1):
emailaddress = normalizeEmail(emailaddress)
if emailAddress is None:
emailaddress = normalizeEmail(emailaddress, self.domainmap)
if emailaddress is None:
return [] #XXXX raise an exception.
if store.has_key(emailaddress):
if self.store.has_key(emailaddress):
result = []
ids = store[emailaddress]
ids = self.store[emailaddress]
for id in bridgedb.Bridges.chopString(ids, bridgedb.Bridges.ID_LEN):
b = self.ring.getBridgeByID(id)
if b != None:
......@@ -149,7 +149,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
return result
pos = self.emailHmac("<%s>%s" % (epoch, emailaddress))
result = ring.getBridges(pos, N)
result = self.ring.getBridges(pos, N)
memo = "".join(b.getID() for b in result)
self.store[emailaddress] = memo
return result
......
......@@ -17,29 +17,32 @@ class Conf:
self.__dict__.update(attrs)
CONFIG = Conf(
RUN_IN_DIR = ".",
BRIDGE_FILES = [ "./cached-descriptors", "./cached-descriptors.new" ],
BRIDGE_PURPOSE = "bridge",
DB_FILE = [ "./bridgedist" ],
DB_LOG_FILE = [ "./bridgedist.log" ],
DB_FILE = "./bridgedist.db",
DB_LOG_FILE = "./bridgedist.log",
N_IP_CLUSTERS = 8,
MASTER_KEY_FILE = [ "./secret_key" ],
N_IP_CLUSTERS = 4,
MASTER_KEY_FILE = "./secret_key",
HTTPS_DIST = True,
HTTPS_SHARE=10,
HTTPS_BIND_IP=None,
HTTPS_PORT=6789,
HTTPS_CERT_FILE="cert",
HTTPS_KEY_FILE="key",
HTTPS_KEY_FILE="privkey.pem",
HTTP_UNENCRYPTED_BIND_IP=None,
HTTP_UNENCRYPTED_PORT=6788,
HTTPS_N_BRIDGES_PER_ANSWER=2,
EMAIL_DIST = True,
EMAIL_SHARE=10,
EMAIL_DOMAINS = [ "gmail.com", "yahoo.com" ],
EMAIL_DOMAINS = [ "gmail.com", "yahoo.com", "catbus.wangafu.net" ],
EMAIL_DOMAIN_MAP = { "mail.google.com" : "gmail.com",
"googlemail.com" : "gmail.com", },
EMAIL_RESTRICT_IPS=[],
EMAIL_BIND_IP=None,
EMAIL_PORT=6725,
EMAIL_N_BRIDGES_PER_ANSWER=2,
......@@ -64,7 +67,7 @@ def getKey(fname):
True
"""
try:
f = open(fname, 'r')
f = open(fname, 'rb')
except IOError:
k = os.urandom(32)
flags = os.O_WRONLY|os.O_TRUNC|os.O_CREAT|getattr(os, "O_BIN", 0)
......@@ -85,28 +88,40 @@ def load(cfg, splitter):
f.close()
def startup(cfg):
key = getKey(MASTER_KEY_FILE)
cfg.BRIDGE_FILES = [ os.path.expanduser(fn) for fn in cfg.BRIDGE_FILES ]
for key in ("RUN_IN_DIR", "DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE",
"HTTPS_CERT_FILE", "HTTPS_KEY_FILE"):
v = getattr(cfg, key)
if v:
setattr(cfg, key, os.path.expanduser(v))
if cfg.RUN_IN_DIR:
os.chdir(cfg.RUN_IN_DIR)
key = getKey(cfg.MASTER_KEY_FILE)
dblogfile = None
emailDistributor = ipDistributor = None
baseStore = store = anydbm.open(cfg.DB_FILE, "c", 0600)
if DB_LOG_FILE:
if cfg.DB_LOG_FILE:
dblogfile = open(cfg.DB_LOG_FILE, "a+", 0)
store = LogDB("db", store, dblogfile)
store = Bridges.LogDB("db", store, dblogfile)
splitter = Bridges.BridgeSplitter(Bridges.get_hmac(key, "Splitter-Key"),
Bridges.PrefixStore(store, "sp|"))
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
ipDistrbutor = Dist.ipBasedDistributor(Dist.uniformMap,
Dist.N_IP_CLUSTERS,
Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"))
ipDistributor = Dist.IPBasedDistributor(
Dist.uniformMap,
cfg.N_IP_CLUSTERS,
Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"))
splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE)
webSchedule = Time.IntervalSchedule("day", 2)
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
for d in cfg.EMAIL_DOMAINS:
cfg.EMAIL_DOMAIN_MAP[d] = d
emailDistributor = Dist.emailBasedDistributor(
emailDistributor = Dist.EmailBasedDistributor(
Bridges.get_hmac(key, "Email-Dist-Key"),
Bridges.PrefixStore(store, "em|"),
cfg.EMAIL_DOMAIN_MAP.copy())
......@@ -118,11 +133,18 @@ def startup(cfg):
"unallocated",
cfg.RESERVED_SHARE)
stats = Bridges.BridgeTracker(Bridges.PrefixStore(store, "fs"),
Bridges.PrefixStore(store, "ls"))
stats = Bridges.BridgeTracker(Bridges.PrefixStore(store, "fs|"),
Bridges.PrefixStore(store, "ls|"))
splitter.addTracker(stats)
print "Loading bridges"
load(cfg, splitter)
print "%d bridges loaded" % len(splitter)
if emailDistributor:
print "%d for email" % len(emailDistributor.ring)
if ipDistributor:
print "%d for web:" % len(ipDistributor.splitter)
print " by location set:", " ".join(str(len(r)) for r in ipDistributor.rings)
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
Server.addWebServer(cfg, ipDistributor, webSchedule)
......@@ -131,9 +153,21 @@ def startup(cfg):
Server.addSMTPServer(cfg, emailDistributor, emailSchedule)
try:
Server.run()
print "Starting reactors."
Server.runServers()
finally:
baseStore.close()
if dblogfile is not None:
dblogfile.close()
if __name__ == '__main__':
if len(sys.argv) != 2:
print "Syntax: %s [config file]" % sys.argv[0]
sys.exit(1)
if sys.argv[1] == "TESTING":
configuration = CONFIG
else:
configuration = {}
execfile(sys.argv[1], configuration)
startup(CONFIG)
......@@ -15,6 +15,8 @@ import twisted.web.resource
import twisted.web.server
import twisted.mail.smtp
import bridgedb.Dist
class WebResource(twisted.web.resource.Resource):
isLeaf = True
......@@ -36,7 +38,7 @@ class WebResource(twisted.web.resource.Resource):
return "<html><body><pre>%s</pre></body></html>" % answer
def addWebServer(cfg, dist, sched):
from twised.web.server import Site
from twisted.web.server import Site
resource = WebResource(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER)
site = Site(resource)
if cfg.HTTP_UNENCRYPTED_PORT:
......@@ -53,6 +55,7 @@ def addWebServer(cfg, dist, sched):
class MailFile:
def __init__(self, lines):
self.lines = lines
self.idx = 0
def readline(self):
try :
......@@ -64,59 +67,64 @@ class MailFile:
def getMailResponse(lines, ctx):
# Extract data from the headers.
msg = rfc822(MailFile(lines))
msg = rfc822.Message(MailFile(lines))
subject = msg.getheader("Subject", None)
if not subject: subject = "[no subject]"
clientFromAddr = msg.getaddr("From")
clientSenderAddr = msg.getaddr("Sender")
msgID = msg.getheader("Message-ID")
if clientSenderAddr:
if clientSenderAddr and clientSenderAddr[1]:
clientAddr = clientSenderAddr[1]
elif clientFromAddr:
elif clientFromAddr and clientFromAddr[1]:
clientAddr = clientFromAddr[1]
else:
return None
print "No from header. WTF."
return None,None
for ln in lines:
if ln.strip() in ("get bridges", "Subject: get bridges"):
break
else:
return None
print "No request for bridges."
return None,None
try:
interval = ctx.schedule.getInterval(time.time())
bridges = ctx.distributor.getBridgesForEmail(clientAddr,
interval, ctx.N)
except bridgedb.Dist.BadEmail:
return None
except bridgedb.Dist.BadEmail, e:
print "Bad email addr in request: %s"%e
return None, None
if not bridges:
return None
print "No bridges available."
return None, None
# Generate the message.
f = StringIO()
w = MimeWriter.MimeWriter(f)
w.addHeader("From", ctx.fromAddr)
w.addHeader("To", clientAddr)
w.addHeader("Message-ID", twisted.mail.smtp.messageid())
w.addheader("From", ctx.fromAddr)
w.addheader("To", clientAddr)
w.addheader("Message-ID", twisted.mail.smtp.messageid())
if not subject.startswith("Re:"): subject = "Re: %s"%subject
w.addHeader("Subject", subject)
w.addHeader("In-Reply-To", msgID)
w.addHeader("Date", twisted.mail.smtp.rfc822date())
w.addheader("Subject", subject)
w.addheader("In-Reply-To", msgID)
w.addheader("Date", twisted.mail.smtp.rfc822date())
body = w.startbody("text/plain")
for b in bridges:
body.write("%s\n" % b.getConfigLine())
f.seek(0)
return f
return clientAddr, f
def replyToMail(lines, ctx):
sendToUser, response = getMailResponse(lines)
sendToUser, response = getMailResponse(lines, ctx)
if response is None:
return
response.seek(0)
d = Deferred()
factory = twisted.mail.smtp.SMTPSenderFactory(
ctx.fromAddr,
sendToUser,
StringIO(response),
response,
d)
reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory)
return d
......@@ -143,7 +151,7 @@ class MailMessage:
def lineReceived(self, line):
self.nBytes += len(line)
if self.nBytes > ctx.maximumSize:
if self.nBytes > self.ctx.maximumSize:
self.ignoring = True
else:
self.lines.append(line)
......@@ -151,7 +159,7 @@ class MailMessage:
def eomReceived(self):
if not self.ignoring:
replyToMail(self.lines, self.ctx)
return defer.succeed(None)
return twisted.internet.defer.succeed(None)
def connectionLost(self):
pass
......@@ -169,7 +177,7 @@ class MailDelivery:
if user.dest.local != self.ctx.username:
raise twisted.mail.smtp.SMTPBadRcpt(user)
return lambda: MailMessage(self.ctx)
class MailFactory(twisted.mail.smtp.SMTPFactory):
def __init__(self, *a, **kw):
twisted.mail.smtp.SMTPFactory.__init__(self, *a, **kw)
......@@ -185,7 +193,7 @@ class MailFactory(twisted.mail.smtp.SMTPFactory):
return p
def addSMTPServer(cfg, dist, sched):
ctx = MailContext(cfg)
ctx = MailContext(cfg, dist, sched)
factory = MailFactory()
factory.setBridgeDBContext(ctx)
ip = cfg.EMAIL_BIND_IP or ""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment