r17150@catbus: nickm | 2007-12-14 15:32:46 -0500

 Fix bugs to the point where everything in bridgedb works; check in an initial configuration file.


svn:r12812

TODO

@@ -2,6 +2,10 @@
 For dec:
 - reload bridges on sighup
 - write a README
+- make the configuration logic a little more sensible.
+- proper logging
+- check that incoming IP of email is sane.
+- check more email headers for sanity
 
 Later:
 - document stuff better
@@ -13,4 +17,3 @@ Later:
 - decent template for web interface
 - decent template for mail interface
 - implement 'help' command
- 
\ No newline at end of file

bridgedb.conf

+
+#==========
+# General-purpose options.
+
+# We chdir to this directory when we start; all files with relative
+# pathnames are created under this directory
+RUN_IN_DIR = "~/run/"
+
+# Files from which we read descriptors on start and SIGHUP
+# XXXX SIGHUP not implemented.
+BRIDGE_FILES = [ "./cached-descriptors", "./cached-descriptors.new" ]
+
+# Only consider routers whose purpose matches this string.
+BRIDGE_PURPOSE = "bridge"
+# File to store persistent info in.
+DB_FILE = "./bridgedist.db"
+# File to log changes to persistent info in.  For debugging and bugfixing.
+DB_LOG_FILE = "./bridgedist.log"
+# File in which we store our secret HMAC root key.
+MASTER_KEY_FILE = "./secret_key"
+
+# How many clusters do we group IPs in when distributing bridges based on IP?
+N_IP_CLUSTERS = 4
+
+#==========
+# Options related to HTTPS
+
+# True if we are enableing distribution via HTTP or HTTPS; False otherwise.
+HTTPS_DIST = True
+# What proportion of bridges do we allocate to HTTP distribution?  See
+# EMAIL_SHARE and RESERVED_SHARE.
+HTTPS_SHARE=10
+# An IP address (form "1.2.3.4") where we listen for HTTPS connections.
+# "None" to listen on the default interface.
+HTTPS_BIND_IP=None
+# Port to listen on for incoming HTTPS connections
+HTTPS_PORT=6789
+# Certificate file
+HTTPS_CERT_FILE="cert"
+# Private key file.
+HTTPS_KEY_FILE="privkey.pem"
+# IP and port to listen on for unencrypted HTTP connections. Debugging only.
+HTTP_UNENCRYPTED_BIND_IP=None
+HTTP_UNENCRYPTED_PORT=None
+# How many bridges do we give back in an answer?
+HTTPS_N_BRIDGES_PER_ANSWER=2
+
+#==========
+# Options related to Email
+
+# True if we are enableing distribution via Email; false otherwise.
+EMAIL_DIST = True
+# What proportion of bridges do we allocate to Email distribution?  See
+# HTTPS_SHARE and RESERVED_SHARE.
+EMAIL_SHARE=10
+# Canonical versions of domains that we will reply to.
+EMAIL_DOMAINS = [ "gmail.com", "yahoo.com" ]
+# Map from unofficial domain to canonical domain.
+EMAIL_DOMAIN_MAP = { "mail.google.com" : "gmail.com",
+                     "googlemail.com" : "gmail.com",
+                     }
+# If there are any IPS in this list, only allow incoming connections from
+# those IPs.
+EMAIL_RESTRICT_IPS=[]
+# IP and port to listen on for email connections. Debugging only.
+EMAIL_BIND_IP=None
+EMAIL_PORT=6725
+# How many bridges to we give back in an answer?
+EMAIL_N_BRIDGES_PER_ANSWER=2
+
+#==========
+# Options related to unallocated bridges.
+
+# We split bridges into a group accessible by HTTPS queries; a group
+# accessible by email queries; and a group that we don't assign to any
+# query mechanism.  Once a bridge is assigned to either of the first
+# two groups, it stays there persistently.  The bridges are allocated
+# to these groups in an proportion of
+#   HTTPS_SHARE : EMAIL_SHARE : RESERVED_SHARE
+RESERVED_SHARE=2
+
+

lib/bridgedb/Bridges.py

@@ -50,7 +50,7 @@ def is_valid_fingerprint(fp):
     if len(fp) != HEX_FP_LEN:
         return False
     try:
-        toHex(fp)
+        fromHex(fp)
     except TypeError:
         return False
     else:
@@ -89,8 +89,9 @@ class Bridge:
                 raise TypeError("Bridge with invalid ID")
             self.fingerprint = toHex(id_digest)
         elif fingerprint is not None:
-            if not is_valid_fingerprint(fromHex(fingerprint)):
-                raise TypeError("Bridge with invalid fingerprint")
+            if not is_valid_fingerprint(fingerprint):
+                raise TypeError("Bridge with invalid fingerprint (%r)"%
+                                fingerprint)
             self.fingerprint = fingerprint.lower()
         else:
             raise TypeError("Bridge with no ID")
@@ -153,6 +154,9 @@ class BridgeRing(BridgeHolder):
         self.isSorted = False
         self.sortedKeys = []
 
+    def __len__(self):
+        return len(self.bridgesByID)
+
     def insert(self, bridge):
         id = bridge.getID()
         pos = self.hmac(id)
@@ -243,9 +247,9 @@ class PrefixStore:
         n = len(self._p)
         return [ k[n:] for k in self._d.keys() if k.startswith(self._p) ]
 
-def FixedBridgeSplitter(BridgeHolder):
+class FixedBridgeSplitter(BridgeHolder):
     def __init__(self, key, rings):
-        self.hmac = get_mac_fn(key, hex=True)
+        self.hmac = get_hmac_fn(key, hex=True)
         self.rings = rings[:]
         for r in self.rings:
             assert(isinstance(r, BridgeHolder))
@@ -255,7 +259,14 @@ def FixedBridgeSplitter(BridgeHolder):
         digest = self.hmac(bridge.getID())
         pos = long( digest[:8], 16 )
         which = pos % len(self.rings)
-        self.ring[which].insert(bridge)
+        self.rings[which].insert(bridge)
+
+    def __len__(self):
+        n = 0
+        for r in self.rings:
+            n += len(r)
+        return n
+
 
 class UnallocatedHolder(BridgeHolder):
     def insert(self, bridge):
@@ -278,9 +289,9 @@ class BridgeTracker:
         # The first-seen time only gets updated if it wasn't already set.
         self.firstSeenStore.setdefault(bridgeID, now)
 
-def BridgeSplitter(BridgeHolder):
+class BridgeSplitter(BridgeHolder):
     def __init__(self, key, store):
-        self.hmac = hmac.new(key, digestmod=DIGESTMOD)
+        self.hmac = get_hmac_fn(key, hex=True)
         self.store = store
         self.ringsByName = {}
         self.totalP = 0
@@ -288,6 +299,12 @@ def BridgeSplitter(BridgeHolder):
         self.rings = []
         self.statsHolders = []
 
+    def __len__(self):
+        n = 0
+        for r in self.rings:
+            n += len(r)
+        return n
+
     def addRing(self, ring, ringname, p=1):
         assert isinstance(ring, BridgeHolder)
         self.ringsByName[ringname] = ring
@@ -310,7 +327,7 @@ def BridgeSplitter(BridgeHolder):
         else:
             pos = self.hmac(bridgeID)
             n = int(pos[:8], 16) % self.totalP
-            pos = bisect.bisect_right(self.pValues, p) - 1
+            pos = bisect.bisect_right(self.pValues, n) - 1
             assert 0 <= pos < len(self.rings)
             ringname = self.rings[pos]
             ring = self.ringsByName.get(ringname)

lib/bridgedb/Dist.py

@@ -29,10 +29,10 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder):
         self.splitter = bridgedb.Bridges.FixedBridgeSplitter(key2, self.rings)
 
         key3 = bridgedb.Bridges.get_hmac(key, "Order-Areas-In-Rings")
-        self.areaOrderHmac = bridgedb.Bridges.get_hmac_fn(key3, hex=True)
+        self.areaOrderHmac = bridgedb.Bridges.get_hmac_fn(key3, hex=False)
 
         key4 = bridgedb.Bridges.get_hmac(key, "Assign-Areas-To-Rings")
-        self.areaClusterHmac = bridgedb.Bridges.get_hmac_fun(key4, hex=True)
+        self.areaClusterHmac = bridgedb.Bridges.get_hmac_fn(key4, hex=True)
 
     def insert(self, bridge):
         self.splitter.insert(bridge)
@@ -125,10 +125,10 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
     def __init__(self, key, store, domainmap):
 
         key1 = bridgedb.Bridges.get_hmac(key, "Map-Addresses-To-Ring")
-        self.emailHmac = bridgedb.Bridges.get_hmac_fn(key1, hex=1)
+        self.emailHmac = bridgedb.Bridges.get_hmac_fn(key1, hex=False)
 
         key2 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring")
-        self.ring = bridgedb.Bridges.BrigeRing(key2)
+        self.ring = bridgedb.Bridges.BridgeRing(key2)
         self.store = store
         self.domainmap = domainmap
 
@@ -136,12 +136,12 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
         self.ring.insert(bridge)
 
     def getBridgesForEmail(self, emailaddress, epoch, N=1):
-        emailaddress = normalizeEmail(emailaddress)
-        if emailAddress is None:
+        emailaddress = normalizeEmail(emailaddress, self.domainmap)
+        if emailaddress is None:
             return [] #XXXX raise an exception.
-        if store.has_key(emailaddress):
+        if self.store.has_key(emailaddress):
             result = []
-            ids = store[emailaddress]
+            ids = self.store[emailaddress]
             for id in bridgedb.Bridges.chopString(ids, bridgedb.Bridges.ID_LEN):
                 b = self.ring.getBridgeByID(id)
                 if b != None:
@@ -149,7 +149,7 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
             return result
 
         pos = self.emailHmac("<%s>%s" % (epoch, emailaddress))
-        result = ring.getBridges(pos, N)
+        result = self.ring.getBridges(pos, N)
         memo = "".join(b.getID() for b in result)
         self.store[emailaddress] = memo
         return result

lib/bridgedb/Main.py

@@ -17,29 +17,32 @@ class Conf:
         self.__dict__.update(attrs)
 
 CONFIG = Conf(
+    RUN_IN_DIR = ".",
+
     BRIDGE_FILES = [ "./cached-descriptors", "./cached-descriptors.new" ],
     BRIDGE_PURPOSE = "bridge",
-    DB_FILE = [ "./bridgedist" ],
-    DB_LOG_FILE = [ "./bridgedist.log" ],
+    DB_FILE = "./bridgedist.db",
+    DB_LOG_FILE = "./bridgedist.log",
 
-    N_IP_CLUSTERS = 8,
-    MASTER_KEY_FILE = [ "./secret_key" ],
+    N_IP_CLUSTERS = 4,
+    MASTER_KEY_FILE = "./secret_key",
 
     HTTPS_DIST = True,
     HTTPS_SHARE=10,
     HTTPS_BIND_IP=None,
     HTTPS_PORT=6789,
     HTTPS_CERT_FILE="cert",
-    HTTPS_KEY_FILE="key",
+    HTTPS_KEY_FILE="privkey.pem",
     HTTP_UNENCRYPTED_BIND_IP=None,
     HTTP_UNENCRYPTED_PORT=6788,
     HTTPS_N_BRIDGES_PER_ANSWER=2,
 
     EMAIL_DIST = True,
     EMAIL_SHARE=10,
-    EMAIL_DOMAINS = [ "gmail.com", "yahoo.com" ],
+    EMAIL_DOMAINS = [ "gmail.com", "yahoo.com", "catbus.wangafu.net" ],
     EMAIL_DOMAIN_MAP = { "mail.google.com" : "gmail.com",
                          "googlemail.com" : "gmail.com", },
+    EMAIL_RESTRICT_IPS=[],
     EMAIL_BIND_IP=None,
     EMAIL_PORT=6725,
     EMAIL_N_BRIDGES_PER_ANSWER=2,
@@ -64,7 +67,7 @@ def getKey(fname):
     True
     """
     try:
-        f = open(fname, 'r')
+        f = open(fname, 'rb')
     except IOError:
         k = os.urandom(32)
         flags = os.O_WRONLY|os.O_TRUNC|os.O_CREAT|getattr(os, "O_BIN", 0)
@@ -85,28 +88,40 @@ def load(cfg, splitter):
         f.close()
 
 def startup(cfg):
-    key = getKey(MASTER_KEY_FILE)
+    cfg.BRIDGE_FILES = [ os.path.expanduser(fn) for fn in cfg.BRIDGE_FILES ]
+    for key in ("RUN_IN_DIR", "DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE",
+                "HTTPS_CERT_FILE", "HTTPS_KEY_FILE"):
+        v = getattr(cfg, key)
+        if v:
+            setattr(cfg, key, os.path.expanduser(v))
+
+    if cfg.RUN_IN_DIR:
+        os.chdir(cfg.RUN_IN_DIR)
+
+    key = getKey(cfg.MASTER_KEY_FILE)
     dblogfile = None
+    emailDistributor = ipDistributor = None
 
     baseStore = store = anydbm.open(cfg.DB_FILE, "c", 0600)
-    if DB_LOG_FILE:
+    if cfg.DB_LOG_FILE:
         dblogfile = open(cfg.DB_LOG_FILE, "a+", 0)
-        store = LogDB("db", store, dblogfile)
+        store = Bridges.LogDB("db", store, dblogfile)
 
     splitter = Bridges.BridgeSplitter(Bridges.get_hmac(key, "Splitter-Key"),
                                       Bridges.PrefixStore(store, "sp|"))
 
     if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
-        ipDistrbutor = Dist.ipBasedDistributor(Dist.uniformMap,
-                                 Dist.N_IP_CLUSTERS,
-                                 Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"))
+        ipDistributor = Dist.IPBasedDistributor(
+            Dist.uniformMap,
+            cfg.N_IP_CLUSTERS,
+            Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"))
         splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE)
         webSchedule = Time.IntervalSchedule("day", 2)
 
     if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
         for d in cfg.EMAIL_DOMAINS:
             cfg.EMAIL_DOMAIN_MAP[d] = d
-        emailDistributor = Dist.emailBasedDistributor(
+        emailDistributor = Dist.EmailBasedDistributor(
             Bridges.get_hmac(key, "Email-Dist-Key"),
             Bridges.PrefixStore(store, "em|"),
             cfg.EMAIL_DOMAIN_MAP.copy())
@@ -118,11 +133,18 @@ def startup(cfg):
                          "unallocated",
                          cfg.RESERVED_SHARE)
 
-    stats = Bridges.BridgeTracker(Bridges.PrefixStore(store, "fs"),
-                                  Bridges.PrefixStore(store, "ls"))
+    stats = Bridges.BridgeTracker(Bridges.PrefixStore(store, "fs|"),
+                                  Bridges.PrefixStore(store, "ls|"))
     splitter.addTracker(stats)
 
+    print "Loading bridges"
     load(cfg, splitter)
+    print "%d bridges loaded" % len(splitter)
+    if emailDistributor:
+        print "%d for email" % len(emailDistributor.ring)
+    if ipDistributor:
+        print "%d for web:" % len(ipDistributor.splitter)
+        print "  by location set:", " ".join(str(len(r)) for r in ipDistributor.rings)
 
     if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
         Server.addWebServer(cfg, ipDistributor, webSchedule)
@@ -131,9 +153,21 @@ def startup(cfg):
         Server.addSMTPServer(cfg, emailDistributor, emailSchedule)
 
     try:
-        Server.run()
+        print "Starting reactors."
+        Server.runServers()
     finally:
         baseStore.close()
         if dblogfile is not None:
             dblogfile.close()
 
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print "Syntax: %s [config file]" % sys.argv[0]
+        sys.exit(1)
+    if sys.argv[1] == "TESTING":
+        configuration = CONFIG
+    else:
+        configuration = {}
+        execfile(sys.argv[1], configuration)
+
+    startup(CONFIG)

lib/bridgedb/Server.py

@@ -15,6 +15,8 @@ import twisted.web.resource
 import twisted.web.server
 import twisted.mail.smtp
 
+import bridgedb.Dist
+
 class WebResource(twisted.web.resource.Resource):
     isLeaf = True
 
@@ -36,7 +38,7 @@ class WebResource(twisted.web.resource.Resource):
         return "<html><body><pre>%s</pre></body></html>" % answer
 
 def addWebServer(cfg, dist, sched):
-    from twised.web.server import Site
+    from twisted.web.server import Site
     resource = WebResource(dist, sched, cfg.HTTPS_N_BRIDGES_PER_ANSWER)
     site = Site(resource)
     if cfg.HTTP_UNENCRYPTED_PORT:
@@ -53,6 +55,7 @@ def addWebServer(cfg, dist, sched):
 
 class MailFile:
     def __init__(self, lines):
+        self.lines = lines
         self.idx = 0
     def readline(self):
         try :
@@ -64,59 +67,64 @@ class MailFile:
 
 def getMailResponse(lines, ctx):
     # Extract data from the headers.
-    msg = rfc822(MailFile(lines))
+    msg = rfc822.Message(MailFile(lines))
     subject = msg.getheader("Subject", None)
     if not subject: subject = "[no subject]"
     clientFromAddr = msg.getaddr("From")
     clientSenderAddr = msg.getaddr("Sender")
     msgID = msg.getheader("Message-ID")
-    if clientSenderAddr:
+    if clientSenderAddr and clientSenderAddr[1]:
         clientAddr = clientSenderAddr[1]
-    elif clientFromAddr:
+    elif clientFromAddr and clientFromAddr[1]:
         clientAddr = clientFromAddr[1]
     else:
-        return None
+        print "No from header. WTF."
+        return None,None
     for ln in lines:
         if ln.strip() in ("get bridges", "Subject: get bridges"):
             break
     else:
-        return None
+        print "No request for bridges."
+        return None,None
 
     try:
         interval = ctx.schedule.getInterval(time.time())
         bridges = ctx.distributor.getBridgesForEmail(clientAddr,
                                                      interval, ctx.N)
-    except bridgedb.Dist.BadEmail:
-        return None
+    except bridgedb.Dist.BadEmail, e:
+        print "Bad email addr in request: %s"%e
+        return None, None
     if not bridges:
-        return None
+        print "No bridges available."
+        return None, None
 
     # Generate the message.
     f = StringIO()
     w = MimeWriter.MimeWriter(f)
-    w.addHeader("From", ctx.fromAddr)
-    w.addHeader("To", clientAddr)
-    w.addHeader("Message-ID", twisted.mail.smtp.messageid())
+    w.addheader("From", ctx.fromAddr)
+    w.addheader("To", clientAddr)
+    w.addheader("Message-ID", twisted.mail.smtp.messageid())
     if not subject.startswith("Re:"): subject = "Re: %s"%subject
-    w.addHeader("Subject", subject)
-    w.addHeader("In-Reply-To", msgID)
-    w.addHeader("Date", twisted.mail.smtp.rfc822date())
+    w.addheader("Subject", subject)
+    w.addheader("In-Reply-To", msgID)
+    w.addheader("Date", twisted.mail.smtp.rfc822date())
     body = w.startbody("text/plain")
     for b in bridges:
         body.write("%s\n" % b.getConfigLine())
 
     f.seek(0)
-    return f
+    return clientAddr, f
 
 def replyToMail(lines, ctx):
-    sendToUser, response = getMailResponse(lines)
+    sendToUser, response = getMailResponse(lines, ctx)
     if response is None:
         return
+    response.seek(0)
     d = Deferred()
     factory = twisted.mail.smtp.SMTPSenderFactory(
         ctx.fromAddr,
         sendToUser,
-        StringIO(response),
+        response,
         d)
     reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory)
     return d
@@ -143,7 +151,7 @@ class MailMessage:
 
     def lineReceived(self, line):
         self.nBytes += len(line)
-        if self.nBytes > ctx.maximumSize:
+        if self.nBytes > self.ctx.maximumSize:
             self.ignoring = True
         else:
             self.lines.append(line)
@@ -151,7 +159,7 @@ class MailMessage:
     def eomReceived(self):
         if not self.ignoring:
             replyToMail(self.lines, self.ctx)
-        return defer.succeed(None)
+        return twisted.internet.defer.succeed(None)
 
     def connectionLost(self):
         pass
@@ -169,7 +177,7 @@ class MailDelivery:
         if user.dest.local != self.ctx.username:
             raise twisted.mail.smtp.SMTPBadRcpt(user)
         return lambda: MailMessage(self.ctx)
-
+    
 class MailFactory(twisted.mail.smtp.SMTPFactory):
     def __init__(self, *a, **kw):
         twisted.mail.smtp.SMTPFactory.__init__(self, *a, **kw)
@@ -185,7 +193,7 @@ class MailFactory(twisted.mail.smtp.SMTPFactory):
         return p
 
 def addSMTPServer(cfg, dist, sched):
-    ctx = MailContext(cfg)
+    ctx = MailContext(cfg, dist, sched)
     factory = MailFactory()
     factory.setBridgeDBContext(ctx)
     ip = cfg.EMAIL_BIND_IP or ""