Commit 9ba4b0e7 authored by Christian Fromme's avatar Christian Fromme
Browse files

Adding the code that Nick reviewed to the bug_1612 branch

parent 691bf8c5
......@@ -140,3 +140,4 @@ EMAIL_INCLUDE_FINGERPRINTS=False
# HTTPS_SHARE : EMAIL_SHARE : RESERVED_SHARE
RESERVED_SHARE=2
FILE_DISTRIBUTORS = {}
......@@ -49,6 +49,7 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder):
self.rings[-1].setName("IP ring %s"%len(self.rings))
n = nClusters
for c in ipCategories:
logging.info("Building ring: Order-Bridges-In-Ring-%d"%n)
key1 = bridgedb.Bridges.get_hmac(key, "Order-Bridges-In-Ring-%d"%n)
ring = bridgedb.Bridges.BridgeRing(key1, answerParameters)
self.categoryRings.append( ring )
......@@ -86,8 +87,12 @@ class IPBasedDistributor(bridgedb.Bridges.BridgeHolder):
area = self.areaMapper(ip)
logging.info("area is %s" % area)
for category, ring in self.categories:
logging.info("---------------------------------")
if category.contains(ip):
logging.info("category<%s>%s"%(epoch,area))
pos = self.areaOrderHmac("category<%s>%s"%(epoch,area))
return ring.getBridges(pos, N)
......@@ -276,3 +281,163 @@ class EmailBasedDistributor(bridgedb.Bridges.BridgeHolder):
raise
else:
db.commit()
class FileDistributorBean:
"""A file distributor bean
"""
def __init__(self, name, needed):
self.name = name
if needed == "*":
# Set to rediculously high number
needed = 1000000
self.needed = int(needed)
self.allocated = 0
class FileDistributor:
"""FileDistributor reads a number of file distributors from the config.
They're expected to be in the following format:
FILE_DISTRIBUTORS = { "name1": 10, "name2": 15, "foobar": 3 }
This syntax means that certain distributors ("name1", "name2" and so on)
are given a number of bridges (10, 15 and so on). Names can be anything.
The name will later be the prefix of the file that is written with the
assigned number of bridges in it. Instead of a number, a wildcard item
("*") is allowed, too. This means that that file distributor will get
maximum bridges (as many as are left in the unallocated bucket).
The files will be written in ip:port format, one bridge per line.
The way this works internally is as follows:
First of all, the assignBridgesToDistributors() routine runs through
the database of bridges and looks up the 'distributor' field of each
bridge. Unallocated bridges are sent to a pool for later assignement.
Already allocated bridges for file distributors are sorted and checked.
They're checked for whether the distributor still exists in the current
config and also whether the number of assigned bridges is still valid.
If either the distributor is not existing anymore or too many bridges
are currently assigned to her, bridges will go to the unassigned pool.
In the second step, after bridges are sorted and the unassigned pool is
ready, the assignBridgesToDistributors() routine assigns one bridge
from the unassigned pool to a known distributor at a time until it
either runs out of bridges in the unallocated pool or the number of
needed bridges for that distributor is fullfilled.
When all bridges are assigned in this way, they then can then be dumped
into files by calling the dumpBridges() routine.
"""
def __init__(self, cfg):
self.cfg = cfg
self.distributorList = []
self.unallocatedList = []
self.unallocated_available = False
self.db = bridgedb.Storage.Database(self.cfg.DB_FILE+".sqlite",
self.cfg.DB_FILE)
def __del__(self):
self.db.close()
def addToUnallocatedList(self, id):
"""Add a bridge by database id into the unallocated pool
"""
try:
self.db.updateDistributorForId("unallocated", id)
except:
self.db.rollback()
raise
else:
self.db.commit()
self.unallocatedList.append(id)
self.unallocated_available = True
def knownFileDistributor(self, distributor):
"""Do we know this distributor?
"""
for d in self.distributorList:
if d.name == distributor:
return d
return None
def assignUnallocatedBridge(self, distributor):
"""Assign an unallocated bridge to a certain distributor
"""
distributor.allocated += 1
id = self.unallocatedList.pop()
#print "ID: %d NAME: %s" % (id, distributor.name)
try:
self.db.updateDistributorForId(distributor.name, id)
except:
self.db.rollback()
raise
else:
self.db.commit()
if len(self.unallocatedList) < 1:
self.unallocated_available = False
return True
def assignBridgesToDistributors(self):
"""Read file distributors from the configuration, sort them and write
necessary changes to the database
"""
# Build distributor list
for k, v in self.cfg.FILE_DISTRIBUTORS.items():
d = FileDistributorBean(k, v)
self.distributorList.append(d)
# Loop through all bridges and sort out our distributors
allBridges = self.db.getAllBridges()
for bridge in allBridges:
distributor = bridge[4]
if distributor == "unallocated":
self.addToUnallocatedList(bridge[0])
continue
# Check if we know this distributor
d = self.knownFileDistributor(distributor)
if d is not None:
# Does this distributor need another one?
# We assume that d.allocated is 0 in the beginning
if d.allocated < d.needed:
d.allocated += 1
else:
self.addToUnallocatedList(bridge[0])
# We don't know it. Maybe an old entry. Free it.
else:
# DON'T free https or email allocations!
if distributor != "https" and distributor != "email":
self.addToUnallocatedList(bridge[0])
# Loop though distributorList while we have and need unallocated
# bridges, assign one bridge at a time
while self.unallocated_available and len(self.distributorList) > 0:
for d in self.distributorList:
if d.allocated < d.needed:
if not self.assignUnallocatedBridge(d):
print "Couldn't assign unallocated bridge to %s" % d.name
else:
# When we have enough bridges, remove from list
self.distributorList.remove(d)
def dumpBridges(self):
"""Dump all known file distributors to files
"""
# Dump https, email and unreserved, too
self.cfg.FILE_DISTRIBUTORS["https"] = 0
self.cfg.FILE_DISTRIBUTORS["email"] = 0
self.cfg.FILE_DISTRIBUTORS["unallocated"] = 0
# Loop through all distributors and dump their bridges to files
for distributor, _ in self.cfg.FILE_DISTRIBUTORS.items():
fileName = distributor + "-" + time.strftime("%Y-%m-%d") + ".brdgs"
f = open(fileName, 'w')
f.write("Here are your bridges, %s:\n" % distributor)
bForDistributor = self.db.getBridgesForDistributor(distributor)
print "Dumping %d bridges for %s to %s" % (len(bForDistributor), distributor, fileName)
for bridge in bForDistributor:
line = "%s:%s" % (bridge[2], bridge[3])
f.write(line + '\n')
f.close
......@@ -18,6 +18,7 @@ import bridgedb.Dist as Dist
import bridgedb.Time as Time
import bridgedb.Server as Server
import bridgedb.Storage
import bridgedb.Opt as Opt
class Conf:
"""A configuration object. Holds unvalidated attributes.
......@@ -88,6 +89,8 @@ CONFIG = Conf(
EMAIL_INCLUDE_FINGERPRINTS = False,
RESERVED_SHARE=2,
FILE_DISTRIBUTORS = {}
)
def configureLogging(cfg):
......@@ -287,6 +290,12 @@ def startup(cfg):
logging.info("%d for web:", len(ipDistributor.splitter))
logging.info(" by location set: %s",
" ".join(str(len(r)) for r in ipDistributor.rings))
logging.info(" by category set: %s",
" ".join(str(len(r)) for r in ipDistributor.categoryRings))
logging.info("Here are all known bridges in the category section:")
for r in ipDistributor.categoryRings:
for name, b in r.bridges.items():
logging.info("%s" % b.getConfigLine(True))
global _reloadFn
_reloadFn = reload
......@@ -316,18 +325,26 @@ def run():
"""Parse the command line to determine where the configuration is.
Parse the configuration, and start the servers.
"""
if len(sys.argv) != 2:
options, arguments = Opt.parseOpts()
configuration = {}
if options.testing:
configuration = CONFIG
elif not options.configfile:
print "Syntax: %s [config file]" % sys.argv[0]
sys.exit(1)
if sys.argv[1] == "TESTING":
configuration = CONFIG
else:
configuration = {}
execfile(sys.argv[1], configuration)
configFile = options.configfile
execfile(configFile, configuration)
C = Conf(**configuration)
configuration = C
startup(configuration)
if options.dumpbridges:
fileDistributor = Dist.FileDistributor(configuration)
fileDistributor.assignBridgesToDistributors()
fileDistributor.dumpBridges()
else:
startup(configuration)
if __name__ == '__main__':
run()
......@@ -204,6 +204,22 @@ class Database:
cur.execute("INSERT OR REPLACE INTO EmailedBridges "
"(email,when_mailed) VALUES (?,?)", (addr, t))
def getAllBridges(self):
cur = self._cur
cur.execute("SELECT * FROM Bridges")
return cur.fetchall()
def getBridgesForDistributor(self, distributor):
cur = self._cur
cur.execute("SELECT * FROM Bridges WHERE "
"distributor = ?", (distributor, ))
return cur.fetchall()
def updateDistributorForId(self, distributor, id):
cur = self._cur
cur.execute("UPDATE Bridges SET distributor = ? WHERE id = ?",
(distributor, id))
def openDatabase(sqlite_file):
conn = sqlite3.Connection(sqlite_file)
cur = conn.cursor()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment