Commit 4f272b1b authored by Mike Perry's avatar Mike Perry
Browse files

Add a 5th scanner for Unmeasured=1 nodes.

parent 8d091d36
......@@ -95,9 +95,13 @@ def read_config(filename):
pid_file = config.get('BwAuthority', 'pid_file')
db_url = config.get('BwAuthority', 'db_url')
only_unmeasured = config.getint('BwAuthority', 'only_unmeasured')
min_unmeasured = config.getint('BwAuthority', 'min_unmeasured')
return (start_pct,stop_pct,nodes_per_slice,save_every,
circs_per_node,out_dir,max_fetch_time,tor_dir,
sleep_start,sleep_stop,min_streams,pid_file,db_url)
sleep_start,sleep_stop,min_streams,pid_file,db_url,only_unmeasured,
min_unmeasured)
def choose_url(percentile):
# TODO: Maybe we don't want to read the file *every* time?
......@@ -205,7 +209,7 @@ class BwScanHandler(ScanSupport.SQLScanHandler):
def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
max_fetch_time, sleep_start_tp, sleep_stop_tp, slice_num,
min_streams, sql_file):
min_streams, sql_file, only_unmeasured):
hdlr.set_pct_rstr(start_pct, stop_pct)
attempt = 0
......@@ -214,8 +218,6 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
if hdlr.is_count_met(circs_per_node, successful): break
hdlr.wait_for_consensus()
# Check local time. Do not scan between 01:30 and 05:30 local time
lt = time.localtime()
sleep_start = time.mktime(lt[0:3]+sleep_start_tp+(0,0,0)+(lt[-1],))
......@@ -239,7 +241,14 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
# stream.. however, we count it as 'successful' below
timer = threading.Timer(max_fetch_time, lambda: hdlr.close_streams(7))
timer.start()
url = choose_url(start_pct)
# Always use median URL size for unmeasured nodes
# They may be too slow..
if only_unmeasured:
url = choose_url(50)
else:
url = choose_url(start_pct)
plog("DEBUG", "Launching stream request for url "+url+" in "+str(start_pct)+'-'+str(stop_pct) + '%')
ret = http_request(url)
timer.cancel()
......@@ -297,7 +306,8 @@ def main(argv):
TorUtil.read_config(argv[1])
(start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir,
max_fetch_time,tor_dir,sleep_start,sleep_stop,
min_streams,pid_file_name,db_url) = read_config(argv[1])
min_streams,pid_file_name,db_url,only_unmeasured,
min_unmeasured) = read_config(argv[1])
# make sure necessary out_dir directory exists
path = os.getcwd()+'/'+out_dir
......@@ -332,9 +342,21 @@ def main(argv):
socket.socket = socks.socksocket
plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port))
hdlr.schedule_selmgr(lambda s: setattr(s, "only_unmeasured", only_unmeasured))
hdlr.wait_for_consensus()
# We should go to sleep if there are less than 5 unmeasured nodes after
# consensus update
while min_unmeasured and hdlr.get_unmeasured() < min_unmeasured:
plog("NOTICE", "Less than "+str(min_unmeasured)+" unmeasured nodes ("+str(hdlr.get_unmeasured())+"). Sleeping for a bit")
time.sleep(3600) # Until next consensus arrives
plog("NOTICE", "Woke up from waiting for more unmeasured nodes. Getting consensus and checking again")
hdlr.wait_for_consensus()
pct_step = hdlr.rank_to_percent(nodes_per_slice)
plog("INFO", "Percent per slice is: "+str(pct_step))
if pct_step > 100: pct_step = 100
# check to see if we are done
if (slice_num * pct_step + start_pct > stop_pct):
......@@ -344,7 +366,7 @@ def main(argv):
plog("DEBUG", "Starting slice number %s" % slice_num)
successful = speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node,
save_every, out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num,
min_streams, sql_file)
min_streams, sql_file, only_unmeasured)
# For debugging memory leak..
#TorUtil.dump_class_ref_counts(referrer_depth=1)
......
......@@ -26,3 +26,5 @@ min_streams = 1
max_fetch_time = 600
sleep_start = 01:30
sleep_stop = 01:30
only_unmeasured = 0
min_unmeasured = 0
......@@ -26,3 +26,5 @@ min_streams = 1
max_fetch_time = 300
sleep_start = 01:30
sleep_stop = 01:30
only_unmeasured = 0
min_unmeasured = 0
......@@ -26,3 +26,5 @@ min_streams = 1
max_fetch_time = 300
sleep_start = 01:30
sleep_stop = 01:30
only_unmeasured = 0
min_unmeasured = 0
......@@ -26,3 +26,5 @@ min_streams = 1
max_fetch_time = 300
sleep_start = 01:30
sleep_stop = 01:30
only_unmeasured = 0
min_unmeasured = 0
[TorCtl]
loglevel=WARN
tor_host = 127.0.0.1
tor_port = 9110
control_host = 127.0.0.1
control_port = 9111
control_pass =
# XXX: Unused
meta_host = 127.0.0.1
meta_port = 9112
[BwAuthority]
out_dir = ./data/scanner.5/scan-data
pid_file = ./data/scanner.5/bwauthority.pid
# if db_url is unset bwauthority will default to sqlite
db_url =
#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4
#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4
tor_dir = ./data/tor
start_pct = 0
stop_pct = 100
save_every = 0
nodes_per_slice = 50
circs_per_node = 5
min_streams = 1
max_fetch_time = 360
sleep_start = 01:30
sleep_stop = 01:30
only_unmeasured = 1
min_unmeasured = 5
#!/bin/sh
# Number of scanners to run.
SCANNER_COUNT=4
SCANNER_COUNT=5
# This tor must have the w status line fix as well as the stream bw fix
# Ie git master or 0.2.2.x
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment