Add a 5th scanner for Unmeasured=1 nodes.

NetworkScanners/BwAuthority/bwauthority_child.py

@@ -95,9 +95,13 @@ def read_config(filename):
   pid_file = config.get('BwAuthority', 'pid_file')
   db_url = config.get('BwAuthority', 'db_url')
 
+  only_unmeasured = config.getint('BwAuthority', 'only_unmeasured')
+  min_unmeasured = config.getint('BwAuthority', 'min_unmeasured')
+
   return (start_pct,stop_pct,nodes_per_slice,save_every,
             circs_per_node,out_dir,max_fetch_time,tor_dir,
-            sleep_start,sleep_stop,min_streams,pid_file,db_url)
+            sleep_start,sleep_stop,min_streams,pid_file,db_url,only_unmeasured,
+            min_unmeasured)
 
 def choose_url(percentile):
   # TODO: Maybe we don't want to read the file *every* time?
@@ -205,7 +209,7 @@ class BwScanHandler(ScanSupport.SQLScanHandler):
 
 def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
               max_fetch_time, sleep_start_tp, sleep_stop_tp, slice_num,
-              min_streams, sql_file):
+              min_streams, sql_file, only_unmeasured):
   hdlr.set_pct_rstr(start_pct, stop_pct)
 
   attempt = 0
@@ -214,8 +218,6 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
     if hdlr.is_count_met(circs_per_node, successful): break
     hdlr.wait_for_consensus()
 
-
-
     # Check local time. Do not scan between 01:30 and 05:30 local time
     lt = time.localtime()
     sleep_start = time.mktime(lt[0:3]+sleep_start_tp+(0,0,0)+(lt[-1],))
@@ -239,7 +241,14 @@ def speedrace(hdlr, start_pct, stop_pct, circs_per_node, save_every, out_dir,
     # stream.. however, we count it as 'successful' below
     timer = threading.Timer(max_fetch_time, lambda: hdlr.close_streams(7))
     timer.start()
-    url = choose_url(start_pct)
+
+    # Always use median URL size for unmeasured nodes
+    # They may be too slow..
+    if only_unmeasured:
+      url = choose_url(50)
+    else:
+      url = choose_url(start_pct)
+
     plog("DEBUG", "Launching stream request for url "+url+" in "+str(start_pct)+'-'+str(stop_pct) + '%')
     ret = http_request(url)
     timer.cancel()
@@ -297,7 +306,8 @@ def main(argv):
   TorUtil.read_config(argv[1])
   (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir,
       max_fetch_time,tor_dir,sleep_start,sleep_stop,
-             min_streams,pid_file_name,db_url) = read_config(argv[1])
+             min_streams,pid_file_name,db_url,only_unmeasured,
+             min_unmeasured) = read_config(argv[1])
 
   # make sure necessary out_dir directory exists
   path = os.getcwd()+'/'+out_dir
@@ -332,9 +342,21 @@ def main(argv):
     socket.socket = socks.socksocket
     plog("INFO", "Set socks proxy to "+TorUtil.tor_host+":"+str(TorUtil.tor_port))
 
+    hdlr.schedule_selmgr(lambda s: setattr(s, "only_unmeasured", only_unmeasured))
+
     hdlr.wait_for_consensus()
 
+    # We should go to sleep if there are less than 5 unmeasured nodes after
+    # consensus update
+    while min_unmeasured and hdlr.get_unmeasured() < min_unmeasured:
+      plog("NOTICE", "Less than "+str(min_unmeasured)+" unmeasured nodes ("+str(hdlr.get_unmeasured())+"). Sleeping for a bit")
+      time.sleep(3600) # Until next consensus arrives
+      plog("NOTICE", "Woke up from waiting for more unmeasured nodes. Getting consensus and checking again")
+      hdlr.wait_for_consensus()
+
     pct_step = hdlr.rank_to_percent(nodes_per_slice)
+    plog("INFO", "Percent per slice is: "+str(pct_step))
+    if pct_step > 100: pct_step = 100
 
     # check to see if we are done
     if (slice_num * pct_step + start_pct > stop_pct):
@@ -344,7 +366,7 @@ def main(argv):
     plog("DEBUG", "Starting slice number %s" % slice_num)
     successful = speedrace(hdlr, slice_num*pct_step + start_pct, (slice_num + 1)*pct_step + start_pct, circs_per_node,
               save_every, out_dir, max_fetch_time, sleep_start, sleep_stop, slice_num,
-              min_streams, sql_file)
+              min_streams, sql_file, only_unmeasured)
 
     # For debugging memory leak..
     #TorUtil.dump_class_ref_counts(referrer_depth=1)

NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg

@@ -26,3 +26,5 @@ min_streams = 1
 max_fetch_time = 600
 sleep_start = 01:30
 sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0

NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg

@@ -26,3 +26,5 @@ min_streams = 1
 max_fetch_time = 300
 sleep_start = 01:30
 sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0

NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg

@@ -26,3 +26,5 @@ min_streams = 1
 max_fetch_time = 300
 sleep_start = 01:30
 sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0

NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg

@@ -26,3 +26,5 @@ min_streams = 1
 max_fetch_time = 300
 sleep_start = 01:30
 sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0

NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg

+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9110
+control_host = 127.0.0.1
+control_port = 9111
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9112
+
+[BwAuthority]
+out_dir = ./data/scanner.5/scan-data
+pid_file = ./data/scanner.5/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url = 
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4
+tor_dir = ./data/tor
+start_pct = 0
+stop_pct = 100
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 360
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 1
+min_unmeasured = 5

NetworkScanners/BwAuthority/run_scan.sh

 #!/bin/sh
 
 # Number of scanners to run.
-SCANNER_COUNT=4
+SCANNER_COUNT=5
 
 # This tor must have the w status line fix as well as the stream bw fix
 # Ie git master or 0.2.2.x