Double the scanners, double the tors

Also update aggregate.py to work with a two-tor system

NetworkScanners/BwAuthority/aggregate.py

@@ -362,7 +362,8 @@ def write_file_list(datadir):
 
 def main(argv):
   TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg")
-  TorUtil.loglevel = "NOTICE"
+  TorUtil.loglevel = "DEBUG"
+  TorUtil.logfile = "data/aggregate-debug.log"
 
   (branch, head) = TorUtil.get_git_version(PATH_TO_TORFLOW_REPO)
   plog('INFO', 'TorFlow Version: %s' % branch+' '+head)
@@ -373,7 +374,7 @@ def main(argv):
   s.connect((TorUtil.control_host,TorUtil.control_port))
   c = TorCtl.Connection(s)
   c.debug(file(argv[1]+"/aggregate-control.log", "w", buffering=0))
-  c.authenticate_cookie(file(argv[1]+"/tor/control_auth_cookie",
+  c.authenticate_cookie(file(argv[1]+"/tor.1/control_auth_cookie",
                          "r"))
 
   ns_list = c.get_network_status()
@@ -847,6 +848,7 @@ def main(argv):
   plog("INFO", "Avg network +pid_error=+"+str(sum(map(lambda n: n.pid_error, p_nodes))/len(p_nodes)))
   plog("INFO", "Avg network -pid_error="+str(sum(map(lambda n: n.pid_error, n_nodes))/len(n_nodes)))
 
+
   plog("INFO",
        "Measured "+str(measured_pct) +"% of all tor nodes ("
        +str(measured_bw_pct)+"% of previous consensus bw).")

NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg

@@ -16,9 +16,9 @@ pid_file = ./data/scanner.1/bwauthority.pid
 db_url =
 #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1
 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1
-tor_dir = ./data/tor
+tor_dir = ./data/tor.1
 start_pct = 0
-stop_pct = 10
+stop_pct = 12
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5

NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg

@@ -16,9 +16,9 @@ pid_file = ./data/scanner.2/bwauthority.pid
 db_url =
 #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan2
 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan2
-tor_dir = ./data/tor
-start_pct = 10
-stop_pct = 40
+tor_dir = ./data/tor.1
+start_pct = 12
+stop_pct = 25
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5

NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg

@@ -16,9 +16,9 @@ pid_file = ./data/scanner.3/bwauthority.pid
 db_url = 
 #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan3
 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan3
-tor_dir = ./data/tor
-start_pct = 30
-stop_pct = 60
+tor_dir = ./data/tor.1
+start_pct = 25
+stop_pct = 37
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5

NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg

@@ -16,9 +16,9 @@ pid_file = ./data/scanner.4/bwauthority.pid
 db_url = 
 #db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4
 #db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4
-tor_dir = ./data/tor
-start_pct = 60
-stop_pct = 100
+tor_dir = ./data/tor.1
+start_pct = 37
+stop_pct = 50
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5

NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg

 [TorCtl]
 loglevel=WARN
 tor_host = 127.0.0.1
-tor_port = 9110
+tor_port = 9120
 control_host = 127.0.0.1
-control_port = 9111
+control_port = 9121
 control_pass = 
 # XXX: Unused
 meta_host = 127.0.0.1
-meta_port = 9112
+meta_port = 9122
 
 [BwAuthority]
 out_dir = ./data/scanner.5/scan-data
 pid_file = ./data/scanner.5/bwauthority.pid
 # if db_url is unset bwauthority will default to sqlite
-db_url = 
-#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4
-#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4
-tor_dir = ./data/tor
-start_pct = 0
-stop_pct = 100
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan5
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan5
+tor_dir = ./data/tor.2
+start_pct = 50
+stop_pct = 62
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5
 min_streams = 1
-max_fetch_time = 360
+max_fetch_time = 300
 sleep_start = 01:30
 sleep_stop = 01:30
-only_unmeasured = 1
-min_unmeasured = 5
+only_unmeasured = 0
+min_unmeasured = 0
+

NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg

+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.6/scan-data
+pid_file = ./data/scanner.6/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan6
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan6
+tor_dir = ./data/tor.2
+start_pct = 62
+stop_pct = 75
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0

NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg

+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.7/scan-data
+pid_file = ./data/scanner.7/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url = 
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan7
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan7
+tor_dir = ./data/tor.2
+start_pct = 75
+stop_pct = 87
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0

NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg

+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.8/scan-data
+pid_file = ./data/scanner.8/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url = 
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan8
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan8
+tor_dir = ./data/tor.2
+start_pct = 87
+stop_pct = 100
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0

NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg

+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.9/scan-data
+pid_file = ./data/scanner.9/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan9
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan9
+tor_dir = ./data/tor.2
+start_pct = 0
+stop_pct = 100
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 360
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 1
+min_unmeasured = 5

NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore

+bws-*
+sql-*
\ No newline at end of file

NetworkScanners/BwAuthority/data/tor/torrc → NetworkScanners/BwAuthority/data/tor.1/torrc

@@ -2,9 +2,9 @@
 SocksPort 9110 # what port to advertise for application connections
 ControlPort 9111
 SafeLogging 0
-Log notice file ./data/tor/tor.log
-DataDirectory ./data/tor
-PidFile ./data/tor/tor.pid
+Log notice file ./data/tor.1/tor.log
+DataDirectory ./data/tor.1
+PidFile ./data/tor.1/tor.pid
 CookieAuthentication 1
 FetchUselessDescriptors 1
 __LeaveStreamsUnattached 1

NetworkScanners/BwAuthority/data/tor.2/torrc

+## Configuration file for metatroller on alternate prot
+SocksPort 9120 # what port to advertise for application connections
+ControlPort 9121
+SafeLogging 0
+Log notice file ./data/tor.2/tor.log
+DataDirectory ./data/tor.2
+PidFile ./data/tor.2/tor.pid
+CookieAuthentication 1
+FetchUselessDescriptors 1
+__LeaveStreamsUnattached 1
+
+# Bad idea? Too much consensus update activity?
+FetchDirInfoExtraEarly 1
+FetchDirInfoEarly 1
+
+# Use CREATE cells for first hop, so we can still measure CPU overload via 
+# circ failure
+FastFirstHopPK 0

NetworkScanners/BwAuthority/run_scan.sh

 #!/bin/sh
 
-# Number of scanners to run.
-SCANNER_COUNT=5
+# Number of applications to run.
+SCANNERS_PER_TOR_COUNT=4
+TOR_COUNT=2
+SCANNER_COUNT=$(($SCANNERS_PER_TOR_COUNT * $TOR_COUNT + 1))
 
 # This tor must have the w status line fix as well as the stream bw fix
 # Ie git master or 0.2.2.x
@@ -19,15 +21,17 @@ for n in `seq $SCANNER_COUNT`; do
 done
 
 KILLED_TOR=false
-if [ -f "./data/tor/tor.pid" ]; then
-  PID=`cat ./data/tor/tor.pid`
-  if kill -0 "$PID" 2>/dev/null; then # it is a running process and we may send signals to it
-    kill $PID
-    if [ $? -eq 0 ]; then
-      KILLED_TOR=true
+for n in `seq $TOR_COUNT`; do
+  PIDFILE=./data/tor.${n}/tor.pid
+  if [ -f $PIDFILE ]; then
+    if kill -0 `head -1 $PIDFILE` 2>/dev/null; then # it is a running process and we may send signals to it
+  	  kill `head -1 $PIDFILE`
+  	  if [ $? -eq 0 ]; then
+  	    KILLED_TOR=true
+  	  fi
     fi
   fi
-fi
+done
 
 sleep 5
 
@@ -35,9 +39,10 @@ sleep 5
 # A more accurate resume could be implemented in bwauthority.py
 find data/scanner.* -name .svn -prune -o -type f -a ! -name '*-done-*' -a ! -name bwauthority.cfg -a ! -name .gitignore -exec rm {} +
 
-rm -f ./data/tor/tor.log
-
-$TOR_EXE -f ./data/tor/torrc &
+for n in `seq $TOR_COUNT`; do
+	rm -f ./data/tor.${n}/tor.log
+	$TOR_EXE -f ./data/tor.${n}/torrc &
+done
 
 # If this is a fresh start, we should allow the tors time to download
 # new descriptors.