added a large file download for bandwidth/throttling tests

README

@@ -10,7 +10,7 @@ Add to crontab to run hourly tests:
 	0 */1 * * * cd ~/kz && ./bridgetest.sh [SITENAME]
 
 Generate a CSV file from logs:
-	find log -name '*.log' | sort | ./makecsv > bridgetest.csv
+	find log -name '*.log' | sort | ./makecsv > bridgetest.csv 2> bandwidth.csv
 
 Make a graph:
 	Rscript graph.R bridgetest.csv

bridgetest

@@ -43,6 +43,22 @@ def start_tcpdump(basename, addr):
         stderr=open(basename + ".tcpdump.err", "w"))
     return p
 
+def download_file():
+    logging.info("Attempting to download large file")
+    
+    try:
+        start = time.time()
+        subprocess.check_call(["/usr/bin/torsocks", "wget", "-O", "/dev/null", "http://mirror.cs.uwaterloo.ca/raspbian/images/apc_apricot_r1.zip"])
+        stop = time.time()
+        download_time = stop - start
+    except subprocess.CalledProcessError as e:
+        logging.info("failed to download file: %s", e)
+        download_time = -1
+
+    finally:
+        return download_time
+
+
 def start_tor(tor_config):
     assert "DataDirectory" in tor_config
 
@@ -133,6 +149,13 @@ with open(bridge_lines_file) as f:
             except OSError as err:
                 logging.info("failed to start tor: %s" % err)
                 continue
+
+            #Now try a large file download
+            download_time = download_file()
+            f = open(os.path.join(".", "%s.log" % nickname), 'a')
+            f.write("Downloaded " + str(download_time) + "s\n")
+            f.close()
+
             tor_proc.terminate()
             tor_proc.wait()
         finally:

makecsv

@@ -12,29 +12,43 @@ locale.setlocale(locale.LC_ALL, "C")
 
 # Dec 01 20:57:53.000 [notice] Bootstrapped 0%: Starting
 bootstrapped_re = re.compile(r'^(\w+ \d+ \d\d:\d\d:\d\d\.\d\d\d) \[\w+\] Bootstrapped (\d+)%')
+bandwidth_re = re.compile(r'^Downloaded (-?\d+.\d+)s')
 
 csvW = csv.DictWriter(sys.stdout, fieldnames=("timestamp", "site", "runid", "nickname", "percent"))
 csvW.writeheader()
 
+csvE = csv.DictWriter(sys.stderr, fieldnames=("site", "runid", "nickname", "download"))
+csvE.writeheader()
+
 def process_log(f, site, runid, nickname):
     for line in f:
         m = bootstrapped_re.match(line)
-        if m is None:
-            continue
-
-        timestamp = datetime.datetime.strptime(m.group(1), "%b %d %H:%M:%S.%f")
-        # tor logs don't contain the year, so grab it from the runid.
-        timestamp = timestamp.replace(year=int(runid[:4]))
-        percent = m.group(2)
-
-        row = {
-            "timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
-            "site": site,
-            "runid": runid,
-            "nickname": nickname,
-            "percent": percent,
-        }
-        csvW.writerow(row)
+        n = bandwidth_re.match(line)
+        if m is not None:
+
+            timestamp = datetime.datetime.strptime(m.group(1), "%b %d %H:%M:%S.%f")
+            # tor logs don't contain the year, so grab it from the runid.
+            timestamp = timestamp.replace(year=int(runid[:4]))
+            percent = m.group(2)
+
+            row = {
+                "timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
+                "site": site,
+                "runid": runid,
+                "nickname": nickname,
+                "percent": percent,
+            }
+            csvW.writerow(row)
+        elif n is not None:
+            download_time = n.group(1)
+            row = {
+                "site": site,
+                "runid": runid,
+                "nickname": nickname,
+                "download": download_time,
+            }
+            csvE.writerow(row)
+
 
 for filename in sys.stdin:
     filename = filename.strip()