fork of https://www.bamsoftware.com/git/repo.eecs.berkeley.edu/proxy-probe.git/

README

+[SITENAME] is an arbitrary identifier for the probe site
+that you have to choose.
+
+Add to crontab to run hourly tests:
+	0 */1 * * * cd ~/kz && ./bridgetest.sh [SITENAME]
+
+Generate a CSV file from logs:
+	find log -name '*.log' | sort | ./makecsv > bridgetest.csv
+
+Make a graph:
+	Rscript graph.R bridgetest.csv

bridgetest

+#!/usr/bin/env python2
+
+import errno
+import logging
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+import time
+
+import stem.process
+
+BRIDGE_LINES = (
+)
+
+START_TOR_TIMEOUT = 3 * 60
+CIRCUIT_BUILD_TIMEOUT = 60
+OBFS4PROXY_PATH = "/usr/bin/obfs4proxy"
+
+def makedirs(path):
+    try:
+        return os.makedirs(path)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+def get_address_from_bridge_line(bridge_line):
+    host, port = bridge_line.split()[1].split(":", 1)
+    port = int(port)
+    return (host, port)
+
+def start_tcpdump(basename, addr):
+    bpf = "(src host %(host)s and src port %(port)d) or (dst host %(host)s and dst port %(port)d)" % {"host": addr[0], "port": addr[1]}
+    # http://packetlife.net/blog/2010/mar/19/sniffing-wireshark-non-root-user/
+    # groupadd tcpdump
+    # usermod -a -G tcpdump user
+    # chgrp tcpdump /usr/sbin/tcpdump
+    # setcap cap_net_raw,cap_net_admin=eip /usr/sbin/tcpdump
+    p = subprocess.Popen(["/usr/sbin/tcpdump", "-i", "tun0", "-U", "-B", "4096", "-c", "100", "-w", basename + ".pcap", bpf],
+        stdout=open(basename + ".tcpdump.out", "w"),
+        stderr=open(basename + ".tcpdump.err", "w"))
+    return p
+
+def start_tor(tor_config):
+    assert "DataDirectory" in tor_config
+
+    config = {
+        "SOCKSPort": "auto",
+        "ControlPort": "auto",
+        "CookieAuthentication": "1",
+        "LearnCircuitBuildTimeout": "0",
+        "CircuitBuildTimeout": str(CIRCUIT_BUILD_TIMEOUT),
+        "FetchHidServDescriptors": "0",
+        "ClientTransportPlugin": "obfs4 exec %s" % OBFS4PROXY_PATH,
+        "LogTimeGranularity": "1",
+        "Log": "notice stdout",
+    }
+    config.update(tor_config)
+
+    class Ports(object):
+        socks = None
+        control = None
+    ports = Ports()
+    socks_re = re.compile(r'\bSocks listener listening on port ([0-9]+)\.')
+    control_re = re.compile(r'\bControl listener listening on port ([0-9]+)\.')
+    def init_msg_handler(line):
+        logging.info("tor: %s" % line.encode("unicode_escape"))
+        m = socks_re.search(line)
+        if m is not None:
+            assert ports.socks is None
+            ports.socks = int(m.group(1))
+        m = control_re.search(line)
+        if m is not None:
+            assert ports.control is None
+            ports.control = int(m.group(1))
+
+    logging.info("starting tor with configuration %r" % config)
+
+    proc = stem.process.launch_tor_with_config(
+        config,
+        timeout=START_TOR_TIMEOUT,
+        take_ownership=True,
+        init_msg_handler=init_msg_handler,
+    )
+
+    assert ports.socks is not None
+    assert ports.control is not None
+
+    return proc, ports.socks, ports.control
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s.%(msecs)03d %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logging.Formatter.converter = time.gmtime
+
+# Set timezone to be inherited by tor processes.
+os.environ["TZ"] = "UTC"
+time.tzset()
+
+logging.info("starting")
+
+for nickname, bridge_line in BRIDGE_LINES:
+    datadir = tempfile.mkdtemp(prefix="datadir.", dir=".")
+    logging.info("created temporary DataDirectory %r", datadir)
+    tcpdump_proc = None
+    try:
+        logging.info("starting tcpdump for bridge %r" % nickname)
+        addr = get_address_from_bridge_line(bridge_line)
+        try:
+            tcpdump_proc = start_tcpdump(nickname, addr)
+        except OSError as e:
+            logging.info("failed to start tcpdump: %s", e)
+            # Carry on regardless.
+
+        logging.info("starting tor for bridge %r" % nickname)
+        logging.info("Bridge %s" % bridge_line)
+        tor_config = {
+            "DataDirectory": datadir,
+            "Log": "notice file %s" % os.path.join(".", "%s.log" % nickname),
+            "UseBridges": "1",
+            "Bridge": bridge_line,
+        }
+        try:
+            tor_proc, _, _ = start_tor(tor_config)
+        except OSError as err:
+            logging.info("failed to start tor: %s" % err)
+            continue
+        tor_proc.terminate()
+        tor_proc.wait()
+    finally:
+        logging.info("deleting temporary DataDirectory %r", datadir)
+        shutil.rmtree(datadir)
+        if tcpdump_proc is not None:
+            tcpdump_proc.terminate()

bridgetest.sh

+#!/bin/bash
+
+set -e
+
+SITE="${1:?}"
+
+dirname="$PWD"
+logdirname="log/$SITE/$(date -u +%Y%m%d-%H%M)"
+mkdir -p "$logdirname"
+cd "$logdirname" && "$dirname/bridgetest"

graph.R

+library(data.table)
+library(ggplot2)
+
+
+gala.vpn.working <- {
+	# Evaluate these as.POSIXct calls in the closure's environment
+	# so they won't be re-evaluated on every call.
+	pidbug.start <- as.POSIXct("2016-12-28 03:25:06", tz="UTC")
+	pidbug.end <- as.POSIXct("2017-01-12 19:00:37", tz="UTC")
+	off.20170408T062000 <- as.POSIXct("2017-04-08 06:20:00", tz="UTC")
+	on.20170426T124637 <- as.POSIXct("2017-04-26 12:46:37", tz="UTC")
+	off.20170502T051500 <- as.POSIXct("2017-05-02 05:15:00", tz="UTC")
+	function(timestamp) {
+		(timestamp < pidbug.start) |
+		(pidbug.end <= timestamp & timestamp < off.20170408T062000) |
+		(on.20170426T124637 <= timestamp & timestamp < off.20170502T051500)
+	}
+}
+
+
+args <- commandArgs(trailingOnly=T)
+
+x <- data.table()
+for (filename in args) {
+	x <- rbind(x, fread(filename))
+}
+x$timestamp <- as.POSIXct(x$timestamp, tz="UTC")
+# Filter out the times the gala VPN wasn't working
+# (otherwise it looks like timeouts; i.e. blocking)
+x <- x[site!="gala" | gala.vpn.working(timestamp)]
+
+x.max <- x[ , .SD[which.max(percent)], by=.(site, runid, nickname)]
+setkey(x.max, site, runid, nickname)
+
+
+cat("
+{{{#!html
+<table class=\"wiki\">
+<tr><th>bridge</th><th>US average bootstrap %</th><th>KZ average bootstrap %</th></tr>
+")
+ramp <- colorRamp(c("#d6756b", "#f7fbff"))
+summ <- x.max[gala.vpn.working(timestamp), .(.N, avg.percent=mean(percent)), by=.(site, nickname)]
+for (nick in unique(x$nickname)) {
+	bear <- summ[site=="bear" & nickname==nick]
+	gala <- summ[site=="gala" & nickname==nick]
+	cat(sprintf("<tr><td>%s</td><td align=right style=\"background: %s\">%.2f%%</td><td align=right style=\"background: %s\">%.2f%%</td></tr>\n",
+		nick, rgb(ramp(bear$avg.percent/100)/255), bear$avg.percent, rgb(ramp(gala$avg.percent/100)/255), gala$avg.percent))
+}
+cat("</table>
+}}}
+")
+
+
+pdf(width=8.5, height=14)
+
+# runids <- unique(x$runid)
+# runids <- runids[order(runids)]
+# p <- ggplot(x[x$runid %in% runids[(length(runids)-2):(length(runids)-1)], ])
+# p <- p + geom_step(aes(timestamp, percent, group=sprintf("%s-%s", runid, nickname), color=nickname))
+# p <- p + scale_y_continuous(limits=c(0, 100), breaks=seq(0, 100, 10))
+# p <- p + theme_bw()
+# p
+
+# p <- ggplot(x.max)
+# p <- p + geom_point(aes(nickname, percent, color=site), alpha=0.4, size=0.7, position=position_jitter(width=0.3, height=0))
+# p <- p + scale_y_continuous(limits=c(0, 100))
+# p <- p + coord_flip()
+# p <- p + theme_bw()
+# p <- p + guides(color=guide_legend(override.aes=list(alpha=1, size=2)))
+# p
+
+tmp <- x.max
+tmp$site <- factor(tmp$site, levels=c("bear", "gala"), labels=c("US", "KZ"))
+p <- ggplot(tmp)
+p <- p + geom_point(aes(timestamp, percent, color=site, shape=site, size=site), alpha=0.4)
+p <- p + facet_grid(nickname ~ .)
+p <- p + scale_y_continuous(limits=c(0, 105))
+p <- p + scale_color_brewer(palette="Set1")
+p <- p + scale_shape_manual(values=c(US=4, KZ=16))
+p <- p + scale_size_manual(values=c(US=1.0, KZ=1.0))
+p <- p + theme_bw()
+p <- p + theme(strip.text.y=element_text(angle=0))
+p <- p + theme(legend.position="top")
+p <- p + guides(color=guide_legend(override.aes=list(alpha=1, size=2.5)))
+p

makecsv

+#!/usr/bin/env python2
+
+import csv
+import datetime
+import locale
+import os.path
+import re
+import sys
+
+# For strptime.
+locale.setlocale(locale.LC_ALL, "C")
+
+# Dec 01 20:57:53.000 [notice] Bootstrapped 0%: Starting
+bootstrapped_re = re.compile(r'^(\w+ \d+ \d\d:\d\d:\d\d\.\d\d\d) \[\w+\] Bootstrapped (\d+)%')
+
+csvW = csv.DictWriter(sys.stdout, fieldnames=("timestamp", "site", "runid", "nickname", "percent"))
+csvW.writeheader()
+
+def process_log(f, site, runid, nickname):
+    for line in f:
+        m = bootstrapped_re.match(line)
+        if m is None:
+            continue
+
+        timestamp = datetime.datetime.strptime(m.group(1), "%b %d %H:%M:%S.%f")
+        # tor logs don't contain the year, so grab it from the runid.
+        timestamp = timestamp.replace(year=int(runid[:4]))
+        percent = m.group(2)
+
+        row = {
+            "timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
+            "site": site,
+            "runid": runid,
+            "nickname": nickname,
+            "percent": percent,
+        }
+        csvW.writerow(row)
+
+for filename in sys.stdin:
+    filename = filename.strip()
+
+    nickname, ext = os.path.splitext(os.path.basename(filename))
+
+    if ext != ".log":
+        continue
+    if nickname == "main":
+        continue
+
+    parent = os.path.dirname(filename)
+    runid = os.path.basename(parent)
+    parent = os.path.dirname(parent)
+    site = os.path.basename(parent)
+
+    with open(filename) as f:
+        process_log(f, site, runid, nickname)