Commit 2b493227 authored by David Fifield's avatar David Fifield Committed by Cecylia Bocovich
Browse files
parents
[SITENAME] is an arbitrary identifier for the probe site
that you have to choose.
Add to crontab to run hourly tests:
0 */1 * * * cd ~/kz && ./bridgetest.sh [SITENAME]
Generate a CSV file from logs:
find log -name '*.log' | sort | ./makecsv > bridgetest.csv
Make a graph:
Rscript graph.R bridgetest.csv
#!/usr/bin/env python2
import errno
import logging
import os
import re
import shutil
import subprocess
import tempfile
import time
import stem.process
BRIDGE_LINES = (
)
START_TOR_TIMEOUT = 3 * 60
CIRCUIT_BUILD_TIMEOUT = 60
OBFS4PROXY_PATH = "/usr/bin/obfs4proxy"
def makedirs(path):
try:
return os.makedirs(path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
def get_address_from_bridge_line(bridge_line):
host, port = bridge_line.split()[1].split(":", 1)
port = int(port)
return (host, port)
def start_tcpdump(basename, addr):
bpf = "(src host %(host)s and src port %(port)d) or (dst host %(host)s and dst port %(port)d)" % {"host": addr[0], "port": addr[1]}
# http://packetlife.net/blog/2010/mar/19/sniffing-wireshark-non-root-user/
# groupadd tcpdump
# usermod -a -G tcpdump user
# chgrp tcpdump /usr/sbin/tcpdump
# setcap cap_net_raw,cap_net_admin=eip /usr/sbin/tcpdump
p = subprocess.Popen(["/usr/sbin/tcpdump", "-i", "tun0", "-U", "-B", "4096", "-c", "100", "-w", basename + ".pcap", bpf],
stdout=open(basename + ".tcpdump.out", "w"),
stderr=open(basename + ".tcpdump.err", "w"))
return p
def start_tor(tor_config):
assert "DataDirectory" in tor_config
config = {
"SOCKSPort": "auto",
"ControlPort": "auto",
"CookieAuthentication": "1",
"LearnCircuitBuildTimeout": "0",
"CircuitBuildTimeout": str(CIRCUIT_BUILD_TIMEOUT),
"FetchHidServDescriptors": "0",
"ClientTransportPlugin": "obfs4 exec %s" % OBFS4PROXY_PATH,
"LogTimeGranularity": "1",
"Log": "notice stdout",
}
config.update(tor_config)
class Ports(object):
socks = None
control = None
ports = Ports()
socks_re = re.compile(r'\bSocks listener listening on port ([0-9]+)\.')
control_re = re.compile(r'\bControl listener listening on port ([0-9]+)\.')
def init_msg_handler(line):
logging.info("tor: %s" % line.encode("unicode_escape"))
m = socks_re.search(line)
if m is not None:
assert ports.socks is None
ports.socks = int(m.group(1))
m = control_re.search(line)
if m is not None:
assert ports.control is None
ports.control = int(m.group(1))
logging.info("starting tor with configuration %r" % config)
proc = stem.process.launch_tor_with_config(
config,
timeout=START_TOR_TIMEOUT,
take_ownership=True,
init_msg_handler=init_msg_handler,
)
assert ports.socks is not None
assert ports.control is not None
return proc, ports.socks, ports.control
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s.%(msecs)03d %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logging.Formatter.converter = time.gmtime
# Set timezone to be inherited by tor processes.
os.environ["TZ"] = "UTC"
time.tzset()
logging.info("starting")
for nickname, bridge_line in BRIDGE_LINES:
datadir = tempfile.mkdtemp(prefix="datadir.", dir=".")
logging.info("created temporary DataDirectory %r", datadir)
tcpdump_proc = None
try:
logging.info("starting tcpdump for bridge %r" % nickname)
addr = get_address_from_bridge_line(bridge_line)
try:
tcpdump_proc = start_tcpdump(nickname, addr)
except OSError as e:
logging.info("failed to start tcpdump: %s", e)
# Carry on regardless.
logging.info("starting tor for bridge %r" % nickname)
logging.info("Bridge %s" % bridge_line)
tor_config = {
"DataDirectory": datadir,
"Log": "notice file %s" % os.path.join(".", "%s.log" % nickname),
"UseBridges": "1",
"Bridge": bridge_line,
}
try:
tor_proc, _, _ = start_tor(tor_config)
except OSError as err:
logging.info("failed to start tor: %s" % err)
continue
tor_proc.terminate()
tor_proc.wait()
finally:
logging.info("deleting temporary DataDirectory %r", datadir)
shutil.rmtree(datadir)
if tcpdump_proc is not None:
tcpdump_proc.terminate()
#!/bin/bash
set -e
SITE="${1:?}"
dirname="$PWD"
logdirname="log/$SITE/$(date -u +%Y%m%d-%H%M)"
mkdir -p "$logdirname"
cd "$logdirname" && "$dirname/bridgetest"
library(data.table)
library(ggplot2)
gala.vpn.working <- {
# Evaluate these as.POSIXct calls in the closure's environment
# so they won't be re-evaluated on every call.
pidbug.start <- as.POSIXct("2016-12-28 03:25:06", tz="UTC")
pidbug.end <- as.POSIXct("2017-01-12 19:00:37", tz="UTC")
off.20170408T062000 <- as.POSIXct("2017-04-08 06:20:00", tz="UTC")
on.20170426T124637 <- as.POSIXct("2017-04-26 12:46:37", tz="UTC")
off.20170502T051500 <- as.POSIXct("2017-05-02 05:15:00", tz="UTC")
function(timestamp) {
(timestamp < pidbug.start) |
(pidbug.end <= timestamp & timestamp < off.20170408T062000) |
(on.20170426T124637 <= timestamp & timestamp < off.20170502T051500)
}
}
args <- commandArgs(trailingOnly=T)
x <- data.table()
for (filename in args) {
x <- rbind(x, fread(filename))
}
x$timestamp <- as.POSIXct(x$timestamp, tz="UTC")
# Filter out the times the gala VPN wasn't working
# (otherwise it looks like timeouts; i.e. blocking)
x <- x[site!="gala" | gala.vpn.working(timestamp)]
x.max <- x[ , .SD[which.max(percent)], by=.(site, runid, nickname)]
setkey(x.max, site, runid, nickname)
cat("
{{{#!html
<table class=\"wiki\">
<tr><th>bridge</th><th>US average bootstrap %</th><th>KZ average bootstrap %</th></tr>
")
ramp <- colorRamp(c("#d6756b", "#f7fbff"))
summ <- x.max[gala.vpn.working(timestamp), .(.N, avg.percent=mean(percent)), by=.(site, nickname)]
for (nick in unique(x$nickname)) {
bear <- summ[site=="bear" & nickname==nick]
gala <- summ[site=="gala" & nickname==nick]
cat(sprintf("<tr><td>%s</td><td align=right style=\"background: %s\">%.2f%%</td><td align=right style=\"background: %s\">%.2f%%</td></tr>\n",
nick, rgb(ramp(bear$avg.percent/100)/255), bear$avg.percent, rgb(ramp(gala$avg.percent/100)/255), gala$avg.percent))
}
cat("</table>
}}}
")
pdf(width=8.5, height=14)
# runids <- unique(x$runid)
# runids <- runids[order(runids)]
# p <- ggplot(x[x$runid %in% runids[(length(runids)-2):(length(runids)-1)], ])
# p <- p + geom_step(aes(timestamp, percent, group=sprintf("%s-%s", runid, nickname), color=nickname))
# p <- p + scale_y_continuous(limits=c(0, 100), breaks=seq(0, 100, 10))
# p <- p + theme_bw()
# p
# p <- ggplot(x.max)
# p <- p + geom_point(aes(nickname, percent, color=site), alpha=0.4, size=0.7, position=position_jitter(width=0.3, height=0))
# p <- p + scale_y_continuous(limits=c(0, 100))
# p <- p + coord_flip()
# p <- p + theme_bw()
# p <- p + guides(color=guide_legend(override.aes=list(alpha=1, size=2)))
# p
tmp <- x.max
tmp$site <- factor(tmp$site, levels=c("bear", "gala"), labels=c("US", "KZ"))
p <- ggplot(tmp)
p <- p + geom_point(aes(timestamp, percent, color=site, shape=site, size=site), alpha=0.4)
p <- p + facet_grid(nickname ~ .)
p <- p + scale_y_continuous(limits=c(0, 105))
p <- p + scale_color_brewer(palette="Set1")
p <- p + scale_shape_manual(values=c(US=4, KZ=16))
p <- p + scale_size_manual(values=c(US=1.0, KZ=1.0))
p <- p + theme_bw()
p <- p + theme(strip.text.y=element_text(angle=0))
p <- p + theme(legend.position="top")
p <- p + guides(color=guide_legend(override.aes=list(alpha=1, size=2.5)))
p
#!/usr/bin/env python2
import csv
import datetime
import locale
import os.path
import re
import sys
# For strptime.
locale.setlocale(locale.LC_ALL, "C")
# Dec 01 20:57:53.000 [notice] Bootstrapped 0%: Starting
bootstrapped_re = re.compile(r'^(\w+ \d+ \d\d:\d\d:\d\d\.\d\d\d) \[\w+\] Bootstrapped (\d+)%')
csvW = csv.DictWriter(sys.stdout, fieldnames=("timestamp", "site", "runid", "nickname", "percent"))
csvW.writeheader()
def process_log(f, site, runid, nickname):
for line in f:
m = bootstrapped_re.match(line)
if m is None:
continue
timestamp = datetime.datetime.strptime(m.group(1), "%b %d %H:%M:%S.%f")
# tor logs don't contain the year, so grab it from the runid.
timestamp = timestamp.replace(year=int(runid[:4]))
percent = m.group(2)
row = {
"timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
"site": site,
"runid": runid,
"nickname": nickname,
"percent": percent,
}
csvW.writerow(row)
for filename in sys.stdin:
filename = filename.strip()
nickname, ext = os.path.splitext(os.path.basename(filename))
if ext != ".log":
continue
if nickname == "main":
continue
parent = os.path.dirname(filename)
runid = os.path.basename(parent)
parent = os.path.dirname(parent)
site = os.path.basename(parent)
with open(filename) as f:
process_log(f, site, runid, nickname)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment