Merge branch 'issue/3' into 'master'

Issue/3

Closes #3

See merge request !1

README.md

 Bridgestrap
 ===========
 
-Bridgestrap implements a API (for machines) and a Web interface (for people) to
-test a given bridge line by spawning a tor instance and having it bootstrap
-over the bridge line.
+Bridgestrap implements an API (for machines) and a Web interface (for people)
+to test Tor bridge lines by making a Tor instance fetch the bridges'
+descriptor.
 
 Installation
 ------------
@@ -24,17 +24,17 @@ the address and port that bridgestrap is listening on.  Use the argument
 Input
 -----
 
-Clients send bridge lines to the following API, using an HTTP GET request, and
-place the bridge line in the request body:
+Clients send one or more bridge lines to the following API, using an HTTP GET
+request, and place the bridge lines in the request body:
 
       https://HOST/bridge-state
 
 The request body must look as follows:
 
-      {"bridge_line": "BRIDGE_LINE"}
+      {"bridge_lines": ["BRIDGE_LINE_1", ..., "BRIDGE_LINE_N"]}
 
-The value of "bridge_line" can be any bridge line (excluding the "Bridge"
-prefix) that tor accepts.  Here are a few examples:
+The "BRIDGE_LINE" strings in the list may contain any bridge line (excluding
+the "Bridge" prefix) that tor accepts.  Here are a few examples:
 
 * `1.2.3.4:1234`
 * `1.2.3.4:1234 1234567890ABCDEF1234567890ABCDEF12345678`
@@ -42,7 +42,10 @@ prefix) that tor accepts.  Here are a few examples:
 
 You can test bridgestrap's API over the command line as follows:
 
-      curl -X GET localhost:5000/bridge-state -d '{"bridge_line": "BRIDGE_LINE"}'
+      curl -X GET localhost:5000/bridge-state -d '{"bridge_lines": ["BRIDGE_LINE"]}'
+
+You can also use the script test-bridge-lines in the "script" directory to test
+a batch of bridge lines.
 
 Output
 ------
@@ -50,37 +53,59 @@ Output
 The service responds with the following JSON:
 
       {
-        "functional": BOOL,
-        "error": "STRING", (only present if "functional" is false.)
+        "bridge_results": {
+          "BRIDGE_LINE_1": {
+            "functional: BOOL,
+            "error": "STRING", (only present if "functional" is false)
+          },
+          ...
+          "BRIDGE_LINE_N": {
+            ...
+          }
+        },
+        "error": "STRING", (only present if the entire test failed)
         "time": FLOAT
       }
 
-If tor could bootstrap over the given bridge line, "functional" is "true" and
-"false" otherwise.  If "functional" is "false", "error" will contain an error
-string.  "time" is a float that represents the number of seconds that
-bridgestrap's test took.
+In a nutshell, the "bridge_results" dictionary maps bridge lines (as they were
+provided in the request) to a dictionary consisting of two keys: "functional"
+is set to "true" if tor could fetch the bridge's descriptor.  If tor was unable
+to fetch the bridge's descriptor, "functional" is set to "false" and the
+"error" key maps to an error string.
+
+In addition to the "bridge_results" dictionary, the response may contain an
+optional "error" key if the entire test failed (e.g. if bridgestrap failed to
+communicate with its tor instance).  Finally, "time" is a float that represents
+the number of seconds that the test took.
 
 Here are a few examples:
 
     {
-      "functional":false,
-      "error":"Invalid JSON request.",
-      "time":0
+      "bridge_results": {
+      },
+      "error": "something truly ominous happened",
+      "time": 0.32
     }
 
     {
-      "functional":false,
-      "error":"Oct 23 17:36:57.000 [warn] Problem bootstrapping. Stuck at 10%: Finishing handshake with directory server. (DONE; DONE; count 1; recommendation warn; host [REDACTED])",
-      "time":32.31
+      "bridge_results": {
+        "obfs4 1.2.3.4:1234 cert=fJRlJc0T7i2Qkw3SyLQq+M6iTGs9ghLHK65LBy/MQewXJpNOKFq63Om1JHVkLlrmEBbX1w iat-mode=0": {
+          "functional": true
+        },
+        "1.2.3.4:1234": {
+          "functional": false,
+          "error": "timed out waiting for bridge descriptor"
         }
-
-    {
-      "functional":false,
-      "error":"Oct 23 17:34:57.680 [warn] Too few items to Bridge line.",
-      "time":0.013
+      },
+      "time": 3.1824
     }
 
     {
-      "functional":true,
-      "time":13.161
+      "bridge_results": {
+        "1.2.3.4:1234 1234567890ABCDEF1234567890ABCDEF12345678": {
+          "functional": false,
+          "error": "timed out waiting for bridge descriptor"
+        }
+      },
+      "time": 0
     }

cache.go

+package main
+
+import (
+	"encoding/gob"
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"sync"
+	"time"
+)
+
+const (
+	// Cache test results for one week.
+	CacheValidity = 7 * 24 * time.Hour
+)
+
+var cacheMutex sync.Mutex
+var cache TestCache = make(TestCache)
+
+// Regular expression that captures the address:port part of a bridge line (for
+// both IPv4 and IPv6 addresses).
+var AddrPortBridgeLine = regexp.MustCompile(`[0-9a-z\[\]\.:]+:[0-9]{1,5}`)
+
+// CacheEntry represents an entry in our cache of bridges that we recently
+// tested.  Error is nil if a bridge works, and otherwise holds an error
+// string.  Time determines when we tested the bridge.
+type CacheEntry struct {
+	// We're using a string instead of an error here because golang's gob
+	// package doesn't know how to deal with an error:
+	// <https://github.com/golang/go/issues/23340>
+	Error string
+	Time  time.Time
+}
+
+// bridgeLineToAddrPort takes a bridge line as input and returns a string
+// consisting of the bridge's addr:port (for both IPv4 and IPv6 addresses).
+func bridgeLineToAddrPort(bridgeLine string) (string, error) {
+
+	result := string(AddrPortBridgeLine.Find([]byte(bridgeLine)))
+	if result == "" {
+		return result, fmt.Errorf("could not extract addr:port from bridge line")
+	} else {
+		return result, nil
+	}
+}
+
+// TestCache maps a bridge's addr:port tuple to a cache entry.
+type TestCache map[string]*CacheEntry
+
+// WriteToDisk writes our test result cache to disk, allowing it to persist
+// across program restarts.
+func (tc *TestCache) WriteToDisk(cacheFile string) error {
+
+	fh, err := os.Create(cacheFile)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	enc := gob.NewEncoder(fh)
+	cacheMutex.Lock()
+	err = enc.Encode(*tc)
+	if err == nil {
+		log.Printf("Wrote cache with %d elements to %q.",
+			len(*tc), cacheFile)
+	}
+	cacheMutex.Unlock()
+
+	return err
+}
+
+// ReadFromDisk reads our test result cache from disk.
+func (tc *TestCache) ReadFromDisk(cacheFile string) error {
+
+	fh, err := os.Open(cacheFile)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	dec := gob.NewDecoder(fh)
+	cacheMutex.Lock()
+	err = dec.Decode(tc)
+	if err == nil {
+		log.Printf("Read cache with %d elements from %q.",
+			len(*tc), cacheFile)
+	}
+	cacheMutex.Unlock()
+
+	return err
+}
+
+// IsCached returns a cache entry if the given bridge line has been tested
+// recently (as determined by CacheValidity), and nil otherwise.
+func (tc *TestCache) IsCached(bridgeLine string) *CacheEntry {
+
+	// First, prune expired cache entries.
+	now := time.Now().UTC()
+	cacheMutex.Lock()
+	for index, entry := range *tc {
+		if entry.Time.Before(now.Add(-CacheValidity)) {
+			delete(*tc, index)
+		}
+	}
+	cacheMutex.Unlock()
+
+	addrPort, err := bridgeLineToAddrPort(bridgeLine)
+	if err != nil {
+		return nil
+	}
+
+	cacheMutex.Lock()
+	var r *CacheEntry = (*tc)[addrPort]
+	cacheMutex.Unlock()
+
+	return r
+}
+
+// AddEntry adds an entry for the given bridge and test result to our cache.
+func (tc *TestCache) AddEntry(bridgeLine string, result error) {
+
+	addrPort, err := bridgeLineToAddrPort(bridgeLine)
+	if err != nil {
+		return
+	}
+
+	var errorStr string
+	if result == nil {
+		errorStr = ""
+	} else {
+		errorStr = result.Error()
+	}
+	cacheMutex.Lock()
+	(*tc)[addrPort] = &CacheEntry{errorStr, time.Now()}
+	cacheMutex.Unlock()
+}

cache_test.go

+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestCacheFunctions(t *testing.T) {
+
+	cache := make(TestCache)
+	bridgeLine := "obfs4 127.0.0.1:1 cert=foo iat-mode=0"
+
+	e := cache.IsCached(bridgeLine)
+	if e != nil {
+		t.Errorf("Cache is empty but marks bridge line as existing.")
+	}
+
+	cache.AddEntry(bridgeLine, nil)
+	e = cache.IsCached(bridgeLine)
+	if e == nil {
+		t.Errorf("Could not retrieve existing element from cache.")
+	}
+
+	testError := fmt.Errorf("bridge is on fire")
+	cache.AddEntry(bridgeLine, testError)
+	e = cache.IsCached(bridgeLine)
+	if e.Error != testError.Error() {
+		t.Errorf("Got test result %q but expected %q.", e.Error, testError)
+	}
+}
+
+func TestCacheExpiration(t *testing.T) {
+
+	cache := make(TestCache)
+
+	const shortForm = "2006-Jan-02"
+	expiry, _ := time.Parse(shortForm, "2000-Jan-01")
+	bridgeLine1 := "1.1.1.1:1111"
+	cache[bridgeLine1] = &CacheEntry{"", expiry}
+
+	bridgeLine2 := "2.2.2.2:2222"
+	cache[bridgeLine2] = &CacheEntry{"", time.Now()}
+
+	e := cache.IsCached(bridgeLine1)
+	if e != nil {
+		t.Errorf("Expired cache entry was not successfully pruned.")
+	}
+
+	e = cache.IsCached(bridgeLine2)
+	if e == nil {
+		t.Errorf("Valid cache entry was incorrectly pruned.")
+	}
+}
+
+func BenchmarkIsCached(b *testing.B) {
+
+	getRandAddrPort := func() string {
+		return fmt.Sprintf("%d.%d.%d.%d:%d",
+			rand.Intn(256), rand.Intn(256), rand.Intn(256), rand.Intn(256), rand.Intn(65536))
+	}
+	getRandError := func() error {
+		errors := []error{nil, errors.New("censorship"), errors.New("no censorship")}
+		return errors[rand.Intn(len(errors))]
+	}
+
+	numCacheEntries := 10000
+	cache := make(TestCache)
+	for i := 0; i < numCacheEntries; i++ {
+		cache.AddEntry(getRandAddrPort(), getRandError())
+	}
+
+	// How long does it take to iterate over numCacheEntries cache entries?
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		cache.IsCached("invalid bridge line")
+	}
+}
+
+func TestCacheSerialisation(t *testing.T) {
+
+	cache := make(TestCache)
+	testError := fmt.Errorf("foo")
+	cache.AddEntry("1.1.1.1:1", testError)
+	cache.AddEntry("2.2.2.2:2", fmt.Errorf("bar"))
+
+	tmpFh, err := ioutil.TempFile(os.TempDir(), "cache-file-")
+	if err != nil {
+		t.Errorf("Could not create temporary file for test: %s", err)
+	}
+	defer os.Remove(tmpFh.Name())
+
+	err = cache.WriteToDisk(tmpFh.Name())
+	if err != nil {
+		t.Errorf("Failed to write cache to disk: %s", err)
+	}
+	err = cache.ReadFromDisk(tmpFh.Name())
+	if err != nil {
+		t.Errorf("Failed to read cache from disk: %s", err)
+	}
+
+	if len(cache) != 2 {
+		t.Errorf("Cache supposed to contain but two elements but has %d.", len(cache))
+	}
+
+	e1 := cache.IsCached("1.1.1.1:1")
+	if e1 == nil {
+		t.Errorf("Cache element supposed to exist but doesn't.")
+	}
+	if e1.Error != testError.Error() {
+		t.Errorf("Error string expected to be %q but is %q.", testError, e1.Error)
+	}
+}
+
+func TestCacheConcurrency(t *testing.T) {
+
+	cache := make(TestCache)
+	max := 10000
+	doneReading := make(chan bool)
+	doneWriting := make(chan bool)
+
+	// Trigger many concurrent reads and writes, to verify that there are no
+	// synchronisation issues.
+	go func() {
+		for i := 0; i < max; i++ {
+			ipAddr := net.IPv4(byte((i>>24)&0xff),
+				byte((i>>16)&0xff),
+				byte((i>>8)&0xff),
+				byte(i&0xff))
+			cache.AddEntry(fmt.Sprintf("%s:1234", ipAddr.String()), nil)
+		}
+		doneWriting <- true
+	}()
+
+	go func() {
+		for i := 0; i < max; i++ {
+			ipAddr := net.IPv4(byte((i>>24)&0xff),
+				byte((i>>16)&0xff),
+				byte((i>>8)&0xff),
+				byte(i&0xff))
+			cache.IsCached(fmt.Sprintf("%s:1234", ipAddr.String()))
+		}
+		doneReading <- true
+	}()
+
+	<-doneReading
+	<-doneWriting
+}

events.go

+package main
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"math"
+	"math/rand"
+	"regexp"
+	"strconv"
+)
+
+const (
+	// The number of hex digits in a bridge's fingerprint, e.g.:
+	// 0123456789ABCDEF0123456789ABCDEF01234567
+	BridgeFingerprintLen = 40
+	BridgeStatePending   = iota
+	BridgeStateSuccess
+	BridgeStateFailure
+)
+
+// Examples of ORCONN events:
+//   650 ORCONN 90.41.70.32:7434 LAUNCHED ID=75
+//   650 ORCONN $D9A82D2F9C2F65A18407B1D2B764F130847F8B5D LAUNCHED ID=38
+//   650 ORCONN $D9A82D2F9C2F65A18407B1D2B764F130847F8B5D~dragon CLOSED REASON=DONE ID=42
+//   650 ORCONN $9695DFC35FFEB861329B9F1AB04C46397020CE31~moria1 CLOSED REASON=IOERROR ID=1833
+//   650 ORCONN 128.31.0.33:9101 FAILED REASON=TIMEOUT NCIRCS=1 ID=1836
+//   650 ORCONN $D9A82D2F9C2F65A18407B1D2B764F130847F8B5D~dragon CONNECTED ID=42
+var OrConnFields = regexp.MustCompile(`ORCONN ([^ ]*) ([^ ]*).*ID=([0-9]*)`)
+var OrConnEvent = regexp.MustCompile(`^650 ORCONN`)
+var OrConnReasonField = regexp.MustCompile(`^650 ORCONN.*REASON=([A-Z]*)`)
+var NewDescEvent = regexp.MustCompile(`^650 NEWDESC`)
+var Fingerprint = regexp.MustCompile(`([A-F0-9]{40})`)
+
+// extractFingerprint extracts a bridge's fingerprint from the given ORCONN or
+// NEWDESC line.
+func extractFingerprint(line string) (string, error) {
+
+	result := string(Fingerprint.Find([]byte(line)))
+
+	if result == "" {
+		return result, errors.New("could not extract fingerprint from line")
+	} else {
+		return result, nil
+	}
+}
+
+// getFailureDesc takes as input an ORCONN line and maps the error code to a
+// more descriptive string.
+func getFailureDesc(line string) (string, error) {
+
+	// See the following part of our control specification:
+	// https://gitweb.torproject.org/torspec.git/tree/control-spec.txt?id=1ecf3f66586816fc718e38f8cd7cbb23fa9b81f5#n2472
+	var reasons = map[string]string{
+		"DONE":           "The OR connection has shut down cleanly.",
+		"CONNECTREFUSED": "We got an ECONNREFUSED while connecting to the target OR.",
+		"IDENTITY":       "We connected to the OR, but found that its identity was not what we expected.",
+		"CONNECTRESET":   "We got an ECONNRESET or similar IO error from the connection with the OR.",
+		"TIMEOUT":        "We got an ETIMEOUT or similar IO error from the connection with the OR, or we're closing the connection for being idle for too long.",
+		"NOROUTE":        "We got an ENOTCONN, ENETUNREACH, ENETDOWN, EHOSTUNREACH, or similar error while connecting to the OR.",
+		"IOERROR":        "We got some other IO error on our connection to the OR.",
+		"RESOURCELIMIT":  "We don't have enough operating system resources (file descriptors, buffers, etc) to connect to the OR.",
+		"PT_MISSING":     "No pluggable transport was available.",
+		"MISC":           "The OR connection closed for some other reason.",
+	}
+
+	matches := OrConnReasonField.FindStringSubmatch(line)
+	expectedMatches := 2
+	if len(matches) != expectedMatches {
+		return "", fmt.Errorf("expected %d but got %d matches", expectedMatches, len(matches))
+	}
+
+	desc, exists := reasons[matches[1]]
+	if !exists {
+		return "", fmt.Errorf("could not find reason for %q", matches[1])
+	}
+
+	return desc, nil
+}
+
+// calcMatchLength determines the number of digits that we should compare for
+// in an ORCONN LAUNCHED event.
+func calcMatchLength(target1, target2 string) int {
+
+	// Start with min(target1, target2).
+	length := len(target1)
+	if len(target2) < len(target1) {
+		length = len(target2)
+	}
+
+	// We're dealing with a "$fingerprint~name" pattern.
+	if length > BridgeFingerprintLen+1 {
+		length = BridgeFingerprintLen + 1
+	}
+
+	return length
+}
+
+// TorEventState represents a state machine that we use to parse ORCONN and
+// NEWDESC events.
+type TorEventState struct {
+	ConnIds     map[int]bool
+	State       int
+	Reason      string
+	Fingerprint string
+	Target      string // If present, the fingerprint; otherwise address:port.
+	TestId      int
+}
+
+// NewTorEventState returns a new TorEventState struct.
+func NewTorEventState(target string) *TorEventState {
+
+	testId := rand.Intn(math.MaxInt32)
+	log.Printf("%x: Creating new TorEventState with %s bridge identifier.", testId, target)
+	return &TorEventState{ConnIds: make(map[int]bool),
+		Target: target,
+		TestId: testId,
+		State:  BridgeStatePending}
+}
+
+// Feed takes as input a new Tor event line.
+func (t *TorEventState) Feed(line string) {
+
+	if OrConnEvent.MatchString(line) {
+		t.processOrConnLine(line)
+	} else if NewDescEvent.MatchString(line) {
+		t.processNewDescLine(line)
+	} else {
+		log.Printf("%x: Bug: Received an unexpected event %q.", t.TestId, line)
+	}
+}
+
+// processOrConnLine processes ORCONN lines.
+func (t *TorEventState) processOrConnLine(line string) {
+
+	matches := OrConnFields.FindStringSubmatch(line)
+	if len(matches) != 4 {
+		log.Printf("%x: Bug: Unexpected number of substring matches in %q", t.TestId, line)
+		return
+	}
+
+	target := matches[1]
+	eventType := matches[2]
+	i, err := strconv.Atoi(matches[3])
+	if err != nil {
+		log.Printf("%x: Bug: Could not convert %q to integer: %s", t.TestId, matches[2], err)
+		return
+	}
+
+	// Are we dealing with a new ORCONN for our bridge line?  If so, add its ID
+	// to our map, so we can keep track of it.
+	if eventType == "LAUNCHED" {
+		matchLen := calcMatchLength(target, t.Target)
+		if target == t.Target[:matchLen] {
+			log.Printf("%x: Adding ID %d to map.", t.TestId, i)
+			t.ConnIds[i] = true
+		}
+	}
+
+	// Are we dealing with an ORCONN for a bridge line that isn't ours?  If so,
+	// let's get outta here.
+	if _, exists := t.ConnIds[i]; !exists {
+		return
+	}
+
+	// Now decide what to do.  Here are the event types we're dealing with:
+	// https://gitweb.torproject.org/torspec.git/tree/control-spec.txt#n2448
+	switch eventType {
+	case "FAILED":
+		// An ORCONN failed.  Was it ours?
+		if _, exists := t.ConnIds[i]; exists {
+			log.Printf("%x: Setting ORCONN failure.", t.TestId)
+			t.State = BridgeStateFailure
+		}
+
+		// Extract the "REASON" field to learn what happened.
+		desc, err := getFailureDesc(line)
+		if err != nil {
+			log.Printf("%x: Bug: %s", t.TestId, err)
+		} else {
+			log.Printf("%x: ORCONN failed because: %s", t.TestId, desc)
+		}
+		t.Reason = desc
+	case "CONNECTED":
+		fingerprint, err := extractFingerprint(line)
+		if err == nil {
+			log.Printf("%x: Setting fingerprint to %s.", t.TestId, fingerprint)
+			t.Fingerprint = fingerprint
+		} else {
+			log.Printf("%x: Bug: Failed to extract fingerprint from %q.", t.TestId, line)
+		}
+
+		// An ORCONN succeeded.  Was it ours?
+		if _, exists := t.ConnIds[i]; exists {
+			log.Printf("%x: ORCONN success.  One step closer to NEWDESC.", t.TestId)
+		}
+	}
+}
+
+// processNewDescLine processes NEWDESC lines.
+func (t *TorEventState) processNewDescLine(line string) {
+
+	// Examples of valid NEWDESC events:
+	//   650 NEWDESC $CDF2E852BF539B82BD10E27E9115A31734E378C2~Lisbeth
+	//   650 NEWDESC $CDF2E852BF539B82BD10E27E9115A31734E378C2
+	fingerprint, err := extractFingerprint(line)
+	if err != nil {
+		log.Printf("%x: Bug: Could not extract fingerprint from %q.", t.TestId, line)
+		return
+	}
+
+	// Is the NEWDESC event ours?
+	if fingerprint == t.Fingerprint {
+		log.Printf("%x: Received NEWDESC event for our bridge.", t.TestId)
+		t.State = BridgeStateSuccess
+	}
+}

events_test.go

+package main