Add 3 type interactive connectivity type support for broker, client, and proxy (e152e8ad) · Commits · The Tor Project / Anti-censorship / Pluggable Transports / Snowflake

broker/broker.go

+38 −21

Changes for broker/broker.go: 38 added lines, 21 removed lines.
Original line number	Diff line number	Diff line
		@@ -23,22 +23,22 @@ import (
		"syscall"
		"time"

		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/ipsetsink/sinkcluster"

		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/bridgefingerprint"

		"github.com/aws/aws-sdk-go-v2/aws"
		"github.com/aws/aws-sdk-go-v2/config"
		"github.com/aws/aws-sdk-go-v2/service/sqs"
		"github.com/prometheus/client_golang/prometheus/promhttp"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil/safelog"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/namematcher"
		"golang.org/x/crypto/acme/autocert"

		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/bridgefingerprint"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/ipsetsink/sinkcluster"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/namematcher"
		)

		type BrokerContext struct {
		unrestrictedPool *SnowflakePool
		restrictedPool *SnowflakePool
		openPool *SnowflakePool
		moderatePool *SnowflakePool
		strictPool *SnowflakePool
		// Maps keeping track of snowflakeIDs required to match SDP answers from
		// the second http POST. Restricted snowflakes can only be matched up with
		// clients behind an unrestricted NAT.
		@@ -77,8 +77,9 @@ func NewBrokerContext(
		bridgeListHolder.LoadBridgeInfo(bytes.NewReader([]byte(DefaultBridges)))

		return &BrokerContext{
		unrestrictedPool: NewSnowflakePool(),
		restrictedPool: NewSnowflakePool(),
		openPool: NewSnowflakePool(),
		moderatePool: NewSnowflakePool(),
		strictPool: NewSnowflakePool(),
		idToSnowflake: make(map[string]*Snowflake),
		proxyPolls: make(chan *ProxyPoll),
		metrics: metrics,
		@@ -135,10 +136,22 @@ func (ctx *BrokerContext) Broker() {
		}

		func (ctx BrokerContext) GetPool(poll ProxyPoll) *SnowflakePool {
		if poll.natType == NATUnrestricted {
		return ctx.unrestrictedPool
		switch poll.natType {
		case NATUnrestricted:
		fallthrough
		case NAT3Open:
		return ctx.openPool
		case NAT3Moderate:
		return ctx.moderatePool
		case NATRestricted:
		fallthrough
		case NAT3Strict:
		fallthrough
		case NATUnknown:
		fallthrough
		default:
		return ctx.strictPool
		}
		return ctx.restrictedPool
		}

		func (ctx *BrokerContext) InstallBridgeListProfile(reader io.Reader) error {
		@@ -158,8 +171,9 @@ func (ctx BrokerContext) CheckProxyRelayPattern(pattern string) bool {
		}

		type pollIntervalConfig struct {
		UnrestrictedPollInterval time.Duration `json:"unrestricted_poll_interval"`
		RestrictedPollInterval time.Duration `json:"restricted_poll_interval"`
		OpenPollInterval time.Duration `json:"open_poll_interval"`
		ModeratePollInterval time.Duration `json:"moderate_poll_interval"`
		StrictPollInterval time.Duration `json:"strict_poll_interval"`
		}

		func (ctx *BrokerContext) LoadPollIntervalFromFile(filename string) error {
		@@ -171,10 +185,12 @@ func (ctx *BrokerContext) LoadPollIntervalFromFile(filename string) error {
		if err := json.Unmarshal(str, &config); err != nil {
		return err
		}
		ctx.unrestrictedPool.SetPollInterval(config.UnrestrictedPollInterval * time.Millisecond)
		ctx.restrictedPool.SetPollInterval(config.RestrictedPollInterval * time.Millisecond)
		log.Printf("Loaded unrestricted poll interval: %d ms", config.UnrestrictedPollInterval)
		log.Printf("Loaded restricted poll interval: %d ms", config.RestrictedPollInterval)
		ctx.openPool.SetPollInterval(config.OpenPollInterval * time.Millisecond)
		ctx.moderatePool.SetPollInterval(config.ModeratePollInterval * time.Millisecond)
		ctx.strictPool.SetPollInterval(config.StrictPollInterval * time.Millisecond)
		log.Printf("Loaded open poll interval: %d ms", config.OpenPollInterval)
		log.Printf("Loaded moderate poll interval: %d ms", config.ModeratePollInterval)
		log.Printf("Loaded strict poll interval: %d ms", config.StrictPollInterval)
		return nil
		}

		@@ -272,7 +288,7 @@ func main() {
		if ipCountPrefix != "" {
		var err error
		files := make(map[string]*os.File)
		for _, name := range []string{"restricted", "unrestricted",
		for _, name := range []string{"strict", "open", "moderate",
		"standalone", "browser", "mobile", "unknown"} {

		files[name], err = os.OpenFile(fmt.Sprintf("%s-%s-%s.log", ipCountPrefix,
		@@ -288,8 +304,9 @@ func main() {
		}
		ctx.metrics.distinctIPWriter = sinkcluster.NewClusterWriter(
		map[string]sinkcluster.WriteSyncer{
		"restricted": files["restricted"],
		"unrestricted": files["unrestricted"],
		"strict": files["strict"],
		"moderate": files["moderate"],
		"open": files["open"],
		"standalone": files["standalone"],
		"browser": files["browser"],
		"mobile": files["mobile"],

broker/ipc.go

+41 −13

Changes for broker/ipc.go: 41 added lines, 13 removed lines.
Original line number	Diff line number	Diff line
		@@ -6,12 +6,13 @@ import (
		"log"
		"time"

		"github.com/prometheus/client_golang/prometheus"

		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/bridgefingerprint"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/constants"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/util"

		"github.com/prometheus/client_golang/prometheus"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/messages"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/nat"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/util"
		)

		const (
		@@ -20,9 +21,15 @@ const (

		MaxPollInterval = time.Hour

		NATUnknown = "unknown"
		NATRestricted = "restricted"
		NATUnrestricted = "unrestricted"
		NATUnknown = nat.NATUnknown
		NATRestricted = nat.NATRestricted
		NATUnrestricted = nat.NATUnrestricted

		// Inlined from common/nat

		NAT3Open = nat.NAT3Open
		NAT3Moderate = nat.NAT3Moderate
		NAT3Strict = nat.NAT3Strict
		)

		type IPC struct {
		@@ -32,6 +39,7 @@ type IPC struct {
		func (i IPC) Debug(_ interface{}, response string) error {
		var unknowns int
		var natRestricted, natUnrestricted, natUnknown int
		var natOpen, natModerate, natStrict int
		proxyTypes := make(map[string]int)

		i.ctx.snowflakeLock.Lock()
		@@ -48,6 +56,12 @@ func (i IPC) Debug(_ interface{}, response string) error {
		natRestricted++
		case NATUnrestricted:
		natUnrestricted++
		case NAT3Open:
		natOpen++
		case NAT3Moderate:
		natModerate++
		case NAT3Strict:
		natStrict++
		default:
		natUnknown++
		}
		@@ -63,6 +77,10 @@ func (i IPC) Debug(_ interface{}, response string) error {
		s += fmt.Sprintf("\nNAT Types available:")
		s += fmt.Sprintf("\n\trestricted: %d", natRestricted)
		s += fmt.Sprintf("\n\tunrestricted: %d", natUnrestricted)
		s += fmt.Sprintf("\n\tstrict: %d", natStrict)
		s += fmt.Sprintf("\n\tmoderate: %d", natModerate)
		s += fmt.Sprintf("\n\topen: %d", natOpen)

		s += fmt.Sprintf("\n\tunknown: %d", natUnknown)

		*response = s
		@@ -105,7 +123,7 @@ func (i IPC) ProxyPolls(arg messages.Arg, response []byte) error {
		log.Println("Warning: cannot process proxy IP: ", err.Error())
		} else {
		i.ctx.metrics.UpdateProxyStats(remoteIP, req.Type, req.NAT)
		go i.ctx.metrics.RecordIPAddress(remoteIP, req.NAT == NATUnrestricted, req.Type)
		go i.ctx.metrics.RecordIPAddress(remoteIP, req.NAT, req.Type)
		}

		var b []byte
		@@ -251,17 +269,27 @@ func (i IPC) ClientOffers(arg messages.Arg, response []byte) error {
		}

		func (i IPC) matchSnowflake(natType string) Snowflake {
		// Proiritize known restricted snowflakes for unrestricted clients
		if natType == NATUnrestricted {
		snowflake := i.ctx.restrictedPool.Pop()
		if snowflake != nil {
		// Match strict, moderate, open pool in order, and skip any non-working pools
		switch natType {
		case NATUnrestricted:
		fallthrough
		case NAT3Open:
		if snowflake := i.ctx.strictPool.Pop(); snowflake != nil {
		return snowflake
		}
		if snowflake := i.ctx.moderatePool.Pop(); snowflake != nil {
		return snowflake
		}

		snowflake := i.ctx.unrestrictedPool.Pop()
		return i.ctx.openPool.Pop()
		case NAT3Moderate:
		if snowflake := i.ctx.moderatePool.Pop(); snowflake != nil {
		return snowflake
		}
		return i.ctx.openPool.Pop()
		default:
		return i.ctx.openPool.Pop()
		}
		}

		func (i IPC) ProxyAnswers(arg messages.Arg, response []byte) error {
		req, err := messages.DecodeProxyAnswerRequest(arg.Body)

broker/metrics.go

+41 −8

Changes for broker/metrics.go: 41 added lines, 8 removed lines.
Original line number	Diff line number	Diff line
		@@ -18,6 +18,7 @@ import (
		"github.com/prometheus/client_golang/prometheus"
		"gitlab.torproject.org/tpo/anti-censorship/geoip"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil/safeprom"

		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/ipsetsink/sinkcluster"
		"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/messages"
		)
		@@ -107,6 +108,12 @@ func (m *Metrics) UpdateProxyStats(addr string, proxyType string, natType string
		m.IncrementCounter("proxy-nat-restricted")
		case NATUnrestricted:
		m.IncrementCounter("proxy-nat-unrestricted")
		case NAT3Strict:
		m.IncrementCounter("proxy-nat-strict")
		case NAT3Moderate:
		m.IncrementCounter("proxy-nat-moderate")
		case NAT3Open:
		m.IncrementCounter("proxy-nat-open")
		default:
		m.IncrementCounter("proxy-nat-unknown")
		}
		@@ -133,10 +140,19 @@ func (m *Metrics) UpdateClientStats(addr string, rendezvousMethod messages.Rende
		switch status {
		case "denied":
		m.IncrementCounter("client-denied")
		if natType == NATUnrestricted {
		m.IncrementCounter("client-unrestricted-denied")
		} else {
		switch natType {
		case NATRestricted:
		m.IncrementCounter("client-restricted-denied")
		case NATUnrestricted:
		m.IncrementCounter("client-unrestricted-denied")
		case NAT3Strict:
		m.IncrementCounter("client-nat-strict-denied")
		case NAT3Moderate:
		m.IncrementCounter("client-nat-moderate-denied")
		case NAT3Open:
		m.IncrementCounter("client-nat-open-denied")
		default:
		m.IncrementCounter("client-nat-unknown-denied")
		}
		case "matched":
		m.IncrementCounter("client-match")
		@@ -258,6 +274,11 @@ func (m *Metrics) printMetrics() {
		m.logger.Println("client-denied-count", binCount(m.loadAndZero("client-denied")))
		m.logger.Println("client-restricted-denied-count", binCount(m.loadAndZero("client-restricted-denied")))
		m.logger.Println("client-unrestricted-denied-count", binCount(m.loadAndZero("client-unrestricted-denied")))
		m.logger.Println("client-nat-strict-denied-count", binCount(m.loadAndZero("client-nat-strict-denied")))
		m.logger.Println("client-nat-moderate-denied-count", binCount(m.loadAndZero("client-nat-moderate-denied")))
		m.logger.Println("client-nat-open-denied-count", binCount(m.loadAndZero("client-nat-open-denied")))
		m.logger.Println("client-nat-unknown-denied-count", binCount(m.loadAndZero("client-nat-unknown-denied")))

		m.logger.Println("client-snowflake-match-count", binCount(m.loadAndZero("client-match")))
		m.logger.Println("client-snowflake-timeout-count", binCount(m.loadAndZero("client-timeout")))

		@@ -271,6 +292,9 @@ func (m *Metrics) printMetrics() {
		m.logger.Println("snowflake-ips-nat-restricted", m.loadAndZero("proxy-nat-restricted"))
		m.logger.Println("snowflake-ips-nat-unrestricted", m.loadAndZero("proxy-nat-unrestricted"))
		m.logger.Println("snowflake-ips-nat-unknown", m.loadAndZero("proxy-nat-unknown"))
		m.logger.Println("snowflake-ips-nat-strict", m.loadAndZero("proxy-nat-strict"))
		m.logger.Println("snowflake-ips-nat-moderate", m.loadAndZero("proxy-nat-moderate"))
		m.logger.Println("snowflake-ips-nat-open", m.loadAndZero("proxy-nat-open"))

		m.ips.Clear()
		}
		@@ -385,12 +409,21 @@ func initPrometheus() *PromMetrics {
		return promMetrics
		}

		func (m *Metrics) RecordIPAddress(ip string, restricted bool, proxyType string) {
		func (m *Metrics) RecordIPAddress(ip string, natType string, proxyType string) {
		if m.distinctIPWriter != nil {
		if restricted {
		m.distinctIPWriter.AddIPToSet("restricted", ip)
		} else {
		m.distinctIPWriter.AddIPToSet("unrestricted", ip)
		switch natType {
		case NAT3Open:
		fallthrough
		case NATUnrestricted:
		m.distinctIPWriter.AddIPToSet("open", ip)
		case NAT3Moderate:
		m.distinctIPWriter.AddIPToSet("moderate", ip)
		case NATRestricted:
		fallthrough
		case NAT3Strict:
		fallthrough
		default:
		m.distinctIPWriter.AddIPToSet("strict", ip)
		}
		switch proxyType {
		case "standalone":

broker/metrics_test.go

+42 −0

Changes for broker/metrics_test.go: 42 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		package main

		import (
		"bytes"
		"log"
		"sync"
		"testing"

		. "github.com/smartystreets/goconvey/convey"
		)

		func TestDeniedClientNATMetrics(t *testing.T) {
		Convey("Denied clients are counted by every supported NAT type", t, func() {
		buf := new(bytes.Buffer)
		metrics, err := NewMetrics(log.New(buf, "", 0))
		So(err, ShouldBeNil)

		for _, natType := range []string{
		NATRestricted,
		NATUnrestricted,
		NAT3Strict,
		NAT3Moderate,
		NAT3Open,
		NATUnknown,
		} {
		metrics.UpdateClientStats("192.0.2.1", "http", natType, "denied")
		}

		metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, `client-denied-count 8
		client-restricted-denied-count 8
		client-unrestricted-denied-count 8
		client-nat-strict-denied-count 8
		client-nat-moderate-denied-count 8
		client-nat-open-denied-count 8
		client-nat-unknown-denied-count 8
		`)

		buf.Reset()
		metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, `client-denied-count 0
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 0
		`)
		})
		}

		func TestFormatAndClearCountryStats(t *testing.T) {
		Convey("given a mapping of country stats", t, func() {
		stats := new(sync.Map)

broker/snowflake-broker_test.go

+61 −15

Changes for broker/snowflake-broker_test.go: 61 added lines, 15 removed lines.
Original line number	Diff line number	Diff line
		@@ -92,10 +92,10 @@ func TestBroker(t *testing.T) {
		i := &IPC{ctx}

		Convey("Adds Snowflake", func() {
		So(ctx.unrestrictedPool.h.Len(), ShouldEqual, 0)
		So(ctx.openPool.h.Len(), ShouldEqual, 0)
		So(len(ctx.idToSnowflake), ShouldEqual, 0)
		addFakeSnowflake(ctx)
		So(ctx.unrestrictedPool.h.Len(), ShouldEqual, 1)
		So(ctx.openPool.h.Len(), ShouldEqual, 1)
		So(len(ctx.idToSnowflake), ShouldEqual, 1)
		})

		@@ -109,13 +109,13 @@ func TestBroker(t *testing.T) {
		close(ctx.proxyPolls)
		}(ctx)
		ctx.Broker()
		So(ctx.unrestrictedPool.h.Len(), ShouldEqual, 1)
		snowflake := ctx.unrestrictedPool.Pop()
		So(ctx.openPool.h.Len(), ShouldEqual, 1)
		snowflake := ctx.openPool.Pop()
		snowflake.offerChannel <- &ClientOffer{sdp: []byte("test offer")}
		offer := <-p.offerChannel
		So(ctx.idToSnowflake["test"], ShouldNotBeNil)
		So(offer.sdp, ShouldResemble, []byte("test offer"))
		So(ctx.unrestrictedPool.h.Len(), ShouldEqual, 0)
		So(ctx.openPool.h.Len(), ShouldEqual, 0)
		})

		Convey("Request an offer from the Snowflake Heap", func() {
		@@ -147,8 +147,12 @@ func TestBroker(t *testing.T) {
		// Ensure that denial is correctly recorded in metrics
		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, `client-denied-count 8
		client-restricted-denied-count 8
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 8
		client-snowflake-match-count 0
		client-snowflake-timeout-count 0
		client-http-count 8
		@@ -180,6 +184,10 @@ client-sqs-ips
		So(buf.String(), ShouldContainSubstring, `client-denied-count 0
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 0
		client-snowflake-match-count 8
		client-snowflake-timeout-count 0
		client-http-count 8
		@@ -252,6 +260,10 @@ client-sqs-ips
		So(buf.String(), ShouldContainSubstring, `client-denied-count 8
		client-restricted-denied-count 8
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 0
		client-snowflake-match-count 0
		client-snowflake-timeout-count 0
		client-http-count 8
		@@ -283,6 +295,10 @@ client-sqs-ips
		So(buf.String(), ShouldContainSubstring, `client-denied-count 0
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 0
		client-snowflake-match-count 8
		client-snowflake-timeout-count 0
		client-http-count 8
		@@ -343,8 +359,12 @@ client-sqs-ips
		// Ensure that denial is correctly recorded in metrics
		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, `client-denied-count 8
		client-restricted-denied-count 8
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 8
		client-snowflake-match-count 0
		client-snowflake-timeout-count 0
		client-http-count 0
		@@ -378,6 +398,10 @@ client-sqs-ips
		So(buf.String(), ShouldContainSubstring, `client-denied-count 0
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 0
		client-snowflake-match-count 8
		client-snowflake-timeout-count 0
		client-http-count 0
		@@ -429,8 +453,12 @@ client-sqs-ips

		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, `client-denied-count 8
		client-restricted-denied-count 8
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 8
		client-snowflake-match-count 0
		client-snowflake-timeout-count 0
		client-http-count 0
		@@ -823,6 +851,10 @@ snowflake-proxy-rejected-for-relay-url-count 0
		client-denied-count 0
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 0
		client-snowflake-match-count 0
		client-snowflake-timeout-count 0
		client-http-count 0
		@@ -834,6 +866,9 @@ client-sqs-ips
		snowflake-ips-nat-restricted 0
		snowflake-ips-nat-unrestricted 0
		snowflake-ips-nat-unknown 4
		snowflake-ips-nat-strict 0
		snowflake-ips-nat-moderate 0
		snowflake-ips-nat-open 0
		`)
		})

		@@ -850,8 +885,12 @@ snowflake-ips-nat-unknown 4

		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, `client-denied-count 8
		client-restricted-denied-count 8
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 8
		client-snowflake-match-count 0
		client-snowflake-timeout-count 0
		client-http-count 8
		@@ -879,6 +918,10 @@ snowflake-proxy-rejected-for-relay-url-count 0
		client-denied-count 0
		client-restricted-denied-count 0
		client-unrestricted-denied-count 0
		client-nat-strict-denied-count 0
		client-nat-moderate-denied-count 0
		client-nat-open-denied-count 0
		client-nat-unknown-denied-count 0
		client-snowflake-match-count 0
		client-snowflake-timeout-count 0
		client-http-count 0
		@@ -890,6 +933,9 @@ client-sqs-ips
		snowflake-ips-nat-restricted 0
		snowflake-ips-nat-unrestricted 0
		snowflake-ips-nat-unknown 0
		snowflake-ips-nat-strict 0
		snowflake-ips-nat-moderate 0
		snowflake-ips-nat-open 0
		`)
		})
		//Test addition of client matches
		@@ -912,7 +958,7 @@ snowflake-ips-nat-unknown 0
		<-done

		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, "client-denied-count 0\nclient-restricted-denied-count 0\nclient-unrestricted-denied-count 0\nclient-snowflake-match-count 8")
		So(buf.String(), ShouldContainSubstring, "client-denied-count 0\nclient-restricted-denied-count 0\nclient-unrestricted-denied-count 0\nclient-nat-strict-denied-count 0\nclient-nat-moderate-denied-count 0\nclient-nat-open-denied-count 0\nclient-nat-unknown-denied-count 0\nclient-snowflake-match-count 8")
		})
		//Test rounding boundary
		Convey("binning boundary", func() {
		@@ -981,7 +1027,7 @@ snowflake-ips-nat-unknown 0

		buf.Reset()
		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, "client-denied-count 16\nclient-restricted-denied-count 16\nclient-unrestricted-denied-count 0\n")
		So(buf.String(), ShouldContainSubstring, "client-denied-count 16\nclient-restricted-denied-count 16\nclient-unrestricted-denied-count 0\nclient-nat-strict-denied-count 0\nclient-nat-moderate-denied-count 0\nclient-nat-open-denied-count 0\nclient-nat-unknown-denied-count 0\n")
		})

		//Test unique ip
		@@ -1048,7 +1094,7 @@ snowflake-ips-nat-unknown 0
		<-done

		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, "snowflake-ips-nat-restricted 1\nsnowflake-ips-nat-unrestricted 1\nsnowflake-ips-nat-unknown 0")
		So(buf.String(), ShouldContainSubstring, "snowflake-ips-nat-restricted 1\nsnowflake-ips-nat-unrestricted 1\nsnowflake-ips-nat-unknown 0\nsnowflake-ips-nat-strict 0\nsnowflake-ips-nat-moderate 0\nsnowflake-ips-nat-open 0")
		})

		Convey("client failures by NAT type", func() {
		@@ -1061,7 +1107,7 @@ snowflake-ips-nat-unknown 0
		clientOffers(i, w, r)

		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, "client-denied-count 8\nclient-restricted-denied-count 8\nclient-unrestricted-denied-count 0\nclient-snowflake-match-count 0")
		So(buf.String(), ShouldContainSubstring, "client-denied-count 8\nclient-restricted-denied-count 8\nclient-unrestricted-denied-count 0\nclient-nat-strict-denied-count 0\nclient-nat-moderate-denied-count 0\nclient-nat-open-denied-count 0\nclient-nat-unknown-denied-count 0\nclient-snowflake-match-count 0")

		buf.Reset()

		@@ -1073,7 +1119,7 @@ snowflake-ips-nat-unknown 0
		clientOffers(i, w, r)

		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, "client-denied-count 8\nclient-restricted-denied-count 0\nclient-unrestricted-denied-count 8\nclient-snowflake-match-count 0")
		So(buf.String(), ShouldContainSubstring, "client-denied-count 8\nclient-restricted-denied-count 0\nclient-unrestricted-denied-count 8\nclient-nat-strict-denied-count 0\nclient-nat-moderate-denied-count 0\nclient-nat-open-denied-count 0\nclient-nat-unknown-denied-count 0\nclient-snowflake-match-count 0")

		buf.Reset()

		@@ -1085,7 +1131,7 @@ snowflake-ips-nat-unknown 0
		clientOffers(i, w, r)

		ctx.metrics.printMetrics()
		So(buf.String(), ShouldContainSubstring, "client-denied-count 8\nclient-restricted-denied-count 8\nclient-unrestricted-denied-count 0\nclient-snowflake-match-count 0")
		So(buf.String(), ShouldContainSubstring, "client-denied-count 8\nclient-restricted-denied-count 0\nclient-unrestricted-denied-count 0\nclient-nat-strict-denied-count 0\nclient-nat-moderate-denied-count 0\nclient-nat-open-denied-count 0\nclient-nat-unknown-denied-count 8\nclient-snowflake-match-count 0")
		})
		Convey("that seen IPs map is cleared after each print", func() {
		w := httptest.NewRecorder()