uTLS Round-Tripper for Snowflake Client

shelikhoorequested to merge
shelikhoo/snowflake:dev-utls into main

This merge request bring uTLS support to the snowflake. The design of this part of the code is influenced by both @dcf and @max-b's design and reused a significant part of these existing codes.

This merge request adds two options to the snowflake client: -utls-imitate left empty to disable uTLS, input name of ClientHelloID to avoid -utls-nosni remove SNI from TLS Client Hello when using uTLS.

Highlight: Preflight connections that detect negotiated ALPN are reused for data transfer whenever possible. Any unexpected negotiated ALPN will be automatically adapted. This reduces the amount of anomaly observable to adversaries. As a side-effect, in the corner case that a server changes its TLS configuration while the client is running, the round-tripper will no longer be stuck in a malfunctioning state. A remove SNI setting is added to disable SNI extension on uTLS connections. Some CDN like CDN77, Azure allow domain fronting without SNI:Discussion Geph Sourcecode.

Things to look at: Currently, uTLS is not enabled by default. We will have a discussion on which fingerprint to use by default, or use dcf1's weighted random method to use a combination of them.

(Close #40054 (closed))

Edited by shelikhoo

Merge request reports

client/lib/rendezvous.go

+24 −4
Changes for client/lib/rendezvous.go: 24 added lines, 4 removed lines.
Original line number Diff line number Diff line
@@ -5,6 +5,8 @@ package snowflake_client 


import (

	"errors"

	"fmt"



	"log"

	"net/http"

	"sync"
@@ -14,7 +16,9 @@ import ( 
	"git.torproject.org/pluggable-transports/snowflake.git/v2/common/messages"

	"git.torproject.org/pluggable-transports/snowflake.git/v2/common/nat"

	"git.torproject.org/pluggable-transports/snowflake.git/v2/common/util"

	utlsutil "git.torproject.org/pluggable-transports/snowflake.git/v2/common/utls"

	"github.com/pion/webrtc/v3"

	utls "github.com/refraction-networking/utls"

)



const (
@@ -51,10 +55,15 @@ func createBrokerTransport() http.RoundTripper { 
	return transport

}



// NewBrokerChannel construct a new BrokerChannel, where:

func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) (*BrokerChannel, error) {

	return NewBrokerChannelWithUTLSSettings(broker, ampCache, front, keepLocalAddresses, "", false)

}



// NewBrokerChannelWithUTLSSettings construct a new BrokerChannel, where:

// |broker| is the full URL of the facilitating program which assigns proxies

// to clients, and |front| is the option fronting domain.

func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) (*BrokerChannel, error) {

func NewBrokerChannelWithUTLSSettings(broker, ampCache, front string, keepLocalAddresses bool,

	uTLSClientID string, removeSNI bool) (*BrokerChannel, error) {

	log.Println("Rendezvous using Broker at:", broker)

	if ampCache != "" {

		log.Println("Through AMP cache at:", ampCache)
@@ -63,12 +72,23 @@ func NewBrokerChannel(broker, ampCache, front string, keepLocalAddresses bool) ( 
		log.Println("Domain fronting using:", front)

	}



	brokerTransport := createBrokerTransport()



	if uTLSClientID != "" {

		utlsClientHelloID, err := utlsutil.NameToUTLSID(uTLSClientID)

		if err != nil {

			return nil, fmt.Errorf("unable to create broker channel: %v", err)

		}

		config := &utls.Config{}

		brokerTransport = utlsutil.NewUTLSHTTPRoundTripper(utlsClientHelloID, config, brokerTransport, removeSNI)

	}



	var rendezvous RendezvousMethod

	var err error

	if ampCache != "" {

		rendezvous, err = newAMPCacheRendezvous(broker, ampCache, front, createBrokerTransport())

		rendezvous, err = newAMPCacheRendezvous(broker, ampCache, front, brokerTransport)

	} else {

		rendezvous, err = newHTTPRendezvous(broker, front, createBrokerTransport())

		rendezvous, err = newHTTPRendezvous(broker, front, brokerTransport)

	}

	if err != nil {

		return nil, err

client/lib/snowflake.go

+9 −2
Changes for client/lib/snowflake.go: 9 added lines, 2 removed lines.
Original line number Diff line number Diff line
@@ -97,6 +97,12 @@ type ClientConfig struct { 
	// Max is the maximum number of snowflake proxy peers that the client should attempt to

	// connect to. Defaults to 1.

	Max int

	// UTLSClientID is the type of user application that snowflake should imitate.

	// If an empty value is provided, it will use Go's default TLS implementation

	UTLSClientID string

	// UTLSRemoveSNI is the flag to control whether SNI should be removed from Client Hello

	// when uTLS is used.

	UTLSRemoveSNI bool

}



// NewSnowflakeClient creates a new Snowflake transport client that can spawn multiple
@@ -125,8 +131,9 @@ func NewSnowflakeClient(config ClientConfig) (*Transport, error) { 
	}



	// Rendezvous with broker using the given parameters.

	broker, err := NewBrokerChannel(

		config.BrokerURL, config.AmpCacheURL, config.FrontDomain, config.KeepLocalAddresses)

	broker, err := NewBrokerChannelWithUTLSSettings(

		config.BrokerURL, config.AmpCacheURL, config.FrontDomain,

		config.KeepLocalAddresses, config.UTLSClientID, config.UTLSRemoveSNI)

	if err != nil {

		return nil, err

	}

client/snowflake.go

+11 −0
Changes for client/snowflake.go: 11 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -84,6 +84,17 @@ func socksAcceptLoop(ln *pt.SocksListener, config sf.ClientConfig, shutdown chan 
			if arg, ok := conn.Req.Args.Get("url"); ok {

				config.BrokerURL = arg

			}

			if arg, ok := conn.Req.Args.Get("utls-nosni"); ok {

				switch strings.ToLower(arg) {

				case "true":

					fallthrough

				case "yes":

					config.UTLSRemoveSNI = true

				}

			}

			if arg, ok := conn.Req.Args.Get("utls-imitate"); ok {

				config.UTLSClientID = arg

			}

			transport, err := sf.NewSnowflakeClient(config)

			if err != nil {

				conn.Reject()

common/utls/client_hello_id.go

0 → 100644
+38 −0
Changes for common/utls/client_hello_id.go: 38 added lines, 0 removed lines.
Original line number Diff line number Diff line 
package utls



import (

	"errors"

	utls "github.com/refraction-networking/utls"

	"strings"

)



// ported from https://github.com/max-b/snowflake/commit/9dded063cb74c6941a16ad90b9dd0e06e618e55e

var clientHelloIDMap = map[string]utls.ClientHelloID{

	// No HelloCustom: not useful for external configuration.

	// No HelloRandomized: doesn't negotiate consistent ALPN.

	"hellorandomizedalpn":   utls.HelloRandomizedALPN,

	"hellorandomizednoalpn": utls.HelloRandomizedNoALPN,

	"hellofirefox_auto":     utls.HelloFirefox_Auto,

	"hellofirefox_55":       utls.HelloFirefox_55,

	"hellofirefox_56":       utls.HelloFirefox_56,

	"hellofirefox_63":       utls.HelloFirefox_63,

	"hellofirefox_65":       utls.HelloFirefox_65,

	"hellochrome_auto":      utls.HelloChrome_Auto,

	"hellochrome_58":        utls.HelloChrome_58,

	"hellochrome_62":        utls.HelloChrome_62,

	"hellochrome_70":        utls.HelloChrome_70,

	"hellochrome_72":        utls.HelloChrome_72,

	"helloios_auto":         utls.HelloIOS_Auto,

	"helloios_11_1":         utls.HelloIOS_11_1,

	"helloios_12_1":         utls.HelloIOS_12_1,

}



var errNameNotFound = errors.New("client hello name is unrecognized")



func NameToUTLSID(name string) (utls.ClientHelloID, error) {

	normalizedName := strings.ToLower(name)

	if id, ok := clientHelloIDMap[normalizedName]; ok {

		return id, nil

	}

	return utls.ClientHelloID{}, errNameNotFound

}

common/utls/roundtripper.go

0 → 100644
+238 −0
Changes for common/utls/roundtripper.go: 238 added lines, 0 removed lines.
Original line number Diff line number Diff line 
package utls



import (

	"context"

	"crypto/tls"

	"errors"

	"net"

	"net/http"

	"sync"

	"time"



	utls "github.com/refraction-networking/utls"

	"golang.org/x/net/http2"

)



// NewUTLSHTTPRoundTripper creates an instance of RoundTripper that dial to remote HTTPS endpoint with

// an alternative version of TLS implementation that attempts to imitate browsers' fingerprint.

// clientHelloID is the clientHello that uTLS attempts to imitate

// uTlsConfig is the TLS Configuration template

// backdropTransport is the transport that will be used for non-https traffic

// removeSNI indicates not to send Server Name Indication Extension

// returns a RoundTripper: its behaviour is documented at

// https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/merge_requests/76#note_2777161

func NewUTLSHTTPRoundTripper(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config,

	backdropTransport http.RoundTripper, removeSNI bool) http.RoundTripper {

	rtImpl := &uTLSHTTPRoundTripperImpl{

		clientHelloID:     clientHelloID,

		config:            uTlsConfig,

		connectWithH1:     map[string]bool{},

		backdropTransport: backdropTransport,

		pendingConn:       map[pendingConnKey]*unclaimedConnection{},

		removeSNI:         removeSNI,

	}

	rtImpl.init()

	return rtImpl

}



type uTLSHTTPRoundTripperImpl struct {

	clientHelloID utls.ClientHelloID

	config        *utls.Config



	accessConnectWithH1 sync.Mutex

	connectWithH1       map[string]bool



	httpsH1Transport  http.RoundTripper

	httpsH2Transport  http.RoundTripper

	backdropTransport http.RoundTripper



	accessDialingConnection sync.Mutex

	pendingConn             map[pendingConnKey]*unclaimedConnection



	removeSNI bool

}



type pendingConnKey struct {

	isH2 bool

	dest string

}



var errEAGAIN = errors.New("incorrect ALPN negotiated, try again with another ALPN")

var errEAGAINTooMany = errors.New("incorrect ALPN negotiated")

var errExpired = errors.New("connection have expired")



func (r *uTLSHTTPRoundTripperImpl) RoundTrip(req *http.Request) (*http.Response, error) {

	if req.URL.Scheme != "https" {

		return r.backdropTransport.RoundTrip(req)

	}

	for retryCount := 0; retryCount < 5; retryCount++ {

		if r.getShouldConnectWithH1(req.URL.Host) {

			resp, err := r.httpsH1Transport.RoundTrip(req)

			if errors.Is(err, errEAGAIN) {

				continue

			}

			return resp, err

		}

		resp, err := r.httpsH2Transport.RoundTrip(req)

		if errors.Is(err, errEAGAIN) {

			continue

		}

		return resp, err

	}

	return nil, errEAGAINTooMany

}



func (r *uTLSHTTPRoundTripperImpl) getShouldConnectWithH1(domainName string) bool {

	r.accessConnectWithH1.Lock()

	defer r.accessConnectWithH1.Unlock()

	if value, set := r.connectWithH1[domainName]; set {

		return value

	}

	return false

}



func (r *uTLSHTTPRoundTripperImpl) setShouldConnectWithH1(domainName string) {

	r.accessConnectWithH1.Lock()

	defer r.accessConnectWithH1.Unlock()

	r.connectWithH1[domainName] = true

}



func (r *uTLSHTTPRoundTripperImpl) clearShouldConnectWithH1(domainName string) {

	r.accessConnectWithH1.Lock()

	defer r.accessConnectWithH1.Unlock()

	r.connectWithH1[domainName] = false

}



func getPendingConnectionID(dest string, alpnIsH2 bool) pendingConnKey {

	return pendingConnKey{isH2: alpnIsH2, dest: dest}

}



func (r *uTLSHTTPRoundTripperImpl) putConn(addr string, alpnIsH2 bool, conn net.Conn) {

	connId := getPendingConnectionID(addr, alpnIsH2)

	r.pendingConn[connId] = NewUnclaimedConnection(conn, time.Minute)

}

func (r *uTLSHTTPRoundTripperImpl) getConn(addr string, alpnIsH2 bool) net.Conn {

	connId := getPendingConnectionID(addr, alpnIsH2)

	if conn, ok := r.pendingConn[connId]; ok {

		delete(r.pendingConn, connId)

		if claimedConnection, err := conn.claimConnection(); err == nil {

			return claimedConnection

		}

	}

	return nil

}

func (r *uTLSHTTPRoundTripperImpl) dialOrGetTLSWithExpectedALPN(ctx context.Context, addr string, expectedH2 bool) (net.Conn, error) {

	r.accessDialingConnection.Lock()

	defer r.accessDialingConnection.Unlock()



	if r.getShouldConnectWithH1(addr) == expectedH2 {

		return nil, errEAGAIN

	}



	//Get a cached connection if possible to reduce preflight connection closed without sending data

	if gconn := r.getConn(addr, expectedH2); gconn != nil {

		return gconn, nil

	}



	conn, err := r.dialTLS(ctx, addr)

	if err != nil {

		return nil, err

	}



	protocol := conn.ConnectionState().NegotiatedProtocol



	protocolIsH2 := protocol == http2.NextProtoTLS



	if protocolIsH2 == expectedH2 {

		return conn, err

	}



	r.putConn(addr, protocolIsH2, conn)



	if protocolIsH2 {

		r.clearShouldConnectWithH1(addr)

	} else {

		r.setShouldConnectWithH1(addr)

	}



	return nil, errEAGAIN

}



// based on https://repo.or.cz/dnstt.git/commitdiff/d92a791b6864901f9263f7d73d97cfd30ac53b09..98bdffa1706dfc041d1e99b86c47f29d72ad3a0c

// by dcf1

func (r *uTLSHTTPRoundTripperImpl) dialTLS(ctx context.Context, addr string) (*utls.UConn, error) {

	config := r.config.Clone()



	host, _, err := net.SplitHostPort(addr)

	if err != nil {

		return nil, err

	}

	config.ServerName = host



	dialer := &net.Dialer{}

	conn, err := dialer.DialContext(ctx, "tcp", addr)

	if err != nil {

		return nil, err

	}

	uconn := utls.UClient(conn, config, r.clientHelloID)

	if (net.ParseIP(config.ServerName) != nil) || r.removeSNI {

		err := uconn.RemoveSNIExtension()

		if err != nil {

			uconn.Close()

			return nil, err

		}

	}



	err = uconn.Handshake()

	if err != nil {

		return nil, err

	}

	return uconn, nil

}



func (r *uTLSHTTPRoundTripperImpl) init() {

	r.httpsH2Transport = &http2.Transport{

		DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {

			return r.dialOrGetTLSWithExpectedALPN(context.Background(), addr, true)

		},

	}

	r.httpsH1Transport = &http.Transport{

		DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {

			return r.dialOrGetTLSWithExpectedALPN(ctx, addr, false)

		},

	}

}



func NewUnclaimedConnection(conn net.Conn, expireTime time.Duration) *unclaimedConnection {

	c := &unclaimedConnection{

		Conn: conn,

	}

	time.AfterFunc(expireTime, c.tick)

	return c

}



type unclaimedConnection struct {

	net.Conn

	claimed bool

	access  sync.Mutex

}



func (c *unclaimedConnection) claimConnection() (net.Conn, error) {

	c.access.Lock()

	defer c.access.Unlock()

	if !c.claimed {

		c.claimed = true

		return c.Conn, nil

	}

	return nil, errExpired

}



func (c *unclaimedConnection) tick() {

	c.access.Lock()

	defer c.access.Unlock()

	if !c.claimed {

		c.claimed = true

		c.Conn.Close()

		c.Conn = nil

	}

}