Merge branch 'bug4744_squashed'

changes/bug4744

+  o Major features:
+    - Update cipher cipher list to match Firefox 8 and later. Fix for
+      issue 4744.
+

changes/prop198

+  o Removed features:
+
+    - Remove support for clients claiming to support any standard
+      ciphersuites that we can actually provide.  (As of modern
+      OpenSSL versions, it's not necessary to fake any standard
+      ciphersuite, and doing so prevents us from using better
+      ciphersuites in the future, since servers can't know whether an
+      advertised ciphersuite is really supported or not.)  Some
+      hosts--notably, ones with very old versions of OpenSSL or where
+      OpenSSL has been built with ECC disabled-- will stand out
+      because of this change; TBB users should not be affected.
+      This implements the client side of proposal 198.

src/common/ciphers.inc

 /* This is an include file used to define the list of ciphers clients should
  * advertise.  Before including it, you should define the CIPHER and XCIPHER
- * macros. */
+ * macros.
+ *
+ * This file was automatically generated by get_mozilla_ciphers.py.
+ */
 #ifdef TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
     CIPHER(0xc00a, TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_CBC_SHA)
 #else
@@ -11,6 +14,16 @@
 #else
    XCIPHER(0xc014, TLS1_TXT_ECDHE_RSA_WITH_AES_256_CBC_SHA)
 #endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA
+    CIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
+#else
+   XCIPHER(0x0088, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA)
+#endif
+#ifdef TLS1_TXT_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA
+    CIPHER(0x0087, TLS1_TXT_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA)
+#else
+   XCIPHER(0x0087, TLS1_TXT_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA)
+#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_256_SHA
     CIPHER(0x0039, TLS1_TXT_DHE_RSA_WITH_AES_256_SHA)
 #else
@@ -31,6 +44,11 @@
 #else
    XCIPHER(0xc005, TLS1_TXT_ECDH_ECDSA_WITH_AES_256_CBC_SHA)
 #endif
+#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA
+    CIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
+#else
+   XCIPHER(0x0084, TLS1_TXT_RSA_WITH_CAMELLIA_256_CBC_SHA)
+#endif
 #ifdef TLS1_TXT_RSA_WITH_AES_256_SHA
     CIPHER(0x0035, TLS1_TXT_RSA_WITH_AES_256_SHA)
 #else
@@ -56,6 +74,16 @@
 #else
    XCIPHER(0xc013, TLS1_TXT_ECDHE_RSA_WITH_AES_128_CBC_SHA)
 #endif
+#ifdef TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA
+    CIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
+#else
+   XCIPHER(0x0045, TLS1_TXT_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA)
+#endif
+#ifdef TLS1_TXT_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA
+    CIPHER(0x0044, TLS1_TXT_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA)
+#else
+   XCIPHER(0x0044, TLS1_TXT_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA)
+#endif
 #ifdef TLS1_TXT_DHE_RSA_WITH_AES_128_SHA
     CIPHER(0x0033, TLS1_TXT_DHE_RSA_WITH_AES_128_SHA)
 #else
@@ -86,6 +114,16 @@
 #else
    XCIPHER(0xc004, TLS1_TXT_ECDH_ECDSA_WITH_AES_128_CBC_SHA)
 #endif
+#ifdef TLS1_TXT_RSA_WITH_SEED_SHA
+    CIPHER(0x0096, TLS1_TXT_RSA_WITH_SEED_SHA)
+#else
+   XCIPHER(0x0096, TLS1_TXT_RSA_WITH_SEED_SHA)
+#endif
+#ifdef TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA
+    CIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
+#else
+   XCIPHER(0x0041, TLS1_TXT_RSA_WITH_CAMELLIA_128_CBC_SHA)
+#endif
 #ifdef SSL3_TXT_RSA_RC4_128_MD5
     CIPHER(0x0004, SSL3_TXT_RSA_RC4_128_MD5)
 #else
@@ -131,10 +169,11 @@
 #else
    XCIPHER(0xc003, TLS1_TXT_ECDH_ECDSA_WITH_DES_192_CBC3_SHA)
 #endif
-#ifdef SSL3_TXT_RSA_FIPS_WITH_3DES_EDE_CBC_SHA
-    CIPHER(0xfeff, SSL3_TXT_RSA_FIPS_WITH_3DES_EDE_CBC_SHA)
+/* No openssl macro found for 0xfeff */
+#ifdef SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA
+    CIPHER(0xfeff, SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA)
 #else
-   XCIPHER(0xfeff, SSL3_TXT_RSA_FIPS_WITH_3DES_EDE_CBC_SHA)
+   XCIPHER(0xfeff, SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA)
 #endif
 #ifdef SSL3_TXT_RSA_DES_192_CBC3_SHA
     CIPHER(0x000a, SSL3_TXT_RSA_DES_192_CBC3_SHA)

src/common/get_mozilla_ciphers.py

+#!/usr/bin/python
+# coding=utf-8
+# Copyright 2011, The Tor Project, Inc
+# original version by Arturo Filastò
+# See LICENSE for licensing information
+
+# This script parses Firefox and OpenSSL sources, and uses this information
+# to generate a ciphers.inc file.
+#
+# It takes two arguments: the location of a firefox source directory, and the
+# location of an openssl source directory.
+
+import os
+import re
+import sys
+
+if len(sys.argv) != 3:
+    print >>sys.stderr, "Syntax: get_mozilla_ciphers.py <firefox-source-dir> <openssl-source-dir>"
+    sys.exit(1)
+
+ff_root = sys.argv[1]
+ossl_root = sys.argv[2]
+
+def ff(s):
+    return os.path.join(ff_root, s)
+def ossl(s):
+    return os.path.join(ossl_root, s)
+
+#####
+# Read the cpp file to understand what Ciphers map to what name :
+# Make "ciphers" a map from name used in the javascript to a cipher macro name
+fileA = open(ff('security/manager/ssl/src/nsNSSComponent.cpp'),'r')
+
+# The input format is a file containing exactly one section of the form:
+# static CipherPref CipherPrefs[] = {
+#  {"name", MACRO_NAME}, // comment
+#  ...
+#  {NULL, 0}
+# }
+
+inCipherSection = False
+cipherLines = []
+for line in fileA:
+    if line.startswith('static CipherPref CipherPrefs'):
+        # Get the starting boundary of the Cipher Preferences
+        inCipherSection = True
+    elif inCipherSection:
+        line = line.strip()
+        if line.startswith('{NULL, 0}'):
+            # At the ending boundary of the Cipher Prefs
+            break
+        else:
+            cipherLines.append(line)
+fileA.close()
+
+# Parse the lines and put them into a dict
+ciphers = {}
+cipher_pref = {}
+for line in cipherLines:
+    m = re.search(r'^{\s*\"([^\"]+)\",\s*(\S*)\s*}', line)
+    if m:
+        key,value = m.groups()
+        ciphers[key] = value
+        cipher_pref[value] = key
+
+####
+# Now find the correct order for the ciphers
+fileC = open(ff('security/nss/lib/ssl/ssl3con.c'), 'r')
+firefox_ciphers = []
+inEnum=False
+for line in fileC:
+    if not inEnum:
+        if "ssl3CipherSuiteCfg cipherSuites[" in line:
+            inEnum = True
+        continue
+
+    if line.startswith("};"):
+        break
+
+    m = re.match(r'^\s*\{\s*([A-Z_0-9]+),', line)
+    if m:
+        firefox_ciphers.append(m.group(1))
+
+fileC.close()
+
+#####
+# Read the JS file to understand what ciphers are enabled.  The format is
+#  pref("name", true/false);
+# Build a map enabled_ciphers from javascript name to "true" or "false",
+# and an (unordered!) list of the macro names for those ciphers that are
+# enabled.
+fileB = open(ff('netwerk/base/public/security-prefs.js'), 'r')
+
+enabled_ciphers = {}
+for line in fileB:
+    m = re.match(r'pref\(\"([^\"]+)\"\s*,\s*(\S*)\s*\)', line)
+    if not m:
+        continue
+    key, val = m.groups()
+    if key.startswith("security.ssl3"):
+        enabled_ciphers[key] = val
+fileB.close()
+
+used_ciphers = []
+for k, v in enabled_ciphers.items():
+    if v == "true":
+        used_ciphers.append(ciphers[k])
+
+#oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h',
+#               '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h',
+#               '/usr/include/openssl/tls1.h')
+oSSLinclude = ('ssl/ssl3.h', 'ssl/ssl.h',
+               'ssl/ssl2.h', 'ssl/ssl23.h',
+               'ssl/tls1.h')
+
+#####
+# This reads the hex code for the ciphers that are used by firefox.
+# sslProtoD is set to a map from macro name to macro value in sslproto.h;
+# cipher_codes is set to an (unordered!) list of these hex values.
+sslProto = open(ff('security/nss/lib/ssl/sslproto.h'), 'r')
+sslProtoD = {}
+
+for line in sslProto:
+    m = re.match('#define\s+(\S+)\s+(\S+)', line)
+    if m:
+        key, value = m.groups()
+        sslProtoD[key] = value
+sslProto.close()
+
+cipher_codes = []
+for x in used_ciphers:
+    cipher_codes.append(sslProtoD[x].lower())
+
+####
+# Now read through all the openssl include files, and try to find the openssl
+# macro names for those files.
+openssl_macro_by_hex = {}
+all_openssl_macros = {}
+for fl in oSSLinclude:
+    fp = open(ossl(fl), 'r')
+    for line in fp.readlines():
+        m = re.match('#define\s+(\S+)\s+(\S+)', line)
+        if m:
+            value,key = m.groups()
+            if key.startswith('0x') and "_CK_" in value:
+                key = key.replace('0x0300','0x').lower()
+                #print "%s %s" % (key, value)
+                openssl_macro_by_hex[key] = value
+            all_openssl_macros[value]=key
+    fp.close()
+
+# Now generate the output.
+print """\
+/* This is an include file used to define the list of ciphers clients should
+ * advertise.  Before including it, you should define the CIPHER and XCIPHER
+ * macros.
+ *
+ * This file was automatically generated by get_mozilla_ciphers.py.
+ */"""
+# Go in order by the order in CipherPrefs
+for firefox_macro in firefox_ciphers:
+
+    try:
+        js_cipher_name = cipher_pref[firefox_macro]
+    except KeyError:
+        # This one has no javascript preference.
+        continue
+
+    # The cipher needs to be enabled in security-prefs.js
+    if enabled_ciphers.get(js_cipher_name, 'false') != 'true':
+        continue
+
+    hexval = sslProtoD[firefox_macro].lower()
+
+    try:
+        openssl_macro = openssl_macro_by_hex[hexval.lower()]
+        openssl_macro = openssl_macro.replace("_CK_", "_TXT_")
+        if openssl_macro not in all_openssl_macros:
+            raise KeyError()
+        format = {'hex':hexval, 'macro':openssl_macro, 'note':""}
+    except KeyError:
+        # openssl doesn't have a macro for this.
+        format = {'hex':hexval, 'macro':firefox_macro,
+                  'note':"/* No openssl macro found for "+hexval+" */\n"}
+
+    res = """\
+%(note)s#ifdef %(macro)s
+    CIPHER(%(hex)s, %(macro)s)
+#else
+   XCIPHER(%(hex)s, %(macro)s)
+#endif""" % format
+    print res

src/common/tortls.c

@@ -674,6 +674,9 @@ tor_tls_create_certificate(crypto_pk_t *rsa,
  * our OpenSSL doesn't know about. */
 static const char CLIENT_CIPHER_LIST[] =
 #include "./ciphers.inc"
+  /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version
+   * of any cipher we say. */
+  "!SSLv2"
   ;
 #undef CIPHER
 #undef XCIPHER
@@ -1415,11 +1418,35 @@ tor_tls_server_info_callback(const SSL *ssl, int type, int val)
 }
 #endif
 
+/** Explain which ciphers we're missing. */
+static void
+log_unsupported_ciphers(smartlist_t *unsupported)
+{
+  char *joined;
+
+  log_notice(LD_NET, "We weren't able to find support for all of the "
+             "TLS ciphersuites that we wanted to advertise. This won't "
+             "hurt security, but it might make your Tor (if run as a client) "
+             "more easy for censors to block.");
+
+  if (SSLeay() < 0x10000000L) {
+    log_notice(LD_NET, "To correct this, use a more recent OpenSSL, "
+               "built without disabling any secure ciphers or features.");
+  } else {
+    log_notice(LD_NET, "To correct this, use a version of OpenSSL "
+               "built with none of its ciphers disabled.");
+  }
+
+  joined = smartlist_join_strings(unsupported, ":", 0, NULL);
+  log_info(LD_NET, "The unsupported ciphers were: %s", joined);
+  tor_free(joined);
+}
+
 /** Replace *<b>ciphers</b> with a new list of SSL ciphersuites: specifically,
- * a list designed to mimic a common web browser.  Some of the ciphers in the
- * list won't actually be implemented by OpenSSL: that's okay so long as the
- * server doesn't select them, and the server won't select anything besides
- * what's in SERVER_CIPHER_LIST.
+ * a list designed to mimic a common web browser.  We might not be able to do
+ * that if OpenSSL doesn't support all the ciphers we want.  Some of the
+ * ciphers in the list won't actually be implemented by OpenSSL: that's okay
+ * so long as the server doesn't select them.
  *
  * [If the server <b>does</b> select a bogus cipher, we won't crash or
  * anything; we'll just fail later when we try to look up the cipher in
@@ -1431,14 +1458,17 @@ rectify_client_ciphers(STACK_OF(SSL_CIPHER) **ciphers)
 #ifdef V2_HANDSHAKE_CLIENT
   if (PREDICT_UNLIKELY(!CLIENT_CIPHER_STACK)) {
     /* We need to set CLIENT_CIPHER_STACK to an array of the ciphers
-     * we want.*/
+     * we want to use/advertise. */
     int i = 0, j = 0;
+    smartlist_t *unsupported = smartlist_create();
 
     /* First, create a dummy SSL_CIPHER for every cipher. */
     CLIENT_CIPHER_DUMMIES =
       tor_malloc_zero(sizeof(SSL_CIPHER)*N_CLIENT_CIPHERS);
     for (i=0; i < N_CLIENT_CIPHERS; ++i) {
       CLIENT_CIPHER_DUMMIES[i].valid = 1;
+      /* The "3<<24" here signifies that the cipher is supposed to work with
+       * SSL3 and TLS1. */
       CLIENT_CIPHER_DUMMIES[i].id = CLIENT_CIPHER_INFO_LIST[i].id | (3<<24);
       CLIENT_CIPHER_DUMMIES[i].name = CLIENT_CIPHER_INFO_LIST[i].name;
     }
@@ -1453,27 +1483,49 @@ rectify_client_ciphers(STACK_OF(SSL_CIPHER) **ciphers)
     }
 
     /* Then copy as many ciphers as we can from the good list, inserting
-     * dummies as needed. */
-    j=0;
-    for (i = 0; i < N_CLIENT_CIPHERS; ) {
+     * dummies as needed. Let j be an index into list of ciphers we have
+     * (*ciphers) and let i be an index into the ciphers we want
+     * (CLIENT_INFO_CIPHER_LIST).  We are building a list of ciphers in
+     * CLIENT_CIPHER_STACK.
+     */
+    for (i = j = 0; i < N_CLIENT_CIPHERS; ) {
       SSL_CIPHER *cipher = NULL;
       if (j < sk_SSL_CIPHER_num(*ciphers))
         cipher = sk_SSL_CIPHER_value(*ciphers, j);
       if (cipher && ((cipher->id >> 24) & 0xff) != 3) {
-        log_debug(LD_NET, "Skipping v2 cipher %s", cipher->name);
+        /* Skip over non-v3 ciphers entirely.  (This should no longer be
+         * needed, thanks to saying !SSLv2 above.) */
+        log_debug(LD_NET, "Skipping v%d cipher %s",
+                  (int)((cipher->id>>24) & 0xff),
+                  cipher->name);
         ++j;
       } else if (cipher &&
                  (cipher->id & 0xffff) == CLIENT_CIPHER_INFO_LIST[i].id) {
+        /* "cipher" is the cipher we expect. Put it on the list. */
         log_debug(LD_NET, "Found cipher %s", cipher->name);
         sk_SSL_CIPHER_push(CLIENT_CIPHER_STACK, cipher);
         ++j;
         ++i;
-      } else {
+      } else if (!strcmp(CLIENT_CIPHER_DUMMIES[i].name,
+                         "SSL_RSA_FIPS_WITH_3DES_EDE_CBC_SHA")) {
+        /* We found bogus cipher 0xfeff, which OpenSSL doesn't support and
+         * never has.  For this one, we need a dummy. */
         log_debug(LD_NET, "Inserting fake %s", CLIENT_CIPHER_DUMMIES[i].name);
         sk_SSL_CIPHER_push(CLIENT_CIPHER_STACK, &CLIENT_CIPHER_DUMMIES[i]);
         ++i;
+      } else {
+        /* OpenSSL doesn't have this one. */
+        log_debug(LD_NET, "Completely omitting unsupported cipher %s",
+                  CLIENT_CIPHER_INFO_LIST[i].name);
+        smartlist_add(unsupported, (char*) CLIENT_CIPHER_INFO_LIST[i].name);
+        ++i;
       }
     }
+
+    if (smartlist_len(unsupported))
+      log_unsupported_ciphers(unsupported);
+
+    smartlist_free(unsupported);
   }
 
   sk_SSL_CIPHER_free(*ciphers);