New FallbackDir option to add extra directories for bootstraping

This replaces the old FallbackConsensus notion, and should provide a
way -- assuming we pick reasonable nodes! -- to give clients
suggestions of placs to go to get their first consensus.

changes/fallback_dirsource

+  o Major features:
+    - Add a new FallbackDir option to use when we can't use a directory
+      from the consensus (either because we lack a consensus, or because
+      they're all down).  Currently, all authorities are fallbacks by
+      default, and there are no other default fallbacks, but that will
+      change.  This option will allow us to give clients a longer list
+      of servers to try to get a consensus from when first connecting 
+      to the Tor network, and thereby reduce load on the directory
+      authorities.

doc/tor.1.txt

@@ -292,6 +292,11 @@ GENERAL OPTIONS
 **DataDirectory** __DIR__::
     Store working data in DIR (Default: @LOCALSTATEDIR@/lib/tor)
 
+**FallbackDir** __address__:__port__ orport=__port__ id=__fingerprint__::
+    When we're unable to connect to any directory cache for directory info
+    (usually because we don't know about any yet) we try a FallbackDir.
+    By default, the directory authorities are also FallbackDirs.
+
 **DirAuthority** [__nickname__] [**flags**] __address__:__port__ __fingerprint__::
     Use a nonstandard authoritative directory server at the provided address
     and port, with the specified key fingerprint. This option can be repeated

src/or/config.c

@@ -228,6 +228,7 @@ static config_var_t option_vars_[] = {
   V(ExitPortStatistics,          BOOL,     "0"),
   V(ExtendAllowPrivateAddresses, BOOL,     "0"),
   V(ExtraInfoStatistics,         BOOL,     "1"),
+  V(FallbackDir,                 LINELIST, NULL),
 
 #if defined (WINCE)
   V(FallbackNetworkstatusFile,   FILENAME, "fallback-consensus"),
@@ -474,6 +475,8 @@ static char *get_bindaddr_from_transport_listen_line(const char *line,
 static int parse_dir_authority_line(const char *line,
                                  dirinfo_type_t required_type,
                                  int validate_only);
+static int parse_dir_fallback_line(const char *line,
+                                   int validate_only);
 static void port_cfg_free(port_cfg_t *port);
 static int parse_ports(or_options_t *options, int validate_only,
                               char **msg_out, int *n_ports_out);
@@ -756,7 +759,7 @@ static void
 add_default_trusted_dir_authorities(dirinfo_type_t type)
 {
   int i;
-  const char *dirservers[] = {
+  const char *authorities[] = {
     "moria1 orport=9101 no-v2 "
       "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 "
       "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31",
@@ -785,10 +788,27 @@ add_default_trusted_dir_authorities(dirinfo_type_t type)
       "154.35.32.5:80 CF6D 0AAF B385 BE71 B8E1 11FC 5CFF 4B47 9237 33BC",
     NULL
   };
-  for (i=0; dirservers[i]; i++) {
-    if (parse_dir_authority_line(dirservers[i], type, 0)<0) {
-      log_err(LD_BUG, "Couldn't parse internal dirserver line %s",
-              dirservers[i]);
+  for (i=0; authorities[i]; i++) {
+    if (parse_dir_authority_line(authorities[i], type, 0)<0) {
+      log_err(LD_BUG, "Couldn't parse internal DirAuthority line %s",
+              authorities[i]);
+    }
+  }
+}
+
+/** Add the default fallback directory servers into the fallback directory
+ * server list. */
+static void
+add_default_fallback_dir_servers(void)
+{
+  int i;
+  const char *fallback[] = {
+    NULL
+  };
+  for (i=0; fallback[i]; i++) {
+    if (parse_dir_fallback_line(fallback[i], 0)<0) {
+      log_err(LD_BUG, "Couldn't parse internal FallbackDir line %s",
+              fallback[i]);
     }
   }
 }
@@ -798,7 +818,7 @@ add_default_trusted_dir_authorities(dirinfo_type_t type)
  * user if we changed any dangerous ones.
  */
 static int
-validate_dir_authorities(or_options_t *options, or_options_t *old_options)
+validate_dir_servers(or_options_t *options, or_options_t *old_options)
 {
   config_line_t *cl;
 
@@ -842,6 +862,9 @@ validate_dir_authorities(or_options_t *options, or_options_t *old_options)
   for (cl = options->AlternateHSAuthority; cl; cl = cl->next)
     if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0)
       return -1;
+  for (cl = options->FallbackDir; cl; cl = cl->next)
+    if (parse_dir_fallback_line(cl->value, 1)<0)
+      return -1;
   return 0;
 }
 
@@ -849,14 +872,15 @@ validate_dir_authorities(or_options_t *options, or_options_t *old_options)
  * as appropriate.
  */
 static int
-consider_adding_dir_authorities(const or_options_t *options,
-                                const or_options_t *old_options)
+consider_adding_dir_servers(const or_options_t *options,
+                            const or_options_t *old_options)
 {
   config_line_t *cl;
   int need_to_update =
     !smartlist_len(router_get_trusted_dir_servers()) ||
     !smartlist_len(router_get_fallback_dir_servers()) || !old_options ||
     !config_lines_eq(options->DirAuthorities, old_options->DirAuthorities) ||
+    !config_lines_eq(options->FallbackDir, old_options->FallbackDir) ||
     !config_lines_eq(options->AlternateBridgeAuthority,
                      old_options->AlternateBridgeAuthority) ||
     !config_lines_eq(options->AlternateDirAuthority,
@@ -882,6 +906,8 @@ consider_adding_dir_authorities(const or_options_t *options,
       type |= HIDSERV_DIRINFO;
     add_default_trusted_dir_authorities(type);
   }
+  if (!options->FallbackDir)
+    add_default_fallback_dir_servers();
 
   for (cl = options->DirAuthorities; cl; cl = cl->next)
     if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0)
@@ -895,6 +921,9 @@ consider_adding_dir_authorities(const or_options_t *options,
   for (cl = options->AlternateHSAuthority; cl; cl = cl->next)
     if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0)
       return -1;
+  for (cl = options->FallbackDir; cl; cl = cl->next)
+    if (parse_dir_fallback_line(cl->value, 0)<0)
+      return -1;
   return 0;
 }
 
@@ -1217,7 +1246,7 @@ options_act(const or_options_t *old_options)
       return -1;
   }
 
-  if (consider_adding_dir_authorities(options, old_options) < 0)
+  if (consider_adding_dir_servers(options, old_options) < 0)
     return -1;
 
 #ifdef NON_ANONYMOUS_MODE_ENABLED
@@ -2844,8 +2873,9 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (validate_addr_policies(options, msg) < 0)
     return -1;
 
-  if (validate_dir_authorities(options, old_options) < 0)
-    REJECT("Directory authority line did not parse. See logs for details.");
+  if (validate_dir_servers(options, old_options) < 0)
+    REJECT("Directory authority/fallback line did not parse. See logs "
+           "for details.");
 
   if (options->UseBridges && !options->Bridges)
     REJECT("If you set UseBridges, you must specify at least one bridge.");
@@ -4439,7 +4469,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type,
     log_debug(LD_DIR, "Trusted %d dirserver at %s:%d (%s)", (int)type,
               address, (int)dir_port, (char*)smartlist_get(items,0));
     if (!(ds = trusted_dir_server_new(nickname, address, dir_port, or_port,
-                                      digest, v3_digest, type)))
+                                      digest, v3_digest, type, 1.0)))
       goto err;
     dir_server_add(ds);
   }
@@ -4460,6 +4490,88 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type,
   return r;
 }
 
+/** Read the contents of a FallbackDir line from <b>line</b>. If
+ * <b>validate_only</b> is 0, and the line is well-formed, then add the
+ * dirserver described in the line as a fallback directory. Return 0 on
+ * success, or -1 if the line isn't well-formed or if we can't add it. */
+static int
+parse_dir_fallback_line(const char *line,
+                        int validate_only)
+{
+  int r = -1;
+  smartlist_t *items = smartlist_new(), *positional = smartlist_new();
+  int orport = -1;
+  uint16_t dirport;
+  tor_addr_t addr;
+  int ok;
+  char id[DIGEST_LEN];
+  char *address=NULL;
+
+  memset(id, 0, sizeof(id));
+  smartlist_split_string(items, line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+  SMARTLIST_FOREACH_BEGIN(items, const char *, cp) {
+    const char *eq = strchr(cp, '=');
+    ok = 1;
+    if (! eq) {
+      smartlist_add(positional, (char*)cp);
+      continue;
+    }
+    if (!strcmpstart(cp, "orport="))
+      orport = (int)tor_parse_long(cp+strlen("orport="), 10,
+                                   1, 65535, &ok, NULL);
+    else if (!strcmpstart(cp, "id="))
+      ok = !base16_decode(id, DIGEST_LEN,
+                          cp+strlen("id="), strlen(cp)-strlen("id="));
+    if (!ok) {
+      log_warn(LD_CONFIG, "Bad FallbackDir option %s", escaped(cp));
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  if (smartlist_len(positional) != 1) {
+    log_warn(LD_CONFIG, "Couldn't parse FallbackDir line %s", escaped(line));
+    goto end;
+  }
+
+  if (tor_digest_is_zero(id)) {
+    log_warn(LD_CONFIG, "Missing identity on FallbackDir line");
+    goto end;
+  }
+
+  if (orport <= 0) {
+    log_warn(LD_CONFIG, "Missing orport on FallbackDir line");
+    goto end;
+  }
+
+  if (tor_addr_port_split(LOG_INFO, smartlist_get(positional, 0),
+                          &address, &dirport) < 0 ||
+      tor_addr_parse(&addr, address)<0) {
+    log_warn(LD_CONFIG, "Couldn't parse address:port %s on FallbackDir line",
+             (const char*)smartlist_get(positional, 0));
+    goto end;
+  }
+
+  if (!validate_only) {
+    dir_server_t *ds;
+    ds = fallback_dir_server_new(&addr, dirport, orport, id, 1.0);
+    if (!ds) {
+      log_warn(LD_CONFIG, "Couldn't create FallbackDir %s", escaped(line));
+      goto end;
+    }
+    dir_server_add(ds);
+  }
+
+  r = 0;
+
+ end:
+  SMARTLIST_FOREACH(items, char *, cp, tor_free(cp));
+  smartlist_free(items);
+  smartlist_free(positional);
+  tor_free(address);
+  return r;
+}
+
 /** Free all storage held in <b>port</b> */
 static void
 port_cfg_free(port_cfg_t *port)

src/or/or.h

@@ -3428,6 +3428,9 @@ typedef struct {
    * use the "Alternate*Authority" options below instead. */
   config_line_t *DirAuthorities;
 
+  /** List of fallback directory servers */
+  config_line_t *FallbackDir;
+
   /** If set, use these main (currently v3) directory authorities and
    * not the default ones. */
   config_line_t *AlternateDirAuthority;
@@ -4496,6 +4499,7 @@ typedef struct dir_server_t {
   uint32_t addr; /**< IPv4 address. */
   uint16_t dir_port; /**< Directory port. */
   uint16_t or_port; /**< OR port: Used for tunneling connections. */
+  double weight; /** Weight used when selecting this node at random */
   char digest[DIGEST_LEN]; /**< Digest of identity key. */
   char v3_identity_digest[DIGEST_LEN]; /**< Digest of v3 (authority only,
                                         * high-security) identity key. */

src/or/router.c

@@ -737,7 +737,7 @@ init_keys(void)
                                 router_get_advertised_or_port(options),
                                 digest,
                                 v3_digest,
-                                type);
+                                type, 0.0);
     if (!ds) {
       log_err(LD_GENERAL,"We want to be a directory authority, but we "
               "couldn't add ourselves to the authority list. Failing.");

src/or/routerlist.c

@@ -3785,12 +3785,16 @@ dir_server_new(int is_authority,
                const char *hostname,
                uint16_t dir_port, uint16_t or_port,
                const char *digest, const char *v3_auth_digest,
-               dirinfo_type_t type)
+               dirinfo_type_t type,
+               double weight)
 {
   dir_server_t *ent;
   uint32_t a;
   char *hostname_ = NULL;
 
+  if (weight < 0)
+    return NULL;
+
   if (tor_addr_family(addr) == AF_INET)
     a = tor_addr_to_ipv4h(addr);
   else
@@ -3810,6 +3814,7 @@ dir_server_new(int is_authority,
   ent->is_running = 1;
   ent->is_authority = is_authority;
   ent->type = type;
+  ent->weight = weight;
   memcpy(ent->digest, digest, DIGEST_LEN);
   if (v3_auth_digest && (type & V3_DIRINFO))
     memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN);
@@ -3842,7 +3847,7 @@ dir_server_t *
 trusted_dir_server_new(const char *nickname, const char *address,
                        uint16_t dir_port, uint16_t or_port,
                        const char *digest, const char *v3_auth_digest,
-                       dirinfo_type_t type)
+                       dirinfo_type_t type, double weight)
 {
   uint32_t a;
   tor_addr_t addr;
@@ -3869,7 +3874,7 @@ trusted_dir_server_new(const char *nickname, const char *address,
 
   result = dir_server_new(1, nickname, &addr, hostname,
                           dir_port, or_port, digest,
-                          v3_auth_digest, type);
+                          v3_auth_digest, type, weight);
   tor_free(hostname);
   return result;
 }
@@ -3880,10 +3885,10 @@ trusted_dir_server_new(const char *nickname, const char *address,
 dir_server_t *
 fallback_dir_server_new(const tor_addr_t *addr,
                         uint16_t dir_port, uint16_t or_port,
-                        const char *id_digest)
+                        const char *id_digest, double weight)
 {
   return dir_server_new(0, NULL, addr, NULL, dir_port, or_port, id_digest,
-                        NULL, ALL_DIRINFO);
+                        NULL, ALL_DIRINFO, weight);
 }
 
 /** Add a directory server to the global list(s). */

src/or/routerlist.h

@@ -136,10 +136,10 @@ int router_exit_policy_rejects_all(const routerinfo_t *router);
 dir_server_t *trusted_dir_server_new(const char *nickname, const char *address,
                        uint16_t dir_port, uint16_t or_port,
                        const char *digest, const char *v3_auth_digest,
-                       dirinfo_type_t type);
+                       dirinfo_type_t type, double weight);
 dir_server_t *fallback_dir_server_new(const tor_addr_t *addr,
                                       uint16_t dir_port, uint16_t or_port,
-                                      const char *id_digest);
+                                      const char *id_digest, double weight);
 void dir_server_add(dir_server_t *ent);
 
 void authority_cert_free(authority_cert_t *cert);