directory.c 130 KB
Newer Older
1
2
/* Copyright (c) 2001-2004, Roger Dingledine.
 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
3
 * Copyright (c) 2007-2008, The Tor Project, Inc. */
Roger Dingledine's avatar
Roger Dingledine committed
4
/* See LICENSE for licensing information */
5
/* $Id$ */
6
7
const char directory_c_id[] =
  "$Id$";
8
9

#include "or.h"
10
#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
11
12
#include <malloc.h>
#endif
13

Roger Dingledine's avatar
Roger Dingledine committed
14
15
/**
 * \file directory.c
16
17
18
 * \brief Code to send and fetch directories and router
 * descriptors via HTTP.  Directories use dirserv.c to generate the
 * results; clients use routers.c to parse them.
Roger Dingledine's avatar
Roger Dingledine committed
19
 **/
20

Roger Dingledine's avatar
Roger Dingledine committed
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/* In-points to directory.c:
 *
 * - directory_post_to_dirservers(), called from
 *   router_upload_dir_desc_to_dirservers() in router.c
 *   upload_service_descriptor() in rendservice.c
 * - directory_get_from_dirserver(), called from
 *   rend_client_refetch_renddesc() in rendclient.c
 *   run_scheduled_events() in main.c
 *   do_hup() in main.c
 * - connection_dir_process_inbuf(), called from
 *   connection_process_inbuf() in connection.c
 * - connection_dir_finished_flushing(), called from
 *   connection_finished_flushing() in connection.c
 * - connection_dir_finished_connecting(), called from
 *   connection_finished_connecting() in connection.c
 */
37
static void directory_send_command(dir_connection_t *conn,
38
39
                             int purpose, int direct, const char *resource,
                             const char *payload, size_t payload_len,
40
                             int supports_conditional_consensus,
41
                             time_t if_modified_since);
42
static int directory_handle_command(dir_connection_t *conn);
43
static int body_is_plausible(const char *body, size_t body_len, int purpose);
44
45
static int purpose_needs_anonymity(uint8_t dir_purpose,
                                   uint8_t router_purpose);
46
static char *http_get_header(const char *headers, const char *which);
47
static void http_set_address_origin(const char *headers, connection_t *conn);
48
static void connection_dir_download_networkstatus_failed(
49
                               dir_connection_t *conn, int status_code);
50
static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
51
52
static void connection_dir_download_cert_failed(
                               dir_connection_t *conn, int status_code);
53
54
static void dir_networkstatus_download_failed(smartlist_t *failed,
                                              int status_code);
55
static void dir_routerdesc_download_failed(smartlist_t *failed,
56
                                           int status_code,
57
                                           int router_purpose,
58
59
                                           int was_extrainfo,
                                           int was_descriptor_digests);
60
static void note_client_request(int purpose, int compressed, size_t bytes);
61
static int client_likes_consensus(networkstatus_t *v, const char *want_url);
Roger Dingledine's avatar
Roger Dingledine committed
62

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
static void directory_initiate_command_rend(const char *address,
                                            const tor_addr_t *addr,
                                            uint16_t or_port,
                                            uint16_t dir_port,
                                            int supports_conditional_consensus,
                                            int supports_begindir,
                                            const char *digest,
                                            uint8_t dir_purpose,
                                            uint8_t router_purpose,
                                            int anonymized_connection,
                                            const char *resource,
                                            const char *payload,
                                            size_t payload_len,
                                            time_t if_modified_since,
                                            const rend_data_t *rend_query);

79
80
/********* START VARIABLES **********/

81
82
83
/** How far in the future do we allow a directory server to tell us it is
 * before deciding that one of us has the wrong time? */
#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
84

85
86
#define X_ADDRESS_HEADER "X-Your-Address-Is: "

87
88
/** HTTP cache control: how long do we tell proxies they can cache each
 * kind of document we serve? */
89
90
#define FULL_DIR_CACHE_LIFETIME (60*60)
#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
91
#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
92
93
#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
#define ROUTERDESC_CACHE_LIFETIME (30*60)
94
#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
95
96
#define ROBOTS_CACHE_LIFETIME (24*60*60)

97
98
/********* END VARIABLES ************/

99
100
/** Return true iff the directory purpose 'purpose' must use an
 * anonymous connection to a directory. */
101
static int
102
purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
103
{
104
105
  if (get_options()->AllDirActionsPrivate)
    return 1;
106
107
  if (router_purpose == ROUTER_PURPOSE_BRIDGE && has_completed_circuit)
    return 1; /* if no circuits yet, we may need this info to bootstrap. */
108
  if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR ||
109
      dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
110
      dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES ||
111
      dir_purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS ||
112
113
114
115
      dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
      dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES ||
      dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
      dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
116
117
      dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
      dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO)
118
119
120
121
    return 0;
  return 1;
}

122
123
/** Return a newly allocated string describing <b>auth</b>. */
char *
124
125
authority_type_to_string(authority_type_t auth)
{
126
127
128
129
130
131
132
133
134
135
136
137
138
139
  char *result;
  smartlist_t *lst = smartlist_create();
  if (auth & V1_AUTHORITY)
    smartlist_add(lst, (void*)"V1");
  if (auth & V2_AUTHORITY)
    smartlist_add(lst, (void*)"V2");
  if (auth & BRIDGE_AUTHORITY)
    smartlist_add(lst, (void*)"Bridge");
  if (auth & HIDSERV_AUTHORITY)
    smartlist_add(lst, (void*)"Hidden service");
  if (smartlist_len(lst)) {
    result = smartlist_join_strings(lst, ", ", 0, NULL);
  } else {
    result = tor_strdup("[Not an authority]");
140
  }
141
142
  smartlist_free(lst);
  return result;
143
144
}

145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
/** Return a string describing a given directory connection purpose. */
static const char *
dir_conn_purpose_to_string(int purpose)
{
  switch (purpose)
    {
    case DIR_PURPOSE_FETCH_RENDDESC:
      return "hidden-service descriptor fetch";
    case DIR_PURPOSE_UPLOAD_DIR:
      return "server descriptor upload";
    case DIR_PURPOSE_UPLOAD_RENDDESC:
      return "hidden-service descriptor upload";
    case DIR_PURPOSE_UPLOAD_VOTE:
      return "server vote upload";
    case DIR_PURPOSE_UPLOAD_SIGNATURES:
      return "consensus signature upload";
161
    case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS:
162
163
164
165
166
167
168
169
170
171
172
173
174
      return "network-status fetch";
    case DIR_PURPOSE_FETCH_SERVERDESC:
      return "server descriptor fetch";
    case DIR_PURPOSE_FETCH_EXTRAINFO:
      return "extra-info fetch";
    case DIR_PURPOSE_FETCH_CONSENSUS:
      return "consensus network-status fetch";
    case DIR_PURPOSE_FETCH_CERTIFICATE:
      return "authority cert fetch";
    case DIR_PURPOSE_FETCH_STATUS_VOTE:
      return "status vote fetch";
    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
      return "consensus signature fetch";
175
176
177
178
    case DIR_PURPOSE_FETCH_RENDDESC_V2:
      return "hidden-service v2 descriptor fetch";
    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
      return "hidden-service v2 descriptor upload";
179
180
181
182
183
184
    }

  log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
  return "(unknown)";
}

185
186
187
/** Return true iff <b>identity_digest</b> is the digest of a router we
 * believe to support extrainfo downloads.  (If <b>is_authority</b> we do
 * additional checking that's only valid for authorities.) */
188
189
190
191
192
193
194
195
196
197
198
199
200
int
router_supports_extrainfo(const char *identity_digest, int is_authority)
{
  routerinfo_t *ri = router_get_by_digest(identity_digest);

  if (ri) {
    if (ri->caches_extra_info)
      return 1;
    if (is_authority && ri->platform &&
        tor_version_as_new_as(ri->platform, "Tor 0.2.0.0-alpha-dev (r10070)"))
      return 1;
  }
  if (is_authority) {
201
    routerstatus_t *rs = router_get_consensus_status_by_id(identity_digest);
202
    if (rs && rs->version_supports_extrainfo_upload)
203
204
205
206
207
      return 1;
  }
  return 0;
}

208
209
210
211
212
213
214
215
/** Return true iff any trusted directory authority has accepted our
 * server descriptor.
 *
 * We consider any authority sufficient because waiting for all of
 * them means it never happens while any authority is down; we don't
 * go for something more complex in the middle (like \>1/3 or \>1/2 or
 * \>=1/2) because that doesn't seem necessary yet.
 */
216
217
218
219
int
directories_have_accepted_server_descriptor(void)
{
  smartlist_t *servers = router_get_trusted_dir_servers();
220
  or_options_t *options = get_options();
221
  SMARTLIST_FOREACH(servers, trusted_dir_server_t *, d, {
222
    if ((d->type & options->_PublishServerDescriptor) &&
223
224
        d->has_accepted_serverdesc) {
      return 1;
225
226
    }
  });
227
  return 0;
228
229
}

230
/** Start a connection to every suitable directory authority, using
Roger Dingledine's avatar
Roger Dingledine committed
231
232
233
 * connection purpose 'purpose' and uploading the payload 'payload'
 * (length 'payload_len').  The purpose should be one of
 * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'.
234
 *
235
236
237
 * <b>type</b> specifies what sort of dir authorities (V1, V2,
 * HIDSERV, BRIDGE) we should upload to.
 *
238
239
240
241
242
 * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
 * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
 * bytes of <b>payload</b> hold an extra-info document.  Upload the descriptor
 * to all authorities, and the extra-info document to all authorities that
 * support it.
Roger Dingledine's avatar
Roger Dingledine committed
243
244
 */
void
245
246
directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
                             authority_type_t type,
247
                             const char *payload,
248
                             size_t payload_len, size_t extrainfo_len)
Roger Dingledine's avatar
Roger Dingledine committed
249
{
250
  int post_via_tor;
251
  smartlist_t *dirservers = router_get_trusted_dir_servers();
252
  int found = 0;
253
  tor_assert(dirservers);
Nick Mathewson's avatar
Nick Mathewson committed
254
255
256
  /* This tries dirservers which we believe to be down, but ultimately, that's
   * harmless, and we may as well err on the side of getting things uploaded.
   */
257
  SMARTLIST_FOREACH_BEGIN(dirservers, trusted_dir_server_t *, ds) {
258
      routerstatus_t *rs = &(ds->fake_status);
259
      size_t upload_len = payload_len;
260
      tor_addr_t ds_addr;
261
262

      if ((type & ds->type) == 0)
263
        continue;
264

265
      found = 1; /* at least one authority of this type was listed */
266
      if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
267
        ds->has_accepted_serverdesc = 0;
268

269
      if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
270
        upload_len += extrainfo_len;
271
272
        log_info(LD_DIR, "Uploading an extrainfo (length %d)",
                 (int) extrainfo_len);
273
      }
274
      tor_addr_from_ipv4h(&ds_addr, ds->addr);
275
      post_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose) ||
276
        !fascist_firewall_allows_address_dir(&ds_addr, ds->dir_port);
277
278
      directory_initiate_command_routerstatus(rs, dir_purpose,
                                              router_purpose,
279
                                              post_via_tor,
280
                                              NULL, payload, upload_len, 0);
281
  } SMARTLIST_FOREACH_END(ds);
282
  if (!found) {
283
    char *s = authority_type_to_string(type);
284
    log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
285
286
             "of type '%s', but no authorities of that type listed!", s);
    tor_free(s);
287
  }
Roger Dingledine's avatar
Roger Dingledine committed
288
289
}

Roger Dingledine's avatar
Roger Dingledine committed
290
/** Start a connection to a random running directory server, using
291
292
 * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
 * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
Roger Dingledine's avatar
Roger Dingledine committed
293
294
 * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
 * or router_pick_trusteddirserver().
Roger Dingledine's avatar
Roger Dingledine committed
295
296
 */
void
297
directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
298
                             const char *resource, int pds_flags)
Roger Dingledine's avatar
Roger Dingledine committed
299
{
300
  routerstatus_t *rs = NULL;
301
  or_options_t *options = get_options();
302
  int prefer_authority = directory_fetches_from_authorities(options);
303
  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose);
304
  authority_type_t type;
305
  time_t if_modified_since = 0;
306

Roger Dingledine's avatar
Roger Dingledine committed
307
308
  /* FFFF we could break this switch into its own function, and call
   * it elsewhere in directory.c. -RD */
309
  switch (dir_purpose) {
310
    case DIR_PURPOSE_FETCH_EXTRAINFO:
311
312
313
      type = EXTRAINFO_CACHE |
             (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_AUTHORITY :
                                                        V2_AUTHORITY);
314
      break;
315
    case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS:
316
    case DIR_PURPOSE_FETCH_SERVERDESC:
317
318
      type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_AUTHORITY :
                                                        V2_AUTHORITY);
319
320
321
322
      break;
    case DIR_PURPOSE_FETCH_RENDDESC:
      type = HIDSERV_AUTHORITY;
      break;
323
324
    case DIR_PURPOSE_FETCH_STATUS_VOTE:
    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
325
326
      type = V3_AUTHORITY;
      break;
327
328
329
330
    case DIR_PURPOSE_FETCH_CONSENSUS:
    case DIR_PURPOSE_FETCH_CERTIFICATE:
      type = V3_AUTHORITY;
      break;
331
    default:
332
      log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
333
334
      return;
  }
335

336
  if (DIR_PURPOSE_FETCH_CONSENSUS) {
337
    networkstatus_t *v = networkstatus_get_latest_consensus();
338
339
340
341
    if (v)
      if_modified_since = v->valid_after + 180;
  }

342
  if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
343
344
    return;

345
346
347
  if (!get_via_tor) {
    if (options->UseBridges && type != BRIDGE_AUTHORITY) {
      /* want to ask a running bridge for which we have a descriptor. */
Roger Dingledine's avatar
Roger Dingledine committed
348
      /* XXX021 we assume that all of our bridges can answer any
349
       * possible directory question. This won't be true forever. -RD */
350
351
      /* It certainly is not true with conditional consensus downloading,
       * so, for now, never assume the server supports that. */
352
353
      routerinfo_t *ri = choose_random_entry(NULL);
      if (ri) {
354
355
356
        tor_addr_t addr;
        tor_addr_from_ipv4h(&addr, ri->addr);
        directory_initiate_command(ri->address, &addr,
357
                                   ri->or_port, 0,
358
                                   0, /* don't use conditional consensus url */
359
360
361
                                   1, ri->cache_info.identity_digest,
                                   dir_purpose,
                                   router_purpose,
362
                                   0, resource, NULL, 0, if_modified_since);
363
364
365
366
367
368
369
      } else
        log_notice(LD_DIR, "Ignoring directory request, since no bridge "
                           "nodes are available yet.");
      return;
    } else {
      if (prefer_authority || type == BRIDGE_AUTHORITY) {
        /* only ask authdirservers, and don't ask myself */
370
        rs = router_pick_trusteddirserver(type, pds_flags);
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
        if (rs == NULL && (pds_flags & PDS_NO_EXISTING_SERVERDESC_FETCH)) {
          /* We don't want to fetch from any authorities that we're currently
           * fetching server descriptors from, and we got no match.  Did we
           * get no match because all the authorities have connections
           * fetching server descriptors (in which case we should just
           * return,) or because all the authorities are down or on fire or
           * unreachable or something (in which case we should go on with
           * our fallback code)? */
          pds_flags &= ~PDS_NO_EXISTING_SERVERDESC_FETCH;
          rs = router_pick_trusteddirserver(type, pds_flags);
          if (rs) {
            log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
                      "are in use.");
            return;
          }
        }
387
388
389
      }
      if (!rs && type != BRIDGE_AUTHORITY) {
        /* anybody with a non-zero dirport will do */
390
        rs = router_pick_directory_server(type, pds_flags);
391
392
393
        if (!rs) {
          log_info(LD_DIR, "No router found for %s; falling back to "
                   "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
394
          rs = router_pick_trusteddirserver(type, pds_flags);
395
396
397
          if (!rs)
            get_via_tor = 1; /* last resort: try routing it via Tor */
        }
398
      }
399
    }
400
  } else { /* get_via_tor */
401
    /* Never use fascistfirewall; we're going via Tor. */
402
    if (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) {
Roger Dingledine's avatar
Roger Dingledine committed
403
      /* only ask hidserv authorities, any of them will do */
404
405
      pds_flags |= PDS_IGNORE_FASCISTFIREWALL|PDS_ALLOW_SELF;
      rs = router_pick_trusteddirserver(HIDSERV_AUTHORITY, pds_flags);
406
407
    } else {
      /* anybody with a non-zero dirport will do. Disregard firewalls. */
408
409
      pds_flags |= PDS_IGNORE_FASCISTFIREWALL;
      rs = router_pick_directory_server(type, pds_flags);
410
      /* If we have any hope of building an indirect conn, we know some router
Roger Dingledine's avatar
Roger Dingledine committed
411
       * descriptors.  If (rs==NULL), we can't build circuits anyway, so
Roger Dingledine's avatar
Roger Dingledine committed
412
       * there's no point in falling back to the authorities in this case. */
413
    }
414
415
  }

416
  if (rs)
417
    directory_initiate_command_routerstatus(rs, dir_purpose,
418
419
                                            router_purpose,
                                            get_via_tor,
420
421
                                            resource, NULL, 0,
                                            if_modified_since);
422
  else {
423
    log_notice(LD_DIR,
424
425
               "While fetching directory info, "
               "no running dirservers known. Will try again later. "
426
               "(purpose %d)", dir_purpose);
427
    if (!purpose_needs_anonymity(dir_purpose, router_purpose)) {
428
      /* remember we tried them all and failed. */
429
      directory_all_unreachable(time(NULL));
430
    }
431
  }
Roger Dingledine's avatar
Roger Dingledine committed
432
433
}

434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
/** As directory_get_from_dirserver, but initiates a request to <i>every</i>
 * directory authority other than ourself.  Only for use by authorities when
 * searching for missing information while voting. */
void
directory_get_from_all_authorities(uint8_t dir_purpose,
                                   uint8_t router_purpose,
                                   const char *resource)
{
  tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
             dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);

  SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
                    trusted_dir_server_t *, ds,
    {
      routerstatus_t *rs;
      if (router_digest_is_me(ds->digest))
        continue;
      if (!(ds->type & V3_AUTHORITY))
        continue;
453
      rs = &ds->fake_status;
454
      directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose,
455
                                              0, resource, NULL, 0, 0);
456
457
458
    });
}

459
460
/** Same as directory_initiate_command_routerstatus(), but accepts
 * rendezvous data to fetch a hidden service descriptor. */
461
void
462
463
464
465
466
467
468
469
470
directory_initiate_command_routerstatus_rend(routerstatus_t *status,
                                             uint8_t dir_purpose,
                                             uint8_t router_purpose,
                                             int anonymized_connection,
                                             const char *resource,
                                             const char *payload,
                                             size_t payload_len,
                                             time_t if_modified_since,
                                             const rend_data_t *rend_query)
471
{
472
  routerinfo_t *router;
473
  char address_buf[INET_NTOA_BUF_LEN+1];
474
475
  struct in_addr in;
  const char *address;
476
  tor_addr_t addr;
477
478
  router = router_get_by_digest(status->identity_digest);
  if (!router && anonymized_connection) {
Roger Dingledine's avatar
Roger Dingledine committed
479
480
    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
                     "don't have its router descriptor.", status->nickname);
481
482
    return;
  } else if (router) {
483
484
485
486
487
488
    address = router->address;
  } else {
    in.s_addr = htonl(status->addr);
    tor_inet_ntoa(&in, address_buf, sizeof(address_buf));
    address = address_buf;
  }
489
  tor_addr_from_ipv4h(&addr, status->addr);
490
  directory_initiate_command_rend(address, &addr,
491
                             status->or_port, status->dir_port,
492
                             status->version_supports_conditional_consensus,
493
494
                             status->version_supports_begindir,
                             status->identity_digest,
495
496
                             dir_purpose, router_purpose,
                             anonymized_connection, resource,
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
                             payload, payload_len, if_modified_since,
                             rend_query);
}

/** Launch a new connection to the directory server <b>status</b> to
 * upload or download a server or rendezvous
 * descriptor. <b>dir_purpose</b> determines what
 * kind of directory connection we're launching, and must be one of
 * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC|RENDDESC_V2}. <b>router_purpose</b>
 * specifies the descriptor purposes we have in mind (currently only
 * used for FETCH_DIR).
 *
 * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
 * of the HTTP post.  Otherwise, <b>payload</b> should be NULL.
 *
 * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
 * want to fetch.
 */
void
directory_initiate_command_routerstatus(routerstatus_t *status,
                                        uint8_t dir_purpose,
                                        uint8_t router_purpose,
                                        int anonymized_connection,
                                        const char *resource,
                                        const char *payload,
                                        size_t payload_len,
                                        time_t if_modified_since)
{
  directory_initiate_command_routerstatus_rend(status, dir_purpose,
                                          router_purpose,
                                          anonymized_connection, resource,
                                          payload, payload_len,
                                          if_modified_since, NULL);
530
531
}

532
533
534
/** Return true iff <b>conn</b> is the client side of a directory connection
 * we launched to ourself in order to determine the reachability of our
 * dir_port. */
535
536
537
538
539
540
541
542
static int
directory_conn_is_self_reachability_test(dir_connection_t *conn)
{
  if (conn->requested_resource &&
      !strcmpstart(conn->requested_resource,"authority")) {
    routerinfo_t *me = router_get_my_routerinfo();
    if (me &&
        router_digest_is_me(conn->identity_digest) &&
543
        tor_addr_eq_ipv4h(&conn->_base.addr, me->addr) && /*XXXX021 prop 118*/
544
545
546
547
548
549
        me->dir_port == conn->_base.port)
      return 1;
  }
  return 0;
}

550
551
552
/** Called when we are unable to complete the client's request to a directory
 * server due to a network error: Mark the router as down and try again if
 * possible.
553
 */
554
void
555
connection_dir_request_failed(dir_connection_t *conn)
556
{
557
558
559
560
561
562
  if (directory_conn_is_self_reachability_test(conn)) {
    routerinfo_t *me = router_get_my_routerinfo();
    if (me)
      control_event_server_status(LOG_WARN,
                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
                                  me->address, me->dir_port);
563
    return; /* this was a test fetch. don't retry. */
564
  }
565
566
  if (entry_list_can_grow(get_options()))
    router_set_status(conn->identity_digest, 0); /* don't try him again */
567
  if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) {
568
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
569
             conn->_base.address);
570
    connection_dir_download_networkstatus_failed(conn, -1);
571
572
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
573
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
574
             conn->_base.address);
575
    connection_dir_download_routerdesc_failed(conn);
576
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
577
    networkstatus_consensus_download_failed(0);
578
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
579
580
581
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
             conn->_base.address);
    connection_dir_download_cert_failed(conn, 0);
582
583
584
585
586
587
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
    log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
             conn->_base.address);
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
    log_info(LD_DIR, "Giving up downloading votes from '%s'",
             conn->_base.address);
588
589
590
  }
}

591
592
593
/** Called when an attempt to download one or more network status
 * documents on connection <b>conn</b> failed. Decide whether to
 * retry the fetch now, later, or never.
594
 */
595
static void
596
connection_dir_download_networkstatus_failed(dir_connection_t *conn,
597
                                             int status_code)
598
{
599
600
601
602
603
604
  if (!conn->requested_resource) {
    /* We never reached directory_send_command, which means that we never
     * opened a network connection.  Either we're out of sockets, or the
     * network is down.  Either way, retrying would be pointless. */
    return;
  }
605
  if (!strcmpstart(conn->requested_resource, "all")) {
606
607
608
    /* We're a non-authoritative directory cache; try again. Ignore status
     * code, since we don't want to keep trying forever in a tight loop
     * if all the authorities are shutting us out. */
609
610
    smartlist_t *trusted_dirs = router_get_trusted_dir_servers();
    SMARTLIST_FOREACH(trusted_dirs, trusted_dir_server_t *, ds,
611
                      download_status_failed(&ds->v2_ns_dl_status, 0));
612
613
    directory_get_from_dirserver(conn->_base.purpose, conn->router_purpose,
                                 "all.z", 0 /* don't retry_if_no_servers */);
614
  } else if (!strcmpstart(conn->requested_resource, "fp/")) {
Roger Dingledine's avatar
Roger Dingledine committed
615
    /* We were trying to download by fingerprint; mark them all as having
616
617
     * failed, and possibly retry them later.*/
    smartlist_t *failed = smartlist_create();
618
    dir_split_resource_into_fingerprints(conn->requested_resource+3,
619
                                         failed, NULL, 0, 0);
620
    if (smartlist_len(failed)) {
621
      dir_networkstatus_download_failed(failed, status_code);
622
      SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp));
623
    }
624
625
    smartlist_free(failed);
  }
626
627
}

628
/** Called when an attempt to download one or more router descriptors
629
 * or extra-info documents on connection <b>conn</b> failed.
630
631
 */
static void
632
connection_dir_download_routerdesc_failed(dir_connection_t *conn)
633
{
634
  /* No need to increment the failure count for routerdescs, since
635
   * it's not their fault. */
636

637
  /* No need to relaunch descriptor downloads here: we already do it
638
   * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
639
640
  tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
641

642
  (void) conn;
643
644
}

645
646
647
648
649
650
651
652
653
654
/** Called when an attempt to fetch a certificate fails. */
static void
connection_dir_download_cert_failed(dir_connection_t *conn, int status)
{
  smartlist_t *failed;
  tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);

  if (!conn->requested_resource)
    return;
  failed = smartlist_create();
655
  dir_split_resource_into_fingerprints(conn->requested_resource+3,
656
657
658
                                       failed, NULL, 1, 0);
  SMARTLIST_FOREACH(failed, char *, cp,
  {
659
    authority_cert_dl_failed(cp, status);
660
661
662
    tor_free(cp);
  });
  smartlist_free(failed);
663
664

  update_certificate_downloads(time(NULL));
665
666
}

Roger Dingledine's avatar
Roger Dingledine committed
667
668
669
670
671
672
673
674
675
/** Evaluate the situation and decide if we should use an encrypted
 * "begindir-style" connection for this directory request.
 * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
 *    or we're a dir mirror, no.
 * 2) If we prefer to avoid begindir conns, and we're not fetching or
 * publishing a bridge relay descriptor, no.
 * 3) Else yes.
 */
static int
676
677
directory_command_should_use_begindir(or_options_t *options,
                                      const tor_addr_t *addr,
Roger Dingledine's avatar
Roger Dingledine committed
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
                                      int or_port, uint8_t router_purpose,
                                      int anonymized_connection)
{
  if (!or_port)
    return 0; /* We don't know an ORPort -- no chance. */
  if (!anonymized_connection)
    if (!fascist_firewall_allows_address_or(addr, or_port) ||
        directory_fetches_from_authorities(options))
      return 0; /* We're firewalled or are acting like a relay -- also no. */
  if (!options->TunnelDirConns &&
      router_purpose != ROUTER_PURPOSE_BRIDGE)
    return 0; /* We prefer to avoid using begindir conns. Fine. */
  return 1;
}

693
/** Helper for directory_initiate_command_routerstatus: send the
694
 * command to a server whose address is <b>address</b>, whose IP is
695
696
697
 * <b>addr</b>, whose directory port is <b>dir_port</b>, whose tor version
 * <b>supports_begindir</b>, and whose identity key digest is
 * <b>digest</b>. */
698
void
699
directory_initiate_command(const char *address, const tor_addr_t *_addr,
700
                           uint16_t or_port, uint16_t dir_port,
701
                           int supports_conditional_consensus,
702
703
                           int supports_begindir, const char *digest,
                           uint8_t dir_purpose, uint8_t router_purpose,
704
                           int anonymized_connection, const char *resource,
705
706
                           const char *payload, size_t payload_len,
                           time_t if_modified_since)
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
{
  directory_initiate_command_rend(address, _addr, or_port, dir_port,
                             supports_conditional_consensus,
                             supports_begindir, digest, dir_purpose,
                             router_purpose, anonymized_connection,
                             resource, payload, payload_len,
                             if_modified_since, NULL);
}

/** Same as directory_initiate_command(), but accepts rendezvous data to
 * fetch a hidden service descriptor. */
static void
directory_initiate_command_rend(const char *address, const tor_addr_t *_addr,
                                uint16_t or_port, uint16_t dir_port,
                                int supports_conditional_consensus,
                                int supports_begindir, const char *digest,
                                uint8_t dir_purpose, uint8_t router_purpose,
                                int anonymized_connection,
                                const char *resource,
                                const char *payload, size_t payload_len,
                                time_t if_modified_since,
                                const rend_data_t *rend_query)
Roger Dingledine's avatar
Roger Dingledine committed
729
{
730
  dir_connection_t *conn;
731
  or_options_t *options = get_options();
732
  int socket_error = 0;
Roger Dingledine's avatar
Roger Dingledine committed
733
  int use_begindir = supports_begindir &&
734
                     directory_command_should_use_begindir(options, _addr,
Roger Dingledine's avatar
Roger Dingledine committed
735
                       or_port, router_purpose, anonymized_connection);
736
  tor_addr_t addr;
737

738
  tor_assert(address);
739
  tor_assert(_addr);
740
  tor_assert(or_port || dir_port);
741
  tor_assert(digest);
742

743
744
  tor_addr_copy(&addr, _addr);

745
746
  log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
            anonymized_connection, use_begindir);
747

748
  log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
749

750
  conn = dir_connection_new(AF_INET);
751
752

  /* set up conn so it's got all the data we need to remember */
753
  tor_addr_copy(&conn->_base.addr, &addr);
754
  conn->_base.port = use_begindir ? or_port : dir_port;
755
  conn->_base.address = tor_strdup(address);
756
  memcpy(conn->identity_digest, digest, DIGEST_LEN);
757

758
759
  conn->_base.purpose = dir_purpose;
  conn->router_purpose = router_purpose;
760

761
  /* give it an initial state */
762
  conn->_base.state = DIR_CONN_STATE_CONNECTING;
763

764
765
766
  /* decide whether we can learn our IP address from this conn */
  conn->dirconn_direct = !anonymized_connection;

767
768
769
770
  /* copy rendezvous data, if any */
  if (rend_query)
    conn->rend_data = rend_data_dup(rend_query);

771
772
  if (!anonymized_connection && !use_begindir) {
    /* then we want to connect to dirport directly */
773

774
    if (options->HttpProxy) {
775
      tor_addr_from_ipv4h(&addr, options->HttpProxyAddr);
776
      dir_port = options->HttpProxyPort;
777
778
    }

779
    switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr,
780
                               dir_port, &socket_error)) {
Roger Dingledine's avatar
Roger Dingledine committed
781
      case -1:
782
        connection_dir_request_failed(conn); /* retry if we want */
783
784
        /* XXX we only pass 'conn' above, not 'resource', 'payload',
         * etc. So in many situations it can't retry! -RD */
785
        connection_free(TO_CONN(conn));
Roger Dingledine's avatar
Roger Dingledine committed
786
787
        return;
      case 1:
788
789
        /* start flushing conn */
        conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
Roger Dingledine's avatar
Roger Dingledine committed
790
791
        /* fall through */
      case 0:
792
        /* queue the command on the outbuf */
793
        directory_send_command(conn, dir_purpose, 1, resource,
794
795
796
                               payload, payload_len,
                               supports_conditional_consensus,
                               if_modified_since);
797
        connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE);
Roger Dingledine's avatar
Roger Dingledine committed
798
799
800
        /* writable indicates finish, readable indicates broken link,
           error indicates broken link in windowsland. */
    }
801
  } else { /* we want to connect via a tor connection */
802
    edge_connection_t *linked_conn;
803
804
805
806

    /* If it's an anonymized connection, remember the fact that we
     * wanted it for later: maybe we'll want it again soon. */
    if (anonymized_connection && use_begindir)
807
      rep_hist_note_used_internal(time(NULL), 0, 1);
808
809
810
    else if (anonymized_connection && !use_begindir)
      rep_hist_note_used_port(time(NULL), conn->_base.port);

Roger Dingledine's avatar
Roger Dingledine committed
811
    /* make an AP connection
812
     * populate it and add it at the right state
813
     * hook up both sides
Roger Dingledine's avatar
Roger Dingledine committed
814
     */
815
    linked_conn =
816
      connection_ap_make_link(conn->_base.address, conn->_base.port,
817
                              digest, use_begindir, conn->dirconn_direct);
818
    if (!linked_conn) {
819
      log_warn(LD_NET,"Making tunnel to dirserver failed.");
820
      connection_mark_for_close(TO_CONN(conn));
821
822
      return;
    }
823
    connection_link_connections(TO_CONN(conn), TO_CONN(linked_conn));
Roger Dingledine's avatar
Roger Dingledine committed
824

825
    if (connection_add(TO_CONN(conn)) < 0) {
826
      log_warn(LD_NET,"Unable to add connection for link to dirserver.");
827
      connection_mark_for_close(TO_CONN(conn));
828
829
      return;
    }
830
    conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
831
    /* queue the command on the outbuf */
832
    directory_send_command(conn, dir_purpose, 0, resource,
833
834
835
                           payload, payload_len,
                           supports_conditional_consensus,
                           if_modified_since);
836
    connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE);
837
    connection_start_reading(TO_CONN(linked_conn));
838
839
840
  }
}

841
842
843
844
845
846
847
848
849
850
851
852
853
/** Return true iff anything we say on <b>conn</b> is being encrypted before
 * we send it to the client/server. */
int
connection_dir_is_encrypted(dir_connection_t *conn)
{
  /* Right now it's sufficient to see if conn is or has been linked, since
   * the only thing it could be linked to is an edge connection on a
   * circuit, and the only way it could have been unlinked is at the edge
   * connection getting closed.
   */
  return TO_CONN(conn)->linked;
}

854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
/** Helper for sorting
 *
 * sort strings alphabetically
 */
static int
_compare_strs(const void **a, const void **b)
{
  const char *s1 = *a, *s2 = *b;
  return strcmp(s1, s2);
}

/** Return the URL we should use for a consensus download.
 *
 * This url depends on whether or not the server we go to
 * is sufficiently new to support conditional consensus downloading,
 * i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
 */
#define CONDITIONAL_CONSENSUS_FPR_LEN 3
#if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
873
#error "conditional consensus fingerprint length is larger than digest length"
874
875
876
877
878
879
880
881
882
883
884
885
886
887
#endif
static char *
directory_get_consensus_url(int supports_conditional_consensus)
{
  char *url;
  int len;

  if (supports_conditional_consensus) {
    char *authority_id_list;
    smartlist_t *authority_digets = smartlist_create();

    SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
                      trusted_dir_server_t *, ds,
      {
888
889
890
891
892
        char *hex;
        if (!(ds->type & V3_AUTHORITY))
          continue;

        hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
893
        base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
894
                      ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
        smartlist_add(authority_digets, hex);
      });
    smartlist_sort(authority_digets, _compare_strs);
    authority_id_list = smartlist_join_strings(authority_digets,
                                               "+", 0, NULL);

    len = strlen(authority_id_list)+64;
    url = tor_malloc(len);
    tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z",
                 authority_id_list);

    SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp));
    smartlist_free(authority_digets);
    tor_free(authority_id_list);
  } else {
    url = tor_strdup("/tor/status-vote/current/consensus.z");
  }
  return url;
}

915
916
/** Queue an appropriate HTTP command on conn-\>outbuf.  The other args
 * are as in directory_initiate_command.
917
 */
918
static void
919
directory_send_command(dir_connection_t *conn,
920
                       int purpose, int direct, const char *resource,
921
                       const char *payload, size_t payload_len,
922
                       int supports_conditional_consensus,
923
                       time_t if_modified_since)
924
{
925
926
  char proxystring[256];
  char proxyauthstring[256];
927
  char hoststring[128];
928
  char imsstring[RFC1123_TIME_LEN+32];
929
930
  char *url;
  char request[8192];
931
  const char *httpcommand = NULL;
932
  size_t len;
933

934
  tor_assert(conn);
935
  tor_assert(conn->_base.type == CONN_TYPE_DIR);
936

937
  tor_free(conn->requested_resource);
938
939
  if (resource)
    conn->requested_resource = tor_strdup(resource);
940

941
  /* come up with a string for which Host: we want */
942
943
  if (conn->_base.port == 80) {
    strlcpy(hoststring, conn->_base.address, sizeof(hoststring));
944
  } else {
945
    tor_snprintf(hoststring, sizeof(hoststring),"%s:%d",
946
                 conn->_base.address, conn->_base.port);
947
  }
948

949
950
951
952
953
954
955
956
957
  /* Format if-modified-since */
  if (!if_modified_since) {
    imsstring[0] = '\0';
  } else {
    char b[RFC1123_TIME_LEN+1];
    format_rfc1123_time(b, if_modified_since);
    tor_snprintf(imsstring, sizeof(imsstring), "\r\nIf-Modified-Since: %s", b);
  }

958
  /* come up with some proxy lines, if we're using one. */
959
  if (direct && get_options()->HttpProxy) {
960
961
962
    char *base64_authenticator=NULL;
    const char *authenticator = get_options()->HttpProxyAuthenticator;

963
    tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
964
965
966
    if (authenticator) {
      base64_authenticator = alloc_http_authenticator(authenticator);
      if (!base64_authenticator)
967
        log_warn(LD_BUG, "Encoding http authenticator failed");
968
969
970
971
972
973
974
975
976
    }
    if (base64_authenticator) {
      tor_snprintf(proxyauthstring, sizeof(proxyauthstring),
                   "\r\nProxy-Authorization: Basic %s",
                   base64_authenticator);
      tor_free(base64_authenticator);
    } else {
      proxyauthstring[0] = 0;
    }
977
978
  } else {
    proxystring[0] = 0;