directory.c 118 KB
Newer Older
1
2
3
/* Copyright (c) 2001-2004, Roger Dingledine.
 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 * Copyright (c) 2007, The Tor Project, Inc. */
Roger Dingledine's avatar
Roger Dingledine committed
4
/* See LICENSE for licensing information */
5
/* $Id$ */
6
7
const char directory_c_id[] =
  "$Id$";
8
9

#include "or.h"
10
#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
11
12
#include <malloc.h>
#endif
13

Roger Dingledine's avatar
Roger Dingledine committed
14
15
/**
 * \file directory.c
16
17
18
 * \brief Code to send and fetch directories and router
 * descriptors via HTTP.  Directories use dirserv.c to generate the
 * results; clients use routers.c to parse them.
Roger Dingledine's avatar
Roger Dingledine committed
19
 **/
20

Roger Dingledine's avatar
Roger Dingledine committed
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/* In-points to directory.c:
 *
 * - directory_post_to_dirservers(), called from
 *   router_upload_dir_desc_to_dirservers() in router.c
 *   upload_service_descriptor() in rendservice.c
 * - directory_get_from_dirserver(), called from
 *   rend_client_refetch_renddesc() in rendclient.c
 *   run_scheduled_events() in main.c
 *   do_hup() in main.c
 * - connection_dir_process_inbuf(), called from
 *   connection_process_inbuf() in connection.c
 * - connection_dir_finished_flushing(), called from
 *   connection_finished_flushing() in connection.c
 * - connection_dir_finished_connecting(), called from
 *   connection_finished_connecting() in connection.c
 */
37
static void directory_send_command(dir_connection_t *conn,
38
39
40
                             int purpose, int direct, const char *resource,
                             const char *payload, size_t payload_len,
                             time_t if_modified_since);
41
static int directory_handle_command(dir_connection_t *conn);
42
static int body_is_plausible(const char *body, size_t body_len, int purpose);
43
44
static int purpose_needs_anonymity(uint8_t dir_purpose,
                                   uint8_t router_purpose);
45
static char *http_get_header(const char *headers, const char *which);
46
static void http_set_address_origin(const char *headers, connection_t *conn);
47
static void connection_dir_download_networkstatus_failed(
48
                               dir_connection_t *conn, int status_code);
49
static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
50
51
static void connection_dir_download_cert_failed(
                               dir_connection_t *conn, int status_code);
52
53
static void dir_networkstatus_download_failed(smartlist_t *failed,
                                              int status_code);
54
static void dir_routerdesc_download_failed(smartlist_t *failed,
55
                                           int status_code,
56
                                           int router_purpose,
57
58
                                           int was_extrainfo,
                                           int was_descriptor_digests);
59
static void note_request(const char *key, size_t bytes);
Roger Dingledine's avatar
Roger Dingledine committed
60

61
62
/********* START VARIABLES **********/

63
64
65
/** How far in the future do we allow a directory server to tell us it is
 * before deciding that one of us has the wrong time? */
#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
66

67
68
#define X_ADDRESS_HEADER "X-Your-Address-Is: "

69
70
/** HTTP cache control: how long do we tell proxies they can cache each
 * kind of document we serve? */
71
72
73
74
#define FULL_DIR_CACHE_LIFETIME (60*60)
#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
#define ROUTERDESC_CACHE_LIFETIME (30*60)
75
#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
76
77
#define ROBOTS_CACHE_LIFETIME (24*60*60)

78
79
/********* END VARIABLES ************/

80
81
/** Return true iff the directory purpose 'purpose' must use an
 * anonymous connection to a directory. */
82
static int
83
purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
84
{
85
86
  if (get_options()->AllDirActionsPrivate)
    return 1;
87
88
  if (router_purpose == ROUTER_PURPOSE_BRIDGE && has_completed_circuit)
    return 1; /* if no circuits yet, we may need this info to bootstrap. */
89
90
  if (dir_purpose == DIR_PURPOSE_FETCH_DIR ||
      dir_purpose == DIR_PURPOSE_UPLOAD_DIR ||
91
      dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
92
      dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES ||
93
94
      dir_purpose == DIR_PURPOSE_FETCH_RUNNING_LIST ||
      dir_purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS ||
95
96
97
98
      dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
      dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES ||
      dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
      dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
99
100
      dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
      dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO)
101
102
103
104
    return 0;
  return 1;
}

105
106
/** Return a newly allocated string describing <b>auth</b>. */
char *
107
108
authority_type_to_string(authority_type_t auth)
{
109
110
111
112
113
114
115
116
117
118
119
120
121
122
  char *result;
  smartlist_t *lst = smartlist_create();
  if (auth & V1_AUTHORITY)
    smartlist_add(lst, (void*)"V1");
  if (auth & V2_AUTHORITY)
    smartlist_add(lst, (void*)"V2");
  if (auth & BRIDGE_AUTHORITY)
    smartlist_add(lst, (void*)"Bridge");
  if (auth & HIDSERV_AUTHORITY)
    smartlist_add(lst, (void*)"Hidden service");
  if (smartlist_len(lst)) {
    result = smartlist_join_strings(lst, ", ", 0, NULL);
  } else {
    result = tor_strdup("[Not an authority]");
123
  }
124
125
  smartlist_free(lst);
  return result;
126
127
}

128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
/** Return a string describing a given directory connection purpose. */
static const char *
dir_conn_purpose_to_string(int purpose)
{
  switch (purpose)
    {
    case DIR_PURPOSE_FETCH_DIR:
      return "v1 directory fetch";
    case DIR_PURPOSE_FETCH_RENDDESC:
      return "hidden-service descriptor fetch";
    case DIR_PURPOSE_UPLOAD_DIR:
      return "server descriptor upload";
    case DIR_PURPOSE_UPLOAD_RENDDESC:
      return "hidden-service descriptor upload";
    case DIR_PURPOSE_UPLOAD_VOTE:
      return "server vote upload";
    case DIR_PURPOSE_UPLOAD_SIGNATURES:
      return "consensus signature upload";
    case DIR_PURPOSE_FETCH_RUNNING_LIST:
      return "running-routers fetch";
    case DIR_PURPOSE_FETCH_NETWORKSTATUS:
      return "network-status fetch";
    case DIR_PURPOSE_FETCH_SERVERDESC:
      return "server descriptor fetch";
    case DIR_PURPOSE_FETCH_EXTRAINFO:
      return "extra-info fetch";
    case DIR_PURPOSE_FETCH_CONSENSUS:
      return "consensus network-status fetch";
    case DIR_PURPOSE_FETCH_CERTIFICATE:
      return "authority cert fetch";
    case DIR_PURPOSE_FETCH_STATUS_VOTE:
      return "status vote fetch";
    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
      return "consensus signature fetch";
162
163
164
165
    case DIR_PURPOSE_FETCH_RENDDESC_V2:
      return "hidden-service v2 descriptor fetch";
    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
      return "hidden-service v2 descriptor upload";
166
167
168
169
170
171
    }

  log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
  return "(unknown)";
}

172
173
174
/** Return true iff <b>identity_digest</b> is the digest of a router we
 * believe to support extrainfo downloads.  (If <b>is_authority</b> we do
 * additional checking that's only valid for authorities.) */
175
176
177
178
179
180
181
182
183
184
185
186
187
int
router_supports_extrainfo(const char *identity_digest, int is_authority)
{
  routerinfo_t *ri = router_get_by_digest(identity_digest);

  if (ri) {
    if (ri->caches_extra_info)
      return 1;
    if (is_authority && ri->platform &&
        tor_version_as_new_as(ri->platform, "Tor 0.2.0.0-alpha-dev (r10070)"))
      return 1;
  }
  if (is_authority) {
188
    routerstatus_t *rs = router_get_consensus_status_by_id(identity_digest);
189
    if (rs && rs->version_supports_extrainfo_upload)
190
191
192
193
194
      return 1;
  }
  return 0;
}

195
196
197
198
199
200
/** Return true iff all trusted directory servers have accepted our
 * server descriptor. */
int
directories_have_accepted_server_descriptor(void)
{
  smartlist_t *servers = router_get_trusted_dir_servers();
201
  or_options_t *options = get_options();
202
203
204
205
  /* XXX020 If any authority of the needed type is down, this
   * function will never return true. Perhaps we need to be
   * tolerant of down servers? Or even better, should we change
   * this so one successful upload is enough? -RD */
206
  SMARTLIST_FOREACH(servers, trusted_dir_server_t *, d, {
207
    if ((d->type & options->_PublishServerDescriptor) &&
208
209
210
211
212
213
214
        !d->has_accepted_serverdesc) {
      return 0;
    }
  });
  return 1;
}

215
/** Start a connection to every suitable directory authority, using
Roger Dingledine's avatar
Roger Dingledine committed
216
217
218
 * connection purpose 'purpose' and uploading the payload 'payload'
 * (length 'payload_len').  The purpose should be one of
 * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'.
219
 *
220
221
222
 * <b>type</b> specifies what sort of dir authorities (V1, V2,
 * HIDSERV, BRIDGE) we should upload to.
 *
223
224
225
226
227
 * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
 * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
 * bytes of <b>payload</b> hold an extra-info document.  Upload the descriptor
 * to all authorities, and the extra-info document to all authorities that
 * support it.
Roger Dingledine's avatar
Roger Dingledine committed
228
229
 */
void
230
231
directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
                             authority_type_t type,
232
                             const char *payload,
233
                             size_t payload_len, size_t extrainfo_len)
Roger Dingledine's avatar
Roger Dingledine committed
234
{
235
  int post_via_tor;
236
  smartlist_t *dirservers = router_get_trusted_dir_servers();
237
  int found = 0;
238
  tor_assert(dirservers);
Nick Mathewson's avatar
Nick Mathewson committed
239
240
241
  /* This tries dirservers which we believe to be down, but ultimately, that's
   * harmless, and we may as well err on the side of getting things uploaded.
   */
242
243
  SMARTLIST_FOREACH(dirservers, trusted_dir_server_t *, ds,
    {
244
      routerstatus_t *rs = &(ds->fake_status);
245
      size_t upload_len = payload_len;
246
247

      if ((type & ds->type) == 0)
248
        continue;
249

250
      found = 1; /* at least one authority of this type was listed */
251
      if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
252
        ds->has_accepted_serverdesc = 0;
253

254
      if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
255
        upload_len += extrainfo_len;
256
257
        log_info(LD_DIR, "Uploading an extrainfo (length %d)",
                 (int) extrainfo_len);
258
      }
259
      post_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose) ||
260
              !fascist_firewall_allows_address_dir(ds->addr, ds->dir_port);
261
262
      directory_initiate_command_routerstatus(rs, dir_purpose,
                                              router_purpose,
263
                                              post_via_tor,
264
                                              NULL, payload, upload_len, 0);
265
    });
266
  if (!found) {
267
    char *s = authority_type_to_string(type);
268
    log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
269
270
             "of type '%s', but no authorities of that type listed!", s);
    tor_free(s);
271
  }
Roger Dingledine's avatar
Roger Dingledine committed
272
273
}

Roger Dingledine's avatar
Roger Dingledine committed
274
/** Start a connection to a random running directory server, using
275
276
 * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
 * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
277
278
 * If <b>retry_if_no_servers</b>, then if all the possible servers seem
 * down, mark them up and try again.
Roger Dingledine's avatar
Roger Dingledine committed
279
280
 */
void
281
282
directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
                             const char *resource, int retry_if_no_servers)
Roger Dingledine's avatar
Roger Dingledine committed
283
{
284
  routerstatus_t *rs = NULL;
285
  or_options_t *options = get_options();
286
  int prefer_authority = directory_fetches_from_authorities(options);
287
  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose);
288
  authority_type_t type;
289
  int flags = retry_if_no_servers ? PDS_RETRY_IF_NO_SERVERS : 0;
290
  time_t if_modified_since = 0;
291

Roger Dingledine's avatar
Roger Dingledine committed
292
293
  /* FFFF we could break this switch into its own function, and call
   * it elsewhere in directory.c. -RD */
294
  switch (dir_purpose) {
295
    case DIR_PURPOSE_FETCH_EXTRAINFO:
296
297
298
      type = EXTRAINFO_CACHE |
             (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_AUTHORITY :
                                                        V2_AUTHORITY);
299
      break;
300
301
    case DIR_PURPOSE_FETCH_NETWORKSTATUS:
    case DIR_PURPOSE_FETCH_SERVERDESC:
302
303
      type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_AUTHORITY :
                                                        V2_AUTHORITY);
304
305
306
307
308
309
310
311
      break;
    case DIR_PURPOSE_FETCH_DIR:
    case DIR_PURPOSE_FETCH_RUNNING_LIST:
      type = V1_AUTHORITY;
      break;
    case DIR_PURPOSE_FETCH_RENDDESC:
      type = HIDSERV_AUTHORITY;
      break;
312
313
    case DIR_PURPOSE_FETCH_STATUS_VOTE:
    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
314
315
      type = V3_AUTHORITY;
      break;
316
317
318
319
    case DIR_PURPOSE_FETCH_CONSENSUS:
    case DIR_PURPOSE_FETCH_CERTIFICATE:
      type = V3_AUTHORITY;
      break;
320
    default:
321
      log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
322
323
      return;
  }
324

325
326
327
328
329
330
  if (DIR_PURPOSE_FETCH_CONSENSUS) {
    networkstatus_vote_t *v = networkstatus_get_latest_consensus();
    if (v)
      if_modified_since = v->valid_after + 180;
  }

331
  if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
332
333
    return;

334
335
336
  if (!get_via_tor) {
    if (options->UseBridges && type != BRIDGE_AUTHORITY) {
      /* want to ask a running bridge for which we have a descriptor. */
Roger Dingledine's avatar
Roger Dingledine committed
337
      /* XXX021 we assume that all of our bridges can answer any
338
       * possible directory question. This won't be true forever. -RD */
339
340
341
342
343
344
345
      routerinfo_t *ri = choose_random_entry(NULL);
      if (ri) {
        directory_initiate_command(ri->address, ri->addr,
                                   ri->or_port, 0,
                                   1, ri->cache_info.identity_digest,
                                   dir_purpose,
                                   router_purpose,
346
                                   0, resource, NULL, 0, if_modified_since);
347
348
349
350
351
352
353
      } else
        log_notice(LD_DIR, "Ignoring directory request, since no bridge "
                           "nodes are available yet.");
      return;
    } else {
      if (prefer_authority || type == BRIDGE_AUTHORITY) {
        /* only ask authdirservers, and don't ask myself */
354
        rs = router_pick_trusteddirserver(type, flags);
355
356
357
      }
      if (!rs && type != BRIDGE_AUTHORITY) {
        /* anybody with a non-zero dirport will do */
358
        rs = router_pick_directory_server(type, flags);
359
360
361
        if (!rs) {
          log_info(LD_DIR, "No router found for %s; falling back to "
                   "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
362
          rs = router_pick_trusteddirserver(type, flags);
363
364
365
          if (!rs)
            get_via_tor = 1; /* last resort: try routing it via Tor */
        }
366
      }
367
    }
368
  } else { /* get_via_tor */
369
    /* Never use fascistfirewall; we're going via Tor. */
370
    if (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) {
Roger Dingledine's avatar
Roger Dingledine committed
371
      /* only ask hidserv authorities, any of them will do */
372
373
      flags |= PDS_IGNORE_FASCISTFIREWALL|PDS_ALLOW_SELF;
      rs = router_pick_trusteddirserver(HIDSERV_AUTHORITY, flags);
374
375
    } else {
      /* anybody with a non-zero dirport will do. Disregard firewalls. */
376
377
      flags |= PDS_IGNORE_FASCISTFIREWALL;
      rs = router_pick_directory_server(type, flags);
378
      /* If we have any hope of building an indirect conn, we know some router
Roger Dingledine's avatar
Roger Dingledine committed
379
       * descriptors.  If (rs==NULL), we can't build circuits anyway, so
Roger Dingledine's avatar
Roger Dingledine committed
380
       * there's no point in falling back to the authorities in this case. */
381
    }
382
383
  }

384
  if (rs)
385
    directory_initiate_command_routerstatus(rs, dir_purpose,
386
387
                                            router_purpose,
                                            get_via_tor,
388
389
                                            resource, NULL, 0,
                                            if_modified_since);
390
  else {
391
    log_notice(LD_DIR,
392
393
               "While fetching directory info, "
               "no running dirservers known. Will try again later. "
394
               "(purpose %d)", dir_purpose);
395
    if (!purpose_needs_anonymity(dir_purpose, router_purpose)) {
396
      /* remember we tried them all and failed. */
397
      directory_all_unreachable(time(NULL));
398
    }
399
  }
Roger Dingledine's avatar
Roger Dingledine committed
400
401
}

402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
/** As directory_get_from_dirserver, but initiates a request to <i>every</i>
 * directory authority other than ourself.  Only for use by authorities when
 * searching for missing information while voting. */
void
directory_get_from_all_authorities(uint8_t dir_purpose,
                                   uint8_t router_purpose,
                                   const char *resource)
{
  tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
             dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);

  SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
                    trusted_dir_server_t *, ds,
    {
      routerstatus_t *rs;
      if (router_digest_is_me(ds->digest))
        continue;
      if (!(ds->type & V3_AUTHORITY))
        continue;
421
      rs = &ds->fake_status;
422
      directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose,
423
                                              0, resource, NULL, 0, 0);
424
425
426
    });
}

427
428
429
/** Launch a new connection to the directory server <b>status</b> to
 * upload or download a server or rendezvous
 * descriptor. <b>dir_purpose</b> determines what
430
 * kind of directory connection we're launching, and must be one of
431
 * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC|RENDDESC_V2}. <b>router_purpose</b>
432
433
 * specifies the descriptor purposes we have in mind (currently only
 * used for FETCH_DIR).
434
435
436
437
438
439
440
441
442
 *
 * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
 * of the HTTP post.  Otherwise, <b>payload</b> should be NULL.
 *
 * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
 * want to fetch.
 */
void
directory_initiate_command_routerstatus(routerstatus_t *status,
443
444
                                        uint8_t dir_purpose,
                                        uint8_t router_purpose,
445
                                        int anonymized_connection,
446
447
                                        const char *resource,
                                        const char *payload,
448
449
                                        size_t payload_len,
                                        time_t if_modified_since)
450
{
451
  routerinfo_t *router;
452
  char address_buf[INET_NTOA_BUF_LEN+1];
453
454
455
456
457
458
459
460
461
  struct in_addr in;
  const char *address;
  if ((router = router_get_by_digest(status->identity_digest))) {
    address = router->address;
  } else {
    in.s_addr = htonl(status->addr);
    tor_inet_ntoa(&in, address_buf, sizeof(address_buf));
    address = address_buf;
  }
462
463
  directory_initiate_command(address, status->addr,
                             status->or_port, status->dir_port,
464
465
                             status->version_supports_begindir,
                             status->identity_digest,
466
467
                             dir_purpose, router_purpose,
                             anonymized_connection, resource,
468
                             payload, payload_len, if_modified_since);
469
470
}

471
472
473
/** Return true iff <b>conn</b> is the client side of a directory connection
 * we launched to ourself in order to determine the reachability of our
 * dir_port. */
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
static int
directory_conn_is_self_reachability_test(dir_connection_t *conn)
{
  if (conn->requested_resource &&
      !strcmpstart(conn->requested_resource,"authority")) {
    routerinfo_t *me = router_get_my_routerinfo();
    if (me &&
        router_digest_is_me(conn->identity_digest) &&
        me->addr == conn->_base.addr &&
        me->dir_port == conn->_base.port)
      return 1;
  }
  return 0;
}

489
490
491
/** Called when we are unable to complete the client's request to a directory
 * server due to a network error: Mark the router as down and try again if
 * possible.
492
 */
493
void
494
connection_dir_request_failed(dir_connection_t *conn)
495
{
496
497
498
499
500
501
  if (directory_conn_is_self_reachability_test(conn)) {
    routerinfo_t *me = router_get_my_routerinfo();
    if (me)
      control_event_server_status(LOG_WARN,
                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
                                  me->address, me->dir_port);
502
    return; /* this was a test fetch. don't retry. */
503
  }
504
505
  if (entry_list_can_grow(get_options()))
    router_set_status(conn->identity_digest, 0); /* don't try him again */
506
507
  if (conn->_base.purpose == DIR_PURPOSE_FETCH_DIR ||
      conn->_base.purpose == DIR_PURPOSE_FETCH_RUNNING_LIST) {
508
    log_info(LD_DIR, "Giving up on directory server at '%s:%d'; retrying",
509
             conn->_base.address, conn->_base.port);
510
511
    directory_get_from_dirserver(conn->_base.purpose, conn->router_purpose,
                                 NULL, 0 /* don't retry_if_no_servers */);
512
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS) {
513
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
514
             conn->_base.address);
515
    connection_dir_download_networkstatus_failed(conn, -1);
516
517
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
518
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
519
             conn->_base.address);
520
    connection_dir_download_routerdesc_failed(conn);
521
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
522
    networkstatus_consensus_download_failed(0);
523
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
524
525
526
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
             conn->_base.address);
    connection_dir_download_cert_failed(conn, 0);
527
528
529
530
531
532
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
    log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
             conn->_base.address);
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
    log_info(LD_DIR, "Giving up downloading votes from '%s'",
             conn->_base.address);
533
534
535
  }
}

536
537
538
/** Called when an attempt to download one or more network status
 * documents on connection <b>conn</b> failed. Decide whether to
 * retry the fetch now, later, or never.
539
 */
540
static void
541
connection_dir_download_networkstatus_failed(dir_connection_t *conn,
542
                                             int status_code)
543
{
544
545
546
547
548
549
  if (!conn->requested_resource) {
    /* We never reached directory_send_command, which means that we never
     * opened a network connection.  Either we're out of sockets, or the
     * network is down.  Either way, retrying would be pointless. */
    return;
  }
550
  if (!strcmpstart(conn->requested_resource, "all")) {
551
552
553
    /* We're a non-authoritative directory cache; try again. Ignore status
     * code, since we don't want to keep trying forever in a tight loop
     * if all the authorities are shutting us out. */
554
555
    smartlist_t *trusted_dirs = router_get_trusted_dir_servers();
    SMARTLIST_FOREACH(trusted_dirs, trusted_dir_server_t *, ds,
556
                      download_status_failed(&ds->v2_ns_dl_status, 0));
557
558
    directory_get_from_dirserver(conn->_base.purpose, conn->router_purpose,
                                 "all.z", 0 /* don't retry_if_no_servers */);
559
  } else if (!strcmpstart(conn->requested_resource, "fp/")) {
Roger Dingledine's avatar
Roger Dingledine committed
560
    /* We were trying to download by fingerprint; mark them all as having
561
562
     * failed, and possibly retry them later.*/
    smartlist_t *failed = smartlist_create();
563
    dir_split_resource_into_fingerprints(conn->requested_resource+3,
564
                                         failed, NULL, 0, 0);
565
    if (smartlist_len(failed)) {
566
      dir_networkstatus_download_failed(failed, status_code);
567
      SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp));
568
    }
569
570
    smartlist_free(failed);
  }
571
572
}

573
/** Called when an attempt to download one or more router descriptors
574
 * or extra-info documents on connection <b>conn</b> failed.
575
576
 */
static void
577
connection_dir_download_routerdesc_failed(dir_connection_t *conn)
578
{
579
  /* No need to increment the failure count for routerdescs, since
580
   * it's not their fault. */
581

582
583
  /* No need to relaunch descriptor downloads here: we already do it
   * every 10 seconds (DESCRIPTOR_RETRY_INTERVAL) in main.c. */
584
585
  tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
586

587
  (void) conn;
588
589
}

590
591
592
593
594
595
596
597
598
599
/** Called when an attempt to fetch a certificate fails. */
static void
connection_dir_download_cert_failed(dir_connection_t *conn, int status)
{
  smartlist_t *failed;
  tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);

  if (!conn->requested_resource)
    return;
  failed = smartlist_create();
600
  dir_split_resource_into_fingerprints(conn->requested_resource+3,
601
602
603
604
605
606
607
608
609
                                       failed, NULL, 1, 0);
  SMARTLIST_FOREACH(failed, char *, cp,
  {
    trusted_dir_server_t *dir = trusteddirserver_get_by_v3_auth_digest(cp);
    if (dir)
      download_status_failed(&dir->cert_dl_status, status);
    tor_free(cp);
  });
  smartlist_free(failed);
610
611

  update_certificate_downloads(time(NULL));
612
613
}

614
/** Helper for directory_initiate_command_routerstatus: send the
615
 * command to a server whose address is <b>address</b>, whose IP is
616
617
618
 * <b>addr</b>, whose directory port is <b>dir_port</b>, whose tor version
 * <b>supports_begindir</b>, and whose identity key digest is
 * <b>digest</b>. */
619
void
620
directory_initiate_command(const char *address, uint32_t addr,
621
                           uint16_t or_port, uint16_t dir_port,
622
623
                           int supports_begindir, const char *digest,
                           uint8_t dir_purpose, uint8_t router_purpose,
624
                           int anonymized_connection, const char *resource,
625
626
                           const char *payload, size_t payload_len,
                           time_t if_modified_since)
Roger Dingledine's avatar
Roger Dingledine committed
627
{
628
  dir_connection_t *conn;
629
  or_options_t *options = get_options();
630
631
632
  int use_begindir = supports_begindir && or_port &&
                     (options->TunnelDirConns ||
                      router_purpose == ROUTER_PURPOSE_BRIDGE) &&
633
634
                     (anonymized_connection ||
                      fascist_firewall_allows_address_or(addr, or_port));
635

636
637
  tor_assert(address);
  tor_assert(addr);
638
  tor_assert(or_port || dir_port);
639
  tor_assert(digest);
640

641
642
  log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
            anonymized_connection, use_begindir);
643

644
  log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
645

646
  conn = TO_DIR_CONN(connection_new(CONN_TYPE_DIR, AF_INET));
647
648

  /* set up conn so it's got all the data we need to remember */
649
  conn->_base.addr = addr;
650
  conn->_base.port = use_begindir ? or_port : dir_port;
651
  conn->_base.address = tor_strdup(address);
652
  memcpy(conn->identity_digest, digest, DIGEST_LEN);
653

654
655
  conn->_base.purpose = dir_purpose;
  conn->router_purpose = router_purpose;
656

657
  /* give it an initial state */
658
  conn->_base.state = DIR_CONN_STATE_CONNECTING;
659

660
661
662
663
664
  /* decide whether we can learn our IP address from this conn */
  conn->dirconn_direct = !anonymized_connection;

  if (!anonymized_connection && !use_begindir) {
    /* then we want to connect to dirport directly */
665

666
667
668
    if (options->HttpProxy) {
      addr = options->HttpProxyAddr;
      dir_port = options->HttpProxyPort;
669
670
    }

671
672
    switch (connection_connect(TO_CONN(conn), conn->_base.address, addr,
                               dir_port)) {
Roger Dingledine's avatar
Roger Dingledine committed
673
      case -1:
674
        connection_dir_request_failed(conn); /* retry if we want */
675
676
        /* XXX we only pass 'conn' above, not 'resource', 'payload',
         * etc. So in many situations it can't retry! -RD */
677
        connection_free(TO_CONN(conn));
Roger Dingledine's avatar
Roger Dingledine committed
678
679
        return;
      case 1:
680
681
        /* start flushing conn */
        conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
Roger Dingledine's avatar
Roger Dingledine committed
682
683
        /* fall through */
      case 0:
684
        /* queue the command on the outbuf */
685
        directory_send_command(conn, dir_purpose, 1, resource,
686
                               payload, payload_len, if_modified_since);
687
        connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE);
Roger Dingledine's avatar
Roger Dingledine committed
688
689
690
        /* writable indicates finish, readable indicates broken link,
           error indicates broken link in windowsland. */
    }
691
  } else { /* we want to connect via a tor connection */
692
    edge_connection_t *linked_conn;
Roger Dingledine's avatar
Roger Dingledine committed
693
    /* make an AP connection
694
     * populate it and add it at the right state
695
     * hook up both sides
Roger Dingledine's avatar
Roger Dingledine committed
696
     */
697
    linked_conn =
698
      connection_ap_make_link(conn->_base.address, conn->_base.port,
699
                              digest, use_begindir, conn->dirconn_direct);
700
    if (!linked_conn) {
701
      log_warn(LD_NET,"Making tunnel to dirserver failed.");
702
      connection_mark_for_close(TO_CONN(conn));
703
704
      return;
    }
705
    connection_link_connections(TO_CONN(conn), TO_CONN(linked_conn));
Roger Dingledine's avatar
Roger Dingledine committed
706

707
    if (connection_add(TO_CONN(conn)) < 0) {
708
      log_warn(LD_NET,"Unable to add connection for link to dirserver.");
709
      connection_mark_for_close(TO_CONN(conn));
710
711
      return;
    }
712
    conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
713
    /* queue the command on the outbuf */
714
    directory_send_command(conn, dir_purpose, 0, resource,
715
                           payload, payload_len, if_modified_since);
716
    connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE);
717
    connection_start_reading(TO_CONN(linked_conn));
718
719
720
  }
}

721
722
/** Queue an appropriate HTTP command on conn-\>outbuf.  The other args
 * are as in directory_initiate_command.
723
 */
724
static void
725
directory_send_command(dir_connection_t *conn,
726
                       int purpose, int direct, const char *resource,
727
728
                       const char *payload, size_t payload_len,
                       time_t if_modified_since)
729
{
730
731
  char proxystring[256];
  char proxyauthstring[256];
732
  char hoststring[128];
733
  char imsstring[RFC1123_TIME_LEN+32];
734
735
  char *url;
  char request[8192];
736
  const char *httpcommand = NULL;
737
  size_t len;
738

739
  tor_assert(conn);
740
  tor_assert(conn->_base.type == CONN_TYPE_DIR);
741

742
  tor_free(conn->requested_resource);
743
744
  if (resource)
    conn->requested_resource = tor_strdup(resource);
745

746
  /* come up with a string for which Host: we want */
747
748
  if (conn->_base.port == 80) {
    strlcpy(hoststring, conn->_base.address, sizeof(hoststring));
749
  } else {
750
    tor_snprintf(hoststring, sizeof(hoststring),"%s:%d",
751
                 conn->_base.address, conn->_base.port);
752
  }
753

754
755
756
757
758
759
760
761
762
  /* Format if-modified-since */
  if (!if_modified_since) {
    imsstring[0] = '\0';
  } else {
    char b[RFC1123_TIME_LEN+1];
    format_rfc1123_time(b, if_modified_since);
    tor_snprintf(imsstring, sizeof(imsstring), "\r\nIf-Modified-Since: %s", b);
  }

763
  /* come up with some proxy lines, if we're using one. */
764
  if (direct && get_options()->HttpProxy) {
765
766
767
    char *base64_authenticator=NULL;
    const char *authenticator = get_options()->HttpProxyAuthenticator;

768
    tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
769
770
771
    if (authenticator) {
      base64_authenticator = alloc_http_authenticator(authenticator);
      if (!base64_authenticator)
772
        log_warn(LD_BUG, "Encoding http authenticator failed");
773
774
775
776
777
778
779
780
781
    }
    if (base64_authenticator) {
      tor_snprintf(proxyauthstring, sizeof(proxyauthstring),
                   "\r\nProxy-Authorization: Basic %s",
                   base64_authenticator);
      tor_free(base64_authenticator);
    } else {
      proxyauthstring[0] = 0;
    }
782
783
  } else {
    proxystring[0] = 0;
784
    proxyauthstring[0] = 0;
785
786
  }

787
  switch (purpose) {
Roger Dingledine's avatar
Roger Dingledine committed
788
    case DIR_PURPOSE_FETCH_DIR:
789
790
      tor_assert(!resource);
      tor_assert(!payload);
791
      httpcommand = "GET";
792
      url = tor_strdup("/tor/dir.z");
793
794
      break;
    case DIR_PURPOSE_FETCH_RUNNING_LIST:
795
796
      tor_assert(!resource);
      tor_assert(!payload);
797
      httpcommand = "GET";
798
799
800
      url = tor_strdup("/tor/running-routers");
      break;
    case DIR_PURPOSE_FETCH_NETWORKSTATUS:
801
      httpcommand = "GET";
802
803
804
805
      len = strlen(resource)+32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/status/%s", resource);
      break;
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
    case DIR_PURPOSE_FETCH_CONSENSUS:
      tor_assert(!resource);
      tor_assert(!payload);
      httpcommand = "GET";
      url = tor_strdup("/tor/status-vote/current/consensus.z");
      break;
    case DIR_PURPOSE_FETCH_CERTIFICATE:
      tor_assert(resource);
      tor_assert(!payload);
      httpcommand = "GET";
      len = strlen(resource)+32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/keys/%s", resource);
      break;
    case DIR_PURPOSE_FETCH_STATUS_VOTE:
      tor_assert(resource);
      tor_assert(!payload);
      httpcommand = "GET";
      len = strlen(resource)+32;
      url = tor_malloc(len);
826
      tor_snprintf(url, len, "/tor/status-vote/next/%s.z", resource);
827
828
829
830
831
832
833
      break;
    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
      tor_assert(!resource);
      tor_assert(!payload);
      httpcommand = "GET";
      url = tor_strdup("/tor/status-vote/next/consensus-signatures.z");
      break;
834
    case DIR_PURPOSE_FETCH_SERVERDESC:
835
      httpcommand = "GET";
836
837
838
      len = strlen(resource)+32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/server/%s", resource);
839
      break;
840
841
842
843
844
845
    case DIR_PURPOSE_FETCH_EXTRAINFO:
      httpcommand = "GET";
      len = strlen(resource)+32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/extra/%s", resource);
      break;
Roger Dingledine's avatar
Roger Dingledine committed
846
    case DIR_PURPOSE_UPLOAD_DIR:
847
      tor_assert(!resource);
Roger Dingledine's avatar
Roger Dingledine committed
848
      tor_assert(payload);
849
      httpcommand = "POST";
850
      url = tor_strdup("/tor/");
Roger Dingledine's avatar
Roger Dingledine committed
851
      break;
852
853
854
855
856
857
    case DIR_PURPOSE_UPLOAD_VOTE:
      tor_assert(!resource);
      tor_assert(payload);
      httpcommand = "POST";
      url = tor_strdup("/tor/post/vote");
      break;
858
859
860
861
    case DIR_PURPOSE_UPLOAD_SIGNATURES:
      tor_assert(!resource);
      tor_assert(payload);
      httpcommand = "POST";
862
      url = tor_strdup("/tor/post/consensus-signature");
863
      break;
864
    case DIR_PURPOSE_FETCH_RENDDESC:
865
866
      tor_assert(resource);
      tor_assert(!payload);
867
868

      /* this must be true or we wouldn't be doing the lookup */
869
      tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32);
870
      /* This breaks the function abstraction. */
871
      strlcpy(conn->rend_query, resource, sizeof(conn->rend_query));
872

873
      httpcommand = "GET";
Nick Mathewson's avatar
Nick Mathewson committed
874
      /* Request the most recent versioned descriptor. */
875
876
      // (XXXX We were going to switch this to fetch rendezvous1 descriptors,
      // but that never got testing, and it wasn't a good design.)
877
878
879
      len = strlen(resource)+32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/rendezvous/%s", resource);
Roger Dingledine's avatar
Roger Dingledine committed
880
      break;
881
882
883
    case DIR_PURPOSE_FETCH_RENDDESC_V2:
      tor_assert(resource);
      tor_assert(!payload);
884
      tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32);
885
886
887
888
889
      httpcommand = "GET";
      len = strlen(resource) + 32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/rendezvous2/%s", resource);
      break;
890
    case DIR_PURPOSE_UPLOAD_RENDDESC:
891
      tor_assert(!resource);
Roger Dingledine's avatar
Roger Dingledine committed
892
      tor_assert(payload);
893
      httpcommand = "POST";
894
      url = tor_strdup("/tor/rendezvous/publish");
895
      break;
896
897
898
899
900
901
    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
      tor_assert(!resource);
      tor_assert(payload);
      httpcommand = "POST";
      url = tor_strdup("/tor/rendezvous2/publish");
      break;
902
903
904
    default:
      tor_assert(0);
      return;
905
  }
906
907

  if (strlen(proxystring) + strlen(url) >= 4096) {
908
    log_warn(LD_BUG,
Roger Dingledine's avatar
Roger Dingledine committed
909
             "Squid does not like URLs longer than 4095 bytes, and this "
910
911
             "one is %d bytes long: %s%s",
             (int)(strlen(proxystring) + strlen(url)), proxystring, url);
912
913
  }

914
  tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
915
916
  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
  connection_write_to_buf(url, strlen(url), TO_CONN(conn));
917
  tor_free(url);
918

919
920
  if (!strcmp(httpcommand, "GET") && !payload) {
    tor_snprintf(request, sizeof(request),
921
                 " HTTP/1.0\r\nHost: %s%s%s\r\n\r\n",
922
                 hoststring,