directory.c 132 KB
Newer Older
1
2
/* Copyright (c) 2001-2004, Roger Dingledine.
 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
Karsten Loesing's avatar
Karsten Loesing committed
3
 * Copyright (c) 2007-2009, The Tor Project, Inc. */
Roger Dingledine's avatar
Roger Dingledine committed
4
/* See LICENSE for licensing information */
5
6

#include "or.h"
7
#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
8
#ifndef OPENBSD
9
10
#include <malloc.h>
#endif
11
#endif
12

Roger Dingledine's avatar
Roger Dingledine committed
13
14
/**
 * \file directory.c
15
16
17
 * \brief Code to send and fetch directories and router
 * descriptors via HTTP.  Directories use dirserv.c to generate the
 * results; clients use routers.c to parse them.
Roger Dingledine's avatar
Roger Dingledine committed
18
 **/
19

Roger Dingledine's avatar
Roger Dingledine committed
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
/* In-points to directory.c:
 *
 * - directory_post_to_dirservers(), called from
 *   router_upload_dir_desc_to_dirservers() in router.c
 *   upload_service_descriptor() in rendservice.c
 * - directory_get_from_dirserver(), called from
 *   rend_client_refetch_renddesc() in rendclient.c
 *   run_scheduled_events() in main.c
 *   do_hup() in main.c
 * - connection_dir_process_inbuf(), called from
 *   connection_process_inbuf() in connection.c
 * - connection_dir_finished_flushing(), called from
 *   connection_finished_flushing() in connection.c
 * - connection_dir_finished_connecting(), called from
 *   connection_finished_connecting() in connection.c
 */
36
static void directory_send_command(dir_connection_t *conn,
37
38
                             int purpose, int direct, const char *resource,
                             const char *payload, size_t payload_len,
39
                             int supports_conditional_consensus,
40
                             time_t if_modified_since);
41
static int directory_handle_command(dir_connection_t *conn);
42
static int body_is_plausible(const char *body, size_t body_len, int purpose);
43
44
static int purpose_needs_anonymity(uint8_t dir_purpose,
                                   uint8_t router_purpose);
45
static char *http_get_header(const char *headers, const char *which);
46
static void http_set_address_origin(const char *headers, connection_t *conn);
47
static void connection_dir_download_networkstatus_failed(
48
                               dir_connection_t *conn, int status_code);
49
static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
50
51
static void connection_dir_download_cert_failed(
                               dir_connection_t *conn, int status_code);
52
53
static void dir_networkstatus_download_failed(smartlist_t *failed,
                                              int status_code);
54
static void dir_routerdesc_download_failed(smartlist_t *failed,
55
                                           int status_code,
56
                                           int router_purpose,
57
58
                                           int was_extrainfo,
                                           int was_descriptor_digests);
59
static void note_client_request(int purpose, int compressed, size_t bytes);
60
static int client_likes_consensus(networkstatus_t *v, const char *want_url);
Roger Dingledine's avatar
Roger Dingledine committed
61

62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
static void directory_initiate_command_rend(const char *address,
                                            const tor_addr_t *addr,
                                            uint16_t or_port,
                                            uint16_t dir_port,
                                            int supports_conditional_consensus,
                                            int supports_begindir,
                                            const char *digest,
                                            uint8_t dir_purpose,
                                            uint8_t router_purpose,
                                            int anonymized_connection,
                                            const char *resource,
                                            const char *payload,
                                            size_t payload_len,
                                            time_t if_modified_since,
                                            const rend_data_t *rend_query);

78
79
/********* START VARIABLES **********/

80
81
82
/** How far in the future do we allow a directory server to tell us it is
 * before deciding that one of us has the wrong time? */
#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
83

84
85
#define X_ADDRESS_HEADER "X-Your-Address-Is: "

86
87
/** HTTP cache control: how long do we tell proxies they can cache each
 * kind of document we serve? */
88
89
#define FULL_DIR_CACHE_LIFETIME (60*60)
#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
90
#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
91
92
#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
#define ROUTERDESC_CACHE_LIFETIME (30*60)
93
#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
94
95
#define ROBOTS_CACHE_LIFETIME (24*60*60)

96
97
/********* END VARIABLES ************/

98
99
/** Return true iff the directory purpose 'purpose' must use an
 * anonymous connection to a directory. */
100
static int
101
purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
102
{
103
104
  if (get_options()->AllDirActionsPrivate)
    return 1;
105
106
  if (router_purpose == ROUTER_PURPOSE_BRIDGE && has_completed_circuit)
    return 1; /* if no circuits yet, we may need this info to bootstrap. */
107
  if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR ||
108
      dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
109
      dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES ||
110
      dir_purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS ||
111
112
113
114
      dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
      dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES ||
      dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS ||
      dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE ||
115
116
      dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
      dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO)
117
118
119
120
    return 0;
  return 1;
}

121
122
/** Return a newly allocated string describing <b>auth</b>. */
char *
123
124
authority_type_to_string(authority_type_t auth)
{
125
126
127
128
129
130
131
132
133
134
135
136
137
138
  char *result;
  smartlist_t *lst = smartlist_create();
  if (auth & V1_AUTHORITY)
    smartlist_add(lst, (void*)"V1");
  if (auth & V2_AUTHORITY)
    smartlist_add(lst, (void*)"V2");
  if (auth & BRIDGE_AUTHORITY)
    smartlist_add(lst, (void*)"Bridge");
  if (auth & HIDSERV_AUTHORITY)
    smartlist_add(lst, (void*)"Hidden service");
  if (smartlist_len(lst)) {
    result = smartlist_join_strings(lst, ", ", 0, NULL);
  } else {
    result = tor_strdup("[Not an authority]");
139
  }
140
141
  smartlist_free(lst);
  return result;
142
143
}

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
/** Return a string describing a given directory connection purpose. */
static const char *
dir_conn_purpose_to_string(int purpose)
{
  switch (purpose)
    {
    case DIR_PURPOSE_FETCH_RENDDESC:
      return "hidden-service descriptor fetch";
    case DIR_PURPOSE_UPLOAD_DIR:
      return "server descriptor upload";
    case DIR_PURPOSE_UPLOAD_RENDDESC:
      return "hidden-service descriptor upload";
    case DIR_PURPOSE_UPLOAD_VOTE:
      return "server vote upload";
    case DIR_PURPOSE_UPLOAD_SIGNATURES:
      return "consensus signature upload";
160
    case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS:
161
162
163
164
165
166
167
168
169
170
171
172
173
      return "network-status fetch";
    case DIR_PURPOSE_FETCH_SERVERDESC:
      return "server descriptor fetch";
    case DIR_PURPOSE_FETCH_EXTRAINFO:
      return "extra-info fetch";
    case DIR_PURPOSE_FETCH_CONSENSUS:
      return "consensus network-status fetch";
    case DIR_PURPOSE_FETCH_CERTIFICATE:
      return "authority cert fetch";
    case DIR_PURPOSE_FETCH_STATUS_VOTE:
      return "status vote fetch";
    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
      return "consensus signature fetch";
174
175
176
177
    case DIR_PURPOSE_FETCH_RENDDESC_V2:
      return "hidden-service v2 descriptor fetch";
    case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
      return "hidden-service v2 descriptor upload";
178
179
180
181
182
183
    }

  log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
  return "(unknown)";
}

184
185
186
/** Return true iff <b>identity_digest</b> is the digest of a router we
 * believe to support extrainfo downloads.  (If <b>is_authority</b> we do
 * additional checking that's only valid for authorities.) */
187
188
189
190
191
192
193
194
195
196
197
198
199
int
router_supports_extrainfo(const char *identity_digest, int is_authority)
{
  routerinfo_t *ri = router_get_by_digest(identity_digest);

  if (ri) {
    if (ri->caches_extra_info)
      return 1;
    if (is_authority && ri->platform &&
        tor_version_as_new_as(ri->platform, "Tor 0.2.0.0-alpha-dev (r10070)"))
      return 1;
  }
  if (is_authority) {
200
    routerstatus_t *rs = router_get_consensus_status_by_id(identity_digest);
201
    if (rs && rs->version_supports_extrainfo_upload)
202
203
204
205
206
      return 1;
  }
  return 0;
}

207
208
209
210
211
212
213
214
/** Return true iff any trusted directory authority has accepted our
 * server descriptor.
 *
 * We consider any authority sufficient because waiting for all of
 * them means it never happens while any authority is down; we don't
 * go for something more complex in the middle (like \>1/3 or \>1/2 or
 * \>=1/2) because that doesn't seem necessary yet.
 */
215
216
217
218
int
directories_have_accepted_server_descriptor(void)
{
  smartlist_t *servers = router_get_trusted_dir_servers();
219
  or_options_t *options = get_options();
220
  SMARTLIST_FOREACH(servers, trusted_dir_server_t *, d, {
221
    if ((d->type & options->_PublishServerDescriptor) &&
222
223
        d->has_accepted_serverdesc) {
      return 1;
224
225
    }
  });
226
  return 0;
227
228
}

229
/** Start a connection to every suitable directory authority, using
Roger Dingledine's avatar
Roger Dingledine committed
230
231
232
 * connection purpose 'purpose' and uploading the payload 'payload'
 * (length 'payload_len').  The purpose should be one of
 * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'.
233
 *
234
235
236
 * <b>type</b> specifies what sort of dir authorities (V1, V2,
 * HIDSERV, BRIDGE) we should upload to.
 *
237
238
239
240
241
 * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
 * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
 * bytes of <b>payload</b> hold an extra-info document.  Upload the descriptor
 * to all authorities, and the extra-info document to all authorities that
 * support it.
Roger Dingledine's avatar
Roger Dingledine committed
242
243
 */
void
244
245
directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
                             authority_type_t type,
246
                             const char *payload,
247
                             size_t payload_len, size_t extrainfo_len)
Roger Dingledine's avatar
Roger Dingledine committed
248
{
249
  int post_via_tor;
250
  smartlist_t *dirservers = router_get_trusted_dir_servers();
251
  int found = 0;
252
  tor_assert(dirservers);
Nick Mathewson's avatar
Nick Mathewson committed
253
254
255
  /* This tries dirservers which we believe to be down, but ultimately, that's
   * harmless, and we may as well err on the side of getting things uploaded.
   */
256
  SMARTLIST_FOREACH_BEGIN(dirservers, trusted_dir_server_t *, ds) {
257
      routerstatus_t *rs = &(ds->fake_status);
258
      size_t upload_len = payload_len;
259
      tor_addr_t ds_addr;
260
261

      if ((type & ds->type) == 0)
262
        continue;
263

264
      found = 1; /* at least one authority of this type was listed */
265
      if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
266
        ds->has_accepted_serverdesc = 0;
267

268
      if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
269
        upload_len += extrainfo_len;
270
        log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
271
                 (int) extrainfo_len);
272
      }
273
      tor_addr_from_ipv4h(&ds_addr, ds->addr);
274
      post_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose) ||
275
        !fascist_firewall_allows_address_dir(&ds_addr, ds->dir_port);
276
277
      directory_initiate_command_routerstatus(rs, dir_purpose,
                                              router_purpose,
278
                                              post_via_tor,
279
                                              NULL, payload, upload_len, 0);
280
  } SMARTLIST_FOREACH_END(ds);
281
  if (!found) {
282
    char *s = authority_type_to_string(type);
283
    log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
284
285
             "of type '%s', but no authorities of that type listed!", s);
    tor_free(s);
286
  }
Roger Dingledine's avatar
Roger Dingledine committed
287
288
}

Roger Dingledine's avatar
Roger Dingledine committed
289
/** Start a connection to a random running directory server, using
290
291
 * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
 * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
Roger Dingledine's avatar
Roger Dingledine committed
292
293
 * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
 * or router_pick_trusteddirserver().
Roger Dingledine's avatar
Roger Dingledine committed
294
295
 */
void
296
directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
297
                             const char *resource, int pds_flags)
Roger Dingledine's avatar
Roger Dingledine committed
298
{
299
  routerstatus_t *rs = NULL;
300
  or_options_t *options = get_options();
301
  int prefer_authority = directory_fetches_from_authorities(options);
302
  int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose);
303
  authority_type_t type;
304
  time_t if_modified_since = 0;
305

Roger Dingledine's avatar
Roger Dingledine committed
306
307
  /* FFFF we could break this switch into its own function, and call
   * it elsewhere in directory.c. -RD */
308
  switch (dir_purpose) {
309
    case DIR_PURPOSE_FETCH_EXTRAINFO:
310
311
312
      type = EXTRAINFO_CACHE |
             (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_AUTHORITY :
                                                        V2_AUTHORITY);
313
      break;
314
    case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS:
315
    case DIR_PURPOSE_FETCH_SERVERDESC:
316
317
      type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_AUTHORITY :
                                                        V2_AUTHORITY);
318
319
320
321
      break;
    case DIR_PURPOSE_FETCH_RENDDESC:
      type = HIDSERV_AUTHORITY;
      break;
322
323
    case DIR_PURPOSE_FETCH_STATUS_VOTE:
    case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
324
325
      type = V3_AUTHORITY;
      break;
326
327
328
329
    case DIR_PURPOSE_FETCH_CONSENSUS:
    case DIR_PURPOSE_FETCH_CERTIFICATE:
      type = V3_AUTHORITY;
      break;
330
    default:
331
      log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
332
333
      return;
  }
334

335
  if (DIR_PURPOSE_FETCH_CONSENSUS) {
336
    networkstatus_t *v = networkstatus_get_latest_consensus();
337
338
339
340
    if (v)
      if_modified_since = v->valid_after + 180;
  }

341
  if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
342
343
    return;

344
345
346
  if (!get_via_tor) {
    if (options->UseBridges && type != BRIDGE_AUTHORITY) {
      /* want to ask a running bridge for which we have a descriptor. */
347
      /* XXX022 we assume that all of our bridges can answer any
348
       * possible directory question. This won't be true forever. -RD */
349
350
      /* It certainly is not true with conditional consensus downloading,
       * so, for now, never assume the server supports that. */
351
352
      routerinfo_t *ri = choose_random_entry(NULL);
      if (ri) {
353
354
355
        tor_addr_t addr;
        tor_addr_from_ipv4h(&addr, ri->addr);
        directory_initiate_command(ri->address, &addr,
356
                                   ri->or_port, 0,
357
                                   0, /* don't use conditional consensus url */
358
359
360
                                   1, ri->cache_info.identity_digest,
                                   dir_purpose,
                                   router_purpose,
361
                                   0, resource, NULL, 0, if_modified_since);
362
363
364
365
366
367
368
      } else
        log_notice(LD_DIR, "Ignoring directory request, since no bridge "
                           "nodes are available yet.");
      return;
    } else {
      if (prefer_authority || type == BRIDGE_AUTHORITY) {
        /* only ask authdirservers, and don't ask myself */
369
        rs = router_pick_trusteddirserver(type, pds_flags);
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
        if (rs == NULL && (pds_flags & PDS_NO_EXISTING_SERVERDESC_FETCH)) {
          /* We don't want to fetch from any authorities that we're currently
           * fetching server descriptors from, and we got no match.  Did we
           * get no match because all the authorities have connections
           * fetching server descriptors (in which case we should just
           * return,) or because all the authorities are down or on fire or
           * unreachable or something (in which case we should go on with
           * our fallback code)? */
          pds_flags &= ~PDS_NO_EXISTING_SERVERDESC_FETCH;
          rs = router_pick_trusteddirserver(type, pds_flags);
          if (rs) {
            log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
                      "are in use.");
            return;
          }
        }
386
387
388
      }
      if (!rs && type != BRIDGE_AUTHORITY) {
        /* anybody with a non-zero dirport will do */
389
        rs = router_pick_directory_server(type, pds_flags);
390
391
392
        if (!rs) {
          log_info(LD_DIR, "No router found for %s; falling back to "
                   "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
393
          rs = router_pick_trusteddirserver(type, pds_flags);
394
395
396
          if (!rs)
            get_via_tor = 1; /* last resort: try routing it via Tor */
        }
397
      }
398
    }
399
  } else { /* get_via_tor */
400
    /* Never use fascistfirewall; we're going via Tor. */
401
    if (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) {
Roger Dingledine's avatar
Roger Dingledine committed
402
      /* only ask hidserv authorities, any of them will do */
403
404
      pds_flags |= PDS_IGNORE_FASCISTFIREWALL|PDS_ALLOW_SELF;
      rs = router_pick_trusteddirserver(HIDSERV_AUTHORITY, pds_flags);
405
406
    } else {
      /* anybody with a non-zero dirport will do. Disregard firewalls. */
407
408
      pds_flags |= PDS_IGNORE_FASCISTFIREWALL;
      rs = router_pick_directory_server(type, pds_flags);
409
      /* If we have any hope of building an indirect conn, we know some router
Roger Dingledine's avatar
Roger Dingledine committed
410
       * descriptors.  If (rs==NULL), we can't build circuits anyway, so
Roger Dingledine's avatar
Roger Dingledine committed
411
       * there's no point in falling back to the authorities in this case. */
412
    }
413
414
  }

415
  if (rs)
416
    directory_initiate_command_routerstatus(rs, dir_purpose,
417
418
                                            router_purpose,
                                            get_via_tor,
419
420
                                            resource, NULL, 0,
                                            if_modified_since);
421
  else {
422
    log_notice(LD_DIR,
423
424
               "While fetching directory info, "
               "no running dirservers known. Will try again later. "
425
               "(purpose %d)", dir_purpose);
426
    if (!purpose_needs_anonymity(dir_purpose, router_purpose)) {
427
      /* remember we tried them all and failed. */
428
      directory_all_unreachable(time(NULL));
429
    }
430
  }
Roger Dingledine's avatar
Roger Dingledine committed
431
432
}

433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
/** As directory_get_from_dirserver, but initiates a request to <i>every</i>
 * directory authority other than ourself.  Only for use by authorities when
 * searching for missing information while voting. */
void
directory_get_from_all_authorities(uint8_t dir_purpose,
                                   uint8_t router_purpose,
                                   const char *resource)
{
  tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
             dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);

  SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
                    trusted_dir_server_t *, ds,
    {
      routerstatus_t *rs;
      if (router_digest_is_me(ds->digest))
        continue;
      if (!(ds->type & V3_AUTHORITY))
        continue;
452
      rs = &ds->fake_status;
453
      directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose,
454
                                              0, resource, NULL, 0, 0);
455
456
457
    });
}

458
459
/** Same as directory_initiate_command_routerstatus(), but accepts
 * rendezvous data to fetch a hidden service descriptor. */
460
void
461
462
463
464
465
466
467
468
469
directory_initiate_command_routerstatus_rend(routerstatus_t *status,
                                             uint8_t dir_purpose,
                                             uint8_t router_purpose,
                                             int anonymized_connection,
                                             const char *resource,
                                             const char *payload,
                                             size_t payload_len,
                                             time_t if_modified_since,
                                             const rend_data_t *rend_query)
470
{
471
  routerinfo_t *router;
472
  char address_buf[INET_NTOA_BUF_LEN+1];
473
474
  struct in_addr in;
  const char *address;
475
  tor_addr_t addr;
476
477
  router = router_get_by_digest(status->identity_digest);
  if (!router && anonymized_connection) {
Roger Dingledine's avatar
Roger Dingledine committed
478
479
    log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
                     "don't have its router descriptor.", status->nickname);
480
481
    return;
  } else if (router) {
482
483
484
485
486
487
    address = router->address;
  } else {
    in.s_addr = htonl(status->addr);
    tor_inet_ntoa(&in, address_buf, sizeof(address_buf));
    address = address_buf;
  }
488
  tor_addr_from_ipv4h(&addr, status->addr);
489
  directory_initiate_command_rend(address, &addr,
490
                             status->or_port, status->dir_port,
491
                             status->version_supports_conditional_consensus,
492
493
                             status->version_supports_begindir,
                             status->identity_digest,
494
495
                             dir_purpose, router_purpose,
                             anonymized_connection, resource,
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
                             payload, payload_len, if_modified_since,
                             rend_query);
}

/** Launch a new connection to the directory server <b>status</b> to
 * upload or download a server or rendezvous
 * descriptor. <b>dir_purpose</b> determines what
 * kind of directory connection we're launching, and must be one of
 * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC|RENDDESC_V2}. <b>router_purpose</b>
 * specifies the descriptor purposes we have in mind (currently only
 * used for FETCH_DIR).
 *
 * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
 * of the HTTP post.  Otherwise, <b>payload</b> should be NULL.
 *
 * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
 * want to fetch.
 */
void
directory_initiate_command_routerstatus(routerstatus_t *status,
                                        uint8_t dir_purpose,
                                        uint8_t router_purpose,
                                        int anonymized_connection,
                                        const char *resource,
                                        const char *payload,
                                        size_t payload_len,
                                        time_t if_modified_since)
{
  directory_initiate_command_routerstatus_rend(status, dir_purpose,
                                          router_purpose,
                                          anonymized_connection, resource,
                                          payload, payload_len,
                                          if_modified_since, NULL);
529
530
}

531
532
533
/** Return true iff <b>conn</b> is the client side of a directory connection
 * we launched to ourself in order to determine the reachability of our
 * dir_port. */
534
535
536
537
538
539
540
541
static int
directory_conn_is_self_reachability_test(dir_connection_t *conn)
{
  if (conn->requested_resource &&
      !strcmpstart(conn->requested_resource,"authority")) {
    routerinfo_t *me = router_get_my_routerinfo();
    if (me &&
        router_digest_is_me(conn->identity_digest) &&
542
        tor_addr_eq_ipv4h(&conn->_base.addr, me->addr) && /*XXXX prop 118*/
543
544
545
546
547
548
        me->dir_port == conn->_base.port)
      return 1;
  }
  return 0;
}

549
550
551
/** Called when we are unable to complete the client's request to a directory
 * server due to a network error: Mark the router as down and try again if
 * possible.
552
 */
553
void
554
connection_dir_request_failed(dir_connection_t *conn)
555
{
556
557
558
559
560
561
  if (directory_conn_is_self_reachability_test(conn)) {
    routerinfo_t *me = router_get_my_routerinfo();
    if (me)
      control_event_server_status(LOG_WARN,
                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
                                  me->address, me->dir_port);
562
    return; /* this was a test fetch. don't retry. */
563
  }
564
565
  if (entry_list_can_grow(get_options()))
    router_set_status(conn->identity_digest, 0); /* don't try him again */
566
  if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) {
567
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
568
             conn->_base.address);
569
    connection_dir_download_networkstatus_failed(conn, -1);
570
571
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
572
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
573
             conn->_base.address);
574
    connection_dir_download_routerdesc_failed(conn);
575
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
576
    networkstatus_consensus_download_failed(0);
577
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
578
579
580
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
             conn->_base.address);
    connection_dir_download_cert_failed(conn, 0);
581
582
583
584
585
586
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
    log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
             conn->_base.address);
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
    log_info(LD_DIR, "Giving up downloading votes from '%s'",
             conn->_base.address);
587
588
589
  }
}

590
591
592
/** Called when an attempt to download one or more network status
 * documents on connection <b>conn</b> failed. Decide whether to
 * retry the fetch now, later, or never.
593
 */
594
static void
595
connection_dir_download_networkstatus_failed(dir_connection_t *conn,
596
                                             int status_code)
597
{
598
599
600
601
602
603
  if (!conn->requested_resource) {
    /* We never reached directory_send_command, which means that we never
     * opened a network connection.  Either we're out of sockets, or the
     * network is down.  Either way, retrying would be pointless. */
    return;
  }
604
  if (!strcmpstart(conn->requested_resource, "all")) {
605
606
607
    /* We're a non-authoritative directory cache; try again. Ignore status
     * code, since we don't want to keep trying forever in a tight loop
     * if all the authorities are shutting us out. */
608
609
    smartlist_t *trusted_dirs = router_get_trusted_dir_servers();
    SMARTLIST_FOREACH(trusted_dirs, trusted_dir_server_t *, ds,
610
                      download_status_failed(&ds->v2_ns_dl_status, 0));
611
612
    directory_get_from_dirserver(conn->_base.purpose, conn->router_purpose,
                                 "all.z", 0 /* don't retry_if_no_servers */);
613
  } else if (!strcmpstart(conn->requested_resource, "fp/")) {
Roger Dingledine's avatar
Roger Dingledine committed
614
    /* We were trying to download by fingerprint; mark them all as having
615
616
     * failed, and possibly retry them later.*/
    smartlist_t *failed = smartlist_create();
617
    dir_split_resource_into_fingerprints(conn->requested_resource+3,
618
                                         failed, NULL, 0, 0);
619
    if (smartlist_len(failed)) {
620
      dir_networkstatus_download_failed(failed, status_code);
621
      SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp));
622
    }
623
624
    smartlist_free(failed);
  }
625
626
}

627
/** Called when an attempt to download one or more router descriptors
628
 * or extra-info documents on connection <b>conn</b> failed.
629
630
 */
static void
631
connection_dir_download_routerdesc_failed(dir_connection_t *conn)
632
{
633
  /* No need to increment the failure count for routerdescs, since
634
   * it's not their fault. */
635

636
  /* No need to relaunch descriptor downloads here: we already do it
637
   * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
638
639
  tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
             conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
640

641
  (void) conn;
642
643
}

644
645
646
647
648
649
650
651
652
653
/** Called when an attempt to fetch a certificate fails. */
static void
connection_dir_download_cert_failed(dir_connection_t *conn, int status)
{
  smartlist_t *failed;
  tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);

  if (!conn->requested_resource)
    return;
  failed = smartlist_create();
654
  dir_split_resource_into_fingerprints(conn->requested_resource+3,
655
656
657
                                       failed, NULL, 1, 0);
  SMARTLIST_FOREACH(failed, char *, cp,
  {
658
    authority_cert_dl_failed(cp, status);
659
660
661
    tor_free(cp);
  });
  smartlist_free(failed);
662
663

  update_certificate_downloads(time(NULL));
664
665
}

Roger Dingledine's avatar
Roger Dingledine committed
666
667
668
669
670
671
672
673
674
/** Evaluate the situation and decide if we should use an encrypted
 * "begindir-style" connection for this directory request.
 * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
 *    or we're a dir mirror, no.
 * 2) If we prefer to avoid begindir conns, and we're not fetching or
 * publishing a bridge relay descriptor, no.
 * 3) Else yes.
 */
static int
675
676
directory_command_should_use_begindir(or_options_t *options,
                                      const tor_addr_t *addr,
Roger Dingledine's avatar
Roger Dingledine committed
677
678
679
680
681
682
683
                                      int or_port, uint8_t router_purpose,
                                      int anonymized_connection)
{
  if (!or_port)
    return 0; /* We don't know an ORPort -- no chance. */
  if (!anonymized_connection)
    if (!fascist_firewall_allows_address_or(addr, or_port) ||
684
685
        directory_fetches_from_authorities(options) ||
        (server_mode(options) && !options->Address))
Roger Dingledine's avatar
Roger Dingledine committed
686
687
688
689
690
691
692
      return 0; /* We're firewalled or are acting like a relay -- also no. */
  if (!options->TunnelDirConns &&
      router_purpose != ROUTER_PURPOSE_BRIDGE)
    return 0; /* We prefer to avoid using begindir conns. Fine. */
  return 1;
}

693
/** Helper for directory_initiate_command_routerstatus: send the
694
 * command to a server whose address is <b>address</b>, whose IP is
695
696
697
 * <b>addr</b>, whose directory port is <b>dir_port</b>, whose tor version
 * <b>supports_begindir</b>, and whose identity key digest is
 * <b>digest</b>. */
698
void
699
directory_initiate_command(const char *address, const tor_addr_t *_addr,
700
                           uint16_t or_port, uint16_t dir_port,
701
                           int supports_conditional_consensus,
702
703
                           int supports_begindir, const char *digest,
                           uint8_t dir_purpose, uint8_t router_purpose,
704
                           int anonymized_connection, const char *resource,
705
706
                           const char *payload, size_t payload_len,
                           time_t if_modified_since)
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
{
  directory_initiate_command_rend(address, _addr, or_port, dir_port,
                             supports_conditional_consensus,
                             supports_begindir, digest, dir_purpose,
                             router_purpose, anonymized_connection,
                             resource, payload, payload_len,
                             if_modified_since, NULL);
}

/** Same as directory_initiate_command(), but accepts rendezvous data to
 * fetch a hidden service descriptor. */
static void
directory_initiate_command_rend(const char *address, const tor_addr_t *_addr,
                                uint16_t or_port, uint16_t dir_port,
                                int supports_conditional_consensus,
                                int supports_begindir, const char *digest,
                                uint8_t dir_purpose, uint8_t router_purpose,
                                int anonymized_connection,
                                const char *resource,
                                const char *payload, size_t payload_len,
                                time_t if_modified_since,
                                const rend_data_t *rend_query)
Roger Dingledine's avatar
Roger Dingledine committed
729
{
730
  dir_connection_t *conn;
731
  or_options_t *options = get_options();
732
  int socket_error = 0;
Roger Dingledine's avatar
Roger Dingledine committed
733
  int use_begindir = supports_begindir &&
734
                     directory_command_should_use_begindir(options, _addr,
Roger Dingledine's avatar
Roger Dingledine committed
735
                       or_port, router_purpose, anonymized_connection);
736
  tor_addr_t addr;
737

738
  tor_assert(address);
739
  tor_assert(_addr);
740
  tor_assert(or_port || dir_port);
741
  tor_assert(digest);
742

743
744
  tor_addr_copy(&addr, _addr);

745
746
  log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
            anonymized_connection, use_begindir);
747

748
  log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
749

750
  conn = dir_connection_new(AF_INET);
751
752

  /* set up conn so it's got all the data we need to remember */
753
  tor_addr_copy(&conn->_base.addr, &addr);
754
  conn->_base.port = use_begindir ? or_port : dir_port;
755
  conn->_base.address = tor_strdup(address);
756
  memcpy(conn->identity_digest, digest, DIGEST_LEN);
757

758
759
  conn->_base.purpose = dir_purpose;
  conn->router_purpose = router_purpose;
760

761
  /* give it an initial state */
762
  conn->_base.state = DIR_CONN_STATE_CONNECTING;
763

764
765
766
  /* decide whether we can learn our IP address from this conn */
  conn->dirconn_direct = !anonymized_connection;

767
768
769
770
  /* copy rendezvous data, if any */
  if (rend_query)
    conn->rend_data = rend_data_dup(rend_query);

771
772
  if (!anonymized_connection && !use_begindir) {
    /* then we want to connect to dirport directly */
773

774
    if (options->HttpProxy) {
775
      tor_addr_from_ipv4h(&addr, options->HttpProxyAddr);
776
      dir_port = options->HttpProxyPort;
777
778
    }

779
    switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr,
780
                               dir_port, &socket_error)) {
Roger Dingledine's avatar
Roger Dingledine committed
781
      case -1:
782
        connection_dir_request_failed(conn); /* retry if we want */
783
784
        /* XXX we only pass 'conn' above, not 'resource', 'payload',
         * etc. So in many situations it can't retry! -RD */
785
        connection_free(TO_CONN(conn));
Roger Dingledine's avatar
Roger Dingledine committed
786
787
        return;
      case 1:
788
789
        /* start flushing conn */
        conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
Roger Dingledine's avatar
Roger Dingledine committed
790
791
        /* fall through */
      case 0:
792
        /* queue the command on the outbuf */
793
        directory_send_command(conn, dir_purpose, 1, resource,
794
795
796
                               payload, payload_len,
                               supports_conditional_consensus,
                               if_modified_since);
797
        connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
Roger Dingledine's avatar
Roger Dingledine committed
798
799
800
        /* writable indicates finish, readable indicates broken link,
           error indicates broken link in windowsland. */
    }
801
  } else { /* we want to connect via a tor connection */
802
    edge_connection_t *linked_conn;
803
804
805
806

    /* If it's an anonymized connection, remember the fact that we
     * wanted it for later: maybe we'll want it again soon. */
    if (anonymized_connection && use_begindir)
807
      rep_hist_note_used_internal(time(NULL), 0, 1);
808
809
810
    else if (anonymized_connection && !use_begindir)
      rep_hist_note_used_port(time(NULL), conn->_base.port);

Roger Dingledine's avatar
Roger Dingledine committed
811
    /* make an AP connection
812
     * populate it and add it at the right state
813
     * hook up both sides
Roger Dingledine's avatar
Roger Dingledine committed
814
     */
815
    linked_conn =
816
      connection_ap_make_link(conn->_base.address, conn->_base.port,
817
                              digest, use_begindir, conn->dirconn_direct);
818
    if (!linked_conn) {
819
      log_warn(LD_NET,"Making tunnel to dirserver failed.");
820
      connection_mark_for_close(TO_CONN(conn));
821
822
      return;
    }
823
    connection_link_connections(TO_CONN(conn), TO_CONN(linked_conn));
Roger Dingledine's avatar
Roger Dingledine committed
824

825
    if (connection_add(TO_CONN(conn)) < 0) {
826
      log_warn(LD_NET,"Unable to add connection for link to dirserver.");
827
      connection_mark_for_close(TO_CONN(conn));
828
829
      return;
    }
830
    conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
831
    /* queue the command on the outbuf */
832
    directory_send_command(conn, dir_purpose, 0, resource,
833
834
835
                           payload, payload_len,
                           supports_conditional_consensus,
                           if_modified_since);
836
    connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
837
    connection_start_reading(TO_CONN(linked_conn));
838
839
840
  }
}

841
842
843
844
845
846
847
848
849
850
851
852
853
/** Return true iff anything we say on <b>conn</b> is being encrypted before
 * we send it to the client/server. */
int
connection_dir_is_encrypted(dir_connection_t *conn)
{
  /* Right now it's sufficient to see if conn is or has been linked, since
   * the only thing it could be linked to is an edge connection on a
   * circuit, and the only way it could have been unlinked is at the edge
   * connection getting closed.
   */
  return TO_CONN(conn)->linked;
}

854
855
856
857
858
859
860
861
862
863
864
/** Helper for sorting
 *
 * sort strings alphabetically
 */
static int
_compare_strs(const void **a, const void **b)
{
  const char *s1 = *a, *s2 = *b;
  return strcmp(s1, s2);
}

865
866
867
868
869
#define CONDITIONAL_CONSENSUS_FPR_LEN 3
#if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
#error "conditional consensus fingerprint length is larger than digest length"
#endif

870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
/** Return the URL we should use for a consensus download.
 *
 * This url depends on whether or not the server we go to
 * is sufficiently new to support conditional consensus downloading,
 * i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
 */
static char *
directory_get_consensus_url(int supports_conditional_consensus)
{
  char *url;
  int len;

  if (supports_conditional_consensus) {
    char *authority_id_list;
    smartlist_t *authority_digets = smartlist_create();

    SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
                      trusted_dir_server_t *, ds,
      {
889
890
891
892
893
        char *hex;
        if (!(ds->type & V3_AUTHORITY))
          continue;

        hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
894
        base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
895
                      ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
        smartlist_add(authority_digets, hex);
      });
    smartlist_sort(authority_digets, _compare_strs);
    authority_id_list = smartlist_join_strings(authority_digets,
                                               "+", 0, NULL);

    len = strlen(authority_id_list)+64;
    url = tor_malloc(len);
    tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z",
                 authority_id_list);

    SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp));
    smartlist_free(authority_digets);
    tor_free(authority_id_list);
  } else {
    url = tor_strdup("/tor/status-vote/current/consensus.z");
  }
  return url;
}

916
917
/** Queue an appropriate HTTP command on conn-\>outbuf.  The other args
 * are as in directory_initiate_command.
918
 */
919
static void
920
directory_send_command(dir_connection_t *conn,
921
                       int purpose, int direct, const char *resource,
922
                       const char *payload, size_t payload_len,
923
                       int supports_conditional_consensus,
924
                       time_t if_modified_since)
925
{
926
927
  char proxystring[256];
  char proxyauthstring[256];
928
  char hoststring[128];
929
  char imsstring[RFC1123_TIME_LEN+32];
930
931
  char *url;
  char request[8192];
932
  const char *httpcommand = NULL;
933
  size_t len;
934

935
  tor_assert(conn);
936
  tor_assert(conn->_base.type == CONN_TYPE_DIR);
937

938
  tor_free(conn->requested_resource);
939
940
  if (resource)
    conn->requested_resource = tor_strdup(resource);
941

942
  /* come up with a string for which Host: we want */
943
944
  if (conn->_base.port == 80) {
    strlcpy(hoststring, conn->_base.address, sizeof(hoststring));
945
  } else {
946
    tor_snprintf(hoststring, sizeof(hoststring),"%s:%d",
947
                 conn->_base.address, conn->_base.port);
948
  }
949

950
951
952
953
954
955
956
957
958
  /* Format if-modified-since */
  if (!if_modified_since) {
    imsstring[0] = '\0';
  } else {
    char b[RFC1123_TIME_LEN+1];
    format_rfc1123_time(b, if_modified_since);
    tor_snprintf(imsstring, sizeof(imsstring), "\r\nIf-Modified-Since: %s", b);
  }

959
  /* come up with some proxy lines, if we're using one. */
960
  if (direct && get_options()->HttpProxy) {
961
962
963
    char *base64_authenticator=NULL;
    const char *authenticator = get_options()->HttpProxyAuthenticator;

964
    tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
965
966
967
    if (authenticator) {
      base64_authenticator = alloc_http_authenticator(authenticator);
      if (!base64_authenticator)
968
        log_warn(LD_BUG, "Encoding http authenticator failed");
969
970
971
972
973