directory.c 68.9 KB
Newer Older
Roger Dingledine's avatar
Roger Dingledine committed
1
/* Copyright 2001-2004 Roger Dingledine.
Roger Dingledine's avatar
Roger Dingledine committed
2
 * Copyright 2004-2006 Roger Dingledine, Nick Mathewson. */
Roger Dingledine's avatar
Roger Dingledine committed
3
/* See LICENSE for licensing information */
4
/* $Id$ */
5
6
const char directory_c_id[] =
  "$Id$";
7
8
9

#include "or.h"

Roger Dingledine's avatar
Roger Dingledine committed
10
11
/**
 * \file directory.c
12
13
14
 * \brief Code to send and fetch directories and router
 * descriptors via HTTP.  Directories use dirserv.c to generate the
 * results; clients use routers.c to parse them.
Roger Dingledine's avatar
Roger Dingledine committed
15
 **/
16

Roger Dingledine's avatar
Roger Dingledine committed
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
/* In-points to directory.c:
 *
 * - directory_post_to_dirservers(), called from
 *   router_upload_dir_desc_to_dirservers() in router.c
 *   upload_service_descriptor() in rendservice.c
 * - directory_get_from_dirserver(), called from
 *   rend_client_refetch_renddesc() in rendclient.c
 *   run_scheduled_events() in main.c
 *   do_hup() in main.c
 * - connection_dir_process_inbuf(), called from
 *   connection_process_inbuf() in connection.c
 * - connection_dir_finished_flushing(), called from
 *   connection_finished_flushing() in connection.c
 * - connection_dir_finished_connecting(), called from
 *   connection_finished_connecting() in connection.c
 */
33
34
35
36
static void
directory_initiate_command(const char *address, uint32_t addr, uint16_t port,
                           const char *platform,
                           const char *digest, uint8_t purpose,
37
                           int private_connection, const char *resource,
38
                           const char *payload, size_t payload_len);
39

40
static void
41
directory_send_command(dir_connection_t *conn, const char *platform,
42
43
                       int purpose, const char *resource,
                       const char *payload, size_t payload_len);
44
static int directory_handle_command(dir_connection_t *conn);
45
static int body_is_plausible(const char *body, size_t body_len, int purpose);
46
static int purpose_is_private(uint8_t purpose);
47
static char *http_get_header(const char *headers, const char *which);
48
static void http_set_address_origin(const char *headers, connection_t *conn);
49
50
static void connection_dir_download_networkstatus_failed(
                                                      dir_connection_t *conn);
51
static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
52
static void dir_networkstatus_download_failed(smartlist_t *failed);
53
static void dir_routerdesc_download_failed(smartlist_t *failed);
54
static void note_request(const char *key, size_t bytes);
Roger Dingledine's avatar
Roger Dingledine committed
55

56
57
/********* START VARIABLES **********/

58
59
60
/** How far in the future do we allow a directory server to tell us it is
 * before deciding that one of us has the wrong time? */
#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
61

62
63
#define X_ADDRESS_HEADER "X-Your-Address-Is: "

64
65
/********* END VARIABLES ************/

66
67
/** Return true iff the directory purpose 'purpose' must use an
 * anonymous connection to a directory. */
68
static int
69
70
purpose_is_private(uint8_t purpose)
{
71
72
  if (get_options()->AllDirActionsPrivate)
    return 1;
73
74
  if (purpose == DIR_PURPOSE_FETCH_DIR ||
      purpose == DIR_PURPOSE_UPLOAD_DIR ||
75
76
77
      purpose == DIR_PURPOSE_FETCH_RUNNING_LIST ||
      purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS ||
      purpose == DIR_PURPOSE_FETCH_SERVERDESC)
78
79
80
81
    return 0;
  return 1;
}

Roger Dingledine's avatar
Roger Dingledine committed
82
83
84
85
86
87
88
/** Start a connection to every known directory server, using
 * connection purpose 'purpose' and uploading the payload 'payload'
 * (length 'payload_len').  The purpose should be one of
 * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'.
 */
void
directory_post_to_dirservers(uint8_t purpose, const char *payload,
89
                             size_t payload_len)
Roger Dingledine's avatar
Roger Dingledine committed
90
{
91
  smartlist_t *dirservers;
92
  int post_via_tor;
93
  int post_to_hidserv_only;
Roger Dingledine's avatar
Roger Dingledine committed
94

95
  dirservers = router_get_trusted_dir_servers();
96
  tor_assert(dirservers);
97
  /* Only old dirservers handle rendezvous descriptor publishing. */
98
  post_to_hidserv_only = (purpose == DIR_PURPOSE_UPLOAD_RENDDESC);
Nick Mathewson's avatar
Nick Mathewson committed
99
100
101
  /* This tries dirservers which we believe to be down, but ultimately, that's
   * harmless, and we may as well err on the side of getting things uploaded.
   */
102
103
  SMARTLIST_FOREACH(dirservers, trusted_dir_server_t *, ds,
    {
104
      routerstatus_t *rs = &(ds->fake_status);
105
106
107
108
      if (post_to_hidserv_only && !ds->is_hidserv_authority)
        continue;
      if (!post_to_hidserv_only &&
          !(ds->is_v1_authority || ds->is_v2_authority))
109
        continue;
110
      post_via_tor = purpose_is_private(purpose) ||
111
              !fascist_firewall_allows_address_dir(ds->addr, ds->dir_port);
112
113
      directory_initiate_command_routerstatus(rs, purpose, post_via_tor,
                                              NULL, payload, payload_len);
114
    });
Roger Dingledine's avatar
Roger Dingledine committed
115
116
}

Roger Dingledine's avatar
Roger Dingledine committed
117
/** Start a connection to a random running directory server, using
118
 * connection purpose 'purpose' and requesting 'resource'.
119
120
 * If <b>retry_if_no_servers</b>, then if all the possible servers seem
 * down, mark them up and try again.
Roger Dingledine's avatar
Roger Dingledine committed
121
122
 */
void
123
124
directory_get_from_dirserver(uint8_t purpose, const char *resource,
                             int retry_if_no_servers)
Roger Dingledine's avatar
Roger Dingledine committed
125
{
126
  routerstatus_t *rs = NULL;
127
  or_options_t *options = get_options();
128
  int prefer_authority = server_mode(options) && options->DirPort != 0;
129
  int directconn = !purpose_is_private(purpose);
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  authority_type_t type;

  switch (purpose) {
    case DIR_PURPOSE_FETCH_NETWORKSTATUS:
    case DIR_PURPOSE_FETCH_SERVERDESC:
      type = V2_AUTHORITY;
      break;
    case DIR_PURPOSE_FETCH_DIR:
    case DIR_PURPOSE_FETCH_RUNNING_LIST:
      type = V1_AUTHORITY;
      break;
    case DIR_PURPOSE_FETCH_RENDDESC:
      type = HIDSERV_AUTHORITY;
      break;
    default:
      log_warn(LD_BUG, "Unexpected purpose %d", (int)purpose);
      return;
  }
148

149
  if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
150
151
    return;

152
  if (directconn) {
153
    if (prefer_authority) {
154
      /* only ask authdirservers, and don't ask myself */
155
      rs = router_pick_trusteddirserver(type, 1, 1,
156
                                        retry_if_no_servers);
157
    }
158
    if (!rs) {
159
      /* anybody with a non-zero dirport will do */
160
      rs = router_pick_directory_server(1, 1, type==V2_AUTHORITY,
161
162
                                        retry_if_no_servers);
      if (!rs) {
163
164
165
166
167
168
169
170
171
        const char *which;
        if (purpose == DIR_PURPOSE_FETCH_DIR)
          which = "directory";
        else if (purpose == DIR_PURPOSE_FETCH_RUNNING_LIST)
          which = "status list";
        else if (purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS)
          which = "network status";
        else // if (purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS)
          which = "server descriptors";
172
173
174
        log_info(LD_DIR,
                 "No router found for %s; falling back to dirserver list",
                 which);
175
        rs = router_pick_trusteddirserver(type, 1, 1,
176
                                          retry_if_no_servers);
177
        if (!rs)
178
          directconn = 0; /* last resort: try routing it via Tor */
179
      }
180
    }
181
  }
182
  if (!directconn) {
183
    /* Never use fascistfirewall; we're going via Tor. */
184
185
    if (purpose == DIR_PURPOSE_FETCH_RENDDESC) {
      /* only ask authdirservers, any of them will do */
186
187
      rs = router_pick_trusteddirserver(HIDSERV_AUTHORITY, 0, 0,
                                        retry_if_no_servers);
188
189
    } else {
      /* anybody with a non-zero dirport will do. Disregard firewalls. */
190
      rs = router_pick_directory_server(1, 0, type == V2_AUTHORITY,
191
                                        retry_if_no_servers);
192
      /* If we have any hope of building an indirect conn, we know some router
Roger Dingledine's avatar
Roger Dingledine committed
193
       * descriptors.  If (rs==NULL), we can't build circuits anyway, so
194
       * there's no point in falling back to the authorities in this case.  */
195
    }
196
197
  }

198
199
200
  if (rs)
    directory_initiate_command_routerstatus(rs, purpose, !directconn,
                                            resource, NULL, 0);
201
  else {
202
203
204
    log_notice(LD_DIR,
               "No running dirservers known. Will try again later. "
               "(purpose %d)", purpose);
205
    if (!purpose_is_private(purpose)) {
206
      /* remember we tried them all and failed. */
207
      directory_all_unreachable(time(NULL));
208
    }
209
  }
Roger Dingledine's avatar
Roger Dingledine committed
210
211
}

Nick Mathewson's avatar
Nick Mathewson committed
212
213
/** Launch a new connection to the directory server <b>router</b> to upload or
 * download a service or rendezvous descriptor. <b>purpose</b> determines what
214
215
216
 * kind of directory connection we're launching, and must be one of
 * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC}.
 *
Nick Mathewson's avatar
Nick Mathewson committed
217
 * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
218
219
220
221
 * of the HTTP post.  Otherwise, <b>payload</b> should be NULL.
 *
 * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
 * want to fetch.
222
 */
223
void
224
225
226
227
228
229
directory_initiate_command_router(routerinfo_t *router,
                                  uint8_t purpose,
                                  int private_connection,
                                  const char *resource,
                                  const char *payload,
                                  size_t payload_len)
230
231
{
  directory_initiate_command(router->address, router->addr, router->dir_port,
232
233
234
235
                             router->platform,
                             router->cache_info.identity_digest,
                             purpose, private_connection, resource,
                             payload, payload_len);
236
237
}

238
/** Launch a new connection to the directory server <b>status</b> to upload or
Roger Dingledine's avatar
Roger Dingledine committed
239
 * download a server or rendezvous descriptor. <b>purpose</b> determines what
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 * kind of directory connection we're launching, and must be one of
 * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC}.
 *
 * When uploading, <b>payload</b> and <b>payload_len</b> determine the content
 * of the HTTP post.  Otherwise, <b>payload</b> should be NULL.
 *
 * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we
 * want to fetch.
 */
void
directory_initiate_command_routerstatus(routerstatus_t *status,
                                        uint8_t purpose,
                                        int private_connection,
                                        const char *resource,
                                        const char *payload,
                                        size_t payload_len)
256
{
257
258
  const char *platform = NULL;
  routerinfo_t *router;
259
  char address_buf[INET_NTOA_BUF_LEN+1];
260
261
262
263
264
265
266
267
268
269
270
271
272
273
  struct in_addr in;
  const char *address;
  if ((router = router_get_by_digest(status->identity_digest))) {
    platform = router->platform;
    address = router->address;
  } else {
    in.s_addr = htonl(status->addr);
    tor_inet_ntoa(&in, address_buf, sizeof(address_buf));
    address = address_buf;
  }
  directory_initiate_command(address, status->addr, status->dir_port,
                             platform, status->identity_digest,
                             purpose, private_connection, resource,
                             payload, payload_len);
274
275
}

276
/** Called when we are unable to complete the client's request to a
277
278
 * directory server: Mark the router as down and try again if possible.
 */
279
void
280
connection_dir_request_failed(dir_connection_t *conn)
281
{
282
283
  if (router_digest_is_me(conn->identity_digest))
    return; /* this was a test fetch. don't retry. */
284
  router_set_status(conn->identity_digest, 0); /* don't try him again */
285
286
  if (conn->_base.purpose == DIR_PURPOSE_FETCH_DIR ||
      conn->_base.purpose == DIR_PURPOSE_FETCH_RUNNING_LIST) {
287
    log_info(LD_DIR, "Giving up on directory server at '%s:%d'; retrying",
288
289
             conn->_base.address, conn->_base.port);
    directory_get_from_dirserver(conn->_base.purpose, NULL,
290
                                 0 /* don't retry_if_no_servers */);
291
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_NETWORKSTATUS) {
292
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
293
             conn->_base.address);
294
    connection_dir_download_networkstatus_failed(conn);
295
  } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC) {
296
    log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
297
             conn->_base.address);
298
    connection_dir_download_routerdesc_failed(conn);
299
300
301
  }
}

302
303
304
/** Called when an attempt to download one or more network status
 * documents on connection <b>conn</b> failed. Decide whether to
 * retry the fetch now, later, or never.
305
 */
306
static void
307
connection_dir_download_networkstatus_failed(dir_connection_t *conn)
308
{
309
310
311
312
313
314
  if (!conn->requested_resource) {
    /* We never reached directory_send_command, which means that we never
     * opened a network connection.  Either we're out of sockets, or the
     * network is down.  Either way, retrying would be pointless. */
    return;
  }
315
316
  if (!strcmpstart(conn->requested_resource, "all")) {
    /* We're a non-authoritative directory cache; try again. */
317
318
319
    smartlist_t *trusted_dirs = router_get_trusted_dir_servers();
    SMARTLIST_FOREACH(trusted_dirs, trusted_dir_server_t *, ds,
                      ++ds->n_networkstatus_failures);
320
    directory_get_from_dirserver(conn->_base.purpose, "all.z",
321
322
                                 0 /* don't retry_if_no_servers */);
  } else if (!strcmpstart(conn->requested_resource, "fp/")) {
Roger Dingledine's avatar
Roger Dingledine committed
323
    /* We were trying to download by fingerprint; mark them all as having
324
325
     * failed, and possibly retry them later.*/
    smartlist_t *failed = smartlist_create();
326
    dir_split_resource_into_fingerprints(conn->requested_resource+3,
327
                                         failed, NULL, 0, 0);
328
329
330
    if (smartlist_len(failed)) {
      dir_networkstatus_download_failed(failed);
      SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp));
331
    }
332
333
    smartlist_free(failed);
  }
334
335
}

336
/** Called when an attempt to download one or more router descriptors
337
338
339
 * on connection <b>conn</b> failed.
 */
static void
340
connection_dir_download_routerdesc_failed(dir_connection_t *conn)
341
{
342
343
  /* Try again. No need to increment the failure count for routerdescs, since
   * it's not their fault.*/
344
  /* update_router_descriptor_downloads(time(NULL)); */
345
346
  (void) conn;
  /* XXXX Why did the above get commented out? -NM */
347
348
}

349
/** Helper for directory_initiate_command_(router|trusted_dir): send the
350
351
352
353
 * command to a server whose address is <b>address</b>, whose IP is
 * <b>addr</b>, whose directory port is <b>dir_port</b>, whose tor version is
 * <b>platform</b>, and whose identity key digest is <b>digest</b>. The
 * <b>platform</b> argument is optional; the others are required. */
354
355
356
357
static void
directory_initiate_command(const char *address, uint32_t addr,
                           uint16_t dir_port, const char *platform,
                           const char *digest, uint8_t purpose,
358
                           int private_connection, const char *resource,
359
                           const char *payload, size_t payload_len)
Roger Dingledine's avatar
Roger Dingledine committed
360
{
361
  dir_connection_t *conn;
362

363
364
365
366
  tor_assert(address);
  tor_assert(addr);
  tor_assert(dir_port);
  tor_assert(digest);
367

368
  switch (purpose) {
369
    case DIR_PURPOSE_FETCH_DIR:
370
      log_debug(LD_DIR,"initiating directory fetch");
371
372
      break;
    case DIR_PURPOSE_FETCH_RENDDESC:
373
      log_debug(LD_DIR,"initiating hidden-service descriptor fetch");
374
375
      break;
    case DIR_PURPOSE_UPLOAD_DIR:
376
      log_debug(LD_OR,"initiating server descriptor upload");
377
378
      break;
    case DIR_PURPOSE_UPLOAD_RENDDESC:
379
      log_debug(LD_REND,"initiating hidden-service descriptor upload");
380
      break;
381
    case DIR_PURPOSE_FETCH_RUNNING_LIST:
382
      log_debug(LD_DIR,"initiating running-routers fetch");
383
      break;
384
    case DIR_PURPOSE_FETCH_NETWORKSTATUS:
385
      log_debug(LD_DIR,"initiating network-status fetch");
386
387
      break;
    case DIR_PURPOSE_FETCH_SERVERDESC:
388
      log_debug(LD_DIR,"initiating server descriptor fetch");
389
      break;
390
    default:
391
      log_err(LD_BUG, "Unrecognized directory connection purpose.");
392
      tor_assert(0);
393
  }
394

395
  conn = TO_DIR_CONN(connection_new(CONN_TYPE_DIR));
396
397

  /* set up conn so it's got all the data we need to remember */
398
399
400
  conn->_base.addr = addr;
  conn->_base.port = dir_port;
  conn->_base.address = tor_strdup(address);
401
  memcpy(conn->identity_digest, digest, DIGEST_LEN);
402

403
  conn->_base.purpose = purpose;
404

405
  /* give it an initial state */
406
  conn->_base.state = DIR_CONN_STATE_CONNECTING;
407
  conn->dirconn_direct = (private_connection == 0);
408

409
  if (!private_connection) {
Roger Dingledine's avatar
Roger Dingledine committed
410
    /* then we want to connect directly */
411
412
413
414
415
416

    if (get_options()->HttpProxy) {
      addr = get_options()->HttpProxyAddr;
      dir_port = get_options()->HttpProxyPort;
    }

417
418
    switch (connection_connect(TO_CONN(conn), conn->_base.address, addr,
                               dir_port)) {
Roger Dingledine's avatar
Roger Dingledine committed
419
      case -1:
420
        connection_dir_request_failed(conn); /* retry if we want */
421
        connection_free(TO_CONN(conn));
Roger Dingledine's avatar
Roger Dingledine committed
422
423
        return;
      case 1:
424
425
        /* start flushing conn */
        conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
Roger Dingledine's avatar
Roger Dingledine committed
426
427
        /* fall through */
      case 0:
428
        /* queue the command on the outbuf */
429
430
        directory_send_command(conn, platform, purpose, resource,
                               payload, payload_len);
431
        connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE);
Roger Dingledine's avatar
Roger Dingledine committed
432
433
434
435
436
        /* writable indicates finish, readable indicates broken link,
           error indicates broken link in windowsland. */
    }
  } else { /* we want to connect via tor */
    /* make an AP connection
437
     * populate it and add it at the right state
Roger Dingledine's avatar
Roger Dingledine committed
438
439
     * socketpair and hook up both sides
     */
440
441
442
    conn->_base.s = connection_ap_make_bridge(conn->_base.address,
                                              conn->_base.port);
    if (conn->_base.s < 0) {
443
      log_warn(LD_NET,"Making AP bridge to dirserver failed.");
444
      connection_mark_for_close(TO_CONN(conn));
445
446
      return;
    }
Roger Dingledine's avatar
Roger Dingledine committed
447

448
    if (connection_add(TO_CONN(conn)) < 0) {
449
      log_warn(LD_NET,"Unable to add AP bridge to dirserver.");
450
      connection_mark_for_close(TO_CONN(conn));
451
452
      return;
    }
453
    conn->_base.state = DIR_CONN_STATE_CLIENT_SENDING;
454
    /* queue the command on the outbuf */
455
456
    directory_send_command(conn, platform, purpose, resource,
                           payload, payload_len);
457
    connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE);
458
459
460
  }
}

461
462
/** Queue an appropriate HTTP command on conn-\>outbuf.  The other args
 * are as in directory_initiate_command.
463
 */
464
static void
465
directory_send_command(dir_connection_t *conn, const char *platform,
466
                       int purpose, const char *resource,
467
468
                       const char *payload, size_t payload_len)
{
469
470
  char proxystring[256];
  char proxyauthstring[256];
471
  char hoststring[128];
472
473
  char *url;
  char request[8192];
474
  const char *httpcommand = NULL;
475
  size_t len;
476

477
  tor_assert(conn);
478
  tor_assert(conn->_base.type == CONN_TYPE_DIR);
479

480
  tor_free(conn->requested_resource);
481
482
  if (resource)
    conn->requested_resource = tor_strdup(resource);
483

484
  /* come up with a string for which Host: we want */
485
486
  if (conn->_base.port == 80) {
    strlcpy(hoststring, conn->_base.address, sizeof(hoststring));
487
  } else {
488
    tor_snprintf(hoststring, sizeof(hoststring),"%s:%d",
489
                 conn->_base.address, conn->_base.port);
490
  }
491
492

  /* come up with some proxy lines, if we're using one. */
493
  if (get_options()->HttpProxy) {
494
495
496
    char *base64_authenticator=NULL;
    const char *authenticator = get_options()->HttpProxyAuthenticator;

497
    tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
498
499
500
    if (authenticator) {
      base64_authenticator = alloc_http_authenticator(authenticator);
      if (!base64_authenticator)
501
        log_warn(LD_BUG, "Encoding http authenticator failed");
502
503
504
505
506
507
508
509
510
    }
    if (base64_authenticator) {
      tor_snprintf(proxyauthstring, sizeof(proxyauthstring),
                   "\r\nProxy-Authorization: Basic %s",
                   base64_authenticator);
      tor_free(base64_authenticator);
    } else {
      proxyauthstring[0] = 0;
    }
511
512
  } else {
    proxystring[0] = 0;
513
    proxyauthstring[0] = 0;
514
515
  }

516
  switch (purpose) {
Roger Dingledine's avatar
Roger Dingledine committed
517
    case DIR_PURPOSE_FETCH_DIR:
518
519
      tor_assert(!resource);
      tor_assert(!payload);
520
521
      log_debug(LD_DIR,
                "Asking for compressed directory from server running %s",
522
                platform?escaped(platform):"<unknown version>");
523
      httpcommand = "GET";
524
      url = tor_strdup("/tor/dir.z");
525
526
      break;
    case DIR_PURPOSE_FETCH_RUNNING_LIST:
527
528
      tor_assert(!resource);
      tor_assert(!payload);
529
      httpcommand = "GET";
530
531
532
      url = tor_strdup("/tor/running-routers");
      break;
    case DIR_PURPOSE_FETCH_NETWORKSTATUS:
533
      httpcommand = "GET";
534
535
536
537
538
      len = strlen(resource)+32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/status/%s", resource);
      break;
    case DIR_PURPOSE_FETCH_SERVERDESC:
539
      httpcommand = "GET";
540
541
542
      len = strlen(resource)+32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/server/%s", resource);
543
      break;
Roger Dingledine's avatar
Roger Dingledine committed
544
    case DIR_PURPOSE_UPLOAD_DIR:
545
      tor_assert(!resource);
Roger Dingledine's avatar
Roger Dingledine committed
546
      tor_assert(payload);
547
      httpcommand = "POST";
548
      url = tor_strdup("/tor/");
Roger Dingledine's avatar
Roger Dingledine committed
549
      break;
550
    case DIR_PURPOSE_FETCH_RENDDESC:
551
552
      tor_assert(resource);
      tor_assert(!payload);
553
554

      /* this must be true or we wouldn't be doing the lookup */
555
      tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN);
556
      /* This breaks the function abstraction. */
557
      strlcpy(conn->rend_query, resource, sizeof(conn->rend_query));
558

559
      httpcommand = "GET";
Nick Mathewson's avatar
Nick Mathewson committed
560
      /* Request the most recent versioned descriptor. */
561
562
      // (XXXX We were going to switch this to fetch rendezvous1 descriptors,
      // but that never got testing, and it wasn't a good design.)
563
564
565
      len = strlen(resource)+32;
      url = tor_malloc(len);
      tor_snprintf(url, len, "/tor/rendezvous/%s", resource);
Roger Dingledine's avatar
Roger Dingledine committed
566
      break;
567
    case DIR_PURPOSE_UPLOAD_RENDDESC:
568
      tor_assert(!resource);
Roger Dingledine's avatar
Roger Dingledine committed
569
      tor_assert(payload);
570
      httpcommand = "POST";
571
      url = tor_strdup("/tor/rendezvous/publish");
572
      break;
573
574
575
    default:
      tor_assert(0);
      return;
576
  }
577
578

  if (strlen(proxystring) + strlen(url) >= 4096) {
579
580
581
582
    log_warn(LD_BUG,
             "Bug: squid does not like URLs longer than 4095 bytes, this "
             "one is %d bytes long: %s%s",
             (int)(strlen(proxystring) + strlen(url)), proxystring, url);
583
584
  }

585
  tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
586
587
  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
  connection_write_to_buf(url, strlen(url), TO_CONN(conn));
588
  tor_free(url);
589

590
591
592
593
594
595
596
597
598
599
600
601
  if (!strcmp(httpcommand, "GET") && !payload) {
    tor_snprintf(request, sizeof(request),
                 " HTTP/1.0\r\nHost: %s%s\r\n\r\n",
                 hoststring,
                 proxyauthstring);
  } else {
    tor_snprintf(request, sizeof(request),
                 " HTTP/1.0\r\nContent-Length: %lu\r\nHost: %s%s\r\n\r\n",
                 payload ? (unsigned long)payload_len : 0,
                 hoststring,
                 proxyauthstring);
  }
602
  connection_write_to_buf(request, strlen(request), TO_CONN(conn));
603

604
  if (payload) {
605
    /* then send the payload afterwards too */
606
    connection_write_to_buf(payload, payload_len, TO_CONN(conn));
607
  }
Roger Dingledine's avatar
Roger Dingledine committed
608
609
}

610
/** Parse an HTTP request string <b>headers</b> of the form
611
 * \verbatim
612
 * "\%s [http[s]://]\%s HTTP/1..."
613
 * \endverbatim
614
 * If it's well-formed, strdup the second \%s into *<b>url</b>, and
615
 * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
616
 * so it does. Return 0.
Roger Dingledine's avatar
Roger Dingledine committed
617
618
 * Otherwise, return -1.
 */
Roger Dingledine's avatar
Roger Dingledine committed
619
620
621
static int
parse_http_url(char *headers, char **url)
{
622
  char *s, *start, *tmp;
Roger Dingledine's avatar
Roger Dingledine committed
623
624
625
626
627
628
629

  s = (char *)eat_whitespace_no_nl(headers);
  if (!*s) return -1;
  s = (char *)find_whitespace(s); /* get past GET/POST */
  if (!*s) return -1;
  s = (char *)eat_whitespace_no_nl(s);
  if (!*s) return -1;
630
  start = s; /* this is it, assuming it's valid */
631
  s = (char *)find_whitespace(start);
Roger Dingledine's avatar
Roger Dingledine committed
632
  if (!*s) return -1;
633
634

  /* tolerate the http[s] proxy style of putting the hostname in the url */
635
  if (s-start >= 4 && !strcmpstart(start,"http")) {
636
    tmp = start + 4;
637
    if (*tmp == 's')
638
      tmp++;
639
    if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
640
      tmp = strchr(tmp+3, '/');
641
      if (tmp && tmp < s) {
642
        log_debug(LD_DIR,"Skipping over 'http[s]://hostname' string");
643
644
645
646
647
        start = tmp;
      }
    }
  }

648
  if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */
649
    *url = tor_malloc(s - start + 5);
Nick Mathewson's avatar
Nick Mathewson committed
650
651
    strlcpy(*url,"/tor", s-start+5);
    strlcat((*url)+4, start, s-start+1);
652
653
654
  } else {
    *url = tor_strndup(start, s-start);
  }
655
656
657
  return 0;
}

658
659
660
/** Return a copy of the first HTTP header in <b>headers</b> whose key is
 * <b>which</b>.  The key should be given with a terminating colon and space;
 * this function copies everything after, up to but not including the
661
 * following \\r\\n. */
662
663
664
665
666
667
668
669
static char *
http_get_header(const char *headers, const char *which)
{
  const char *cp = headers;
  while (cp) {
    if (!strcmpstart(cp, which)) {
      char *eos;
      cp += strlen(which);
670
      if ((eos = strchr(cp,'\r')))
671
672
673
674
        return tor_strndup(cp, eos-cp);
      else
        return tor_strdup(cp);
    }
675
    cp = strchr(cp, '\n');
676
677
678
679
680
681
    if (cp)
      ++cp;
  }
  return NULL;
}

682
/** If <b>headers</b> indicates that a proxy was involved, then rewrite
683
684
 * <b>conn</b>-\>address to describe our best guess of the address that
 * originated this HTTP request. */
685
686
static void
http_set_address_origin(const char *headers, connection_t *conn)
687
688
689
690
691
692
693
{
  char *fwd;

  fwd = http_get_header(headers, "Forwarded-For: ");
  if (!fwd)
    fwd = http_get_header(headers, "X-Forwarded-For: ");
  if (fwd) {
694
    tor_free(conn->address);
695
696
    conn->address = tor_strdup(escaped(fwd));
    tor_free(fwd);
697
698
699
  }
}

Nick Mathewson's avatar
Nick Mathewson committed
700
/** Parse an HTTP response string <b>headers</b> of the form
701
 * \verbatim
Nick Mathewson's avatar
Nick Mathewson committed
702
 * "HTTP/1.\%d \%d\%s\r\n...".
703
 * \endverbatim
704
705
706
707
708
709
710
711
712
713
714
 *
 * If it's well-formed, assign the status code to *<b>code</b> and
 * return 0.  Otherwise, return -1.
 *
 * On success: If <b>date</b> is provided, set *date to the Date
 * header in the http headers, or 0 if no such header is found.  If
 * <b>compression</b> is provided, set *<b>compression</b> to the
 * compression method given in the Content-Encoding header, or 0 if no
 * such header is found, or -1 if the value of the header is not
 * recognized.  If <b>reason</b> is provided, strdup the reason string
 * into it.
715
 */
716
int
717
parse_http_response(const char *headers, int *code, time_t *date,
718
                    int *compression, char **reason)
Roger Dingledine's avatar
Roger Dingledine committed
719
{
720
  int n1, n2;
721
  char datestr[RFC1123_TIME_LEN+1];
722
  smartlist_t *parsed_headers;
723
724
  tor_assert(headers);
  tor_assert(code);
725

726
  while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
727

728
  if (sscanf(headers, "HTTP/1.%d %d", &n1, &n2) < 2 ||
729
730
      (n1 != 0 && n1 != 1) ||
      (n2 < 100 || n2 >= 600)) {
731
    log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
732
733
734
    return -1;
  }
  *code = n2;
735

736
737
738
  parsed_headers = smartlist_create();
  smartlist_split_string(parsed_headers, headers, "\n",
                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
739
740
741
742
743
744
745
746
747
748
749
750
751
752
  if (reason) {
    smartlist_t *status_line_elements = smartlist_create();
    tor_assert(smartlist_len(parsed_headers));
    smartlist_split_string(status_line_elements,
                           smartlist_get(parsed_headers, 0),
                           " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
    tor_assert(smartlist_len(status_line_elements) <= 3);
    if (smartlist_len(status_line_elements) == 3) {
      *reason = smartlist_get(status_line_elements, 2);
      smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
    }
    SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
    smartlist_free(status_line_elements);
  }
753
754
  if (date) {
    *date = 0;
755
756
757
    SMARTLIST_FOREACH(parsed_headers, const char *, s,
      if (!strcmpstart(s, "Date: ")) {
        strlcpy(datestr, s+6, sizeof(datestr));
758
759
760
761
762
        /* This will do nothing on failure, so we don't need to check
           the result.   We shouldn't warn, since there are many other valid
           date formats besides the one we use. */
        parse_rfc1123_time(datestr, date);
        break;
763
764
765
766
767
768
      });
  }
  if (compression) {
    const char *enc = NULL;
    SMARTLIST_FOREACH(parsed_headers, const char *, s,
      if (!strcmpstart(s, "Content-Encoding: ")) {
769
        enc = s+18; break;
770
      });
771
    if (!enc || !strcmp(enc, "identity")) {
772
773
774
775
776
777
      *compression = 0;
    } else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) {
      *compression = ZLIB_METHOD;
    } else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) {
      *compression = GZIP_METHOD;
    } else {
778
779
      log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
               escaped(enc));
780
      *compression = -1;
781
782
    }
  }
783
784
  SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
  smartlist_free(parsed_headers);
785

786
787
788
  return 0;
}

789
790
791
792
/** Return true iff <b>body</b> doesn't start with a plausible router or
 * running-list or directory opening.  This is a sign of possible compression.
 **/
static int
793
body_is_plausible(const char *body, size_t len, int purpose)
794
795
{
  int i;
796
797
  if (len == 0)
    return 1; /* empty bodies don't need decompression */
798
799
  if (len < 32)
    return 0;
800
801
802
803
804
805
806
  if (purpose != DIR_PURPOSE_FETCH_RENDDESC) {
    if (!strcmpstart(body,"router") ||
        !strcmpstart(body,"signed-directory") ||
        !strcmpstart(body,"network-status") ||
        !strcmpstart(body,"running-routers"))
    return 1;
    for (i=0;i<32;++i) {
Nick Mathewson's avatar
Nick Mathewson committed
807
      if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
808
809
810
811
        return 0;
    }
    return 1;
  } else {
812
813
814
815
    return 1;
  }
}

816
817
818
/** We are a client, and we've finished reading the server's
 * response. Parse and it and act appropriately.
 *
819
820
821
 * If we're still happy with using this directory server in the future, return
 * 0. Otherwise return -1; and the caller should consider trying the request
 * again.
822
823
 *
 * The caller will take care of marking the connection for close.
824
 */
825
static int
826
connection_dir_client_reached_eof(dir_connection_t *conn)
827
{
828
  char *body;
829
  char *headers;
830
  char *reason = NULL;
831
  size_t body_len=0, orig_len=0;
832
  int status_code;
833
834
  time_t now, date_header=0;
  int delta;
835
  int compression;
836
  int plausible;
837
  int skewed=0;
838
  int allow_partial = conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC;
839
  int was_compressed=0;
840

841
  switch (fetch_from_buf_http(conn->_base.inbuf,
842
                              &headers, MAX_HEADERS_SIZE,
843
844
                              &body, &body_len, MAX_DIR_SIZE,
                              allow_partial)) {
845
    case -1: /* overflow */
846
847
      log_warn(LD_PROTOCOL,
               "'fetch' response too large (server '%s:%d'). Closing.",
848
               conn->_base.address, conn->_base.port);
849
850
      return -1;
    case 0:
851
852
      log_info(LD_HTTP,
               "'fetch' response not all here, but we're at eof. Closing.");
853
854
855
      return -1;
    /* case 1, fall through */
  }
856
  orig_len = body_len;
857

858
  if (parse_http_response(headers, &status_code, &date_header,
859
                          &compression, &reason) < 0) {
860
    log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
861
             conn->_base.address, conn->_base.port);
Roger Dingledine's avatar
Roger Dingledine committed
862
    tor_free(body); tor_free(headers);
863
864
    return -1;
  }