dircache.c 59.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
/* Copyright (c) 2001-2004, Roger Dingledine.
 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 * Copyright (c) 2007-2018, The Tor Project, Inc. */
/* See LICENSE for licensing information */

#define DIRCACHE_PRIVATE

#include "core/or/or.h"

#include "app/config/config.h"
#include "core/mainloop/connection.h"
#include "core/or/relay.h"
#include "feature/dirauth/dirvote.h"
14
#include "feature/dirauth/authmode.h"
15
16
17
18
19
20
21
22
23
24
25
#include "feature/dirauth/process_descs.h"
#include "feature/dircache/conscache.h"
#include "feature/dircache/consdiffmgr.h"
#include "feature/dircache/dircache.h"
#include "feature/dircache/dirserv.h"
#include "feature/dircommon/directory.h"
#include "feature/dircommon/fp_pair.h"
#include "feature/hs/hs_cache.h"
#include "feature/nodelist/authcert.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/routerlist.h"
26
#include "feature/relay/router.h"
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include "feature/rend/rendcache.h"
#include "feature/stats/geoip.h"
#include "feature/stats/rephist.h"
#include "lib/compress/compress.h"

#include "feature/dircache/cached_dir_st.h"
#include "feature/dircommon/dir_connection_st.h"
#include "feature/nodelist/authority_cert_st.h"
#include "feature/nodelist/networkstatus_st.h"
#include "feature/nodelist/routerinfo_st.h"

/** Maximum size, in bytes, for any directory object that we're accepting
 * as an upload. */
#define MAX_DIR_UL_SIZE ((1<<24)-1) /* 16MB-1 */

/** HTTP cache control: how long do we tell proxies they can cache each
 * kind of document we serve? */
#define FULL_DIR_CACHE_LIFETIME (60*60)
#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
#define ROUTERDESC_CACHE_LIFETIME (30*60)
#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
#define ROBOTS_CACHE_LIFETIME (24*60*60)
#define MICRODESC_CACHE_LIFETIME (48*60*60)

/** Parse an HTTP request string <b>headers</b> of the form
 * \verbatim
 * "\%s [http[s]://]\%s HTTP/1..."
 * \endverbatim
 * If it's well-formed, strdup the second \%s into *<b>url</b>, and
 * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
 * so it does. Return 0.
 * Otherwise, return -1.
 */
STATIC int
parse_http_url(const char *headers, char **url)
{
  char *command = NULL;
  if (parse_http_command(headers, &command, url) < 0) {
    return -1;
  }
  if (strcmpstart(*url, "/tor/")) {
    char *new_url = NULL;
    tor_asprintf(&new_url, "/tor%s%s",
                 *url[0] == '/' ? "" : "/",
                 *url);
    tor_free(*url);
    *url = new_url;
  }
  tor_free(command);
  return 0;
}

/** Create an http response for the client <b>conn</b> out of
 * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
 */
static void
write_short_http_response(dir_connection_t *conn, int status,
                       const char *reason_phrase)
{
  char *buf = NULL;
  char *datestring = NULL;

  IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
    reason_phrase = "unspecified";
  }

  if (server_mode(get_options())) {
    /* include the Date: header, but only if we're a relay or bridge */
    char datebuf[RFC1123_TIME_LEN+1];
    format_rfc1123_time(datebuf, time(NULL));
    tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
  }

  tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
               status, reason_phrase, datestring?datestring:"");

  log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
  connection_buf_add(buf, strlen(buf), TO_CONN(conn));

  tor_free(datestring);
  tor_free(buf);
}

/** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
 * with <b>type</b> as the Content-Type.
 *
 * If <b>length</b> is nonnegative, it is the Content-Length.
 * If <b>encoding</b> is provided, it is the Content-Encoding.
 * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
 * up to cache_lifetime seconds.  Otherwise, the content may not be cached. */
static void
write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
                           const char *type, const char *encoding,
                           const char *extra_headers,
                           long cache_lifetime)
{
  char date[RFC1123_TIME_LEN+1];
  time_t now = time(NULL);
  buf_t *buf = buf_new_with_capacity(1024);

  tor_assert(conn);

  format_rfc1123_time(date, now);

  buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
  if (type) {
    buf_add_printf(buf, "Content-Type: %s\r\n", type);
  }
  if (!is_local_addr(&conn->base_.addr)) {
    /* Don't report the source address for a nearby/private connection.
     * Otherwise we tend to mis-report in cases where incoming ports are
     * being forwarded to a Tor server running behind the firewall. */
    buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
  }
  if (encoding) {
    buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
  }
  if (length >= 0) {
    buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
  }
  if (cache_lifetime > 0) {
    char expbuf[RFC1123_TIME_LEN+1];
    format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
    /* We could say 'Cache-control: max-age=%d' here if we start doing
     * http/1.1 */
    buf_add_printf(buf, "Expires: %s\r\n", expbuf);
  } else if (cache_lifetime == 0) {
    /* We could say 'Cache-control: no-cache' here if we start doing
     * http/1.1 */
    buf_add_string(buf, "Pragma: no-cache\r\n");
  }
  if (extra_headers) {
    buf_add_string(buf, extra_headers);
  }
  buf_add_string(buf, "\r\n");

  connection_buf_add_buf(TO_CONN(conn), buf);
  buf_free(buf);
}

/** As write_http_response_header_impl, but sets encoding and content-typed
 * based on whether the response will be <b>compressed</b> or not. */
static void
write_http_response_headers(dir_connection_t *conn, ssize_t length,
                            compress_method_t method,
                            const char *extra_headers, long cache_lifetime)
{
  const char *methodname = compression_method_get_name(method);
  const char *doctype;
  if (method == NO_METHOD)
    doctype = "text/plain";
  else
    doctype = "application/octet-stream";
  write_http_response_header_impl(conn, length,
                                  doctype,
                                  methodname,
                                  extra_headers,
                                  cache_lifetime);
}

/** As write_http_response_headers, but assumes extra_headers is NULL */
static void
write_http_response_header(dir_connection_t *conn, ssize_t length,
                           compress_method_t method,
                           long cache_lifetime)
{
  write_http_response_headers(conn, length, method, NULL, cache_lifetime);
}

/** Array of compression methods to use (if supported) for serving
 * precompressed data, ordered from best to worst. */
static compress_method_t srv_meth_pref_precompressed[] = {
  LZMA_METHOD,
  ZSTD_METHOD,
  ZLIB_METHOD,
  GZIP_METHOD,
  NO_METHOD
};

/** Array of compression methods to use (if supported) for serving
 * streamed data, ordered from best to worst. */
static compress_method_t srv_meth_pref_streaming_compression[] = {
  ZSTD_METHOD,
  ZLIB_METHOD,
  GZIP_METHOD,
  NO_METHOD
};

/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
 * and convert them to a bitfield where compression method x is supported if
 * and only if 1 &lt;&lt; x is set in the bitfield. */
STATIC unsigned
parse_accept_encoding_header(const char *h)
{
  unsigned result = (1u << NO_METHOD);
  smartlist_t *methods = smartlist_new();
  smartlist_split_string(methods, h, ",",
             SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);

  SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
    compress_method_t method = compression_method_get_by_name(m);
    if (method != UNKNOWN_METHOD) {
      tor_assert(((unsigned)method) < 8*sizeof(unsigned));
      result |= (1u << method);
    }
  } SMARTLIST_FOREACH_END(m);
  SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
    tor_free(m);
  } SMARTLIST_FOREACH_END(m);
  smartlist_free(methods);
  return result;
}

/** Decide whether a client would accept the consensus we have.
 *
 * Clients can say they only want a consensus if it's signed by more
 * than half the authorities in a list.  They pass this list in
 * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
 *
 * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
 * of the full authority identity digest. (Only strings of even length,
 * i.e. encodings of full bytes, are handled correctly.  In the case
 * of an odd number of hex digits the last one is silently ignored.)
 *
 * Returns 1 if more than half of the requested authorities signed the
 * consensus, 0 otherwise.
 */
static int
client_likes_consensus(const struct consensus_cache_entry_t *ent,
                       const char *want_url)
{
  smartlist_t *voters = smartlist_new();
  int need_at_least;
  int have = 0;

  if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
    smartlist_free(voters);
    return 1; // We don't know the voters; assume the client won't mind. */
  }

  smartlist_t *want_authorities = smartlist_new();
  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
  need_at_least = smartlist_len(want_authorities)/2+1;

  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {

    SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
      if (!strcasecmpstart(digest, want_digest)) {
        have++;
        break;
      };
    } SMARTLIST_FOREACH_END(digest);

    /* early exit, if we already have enough */
    if (have >= need_at_least)
      break;
  } SMARTLIST_FOREACH_END(want_digest);

  SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
  smartlist_free(want_authorities);
  SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
  smartlist_free(voters);
  return (have >= need_at_least);
}

/** Return the compression level we should use for sending a compressed
 * response of size <b>n_bytes</b>. */
STATIC compression_level_t
choose_compression_level(ssize_t n_bytes)
{
  if (! have_been_under_memory_pressure()) {
    return HIGH_COMPRESSION; /* we have plenty of RAM. */
  } else if (n_bytes < 0) {
    return HIGH_COMPRESSION; /* unknown; might be big. */
  } else if (n_bytes < 1024) {
    return LOW_COMPRESSION;
  } else if (n_bytes < 2048) {
    return MEDIUM_COMPRESSION;
  } else {
    return HIGH_COMPRESSION;
  }
}

/** Information passed to handle a GET request. */
typedef struct get_handler_args_t {
  /** Bitmask of compression methods that the client said (or implied) it
   * supported. */
  unsigned compression_supported;
  /** If nonzero, the time included an if-modified-since header with this
   * value. */
  time_t if_modified_since;
  /** String containing the requested URL or resource. */
  const char *url;
  /** String containing the HTTP headers */
  const char *headers;
} get_handler_args_t;

/** Entry for handling an HTTP GET request.
 *
 * This entry matches a request if "string" is equal to the requested
 * resource, or if "is_prefix" is true and "string" is a prefix of the
 * requested resource.
 *
 * The 'handler' function is called to handle the request.  It receives
 * an arguments structure, and must return 0 on success or -1 if we should
 * close the connection.
 **/
typedef struct url_table_ent_s {
  const char *string;
  int is_prefix;
  int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
} url_table_ent_t;

static int handle_get_frontpage(dir_connection_t *conn,
                                const get_handler_args_t *args);
static int handle_get_current_consensus(dir_connection_t *conn,
                                const get_handler_args_t *args);
static int handle_get_status_vote(dir_connection_t *conn,
                                const get_handler_args_t *args);
static int handle_get_microdesc(dir_connection_t *conn,
                                const get_handler_args_t *args);
static int handle_get_descriptor(dir_connection_t *conn,
                                const get_handler_args_t *args);
static int handle_get_keys(dir_connection_t *conn,
                                const get_handler_args_t *args);
static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
                                       const get_handler_args_t *args);
static int handle_get_robots(dir_connection_t *conn,
                                const get_handler_args_t *args);
static int handle_get_networkstatus_bridges(dir_connection_t *conn,
                                const get_handler_args_t *args);

/** Table for handling GET requests. */
static const url_table_ent_t url_table[] = {
  { "/tor/", 0, handle_get_frontpage },
  { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
  { "/tor/status-vote/current/", 1, handle_get_status_vote },
  { "/tor/status-vote/next/", 1, handle_get_status_vote },
  { "/tor/micro/d/", 1, handle_get_microdesc },
  { "/tor/server/", 1, handle_get_descriptor },
  { "/tor/extra/", 1, handle_get_descriptor },
  { "/tor/keys/", 1, handle_get_keys },
  { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
  { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
  { "/tor/robots.txt", 0, handle_get_robots },
  { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
  { NULL, 0, NULL },
};

/** Helper function: called when a dirserver gets a complete HTTP GET
 * request.  Look for a request for a directory or for a rendezvous
 * service descriptor.  On finding one, write a response into
 * conn-\>outbuf.  If the request is unrecognized, send a 404.
 * Return 0 if we handled this successfully, or -1 if we need to close
 * the connection. */
MOCK_IMPL(STATIC int,
directory_handle_command_get,(dir_connection_t *conn, const char *headers,
                              const char *req_body, size_t req_body_len))
{
  char *url, *url_mem, *header;
  time_t if_modified_since = 0;
  int zlib_compressed_in_url;
  unsigned compression_methods_supported;

  /* We ignore the body of a GET request. */
  (void)req_body;
  (void)req_body_len;

  log_debug(LD_DIRSERV,"Received GET command.");

  conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;

  if (parse_http_url(headers, &url) < 0) {
    write_short_http_response(conn, 400, "Bad request");
    return 0;
  }
  if ((header = http_get_header(headers, "If-Modified-Since: "))) {
    struct tm tm;
    if (parse_http_time(header, &tm) == 0) {
      if (tor_timegm(&tm, &if_modified_since)<0) {
        if_modified_since = 0;
      } else {
        log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
      }
    }
    /* The correct behavior on a malformed If-Modified-Since header is to
     * act as if no If-Modified-Since header had been given. */
    tor_free(header);
  }
  log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));

  url_mem = url;
  {
    size_t url_len = strlen(url);

    zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
    if (zlib_compressed_in_url) {
      url[url_len-2] = '\0';
    }
  }

  if ((header = http_get_header(headers, "Accept-Encoding: "))) {
    compression_methods_supported = parse_accept_encoding_header(header);
    tor_free(header);
  } else {
    compression_methods_supported = (1u << NO_METHOD);
  }
  if (zlib_compressed_in_url) {
    compression_methods_supported |= (1u << ZLIB_METHOD);
  }

  /* Remove all methods that we don't both support. */
  compression_methods_supported &= tor_compress_get_supported_method_bitmask();

  get_handler_args_t args;
  args.url = url;
  args.headers = headers;
  args.if_modified_since = if_modified_since;
  args.compression_supported = compression_methods_supported;

  int i, result = -1;
  for (i = 0; url_table[i].string; ++i) {
    int match;
    if (url_table[i].is_prefix) {
      match = !strcmpstart(url, url_table[i].string);
    } else {
      match = !strcmp(url, url_table[i].string);
    }
    if (match) {
      result = url_table[i].handler(conn, &args);
      goto done;
    }
  }

  /* we didn't recognize the url */
  write_short_http_response(conn, 404, "Not found");
  result = 0;

 done:
  tor_free(url_mem);
  return result;
}

/** Helper function for GET / or GET /tor/
 */
static int
handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
{
  (void) args; /* unused */
  const char *frontpage = get_dirportfrontpage();

  if (frontpage) {
    size_t dlen;
    dlen = strlen(frontpage);
    /* Let's return a disclaimer page (users shouldn't use V1 anymore,
       and caches don't fetch '/', so this is safe). */

    /* [We don't check for write_bucket_low here, since we want to serve
     *  this page no matter what.] */
    write_http_response_header_impl(conn, dlen, "text/html", "identity",
                                    NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
    connection_buf_add(frontpage, dlen, TO_CONN(conn));
  } else {
    write_short_http_response(conn, 404, "Not found");
  }
  return 0;
}

/** Warn that the cached consensus <b>consensus</b> of type
 * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
 * warning to avoid logging an entry on every request.
 */
static void
warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
                          const char *flavor, time_t now)
{
#define TOO_OLD_WARNING_INTERVAL (60*60)
  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
  char timestamp[ISO_TIME_LEN+1];
  time_t valid_until;
  char *dupes;

  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
    return;

  if ((dupes = rate_limit_log(&warned, now))) {
    format_local_iso_time(timestamp, valid_until);
    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
             "serve it to clients. It was valid until %s local time and we "
             "continued to serve it for up to 24 hours after it expired.%s",
             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
    tor_free(dupes);
  }
}

/**
 * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
 * <b>digest</b>. Return 0 on success.  On failure, report that the hash came
 * from <b>location</b>, report that we are taking <b>action</b> with it, and
 * return -1.
 */
static int
parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
                    const char *action)
{
  if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
      DIGEST256_LEN) {
    return 0;
  } else {
    log_fn(LOG_PROTOCOL_WARN, LD_DIR,
           "%s contained bogus digest %s; %s.",
           location, escaped(hex), action);
    return -1;
  }
}

/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
 * set <b>digest_out<b> to a new smartlist containing every 256-bit
 * hex-encoded digest listed in that header and return 0.  Otherwise return
 * -1.  */
static int
parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
{
  char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
  if (hdr == NULL) {
    return -1;
  }
  smartlist_t *hex_digests = smartlist_new();
  *digests_out = smartlist_new();
  smartlist_split_string(hex_digests, hdr, " ",
                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
  SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
    uint8_t digest[DIGEST256_LEN];
    if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
                             "ignoring")) {
      smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
    }
  } SMARTLIST_FOREACH_END(hex);
  SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
  smartlist_free(hex_digests);
  tor_free(hdr);
  return 0;
}

/** Fallback compression method.  The fallback compression method is used in
 * case a client requests a non-compressed document. We only store compressed
 * documents, so we use this compression method to fetch the document and let
 * the spooling system do the streaming decompression.
 */
#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD

/**
 * Try to find the best consensus diff possible in order to serve a client
 * request for a diff from one of the consensuses in <b>digests</b> to the
 * current consensus of flavor <b>flav</b>.  The client supports the
 * compression methods listed in the <b>compression_methods</b> bitfield:
 * place the method chosen (if any) into <b>compression_used_out</b>.
 */
static struct consensus_cache_entry_t *
find_best_diff(const smartlist_t *digests, int flav,
               unsigned compression_methods,
               compress_method_t *compression_used_out)
{
  struct consensus_cache_entry_t *result = NULL;

  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
    unsigned u;
    for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
      compress_method_t method = srv_meth_pref_precompressed[u];
      if (0 == (compression_methods & (1u<<method)))
        continue; // client doesn't like this one, or we don't have it.
      if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
                                     diff_from, DIGEST256_LEN,
                                     method) == CONSDIFF_AVAILABLE) {
        tor_assert_nonfatal(result);
        *compression_used_out = method;
        return result;
      }
    }
  } SMARTLIST_FOREACH_END(diff_from);

  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
    if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
          DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
      tor_assert_nonfatal(result);
      *compression_used_out = FALLBACK_COMPRESS_METHOD;
      return result;
    }
  } SMARTLIST_FOREACH_END(diff_from);

  return NULL;
}

/** Lookup the cached consensus document by the flavor found in <b>flav</b>.
 * The preferred set of compression methods should be listed in the
 * <b>compression_methods</b> bitfield. The compression method chosen (if any)
 * is stored in <b>compression_used_out</b>. */
static struct consensus_cache_entry_t *
find_best_consensus(int flav,
                    unsigned compression_methods,
                    compress_method_t *compression_used_out)
{
  struct consensus_cache_entry_t *result = NULL;
  unsigned u;

  for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
    compress_method_t method = srv_meth_pref_precompressed[u];

    if (0 == (compression_methods & (1u<<method)))
      continue;

    if (consdiffmgr_find_consensus(&result, flav,
                                   method) == CONSDIFF_AVAILABLE) {
      tor_assert_nonfatal(result);
      *compression_used_out = method;
      return result;
    }
  }

  if (consdiffmgr_find_consensus(&result, flav,
        FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
    tor_assert_nonfatal(result);
    *compression_used_out = FALLBACK_COMPRESS_METHOD;
    return result;
  }

  return NULL;
}

/** Try to find the best supported compression method possible from a given
 * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
 * compression method could be found. */
static compress_method_t
find_best_compression_method(unsigned compression_methods, int stream)
{
  unsigned u;
  compress_method_t *methods;
  size_t length;

  if (stream) {
    methods = srv_meth_pref_streaming_compression;
    length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
  } else {
    methods = srv_meth_pref_precompressed;
    length = ARRAY_LENGTH(srv_meth_pref_precompressed);
  }

  for (u = 0; u < length; ++u) {
    compress_method_t method = methods[u];
    if (compression_methods & (1u<<method))
      return method;
  }

  return NO_METHOD;
}

/** Check if any of the digests in <b>digests</b> matches the latest consensus
 *  flavor (given in <b>flavor</b>) that we have available. */
static int
digest_list_contains_best_consensus(consensus_flavor_t flavor,
                                    const smartlist_t *digests)
{
  const networkstatus_t *ns = NULL;

  if (digests == NULL)
    return 0;

  ns = networkstatus_get_latest_consensus_by_flavor(flavor);

  if (ns == NULL)
    return 0;

  SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
    if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
      return 1;
  } SMARTLIST_FOREACH_END(digest);

  return 0;
}

/** Encodes the results of parsing a consensus request to figure out what
 * consensus, and possibly what diffs, the user asked for. */
typedef struct {
  /** name of the flavor to retrieve. */
  char *flavor;
  /** flavor to retrive, as enum. */
  consensus_flavor_t flav;
  /** plus-separated list of authority fingerprints; see
   * client_likes_consensus(). Aliases the URL in the request passed to
   * parse_consensus_request(). */
  const char *want_fps;
  /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
   * to return a diff from. */
  smartlist_t *diff_from_digests;
  /** If true, never send a full consensus. If there is no diff, send
   * a 404 instead. */
  int diff_only;
} parsed_consensus_request_t;

/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
 * it is stack-allocated. */
static void
parsed_consensus_request_clear(parsed_consensus_request_t *req)
{
  if (!req)
    return;
  tor_free(req->flavor);
  if (req->diff_from_digests) {
    SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
    smartlist_free(req->diff_from_digests);
  }
  memset(req, 0, sizeof(parsed_consensus_request_t));
}

/**
 * Parse the URL and relevant headers of <b>args</b> for a current-consensus
 * request to learn what flavor of consensus we want, what keys it must be
 * signed with, and what diffs we would accept (or demand) instead. Return 0
 * on success and -1 on failure.
 */
static int
parse_consensus_request(parsed_consensus_request_t *out,
                        const get_handler_args_t *args)
{
  const char *url = args->url;
  memset(out, 0, sizeof(parsed_consensus_request_t));
  out->flav = FLAV_NS;

  const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
  const char CONSENSUS_FLAVORED_PREFIX[] =
    "/tor/status-vote/current/consensus-";

  /* figure out the flavor if any, and who we wanted to sign the thing */
  const char *after_flavor = NULL;

  if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
    const char *f, *cp;
    f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
    cp = strchr(f, '/');
    if (cp) {
      after_flavor = cp+1;
      out->flavor = tor_strndup(f, cp-f);
    } else {
      out->flavor = tor_strdup(f);
    }
    int flav = networkstatus_parse_flavor_name(out->flavor);
    if (flav < 0)
      flav = FLAV_NS;
    out->flav = flav;
  } else {
    if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
      after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
  }

  /* see whether we've been asked explicitly for a diff from an older
   * consensus. (The user might also have said that a diff would be okay,
   * via X-Or-Diff-From-Consensus */
  const char DIFF_COMPONENT[] = "diff/";
  char *diff_hash_in_url = NULL;
  if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
    after_flavor += strlen(DIFF_COMPONENT);
    const char *cp = strchr(after_flavor, '/');
    if (cp) {
      diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
      out->want_fps = cp+1;
    } else {
      diff_hash_in_url = tor_strdup(after_flavor);
      out->want_fps = NULL;
    }
  } else {
    out->want_fps = after_flavor;
  }

  if (diff_hash_in_url) {
    uint8_t diff_from[DIGEST256_LEN];
    out->diff_from_digests = smartlist_new();
    out->diff_only = 1;
    int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
                                  "rejecting");
    tor_free(diff_hash_in_url);
    if (ok) {
      smartlist_add(out->diff_from_digests,
                    tor_memdup(diff_from, DIGEST256_LEN));
    } else {
      return -1;
    }
  } else {
    parse_or_diff_from_header(&out->diff_from_digests, args->headers);
  }

  return 0;
}

/** Helper function for GET /tor/status-vote/current/consensus
 */
static int
handle_get_current_consensus(dir_connection_t *conn,
                             const get_handler_args_t *args)
{
  const compress_method_t compress_method =
    find_best_compression_method(args->compression_supported, 0);
  const time_t if_modified_since = args->if_modified_since;
  int clear_spool = 0;

  /* v3 network status fetch. */
  long lifetime = NETWORKSTATUS_CACHE_LIFETIME;

  time_t now = time(NULL);
  parsed_consensus_request_t req;

  if (parse_consensus_request(&req, args) < 0) {
    write_short_http_response(conn, 404, "Couldn't parse request");
    goto done;
  }

  if (digest_list_contains_best_consensus(req.flav,
                                          req.diff_from_digests)) {
    write_short_http_response(conn, 304, "Not modified");
    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
    goto done;
  }

  struct consensus_cache_entry_t *cached_consensus = NULL;

  compress_method_t compression_used = NO_METHOD;
  if (req.diff_from_digests) {
    cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
                                      args->compression_supported,
                                      &compression_used);
  }

  if (req.diff_only && !cached_consensus) {
    write_short_http_response(conn, 404, "No such diff available");
    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
    goto done;
  }

  if (! cached_consensus) {
    cached_consensus = find_best_consensus(req.flav,
                                           args->compression_supported,
                                           &compression_used);
  }

  time_t fresh_until, valid_until;
  int have_fresh_until = 0, have_valid_until = 0;
  if (cached_consensus) {
    have_fresh_until =
      !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
    have_valid_until =
      !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
  }

  if (cached_consensus && have_valid_until &&
      !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
    write_short_http_response(conn, 404, "Consensus is too old");
    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
    goto done;
  }

  if (cached_consensus && req.want_fps &&
      !client_likes_consensus(cached_consensus, req.want_fps)) {
    write_short_http_response(conn, 404, "Consensus not signed by sufficient "
                           "number of requested authorities");
    geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
    goto done;
  }

  conn->spool = smartlist_new();
  clear_spool = 1;
  {
    spooled_resource_t *spooled;
    if (cached_consensus) {
      spooled = spooled_resource_new_from_cache_entry(cached_consensus);
      smartlist_add(conn->spool, spooled);
    }
  }

  lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;

  size_t size_guess = 0;
  int n_expired = 0;
  dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
                                              compress_method != NO_METHOD,
                                              &size_guess,
                                              &n_expired);

  if (!smartlist_len(conn->spool) && !n_expired) {
    write_short_http_response(conn, 404, "Not found");
    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
    goto done;
  } else if (!smartlist_len(conn->spool)) {
    write_short_http_response(conn, 304, "Not modified");
    geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
    goto done;
  }

  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
    log_debug(LD_DIRSERV,
              "Client asked for network status lists, but we've been "
              "writing too many bytes lately. Sending 503 Dir busy.");
    write_short_http_response(conn, 503, "Directory busy, try again later");
    geoip_note_ns_response(GEOIP_REJECT_BUSY);
    goto done;
  }

  tor_addr_t addr;
  if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
    geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
                           &addr, NULL,
                           time(NULL));
    geoip_note_ns_response(GEOIP_SUCCESS);
    /* Note that a request for a network status has started, so that we
     * can measure the download time later on. */
    if (conn->dirreq_id)
      geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
    else
      geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
                         DIRREQ_DIRECT);
  }

  /* Use this header to tell caches that the response depends on the
   * X-Or-Diff-From-Consensus header (or lack thereof). */
  const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";

  clear_spool = 0;

  // The compress_method might have been NO_METHOD, but we store the data
  // compressed. Decompress them using `compression_used`. See fallback code in
  // find_best_consensus() and find_best_diff().
  write_http_response_headers(conn, -1,
                             compress_method == NO_METHOD ?
                               NO_METHOD : compression_used,
                             vary_header,
                             smartlist_len(conn->spool) == 1 ? lifetime : 0);

  if (compress_method == NO_METHOD && smartlist_len(conn->spool))
    conn->compress_state = tor_compress_new(0, compression_used,
                                            HIGH_COMPRESSION);

  /* Prime the connection with some data. */
  const int initial_flush_result = connection_dirserv_flushed_some(conn);
  tor_assert_nonfatal(initial_flush_result == 0);
  goto done;

 done:
  parsed_consensus_request_clear(&req);
  if (clear_spool) {
    dir_conn_clear_spool(conn);
  }
  return 0;
}

/** Helper function for GET /tor/status-vote/{current,next}/...
 */
static int
handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
{
  const char *url = args->url;
  {
    ssize_t body_len = 0;
    ssize_t estimated_len = 0;
    int lifetime = 60; /* XXXX?? should actually use vote intervals. */
    /* This smartlist holds strings that we can compress on the fly. */
    smartlist_t *items = smartlist_new();
    /* This smartlist holds cached_dir_t objects that have a precompressed
     * deflated version. */
    smartlist_t *dir_items = smartlist_new();
    dirvote_dirreq_get_status_vote(url, items, dir_items);
    if (!smartlist_len(dir_items) && !smartlist_len(items)) {
      write_short_http_response(conn, 404, "Not found");
For faster browsing, not all history is shown. View entire blame