Commit e1ddee8b authored by Nick Mathewson's avatar Nick Mathewson 🏃
Browse files

Code to generate, store, and parse microdescriptors and consensuses.

The consensus documents are not signed properly, not served, and not
exchanged yet.
parent a8e92ba8
......@@ -684,6 +684,13 @@ tor_digest_is_zero(const char *digest)
return tor_mem_is_zero(digest, DIGEST_LEN);
}
/** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
int
tor_digest256_is_zero(const char *digest)
{
return tor_mem_is_zero(digest, DIGEST256_LEN);
}
/* Helper: common code to check whether the result of a strtol or strtoul or
* strtoll is correct. */
#define CHECK_STRTOX_RESULT() \
......
......@@ -195,6 +195,7 @@ const char *find_whitespace(const char *s) ATTR_PURE;
const char *find_whitespace_eos(const char *s, const char *eos) ATTR_PURE;
int tor_mem_is_zero(const char *mem, size_t len) ATTR_PURE;
int tor_digest_is_zero(const char *digest) ATTR_PURE;
int tor_digest256_is_zero(const char *digest) ATTR_PURE;
char *esc_for_log(const char *string) ATTR_MALLOC;
const char *escaped(const char *string);
struct smartlist_t;
......
......@@ -20,6 +20,7 @@ libtor_a_SOURCES = buffers.c circuitbuild.c circuitlist.c \
connection.c connection_edge.c connection_or.c control.c \
cpuworker.c directory.c dirserv.c dirvote.c \
dns.c dnsserv.c geoip.c hibernate.c main.c $(tor_platform_source) \
microdesc.c \
networkstatus.c onion.c policies.c \
reasons.c relay.c rendcommon.c rendclient.c rendmid.c \
rendservice.c rephist.c router.c routerlist.c routerparse.c \
......
......@@ -1901,10 +1901,11 @@ routerstatus_format_entry(char *buf, size_t buf_len,
tor_inet_ntoa(&in, ipaddr, sizeof(ipaddr));
r = tor_snprintf(buf, buf_len,
"r %s %s %s %s %s %d %d\n",
"r %s %s %s%s%s %s %d %d\n",
rs->nickname,
identity64,
digest64,
(format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
(format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
published,
ipaddr,
(int)rs->or_port,
......@@ -1918,7 +1919,7 @@ routerstatus_format_entry(char *buf, size_t buf_len,
* this here, instead of in the caller. Then we could use the
* networkstatus_type_t values, with an additional control port value
* added -MP */
if (format == NS_V3_CONSENSUS)
if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
return 0;
cp = buf + strlen(buf);
......@@ -2434,6 +2435,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
vote_timing_t timing;
digestmap_t *omit_as_sybil = NULL;
const int vote_on_reachability = running_long_enough_to_decide_unreachable();
smartlist_t *microdescriptors = NULL;
tor_assert(private_key);
tor_assert(cert);
......@@ -2482,11 +2484,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
omit_as_sybil = get_possible_sybil_list(routers);
routerstatuses = smartlist_create();
microdescriptors = smartlist_create();
SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
if (ri->cache_info.published_on >= cutoff) {
routerstatus_t *rs;
vote_routerstatus_t *vrs;
microdesc_t *md;
vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
rs = &vrs->status;
......@@ -2501,9 +2505,30 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_env_t *private_key,
rs->is_running = 0;
vrs->version = version_from_platform(ri->platform);
md = dirvote_create_microdescriptor(ri);
if (md) {
char buf[128];
vote_microdesc_hash_t *h;
dirvote_format_microdesc_vote_line(buf, sizeof(buf), md);
h = tor_malloc(sizeof(vote_microdesc_hash_t));
h->microdesc_hash_line = tor_strdup(buf);
h->next = NULL;
vrs->microdesc = h;
md->last_listed = now;
smartlist_add(microdescriptors, md);
}
smartlist_add(routerstatuses, vrs);
}
});
} SMARTLIST_FOREACH_END(ri);
{
smartlist_t *added =
microdescs_add_list_to_cache(get_microdesc_cache(),
microdescriptors, SAVED_NOWHERE, 0);
smartlist_free(added);
smartlist_free(microdescriptors);
}
smartlist_free(routers);
digestmap_free(omit_as_sybil, NULL);
......
......@@ -24,14 +24,20 @@ static int dirvote_publish_consensus(void);
static char *make_consensus_method_list(int low, int high, const char *sep);
/** The highest consensus method that we currently support. */
#define MAX_SUPPORTED_CONSENSUS_METHOD 7
#define MAX_SUPPORTED_CONSENSUS_METHOD 8
#define MIN_METHOD_FOR_PARAMS 7
/** Lowest consensus method that generates microdescriptors */
#define MIN_METHOD_FOR_MICRODESC 8
/* =====
* Voting
* =====*/
/* Overestimated. */
#define MICRODESC_LINE_LEN 80
/** Return a new string containing the string representation of the vote in
* <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>.
* For v3 authorities. */
......@@ -89,7 +95,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
len = 8192;
len += strlen(version_lines);
len += (RS_ENTRY_LEN)*smartlist_len(rl->routers);
len += (RS_ENTRY_LEN+MICRODESC_LINE_LEN)*smartlist_len(rl->routers);
len += v3_ns->cert->cache_info.signed_descriptor_len;
status = tor_malloc(len);
......@@ -158,15 +164,25 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
outp += cert->cache_info.signed_descriptor_len;
}
SMARTLIST_FOREACH(v3_ns->routerstatus_list, vote_routerstatus_t *, vrs,
{
SMARTLIST_FOREACH_BEGIN(v3_ns->routerstatus_list, vote_routerstatus_t *,
vrs) {
vote_microdesc_hash_t *h;
if (routerstatus_format_entry(outp, endp-outp, &vrs->status,
vrs->version, NS_V3_VOTE) < 0) {
log_warn(LD_BUG, "Unable to print router status.");
goto err;
}
outp += strlen(outp);
});
for (h = vrs->microdesc; h; h = h->next) {
size_t mlen = strlen(h->microdesc_hash_line);
if (outp+mlen >= endp) {
log_warn(LD_BUG, "Can't fit microdesc line in vote.");
}
memcpy(outp, h->microdesc_hash_line, mlen+1);
outp += strlen(outp);
}
} SMARTLIST_FOREACH_END(vrs);
{
char signing_key_fingerprint[FINGERPRINT_LEN+1];
......@@ -189,7 +205,7 @@ format_networkstatus_vote(crypto_pk_env_t *private_signing_key,
outp += strlen(outp);
}
if (router_get_networkstatus_v3_hash(status, digest)<0)
if (router_get_networkstatus_v3_hash(status, digest, DIGEST_SHA1)<0)
goto err;
note_crypto_pk_op(SIGN_DIR);
if (router_append_dirobj_signature(outp,endp-outp,digest, DIGEST_LEN,
......@@ -294,34 +310,8 @@ get_frequent_members(smartlist_t *out, smartlist_t *in, int min)
/** Given a sorted list of strings <b>lst</b>, return the member that appears
* most. Break ties in favor of later-occurring members. */
static const char *
get_most_frequent_member(smartlist_t *lst)
{
const char *most_frequent = NULL;
int most_frequent_count = 0;
const char *cur = NULL;
int count = 0;
SMARTLIST_FOREACH(lst, const char *, s,
{
if (cur && !strcmp(s, cur)) {
++count;
} else {
if (count >= most_frequent_count) {
most_frequent = cur;
most_frequent_count = count;
}
cur = s;
count = 1;
}
});
if (count >= most_frequent_count) {
most_frequent = cur;
most_frequent_count = count;
}
return most_frequent;
}
#define get_most_frequent_member(lst) \
smartlist_get_most_frequent_string(lst)
/** Return 0 if and only if <b>a</b> and <b>b</b> are routerstatuses
* that come from the same routerinfo, with the same derived elements.
......@@ -363,7 +353,8 @@ _compare_vote_rs(const void **_a, const void **_b)
* in favor of smaller descriptor digest.
*/
static vote_routerstatus_t *
compute_routerstatus_consensus(smartlist_t *votes)
compute_routerstatus_consensus(smartlist_t *votes, int consensus_method,
char *microdesc_digest256_out)
{
vote_routerstatus_t *most = NULL, *cur = NULL;
int most_n = 0, cur_n = 0;
......@@ -399,6 +390,26 @@ compute_routerstatus_consensus(smartlist_t *votes)
}
tor_assert(most);
if (consensus_method >= MIN_METHOD_FOR_MICRODESC &&
microdesc_digest256_out) {
smartlist_t *digests = smartlist_create();
const char *best_microdesc_digest;
SMARTLIST_FOREACH(votes, vote_routerstatus_t *, rs, {
char d[DIGEST256_LEN];
if (compare_vote_rs(rs, most))
continue;
if (!vote_routerstatus_find_microdesc_hash(d, rs, consensus_method))
smartlist_add(digests, tor_memdup(d, sizeof(d)));
});
smartlist_sort_digests256(digests);
best_microdesc_digest = smartlist_get_most_frequent_digest256(digests);
if (best_microdesc_digest)
memcpy(microdesc_digest256_out, best_microdesc_digest, DIGEST256_LEN);
SMARTLIST_FOREACH(digests, char *, cp, tor_free(cp));
smartlist_free(digests);
}
return most;
}
......@@ -615,16 +626,20 @@ networkstatus_compute_consensus(smartlist_t *votes,
crypto_pk_env_t *identity_key,
crypto_pk_env_t *signing_key,
const char *legacy_id_key_digest,
crypto_pk_env_t *legacy_signing_key)
crypto_pk_env_t *legacy_signing_key,
consensus_flavor_t flavor)
{
smartlist_t *chunks;
char *result = NULL;
int consensus_method;
time_t valid_after, fresh_until, valid_until;
int vote_seconds, dist_seconds;
char *client_versions = NULL, *server_versions = NULL;
smartlist_t *flags;
const routerstatus_format_type_t rs_format =
flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC;
tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC);
tor_assert(total_authorities >= smartlist_len(votes));
if (!smartlist_len(votes)) {
......@@ -955,6 +970,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
int n_listing = 0;
int i;
char buf[256];
char microdesc_digest[DIGEST256_LEN];
/* Of the next-to-be-considered digest in each voter, which is first? */
SMARTLIST_FOREACH(votes, networkstatus_t *, v, {
......@@ -1021,7 +1037,9 @@ networkstatus_compute_consensus(smartlist_t *votes,
/* Figure out the most popular opinion of what the most recent
* routerinfo and its contents are. */
rs = compute_routerstatus_consensus(matching_descs);
memset(microdesc_digest, 0, sizeof(microdesc_digest));
rs = compute_routerstatus_consensus(matching_descs, consensus_method,
microdesc_digest);
/* Copy bits of that into rs_out. */
tor_assert(!memcmp(lowest_id, rs->status.identity_digest, DIGEST_LEN));
memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN);
......@@ -1182,9 +1200,19 @@ networkstatus_compute_consensus(smartlist_t *votes,
/* Okay!! Now we can write the descriptor... */
/* First line goes into "buf". */
routerstatus_format_entry(buf, sizeof(buf), &rs_out, NULL,
NS_V3_CONSENSUS);
rs_format);
smartlist_add(chunks, tor_strdup(buf));
/* Second line is all flags. The "\n" is missing. */
/* Now an m line, if applicable. */
if (flavor == FLAV_MICRODESC &&
!tor_digest256_is_zero(microdesc_digest)) {
char m[BASE64_DIGEST256_LEN+1], *cp;
const size_t mlen = BASE64_DIGEST256_LEN+5;
digest256_to_base64(m, microdesc_digest);
cp = tor_malloc(mlen);
tor_snprintf(cp, mlen, "m %s\n", m);
smartlist_add(chunks, cp);
}
/* Next line is all flags. The "\n" is missing. */
smartlist_add(chunks,
smartlist_join_strings(chosen_flags, " ", 0, NULL));
/* Now the version line. */
......@@ -1206,7 +1234,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
};
/* Now the exitpolicy summary line. */
if (rs_out.has_exitsummary) {
if (rs_out.has_exitsummary && flavor == FLAV_NS) {
char buf[MAX_POLICY_LINE_LEN+1];
int r = tor_snprintf(buf, sizeof(buf), "p %s\n", rs_out.exitsummary);
if (r<0) {
......@@ -2090,7 +2118,8 @@ dirvote_compute_consensus(void)
consensus_body = networkstatus_compute_consensus(
votes, n_voters,
my_cert->identity_key,
get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign);
get_my_v3_authority_signing_key(), legacy_id_digest, legacy_sign,
FLAV_NS);
}
if (!consensus_body) {
log_warn(LD_DIR, "Couldn't generate a consensus at all!");
......@@ -2389,14 +2418,21 @@ dirvote_get_vote(const char *fp, int flags)
return NULL;
}
int
dirvote_create_microdescriptor(char *out, size_t outlen,
const routerinfo_t *ri)
/** Construct and return a new microdescriptor from a routerinfo <b>ri</b>.
*
* XXX Right now, there is only one way to generate microdescriptors from
* router descriptors. This may change in future consensus methods. If so,
* we'll need an internal way to remember which method we used, and ask for a
* particular method.
**/
microdesc_t *
dirvote_create_microdescriptor(const routerinfo_t *ri)
{
microdesc_t *result = NULL;
char *key = NULL, *summary = NULL, *family = NULL;
char buf[1024];
size_t keylen;
int result = -1;
char *start = out, *end = out+outlen;
char *out = buf, *end = buf+sizeof(buf);
if (crypto_pk_write_public_key_to_string(ri->onion_pkey, &key, &keylen)<0)
goto done;
......@@ -2417,7 +2453,19 @@ dirvote_create_microdescriptor(char *out, size_t outlen,
goto done;
out += strlen(out);
}
result = out - start;
*out = '\0'; /* Make sure it's nul-terminated. This should be a no-op */
{
smartlist_t *lst = microdescs_parse_from_string(buf, out, 0, 1);
if (smartlist_len(lst) != 1) {
log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse.");
SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md));
smartlist_free(lst);
goto done;
}
result = smartlist_get(lst, 0);
smartlist_free(lst);
}
done:
tor_free(key);
......@@ -2426,28 +2474,23 @@ dirvote_create_microdescriptor(char *out, size_t outlen,
return result;
}
/** Lowest consensus method that generates microdescriptors */
#define MIN_CM_MICRODESC 7
/** Cached space-separated string to hold */
static char *microdesc_consensus_methods = NULL;
/** DOCDOC */
int
dirvote_format_microdescriptor_vote_line(char *out, size_t out_len,
const char *microdesc,
size_t microdescriptor_len)
dirvote_format_microdesc_vote_line(char *out, size_t out_len,
const microdesc_t *md)
{
char d[DIGEST256_LEN];
char d64[BASE64_DIGEST256_LEN];
if (!microdesc_consensus_methods) {
microdesc_consensus_methods =
make_consensus_method_list(MIN_CM_MICRODESC,
make_consensus_method_list(MIN_METHOD_FOR_MICRODESC,
MAX_SUPPORTED_CONSENSUS_METHOD,
",");
tor_assert(microdesc_consensus_methods);
}
if (crypto_digest256(d, microdesc, microdescriptor_len, DIGEST_SHA256)<0)
return -1;
if (digest256_to_base64(d64, d)<0)
if (digest256_to_base64(d64, md->digest)<0)
return -1;
if (tor_snprintf(out, out_len, "m %s sha256=%s\n",
......@@ -2457,3 +2500,44 @@ dirvote_format_microdescriptor_vote_line(char *out, size_t out_len,
return strlen(out);
}
/** DOCDOC */
int
vote_routerstatus_find_microdesc_hash(char *digest256_out,
const vote_routerstatus_t *vrs,
int method)
{
/* XXXX only returns the sha256 method. */
const vote_microdesc_hash_t *h;
char mstr[64];
size_t mlen;
tor_snprintf(mstr, sizeof(mstr), "%d", method);
mlen = strlen(mstr);
for (h = vrs->microdesc; h; h = h->next) {
const char *cp = h->microdesc_hash_line;
size_t num_len;
/* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in
* the first part. */
while (1) {
num_len = strspn(cp, "1234567890");
if (num_len == mlen && !memcmp(mstr, cp, mlen)) {
/* This is the line. */
char buf[BASE64_DIGEST256_LEN+1];
/* XXXX ignores extraneous stuff if the digest is too long. This
* seems harmless enough, right? */
cp = strstr(cp, " sha256=");
if (!cp)
return -1;
cp += strlen(" sha256=");
strlcpy(buf, cp, sizeof(buf));
return digest256_from_base64(digest256_out, buf);
}
if (num_len == 0 || cp[num_len] != ',')
break;
cp += num_len + 1;
}
}
return -1;
}
/* Copyright (c) 2009, The Tor Project, Inc. */
/* See LICENSE for licensing information */
#include "or.h"
/** DOCDOC everything here. */
#define MICRODESC_IN_CONSENSUS 1
#define MICRODESC_IN_VOTE 2
struct microdesc_cache_t {
HT_HEAD(microdesc_map, microdesc_t) map;
char *cache_fname;
char *journal_fname;
tor_mmap_t *cache_content;
size_t journal_len;
};
static INLINE unsigned int
_microdesc_hash(microdesc_t *md)
{
unsigned *d = (unsigned*)md->digest;
#if SIZEOF_INT == 4
return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7];
#else
return d[0] ^ d[1] ^ d[2] ^ d[3];
#endif
}
static INLINE int
_microdesc_eq(microdesc_t *a, microdesc_t *b)
{
return !memcmp(a->digest, b->digest, DIGEST256_LEN);
}
HT_PROTOTYPE(microdesc_map, microdesc_t, node,
_microdesc_hash, _microdesc_eq);
HT_GENERATE(microdesc_map, microdesc_t, node,
_microdesc_hash, _microdesc_eq, 0.6,
_tor_malloc, _tor_realloc, _tor_free);
static int
dump_microdescriptor(FILE *f, microdesc_t *md)
{
/* XXXX drops unkown annotations. */
if (md->last_listed) {
char buf[ISO_TIME_LEN+1];
format_iso_time(buf, md->last_listed);
fprintf(f, "@last-listed %s\n", buf);
}
md->off = (off_t) ftell(f);
fwrite(md->body, 1, md->bodylen, f);
return 0;
}
static microdesc_cache_t *the_microdesc_cache = NULL;
microdesc_cache_t *
get_microdesc_cache(void)
{
if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t));
HT_INIT(microdesc_map, &cache->map);
cache->cache_fname = get_datadir_fname("cached-microdescs");
cache->journal_fname = get_datadir_fname("cached-microdescs.new");
microdesc_cache_reload(cache);
the_microdesc_cache = cache;
}
return the_microdesc_cache;
}
/* There are three sources of microdescriptors:
1) Generated us while acting as a directory authority.
2) Loaded from the cache on disk.
3) Downloaded.
*/
/* Returns list of added microdesc_t. */
smartlist_t *
microdescs_add_to_cache(microdesc_cache_t *cache,
const char *s, const char *eos, saved_location_t where,
int no_save)
{
/*XXXX need an argument that sets last_listed as appropriate. */
smartlist_t *descriptors, *added;
const int allow_annotations = (where != SAVED_NOWHERE);
const int copy_body = (where != SAVED_IN_CACHE);
descriptors = microdescs_parse_from_string(s, eos,
allow_annotations,
copy_body);
added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
smartlist_free(descriptors);
return added;
}
/* Returns list of added microdesc_t. Frees any not added. */
smartlist_t *
microdescs_add_list_to_cache(microdesc_cache_t *cache,
smartlist_t *descriptors, saved_location_t where,
int no_save)
{
smartlist_t *added;
open_file_t *open_file = NULL;
FILE *f = NULL;
// int n_added = 0;
if (where == SAVED_NOWHERE && !no_save) {
f = start_writing_to_stdio_file(cache->journal_fname, OPEN_FLAGS_APPEND,
0600, &open_file);
if (!f)
log_warn(LD_DIR, "Couldn't append to journal in %s",
cache->journal_fname);
}
added = smartlist_create();
SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
microdesc_t *md2;
md2 = HT_FIND(microdesc_map, &cache->map, md);
if (md2) {
/* We already had this one. */
if (md2->last_listed < md->last_listed)
md2->last_listed = md->last_listed;
microdesc_free(md);
continue;
}
/* Okay, it's a new one. */
if (f) {
dump_microdescriptor(f, md);
md->saved_location = SAVED_IN_JOURNAL;
} else {
md->saved_location = where;
}
md->no_save = no_save;
HT_INSERT(microdesc_map, &cache->map, md);
smartlist_add(added, md);
} SMARTLIST_FOREACH_END(md);
finish_writing_to_file(open_file); /*XXX Check me.*/
return added;
}
void
microdesc_cache_clear(microdesc_cache_t *cache)
{
microdesc_t **entry, **next;
for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
microdesc_free(*entry);
}
if (cache->cache_content) {
tor_munmap_file(cache->cache_content);
cache->cache_content = NULL;
}
}
int
microdesc_cache_reload(microdesc_cache_t *cache)
{
struct stat st;
char *journal_content;
smartlist_t *added;
tor_mmap_t *mm;
int total = 0;
microdesc_cache_clear(cache);