Merge branch 'microdesc_dl_v2' (d6e255ed) · Commits · The Tor Project / Core / Tor

changes/microdesc_dl

0 → 100644

+4 −0

Original line number	Diff line number	Diff line
		o Major features:
		- Caches now download and cache all the consensus flavors that
		they know about. This allows them to assess which microdescriptors
		they need to fetch.

src/or/connection.c

+27 −0

Original line number	Diff line number	Diff line
		@@ -3397,6 +3397,33 @@ connection_get_by_type_state_rendquery(int type, int state,
		return NULL;
		}

		/** Return a directory connection (if any one exists) that is fetching
		* the item described by <b>state</b>/<b>resource</b> */
		dir_connection_t *
		connection_dir_get_by_purpose_and_resource(int purpose,
		const char *resource)
		{
		smartlist_t *conns = get_connection_array();

		SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
		dir_connection_t *dirconn;
		if (conn->type != CONN_TYPE_DIR \|\| conn->marked_for_close \|\|
		conn->purpose != purpose)
		continue;
		dirconn = TO_DIR_CONN(conn);
		if (dirconn->requested_resource == NULL) {
		if (resource == NULL)
		return dirconn;
		} else if (resource) {
		if (0 == strcmp(resource, dirconn->requested_resource))
		return dirconn;
		}
		} SMARTLIST_FOREACH_END(conn);

		return NULL;
		}


		/** Return an open, non-marked connection of a given type and purpose, or NULL
		* if no such connection exists. */
		connection_t *

src/or/connection.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -124,6 +124,8 @@ connection_t *connection_get_by_type_addr_port_purpose(int type,
		connection_t *connection_get_by_type_state(int type, int state);
		connection_t *connection_get_by_type_state_rendquery(int type, int state,
		const char *rendquery);
		dir_connection_t *connection_dir_get_by_purpose_and_resource(
		int state, const char *resource);

		#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR)
		int connection_is_listener(connection_t *conn);

src/or/directory.c

+143 −26

Original line number	Diff line number	Diff line
		@@ -15,6 +15,7 @@
		#include "dirvote.h"
		#include "geoip.h"
		#include "main.h"
		#include "microdesc.h"
		#include "networkstatus.h"
		#include "policies.h"
		#include "rendclient.h"
		@@ -78,6 +79,8 @@ static void dir_routerdesc_download_failed(smartlist_t *failed,
		int router_purpose,
		int was_extrainfo,
		int was_descriptor_digests);
		static void dir_microdesc_download_failed(smartlist_t *failed,
		int status_code);
		static void note_client_request(int purpose, int compressed, size_t bytes);
		static int client_likes_consensus(networkstatus_t v, const char want_url);

		@@ -137,7 +140,8 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
		dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS \|\|
		dir_purpose == DIR_PURPOSE_FETCH_CERTIFICATE \|\|
		dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC \|\|
		dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO)
		dir_purpose == DIR_PURPOSE_FETCH_EXTRAINFO \|\|
		dir_purpose == DIR_PURPOSE_FETCH_MICRODESC)
		return 0;
		return 1;
		}
		@@ -201,6 +205,8 @@ dir_conn_purpose_to_string(int purpose)
		return "hidden-service v2 descriptor fetch";
		case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
		return "hidden-service v2 descriptor upload";
		case DIR_PURPOSE_FETCH_MICRODESC:
		return "microdescriptor fetch";
		}

		log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
		@@ -355,15 +361,33 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
		case DIR_PURPOSE_FETCH_CERTIFICATE:
		type = V3_AUTHORITY;
		break;
		case DIR_PURPOSE_FETCH_MICRODESC:
		type = V3_AUTHORITY;
		break;
		default:
		log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
		return;
		}

		if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
		networkstatus_t *v = networkstatus_get_latest_consensus();
		int flav = FLAV_NS;
		networkstatus_t *v;
		if (resource)
		flav = networkstatus_parse_flavor_name(resource);

		if (flav != -1) {
		/* IF we have a parsed consensus of this type, we can do an
		* if-modified-time based on it. */
		v = networkstatus_get_latest_consensus_by_flavor(flav);
		if (v)
		if_modified_since = v->valid_after + 180;
		} else {
		/* Otherwise it might be a consensus we don't parse, but which we
		* do cache. Look at the cached copy, perhaps. */
		cached_dir_t *cd = dirserv_get_consensus(resource ? resource : "ns");
		if (cd)
		if_modified_since = cd->published + 180;
		}
		}

		if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
		@@ -395,7 +419,8 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
		if (prefer_authority \|\| type == BRIDGE_AUTHORITY) {
		/* only ask authdirservers, and don't ask myself */
		rs = router_pick_trusteddirserver(type, pds_flags);
		if (rs == NULL && (pds_flags & PDS_NO_EXISTING_SERVERDESC_FETCH)) {
		if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH\|
		PDS_NO_EXISTING_MICRODESC_FETCH))) {
		/* We don't want to fetch from any authorities that we're currently
		* fetching server descriptors from, and we got no match. Did we
		* get no match because all the authorities have connections
		@@ -403,7 +428,8 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
		* return,) or because all the authorities are down or on fire or
		* unreachable or something (in which case we should go on with
		* our fallback code)? */
		pds_flags &= ~PDS_NO_EXISTING_SERVERDESC_FETCH;
		pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH\|
		PDS_NO_EXISTING_MICRODESC_FETCH);
		rs = router_pick_trusteddirserver(type, pds_flags);
		if (rs) {
		log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
		@@ -592,15 +618,19 @@ connection_dir_request_failed(dir_connection_t *conn)
		connection_dir_download_networkstatus_failed(conn, -1);
		} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC \|\|
		conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
		log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
		log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
		"directory server at '%s'; retrying",
		conn->_base.address);
		if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
		connection_dir_bridge_routerdesc_failed(conn);
		connection_dir_download_routerdesc_failed(conn);
		} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
		networkstatus_consensus_download_failed(0);
		const char *flavname =
		conn->requested_resource ? conn->requested_resource : "ns";
		networkstatus_consensus_download_failed(0, flavname);
		} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
		log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
		log_info(LD_DIR, "Giving up on certificate fetch from directory server "
		"at '%s'; retrying",
		conn->_base.address);
		connection_dir_download_cert_failed(conn, 0);
		} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
		@@ -609,6 +639,10 @@ connection_dir_request_failed(dir_connection_t *conn)
		} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
		log_info(LD_DIR, "Giving up downloading votes from '%s'",
		conn->_base.address);
		} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
		log_info(LD_DIR, "Giving up on downloading microdescriptors from "
		" directory server at '%s'; will retry", conn->_base.address);
		connection_dir_download_routerdesc_failed(conn);
		}
		}

		@@ -679,7 +713,8 @@ connection_dir_download_routerdesc_failed(dir_connection_t *conn)
		/* No need to relaunch descriptor downloads here: we already do it
		* every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
		tor_assert(conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC \|\|
		conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
		conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO \|\|
		conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC);

		(void) conn;
		}
		@@ -955,12 +990,16 @@ _compare_strs(const void a, const void b)
		* This url depends on whether or not the server we go to
		* is sufficiently new to support conditional consensus downloading,
		* i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
		*
		* If 'resource' is provided, it is the name of a consensus flavor to request.
		*/
		static char *
		directory_get_consensus_url(int supports_conditional_consensus)
		directory_get_consensus_url(int supports_conditional_consensus,
		const char *resource)
		{
		char *url;
		size_t len;
		char *url = NULL;
		const char *hyphen = resource ? "-" : "";
		const char *flavor = resource ? resource : "";

		if (supports_conditional_consensus) {
		char *authority_id_list;
		@@ -982,16 +1021,15 @@ directory_get_consensus_url(int supports_conditional_consensus)
		authority_id_list = smartlist_join_strings(authority_digests,
		"+", 0, NULL);

		len = strlen(authority_id_list)+64;
		url = tor_malloc(len);
		tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z",
		authority_id_list);
		tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
		hyphen, flavor, authority_id_list);

		SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
		smartlist_free(authority_digests);
		tor_free(authority_id_list);
		} else {
		url = tor_strdup("/tor/status-vote/current/consensus.z");
		tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s.z",
		hyphen, flavor);
		}
		return url;
		}
		@@ -1072,10 +1110,11 @@ directory_send_command(dir_connection_t *conn,
		tor_snprintf(url, len, "/tor/status/%s", resource);
		break;
		case DIR_PURPOSE_FETCH_CONSENSUS:
		tor_assert(!resource);
		/* resource is optional. If present, it's a flavor name */
		tor_assert(!payload);
		httpcommand = "GET";
		url = directory_get_consensus_url(supports_conditional_consensus);
		url = directory_get_consensus_url(supports_conditional_consensus,
		resource);
		log_info(LD_DIR, "Downloading consensus from %s using %s",
		hoststring, url);
		break;
		@@ -1115,6 +1154,11 @@ directory_send_command(dir_connection_t *conn,
		url = tor_malloc(len);
		tor_snprintf(url, len, "/tor/extra/%s", resource);
		break;
		case DIR_PURPOSE_FETCH_MICRODESC:
		tor_assert(resource);
		httpcommand = "GET";
		tor_asprintf(&url, "/tor/micro/%s.z", resource);
		break;
		case DIR_PURPOSE_UPLOAD_DIR:
		tor_assert(!resource);
		tor_assert(payload);
		@@ -1390,6 +1434,9 @@ body_is_plausible(const char *body, size_t len, int purpose)
		return 1; /* empty bodies don't need decompression */
		if (len < 32)
		return 0;
		if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
		return (!strcmpstart(body,"onion-key"));
		}
		if (purpose != DIR_PURPOSE_FETCH_RENDDESC) {
		if (!strcmpstart(body,"router") \|\|
		!strcmpstart(body,"signed-directory") \|\|
		@@ -1466,7 +1513,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
		int plausible;
		int skewed=0;
		int allow_partial = (conn->_base.purpose == DIR_PURPOSE_FETCH_SERVERDESC \|\|
		conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
		conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO \|\|
		conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC);
		int was_compressed=0;
		time_t now = time(NULL);

		@@ -1690,6 +1738,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)

		if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
		int r;
		const char *flavname =
		conn->requested_resource ? conn->requested_resource : "ns";
		if (status_code != 200) {
		int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
		log(severity, LD_DIR,
		@@ -1698,22 +1748,24 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
		status_code, escaped(reason), conn->_base.address,
		conn->_base.port);
		tor_free(body); tor_free(headers); tor_free(reason);
		networkstatus_consensus_download_failed(status_code);
		networkstatus_consensus_download_failed(status_code, flavname);
		return -1;
		}
		log_info(LD_DIR,"Received consensus directory (size %d) from server "
		"'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
		if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) {
		if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) {
		log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
		"Unable to load consensus directory downloaded from "
		"Unable to load %s consensus directory downloaded from "
		"server '%s:%d'. I'll try again soon.",
		conn->_base.address, conn->_base.port);
		flavname, conn->_base.address, conn->_base.port);
		tor_free(body); tor_free(headers); tor_free(reason);
		networkstatus_consensus_download_failed(0);
		networkstatus_consensus_download_failed(0, flavname);
		return -1;
		}
		/* launches router downloads as needed */
		routers_update_all_from_networkstatus(now, 3);
		update_microdescs_from_networkstatus(now);
		update_microdesc_downloads(now);
		directory_info_has_arrived(now, 0);
		log_info(LD_DIR, "Successfully loaded consensus.");
		}
		@@ -1863,6 +1915,41 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
		if (directory_conn_is_self_reachability_test(conn))
		router_dirport_found_reachable();
		}
		if (conn->_base.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
		smartlist_t *which = NULL;
		log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
		"size %d) from server '%s:%d'",
		status_code, (int)body_len, conn->_base.address, conn->_base.port);
		tor_assert(conn->requested_resource &&
		!strcmpstart(conn->requested_resource, "d/"));
		which = smartlist_create();
		dir_split_resource_into_fingerprints(conn->requested_resource+2,
		which, NULL,
		DSR_DIGEST256\|DSR_BASE64);
		if (status_code != 200) {
		log_info(LD_DIR, "Received status code %d (%s) from server "
		"'%s:%d' while fetching \"/tor/micro/%s\". I'll try again "
		"soon.",
		status_code, escaped(reason), conn->_base.address,
		(int)conn->_base.port, conn->requested_resource);
		dir_microdesc_download_failed(which, status_code);
		SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
		smartlist_free(which);
		tor_free(body); tor_free(headers); tor_free(reason);
		return 0;
		} else {
		smartlist_t *mds;
		mds = microdescs_add_to_cache(get_microdesc_cache(),
		body, body+body_len, SAVED_NOWHERE, 0,
		now, which);
		if (smartlist_len(which)) {
		/* Mark remaining ones as failed. */
		dir_microdesc_download_failed(which, status_code);
		}
		SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
		smartlist_free(which);
		}
		}

		if (conn->_base.purpose == DIR_PURPOSE_UPLOAD_DIR) {
		switch (status_code) {
		@@ -3589,6 +3676,36 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
		* every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
		}

		/* DOCDOC NM */
		static void
		dir_microdesc_download_failed(smartlist_t *failed,
		int status_code)
		{
		networkstatus_t *consensus
		= networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
		routerstatus_t *rs;
		download_status_t *dls;
		time_t now = time(NULL);
		int server = directory_fetches_from_authorities(get_options());

		if (! consensus)
		return;
		SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
		rs = router_get_consensus_status_by_descriptor_digest(consensus, d);
		if (!rs)
		continue;
		dls = &rs->dl_status;
		if (dls->n_download_failures >= MAX_MICRODESC_DOWNLOAD_FAILURES)
		continue;
		{
		char buf[BASE64_DIGEST256_LEN+1];
		digest256_to_base64(buf, d);
		download_status_increment_failure(dls, status_code, buf,
		server, now);
		}
		} SMARTLIST_FOREACH_END(d);
		}

		/** Helper. Compare two fp_pair_t objects, and return -1, 0, or 1 as
		* appropriate. */
		static int

src/or/dirserv.c

+2 −1

Original line number	Diff line number	Diff line
		@@ -1263,7 +1263,8 @@ static cached_dir_t cached_runningrouters;
		* cached_dir_t. */
		static digestmap_t *cached_v2_networkstatus = NULL;

		/** Map from flavor name to the v3 consensuses that we're currently serving. */
		/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
		* currently serving. */
		static strmap_t *cached_consensuses = NULL;

		/** Possibly replace the contents of <b>d</b> with the value of