Code to download, parse, and store microdesc consensuses (4e0f7f4f) · Commits · Legacy / gitolite / user / hiviah / rpm-tor

changes/microdesc_dl

0 → 100644

+4 −0

Changes for changes/microdesc_dl: 4 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		o Major features:
		- Caches now download and cache all the consensus flavors that
		they know about. This allows them to assess which microdescriptors
		they need to fetch.

src/or/directory.c

+40 −17

Changes for src/or/directory.c: 40 added lines, 17 removed lines.
Original line number	Diff line number	Diff line
		@@ -361,9 +361,24 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose,
		}

		if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
		networkstatus_t *v = networkstatus_get_latest_consensus();
		int flav = FLAV_NS;
		networkstatus_t *v;
		if (resource)
		flav = networkstatus_parse_flavor_name(resource);

		if (flav != -1) {
		/* IF we have a parsed consensus of this type, we can do an
		* if-modified-time based on it. */
		v = networkstatus_get_latest_consensus_by_flavor(flav);
		if (v)
		if_modified_since = v->valid_after + 180;
		} else {
		/* Otherwise it might be a consensus we don't parse, but which we
		* do cache. Look at the cached copy, perhaps. */
		cached_dir_t *cd = dirserv_get_consensus(resource ? resource : "ns");
		if (cd)
		if_modified_since = cd->published + 180;
		}
		}

		if (!options->FetchServerDescriptors && type != HIDSERV_AUTHORITY)
		@@ -598,7 +613,9 @@ connection_dir_request_failed(dir_connection_t *conn)
		connection_dir_bridge_routerdesc_failed(conn);
		connection_dir_download_routerdesc_failed(conn);
		} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
		networkstatus_consensus_download_failed(0);
		const char *flavname =
		conn->requested_resource ? conn->requested_resource : "ns";
		networkstatus_consensus_download_failed(0, flavname);
		} else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
		log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
		conn->_base.address);
		@@ -955,12 +972,16 @@ _compare_strs(const void a, const void b)
		* This url depends on whether or not the server we go to
		* is sufficiently new to support conditional consensus downloading,
		* i.e. GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
		*
		* If 'resource' is provided, it is the name of a consensus flavor to request.
		*/
		static char *
		directory_get_consensus_url(int supports_conditional_consensus)
		directory_get_consensus_url(int supports_conditional_consensus,
		const char *resource)
		{
		char *url;
		size_t len;
		char *url = NULL;
		const char *hyphen = resource ? "-" : "";
		const char *flavor = resource ? resource : "";

		if (supports_conditional_consensus) {
		char *authority_id_list;
		@@ -982,16 +1003,15 @@ directory_get_consensus_url(int supports_conditional_consensus)
		authority_id_list = smartlist_join_strings(authority_digests,
		"+", 0, NULL);

		len = strlen(authority_id_list)+64;
		url = tor_malloc(len);
		tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z",
		authority_id_list);
		tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
		hyphen, flavor, authority_id_list);

		SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
		smartlist_free(authority_digests);
		tor_free(authority_id_list);
		} else {
		url = tor_strdup("/tor/status-vote/current/consensus.z");
		tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s.z",
		hyphen, flavor);
		}
		return url;
		}
		@@ -1072,10 +1092,11 @@ directory_send_command(dir_connection_t *conn,
		tor_snprintf(url, len, "/tor/status/%s", resource);
		break;
		case DIR_PURPOSE_FETCH_CONSENSUS:
		tor_assert(!resource);
		/* resource is optional. If present, it's a flavor name */
		tor_assert(!payload);
		httpcommand = "GET";
		url = directory_get_consensus_url(supports_conditional_consensus);
		url = directory_get_consensus_url(supports_conditional_consensus,
		resource);
		log_info(LD_DIR, "Downloading consensus from %s using %s",
		hoststring, url);
		break;
		@@ -1690,6 +1711,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn)

		if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
		int r;
		const char *flavname =
		conn->requested_resource ? conn->requested_resource : "ns";
		if (status_code != 200) {
		int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
		log(severity, LD_DIR,
		@@ -1698,18 +1721,18 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
		status_code, escaped(reason), conn->_base.address,
		conn->_base.port);
		tor_free(body); tor_free(headers); tor_free(reason);
		networkstatus_consensus_download_failed(status_code);
		networkstatus_consensus_download_failed(status_code, flavname);
		return -1;
		}
		log_info(LD_DIR,"Received consensus directory (size %d) from server "
		"'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
		if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) {
		if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) {
		log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
		"Unable to load consensus directory downloaded from "
		"Unable to load %s consensus directory downloaded from "
		"server '%s:%d'. I'll try again soon.",
		conn->_base.address, conn->_base.port);
		flavname, conn->_base.address, conn->_base.port);
		tor_free(body); tor_free(headers); tor_free(reason);
		networkstatus_consensus_download_failed(0);
		networkstatus_consensus_download_failed(0, flavname);
		return -1;
		}
		/* launches router downloads as needed */

src/or/dirserv.c

+2 −1

Changes for src/or/dirserv.c: 2 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -1263,7 +1263,8 @@ static cached_dir_t cached_runningrouters;
		* cached_dir_t. */
		static digestmap_t *cached_v2_networkstatus = NULL;

		/** Map from flavor name to the v3 consensuses that we're currently serving. */
		/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
		* currently serving. */
		static strmap_t *cached_consensuses = NULL;

		/** Possibly replace the contents of <b>d</b> with the value of

src/or/networkstatus.c

+116 −37

Changes for src/or/networkstatus.c: 116 added lines, 37 removed lines.
Original line number	Diff line number	Diff line
		@@ -44,8 +44,19 @@ static strmap_t *named_server_map = NULL;
		* as unnamed for some server in the consensus. */
		static strmap_t *unnamed_server_map = NULL;

		/** Most recently received and validated v3 consensus network status. */
		static networkstatus_t *current_consensus = NULL;
		/** Most recently received and validated v3 consensus network status,
		* of whichever type we are using for our own circuits. This will be the same
		* as one of current_ns_consensus or current_md_consensus.
		*/
		#define current_consensus current_ns_consensus

		/** Most recently received and validated v3 "ns"-flavored consensus network
		* status. */
		static networkstatus_t *current_ns_consensus = NULL;

		/** Most recently received and validated v3 "microdec"-flavored consensus
		* network status. */
		static networkstatus_t *current_md_consensus = NULL;

		/** A v3 consensus networkstatus that we've received, but which we don't
		* have enough certificates to be happy about. */
		@@ -1199,6 +1210,24 @@ update_v2_networkstatus_cache_downloads(time_t now)
		}
		}

		static int
		we_want_to_fetch_flavor(or_options_t *options, int flavor)
		{
		if (flavor < 0 \|\| flavor > N_CONSENSUS_FLAVORS) {
		/* This flavor is crazy; we don't want it */
		/XXXX handle unrecognized flavors later /
		return 0;
		}
		if (authdir_mode_v3(options) \|\| directory_caches_dir_info(options)) {
		/* We want to serve all flavors to others, regardless if we would use
		* it ourselves. */
		return 1;
		}
		/* Otherwise, we want the flavor only if we want to use it to build
		* circuits. */
		return (flavor == USABLE_CONSENSUS_FLAVOR);
		}

		/** How many times will we try to fetch a consensus before we give up? */
		#define CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES 8
		/** How long will we hang onto a possibly live consensus for which we're
		@@ -1211,49 +1240,66 @@ static void
		update_consensus_networkstatus_downloads(time_t now)
		{
		int i;
		or_options_t *options = get_options();

		if (!networkstatus_get_live_consensus(now))
		time_to_download_next_consensus = now; /* No live consensus? Get one now!*/
		if (time_to_download_next_consensus > now)
		return; /* Wait until the current consensus is older. */
		/* XXXXNM Microdescs: may need to download more types. */
		if (!download_status_is_ready(&consensus_dl_status[FLAV_NS], now,
		CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
		return; /* We failed downloading a consensus too recently. */
		if (connection_get_by_type_purpose(CONN_TYPE_DIR,
		DIR_PURPOSE_FETCH_CONSENSUS))
		return; /* There's an in-progress download.*/

		for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
		consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];
		/* XXXX need some way to download unknown flavors if we are caching. */
		const char *resource;
		consensus_waiting_for_certs_t *waiting;

		if (! we_want_to_fetch_flavor(options, i))
		continue;

		resource = i==FLAV_NS ? NULL : networkstatus_get_flavor_name(i);

		if (!download_status_is_ready(&consensus_dl_status[i], now,
		CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES))
		continue; /* We failed downloading a consensus too recently. */
		if (connection_dir_get_by_purpose_and_resource(
		DIR_PURPOSE_FETCH_CONSENSUS, resource))
		continue; /* There's an in-progress download.*/

		waiting = &consensus_waiting_for_certs[i];
		if (waiting->consensus) {
		/* XXXX make sure this doesn't delay sane downloads. */
		if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now)
		return; /* We're still getting certs for this one. */
		else {
		if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) {
		continue; /* We're still getting certs for this one. */
		} else {
		if (!waiting->dl_failed) {
		download_status_failed(&consensus_dl_status[FLAV_NS], 0);
		download_status_failed(&consensus_dl_status[i], 0);
		waiting->dl_failed=1;
		}
		}
		}
		}

		log_info(LD_DIR, "Launching networkstatus consensus download.");
		log_info(LD_DIR, "Launching %s networkstatus consensus download.",
		networkstatus_get_flavor_name(i));

		directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS,
		ROUTER_PURPOSE_GENERAL, NULL,
		ROUTER_PURPOSE_GENERAL, resource,
		PDS_RETRY_IF_NO_SERVERS);
		}
		}

		/** Called when an attempt to download a consensus fails: note that the
		* failure occurred, and possibly retry. */
		void
		networkstatus_consensus_download_failed(int status_code)
		networkstatus_consensus_download_failed(int status_code, const char *flavname)
		{
		/* XXXXNM Microdescs: may need to handle more types. */
		download_status_failed(&consensus_dl_status[FLAV_NS], status_code);
		int flav = networkstatus_parse_flavor_name(flavname);
		if (flav >= 0) {
		tor_assert(flav < N_CONSENSUS_FLAVORS);
		/* XXXX handle unrecognized flavors */
		download_status_failed(&consensus_dl_status[flav], status_code);
		/* Retry immediately, if appropriate. */
		update_consensus_networkstatus_downloads(time(NULL));
		}
		}

		/** How long do we (as a cache) wait after a consensus becomes non-fresh
		* before trying to fetch another? */
		@@ -1373,7 +1419,10 @@ update_certificate_downloads(time_t now)
		now);
		}

		authority_certs_fetch_missing(current_consensus, now);
		if (current_ns_consensus)
		authority_certs_fetch_missing(current_ns_consensus, now);
		if (current_ns_consensus)
		authority_certs_fetch_missing(current_md_consensus, now);
		}

		/** Return 1 if we have a consensus but we don't have enough certificates
		@@ -1405,6 +1454,18 @@ networkstatus_get_latest_consensus(void)
		return current_consensus;
		}

		/** DOCDOC */
		networkstatus_t *
		networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
		{
		if (f == FLAV_NS)
		return current_ns_consensus;
		else if (f == FLAV_MICRODESC)
		return current_md_consensus;
		else
		tor_assert(0);
		}

		/** Return the most recent consensus that we have downloaded, or NULL if it is
		* no longer live. */
		networkstatus_t *
		@@ -1569,6 +1630,7 @@ networkstatus_set_current_consensus(const char *consensus,
		const digests_t *current_digests = NULL;
		consensus_waiting_for_certs_t *waiting = NULL;
		time_t current_valid_after = 0;
		int free_consensus = 1;

		if (flav < 0) {
		/* XXXX we don't handle unrecognized flavors yet. */
		@@ -1614,9 +1676,16 @@ networkstatus_set_current_consensus(const char *consensus,
		if (!strcmp(flavor, "ns")) {
		consensus_fname = get_datadir_fname("cached-consensus");
		unverified_fname = get_datadir_fname("unverified-consensus");
		if (current_consensus) {
		current_digests = &current_consensus->digests;
		current_valid_after = current_consensus->valid_after;
		if (current_ns_consensus) {
		current_digests = &current_ns_consensus->digests;
		current_valid_after = current_ns_consensus->valid_after;
		}
		} else if (!strcmp(flavor, "microdesc")) {
		consensus_fname = get_datadir_fname("cached-microdesc-consensus");
		unverified_fname = get_datadir_fname("unverified-microdesc-consensus");
		if (current_md_consensus) {
		current_digests = &current_md_consensus->digests;
		current_valid_after = current_md_consensus->valid_after;
		}
		} else {
		cached_dir_t *cur;
		@@ -1702,11 +1771,21 @@ networkstatus_set_current_consensus(const char *consensus,

		if (flav == USABLE_CONSENSUS_FLAVOR) {
		notify_control_networkstatus_changed(current_consensus, c);

		if (current_consensus) {
		networkstatus_copy_old_consensus_info(c, current_consensus);
		networkstatus_vote_free(current_consensus);
		}
		if (flav == FLAV_NS) {
		if (current_ns_consensus) {
		networkstatus_copy_old_consensus_info(c, current_ns_consensus);
		networkstatus_vote_free(current_ns_consensus);
		}
		current_ns_consensus = c;
		free_consensus = 0; /* avoid free */
		} else if (flav == FLAV_MICRODESC) {
		if (current_md_consensus) {
		networkstatus_copy_old_consensus_info(c, current_md_consensus);
		networkstatus_vote_free(current_md_consensus);
		}
		current_md_consensus = c;
		free_consensus = 0; /* avoid free */
		}

		waiting = &consensus_waiting_for_certs[flav];
		@@ -1739,11 +1818,9 @@ networkstatus_set_current_consensus(const char *consensus,
		}

		if (flav == USABLE_CONSENSUS_FLAVOR) {
		current_consensus = c;
		c = NULL; /* Prevent free. */

		/* XXXXNM Microdescs: needs a non-ns variant. */
		update_consensus_networkstatus_fetch_time(now);

		dirvote_recalculate_timing(options, now);
		routerstatus_list_update_named_server_map();
		cell_ewma_set_scale_factor(options, current_consensus);
		@@ -1758,11 +1835,11 @@ networkstatus_set_current_consensus(const char *consensus,
		write_str_to_file(consensus_fname, consensus, 0);
		}

		if (ftime_definitely_before(now, current_consensus->valid_after)) {
		if (ftime_definitely_before(now, c->valid_after)) {
		char tbuf[ISO_TIME_LEN+1];
		char dbuf[64];
		long delta = now - current_consensus->valid_after;
		format_iso_time(tbuf, current_consensus->valid_after);
		long delta = now - c->valid_after;
		format_iso_time(tbuf, c->valid_after);
		format_time_interval(dbuf, sizeof(dbuf), delta);
		log_warn(LD_GENERAL, "Our clock is %s behind the time published in the "
		"consensus network status document (%s GMT). Tor needs an "
		@@ -1776,6 +1853,7 @@ networkstatus_set_current_consensus(const char *consensus,

		result = 0;
		done:
		if (free_consensus)
		networkstatus_vote_free(c);
		tor_free(consensus_fname);
		tor_free(unverified_fname);
		@@ -2028,7 +2106,7 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers,
		void
		signed_descs_update_status_from_consensus_networkstatus(smartlist_t *descs)
		{
		networkstatus_t *ns = current_consensus;
		networkstatus_t *ns = current_ns_consensus;
		if (!ns)
		return;

		@@ -2264,8 +2342,9 @@ networkstatus_free_all(void)

		digestmap_free(v2_download_status_map, _tor_free);
		v2_download_status_map = NULL;
		networkstatus_vote_free(current_consensus);
		current_consensus = NULL;
		networkstatus_vote_free(current_ns_consensus);
		networkstatus_vote_free(current_md_consensus);
		current_md_consensus = current_ns_consensus = NULL;

		for (i=0; i < N_CONSENSUS_FLAVORS; ++i) {
		consensus_waiting_for_certs_t *waiting = &consensus_waiting_for_certs[i];

src/or/networkstatus.h

+4 −1

Changes for src/or/networkstatus.h: 4 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -53,7 +53,8 @@ routerstatus_t router_get_consensus_status_by_nickname(const char nickname,
		int warn_if_unnamed);
		const char networkstatus_get_router_digest_by_nickname(const char nickname);
		int networkstatus_nickname_is_unnamed(const char *nickname);
		void networkstatus_consensus_download_failed(int status_code);
		void networkstatus_consensus_download_failed(int status_code,
		const char *flavname);
		void update_consensus_networkstatus_fetch_time(time_t now);
		int should_delay_dir_fetches(or_options_t *options);
		void update_networkstatus_downloads(time_t now);
		@@ -61,6 +62,8 @@ void update_certificate_downloads(time_t now);
		int consensus_is_waiting_for_certs(void);
		networkstatus_v2_t networkstatus_v2_get_by_digest(const char digest);
		networkstatus_t *networkstatus_get_latest_consensus(void);
		networkstatus_t *networkstatus_get_latest_consensus_by_flavor(
		consensus_flavor_t f);
		networkstatus_t *networkstatus_get_live_consensus(time_t now);
		networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now);
		#define NSSET_FROM_CACHE 1