Merge branch 'download-schedule' into 'main' (5484bcc2) · Commits · The Tor Project / Core / Arti

crates/arti-client/src/config.rs

+9 −5

Original line number	Original line	Diff line number	Diff line
	@@ -341,8 +341,8 @@ impl TryInto<dir::DirMgrConfig> for &TorClientConfig {
	#[rustfmt::skip]		#[rustfmt::skip]
	fn try_into(self) -> Result<dir::DirMgrConfig, ConfigBuildError> {		fn try_into(self) -> Result<dir::DirMgrConfig, ConfigBuildError> {
	Ok(dir::DirMgrConfig {		Ok(dir::DirMgrConfig {
	network_config: self.tor_network .clone(),		network: self.tor_network .clone(),
	schedule_config: self.download_schedule .clone(),		schedule: self.download_schedule .clone(),
	cache_path: self.storage.expand_cache_dir()?,		cache_path: self.storage.expand_cache_dir()?,
	override_net_params: self.override_net_params.clone(),		override_net_params: self.override_net_params.clone(),
	extensions: Default::default(),		extensions: Default::default(),
	@@ -383,7 +383,6 @@ mod test {

	#[test]		#[test]
	fn builder() {		fn builder() {
	use tor_dirmgr::DownloadSchedule;
	let sec = std::time::Duration::from_secs(1);		let sec = std::time::Duration::from_secs(1);

	let auth = dir::Authority::builder()		let auth = dir::Authority::builder()
	@@ -402,9 +401,14 @@ mod test {
	bld.storage()		bld.storage()
	.cache_dir(CfgPath::new("/var/tmp/foo".to_owned()))		.cache_dir(CfgPath::new("/var/tmp/foo".to_owned()))
	.state_dir(CfgPath::new("/var/tmp/bar".to_owned()));		.state_dir(CfgPath::new("/var/tmp/bar".to_owned()));
			bld.download_schedule().retry_certs().attempts(10);
			bld.download_schedule().retry_certs().initial_delay(sec);
			bld.download_schedule().retry_certs().parallelism(3);
			bld.download_schedule().retry_microdescs().attempts(30);
	bld.download_schedule()		bld.download_schedule()
	.retry_certs(DownloadSchedule::new(10, sec, 3))		.retry_microdescs()
	.retry_microdescs(DownloadSchedule::new(30, 10 * sec, 9));		.initial_delay(10 * sec);
			bld.download_schedule().retry_microdescs().parallelism(9);
	bld.override_net_params()		bld.override_net_params()
	.insert("wombats-per-quokka".to_owned(), 7);		.insert("wombats-per-quokka".to_owned(), 7);
	bld.path_rules()		bld.path_rules()

crates/arti-config/src/arti_defaults.toml

+4 −4

Original line number	Original line	Diff line number	Diff line
	@@ -83,16 +83,16 @@ state_dir = "${ARTI_LOCAL_DATA}"
	[download_schedule]		[download_schedule]

	# How to retry our initial bootstrapping when we're trying to start up.		# How to retry our initial bootstrapping when we're trying to start up.
	retry_bootstrap = { num_retries = 128, initial_delay = "1 sec" }		retry_bootstrap = { attempts = 128, initial_delay = "1 sec" }

	# How to retry a single consensus download.		# How to retry a single consensus download.
	retry_consensus = { num_retries = 3, initial_delay = "1 sec" }		retry_consensus = { attempts = 3, initial_delay = "1 sec" }

	# How to retry a set of authority certificate downloads.		# How to retry a set of authority certificate downloads.
	retry_certs = { num_retries = 3, initial_delay = "1 sec" }		retry_certs = { attempts = 3, initial_delay = "1 sec" }

	# How to retry a set of microdescriptor downloads.		# How to retry a set of microdescriptor downloads.
	retry_microdescs = { num_retries = 3, initial_delay = "1 sec", parallelism = 4 }		retry_microdescs = { attempts = 3, initial_delay = "1 sec", parallelism = 4 }

	# Tells the circuit manager rule for constructing circuit paths		# Tells the circuit manager rule for constructing circuit paths
	[path_rules]		[path_rules]

crates/arti/src/cfg.rs

+16 −3

Original line number	Original line	Diff line number	Diff line
	@@ -219,7 +219,6 @@ mod test {

	#[test]		#[test]
	fn builder() {		fn builder() {
	use arti_client::config::dir::DownloadSchedule;
	use tor_config::CfgPath;		use tor_config::CfgPath;
	let sec = std::time::Duration::from_secs(1);		let sec = std::time::Duration::from_secs(1);

	@@ -245,10 +244,24 @@ mod test {
	.storage()		.storage()
	.cache_dir(CfgPath::new("/var/tmp/foo".to_owned()))		.cache_dir(CfgPath::new("/var/tmp/foo".to_owned()))
	.state_dir(CfgPath::new("/var/tmp/bar".to_owned()));		.state_dir(CfgPath::new("/var/tmp/bar".to_owned()));
			bld.tor().download_schedule().retry_certs().attempts(10);
	bld.tor()		bld.tor()
	.download_schedule()		.download_schedule()
	.retry_certs(DownloadSchedule::new(10, sec, 3))		.retry_certs()
	.retry_microdescs(DownloadSchedule::new(30, 10 * sec, 9));		.initial_delay(sec);
			bld.tor().download_schedule().retry_certs().parallelism(3);
			bld.tor()
			.download_schedule()
			.retry_microdescs()
			.attempts(30);
			bld.tor()
			.download_schedule()
			.retry_microdescs()
			.initial_delay(10 * sec);
			bld.tor()
			.download_schedule()
			.retry_microdescs()
			.parallelism(9);
	bld.tor()		bld.tor()
	.override_net_params()		.override_net_params()
	.insert("wombats-per-quokka".to_owned(), 7);		.insert("wombats-per-quokka".to_owned(), 7);

crates/tor-dirmgr/src/config.rs

+52 −86

Original line number	Original line	Diff line number	Diff line
	@@ -8,10 +8,10 @@
	//! The types in this module are re-exported from `arti-client`: any changes		//! The types in this module are re-exported from `arti-client`: any changes
	//! here must be reflected in the version of `arti-client`.		//! here must be reflected in the version of `arti-client`.

	use crate::authority::AuthorityList;		use crate::authority::{Authority, AuthorityList};
	use crate::retry::DownloadSchedule;		use crate::retry::{DownloadSchedule, DownloadScheduleBuilder};
	use crate::storage::DynStore;		use crate::storage::DynStore;
	use crate::{Authority, AuthorityListBuilder, Result};		use crate::{AuthorityListBuilder, Result};
	use tor_config::ConfigBuildError;		use tor_config::ConfigBuildError;
	use tor_guardmgr::fallback::FallbackListBuilder;		use tor_guardmgr::fallback::FallbackListBuilder;
	use tor_netdoc::doc::netstatus;		use tor_netdoc::doc::netstatus;
	@@ -110,34 +110,30 @@ impl NetworkConfigBuilder {
	#[builder(derive(Deserialize))]		#[builder(derive(Deserialize))]
	pub struct DownloadScheduleConfig {		pub struct DownloadScheduleConfig {
	/// Top-level configuration for how to retry our initial bootstrap attempt.		/// Top-level configuration for how to retry our initial bootstrap attempt.
	#[serde(default = "default_retry_bootstrap")]		#[builder(
	#[builder(default = "default_retry_bootstrap()")]		sub_builder,
	retry_bootstrap: DownloadSchedule,		field(build = "self.retry_bootstrap.build_retry_bootstrap()?")
			)]
			#[builder_field_attr(serde(default))]
			pub(crate) retry_bootstrap: DownloadSchedule,

	/// Configuration for how to retry a consensus download.		/// Configuration for how to retry a consensus download.
	#[serde(default)]		#[builder(sub_builder)]
	#[builder(default)]		#[builder_field_attr(serde(default))]
	retry_consensus: DownloadSchedule,		pub(crate) retry_consensus: DownloadSchedule,

	/// Configuration for how to retry an authority cert download.		/// Configuration for how to retry an authority cert download.
	#[serde(default)]		#[builder(sub_builder)]
	#[builder(default)]		#[builder_field_attr(serde(default))]
	retry_certs: DownloadSchedule,		pub(crate) retry_certs: DownloadSchedule,

	/// Configuration for how to retry a microdescriptor download.		/// Configuration for how to retry a microdescriptor download.
	#[serde(default = "default_microdesc_schedule")]		#[builder(
	#[builder(default = "default_microdesc_schedule()")]		sub_builder,
	retry_microdescs: DownloadSchedule,		field(build = "self.retry_microdescs.build_retry_microdescs()?")
	}		)]
			#[builder_field_attr(serde(default))]
	/// Default value for retry_bootstrap in DownloadScheduleConfig.		pub(crate) retry_microdescs: DownloadSchedule,
	fn default_retry_bootstrap() -> DownloadSchedule {
	DownloadSchedule::new(128, std::time::Duration::new(1, 0), 1)
	}

	/// Default value for microdesc_bootstrap in DownloadScheduleConfig.
	fn default_microdesc_schedule() -> DownloadSchedule {
	DownloadSchedule::new(3, std::time::Duration::new(1, 0), 4)
	}		}

	impl Default for DownloadScheduleConfig {		impl Default for DownloadScheduleConfig {
	@@ -186,7 +182,7 @@ pub struct DirMgrConfig {
	pub cache_path: PathBuf,		pub cache_path: PathBuf,

	/// Configuration information about the network.		/// Configuration information about the network.
	pub network_config: NetworkConfig,		pub network: NetworkConfig,

	/// Configuration information about when we download things.		/// Configuration information about when we download things.
	///		///
	@@ -198,7 +194,7 @@ pub struct DirMgrConfig {
	/// on in-progress attempts as well, at least at the top level. Users		/// on in-progress attempts as well, at least at the top level. Users
	/// should _not_ assume that the effect of changing this option will always		/// should _not_ assume that the effect of changing this option will always
	/// be delayed.)		/// be delayed.)
	pub schedule_config: DownloadScheduleConfig,		pub schedule: DownloadScheduleConfig,

	/// A map of network parameters that we're overriding from their settings in		/// A map of network parameters that we're overriding from their settings in
	/// the consensus.		/// the consensus.
	@@ -230,30 +226,14 @@ impl DirMgrConfig {
	)?))		)?))
	}		}

	/// Return the configured cache path.
	pub fn cache_path(&self) -> &std::path::Path {
	self.cache_path.as_ref()
	}

	/// Return a slice of the configured authorities		/// Return a slice of the configured authorities
	pub fn authorities(&self) -> &[Authority] {		pub fn authorities(&self) -> &[Authority] {
	&self.network_config.authorities		&self.network.authorities
	}		}

	/// Return the configured set of fallback directories		/// Return the configured set of fallback directories
	pub fn fallbacks(&self) -> &tor_guardmgr::fallback::FallbackList {		pub fn fallbacks(&self) -> &tor_guardmgr::fallback::FallbackList {
	&self.network_config.fallbacks		&self.network.fallbacks
	}

	/// Return set of configured networkstatus parameter overrides.
	pub fn override_net_params(&self) -> &netstatus::NetParams<i32> {
	&self.override_net_params
	}

	/// Return the schedule configuration we should use to decide when to
	/// attempt and retry downloads.
	pub fn schedule(&self) -> &DownloadScheduleConfig {
	&self.schedule_config
	}		}

	/// Construct a new configuration object where all replaceable fields in		/// Construct a new configuration object where all replaceable fields in
	@@ -263,11 +243,11 @@ impl DirMgrConfig {
	pub(crate) fn update_from_config(&self, new_config: &DirMgrConfig) -> DirMgrConfig {		pub(crate) fn update_from_config(&self, new_config: &DirMgrConfig) -> DirMgrConfig {
	DirMgrConfig {		DirMgrConfig {
	cache_path: self.cache_path.clone(),		cache_path: self.cache_path.clone(),
	network_config: NetworkConfig {		network: NetworkConfig {
	fallbacks: new_config.network_config.fallbacks.clone(),		fallbacks: new_config.network.fallbacks.clone(),
	authorities: self.network_config.authorities.clone(),		authorities: self.network.authorities.clone(),
	},		},
	schedule_config: new_config.schedule_config.clone(),		schedule: new_config.schedule.clone(),
	override_net_params: new_config.override_net_params.clone(),		override_net_params: new_config.override_net_params.clone(),
	extensions: new_config.extensions.clone(),		extensions: new_config.extensions.clone(),
	}		}
	@@ -292,29 +272,6 @@ pub struct DirMgrExtensions {
	pub filter: crate::filter::FilterConfig,		pub filter: crate::filter::FilterConfig,
	}		}

	impl DownloadScheduleConfig {
	/// Return configuration for retrying our entire bootstrap
	/// operation at startup.
	pub(crate) fn retry_bootstrap(&self) -> &DownloadSchedule {
	&self.retry_bootstrap
	}

	/// Return configuration for retrying a consensus download.
	pub(crate) fn retry_consensus(&self) -> &DownloadSchedule {
	&self.retry_consensus
	}

	/// Return configuration for retrying an authority certificate download
	pub(crate) fn retry_certs(&self) -> &DownloadSchedule {
	&self.retry_certs
	}

	/// Return configuration for retrying an authority certificate download
	pub(crate) fn retry_microdescs(&self) -> &DownloadSchedule {
	&self.retry_microdescs
	}
	}

	#[cfg(test)]		#[cfg(test)]
	mod test {		mod test {
	#![allow(clippy::unwrap_used)]		#![allow(clippy::unwrap_used)]
	@@ -384,21 +341,30 @@ mod test {
	let mut bld = DownloadScheduleConfig::builder();		let mut bld = DownloadScheduleConfig::builder();

	let cfg = bld.build().unwrap();		let cfg = bld.build().unwrap();
	assert_eq!(cfg.retry_microdescs().parallelism(), 4);		assert_eq!(cfg.retry_microdescs.parallelism(), 4);
	assert_eq!(cfg.retry_microdescs().n_attempts(), 3);		assert_eq!(cfg.retry_microdescs.n_attempts(), 3);
	assert_eq!(cfg.retry_bootstrap().n_attempts(), 128);		assert_eq!(cfg.retry_bootstrap.n_attempts(), 128);

	bld.retry_consensus(DownloadSchedule::new(7, Duration::new(86400, 0), 1))		bld.retry_consensus().attempts(7);
	.retry_bootstrap(DownloadSchedule::new(4, Duration::new(3600, 0), 1))		bld.retry_consensus().initial_delay(Duration::new(86400, 0));
	.retry_certs(DownloadSchedule::new(5, Duration::new(3600, 0), 1))		bld.retry_consensus().parallelism(1);
	.retry_microdescs(DownloadSchedule::new(6, Duration::new(3600, 0), 0));		bld.retry_bootstrap().attempts(4);
			bld.retry_bootstrap().initial_delay(Duration::new(3600, 0));
			bld.retry_bootstrap().parallelism(1);

			bld.retry_certs().attempts(5);
			bld.retry_certs().initial_delay(Duration::new(3600, 0));
			bld.retry_certs().parallelism(1);
			bld.retry_microdescs().attempts(6);
			bld.retry_microdescs().initial_delay(Duration::new(3600, 0));
			bld.retry_microdescs().parallelism(1);

	let cfg = bld.build().unwrap();		let cfg = bld.build().unwrap();
	assert_eq!(cfg.retry_microdescs().parallelism(), 1); // gets clamped		assert_eq!(cfg.retry_microdescs.parallelism(), 1);
	assert_eq!(cfg.retry_microdescs().n_attempts(), 6);		assert_eq!(cfg.retry_microdescs.n_attempts(), 6);
	assert_eq!(cfg.retry_bootstrap().n_attempts(), 4);		assert_eq!(cfg.retry_bootstrap.n_attempts(), 4);
	assert_eq!(cfg.retry_consensus().n_attempts(), 7);		assert_eq!(cfg.retry_consensus.n_attempts(), 7);
	assert_eq!(cfg.retry_certs().n_attempts(), 5);		assert_eq!(cfg.retry_certs.n_attempts(), 5);

	Ok(())		Ok(())
	}		}
	@@ -411,7 +377,7 @@ mod test {
	bld.override_net_params.set("circwindow".into(), 999);		bld.override_net_params.set("circwindow".into(), 999);
	bld.cache_path = tmp.path().into();		bld.cache_path = tmp.path().into();

	assert_eq!(bld.override_net_params().get("circwindow").unwrap(), &999);		assert_eq!(bld.override_net_params.get("circwindow").unwrap(), &999);

	Ok(())		Ok(())
	}		}

crates/tor-dirmgr/src/lib.rs

+4 −4

Original line number	Original line	Diff line number	Diff line
	@@ -534,7 +534,7 @@ impl<R: Runtime> DirMgr<R> {
	// TODO(nickm): instead of getting this every time we loop, it		// TODO(nickm): instead of getting this every time we loop, it
	// might be a good idea to refresh it with each attempt, at		// might be a good idea to refresh it with each attempt, at
	// least at the point of checking the number of attempts.		// least at the point of checking the number of attempts.
	*dirmgr.config.get().schedule().retry_bootstrap()		dirmgr.config.get().schedule.retry_bootstrap
	};		};
	let mut retry_delay = retry_config.schedule();		let mut retry_delay = retry_config.schedule();

	@@ -607,7 +607,7 @@ impl<R: Runtime> DirMgr<R> {
	// We don't support changing these: doing so basically would require us		// We don't support changing these: doing so basically would require us
	// to abort all our in-progress downloads, since they might be based on		// to abort all our in-progress downloads, since they might be based on
	// no-longer-viable information.		// no-longer-viable information.
	if new_config.cache_path() != config.cache_path() {		if new_config.cache_path != config.cache_path {
	how.cannot_change("storage.cache_path")?;		how.cannot_change("storage.cache_path")?;
	}		}
	if new_config.authorities() != config.authorities() {		if new_config.authorities() != config.authorities() {
	@@ -618,14 +618,14 @@ impl<R: Runtime> DirMgr<R> {
	return Ok(());		return Ok(());
	}		}

	let params_changed = new_config.override_net_params() != config.override_net_params();		let params_changed = new_config.override_net_params != config.override_net_params;

	self.config		self.config
	.map_and_replace(\|cfg\| cfg.update_from_config(new_config));		.map_and_replace(\|cfg\| cfg.update_from_config(new_config));

	if params_changed {		if params_changed {
	let _ignore_err = self.netdir.mutate(\|netdir\| {		let _ignore_err = self.netdir.mutate(\|netdir\| {
	netdir.replace_overridden_parameters(new_config.override_net_params());		netdir.replace_overridden_parameters(&new_config.override_net_params);
	Ok(())		Ok(())
	});		});
	// (It's okay to ignore the error, since it just means that there		// (It's okay to ignore the error, since it just means that there