Commit 4403719f authored by Hiro's avatar Hiro 🏄
Browse files

Add verbose logging in the RelayDescriptorDownloader Class

Modify the downloader algorithm to only download missing descriptors from votes
from the same authority where the vote is coming from.
parent 405c5253
Pipeline #23094 failed with stage
in 6 minutes and 9 seconds
Subproject commit 6e9a32a89e8f48c70988d9f5cb0eb27a14b3ebe8
Subproject commit c5d7511a7300dd535d368ccb3390b52bff453534
......@@ -20,6 +20,7 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
......@@ -113,6 +114,11 @@ public class RelayDescriptorDownloader {
*/
private List<String> authorities;
/**
* Directory authorities that we will not shuffle.
*/
private List<String> authoritiesStatic;
/**
* Fingerprints of directory authorities that we will use to download
* votes without requiring a successfully downloaded consensus.
......@@ -307,6 +313,8 @@ public class RelayDescriptorDownloader {
/* Memorize argument values. */
this.rdp = rdp;
this.authorities = Arrays.asList(authorities);
/* This authorities array will not be shuffled */
this.authoritiesStatic = Arrays.asList(authorities);
this.authorityFingerprints = Arrays.asList(authorityFingerprints);
this.downloadCurrentConsensus = downloadCurrentConsensus;
this.downloadCurrentMicrodescConsensus =
......@@ -346,7 +354,7 @@ public class RelayDescriptorDownloader {
"stats/missing-relay-descriptors");
if (this.missingDescriptorsFile.exists()) {
try {
logger.debug("Reading file {}...",
logger.info("Reading file {}...",
this.missingDescriptorsFile.getAbsolutePath());
BufferedReader br = new BufferedReader(new FileReader(
this.missingDescriptorsFile));
......@@ -394,12 +402,12 @@ public class RelayDescriptorDownloader {
}
}
} else {
logger.debug("Invalid line '{}' in {}. Ignoring.", line,
logger.info("Invalid line '{}' in {}. Ignoring.", line,
this.missingDescriptorsFile.getAbsolutePath());
}
}
br.close();
logger.debug("Finished reading file {}.",
logger.info("Finished reading file {}.",
this.missingDescriptorsFile.getAbsolutePath());
} catch (IOException e) {
logger.warn("Failed to read file {}! This means that we might forget "
......@@ -415,14 +423,14 @@ public class RelayDescriptorDownloader {
"stats/last-downloaded-all-descriptors");
if (this.lastDownloadedAllDescriptorsFile.exists()) {
try {
logger.debug("Reading file {}...",
logger.info("Reading file {}...",
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
BufferedReader br = new BufferedReader(new FileReader(
this.lastDownloadedAllDescriptorsFile));
String line;
while ((line = br.readLine()) != null) {
if (line.split(",").length != 2) {
logger.debug("Invalid line '{}' in {}. Ignoring.", line,
logger.info("Invalid line '{}' in {}. Ignoring.", line,
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
} else {
String[] parts = line.split(",");
......@@ -433,7 +441,7 @@ public class RelayDescriptorDownloader {
}
}
br.close();
logger.debug("Finished reading file {}.",
logger.info("Finished reading file {}.",
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
} catch (IOException e) {
logger.warn("Failed to read file {}! This means that we might "
......@@ -726,7 +734,7 @@ public class RelayDescriptorDownloader {
for (String fingerprint : fingerprints) {
this.requestedVotes++;
this.downloadedVotes +=
this.downloadResourceFromAuthority(authority,
this.downloadResourceFromAuthority(authority,
"/tor/status-vote/current/" + fingerprint);
}
}
......@@ -739,8 +747,8 @@ public class RelayDescriptorDownloader {
this.downloadResourceFromAuthority(authority,
"/tor/status-vote/next/bandwidth");
/* Download either all server and extra-info descriptors or only
* those that we're missing. Start with server descriptors, then
/* Download either all server and extra-info descriptors.
* Start with server descriptors, then
* request extra-info descriptors. Finally, request missing
* microdescriptors. */
for (String type : new String[] { "server", "extra", "micro" }) {
......@@ -765,37 +773,67 @@ public class RelayDescriptorDownloader {
downloadedAllDescriptors;
}
/* Download missing server descriptors, extra-info descriptors,
* and microdescriptors if we're configured to do so. */
} else if ((type.equals("server")
&& this.downloadMissingServerDescriptors)
|| (type.equals("extra") && this.downloadMissingExtraInfos)
|| (type.equals("micro")
&& this.downloadMissingMicrodescriptors)) {
/* Go through the list of missing descriptors of this type
* and combine the descriptor identifiers to a URL of up to
* 96 server or extra-info descriptors or 92 microdescriptors
* that we can download at once. */
SortedSet<String> descriptorIdentifiers = new TreeSet<>();
for (Map.Entry<String, String> e :
this.missingDescriptors.entrySet()) {
if (e.getValue().equals("NA")
&& e.getKey().startsWith(type + ",")
&& this.descriptorCutOff.compareTo(
e.getKey().split(",")[1]) < 0) {
String descriptorIdentifier = e.getKey().split(",")[3];
descriptorIdentifiers.add(descriptorIdentifier);
}
}
/* If a download failed, stop requesting descriptors from this
* authority and move on to the next. */
} catch (IOException e) {
logger.info("Failed downloading from {}!", authority, e);
}
}
/* Download missing server and extra-info descriptors.
* Start with server descriptors, then
* request extra-info descriptors. Finally, request missing
* microdescriptors. */
for (String type : new String[] { "server", "extra", "micro" }) {
if ((type.equals("server")
&& this.downloadMissingServerDescriptors)
|| (type.equals("extra") && this.downloadMissingExtraInfos)
|| (type.equals("micro")
&& this.downloadMissingMicrodescriptors)) {
/* Go through the list of missing descriptors of this type
* and combine the descriptor identifiers to a URL of up to
* 96 server or extra-info descriptors or 92 microdescriptors
* that we can download at once. */
HashMap<String, List<String>> descriptorIdentifiers =
new HashMap<String, List<String>>();
for (Map.Entry<String, String> e :
this.missingDescriptors.entrySet()) {
if (e.getValue().equals("NA")
&& e.getKey().startsWith(type + ",")
&& this.descriptorCutOff.compareTo(
e.getKey().split(",")[1]) < 0) {
String[] line = e.getKey().split(",");
String descriptorIdentifier = line[3];
String reference = line[2];
int authorityIndex =
this.authorityFingerprints.indexOf(reference);
if (authorityIndex != -1) {
String authority = this.authoritiesStatic.get(authorityIndex);
if(!descriptorIdentifiers.containsKey(authority)) {
descriptorIdentifiers.put(authority, new ArrayList<String>());
}
descriptorIdentifiers.get(authority).add(descriptorIdentifier);
}
StringBuilder combinedResource = null;
int descriptorsInCombinedResource = 0;
int requestedDescriptors = 0;
int downloadedDescriptors = 0;
int maxDescriptorsInCombinedResource =
type.equals("micro") ? 92 : 96;
String separator = type.equals("micro") ? "-" : "+";
for (String descriptorIdentifier : descriptorIdentifiers) {
}
}
StringBuilder combinedResource = null;
int descriptorsInCombinedResource = 0;
int requestedDescriptors = 0;
int downloadedDescriptors = 0;
int maxDescriptorsInCombinedResource =
type.equals("micro") ? 92 : 96;
String separator = type.equals("micro") ? "-" : "+";
for (String authority : descriptorIdentifiers.keySet()) {
try {
List<String> descIdentifiers = descriptorIdentifiers.get(authority);
for (String descriptorIdentifier : descIdentifiers) {
if (descriptorsInCombinedResource
>= maxDescriptorsInCombinedResource) {
requestedDescriptors += descriptorsInCombinedResource;
......@@ -819,36 +857,35 @@ public class RelayDescriptorDownloader {
this.downloadResourceFromAuthority(authority,
combinedResource.toString());
}
switch (type) {
case "server":
this.requestedMissingServerDescriptors +=
requestedDescriptors;
this.downloadedMissingServerDescriptors +=
downloadedDescriptors;
break;
case "extra":
this.requestedMissingExtraInfoDescriptors +=
requestedDescriptors;
this.downloadedMissingExtraInfoDescriptors +=
downloadedDescriptors;
break;
case "micro":
this.requestedMissingMicrodescriptors +=
requestedDescriptors;
this.downloadedMissingMicrodescriptors +=
downloadedDescriptors;
break;
default:
/* Unknown type, nothing to do. */
break;
}
/* If a download failed, stop requesting descriptors from this
* authority and move on to the next. */
} catch (IOException e) {
logger.info("Failed downloading from {}!", authority, e);
}
}
/* If a download failed, stop requesting descriptors from this
* authority and move on to the next. */
} catch (IOException e) {
logger.debug("Failed downloading from {}!", authority, e);
switch (type) {
case "server":
this.requestedMissingServerDescriptors +=
requestedDescriptors;
this.downloadedMissingServerDescriptors +=
downloadedDescriptors;
break;
case "extra":
this.requestedMissingExtraInfoDescriptors +=
requestedDescriptors;
this.downloadedMissingExtraInfoDescriptors +=
downloadedDescriptors;
break;
case "micro":
this.requestedMissingMicrodescriptors +=
requestedDescriptors;
this.downloadedMissingMicrodescriptors +=
downloadedDescriptors;
break;
default:
/* Unknown type, nothing to do. */
break;
}
}
}
}
......@@ -982,7 +1019,7 @@ public class RelayDescriptorDownloader {
int missingServerDescriptors = 0;
int missingExtraInfoDescriptors = 0;
try {
logger.debug("Writing file {}...",
logger.info("Writing file {}...",
this.missingDescriptorsFile.getAbsolutePath());
this.missingDescriptorsFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
......@@ -1009,7 +1046,7 @@ public class RelayDescriptorDownloader {
bw.write(key + "," + value + "\n");
}
bw.close();
logger.debug("Finished writing file {}.",
logger.info("Finished writing file {}.",
this.missingDescriptorsFile.getAbsolutePath());
} catch (IOException e) {
logger.warn("Failed writing {}!",
......@@ -1020,7 +1057,7 @@ public class RelayDescriptorDownloader {
* last downloaded all server and extra-info descriptors from them to
* disk. */
try {
logger.debug("Writing file {}...",
logger.info("Writing file {}...",
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
......@@ -1032,7 +1069,7 @@ public class RelayDescriptorDownloader {
bw.write(authority + "," + lastDownloaded + "\n");
}
bw.close();
logger.debug("Finished writing file {}.",
logger.info("Finished writing file {}.",
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
} catch (IOException e) {
logger.warn("Failed writing {}!",
......@@ -1099,4 +1136,3 @@ public class RelayDescriptorDownloader {
this.missingMicrodescriptors.size());
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment