Skip to content
GitLab
Commit 806b47ca authored by Hiro's avatar Hiro 🏄
Browse files

Modify downloader to avoid DOS protections from authorities

parent 405c5253
Pipeline #23224 failed with stage
in 2 minutes and 58 seconds
Hide whitespace changes
Inline Side-by-side
build @ c5d7511a
Compare 6e9a32a8...c5d7511a
Subproject commit 6e9a32a89e8f48c70988d9f5cb0eb27a14b3ebe8
Subproject commit c5d7511a7300dd535d368ccb3390b52bff453534
src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
View file @ 806b47ca
......@@ -18,6 +18,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.TimeUnit;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
......@@ -346,7 +347,7 @@ public class RelayDescriptorDownloader {
"stats/missing-relay-descriptors");
if (this.missingDescriptorsFile.exists()) {
try {
logger.debug("Reading file {}...",
logger.warn("Reading file {}...",
this.missingDescriptorsFile.getAbsolutePath());
BufferedReader br = new BufferedReader(new FileReader(
this.missingDescriptorsFile));
......@@ -394,12 +395,12 @@ public class RelayDescriptorDownloader {
}
}
} else {
logger.debug("Invalid line '{}' in {}. Ignoring.", line,
logger.warn("Invalid line '{}' in {}. Ignoring.", line,
this.missingDescriptorsFile.getAbsolutePath());
}
}
br.close();
logger.debug("Finished reading file {}.",
logger.warn("Finished reading file {}.",
this.missingDescriptorsFile.getAbsolutePath());
} catch (IOException e) {
logger.warn("Failed to read file {}! This means that we might forget "
......@@ -415,14 +416,14 @@ public class RelayDescriptorDownloader {
"stats/last-downloaded-all-descriptors");
if (this.lastDownloadedAllDescriptorsFile.exists()) {
try {
logger.debug("Reading file {}...",
logger.warn("Reading file {}...",
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
BufferedReader br = new BufferedReader(new FileReader(
this.lastDownloadedAllDescriptorsFile));
String line;
while ((line = br.readLine()) != null) {
if (line.split(",").length != 2) {
logger.debug("Invalid line '{}' in {}. Ignoring.", line,
logger.warn("Invalid line '{}' in {}. Ignoring.", line,
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
} else {
String[] parts = line.split(",");
......@@ -433,7 +434,7 @@ public class RelayDescriptorDownloader {
}
}
br.close();
logger.debug("Finished reading file {}.",
logger.warn("Finished reading file {}.",
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
} catch (IOException e) {
logger.warn("Failed to read file {}! This means that we might "
......@@ -443,7 +444,7 @@ public class RelayDescriptorDownloader {
}
}
/* Make a list of at most two directory authorities that we want to
/* Make a list of at most four directory authorities that we want to
* download all server and extra-info descriptors from. */
this.downloadAllDescriptorsFromAuthorities = new HashSet<>();
for (String authority : this.authorities) {
......@@ -452,7 +453,9 @@ public class RelayDescriptorDownloader {
this.downloadAllDescriptorsCutOff) < 0) {
this.downloadAllDescriptorsFromAuthorities.add(authority);
}
if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {
/* I am not sure considering more than 2 authrity per run is improving the
* algorigthm but is certainly increasing badnwidth */
if (this.downloadAllDescriptorsFromAuthorities.size() >= 4) {
break;
}
}
......@@ -726,7 +729,7 @@ public class RelayDescriptorDownloader {
for (String fingerprint : fingerprints) {
this.requestedVotes++;
this.downloadedVotes +=
this.downloadResourceFromAuthority(authority,
this.downloadResourceFromAuthority(authority,
"/tor/status-vote/current/" + fingerprint);
}
}
......@@ -848,7 +851,7 @@ public class RelayDescriptorDownloader {
/* If a download failed, stop requesting descriptors from this
* authority and move on to the next. */
} catch (IOException e) {
logger.debug("Failed downloading from {}!", authority, e);
logger.warn("Failed downloading from {}!", authority, e);
}
}
}
......@@ -875,95 +878,120 @@ public class RelayDescriptorDownloader {
URL url = new URL(fullUrl);
allData = Downloader.downloadFromHttpServer(url, isCompressed);
int receivedDescriptors = 0;
logger.debug("Downloaded {} -> ({} bytes)", fullUrl,
logger.warn("Downloaded {} -> ({} bytes)", fullUrl,
allData == null ? 0 : allData.length);
/* TODO This should be refactored to take into account that the download
* is tried a defined number of times. */
if (null != allData) {
if (resource.startsWith("/tor/status-vote/")) {
this.rdp.parse(allData, null);
receivedDescriptors = 1;
} else if (resource.startsWith("/tor/server/")
|| resource.startsWith("/tor/extra/")) {
if (resource.equals("/tor/server/all")
|| resource.equals("/tor/extra/all")) {
this.lastDownloadedAllDescriptors.put(authority,
this.currentTimestamp);
receivedDescriptors = processDownloadResponse(resource, allData,
authority);
} else {
try {
logger.warn("Downloaded 0 bytes from {}"
+ "will wait 1 minute to avoid DOS protections.", authority);
TimeUnit.MINUTES.sleep(1);
allData = Downloader.downloadFromHttpServer(url, isCompressed);
logger.warn("Downloaded {} -> ({} bytes)", fullUrl,
allData == null ? 0 : allData.length);
if (null != allData) {
receivedDescriptors = processDownloadResponse(resource, allData,
authority);
}
String ascii = new String(allData, StandardCharsets.US_ASCII);
int start;
int sig;
int end = -1;
String startToken = resource.startsWith("/tor/server/")
? "router " : "extra-info ";
String sigToken = "\nrouter-signature\n";
String endToken = "\n-----END SIGNATURE-----\n";
while (end < ascii.length()) {
start = ascii.indexOf(startToken, end);
if (start < 0) {
break;
}
sig = ascii.indexOf(sigToken, start);
if (sig < 0) {
break;
}
sig += sigToken.length();
end = ascii.indexOf(endToken, sig);
if (end < 0) {
break;
}
end += endToken.length();
byte[] descBytes = new byte[end - start];
System.arraycopy(allData, start, descBytes, 0, end - start);
this.rdp.parse(descBytes, null);
receivedDescriptors++;
} catch (InterruptedException e) {
logger.warn("Was interrupted while waiting to retry a download", e);
}
}
return receivedDescriptors;
}
private int processDownloadResponse(String resource, byte[] allData,
String authority) {
int receivedDescriptors = 0;
if (resource.startsWith("/tor/status-vote/")) {
this.rdp.parse(allData, null);
receivedDescriptors = 1;
} else if (resource.startsWith("/tor/server/")
|| resource.startsWith("/tor/extra/")) {
if (resource.equals("/tor/server/all")
|| resource.equals("/tor/extra/all")) {
this.lastDownloadedAllDescriptors.put(authority,
this.currentTimestamp);
}
String ascii = new String(allData, StandardCharsets.US_ASCII);
int start;
int sig;
int end = -1;
String startToken = resource.startsWith("/tor/server/")
? "router " : "extra-info ";
String sigToken = "\nrouter-signature\n";
String endToken = "\n-----END SIGNATURE-----\n";
while (end < ascii.length()) {
start = ascii.indexOf(startToken, end);
if (start < 0) {
break;
}
sig = ascii.indexOf(sigToken, start);
if (sig < 0) {
break;
}
sig += sigToken.length();
end = ascii.indexOf(endToken, sig);
if (end < 0) {
break;
}
} else if (resource.startsWith("/tor/micro/")) {
/* TODO We need to parse microdescriptors ourselves, rather than
* RelayDescriptorParser, because only we know the valid-after
* time(s) of microdesc consensus(es) containing this
* microdescriptor. However, this breaks functional abstraction
* pretty badly. */
SimpleDateFormat parseFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String ascii = new String(allData, StandardCharsets.US_ASCII);
int start;
int end = -1;
String startToken = "onion-key\n";
while (end < ascii.length()) {
start = ascii.indexOf(startToken, end);
if (start < 0) {
end += endToken.length();
byte[] descBytes = new byte[end - start];
System.arraycopy(allData, start, descBytes, 0, end - start);
this.rdp.parse(descBytes, null);
receivedDescriptors++;
}
} else if (resource.startsWith("/tor/micro/")) {
/* TODO We need to parse microdescriptors ourselves, rather than
* RelayDescriptorParser, because only we know the valid-after
* time(s) of microdesc consensus(es) containing this
* microdescriptor. However, this breaks functional abstraction
* pretty badly. */
SimpleDateFormat parseFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String ascii = new String(allData, StandardCharsets.US_ASCII);
int start;
int end = -1;
String startToken = "onion-key\n";
while (end < ascii.length()) {
start = ascii.indexOf(startToken, end);
if (start < 0) {
break;
}
end = ascii.indexOf(startToken, start + 1);
if (end < 0) {
end = ascii.length();
if (end <= start) {
break;
}
end = ascii.indexOf(startToken, start + 1);
if (end < 0) {
end = ascii.length();
if (end <= start) {
break;
}
}
byte[] descBytes = new byte[end - start];
System.arraycopy(allData, start, descBytes, 0, end - start);
String digest256Base64 = Base64.encodeBase64String(
DigestUtils.sha256(descBytes)).replaceAll("=", "");
if (!this.microdescriptorKeys.containsKey(digest256Base64)) {
continue;
}
String digest256Hex = DigestUtils.sha256Hex(descBytes);
for (String microdescriptorKey :
this.microdescriptorKeys.get(digest256Base64)) {
String validAfterTime = microdescriptorKey.split(",")[1];
try {
long validAfter =
parseFormat.parse(validAfterTime).getTime();
this.rdp.storeMicrodescriptor(descBytes, digest256Hex,
digest256Base64, validAfter);
} catch (ParseException e) {
logger.warn("Could not parse valid-after time '{}' in "
+ "microdescriptor key. Not storing microdescriptor.",
validAfterTime, e);
}
}
byte[] descBytes = new byte[end - start];
System.arraycopy(allData, start, descBytes, 0, end - start);
String digest256Base64 = Base64.encodeBase64String(
DigestUtils.sha256(descBytes)).replaceAll("=", "");
if (!this.microdescriptorKeys.containsKey(digest256Base64)) {
continue;
}
String digest256Hex = DigestUtils.sha256Hex(descBytes);
for (String microdescriptorKey :
this.microdescriptorKeys.get(digest256Base64)) {
String validAfterTime = microdescriptorKey.split(",")[1];
try {
long validAfter =
parseFormat.parse(validAfterTime).getTime();
this.rdp.storeMicrodescriptor(descBytes, digest256Hex,
digest256Base64, validAfter);
} catch (ParseException e) {
logger.warn("Could not parse valid-after time '{}' in "
+ "microdescriptor key. Not storing microdescriptor.",
validAfterTime, e);
}
receivedDescriptors++;
}
receivedDescriptors++;
}
}
return receivedDescriptors;
......@@ -982,7 +1010,7 @@ public class RelayDescriptorDownloader {
int missingServerDescriptors = 0;
int missingExtraInfoDescriptors = 0;
try {
logger.debug("Writing file {}...",
logger.warn("Writing file {}...",
this.missingDescriptorsFile.getAbsolutePath());
this.missingDescriptorsFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
......@@ -1009,7 +1037,7 @@ public class RelayDescriptorDownloader {
bw.write(key + "," + value + "\n");
}
bw.close();
logger.debug("Finished writing file {}.",
logger.warn("Finished writing file {}.",
this.missingDescriptorsFile.getAbsolutePath());
} catch (IOException e) {
logger.warn("Failed writing {}!",
......@@ -1020,7 +1048,7 @@ public class RelayDescriptorDownloader {
* last downloaded all server and extra-info descriptors from them to
* disk. */
try {
logger.debug("Writing file {}...",
logger.warn("Writing file {}...",
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
......@@ -1032,7 +1060,7 @@ public class RelayDescriptorDownloader {
bw.write(authority + "," + lastDownloaded + "\n");
}
bw.close();
logger.debug("Finished writing file {}.",
logger.warn("Finished writing file {}.",
this.lastDownloadedAllDescriptorsFile.getAbsolutePath());
} catch (IOException e) {
logger.warn("Failed writing {}!",
......@@ -1099,4 +1127,3 @@ public class RelayDescriptorDownloader {
this.missingMicrodescriptors.size());
}
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment