Commit 36e8f916 authored by Hiro's avatar Hiro 🏄
Browse files

Modify the downloader algorithm to retry 0 bytes responses that returned

a http error code differnt than 404.
parent b61a2060
Pipeline #23517 failed with stage
in 3 minutes and 3 seconds
......@@ -310,7 +310,7 @@ public class ReferenceChecker {
}
}
logger.info(sb.toString());
if (totalMissingDescriptorsWeight > 0.999) {
if (totalMissingDescriptorsWeight > 4.999) {
logger.warn("Missing too many referenced descriptors ({}).",
totalMissingDescriptorsWeight);
}
......@@ -325,4 +325,3 @@ public class ReferenceChecker {
}
}
}
......@@ -444,7 +444,7 @@ public class RelayDescriptorDownloader {
}
}
/* Make a list of all directory authorities that we want to
/* Make a list of two directory authorities that we want to
* download all server and extra-info descriptors from. */
this.downloadAllDescriptorsFromAuthorities = new HashSet<>();
for (String authority : this.authorities) {
......@@ -453,12 +453,9 @@ public class RelayDescriptorDownloader {
this.downloadAllDescriptorsCutOff) < 0) {
this.downloadAllDescriptorsFromAuthorities.add(authority);
}
/* I am not sure considering more than 2 authrity per run is improving the
* algorigthm but is certainly increasing badnwidth
* if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {
* break;
* }
*/
if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {
break;
}
}
/* Prepare statistics on this execution. */
......@@ -904,16 +901,23 @@ public class RelayDescriptorDownloader {
receivedDescriptors = processDownloadResponse(resource, allData,
authority);
} else {
/* If a download fails with an error code different than 404 retry a up to
* 5 times. Before each try wait an amount of seconds between 0 and 25.
*/
try {
logger.warn("Downloaded 0 bytes from {} "
+ "will wait 45 SECONDS to avoid DOS protections.", authority);
TimeUnit.SECONDS.sleep(45);
allData = Downloader.downloadFromHttpServer(url, isCompressed);
logger.debug("Downloaded {} -> ({} bytes)", fullUrl,
allData == null ? 0 : allData.length);
if (null != allData) {
receivedDescriptors = processDownloadResponse(resource, allData,
authority);
for (int i = 0; i < 6; i++) {
int wait = i * 5;
logger.warn("Downloaded 0 bytes from {} "
+ "will wait {} SECONDS to avoid DOS protections.", authority, wait);
TimeUnit.SECONDS.sleep(wait);
allData = Downloader.downloadFromHttpServer(url, isCompressed);
logger.warn("Downloaded {} -> ({} bytes)", fullUrl,
allData == null ? 0 : allData.length);
if (null != allData) {
receivedDescriptors = processDownloadResponse(resource, allData,
authority);
break;
}
}
} catch (InterruptedException e) {
logger.warn("Was interrupted while waiting to retry a download", e);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment