Commit 36e8f916 authored by Hiro's avatar Hiro 🏄
Browse files

Modify the downloader algorithm to retry 0 bytes responses that returned 

a http error code differnt than 404.
parent b61a2060

src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java

+1 −2
Original line number Diff line number Diff line
@@ -310,7 +310,7 @@ public class ReferenceChecker { 
      }

    }

    logger.info(sb.toString());

    if (totalMissingDescriptorsWeight > 0.999) {

    if (totalMissingDescriptorsWeight > 4.999) {

      logger.warn("Missing too many referenced descriptors ({}).",

          totalMissingDescriptorsWeight);

    }
@@ -325,4 +325,3 @@ public class ReferenceChecker { 
    }

  }

}

src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java

+20 −16
Original line number Diff line number Diff line
@@ -444,7 +444,7 @@ public class RelayDescriptorDownloader { 
      }

    }



    /* Make a list of all directory authorities that we want to

    /* Make a list of two directory authorities that we want to

     * download all server and extra-info descriptors from. */

    this.downloadAllDescriptorsFromAuthorities = new HashSet<>();

    for (String authority : this.authorities) {
@@ -453,12 +453,9 @@ public class RelayDescriptorDownloader { 
          this.downloadAllDescriptorsCutOff) < 0) {

        this.downloadAllDescriptorsFromAuthorities.add(authority);

      }

      /* I am not sure considering more than 2 authrity per run is improving the

       * algorigthm but is certainly increasing badnwidth

       * if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {

       *   break;

       * }

       */

      if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {

        break;

      }

    }



    /* Prepare statistics on this execution. */
@@ -904,16 +901,23 @@ public class RelayDescriptorDownloader { 
      receivedDescriptors = processDownloadResponse(resource, allData,

          authority);

    } else {

      /* If a download fails with an error code different than 404 retry a up to

       * 5 times. Before each try wait an amount of seconds between 0 and 25.

       */

      try {

        for (int i = 0; i < 6; i++) {

          int wait = i * 5;

          logger.warn("Downloaded 0 bytes from {} "

            + "will wait 45 SECONDS to avoid DOS protections.", authority);

        TimeUnit.SECONDS.sleep(45);

              + "will wait {} SECONDS to avoid DOS protections.", authority, wait);

          TimeUnit.SECONDS.sleep(wait);

          allData = Downloader.downloadFromHttpServer(url, isCompressed);

        logger.debug("Downloaded {} -> ({} bytes)", fullUrl,

          logger.warn("Downloaded {} -> ({} bytes)", fullUrl,

              allData == null ? 0 : allData.length);

          if (null != allData) {

            receivedDescriptors = processDownloadResponse(resource, allData,

                authority);

            break;

          }

        }

      } catch (InterruptedException e) {

        logger.warn("Was interrupted while waiting to retry a download", e);