Commit 463a7214 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Create utility class for downloading from HTTP servers 

Also skip two tests in ConfigurationTest that together take 53 seconds
which has the effect that we don't run tests very often.

Implements #31599.

Temporary commit: Requires an update to metrics-base that adds
Mockito. Once this is added, update metrics-base and mention the new
dependency in our change log!
parent 18877fdb

src/main/java/org/torproject/metrics/collector/downloader/Downloader.java

0 → 100644
+62 −0
Original line number Diff line number Diff line 
/* Copyright 2019 The Tor Project

 * See LICENSE for licensing information */



package org.torproject.metrics.collector.downloader;



import java.io.BufferedInputStream;

import java.io.ByteArrayOutputStream;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.zip.InflaterInputStream;



/**

 * Utility class for downloading resources from HTTP servers.

 */

public class Downloader {



  /**

   * Download the given URL from an HTTP server and return downloaded bytes.

   *

   * @param url URL to download.

   * @return Downloaded bytes, or {@code null} if the resource was not found.

   * @throws IOException Thrown if anything goes wrong while downloading.

   */

  public static byte[] downloadFromHttpServer(URL url) throws IOException {

    return Downloader.downloadFromHttpServer(url, false);

  }



  /**

   * Download the given URL from an HTTP server, possibly inflate the response,

   * and return downloaded bytes.

   *

   * @param url URL to download.

   * @param isDeflated Whether the response is deflated.

   * @return Downloaded bytes, or {@code null} if the resource was not found.

   * @throws IOException Thrown if anything goes wrong while downloading.

   */

  public static byte[] downloadFromHttpServer(URL url, boolean isDeflated)

      throws IOException {

    ByteArrayOutputStream downloadedBytes = new ByteArrayOutputStream();

    HttpURLConnection huc = (HttpURLConnection) url.openConnection();

    huc.setRequestMethod("GET");

    huc.setReadTimeout(5000);

    huc.connect();

    int response = huc.getResponseCode();

    if (response != 200) {

      return null;

    }

    try (BufferedInputStream in = isDeflated

        ? new BufferedInputStream(new InflaterInputStream(

            huc.getInputStream()))

        : new BufferedInputStream(huc.getInputStream())) {

      int len;

      byte[] data = new byte[1024];

      while ((len = in.read(data, 0, 1024)) >= 0) {

        downloadedBytes.write(data, 0, len);

      }

    }

    return downloadedBytes.toByteArray();

  }

}

src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java

+9 −24
Original line number Diff line number Diff line
@@ -12,16 +12,15 @@ import org.torproject.metrics.collector.conf.Configuration; 
import org.torproject.metrics.collector.conf.ConfigurationException;

import org.torproject.metrics.collector.conf.Key;

import org.torproject.metrics.collector.cron.CollecTorMain;

import org.torproject.metrics.collector.downloader.Downloader;



import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.io.BufferedInputStream;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileWriter;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import java.nio.file.Paths;

import java.text.SimpleDateFormat;
@@ -77,32 +76,18 @@ public class ExitListDownloader extends CollecTorMain { 
    sb.append("Downloaded ").append(dateTimeFormat.format(downloadedDate))

        .append("\n");

    URL url = config.getUrl(Key.ExitlistUrl);

    byte[] downloadedBytes;

    try {

      HttpURLConnection huc = (HttpURLConnection) url.openConnection();

      huc.setRequestMethod("GET");

      huc.setReadTimeout(5000);

      huc.connect();

      int response = huc.getResponseCode();

      if (response != 200) {

        logger.warn("Could not download exit list. Response code {}",

            response);

        return;

      }

      try (BufferedInputStream in = new BufferedInputStream(

          huc.getInputStream())) {

        int len;

        byte[] data = new byte[1024];

        while ((len = in.read(data, 0, 1024)) >= 0) {

          sb.append(new String(data, 0, len));

        }

      }

      downloadedExitList = sb.toString();

      logger.debug("Finished downloading exit list.");

      downloadedBytes = Downloader.downloadFromHttpServer(url);

    } catch (IOException e) {

      logger.warn("Failed downloading exit list", e);

      return;

    }

    if (downloadedExitList == null) {

    if (null != downloadedBytes) {

      sb.append(new String(downloadedBytes));

      downloadedExitList = sb.toString();

      logger.debug("Finished downloading exit list.");

    } else {

      logger.warn("Failed downloading exit list.");

      return;

    }

src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java

+18 −5
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ import org.torproject.metrics.collector.conf.Configuration; 
import org.torproject.metrics.collector.conf.ConfigurationException;

import org.torproject.metrics.collector.conf.Key;

import org.torproject.metrics.collector.cron.CollecTorMain;

import org.torproject.metrics.collector.downloader.Downloader;



import org.slf4j.Logger;

import org.slf4j.LoggerFactory;
@@ -21,7 +22,6 @@ import java.io.File; 
import java.io.FileReader;

import java.io.FileWriter;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.MalformedURLException;

import java.net.URL;
@@ -200,12 +200,25 @@ public class OnionPerfDownloader extends CollecTorMain { 


    /* Download file contents to temporary file. */

    File tempFile = new File(this.recentDirectory, "." + tpfFileName);

    byte[] downloadedBytes;

    try {

      downloadedBytes = Downloader.downloadFromHttpServer(

          new URL(baseUrl + tpfFileName));

    } catch (IOException e) {

      logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, tpfFileName,

          e);

      return;

    }

    if (null == downloadedBytes) {

      logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, tpfFileName);

      return;

    }

    tempFile.getParentFile().mkdirs();

    try (InputStream is = new URL(baseUrl + tpfFileName).openStream()) {

      Files.copy(is, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);

    try {

      Files.write(tempFile.toPath(), downloadedBytes);

    } catch (IOException e) {

      logger.warn("Unable to download '{}{}' to temporary file '{}'.  "

          + "Skipping.", baseUrl, tpfFileName, tempFile, e);

      logger.warn("Unable to write previously downloaded '{}{}' to temporary "

          + "file '{}'. Skipping.", baseUrl, tpfFileName, tempFile, e);

      return;

    }

src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java

+6 −26
Original line number Diff line number Diff line
@@ -3,20 +3,19 @@ 


package org.torproject.metrics.collector.relaydescs;



import org.torproject.metrics.collector.downloader.Downloader;



import org.apache.commons.codec.binary.Base64;

import org.apache.commons.codec.digest.DigestUtils;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.io.BufferedInputStream;

import java.io.BufferedReader;

import java.io.BufferedWriter;

import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.FileReader;

import java.io.FileWriter;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.URL;

import java.nio.charset.StandardCharsets;

import java.text.ParseException;
@@ -33,7 +32,6 @@ import java.util.SortedSet; 
import java.util.TimeZone;

import java.util.TreeMap;

import java.util.TreeSet;

import java.util.zip.InflaterInputStream;



/**

 * Downloads relay descriptors from the directory authorities via HTTP.
@@ -875,29 +873,11 @@ public class RelayDescriptorDownloader { 
    String fullUrl

        = "http://" + authority + resource + (isCompressed ? ".z" : "");

    URL url = new URL(fullUrl);

    HttpURLConnection huc = (HttpURLConnection) url.openConnection();

    huc.setRequestMethod("GET");

    huc.setReadTimeout(5000);

    huc.connect();

    int response = huc.getResponseCode();

    if (response == 200) {

      try (BufferedInputStream in

          = isCompressed ? new BufferedInputStream(new InflaterInputStream(

          huc.getInputStream()))

          : new BufferedInputStream(huc.getInputStream())) {

        ByteArrayOutputStream baos = new ByteArrayOutputStream();

        int len;

        byte[] data = new byte[1024];

        while ((len = in.read(data, 0, 1024)) >= 0) {

          baos.write(data, 0, len);

        }

        allData = baos.toByteArray();

      }

    }

    logger.debug("Downloaded {} -> {} ({} bytes)", fullUrl, response,

        allData == null ? 0 : allData.length);

    allData = Downloader.downloadFromHttpServer(url, isCompressed);

    int receivedDescriptors = 0;

    if (allData != null) {

    logger.debug("Downloaded {} -> ({} bytes)", fullUrl,

        allData == null ? 0 : allData.length);

    if (null != allData) {

      if (resource.startsWith("/tor/status-vote/")) {

        this.rdp.parse(allData, null);

        receivedDescriptors = 1;

src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java

+11 −45
Original line number Diff line number Diff line
@@ -12,18 +12,16 @@ import org.torproject.metrics.collector.conf.Configuration; 
import org.torproject.metrics.collector.conf.ConfigurationException;

import org.torproject.metrics.collector.conf.Key;

import org.torproject.metrics.collector.cron.CollecTorMain;

import org.torproject.metrics.collector.downloader.Downloader;

import org.torproject.metrics.collector.persist.SnowflakeStatsPersistence;



import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.io.BufferedInputStream;

import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStream;

import java.net.HttpURLConnection;

import java.net.URL;

import java.time.LocalDateTime;

import java.util.Arrays;
@@ -60,9 +58,15 @@ public class SnowflakeStatsDownloader extends CollecTorMain { 
    this.recentPathName = config.getPath(Key.RecentPath).toString();

    logger.debug("Downloading snowflake stats...");

    URL url = config.getUrl(Key.SnowflakeStatsUrl);

    ByteArrayOutputStream downloadedSnowflakeStats

        = this.downloadFromHttpServer(url);

    if (null == downloadedSnowflakeStats) {

    byte[] downloadedBytes;

    try {

      downloadedBytes = Downloader.downloadFromHttpServer(url);

    } catch (IOException e) {

      logger.warn("Failed downloading {}.", url, e);

      return;

    }

    if (null == downloadedBytes) {

      logger.warn("Could not download {}.", url);

      return;

    }

    logger.debug("Finished downloading {}.", url);
@@ -72,7 +76,7 @@ public class SnowflakeStatsDownloader extends CollecTorMain { 
    SortedSet<LocalDateTime> snowflakeStatsEnds = new TreeSet<>();

    String outputPathName = config.getPath(Key.OutputPath).toString();

    for (Descriptor descriptor : descriptorParser.parseDescriptors(

        downloadedSnowflakeStats.toByteArray(), null, null)) {

        downloadedBytes, null, null)) {

      if (descriptor instanceof SnowflakeStats) {

        SnowflakeStats snowflakeStats = (SnowflakeStats) descriptor;

        LocalDateTime snowflakeStatsEnd = snowflakeStats.snowflakeStatsEnd();
@@ -105,44 +109,6 @@ public class SnowflakeStatsDownloader extends CollecTorMain { 
    this.cleanUpRsyncDirectory();

  }



  /**

   * Download the given URL from an HTTP server and return a stream with

   * downloaded bytes.

   *

   * <p>If anything goes wrong while downloading, log a warning and return

   * {@code null}.</p>

   *

   * @param url URL to download.

   * @return Stream with downloaded bytes, or {@code null} if an error has

   *     occurred.

   */

  private ByteArrayOutputStream downloadFromHttpServer(URL url) {

    ByteArrayOutputStream downloadedBytes = new ByteArrayOutputStream();

    try  {

      HttpURLConnection huc = (HttpURLConnection) url.openConnection();

      huc.setRequestMethod("GET");

      huc.setReadTimeout(5000);

      huc.connect();

      int response = huc.getResponseCode();

      if (response != 200) {

        logger.warn("Could not download {}. Response code {}", url, response);

        return null;

      }

      try (BufferedInputStream in = new BufferedInputStream(

          huc.getInputStream())) {

        int len;

        byte[] data = new byte[1024];

        while ((len = in.read(data, 0, 1024)) >= 0) {

          downloadedBytes.write(data, 0, len);

        }

      }

    } catch (IOException e) {

      logger.warn("Failed downloading {}.", url, e);

      return null;

    }

    return downloadedBytes;

  }



  /**

   * Write the given byte array(s) to the given file.

   *