Commit 27cbfc83 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Don't write sanitize bridge descriptors to recent/. 

This saves a lot of time and disk space when processing bridge
descriptors in bulk.  There might be other situations (?) when
disabling writing to the recent/ (or out/) directory would be
benefitial.
parent e514d304

src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java

+16 −16
Original line number Diff line number Diff line
@@ -58,7 +58,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
    super(config);

  }



  private String rsyncCatString;

  /*private String rsyncCatString;*/



  private File bridgeDirectoriesDirectory;
@@ -107,11 +107,11 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
    this.sanitizedBridgesDirectory = sanitizedBridgesDirectory;

    this.replaceIpAddressesWithHashes =

        config.getBool(Key.ReplaceIpAddressesWithHashes);

    SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(

    /*SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(

        "yyyy-MM-dd-HH-mm-ss");

    rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));

    this.rsyncCatString = rsyncCatFormat.format(

        System.currentTimeMillis());

        System.currentTimeMillis());*/



    /* Initialize secure random number generator if we need it. */

    if (this.replaceIpAddressesWithHashes) {
@@ -199,7 +199,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { 


    this.checkStaleDescriptors();



    this.cleanUpRsyncDirectory();

    /*this.cleanUpRsyncDirectory();*/

  }



  private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
@@ -583,9 +583,9 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
          this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear

          + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth

          + sday + "-" + stime + "-" + authorityFingerprint);

      File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),

          "bridge-descriptors/statuses/" + tarballFile.getName());

      File[] outputFiles = new File[] { tarballFile, rsyncFile };

      /*File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),

          "bridge-descriptors/statuses/" + tarballFile.getName());*/

      File[] outputFiles = new File[] { tarballFile /*, rsyncFile*/ };

      for (File outputFile : outputFiles) {

        outputFile.getParentFile().mkdirs();

        BufferedWriter bw = new BufferedWriter(new FileWriter(
@@ -948,10 +948,10 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
        + descriptorDigest.charAt(1) + "/"

        + descriptorDigest);

    try {

      File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(),

      /*File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(),

          "bridge-descriptors/server-descriptors/" + this.rsyncCatString

          + "-server-descriptors.tmp");

      File[] outputFiles = new File[] { tarballFile, rsyncCatFile };

          + "-server-descriptors.tmp");*/

      File[] outputFiles = new File[] { tarballFile /*, rsyncCatFile*/ };

      boolean[] append = new boolean[] { false, true };

      for (int i = 0; i < outputFiles.length; i++) {

        File outputFile = outputFiles[i];
@@ -974,7 +974,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
            + "\n");

        bw.close();

      }

    } catch (ConfigurationException | IOException e) {

    } catch (IOException e) {

      logger.warn("Could not write sanitized server "

          + "descriptor to disk.", e);

      return;
@@ -1226,10 +1226,10 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
        + descriptorDigest.charAt(1) + "/"

        + descriptorDigest);

    try {

      File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(),

      /*File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(),

          "bridge-descriptors/extra-infos/" + this.rsyncCatString

          + "-extra-infos.tmp");

      File[] outputFiles = new File[] { tarballFile, rsyncCatFile };

          + "-extra-infos.tmp");*/

      File[] outputFiles = new File[] { tarballFile /*, rsyncCatFile*/ };

      boolean[] append = new boolean[] { false, true };

      for (int i = 0; i < outputFiles.length; i++) {

        File outputFile = outputFiles[i];
@@ -1338,7 +1338,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
  /** Delete all files from the rsync directory that have not been modified

   * in the last three days, and remove the .tmp extension from newly

   * written files. */

  public void cleanUpRsyncDirectory() throws ConfigurationException {

  /*public void cleanUpRsyncDirectory() throws ConfigurationException {

    long cutOffMillis = System.currentTimeMillis()

        - 3L * 24L * 60L * 60L * 1000L;

    Stack<File> allFiles = new Stack<File>();
@@ -1356,6 +1356,6 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
            file.getName().lastIndexOf(".tmp"))));

      }

    }

  }

  }*/

}