Commit 46ab6609 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Add support for Bifroest's bridge descriptor tarballs. 

Implements #20037.
parent 0541b860

CHANGELOG.md

+3 −0
Original line number Diff line number Diff line 
# Changes in version 1.1.0 - 2016-08-31



 * Medium changes

   - Add support for Bifroest's bridge descriptor tarballs.



 * Minor changes

   - Remove quotes around base URL in index.json.

src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java

+3 −2
Original line number Diff line number Diff line
@@ -27,7 +27,7 @@ public class BridgeDescriptorParser { 


  /** Parses the first line of the given descriptor data to determine the

   * descriptor type and passes it to the sanitized bridges writer. */

  public void parse(byte[] allData, String dateTime)

  public void parse(byte[] allData, String dateTime, String authorityFingerprint)

      throws ConfigurationException {

    try {

      BufferedReader br = new BufferedReader(new StringReader(
@@ -45,7 +45,8 @@ public class BridgeDescriptorParser { 
        }

      } else {

        if (this.sbw != null) {

          this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime);

          this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime,

              authorityFingerprint);

        }

      }

    } catch (IOException e) {

src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java

+31 −5
Original line number Diff line number Diff line
@@ -100,9 +100,35 @@ public class BridgeSnapshotReader { 
              }

              BufferedInputStream bis = new BufferedInputStream(tais);

              String fn = pop.getName();

              String dateTime = fn.substring(11, 21) + " "

                    + fn.substring(22, 24) + ":" + fn.substring(24, 26)

                    + ":" + fn.substring(26, 28);

              String[] fnParts = fn.split("-");

              if (fnParts.length != 5) {

                logger.warn("Invalid bridge descriptor tarball file name: "

                    + fn + ".  Skipping.");

                continue;

              }

              String authorityPart = String.format("%s-%s-", fnParts[0],

                  fnParts[1]);

              String datePart = String.format("%s-%s-%s", fnParts[2],

                  fnParts[3], fnParts[4]);

              String authorityFingerprint;

              switch (authorityPart) {

                case "from-tonga-":

                  authorityFingerprint =

                      "4A0CCD2DDC7995083D73F5D667100C8A5831F16D";

                  break;

                case "from-bifroest-":

                  authorityFingerprint =

                      "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1";

                  break;

                default:

                  logger.warn("Did not recognize the bridge authority that "

                      + "generated " + fn + ".  Skipping.");

                  continue;

              }

              String dateTime = datePart.substring(0, 10) + " "

                  + datePart.substring(11, 13) + ":"

                  + datePart.substring(13, 15) + ":"

                  + datePart.substring(15, 17);

              while ((tais.getNextTarEntry()) != null) {

                ByteArrayOutputStream baos = new ByteArrayOutputStream();

                int len;
@@ -130,7 +156,7 @@ public class BridgeSnapshotReader { 
                if (firstLine.startsWith("published ")

                    || firstLine.startsWith("flag-thresholds ")

                    || firstLine.startsWith("r ")) {

                  bdp.parse(allData, dateTime);

                  bdp.parse(allData, dateTime, authorityFingerprint);

                  parsedStatuses++;

                } else if (descriptorImportHistory.contains(fileDigest)) {

                  /* Skip server descriptors or extra-info descriptors if
@@ -167,7 +193,7 @@ public class BridgeSnapshotReader { 
                        DigestUtils.sha(descBytes));

                    if (!descriptorImportHistory.contains(

                        descriptorDigest)) {

                      bdp.parse(descBytes, dateTime);

                      bdp.parse(descBytes, dateTime, authorityFingerprint);

                      descriptorImportHistory.add(descriptorDigest);

                      if (firstLine.startsWith("router ")) {

                        parsedServerDescriptors++;

src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java

+3 −3
Original line number Diff line number Diff line
@@ -379,7 +379,8 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
   * Sanitizes a network status and writes it to disk.

   */

  public void sanitizeAndStoreNetworkStatus(byte[] data,

      String publicationTime) throws ConfigurationException {

      String publicationTime, String authorityFingerprint)

      throws ConfigurationException {



    if (this.persistenceProblemWithSecrets) {

      /* There's a persistence problem, so we shouldn't scrub more IP
@@ -546,8 +547,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { 
      File tarballFile = new File(

          this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear

          + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth

          + sday + "-" + stime + "-"

          + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D");

          + sday + "-" + stime + "-" + authorityFingerprint);

      File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),

          "bridge-descriptors/statuses/" + tarballFile.getName());

      File[] outputFiles = new File[] { tarballFile, rsyncFile };