Parse hidserv-stats in extra-info descriptors.

This patch is loosely based on metrics-web's hidserv module.

CHANGELOG.md

@@ -23,6 +23,7 @@
      extra-info descriptors, and support Ed25519 master keys in votes.
    - Include RSA-1024 signatures of SHA-1 digests of extra-info
      descriptors, which were parsed and discarded before.
+   - Support hidden-service statistics in extra-info descriptors.
 
 
 # Changes in version 1.0.0 - 2015-12-05

src/org/torproject/descriptor/ExtraInfoDescriptor.java

@@ -3,6 +3,7 @@
 package org.torproject.descriptor;
 
 import java.util.List;
+import java.util.Map;
 import java.util.SortedMap;
 
 /* Contains a relay or bridge extra-info descriptor. */
@@ -265,6 +266,37 @@ public interface ExtraInfoDescriptor extends Descriptor {
    * bridge. */
   public List<String> getTransports();
 
+  /* Return the end of the included hidden-service statistics, or -1 if no
+   * hidden-service statistics are included. */
+  public long getHidservStatsEndMillis();
+
+  /* Return the interval length of the included hidden-service statistics
+  * in seconds, or -1 if no hidden-service statistics are included. */
+  public long getHidservStatsIntervalLength();
+
+  /* Return the approximate number of RELAY cells seen in either direction
+   * on a circuit after receiving and successfully processing a
+   * RENDEZVOUS1 cell, or null if no hidden-service statistics are
+   * included. */
+  public Double getHidservRendRelayedCells();
+
+  /* Return the obfuscation parameters applied to the original measurement
+   * value of RELAY cells seen in either direction on a circuit after
+   * receiving and successfully processing a RENDEZVOUS1 cell, or null if
+   * no hidden-service statistics are included.. */
+  public Map<String, Double> getHidservRendRelayedCellsParameters();
+
+  /* Return the approximate number of unique hidden-service identities
+   * seen in descriptors published to and accepted by this hidden-service
+   * directory, or null if no hidden-service statistics are included. */
+  public Double getHidservDirOnionsSeen();
+
+  /* Return the obfuscation parameters applied to the original measurement
+   * value of unique hidden-service identities seen in descriptors
+   * published to and accepted by this hidden-service directory, or null
+   * if no hidden-service statistics are included. */
+  public Map<String, Double> getHidservDirOnionsSeenParameters();
+
   /* Return the signature of the PKCS1-padded extra-info descriptor
    * digest, or null if the descriptor doesn't contain a signature (which
    * is the case in sanitized bridge descriptors). */

src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java

@@ -9,8 +9,10 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Scanner;
 import java.util.Set;
 import java.util.SortedMap;
@@ -165,6 +167,13 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl
         this.parseBridgeIpTransportsLine(line, lineNoOpt, partsNoOpt);
       } else if (keyword.equals("transport")) {
         this.parseTransportLine(line, lineNoOpt, partsNoOpt);
+      } else if (keyword.equals("hidserv-stats-end")) {
+        this.parseHidservStatsEndLine(line, lineNoOpt, partsNoOpt);
+      } else if (keyword.equals("hidserv-rend-relayed-cells")) {
+        this.parseHidservRendRelayedCellsLine(line, lineNoOpt,
+            partsNoOpt);
+      } else if (keyword.equals("hidserv-dir-onions-seen")) {
+        this.parseHidservDirOnionsSeenLine(line, lineNoOpt, partsNoOpt);
       } else if (keyword.equals("identity-ed25519")) {
         this.parseIdentityEd25519Line(line, lineNoOpt, partsNoOpt);
         nextCrypto = "identity-ed25519";
@@ -642,6 +651,46 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl
     this.transports.add(partsNoOpt[1]);
   }
 
+  private void parseHidservStatsEndLine(String line, String lineNoOpt,
+      String[] partsNoOpt) throws DescriptorParseException {
+    long[] parsedStatsEndData = this.parseStatsEndLine(line, partsNoOpt,
+        5);
+    this.hidservStatsEndMillis = parsedStatsEndData[0];
+    this.hidservStatsIntervalLength = parsedStatsEndData[1];
+  }
+
+  private void parseHidservRendRelayedCellsLine(String line,
+      String lineNoOpt, String[] partsNoOpt)
+      throws DescriptorParseException {
+    if (partsNoOpt.length < 2) {
+      throw new DescriptorParseException("Illegal line '" + line + "'.");
+    }
+    try {
+      this.hidservRendRelayedCells = Double.parseDouble(partsNoOpt[1]);
+    } catch (NumberFormatException e) {
+      throw new DescriptorParseException("Illegal line '" + line + "'.");
+    }
+    this.hidservRendRelayedCellsParameters =
+        ParseHelper.parseSpaceSeparatedStringKeyDoubleValueMap(line,
+        partsNoOpt, 2);
+  }
+
+  private void parseHidservDirOnionsSeenLine(String line,
+      String lineNoOpt, String[] partsNoOpt)
+      throws DescriptorParseException {
+    if (partsNoOpt.length < 2) {
+      throw new DescriptorParseException("Illegal line '" + line + "'.");
+    }
+    try {
+      this.hidservDirOnionsSeen = Double.parseDouble(partsNoOpt[1]);
+    } catch (NumberFormatException e) {
+      throw new DescriptorParseException("Illegal line '" + line + "'.");
+    }
+    this.hidservDirOnionsSeenParameters =
+        ParseHelper.parseSpaceSeparatedStringKeyDoubleValueMap(line,
+        partsNoOpt, 2);
+  }
+
   private void parseRouterSignatureLine(String line, String lineNoOpt,
       String[] partsNoOpt) throws DescriptorParseException {
     if (!lineNoOpt.equals("router-signature")) {
@@ -1057,6 +1106,38 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl
     return new ArrayList<String>(this.transports);
   }
 
+  private long hidservStatsEndMillis = -1L;
+  public long getHidservStatsEndMillis() {
+    return this.hidservStatsEndMillis;
+  }
+
+  private long hidservStatsIntervalLength = -1L;
+  public long getHidservStatsIntervalLength() {
+    return this.hidservStatsIntervalLength;
+  }
+
+  private Double hidservRendRelayedCells;
+  public Double getHidservRendRelayedCells() {
+    return this.hidservRendRelayedCells;
+  }
+
+  private Map<String, Double> hidservRendRelayedCellsParameters;
+  public Map<String, Double> getHidservRendRelayedCellsParameters() {
+    return this.hidservRendRelayedCellsParameters == null ? null :
+        new HashMap<>(this.hidservRendRelayedCellsParameters);
+  }
+
+  private Double hidservDirOnionsSeen;
+  public Double getHidservDirOnionsSeen() {
+    return this.hidservDirOnionsSeen;
+  }
+
+  private Map<String, Double> hidservDirOnionsSeenParameters;
+  public Map<String, Double> getHidservDirOnionsSeenParameters() {
+    return this.hidservDirOnionsSeenParameters == null ? null :
+      new HashMap<>(this.hidservDirOnionsSeenParameters);
+  }
+
   private String routerSignature;
   public String getRouterSignature() {
     return this.routerSignature;

src/org/torproject/descriptor/impl/ParseHelper.java

@@ -6,6 +6,7 @@ import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.SortedMap;
@@ -435,6 +436,39 @@ public class ParseHelper {
     return result;
   }
 
+  protected static Map<String, Double>
+      parseSpaceSeparatedStringKeyDoubleValueMap(String line,
+      String[] partsNoOpt, int startIndex)
+      throws DescriptorParseException {
+    Map<String, Double> result = new LinkedHashMap<>();
+    if (partsNoOpt.length < startIndex) {
+      throw new DescriptorParseException("Line '" + line + "' does not "
+          + "contain a key-value list starting at index " + startIndex
+          + ".");
+    }
+    for (int i = startIndex; i < partsNoOpt.length; i++) {
+      String listElement = partsNoOpt[i];
+      String[] keyAndValue = listElement.split("=");
+      String key = null;
+      Double value = null;
+      if (keyAndValue.length == 2) {
+        try {
+          value = Double.parseDouble(keyAndValue[1]);
+          key = keyAndValue[0];
+        } catch (NumberFormatException e) {
+          /* Handle below. */
+        }
+      }
+      if (key == null) {
+        throw new DescriptorParseException("Line '" + line + "' contains "
+            + "an illegal key or value in list element '" + listElement
+            + "'.");
+      }
+      result.put(key, value);
+    }
+    return result;
+  }
+
   public static String
       parseMasterKeyEd25519FromIdentityEd25519CryptoBlock(
       String identityEd25519CryptoBlock) throws DescriptorParseException {

test/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java

@@ -138,6 +138,13 @@ public class ExtraInfoDescriptorImplTest {
       db.bridgeStatsLines = lines;
       return new RelayExtraInfoDescriptorImpl(db.buildDescriptor(), true);
     }
+    private String hidservStatsLines = null;
+    private static ExtraInfoDescriptor createWithHidservStatsLines(
+        String lines) throws DescriptorParseException {
+      DescriptorBuilder db = new DescriptorBuilder();
+      db.hidservStatsLines = lines;
+      return new RelayExtraInfoDescriptorImpl(db.buildDescriptor(), true);
+    }
     private String unrecognizedLine = null;
     private static ExtraInfoDescriptor createWithUnrecognizedLine(
         String line, boolean failUnrecognizedDescriptorLines)
@@ -232,6 +239,9 @@ public class ExtraInfoDescriptorImplTest {
       if (this.bridgeStatsLines != null) {
         sb.append(this.bridgeStatsLines + "\n");
       }
+      if (this.hidservStatsLines != null) {
+        sb.append(this.hidservStatsLines + "\n");
+      }
       if (this.unrecognizedLine != null) {
         sb.append(this.unrecognizedLine + "\n");
       }
@@ -733,6 +743,62 @@ public class ExtraInfoDescriptorImplTest {
     }
   }
 
+  /* Helper class to build a set of hidserv-stats lines based on default
+   * data and modifications requested by test methods. */
+  private static class HidservStatsBuilder {
+    private String hidservStatsEndLine = "hidserv-stats-end 2015-12-03 "
+        + "14:26:56 (86400 s)";
+    private static ExtraInfoDescriptor createWithHidservStatsEndLine(
+        String line) throws DescriptorParseException {
+      HidservStatsBuilder hsb = new HidservStatsBuilder();
+      hsb.hidservStatsEndLine = line;
+      return DescriptorBuilder.createWithHidservStatsLines(
+          hsb.buildHidservStatsLines());
+    }
+    private String hidservRendRelayedCellsLine =
+        "hidserv-rend-relayed-cells 36474281 delta_f=2048 epsilon=0.30 "
+        + "bin_size=1024";
+    private static ExtraInfoDescriptor
+        createWithHidservRendRelayedCellsLine(String line)
+        throws DescriptorParseException {
+      HidservStatsBuilder hsb = new HidservStatsBuilder();
+      hsb.hidservRendRelayedCellsLine = line;
+      return DescriptorBuilder.createWithHidservStatsLines(
+          hsb.buildHidservStatsLines());
+    }
+    private String hidservDirOnionsSeenLine = "hidserv-dir-onions-seen "
+        + "-3 delta_f=8 epsilon=0.30 bin_size=8";
+    private static ExtraInfoDescriptor createWithHidservDirOnionsSeenLine(
+        String line) throws DescriptorParseException {
+      HidservStatsBuilder hsb = new HidservStatsBuilder();
+      hsb.hidservDirOnionsSeenLine = line;
+      return DescriptorBuilder.createWithHidservStatsLines(
+          hsb.buildHidservStatsLines());
+    }
+    private static ExtraInfoDescriptor createWithDefaultLines()
+        throws DescriptorParseException {
+      return DescriptorBuilder.createWithHidservStatsLines(
+          new HidservStatsBuilder().buildHidservStatsLines());
+    }
+    private String buildHidservStatsLines() {
+      StringBuilder sb = new StringBuilder();
+      if (this.hidservStatsEndLine != null) {
+        sb.append(this.hidservStatsEndLine + "\n");
+      }
+      if (this.hidservRendRelayedCellsLine != null) {
+        sb.append(this.hidservRendRelayedCellsLine + "\n");
+      }
+      if (this.hidservDirOnionsSeenLine != null) {
+        sb.append(this.hidservDirOnionsSeenLine + "\n");
+      }
+      String lines = sb.toString();
+      if (lines.endsWith("\n")) {
+        lines = lines.substring(0, lines.length() - 1);
+      }
+      return lines;
+    }
+  }
+
   @Test()
   public void testSampleDescriptor() throws DescriptorParseException {
     DescriptorBuilder db = new DescriptorBuilder();
@@ -1414,6 +1480,69 @@ public class ExtraInfoDescriptorImplTest {
         "bridge-ip-transports meek=32,obfs3_websocket=8,websocket=64");
   }
 
+  @Test()
+  public void testHidservStatsValid() throws DescriptorParseException {
+    ExtraInfoDescriptor descriptor = HidservStatsBuilder.
+        createWithDefaultLines();
+    assertEquals(1449152816000L, descriptor.getHidservStatsEndMillis());
+    assertEquals(86400L, descriptor.getHidservStatsIntervalLength());
+    assertEquals(36474281.0, descriptor.getHidservRendRelayedCells(),
+        0.0001);
+    Map<String, Double> params =
+        descriptor.getHidservRendRelayedCellsParameters();
+    assertTrue(params.containsKey("delta_f"));
+    assertEquals(2048.0, params.remove("delta_f"), 0.0001);
+    assertTrue(params.containsKey("epsilon"));
+    assertEquals(0.3, params.remove("epsilon"), 0.0001);
+    assertTrue(params.containsKey("bin_size"));
+    assertEquals(1024.0, params.remove("bin_size"), 0.0001);
+    assertTrue(params.isEmpty());
+    assertEquals(-3.0, descriptor.getHidservDirOnionsSeen(), 0.0001);
+    params = descriptor.getHidservDirOnionsSeenParameters();
+    assertTrue(params.containsKey("delta_f"));
+    assertEquals(8.0, params.remove("delta_f"), 0.0001);
+    assertTrue(params.containsKey("epsilon"));
+    assertEquals(0.3, params.remove("epsilon"), 0.0001);
+    assertTrue(params.containsKey("bin_size"));
+    assertEquals(8.0, params.remove("bin_size"), 0.0001);
+    assertTrue(params.isEmpty());
+  }
+
+  @Test()
+  public void testHidservStatsEndLineMissing()
+      throws DescriptorParseException {
+    ExtraInfoDescriptor descriptor =
+        HidservStatsBuilder.createWithHidservStatsEndLine(null);
+    assertEquals(-1L, descriptor.getHidservStatsEndMillis());
+    assertEquals(-1L, descriptor.getHidservStatsIntervalLength());
+  }
+
+  @Test()
+  public void testHidservRendRelayedCellsNoParams()
+        throws DescriptorParseException {
+    ExtraInfoDescriptor descriptor =
+        HidservStatsBuilder.createWithHidservRendRelayedCellsLine(
+        "hidserv-rend-relayed-cells 36474281");
+    assertEquals(36474281.0, descriptor.getHidservRendRelayedCells(),
+        0.0001);
+    assertTrue(
+        descriptor.getHidservRendRelayedCellsParameters().isEmpty());
+  }
+
+  @Test(expected = DescriptorParseException.class)
+  public void testHidservDirOnionsSeenCommaSeparatedParams()
+      throws DescriptorParseException {
+    HidservStatsBuilder.createWithHidservDirOnionsSeenLine(
+        "hidserv-dir-onions-seen -3 delta_f=8,epsilon=0.30,bin_size=8");
+  }
+
+  @Test(expected = DescriptorParseException.class)
+  public void testHidservDirOnionsSeenNoDoubleParams()
+      throws DescriptorParseException {
+    HidservStatsBuilder.createWithHidservDirOnionsSeenLine(
+        "hidserv-dir-onions-seen -3 delta_f=A epsilon=B bin_size=C");
+  }
+
   @Test()
   public void testRouterSignatureOpt()
       throws DescriptorParseException {