Introduce a new ExitList.Entry type.

Patch for #17821

CHANGELOG.md

@@ -7,6 +7,11 @@
    - Support parsing of .xz-compressed tarballs using Apache Commons
      Compress and XZ for Java.  Applications only need to add XZ for
      Java as dependency if they want to parse .xz-compressed tarballs.
+   - Introduce a new ExitList.Entry type for exit list entries instead
+     of the ExitListEntry type which is now deprecated.  The main
+     difference between the two is that ExitList.Entry can hold more
+     than one exit address and scan time which were previously parsed
+     as multiple ExitListEntry instances.
 
 
 # Changes in version 1.0.0 - 2015-12-05

src/org/torproject/descriptor/ExitList.java

@@ -2,15 +2,41 @@
  * See LICENSE for licensing information */
 package org.torproject.descriptor;
 
+import java.util.Map;
 import java.util.Set;
 
 /* Exit list containing all known exit scan results at a given time. */
 public interface ExitList extends Descriptor {
 
+  public final static String EOL = "\n";
+
+  /* Exit list entry containing results from a single exit scan. */
+  public interface Entry {
+
+    /* Return the scanned relay's fingerprint. */
+    public String getFingerprint();
+
+    /* Return the publication time of the scanned relay's last known
+     * descriptor. */
+    public long getPublishedMillis();
+
+    /* Return the publication time of the network status that this scan
+     * was based on. */
+    public long getLastStatusMillis();
+
+    /* Return the IP addresses that were determined in the scan. */
+    public Map<String, Long> getExitAddresses();
+  }
+
   /* Return the download time of the exit list. */
   public long getDownloadedMillis();
 
   /* Return the unordered set of exit scan results. */
+  /* Use getEntries instead. */
+  @Deprecated
   public Set<ExitListEntry> getExitListEntries();
+
+  /* Return the unordered set of exit scan results. */
+  public Set<ExitList.Entry> getEntries();
 }
 

src/org/torproject/descriptor/ExitListEntry.java

@@ -3,7 +3,9 @@
 package org.torproject.descriptor;
 
 /* Exit list entry containing results from a single exit scan. */
-public interface ExitListEntry {
+/* Use org.torproject.descriptor.ExitList.Entry instead. */
+@Deprecated
+public interface ExitListEntry extends ExitList.Entry {
 
   /* Return the scanned relay's fingerprint. */
   public String getFingerprint();

src/org/torproject/descriptor/impl/ExitListEntryImpl.java

@@ -3,15 +3,19 @@
 package org.torproject.descriptor.impl;
 
 import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.ExitList;
+
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Scanner;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
 import org.torproject.descriptor.ExitListEntry;
 
-public class ExitListEntryImpl implements ExitListEntry {
+public class ExitListEntryImpl implements ExitListEntry, ExitList.Entry {
 
   private byte[] exitListEntryBytes;
   public byte[] getExitListEntryBytes() {
@@ -26,6 +30,31 @@ public class ExitListEntryImpl implements ExitListEntry {
     return lines;
   }
 
+  @Deprecated
+  private ExitListEntryImpl(String fingerprint, long publishedMillis,
+      long lastStatusMillis, String exitAddress, long scanMillis) {
+    this.fingerprint = fingerprint;
+    this.publishedMillis = publishedMillis;
+    this.lastStatusMillis = lastStatusMillis;
+    this.exitAddresses.put(exitAddress, scanMillis);
+  }
+
+  @Deprecated
+  List<ExitListEntry> oldEntries() {
+    List<ExitListEntry> result = new ArrayList<>();
+    if (this.exitAddresses.size() > 1) {
+      for (Map.Entry<String, Long> entry :
+          this.exitAddresses.entrySet()) {
+        result.add(new ExitListEntryImpl(this.fingerprint,
+            this.publishedMillis, this.lastStatusMillis, entry.getKey(),
+            entry.getValue()));
+      }
+    } else {
+      result.add(this);
+    }
+    return result;
+  }
+
   protected ExitListEntryImpl(byte[] exitListEntryBytes,
       boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
@@ -37,56 +66,63 @@ public class ExitListEntryImpl implements ExitListEntry {
     this.checkAndClearKeywords();
   }
 
-  private SortedSet<String> exactlyOnceKeywords;
+  private SortedSet<String> keywordCountingSet;
   private void initializeKeywords() {
-    this.exactlyOnceKeywords = new TreeSet<String>();
-    this.exactlyOnceKeywords.add("ExitNode");
-    this.exactlyOnceKeywords.add("Published");
-    this.exactlyOnceKeywords.add("LastStatus");
-    this.exactlyOnceKeywords.add("ExitAddress");
+    this.keywordCountingSet = new TreeSet<String>();
+    this.keywordCountingSet.add("ExitNode");
+    this.keywordCountingSet.add("Published");
+    this.keywordCountingSet.add("LastStatus");
+    this.keywordCountingSet.add("ExitAddress");
   }
 
   private void parsedExactlyOnceKeyword(String keyword)
       throws DescriptorParseException {
-    if (!this.exactlyOnceKeywords.contains(keyword)) {
+    if (!this.keywordCountingSet.contains(keyword)) {
       throw new DescriptorParseException("Duplicate '" + keyword
           + "' line in exit list entry.");
     }
-    this.exactlyOnceKeywords.remove(keyword);
+    this.keywordCountingSet.remove(keyword);
   }
 
   private void checkAndClearKeywords() throws DescriptorParseException {
-    for (String missingKeyword : this.exactlyOnceKeywords) {
+    for (String missingKeyword : this.keywordCountingSet) {
       throw new DescriptorParseException("Missing '" + missingKeyword
           + "' line in exit list entry.");
     }
-    this.exactlyOnceKeywords = null;
+    this.keywordCountingSet = null;
   }
 
   private void parseExitListEntryBytes()
       throws DescriptorParseException {
     Scanner s = new Scanner(new String(this.exitListEntryBytes)).
-        useDelimiter("\n");
+        useDelimiter(ExitList.EOL);
     while (s.hasNext()) {
       String line = s.next();
       String[] parts = line.split(" ");
       String keyword = parts[0];
-      if (keyword.equals("ExitNode")) {
-        this.parseExitNodeLine(line, parts);
-      } else if (keyword.equals("Published")) {
-        this.parsePublishedLine(line, parts);
-      } else if (keyword.equals("LastStatus")) {
-        this.parseLastStatusLine(line, parts);
-      } else if (keyword.equals("ExitAddress")) {
-        this.parseExitAddressLine(line, parts);
-      } else if (this.failUnrecognizedDescriptorLines) {
-        throw new DescriptorParseException("Unrecognized line '" + line
-            + "' in exit list entry.");
-      } else {
-        if (this.unrecognizedLines == null) {
-          this.unrecognizedLines = new ArrayList<String>();
-        }
-        this.unrecognizedLines.add(line);
+      switch (keyword) {
+        case "ExitNode":
+          this.parseExitNodeLine(line, parts);
+          break;
+        case "Published":
+          this.parsePublishedLine(line, parts);
+          break;
+        case "LastStatus":
+          this.parseLastStatusLine(line, parts);
+          break;
+        case "ExitAddress":
+          this.parseExitAddressLine(line, parts);
+          break;
+        default:
+          if (this.failUnrecognizedDescriptorLines) {
+            throw new DescriptorParseException("Unrecognized line '"
+                + line + "' in exit list entry.");
+          } else {
+            if (this.unrecognizedLines == null) {
+              this.unrecognizedLines = new ArrayList<>();
+            }
+            this.unrecognizedLines.add(line);
+          }
       }
     }
   }
@@ -130,10 +166,9 @@ public class ExitListEntryImpl implements ExitListEntry {
       throw new DescriptorParseException("Invalid line '" + line + "' in "
           + "exit list entry.");
     }
-    this.parsedExactlyOnceKeyword(parts[0]);
-    this.exitAddress = ParseHelper.parseIpv4Address(line, parts[1]);
-    this.scanMillis = ParseHelper.parseTimestampAtIndex(line, parts,
-        2, 3);
+    this.keywordCountingSet.remove(parts[0]);
+    this.exitAddresses.put(ParseHelper.parseIpv4Address(line, parts[1]),
+        ParseHelper.parseTimestampAtIndex(line, parts, 2, 3));
   }
 
   private String fingerprint;
@@ -153,12 +188,26 @@ public class ExitListEntryImpl implements ExitListEntry {
 
   private String exitAddress;
   public String getExitAddress() {
+    if (null == exitAddress) {
+      Map.Entry<String, Long> randomEntry =
+          this.exitAddresses.entrySet().iterator().next();
+      this.exitAddress = randomEntry.getKey();
+      this.scanMillis = randomEntry.getValue();
+    }
     return this.exitAddress;
   }
 
+  private Map<String, Long> exitAddresses = new HashMap<>();
+  public Map<String, Long> getExitAddresses(){
+    return new HashMap<>(this.exitAddresses);
+  }
+
   private long scanMillis;
   public long getScanMillis() {
-    return this.scanMillis;
+    if (null == exitAddress) {
+      getExitAddress();
+    }
+    return scanMillis;
   }
 }
 

src/org/torproject/descriptor/impl/ExitListImpl.java

@@ -15,7 +15,6 @@ import java.util.TimeZone;
 import org.torproject.descriptor.ExitList;
 import org.torproject.descriptor.ExitListEntry;
 
-/* TODO Add test class. */
 public class ExitListImpl extends DescriptorImpl implements ExitList {
 
   protected ExitListImpl(byte[] rawDescriptorBytes, String fileName,
@@ -52,36 +51,57 @@ public class ExitListImpl extends DescriptorImpl implements ExitList {
       throw new DescriptorParseException("Descriptor is empty.");
     }
     String descriptorString = new String(rawDescriptorBytes);
-    Scanner s = new Scanner(descriptorString).useDelimiter("\n");
+    Scanner s = new Scanner(descriptorString).useDelimiter(EOL);
     StringBuilder sb = new StringBuilder();
+    boolean firstEntry = true;
     while (s.hasNext()) {
       String line = s.next();
+      if (line.startsWith("@")) { /* Skip annotation. */
+        if (!s.hasNext()) {
+          throw new DescriptorParseException("Descriptor is empty.");
+        } else {
+          line = s.next();
+        }
+      }
       String[] parts = line.split(" ");
       String keyword = parts[0];
-      if (keyword.equals("Downloaded")) {
-        this.downloadedMillis = ParseHelper.parseTimestampAtIndex(line,
-            parts, 1, 2);
-      } else if (keyword.equals("ExitNode")) {
-        sb = new StringBuilder();
-        sb.append(line + "\n");
-      } else if (keyword.equals("Published")) {
-        sb.append(line + "\n");
-      } else if (keyword.equals("LastStatus")) {
-        sb.append(line + "\n");
-      } else if (keyword.equals("ExitAddress")) {
-        String exitListEntryString = sb.toString() + line + "\n";
-        byte[] exitListEntryBytes = exitListEntryString.getBytes();
-        this.parseExitListEntry(exitListEntryBytes);
-      } else if (this.failUnrecognizedDescriptorLines) {
-        throw new DescriptorParseException("Unrecognized line '" + line
-            + "' in exit list.");
-      } else {
-        if (this.unrecognizedLines == null) {
-          this.unrecognizedLines = new ArrayList<String>();
-        }
-        this.unrecognizedLines.add(line);
+      switch (keyword) {
+        case "Downloaded":
+          this.downloadedMillis = ParseHelper.parseTimestampAtIndex(line,
+              parts, 1, 2);
+          break;
+        case "ExitNode":
+          if (!firstEntry) {
+            this.parseExitListEntry(sb.toString().getBytes());
+          } else {
+            firstEntry = false;
+          }
+          sb = new StringBuilder();
+          sb.append(line).append(ExitList.EOL);
+          break;
+        case "Published":
+          sb.append(line).append(ExitList.EOL);
+          break;
+        case "LastStatus":
+          sb.append(line).append(ExitList.EOL);
+          break;
+        case "ExitAddress":
+          sb.append(line).append(ExitList.EOL);
+          break;
+        default:
+          if (this.failUnrecognizedDescriptorLines) {
+            throw new DescriptorParseException("Unrecognized line '"
+                + line + "' in exit list.");
+          } else {
+            if (this.unrecognizedLines == null) {
+                this.unrecognizedLines = new ArrayList<String>();
+            }
+            this.unrecognizedLines.add(line);
+          }
       }
     }
+    /* Parse the last entry. */
+    this.parseExitListEntry(sb.toString().getBytes());
   }
 
   protected void parseExitListEntry(byte[] exitListEntryBytes)
@@ -89,6 +109,7 @@ public class ExitListImpl extends DescriptorImpl implements ExitList {
     ExitListEntryImpl exitListEntry = new ExitListEntryImpl(
         exitListEntryBytes, this.failUnrecognizedDescriptorLines);
     this.exitListEntries.add(exitListEntry);
+    this.oldExitListEntries.addAll(exitListEntry.oldEntries());
     List<String> unrecognizedExitListEntryLines = exitListEntry.
         getAndClearUnrecognizedLines();
     if (unrecognizedExitListEntryLines != null) {
@@ -104,10 +125,15 @@ public class ExitListImpl extends DescriptorImpl implements ExitList {
     return this.downloadedMillis;
   }
 
-  private Set<ExitListEntry> exitListEntries =
-      new HashSet<ExitListEntry>();
+  private Set<ExitListEntry> oldExitListEntries = new HashSet<>();
+  @Deprecated
   public Set<ExitListEntry> getExitListEntries() {
-    return new HashSet<ExitListEntry>(this.exitListEntries);
+    return new HashSet<>(this.oldExitListEntries);
+  }
+
+  private Set<ExitList.Entry> exitListEntries = new HashSet<>();
+  public Set<ExitList.Entry> getEntries() {
+    return new HashSet<ExitList.Entry>(this.exitListEntries);
   }
 }
 

test/org/torproject/descriptor/impl/ExitListImplTest.java

+/* Copyright 2015 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.descriptor.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.ExitListEntry;
+
+public class ExitListImplTest {
+
+  @Test()
+  public void testAnnotatedInput() throws Exception {
+    ExitListImpl result = new ExitListImpl((tordnselAnnotation + input)
+        .getBytes("US-ASCII"), fileName, false);
+    assertEquals("Expected one annotation.", 1,
+        result.getAnnotations().size());
+    assertEquals(tordnselAnnotation.substring(0, 18),
+        result.getAnnotations().get(0));
+    assertEquals(1441065722000L, result.getDownloadedMillis());
+    assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
+        result.getUnrecognizedLines().isEmpty());
+    assertEquals("Found: " + result.getExitListEntries(), 7,
+        result.getExitListEntries().size());
+    assertEquals("Found: " + result.getEntries(), 5,
+        result.getEntries().size());
+  }
+
+  @Test()
+  public void testMultipleOldExitAddresses() throws Exception {
+    ExitListImpl result = new ExitListImpl(
+        (tordnselAnnotation + multiExitAddressInput)
+        .getBytes("US-ASCII"), fileName, false);
+    assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
+        result.getUnrecognizedLines().isEmpty());
+    assertEquals("Found: " + result.getExitListEntries(),
+        3, result.getExitListEntries().size());
+    Map<String, Long> testMap = new HashMap();
+    testMap.put("81.7.17.171", 1441044592000L);
+    testMap.put("81.7.17.172", 1441044652000L);
+    testMap.put("81.7.17.173", 1441044712000L);
+    for (ExitListEntry ele : result.getExitListEntries()) {
+      Map<String, Long> map = ele.getExitAddresses();
+      assertEquals("Found: " + map, 1, map.size());
+      Map.Entry<String, Long> ea = map.entrySet().iterator().next();
+      assertTrue("Map: " + testMap,
+          testMap.keySet().contains(ea.getKey()));
+      assertTrue("Map: " + testMap + " exitaddress: " + ea,
+          testMap.values().contains(ea.getValue()));
+      testMap.remove(ea.getKey());
+    }
+    assertTrue("Map: " + testMap, testMap.isEmpty());
+  }
+
+  @Test()
+  public void testMultipleExitAddresses() throws Exception {
+    ExitListImpl result = new ExitListImpl(
+        (tordnselAnnotation + multiExitAddressInput)
+        .getBytes("US-ASCII"), fileName, false);
+    assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
+        result.getUnrecognizedLines().isEmpty());
+    Map<String, Long> map = result.getEntries()
+        .iterator().next().getExitAddresses();
+    assertEquals("Found: " + map, 3, map.size());
+    assertTrue("Map: " + map, map.containsKey("81.7.17.171"));
+    assertTrue("Map: " + map, map.containsKey("81.7.17.172"));
+    assertTrue("Map: " + map, map.containsKey("81.7.17.173"));
+  }
+
+  @Test(expected = DescriptorParseException.class)
+  public void testInsufficientInput0() throws Exception {
+    new ExitListImpl((tordnselAnnotation + insufficientInput[0])
+        .getBytes("US-ASCII"), fileName, false);
+  }
+
+  @Test(expected = DescriptorParseException.class)
+  public void testInsufficientInput1() throws Exception {
+    new ExitListImpl((tordnselAnnotation + insufficientInput[1])
+        .getBytes("US-ASCII"), fileName, false);
+  }
+
+  private static final String tordnselAnnotation = "@type tordnsel 1.0\n";
+  private static final String fileName = "2015-09-01-00-02-02";
+  private static final String[] insufficientInput = new String[] {
+      "Downloaded 2015-09-01 00:02:02\n"
+      + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n"
+      + "Published 2015-08-31 16:17:30\n"
+      + "LastStatus 2015-08-31 17:03:18\n",
+      "Downloaded 2015-09-01 00:02:02\n"
+      + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n"
+      + "LastStatus 2015-08-31 17:03:18\n"
+      + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n" };
+
+  private static final String multiExitAddressInput =
+      "Downloaded 2015-09-01 00:02:02\n"
+      + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n"
+      + "Published 2015-08-31 16:17:30\n"
+      + "LastStatus 2015-08-31 17:03:18\n"
+      + "ExitAddress 81.7.17.171 2015-08-31 18:09:52\n"
+      + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n"
+      + "ExitAddress 81.7.17.173 2015-08-31 18:11:52\n";
+  private static final String input = "Downloaded 2015-09-01 00:02:02\n"
+      + "ExitNode 0011BD2485AD45D984EC4159C88FC066E5E3300E\n"
+      + "Published 2015-08-31 16:17:30\n"
+      + "LastStatus 2015-08-31 17:03:18\n"
+      + "ExitAddress 162.247.72.201 2015-08-31 17:09:23\n"
+      + "ExitNode 0098C475875ABC4AA864738B1D1079F711C38287\n"
+      + "Published 2015-08-31 13:59:24\n"
+      + "LastStatus 2015-08-31 15:03:20\n"
+      + "ExitAddress 162.248.160.151 2015-08-31 15:07:27\n"
+      + "ExitNode 00C4B4731658D3B4987132A3F77100CFCB190D97\n"
+      + "Published 2015-08-31 17:47:52\n"
+      + "LastStatus 2015-08-31 18:03:17\n"
+      + "ExitAddress 81.7.17.171 2015-08-31 18:09:52\n"
+      + "ExitAddress 81.7.17.172 2015-08-31 18:10:52\n"
+      + "ExitAddress 81.7.17.173 2015-08-31 18:11:52\n"
+      + "ExitNode 00F2D93EBAF2F51D6EE4DCB0F37D91D72F824B16\n"
+      + "Published 2015-08-31 14:39:05\n"
+      + "LastStatus 2015-08-31 16:02:18\n"
+      + "ExitAddress 23.239.18.57 2015-08-31 16:06:07\n"
+      + "ExitNode 011B1D1E876B2C835D01FB9D407F2E00B28077F6\n"
+      + "Published 2015-08-31 05:14:35\n"
+      + "LastStatus 2015-08-31 06:03:29\n"
+      + "ExitAddress 104.131.51.150 2015-08-31 06:04:07\n";
+}
+