Add "recommended_version" parameter.

Add a "recommended_version" parameter to return only relays and
bridges running a Tor software version that is recommended or not
recommended by the directory authorities.

Implements #23544.

CHANGELOG.md

@@ -10,6 +10,9 @@
    - Add a "recommended_version" field to bridge details documents
      based on whether the directory authorities recommend the bridge's
      version.
+   - Add a "recommended_version" parameter to return only relays and
+     bridges running a Tor software version that is recommended or not
+     recommended by the directory authorities.
 
 
 # Changes in version 4.3-1.7.1 - 2017-11-17

src/main/java/org/torproject/onionoo/docs/DocumentStore.java

@@ -440,10 +440,12 @@ public class DocumentStore {
     long consensusWeight = -1L;
     long firstSeenMillis = -1L;
     String hostName = null;
+    Boolean recommendedVersion = null;
     SummaryDocument summaryDocument = new SummaryDocument(isRelay,
         nickname, fingerprint, addresses, lastSeenMillis, running,
         relayFlags, consensusWeight, countryCode, firstSeenMillis,
-        asNumber, contact, family, family, version, hostName);
+        asNumber, contact, family, family, version, hostName,
+        recommendedVersion);
     return summaryDocument;
   }
 

src/main/java/org/torproject/onionoo/docs/SummaryDocument.java

@@ -322,6 +322,18 @@ public class SummaryDocument extends Document {
     return this.hostName;
   }
 
+  @Expose
+  @SerializedName("rv")
+  private Boolean recommendedVersion;
+
+  public void setRecommendedVersion(Boolean recommendedVersion) {
+    this.recommendedVersion = recommendedVersion;
+  }
+
+  public Boolean getRecommendedVersion() {
+    return this.recommendedVersion;
+  }
+
   /* The familyFingerprints parameter can go away after September 8, 2015.
    * See above. */
   /** Instantiates a summary document with all given properties. */
@@ -330,7 +342,8 @@ public class SummaryDocument extends Document {
       boolean running, SortedSet<String> relayFlags, long consensusWeight,
       String countryCode, long firstSeenMillis, String asNumber,
       String contact, SortedSet<String> familyFingerprints,
-      SortedSet<String> effectiveFamily, String version, String hostName) {
+      SortedSet<String> effectiveFamily, String version, String hostName,
+      Boolean recommendedVersion) {
     this.setRelay(isRelay);
     this.setNickname(nickname);
     this.setFingerprint(fingerprint);
@@ -347,6 +360,7 @@ public class SummaryDocument extends Document {
     this.setEffectiveFamily(effectiveFamily);
     this.setVersion(version);
     this.setHostName(hostName);
+    this.setRecommendedVersion(recommendedVersion);
   }
 }
 

src/main/java/org/torproject/onionoo/server/NodeIndex.java

@@ -199,5 +199,27 @@ class NodeIndex {
   public Map<String, Set<String>> getRelaysByHostName() {
     return this.relaysByHostName;
   }
+
+  private Map<Boolean, Set<String>> relaysByRecommendedVersion;
+
+  public void setRelaysByRecommendedVersion(
+      Map<Boolean, Set<String>> relaysByRecommendedVersion) {
+    this.relaysByRecommendedVersion = relaysByRecommendedVersion;
+  }
+
+  public Map<Boolean, Set<String>> getRelaysByRecommendedVersion() {
+    return this.relaysByRecommendedVersion;
+  }
+
+  private Map<Boolean, Set<String>> bridgesByRecommendedVersion;
+
+  public void setBridgesByRecommendedVersion(
+      Map<Boolean, Set<String>> bridgesByRecommendedVersion) {
+    this.bridgesByRecommendedVersion = bridgesByRecommendedVersion;
+  }
+
+  public Map<Boolean, Set<String>> getBridgesByRecommendedVersion() {
+    return this.bridgesByRecommendedVersion;
+  }
 }
 

src/main/java/org/torproject/onionoo/server/NodeIndexer.java

@@ -156,6 +156,12 @@ public class NodeIndexer implements ServletContextListener, Runnable {
     Map<String, Set<String>> newRelaysByVersion = new HashMap<>();
     Map<String, Set<String>> newBridgesByVersion = new HashMap<>();
     Map<String, Set<String>> newRelaysByHostName = new HashMap<>();
+    Map<Boolean, Set<String>> newRelaysByRecommendedVersion = new HashMap<>();
+    newRelaysByRecommendedVersion.put(true, new HashSet<>());
+    newRelaysByRecommendedVersion.put(false, new HashSet<>());
+    Map<Boolean, Set<String>> newBridgesByRecommendedVersion = new HashMap<>();
+    newBridgesByRecommendedVersion.put(true, new HashSet<>());
+    newBridgesByRecommendedVersion.put(false, new HashSet<>());
     SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>();
     SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>();
     SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>();
@@ -269,6 +275,12 @@ public class NodeIndexer implements ServletContextListener, Runnable {
         newRelaysByHostName.get(hostNameLowerCase).add(fingerprint);
         newRelaysByHostName.get(hostNameLowerCase).add(hashedFingerprint);
       }
+      Boolean recommendedVersion = entry.getRecommendedVersion();
+      if (null != recommendedVersion) {
+        newRelaysByRecommendedVersion.get(recommendedVersion).add(fingerprint);
+        newRelaysByRecommendedVersion.get(recommendedVersion).add(
+            hashedFingerprint);
+      }
     }
     /* This loop can go away once all Onionoo services had their hourly
      * updater write effective families to summary documents at least
@@ -332,6 +344,13 @@ public class NodeIndexer implements ServletContextListener, Runnable {
         newBridgesByVersion.get(version).add(hashedFingerprint);
         newBridgesByVersion.get(version).add(hashedHashedFingerprint);
       }
+      Boolean recommendedVersion = entry.getRecommendedVersion();
+      if (null != recommendedVersion) {
+        newBridgesByRecommendedVersion.get(recommendedVersion).add(
+            hashedFingerprint);
+        newBridgesByRecommendedVersion.get(recommendedVersion).add(
+            hashedHashedFingerprint);
+      }
     }
     NodeIndex newNodeIndex = new NodeIndex();
     newNodeIndex.setRelayFingerprintSummaryLines(
@@ -353,6 +372,8 @@ public class NodeIndexer implements ServletContextListener, Runnable {
     newNodeIndex.setRelaysByVersion(newRelaysByVersion);
     newNodeIndex.setBridgesByVersion(newBridgesByVersion);
     newNodeIndex.setRelaysByHostName(newRelaysByHostName);
+    newNodeIndex.setRelaysByRecommendedVersion(newRelaysByRecommendedVersion);
+    newNodeIndex.setBridgesByRecommendedVersion(newBridgesByRecommendedVersion);
     synchronized (this) {
       this.lastIndexed = updateStatusMillis;
       this.latestNodeIndex = newNodeIndex;

src/main/java/org/torproject/onionoo/server/RequestHandler.java

@@ -103,6 +103,12 @@ public class RequestHandler {
     this.hostName = hostName;
   }
 
+  private Boolean recommendedVersion;
+
+  public void setRecommendedVersion(Boolean recommendedVersion) {
+    this.recommendedVersion = recommendedVersion;
+  }
+
   private String[] order;
 
   public void setOrder(String[] order) {
@@ -172,6 +178,7 @@ public class RequestHandler {
     this.filterByFamily();
     this.filterByVersion();
     this.filterByHostName();
+    this.filterByRecommendedVersion();
     this.order();
     this.offset();
     this.limit();
@@ -570,6 +577,19 @@ public class RequestHandler {
     this.filteredBridges.clear();
   }
 
+  private void filterByRecommendedVersion() {
+    if (null == this.recommendedVersion) {
+      /* Not filtering by recommended version. */
+      return;
+    }
+    Set<String> keepRelays = this.nodeIndex.getRelaysByRecommendedVersion()
+        .get(this.recommendedVersion);
+    this.filteredRelays.keySet().retainAll(keepRelays);
+    Set<String> keepBridges = this.nodeIndex.getBridgesByRecommendedVersion()
+        .get(this.recommendedVersion);
+    this.filteredBridges.keySet().retainAll(keepBridges);
+  }
+
   private void order() {
     List<SummaryDocument> uniqueRelays = new ArrayList<>();
     List<SummaryDocument> uniqueBridges = new ArrayList<>();

src/main/java/org/torproject/onionoo/server/ResourceServlet.java

@@ -66,9 +66,10 @@ public class ResourceServlet extends HttpServlet {
   private static final long CACHE_INTERVAL = 5L * 60L * 1000L;
 
   private static Set<String> knownParameters = new HashSet<>(
-      Arrays.asList(("type,running,search,lookup,fingerprint,country,as,"
-          + "flag,first_seen_days,last_seen_days,contact,order,limit,"
-          + "offset,fields,family,version,host_name").split(",")));
+      Arrays.asList("type", "running", "search", "lookup", "fingerprint",
+          "country", "as", "flag", "first_seen_days", "last_seen_days",
+          "contact", "order", "limit", "offset", "fields", "family", "version",
+          "host_name", "recommended_version"));
 
   private static Set<String> illegalSearchQualifiers =
       new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -301,6 +302,18 @@ public class ResourceServlet extends HttpServlet {
       }
       rh.setHostName(hostNameParameter);
     }
+    if (parameterMap.containsKey("recommended_version")) {
+      String recommendedVersionParameterValue =
+          parameterMap.get("recommended_version").toLowerCase();
+      boolean recommendedVersionRequested = true;
+      if (recommendedVersionParameterValue.equals("false")) {
+        recommendedVersionRequested = false;
+      } else if (!recommendedVersionParameterValue.equals("true")) {
+        response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+        return;
+      }
+      rh.setRecommendedVersion(recommendedVersionRequested);
+    }
     if (parameterMap.containsKey("order")) {
       String[] order = this.parseOrderParameter(parameterMap.get("order"));
       if (order == null) {

src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java

@@ -93,11 +93,12 @@ public class SummaryDocumentWriter implements DocumentWriter {
       String nickname = nodeStatus.getNickname();
       String version = nodeStatus.getVersion();
       String hostName = nodeStatus.getHostName();
+      Boolean recommendedVersion = nodeStatus.getRecommendedVersion();
       SummaryDocument summaryDocument = new SummaryDocument(isRelay,
           nickname, fingerprint, addresses, lastSeenMillis, running,
           relayFlags, consensusWeight, countryCode, firstSeenMillis,
           asNumber, contact, declaredFamily, effectiveFamily, version,
-          hostName);
+          hostName, recommendedVersion);
       if (this.documentStore.store(summaryDocument, fingerprint)) {
         this.writtenDocuments++;
       }

src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java

@@ -27,7 +27,7 @@ public class SummaryDocumentTest {
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
-        null);
+        null, true);
   }
 
   @Test()

src/test/java/org/torproject/onionoo/server/ResourceServletTest.java

@@ -144,7 +144,7 @@ public class ResourceServletTest {
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })),
-        "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de");
+        "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", true);
     this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A",
         relayTorkaZ);
     org.torproject.onionoo.docs.SummaryDocument relayFerrari458 =
@@ -159,7 +159,7 @@ public class ResourceServletTest {
             "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
         new TreeSet<>(Arrays.asList(new String[] {
             "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null,
-            "c-68-38-171-200.hsd1.in.comcast.net");
+            "c-68-38-171-200.hsd1.in.comcast.net", null);
     this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC",
         relayFerrari458);
     org.torproject.onionoo.docs.SummaryDocument relayTimMayTribute =
@@ -172,7 +172,8 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
         "1024d/51e2a1c7 \"steven j. murdoch\" "
         + "<tor+steven.murdoch@cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
-        new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", null);
+        new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", null,
+        false);
     this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B",
         relayTimMayTribute);
     this.bridges = new TreeMap<>();
@@ -183,7 +184,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-21 18:07:03"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
         null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
-        null, null, "0.2.2.39", null);
+        null, null, "0.2.2.39", null, true);
     this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
         bridgeec2bridgercc7f31fe);
     org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed =
@@ -193,7 +194,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-20 17:37:04"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
         null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
-        null, null, null, null);
+        null, null, null, null, null);
     this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
         bridgeUnnamed);
     org.torproject.onionoo.docs.SummaryDocument bridgegummy =
@@ -204,7 +205,7 @@ public class ResourceServletTest {
         new TreeSet<>(Arrays.asList(new String[] { "Running",
             "Valid" })), -1L, null,
         DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
-        null, "0.2.4.4-alpha-dev", null);
+        null, "0.2.4.4-alpha-dev", null, false);
     this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
         bridgegummy);
   }
@@ -1695,5 +1696,28 @@ public class ResourceServletTest {
   public void testHostNameUmlaut() {
     this.assertErrorStatusCode("/summary?host_name=äöü", 400);
   }
+
+  @Test
+  public void testRecommendedVersionTrue() {
+    this.assertSummaryDocument("/summary?recommended_version=true", 1,
+        new String[] { "TorkaZ" }, 1, new String[] { "ec2bridgercc7f31fe" });
+  }
+
+  @Test
+  public void testRecommendedVersionFalse() {
+    this.assertSummaryDocument("/summary?recommended_version=false", 1,
+        new String[] { "TimMayTribute" }, 1, new String[] { "gummy" });
+  }
+
+  @Test
+  public void testRecommendedVersionTrueCapitalized() {
+    this.assertSummaryDocument("/summary?recommended_version=TRUE", 1,
+        new String[] { "TorkaZ" }, 1,new String[] { "ec2bridgercc7f31fe" });
+  }
+
+  @Test
+  public void testRecommendedVersionNull() {
+    this.assertErrorStatusCode("/summary?recommended_version=null", 400);
+  }
 }
 

src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java

@@ -41,7 +41,7 @@ public class SummaryDocumentComparatorTest {
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
-        null);
+        null, null);
   }
 
   /** Some values for running all comparison types. */