Allows filtering by operating system

The filter is applicable to both bridges and relays. It uses basic
string operations to extract the operating system from the platform line
of descriptors.

Fixes: #6946

CHANGELOG.md

@@ -3,6 +3,7 @@
  * Medium changes
    - Provide more accurate DNS results in "verified_host_names" and
      "unverified_host_names".
+   - Allow filtering by operating system using the new "os" parameter.
 
  * Minor changes
    - Index relays with no known country code or autonomous system

src/main/java/org/torproject/onionoo/docs/DocumentStore.java

@@ -448,6 +448,7 @@ public class DocumentStore {
     SortedSet<String> relayFlags = new TreeSet<>();
     SortedSet<String> family = null;
     String version = null;
+    String operatingSystem = null;
     long lastSeenMillis = -1L;
     long consensusWeight = -1L;
     long firstSeenMillis = -1L;
@@ -458,8 +459,8 @@ public class DocumentStore {
     SummaryDocument summaryDocument = new SummaryDocument(isRelay,
         nickname, fingerprint, addresses, lastSeenMillis, running,
         relayFlags, consensusWeight, countryCode, firstSeenMillis,
-        asNumber, contact, family, family, version, hostName,
-        verifiedHostNames, unverifiedHostNames,
+        asNumber, contact, family, family, version, operatingSystem,
+        hostName, verifiedHostNames, unverifiedHostNames,
         recommendedVersion);
     return summaryDocument;
   }

src/main/java/org/torproject/onionoo/docs/NodeStatus.java

@@ -343,6 +343,16 @@ public class NodeStatus extends Document {
     return this.versionStatus;
   }
 
+  private String operatingSystem;
+
+  public void setOperatingSystem(String operatingSystem) {
+    this.operatingSystem = operatingSystem;
+  }
+
+  public String getOperatingSystem() {
+    return this.operatingSystem;
+  }
+
   /* From exit lists: */
 
   private SortedSet<String> exitAddresses;

src/main/java/org/torproject/onionoo/docs/SummaryDocument.java

@@ -298,6 +298,17 @@ public class SummaryDocument extends Document {
     return this.version;
   }
 
+  @JsonProperty("o")
+  private String operatingSystem;
+
+  public void setOperatingSystem(String operatingSystem) {
+    this.operatingSystem = operatingSystem;
+  }
+
+  public String getOperatingSystem() {
+    return this.operatingSystem;
+  }
+
   @JsonProperty("h")
   private String hostName;
 
@@ -355,9 +366,9 @@ public class SummaryDocument extends Document {
       boolean running, SortedSet<String> relayFlags, long consensusWeight,
       String countryCode, long firstSeenMillis, String asNumber,
       String contact, SortedSet<String> familyFingerprints,
-      SortedSet<String> effectiveFamily, String version, String hostName,
-      List<String> verifiedHostNames, List<String> unverifiedHostNames,
-      Boolean recommendedVersion) {
+      SortedSet<String> effectiveFamily, String version, String operatingSystem,
+      String hostName, List<String> verifiedHostNames,
+      List<String> unverifiedHostNames, Boolean recommendedVersion) {
     this.setRelay(isRelay);
     this.setNickname(nickname);
     this.setFingerprint(fingerprint);
@@ -373,6 +384,7 @@ public class SummaryDocument extends Document {
     this.setFamilyFingerprints(familyFingerprints);
     this.setEffectiveFamily(effectiveFamily);
     this.setVersion(version);
+    this.setOperatingSystem(operatingSystem);
     this.setHostName(hostName);
     this.setVerifiedHostNames(verifiedHostNames);
     this.setUnverifiedHostNames(unverifiedHostNames);

src/main/java/org/torproject/onionoo/server/NodeIndex.java

@@ -190,6 +190,28 @@ class NodeIndex {
     return this.bridgesByVersion;
   }
 
+  private Map<String, Set<String>> relaysByOperatingSystem;
+
+  public void setRelaysByOperatingSystem(
+      Map<String, Set<String>> relaysByOperatingSystem) {
+    this.relaysByOperatingSystem = relaysByOperatingSystem;
+  }
+
+  public Map<String, Set<String>> getRelaysByOperatingSystem() {
+    return this.relaysByOperatingSystem;
+  }
+
+  private Map<String, Set<String>> bridgesByOperatingSystem;
+
+  public void setBridgesByOperatingSystem(
+      Map<String, Set<String>> bridgesByOperatingSystem) {
+    this.bridgesByOperatingSystem = bridgesByOperatingSystem;
+  }
+
+  public Map<String, Set<String>> getBridgesByOperatingSystem() {
+    return this.bridgesByOperatingSystem;
+  }
+
   private Map<String, Set<String>> relaysByHostName;
 
   public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) {

src/main/java/org/torproject/onionoo/server/NodeIndexer.java

@@ -157,6 +157,8 @@ public class NodeIndexer implements ServletContextListener, Runnable {
     Map<String, Set<String>> newRelaysByFamily = new HashMap<>();
     Map<String, Set<String>> newRelaysByVersion = new HashMap<>();
     Map<String, Set<String>> newBridgesByVersion = new HashMap<>();
+    Map<String, Set<String>> newRelaysByOperatingSystem = new HashMap<>();
+    Map<String, Set<String>> newBridgesByOperatingSystem = new HashMap<>();
     Map<String, Set<String>> newRelaysByHostName = new HashMap<>();
     Map<Boolean, Set<String>> newRelaysByRecommendedVersion = new HashMap<>();
     newRelaysByRecommendedVersion.put(true, new HashSet<>());
@@ -281,6 +283,14 @@ public class NodeIndexer implements ServletContextListener, Runnable {
         newRelaysByVersion.get(version).add(fingerprint);
         newRelaysByVersion.get(version).add(hashedFingerprint);
       }
+      String operatingSystem = entry.getOperatingSystem();
+      if (null != operatingSystem) {
+        if (!newRelaysByOperatingSystem.containsKey(operatingSystem)) {
+          newRelaysByOperatingSystem.put(operatingSystem, new HashSet<>());
+        }
+        newRelaysByOperatingSystem.get(operatingSystem).add(fingerprint);
+        newRelaysByOperatingSystem.get(operatingSystem).add(hashedFingerprint);
+      }
       List<String> allHostNames = new ArrayList<>();
       List<String> verifiedHostNames = entry.getVerifiedHostNames();
       if (null != verifiedHostNames) {
@@ -367,6 +377,16 @@ public class NodeIndexer implements ServletContextListener, Runnable {
         newBridgesByVersion.get(version).add(hashedFingerprint);
         newBridgesByVersion.get(version).add(hashedHashedFingerprint);
       }
+      String operatingSystem = entry.getOperatingSystem();
+      if (null != operatingSystem) {
+        if (!newBridgesByOperatingSystem.containsKey(operatingSystem)) {
+          newBridgesByOperatingSystem.put(operatingSystem, new HashSet<>());
+        }
+        newBridgesByOperatingSystem.get(operatingSystem)
+            .add(hashedFingerprint);
+        newBridgesByOperatingSystem.get(operatingSystem)
+            .add(hashedHashedFingerprint);
+      }
       Boolean recommendedVersion = entry.getRecommendedVersion();
       if (null != recommendedVersion) {
         newBridgesByRecommendedVersion.get(recommendedVersion).add(
@@ -394,6 +414,8 @@ public class NodeIndexer implements ServletContextListener, Runnable {
     newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis);
     newNodeIndex.setRelaysByVersion(newRelaysByVersion);
     newNodeIndex.setBridgesByVersion(newBridgesByVersion);
+    newNodeIndex.setRelaysByOperatingSystem(newRelaysByOperatingSystem);
+    newNodeIndex.setBridgesByOperatingSystem(newBridgesByOperatingSystem);
     newNodeIndex.setRelaysByHostName(newRelaysByHostName);
     newNodeIndex.setRelaysByRecommendedVersion(newRelaysByRecommendedVersion);
     newNodeIndex.setBridgesByRecommendedVersion(newBridgesByRecommendedVersion);

src/main/java/org/torproject/onionoo/server/RequestHandler.java

@@ -97,6 +97,12 @@ public class RequestHandler {
     this.version = version;
   }
 
+  private String operatingSystem;
+
+  public void setOperatingSystem(String operatingSystem) {
+    this.operatingSystem = operatingSystem;
+  }
+
   private String hostName;
 
   public void setHostName(String hostName) {
@@ -177,6 +183,7 @@ public class RequestHandler {
     this.filterByContact();
     this.filterByFamily();
     this.filterByVersion();
+    this.filterByOperatingSystem();
     this.filterByHostName();
     this.filterByRecommendedVersion();
     this.order();
@@ -558,6 +565,29 @@ public class RequestHandler {
     this.filteredBridges.keySet().retainAll(keepBridges);
   }
 
+  private void filterByOperatingSystem() {
+    if (null == this.operatingSystem) {
+      /* Not filtering by operating system. */
+      return;
+    }
+    Set<String> keepRelays = new HashSet<>();
+    for (Map.Entry<String, Set<String>> e
+        : this.nodeIndex.getRelaysByOperatingSystem().entrySet()) {
+      if (e.getKey().startsWith(this.operatingSystem)) {
+        keepRelays.addAll(e.getValue());
+      }
+    }
+    this.filteredRelays.keySet().retainAll(keepRelays);
+    Set<String> keepBridges = new HashSet<>();
+    for (Map.Entry<String, Set<String>> e
+        : this.nodeIndex.getBridgesByOperatingSystem().entrySet()) {
+      if (e.getKey().startsWith(this.operatingSystem)) {
+        keepBridges.addAll(e.getValue());
+      }
+    }
+    this.filteredBridges.keySet().retainAll(keepBridges);
+  }
+
   private void filterByHostName() {
     if (this.hostName == null) {
       /* Not filtering by host name. */

src/main/java/org/torproject/onionoo/server/ResourceServlet.java

@@ -69,7 +69,7 @@ public class ResourceServlet extends HttpServlet {
       Arrays.asList("type", "running", "search", "lookup", "fingerprint",
           "country", "as", "flag", "first_seen_days", "last_seen_days",
           "contact", "order", "limit", "offset", "fields", "family", "version",
-          "host_name", "recommended_version"));
+          "os", "host_name", "recommended_version"));
 
   private static Set<String> illegalSearchQualifiers =
       new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -293,6 +293,15 @@ public class ResourceServlet extends HttpServlet {
       }
       rh.setVersion(versionParameter);
     }
+    if (parameterMap.containsKey("os")) {
+      String osParameter = this.parseOperatingSystemParameter(
+              parameterMap.get("os"));
+      if (null == osParameter) {
+        response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+        return;
+      }
+      rh.setOperatingSystem(osParameter);
+    }
     if (parameterMap.containsKey("host_name")) {
       String hostNameParameter = this.parseHostNameParameter(
           parameterMap.get("host_name"));
@@ -607,6 +616,16 @@ public class ResourceServlet extends HttpServlet {
     return parameter;
   }
 
+  private String parseOperatingSystemParameter(String parameter) {
+    for (char c : parameter.toCharArray()) {
+      if (c < 32 || c >= 127) {
+        /* Only accept printable ASCII. */
+        return null;
+      }
+    }
+    return parameter.toLowerCase();
+  }
+
   private static Pattern hostNameParameterPattern =
       Pattern.compile("^[0-9A-Za-z_\\.\\-]+$");
 

src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java

@@ -807,6 +807,12 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
       }
 
       nodeStatus.setContact(detailsStatus.getContact());
+      if (null != detailsStatus.getPlatform()) {
+        String[] platformParts = detailsStatus.getPlatform().split(" on ");
+        if (platformParts.length > 1) {
+          nodeStatus.setOperatingSystem(platformParts[1].toLowerCase());
+        }
+      }
 
       /* Extract tor software version for bridges from their "platform" line.
        * (We already know this for relays from "v" lines in the consensus.) */

src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java

@@ -92,6 +92,7 @@ public class SummaryDocumentWriter implements DocumentWriter {
       SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily();
       String nickname = nodeStatus.getNickname();
       String version = nodeStatus.getVersion();
+      String operatingSystem = nodeStatus.getOperatingSystem();
       String hostName = nodeStatus.getHostName();
       List<String> verifiedHostNames = nodeStatus.getVerifiedHostNames();
       List<String> unverifiedHostNames = nodeStatus.getUnverifiedHostNames();
@@ -100,8 +101,8 @@ public class SummaryDocumentWriter implements DocumentWriter {
           nickname, fingerprint, addresses, lastSeenMillis, running,
           relayFlags, consensusWeight, countryCode, firstSeenMillis,
           asNumber, contact, declaredFamily, effectiveFamily, version,
-          hostName, verifiedHostNames, unverifiedHostNames,
-          recommendedVersion);
+          operatingSystem, hostName, verifiedHostNames,
+          unverifiedHostNames, recommendedVersion);
       if (this.documentStore.store(summaryDocument, fingerprint)) {
         this.writtenDocuments++;
       }

src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java

@@ -27,7 +27,7 @@ public class SummaryDocumentTest {
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
-        null, null, null, true);
+        null, null, null, null, true);
   }
 
   @Test()

src/test/java/org/torproject/onionoo/server/ResourceServletTest.java

@@ -153,7 +153,7 @@ public class ResourceServletTest {
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })),
-        "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de",
+        "0.2.3.25", "linux", "ppp-62-216-201-221.dynamic.mnet-online.de",
         Arrays.asList(
             new String[] { "ppp-62-216-201-221.dynamic.mnet-online.de" }),
         null, true);
@@ -170,7 +170,7 @@ public class ResourceServletTest {
         new TreeSet<String>(Arrays.asList(new String[] {
             "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
         new TreeSet<>(Arrays.asList(new String[] {
-            "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null,
+            "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, null,
             "c-68-38-171-200.hsd1.in.comcast.net",
         Arrays.asList(
             new String[] {"c-68-38-171-200.hsd1.in.comcast.net"}),
@@ -187,8 +187,8 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
         "1024d/51e2a1c7 \"steven j. murdoch\" "
         + "<tor+steven.murdoch@cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
-        new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev", null,
-        null, null, false);
+        new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev",
+        "windows xp", null, null, null, false);
     this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B",
         relayTimMayTribute);
     this.bridges = new TreeMap<>();
@@ -199,7 +199,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-21 18:07:03"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
         null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
-        null, null, "0.2.2.39", null, null, null, true);
+        null, null, "0.2.2.39", null, null, null, null, true);
     this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
         bridgeec2bridgercc7f31fe);
     org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed =
@@ -209,7 +209,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-20 17:37:04"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
         null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
-        null, null, null, null, null, null, null);
+        null, null, null, null, null, null, null, null);
     this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
         bridgeUnnamed);
     org.torproject.onionoo.docs.SummaryDocument bridgegummy =
@@ -220,7 +220,7 @@ public class ResourceServletTest {
         new TreeSet<>(Arrays.asList(new String[] { "Running",
             "Valid" })), -1L, null,
         DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
-        null, "0.2.4.4-alpha-dev", null, null, null, false);
+        null, "0.2.4.4-alpha-dev", "windows 7", null, null, null, false);
     this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
         bridgegummy);
   }
@@ -1648,6 +1648,44 @@ public class ResourceServletTest {
     this.assertErrorStatusCode("/summary?version=*", 400);
   }
 
+  @Test(timeout = 100)
+  public void testOperatingSystemLinux() {
+    this.assertSummaryDocument(
+        "/summary?os=linux", 1, new String[] {"TorkaZ"}, 0, null);
+  }
+
+  @Test(timeout = 100)
+  public void testOperatingSystemLinuxMixedCaps() {
+    this.assertSummaryDocument(
+        "/summary?os=LiNUx", 1, new String[] {"TorkaZ"}, 0, null);
+  }
+
+  @Test(timeout = 100)
+  public void testOperatingSystemLin() {
+    this.assertSummaryDocument(
+        "/summary?os=lin", 1, new String[] {"TorkaZ"}, 0, null);
+  }
+
+  @Test(timeout = 100)
+  public void testOperatingSystemWindows() {
+    this.assertSummaryDocument(
+        "/summary?os=windows", 1, new String[] {"TimMayTribute"},
+        1, new String[] {"gummy"});
+  }
+
+  @Test(timeout = 100)
+  public void testOperatingSystemWindowsExperience() {
+    this.assertSummaryDocument(
+        "/summary?os=windows xp", 1, new String[] {"TimMayTribute"},
+        0, null);
+  }
+
+  @Test(timeout = 100)
+  public void testOperatingSystemWindows7() {
+    this.assertSummaryDocument(
+        "/summary?os=windows 7", 0, null, 1, new String[] {"gummy"});
+  }
+
   @Test
   public void testHostNameDe() {
     this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null);

src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java

@@ -41,7 +41,7 @@ public class SummaryDocumentComparatorTest {
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
-        null, null, null, null);
+        null, null, null, null, null);
   }
 
   /** Some values for running all comparison types. */