Add new "host_name" parameter to filter by host name.

Implements #16553.

CHANGELOG.md

@@ -4,6 +4,7 @@
    - Support quoted qualified search terms.
    - Skip unrecognized descriptors when importing archives rather than
      aborting the entire import.
+   - Add new "host_name" parameter to filter by host name.
 
 
 # Changes in version 4.2-1.6.1 - 2017-10-26

src/main/java/org/torproject/onionoo/docs/DocumentStore.java

@@ -439,10 +439,11 @@ public class DocumentStore {
     long lastSeenMillis = -1L;
     long consensusWeight = -1L;
     long firstSeenMillis = -1L;
+    String hostName = null;
     SummaryDocument summaryDocument = new SummaryDocument(isRelay,
         nickname, fingerprint, addresses, lastSeenMillis, running,
         relayFlags, consensusWeight, countryCode, firstSeenMillis,
-        asNumber, contact, family, family, version);
+        asNumber, contact, family, family, version, hostName);
     return summaryDocument;
   }
 

src/main/java/org/torproject/onionoo/docs/NodeStatus.java

@@ -368,6 +368,16 @@ public class NodeStatus extends Document {
 
   /* Reverse DNS lookup result */
 
+  private String hostName;
+
+  public void setHostName(String hostName) {
+    this.hostName = hostName;
+  }
+
+  public String getHostName() {
+    return this.hostName;
+  }
+
   private long lastRdnsLookup = -1L;
 
   public void setLastRdnsLookup(long lastRdnsLookup) {
@@ -561,6 +571,9 @@ public class NodeStatus extends Document {
       if (parts.length >= 24 && !parts[23].isEmpty()) {
         nodeStatus.setVersion(parts[23]);
       }
+      if (parts.length >= 25 && !parts[24].isEmpty()) {
+        nodeStatus.setHostName(parts[24]);
+      }
       return nodeStatus;
     } catch (NumberFormatException e) {
       log.error("Number format exception while parsing node "
@@ -627,6 +640,8 @@ public class NodeStatus extends Document {
         .append(StringUtils.join(this.getIndirectFamily(), ";"));
     sb.append("\t")
         .append((this.getVersion() != null ? this.getVersion() : ""));
+    sb.append("\t")
+        .append((this.getHostName() != null ? this.getHostName() : ""));
     return sb.toString();
   }
 }

src/main/java/org/torproject/onionoo/docs/SummaryDocument.java

@@ -279,6 +279,16 @@ public class SummaryDocument extends Document {
     return this.v;
   }
 
+  private String h;
+
+  public void setHostName(String hostName) {
+    this.h = hostName;
+  }
+
+  public String getHostName() {
+    return this.h;
+  }
+
   /* The familyFingerprints parameter can go away after September 8, 2015.
    * See above. */
   /** Instantiates a summary document with all given properties. */
@@ -287,7 +297,7 @@ public class SummaryDocument extends Document {
       boolean running, SortedSet<String> relayFlags, long consensusWeight,
       String countryCode, long firstSeenMillis, String asNumber,
       String contact, SortedSet<String> familyFingerprints,
-      SortedSet<String> effectiveFamily, String version) {
+      SortedSet<String> effectiveFamily, String version, String hostName) {
     this.setRelay(isRelay);
     this.setNickname(nickname);
     this.setFingerprint(fingerprint);
@@ -303,6 +313,7 @@ public class SummaryDocument extends Document {
     this.setFamilyFingerprints(familyFingerprints);
     this.setEffectiveFamily(effectiveFamily);
     this.setVersion(version);
+    this.setHostName(hostName);
   }
 }
 

src/main/java/org/torproject/onionoo/server/NodeIndex.java

@@ -179,5 +179,15 @@ class NodeIndex {
   public Map<String, Set<String>> getRelaysByVersion() {
     return this.relaysByVersion;
   }
+
+  private Map<String, Set<String>> relaysByHostName;
+
+  public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) {
+    this.relaysByHostName = relaysByHostName;
+  }
+
+  public Map<String, Set<String>> getRelaysByHostName() {
+    return this.relaysByHostName;
+  }
 }
 

src/main/java/org/torproject/onionoo/server/NodeIndexer.java

@@ -154,6 +154,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
     Map<String, Set<String>> newRelaysByContact = new HashMap<>();
     Map<String, Set<String>> newRelaysByFamily = new HashMap<>();
     Map<String, Set<String>> newRelaysByVersion = new HashMap<>();
+    Map<String, Set<String>> newRelaysByHostName = new HashMap<>();
     SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>();
     SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>();
     SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>();
@@ -258,6 +259,15 @@ public class NodeIndexer implements ServletContextListener, Runnable {
         newRelaysByVersion.get(version).add(fingerprint);
         newRelaysByVersion.get(version).add(hashedFingerprint);
       }
+      String hostName = entry.getHostName();
+      if (null != hostName) {
+        String hostNameLowerCase = hostName.toLowerCase();
+        if (!newRelaysByHostName.containsKey(hostNameLowerCase)) {
+          newRelaysByHostName.put(hostNameLowerCase, new HashSet<>());
+        }
+        newRelaysByHostName.get(hostNameLowerCase).add(fingerprint);
+        newRelaysByHostName.get(hostNameLowerCase).add(hashedFingerprint);
+      }
     }
     /* This loop can go away once all Onionoo services had their hourly
      * updater write effective families to summary documents at least
@@ -332,6 +342,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
     newNodeIndex.setRelaysPublishedMillis(relaysLastValidAfterMillis);
     newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis);
     newNodeIndex.setRelaysByVersion(newRelaysByVersion);
+    newNodeIndex.setRelaysByHostName(newRelaysByHostName);
     synchronized (this) {
       this.lastIndexed = updateStatusMillis;
       this.latestNodeIndex = newNodeIndex;

src/main/java/org/torproject/onionoo/server/RequestHandler.java

@@ -97,6 +97,12 @@ public class RequestHandler {
     this.version = version;
   }
 
+  private String hostName;
+
+  public void setHostName(String hostName) {
+    this.hostName = hostName;
+  }
+
   private String[] order;
 
   public void setOrder(String[] order) {
@@ -165,6 +171,7 @@ public class RequestHandler {
     this.filterByContact();
     this.filterByFamily();
     this.filterByVersion();
+    this.filterByHostName();
     this.order();
     this.offset();
     this.limit();
@@ -537,6 +544,25 @@ public class RequestHandler {
     this.filteredBridges.clear();
   }
 
+  private void filterByHostName() {
+    if (this.hostName == null) {
+      /* Not filtering by host name. */
+      return;
+    }
+    String hostName = this.hostName.toLowerCase();
+    Set<String> removeRelays = new HashSet<>(this.filteredRelays.keySet());
+    for (Map.Entry<String, Set<String>> e :
+        this.nodeIndex.getRelaysByHostName().entrySet()) {
+      if (e.getKey().endsWith(hostName)) {
+        removeRelays.removeAll(e.getValue());
+      }
+    }
+    for (String fingerprint : removeRelays) {
+      this.filteredRelays.remove(fingerprint);
+    }
+    this.filteredBridges.clear();
+  }
+
   private void order() {
     List<SummaryDocument> uniqueRelays = new ArrayList<>();
     List<SummaryDocument> uniqueBridges = new ArrayList<>();

src/main/java/org/torproject/onionoo/server/ResourceServlet.java

@@ -68,7 +68,7 @@ public class ResourceServlet extends HttpServlet {
   private static Set<String> knownParameters = new HashSet<>(
       Arrays.asList(("type,running,search,lookup,fingerprint,country,as,"
           + "flag,first_seen_days,last_seen_days,contact,order,limit,"
-          + "offset,fields,family,version").split(",")));
+          + "offset,fields,family,version,host_name").split(",")));
 
   private static Set<String> illegalSearchQualifiers =
       new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -292,6 +292,15 @@ public class ResourceServlet extends HttpServlet {
       }
       rh.setVersion(versionParameter);
     }
+    if (parameterMap.containsKey("host_name")) {
+      String hostNameParameter = this.parseHostNameParameter(
+          parameterMap.get("host_name"));
+      if (null == hostNameParameter) {
+        response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+        return;
+      }
+      rh.setHostName(hostNameParameter);
+    }
     if (parameterMap.containsKey("order")) {
       String[] order = this.parseOrderParameter(parameterMap.get("order"));
       if (order == null) {
@@ -584,5 +593,16 @@ public class ResourceServlet extends HttpServlet {
     }
     return parameter;
   }
+
+  private static Pattern hostNameParameterPattern =
+      Pattern.compile("^[0-9A-Za-z_\\.\\-]+$");
+
+  private String parseHostNameParameter(String parameter) {
+    if (!hostNameParameterPattern.matcher(parameter).matches()) {
+      /* Host name contains illegal character(s). */
+      return null;
+    }
+    return parameter;
+  }
 }
 

src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java

@@ -849,6 +849,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
       if (this.rdnsLookupResults.containsKey(fingerprint)) {
         String hostName = this.rdnsLookupResults.get(fingerprint);
         detailsStatus.setHostName(hostName);
+        nodeStatus.setHostName(hostName);
         nodeStatus.setLastRdnsLookup(this.startedRdnsLookups);
       }
 

src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java

@@ -92,10 +92,12 @@ public class SummaryDocumentWriter implements DocumentWriter {
       SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily();
       String nickname = nodeStatus.getNickname();
       String version = nodeStatus.getVersion();
+      String hostName = nodeStatus.getHostName();
       SummaryDocument summaryDocument = new SummaryDocument(isRelay,
           nickname, fingerprint, addresses, lastSeenMillis, running,
           relayFlags, consensusWeight, countryCode, firstSeenMillis,
-          asNumber, contact, declaredFamily, effectiveFamily, version);
+          asNumber, contact, declaredFamily, effectiveFamily, version,
+          hostName);
       if (this.documentStore.store(summaryDocument, fingerprint)) {
         this.writtenDocuments++;
       }

src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java

@@ -26,7 +26,8 @@ public class SummaryDocumentTest {
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
-        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null);
+        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
+        null);
   }
 
   @Test()

src/test/java/org/torproject/onionoo/server/ResourceServletTest.java

@@ -143,7 +143,7 @@ public class ResourceServletTest {
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })),
-        "0.2.3.25");
+        "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de");
     org.torproject.onionoo.docs.SummaryDocument relayFerrari458 =
         new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458",
         "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList(
@@ -155,7 +155,8 @@ public class ResourceServletTest {
         new TreeSet<String>(Arrays.asList(new String[] {
             "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
         new TreeSet<>(Arrays.asList(new String[] {
-            "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null);
+            "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null,
+            "c-68-38-171-200.hsd1.in.comcast.net");
     this.relays = new TreeMap<>();
     this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A",
         relayTorkaZ);
@@ -171,7 +172,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
         "1024d/51e2a1c7 \"steven j. murdoch\" "
         + "<tor+steven.murdoch@cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
-        new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25");
+        new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25", null);
     this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B",
         relayTimMayTribute);
     this.bridges = new TreeMap<>();
@@ -182,7 +183,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-21 18:07:03"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
         null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
-        null, null, null);
+        null, null, null, null);
     this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
         bridgeec2bridgercc7f31fe);
     org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed =
@@ -192,7 +193,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-20 17:37:04"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
         null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
-        null, null, null);
+        null, null, null, null);
     this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
         bridgeUnnamed);
     org.torproject.onionoo.docs.SummaryDocument bridgegummy =
@@ -203,7 +204,7 @@ public class ResourceServletTest {
         new TreeSet<>(Arrays.asList(new String[] { "Running",
             "Valid" })), -1L, null,
         DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
-        null, null);
+        null, null, null);
     this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
         bridgegummy);
   }
@@ -1623,5 +1624,71 @@ public class ResourceServletTest {
     /* This is also correct when comparing strings. */
     this.assertErrorStatusCode("/summary?version=*", 400);
   }
+
+  @Test
+  public void testHostNameDe() {
+    this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null);
+  }
+
+  @Test
+  public void testHostNameE() {
+    this.assertSummaryDocument("/summary?host_name=e", 1, null, 0, null);
+  }
+
+  @Test
+  public void testHostNameDotDe() {
+    this.assertSummaryDocument("/summary?host_name=.de", 1, null, 0, null);
+  }
+
+  @Test
+  public void testHostNameOnlineDe() {
+    this.assertSummaryDocument("/summary?host_name=online.de", 1, null, 0,
+        null);
+  }
+
+  @Test
+  public void testHostNameOnlineDeSomeCapitalized() {
+    this.assertSummaryDocument("/summary?host_name=onLiNe.dE", 1, null, 0,
+        null);
+  }
+
+  @Test
+  public void testHostNameOnline() {
+    this.assertSummaryDocument("/summary?host_name=online", 0, null, 0, null);
+  }
+
+  @Test
+  public void testHostNameTorkaZFull() {
+    this.assertSummaryDocument(
+        "/summary?host_name=ppp-62-216-201-221.dynamic.mnet-online.de",
+        1, null, 0, null);
+  }
+
+  @Test
+  public void testHostNameTorkaZSub() {
+    this.assertSummaryDocument(
+        "/summary?host_name=sub.ppp-62-216-201-221.dynamic.mnet-online.de",
+        0, null, 0, null);
+  }
+
+  @Test
+  public void testHostNameNet() {
+    this.assertSummaryDocument("/summary?host_name=net", 1, null, 0, null);
+  }
+
+  @Test
+  public void testHostNameCom() {
+    this.assertSummaryDocument("/summary?host_name=com", 0, null, 0, null);
+  }
+
+  @Test
+  public void testHostNameDollar() {
+    this.assertErrorStatusCode("/summary?host_name=$", 400);
+  }
+
+  @Test
+  public void testHostNameUmlaut() {
+    this.assertErrorStatusCode("/summary?host_name=äöü", 400);
+  }
 }
 

src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java

@@ -40,7 +40,8 @@ public class SummaryDocumentComparatorTest {
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
             "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
         new TreeSet<>(Arrays.asList(
-        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null);
+        new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null,
+        null);
   }
 
   /** Some values for running all comparison types. */