Add new "as_name" parameter.

Implements another part of #23713.

CHANGELOG.md

@@ -6,6 +6,7 @@
      bridges.
    - Add "as" field as a copy of the "as_number" field in preparation
      of removing the "as_number" field in the future.
+   - Add new "as_name" parameter to search relays by AS name.
 
 
 # Changes in version 6.1-1.15.0 - 2018-07-16

src/main/java/org/torproject/onionoo/docs/DocumentStore.java

@@ -415,6 +415,7 @@ public class DocumentStore {
     List<String> addresses = new ArrayList<>();
     String countryCode = null;
     String asNumber = null;
+    String asName = null;
     String contact = null;
     for (String orAddressAndPort : detailsDocument.getOrAddresses()) {
       if (!orAddressAndPort.contains(":")) {
@@ -451,7 +452,7 @@ public class DocumentStore {
     SummaryDocument summaryDocument = new SummaryDocument(isRelay,
         nickname, fingerprint, addresses, lastSeenMillis, running,
         relayFlags, consensusWeight, countryCode, firstSeenMillis,
-        asNumber, contact, family, family, version, operatingSystem,
+        asNumber, asName, contact, family, family, version, operatingSystem,
         hostName, verifiedHostNames, unverifiedHostNames,
         recommendedVersion);
     return summaryDocument;

src/main/java/org/torproject/onionoo/docs/NodeStatus.java

@@ -387,6 +387,16 @@ public class NodeStatus extends Document {
     return this.asNumber;
   }
 
+  private String asName;
+
+  public void setAsName(String asName) {
+    this.asName = asName;
+  }
+
+  public String getAsName() {
+    return this.asName;
+  }
+
   /* Reverse DNS lookup result */
 
   private String hostName;
@@ -618,6 +628,9 @@ public class NodeStatus extends Document {
       if (parts.length >= 26) {
         nodeStatus.setVersionStatus(TorVersionStatus.ofAbbreviation(parts[25]));
       }
+      if (parts.length >= 27) {
+        nodeStatus.setAsName(parts[26]);
+      }
       return nodeStatus;
     } catch (NumberFormatException e) {
       log.error("Number format exception while parsing node "
@@ -688,6 +701,7 @@ public class NodeStatus extends Document {
         .append((this.getHostName() != null ? this.getHostName() : ""));
     sb.append("\t").append(null != this.getVersionStatus()
         ? this.getVersionStatus().getAbbreviation() : "");
+    sb.append("\t").append((this.asName != null ? this.asName : ""));
     return sb.toString();
   }
 }

src/main/java/org/torproject/onionoo/docs/SummaryDocument.java

@@ -190,6 +190,17 @@ public class SummaryDocument extends Document {
     return this.asNumber;
   }
 
+  @JsonProperty("an")
+  private String asName;
+
+  public void setAsName(String asName) {
+    this.asName = asName;
+  }
+
+  public String getAsName() {
+    return this.asName;
+  }
+
   @JsonProperty("fs")
   private String firstSeenMillis;
 
@@ -364,7 +375,7 @@ public class SummaryDocument extends Document {
   public SummaryDocument(boolean isRelay, String nickname,
       String fingerprint, List<String> addresses, long lastSeenMillis,
       boolean running, SortedSet<String> relayFlags, long consensusWeight,
-      String countryCode, long firstSeenMillis, String asNumber,
+      String countryCode, long firstSeenMillis, String asNumber, String asName,
       String contact, SortedSet<String> familyFingerprints,
       SortedSet<String> effectiveFamily, String version, String operatingSystem,
       String hostName, List<String> verifiedHostNames,
@@ -380,6 +391,7 @@ public class SummaryDocument extends Document {
     this.setCountryCode(countryCode);
     this.setFirstSeenMillis(firstSeenMillis);
     this.setAsNumber(asNumber);
+    this.setAsName(asName);
     this.setContact(contact);
     this.setFamilyFingerprints(familyFingerprints);
     this.setEffectiveFamily(effectiveFamily);

src/main/java/org/torproject/onionoo/server/NodeIndex.java

@@ -85,6 +85,16 @@ class NodeIndex {
     return relaysByAsNumber;
   }
 
+  private Map<String, Set<String>> relaysByAsName = null;
+
+  public void setRelaysByAsName(Map<String, Set<String>> relaysByAsName) {
+    this.relaysByAsName = relaysByAsName;
+  }
+
+  public Map<String, Set<String>> getRelaysByAsName() {
+    return relaysByAsName;
+  }
+
   private Map<String, Set<String>> relaysByFlag = null;
 
   public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) {

src/main/java/org/torproject/onionoo/server/NodeIndexer.java

@@ -151,6 +151,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
         new HashMap<>();
     Map<String, Set<String>> newRelaysByCountryCode = new HashMap<>();
     Map<String, Set<String>> newRelaysByAsNumber = new HashMap<>();
+    Map<String, Set<String>> newRelaysByAsName = new HashMap<>();
     Map<String, Set<String>> newRelaysByFlag = new HashMap<>();
     Map<String, Set<String>> newBridgesByFlag = new HashMap<>();
     Map<String, Set<String>> newRelaysByContact = new HashMap<>();
@@ -230,6 +231,12 @@ public class NodeIndexer implements ServletContextListener, Runnable {
       }
       newRelaysByAsNumber.get(asNumber).add(fingerprint);
       newRelaysByAsNumber.get(asNumber).add(hashedFingerprint);
+      String asName = entry.getAsName();
+      if (!newRelaysByAsName.containsKey(asName)) {
+        newRelaysByAsName.put(asName, new HashSet<>());
+      }
+      newRelaysByAsName.get(asName).add(fingerprint);
+      newRelaysByAsName.get(asName).add(hashedFingerprint);
       for (String flag : entry.getRelayFlags()) {
         String flagLowerCase = flag.toLowerCase();
         if (!newRelaysByFlag.containsKey(flagLowerCase)) {
@@ -402,6 +409,7 @@ public class NodeIndexer implements ServletContextListener, Runnable {
         newBridgeFingerprintSummaryLines);
     newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode);
     newNodeIndex.setRelaysByAsNumber(newRelaysByAsNumber);
+    newNodeIndex.setRelaysByAsName(newRelaysByAsName);
     newNodeIndex.setRelaysByFlag(newRelaysByFlag);
     newNodeIndex.setBridgesByFlag(newBridgesByFlag);
     newNodeIndex.setRelaysByContact(newRelaysByContact);

src/main/java/org/torproject/onionoo/server/RequestHandler.java

@@ -78,6 +78,12 @@ public class RequestHandler {
     this.as = as;
   }
 
+  private String[] asName;
+
+  public void setAsName(String[] asName) {
+    this.asName = asName;
+  }
+
   private String flag;
 
   public void setFlag(String flag) {
@@ -177,6 +183,7 @@ public class RequestHandler {
     this.filterByFingerprint();
     this.filterByCountryCode();
     this.filterByAsNumber();
+    this.filterByAsName();
     this.filterByFlag();
     this.filterNodesByFirstSeenDays();
     this.filterNodesByLastSeenDays();
@@ -427,6 +434,28 @@ public class RequestHandler {
     this.filteredBridges.clear();
   }
 
+  private void filterByAsName() {
+    if (this.asName == null) {
+      /* Not filtering by AS name. */
+      return;
+    }
+    Set<String> removeRelays = new HashSet<>();
+    for (Map.Entry<String, Set<String>> e :
+        this.nodeIndex.getRelaysByAsName().entrySet()) {
+      String asName = e.getKey();
+      for (String asNamePart : this.asName) {
+        if (asName == null || !asName.contains(asNamePart.toLowerCase())) {
+          removeRelays.addAll(e.getValue());
+          break;
+        }
+      }
+    }
+    for (String fingerprint : removeRelays) {
+      this.filteredRelays.remove(fingerprint);
+    }
+    this.filteredBridges.clear();
+  }
+
   private void filterByFlag() {
     if (this.flag == null) {
       /* Not filtering by relay flag. */

src/main/java/org/torproject/onionoo/server/ResourceServlet.java

@@ -67,9 +67,9 @@ public class ResourceServlet extends HttpServlet {
 
   private static Set<String> knownParameters = new HashSet<>(
       Arrays.asList("type", "running", "search", "lookup", "fingerprint",
-          "country", "as", "flag", "first_seen_days", "last_seen_days",
-          "contact", "order", "limit", "offset", "fields", "family", "version",
-          "os", "host_name", "recommended_version"));
+          "country", "as", "as_name", "flag", "first_seen_days",
+          "last_seen_days", "contact", "order", "limit", "offset", "fields",
+          "family", "version", "os", "host_name", "recommended_version"));
 
   private static Set<String> illegalSearchQualifiers =
       new HashSet<>(Arrays.asList(("search,fingerprint,order,limit,"
@@ -246,6 +246,15 @@ public class ResourceServlet extends HttpServlet {
       }
       rh.setAs(asNumberParameter);
     }
+    if (parameterMap.containsKey("as_name")) {
+      String[] asNameParameter = this.parseAsNameParameter(
+          parameterMap.get("as_name"));
+      if (null == asNameParameter) {
+        response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+        return;
+      }
+      rh.setAsName(asNameParameter);
+    }
     if (parameterMap.containsKey("flag")) {
       String flagParameter = this.parseFlagParameter(
           parameterMap.get("flag"));
@@ -503,6 +512,16 @@ public class ResourceServlet extends HttpServlet {
     return parameter;
   }
 
+  private String[] parseAsNameParameter(String parameter) {
+    for (char c : parameter.toCharArray()) {
+      if (c < 32 || c >= 127) {
+        /* Only accept printable ASCII. */
+        return null;
+      }
+    }
+    return parameter.toLowerCase().split(" ");
+  }
+
   private static Pattern flagPattern =
       Pattern.compile("^[a-zA-Z0-9]{1,20}$");
 

src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java

@@ -492,6 +492,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
               nodeStatus.getDefaultPolicy());
           updatedNodeStatus.setPortList(nodeStatus.getPortList());
           updatedNodeStatus.setAsNumber(nodeStatus.getAsNumber());
+          updatedNodeStatus.setAsName(nodeStatus.getAsName());
           updatedNodeStatus.setRecommendedVersion(
               nodeStatus.getRecommendedVersion());
           updatedNodeStatus.setVersion(nodeStatus.getVersion());
@@ -879,6 +880,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
         detailsStatus.setAsName(lookupResult.getAsName());
         nodeStatus.setCountryCode(lookupResult.getCountryCode());
         nodeStatus.setAsNumber(lookupResult.getAsNumber());
+        if (null != lookupResult.getAsName()) {
+          nodeStatus.setAsName(lookupResult.getAsName().toLowerCase());
+        }
       }
 
       if (this.consensusWeightFractions.containsKey(fingerprint)) {

src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java

@@ -87,6 +87,7 @@ public class SummaryDocumentWriter implements DocumentWriter {
       String countryCode = nodeStatus.getCountryCode();
       long firstSeenMillis = nodeStatus.getFirstSeenMillis();
       String asNumber = nodeStatus.getAsNumber();
+      String asName = nodeStatus.getAsName();
       String contact = nodeStatus.getContact();
       SortedSet<String> declaredFamily = nodeStatus.getDeclaredFamily();
       SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily();
@@ -100,7 +101,7 @@ public class SummaryDocumentWriter implements DocumentWriter {
       SummaryDocument summaryDocument = new SummaryDocument(isRelay,
           nickname, fingerprint, addresses, lastSeenMillis, running,
           relayFlags, consensusWeight, countryCode, firstSeenMillis,
-          asNumber, contact, declaredFamily, effectiveFamily, version,
+          asNumber, asName, contact, declaredFamily, effectiveFamily, version,
           operatingSystem, hostName, verifiedHostNames,
           unverifiedHostNames, recommendedVersion);
       if (this.documentStore.store(summaryDocument, fingerprint)) {

src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java

@@ -21,6 +21,7 @@ public class SummaryDocumentTest {
         false, new TreeSet<>(Arrays.asList(new String[] { "Running",
             "Valid" })), 20L, "de",
         DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767",
+        "m-net telekommunikations gmbh",
         "torkaz <klaus dot zufall at gmx dot de> "
         + "<fb-token:np5_g_83jmf=>", new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",

src/test/java/org/torproject/onionoo/server/ResourceServletTest.java

@@ -140,6 +140,7 @@ public class ResourceServletTest {
         false, new TreeSet<>(Arrays.asList(new String[] { "Running",
             "Valid" })), 20L, "de",
         DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767",
+        "m-net telekommunikations gmbh",
         "torkaz <klaus dot zufall at gmx dot de> "
         + "<fb-token:np5_g_83jmf=>", new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
@@ -159,7 +160,8 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-24 12:00:00"), true,
         new TreeSet<>(Arrays.asList(new String[] { "Fast", "Named",
             "Running", "V2Dir", "Valid" })), 1140L, "us",
-        DateTimeHelper.parse("2013-04-16 18:00:00"), "AS7922", null,
+        DateTimeHelper.parse("2013-04-16 18:00:00"), "AS7922",
+        "comcast cable communications, llc", null,
         new TreeSet<String>(Arrays.asList(new String[] {
             "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })),
         new TreeSet<>(Arrays.asList(new String[] {
@@ -178,6 +180,7 @@ public class ResourceServletTest {
         new TreeSet<>(Arrays.asList(new String[] { "Fast",
             "Running", "Unnamed", "V2Dir", "Valid" })), 63L, "a1",
         DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830",
+        "liberty global operations b.v.",
         "1024d/51e2a1c7 \"steven j. murdoch\" "
         + "<tor+steven.murdoch@cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>",
         new TreeSet<String>(), new TreeSet<String>(), "0.2.3.24-rc-dev",
@@ -191,7 +194,7 @@ public class ResourceServletTest {
         Arrays.asList(new String[] { "10.199.7.176" }),
         DateTimeHelper.parse("2013-04-21 18:07:03"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
-        null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
+        null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, null,
         null, null, "0.2.2.39", null, null, null, null, true);
     this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
         bridgeec2bridgercc7f31fe);
@@ -201,7 +204,7 @@ public class ResourceServletTest {
         new String[] { "10.0.52.84" }),
         DateTimeHelper.parse("2013-04-20 17:37:04"), false,
         new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L,
-        null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
+        null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, null,
         null, null, null, null, null, null, null, null);
     this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
         bridgeUnnamed);
@@ -212,7 +215,7 @@ public class ResourceServletTest {
         DateTimeHelper.parse("2013-04-24 01:07:04"), true,
         new TreeSet<>(Arrays.asList(new String[] { "Running",
             "Valid" })), -1L, null,
-        DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null,
+        DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, null,
         null, "0.2.4.4-alpha-dev", "windows 7", null, null, null, false);
     this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756",
         bridgegummy);
@@ -1170,6 +1173,30 @@ public class ResourceServletTest {
         "/summary?as=AS 8767", 400);
   }
 
+  @Test(timeout = 100)
+  public void testAsNameComcast() {
+    this.assertSummaryDocument("/summary?as_name=Comcast", 1, null, 0, null);
+  }
+
+  @Test(timeout = 100)
+  public void testAsNameComcastCable() {
+    this.assertSummaryDocument("/summary?as_name=Comcast Cable",
+        1, null, 0, null);
+  }
+
+  @Test(timeout = 100)
+  public void testAsNameCableComcast() {
+    this.assertSummaryDocument("/summary?as_name=Cable Comcast",
+        1, null, 0, null);
+  }
+
+  @Test(timeout = 100)
+  public void testAsNameMit() {
+    this.assertSummaryDocument(
+        "/summary?as_name=Massachusetts Institute of Technology",
+        0, null, 0, null);
+  }
+
   @Test(timeout = 100)
   public void testFlagRunning() {
     this.assertSummaryDocument(

src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java

@@ -35,6 +35,7 @@ public class SummaryDocumentComparatorTest {
         false, new TreeSet<>(Arrays.asList(new String[] { "Running",
             "Valid" })), 20L, "de",
         DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767",
+        "m-net telekommunikations gmbh",
         "torkaz <klaus dot zufall at gmx dot de> "
         + "<fb-token:np5_g_83jmf=>", new TreeSet<>(Arrays.asList(
         new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",