From 48e5ff84d067254339d2e303547a48cc7ef00183 Mon Sep 17 00:00:00 2001 From: "Iain R. Learmonth" Date: Wed, 4 Jul 2018 16:43:23 +0100 Subject: [PATCH] Provides more accurate DNS results This commit adds two new fields: {un,}verified_host_names. Whereas previously InetAddress was used to resolve reverse domain names, this instead changes the lookup mechanism to use JNDI allowing for a deeper view into the DNS. It also accounts for the fact that multiple PTR records are not forbidden in the DNS specification and are often used in shared hosting scenarios. A host name is considered verified if it has a matching forward record. If a PTR value is found to have multiple A records, it will be considered verified if any one of the A records matches the original address. If no matching record is found, it will be reported as an unverified host name. Previously, unverified host names were discarded internally by the InetAddress lookup mechanism and so this data could not be used. To maintain "bug compatibility" with the previous implementation of the "host_name" field, which will now be deprecated, the IP address is returned when a lookup fails. The host_name field continues to be used, but now will consider all verified and unverified host names. If finer grained filtering is needed, then a seperate ticket could be filed for that, but it is unclear that it is useful enough to justify the work. Fixes: #18342 --- CHANGELOG.md | 7 ++ .../onionoo/docs/DetailsDocument.java | 68 ++++++++++++++++++ .../onionoo/docs/DetailsStatus.java | 69 ++++++++++++++++++ .../onionoo/docs/DocumentStore.java | 3 + .../torproject/onionoo/docs/NodeStatus.java | 21 ++++++ .../onionoo/docs/SummaryDocument.java | 25 +++++++ .../onionoo/server/NodeIndexer.java | 14 +++- .../onionoo/server/ResponseBuilder.java | 4 ++ .../updater/NodeDetailsStatusUpdater.java | 39 +++++++++- .../onionoo/updater/RdnsLookupRequest.java | 72 ++++++++++++++++--- .../onionoo/updater/RdnsLookupWorker.java | 20 +++++- .../updater/ReverseDomainNameResolver.java | 35 ++++++++- .../onionoo/writer/DetailsDocumentWriter.java | 4 ++ .../onionoo/writer/SummaryDocumentWriter.java | 5 +- .../onionoo/docs/SummaryDocumentTest.java | 2 +- .../onionoo/server/ResourceServletTest.java | 19 +++-- .../server/SummaryDocumentComparatorTest.java | 2 +- 17 files changed, 384 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bc89857..c9c9aba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version 6.?-1.??.0 - 2018-??-?? + + * Medium changes + - Provide more accurate DNS results in "verified_host_names" and + "unverified_host_names". + + # Changes in version 6.0-1.14.0 - 2018-05-29 * Medium changes diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index e112efed..7dcf38ef 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -242,6 +242,74 @@ public class DetailsDocument extends Document { return unescapeJson(this.host_name); } + private List verified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the verified host names, unless the argument was null in + * which case the verified host names are just set to null. + */ + public void setVerifiedHostNames(List verifiedHostNames) { + if (null == verifiedHostNames) { + this.verified_host_names = null; + return; + } + this.verified_host_names = new ArrayList<>(); + for (String hostName : verifiedHostNames) { + this.verified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List getVerifiedHostNames() { + if (null == this.verified_host_names) { + return null; + } + List verifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.verified_host_names) { + verifiedHostNames.add(unescapeJson(escapedHostName)); + } + return verifiedHostNames; + } + + private List unverified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the unverified host names, unless the argument was null in + * which case the unverified host names are just set to null. + */ + public void setUnverifiedHostNames(List unverifiedHostNames) { + if (null == unverifiedHostNames) { + this.unverified_host_names = null; + return; + } + this.unverified_host_names = new ArrayList<>(); + for (String hostName : unverifiedHostNames) { + this.unverified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List getUnverifiedHostNames() { + if (null == this.unverified_host_names) { + return null; + } + List unverifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.unverified_host_names) { + unverifiedHostNames.add(unescapeJson(escapedHostName)); + } + return unverifiedHostNames; + } + private String last_restarted; public void setLastRestarted(Long lastRestarted) { diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 7ee0bb6c..19a7380e 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -5,6 +5,7 @@ package org.torproject.onionoo.docs; import org.apache.commons.lang3.StringEscapeUtils; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.SortedSet; @@ -531,6 +532,74 @@ public class DetailsStatus extends Document { return unescapeJson(this.host_name); } + private List verified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the verified host names, unless the argument was null in + * which case the verified host names are just set to null. + */ + public void setVerifiedHostNames(List verifiedHostNames) { + if (null == verifiedHostNames) { + this.verified_host_names = null; + return; + } + this.verified_host_names = new ArrayList<>(); + for (String hostName : verifiedHostNames) { + this.verified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List getVerifiedHostNames() { + if (null == this.verified_host_names) { + return null; + } + List verifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.verified_host_names) { + verifiedHostNames.add(unescapeJson(escapedHostName)); + } + return verifiedHostNames; + } + + private List unverified_host_names; + + /** + * Creates a copy of the list with each string escaped for JSON compatibility + * and sets this as the unverified host names, unless the argument was null in + * which case the unverified host names are just set to null. + */ + public void setUnverifiedHostNames(List unverifiedHostNames) { + if (null == unverifiedHostNames) { + this.unverified_host_names = null; + return; + } + this.unverified_host_names = new ArrayList<>(); + for (String hostName : unverifiedHostNames) { + this.unverified_host_names.add(escapeJson(hostName)); + } + } + + /** + * Creates a copy of the list with each string having its escaping for JSON + * compatibility reversed and returns the copy, unless the held reference was + * null in which case null is returned. + */ + public List getUnverifiedHostNames() { + if (null == this.unverified_host_names) { + return null; + } + List unverifiedHostNames = new ArrayList<>(); + for (String escapedHostName : this.unverified_host_names) { + unverifiedHostNames.add(unescapeJson(escapedHostName)); + } + return unverifiedHostNames; + } + private List advertised_or_addresses; public void setAdvertisedOrAddresses(List advertisedOrAddresses) { diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 3fac5c91..0d75bf97 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -452,11 +452,14 @@ public class DocumentStore { long consensusWeight = -1L; long firstSeenMillis = -1L; String hostName = null; + List verifiedHostNames = null; + List unverifiedHostNames = null; Boolean recommendedVersion = null; SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, asNumber, contact, family, family, version, hostName, + verifiedHostNames, unverifiedHostNames, recommendedVersion); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 1ea8b985..64332dad 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -15,6 +15,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -388,6 +389,26 @@ public class NodeStatus extends Document { return this.hostName; } + private List verifiedHostNames; + + public void setVerifiedHostNames(List verifiedHostNames) { + this.verifiedHostNames = verifiedHostNames; + } + + public List getVerifiedHostNames() { + return this.verifiedHostNames; + } + + private List unverifiedHostNames; + + public void setUnverifiedHostNames(List unverifiedHostNames) { + this.unverifiedHostNames = unverifiedHostNames; + } + + public List getUnverifiedHostNames() { + return this.unverifiedHostNames; + } + private long lastRdnsLookup = -1L; public void setLastRdnsLookup(long lastRdnsLookup) { diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 31cd84c0..8b5a5e12 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -309,6 +309,28 @@ public class SummaryDocument extends Document { return this.hostName; } + @JsonProperty("vh") + private List verifiedHostNames; + + public void setVerifiedHostNames(List verifiedHostNames) { + this.verifiedHostNames = verifiedHostNames; + } + + public List getVerifiedHostNames() { + return this.verifiedHostNames; + } + + @JsonProperty("uh") + private List unverifiedHostNames; + + public void setUnverifiedHostNames(List unverifiedHostNames) { + this.unverifiedHostNames = unverifiedHostNames; + } + + public List getUnverifiedHostNames() { + return this.unverifiedHostNames; + } + @JsonProperty("rv") private Boolean recommendedVersion; @@ -334,6 +356,7 @@ public class SummaryDocument extends Document { String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet familyFingerprints, SortedSet effectiveFamily, String version, String hostName, + List verifiedHostNames, List unverifiedHostNames, Boolean recommendedVersion) { this.setRelay(isRelay); this.setNickname(nickname); @@ -351,6 +374,8 @@ public class SummaryDocument extends Document { this.setEffectiveFamily(effectiveFamily); this.setVersion(version); this.setHostName(hostName); + this.setVerifiedHostNames(verifiedHostNames); + this.setUnverifiedHostNames(unverifiedHostNames); this.setRecommendedVersion(recommendedVersion); } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index e95eda99..c95818d9 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -15,8 +15,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -266,8 +268,16 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByVersion.get(version).add(fingerprint); newRelaysByVersion.get(version).add(hashedFingerprint); } - String hostName = entry.getHostName(); - if (null != hostName) { + List allHostNames = new ArrayList<>(); + List verifiedHostNames = entry.getVerifiedHostNames(); + if (null != verifiedHostNames) { + allHostNames.addAll(verifiedHostNames); + } + List unverifiedHostNames = entry.getUnverifiedHostNames(); + if (null != unverifiedHostNames) { + allHostNames.addAll(unverifiedHostNames); + } + for (String hostName : allHostNames) { String hostNameLowerCase = hostName.toLowerCase(); if (!newRelaysByHostName.containsKey(hostNameLowerCase)) { newRelaysByHostName.put(hostNameLowerCase, new HashSet<>()); diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java index 1dcfeebd..dff9899a 100644 --- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java +++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java @@ -295,6 +295,10 @@ public class ResponseBuilder { dd.setConsensusWeight(detailsDocument.getConsensusWeight()); } else if (field.equals("host_name")) { dd.setHostName(detailsDocument.getHostName()); + } else if (field.equals("verified_host_names")) { + dd.setVerifiedHostNames(detailsDocument.getVerifiedHostNames()); + } else if (field.equals("unverified_host_names")) { + dd.setUnverifiedHostNames(detailsDocument.getUnverifiedHostNames()); } else if (field.equals("last_restarted")) { dd.setLastRestarted(detailsDocument.getLastRestarted()); } else if (field.equals("bandwidth_rate")) { diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index da554759..ad2cac7b 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -757,6 +757,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private long startedRdnsLookups = -1L; private SortedMap rdnsLookupResults = new TreeMap<>(); + private SortedMap> rdnsVerifiedLookupResults = + new TreeMap<>(); + private SortedMap> rdnsUnverifiedLookupResults = + new TreeMap<>(); private void finishReverseDomainNameLookups() { this.reverseDomainNameResolver.finishReverseDomainNameLookups(); @@ -764,11 +768,28 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.reverseDomainNameResolver.getLookupStartMillis(); Map lookupResults = this.reverseDomainNameResolver.getLookupResults(); + Map> verifiedLookupResults = + this.reverseDomainNameResolver.getVerifiedLookupResults(); + Map> unverifiedLookupResults = + this.reverseDomainNameResolver.getUnverifiedLookupResults(); for (String fingerprint : this.currentRelays) { NodeStatus nodeStatus = this.knownNodes.get(fingerprint); String hostName = lookupResults.get(nodeStatus.getAddress()); - if (hostName != null) { + List verifiedHostNames = + verifiedLookupResults.get(nodeStatus.getAddress()); + List unverifiedHostNames = + unverifiedLookupResults.get(nodeStatus.getAddress()); + if (null != hostName) { this.rdnsLookupResults.put(fingerprint, hostName); + } else { + /* Maintains bug compatibility with previous implementation */ + this.rdnsLookupResults.put(fingerprint, nodeStatus.getAddress()); + } + if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) { + this.rdnsVerifiedLookupResults.put(fingerprint, verifiedHostNames); + } + if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) { + this.rdnsUnverifiedLookupResults.put(fingerprint, unverifiedHostNames); } } } @@ -878,6 +899,22 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); } + if (this.rdnsVerifiedLookupResults.containsKey(fingerprint)) { + List verifiedHostNames = + this.rdnsVerifiedLookupResults.get(fingerprint); + detailsStatus.setVerifiedHostNames(verifiedHostNames); + nodeStatus.setVerifiedHostNames(verifiedHostNames); + nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); + } + + if (this.rdnsUnverifiedLookupResults.containsKey(fingerprint)) { + List unverifiedHostNames = + this.rdnsUnverifiedLookupResults.get(fingerprint); + detailsStatus.setUnverifiedHostNames(unverifiedHostNames); + nodeStatus.setUnverifiedHostNames(unverifiedHostNames); + nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); + } + if (detailsStatus.getLastSeenMillis() < nodeStatus.getLastSeenMillis()) { if (this.lastSeenMeasured.containsKey(fingerprint)) { diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java index cb9169ce..7f143508 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupRequest.java @@ -3,18 +3,26 @@ package org.torproject.onionoo.updater; -import java.net.InetAddress; -import java.net.UnknownHostException; +import org.torproject.onionoo.util.DomainNameSystem; + +import java.util.ArrayList; +import java.util.List; + +import javax.naming.NamingException; class RdnsLookupRequest extends Thread { - private final ReverseDomainNameResolver reverseDomainNameResolver; + private ReverseDomainNameResolver reverseDomainNameResolver; + + private DomainNameSystem domainNameSystem; private RdnsLookupWorker parent; private String address; - private String hostName; + private List verifiedHostNames; + + private List unverifiedHostNames; private long lookupStartedMillis = -1L; @@ -24,6 +32,8 @@ class RdnsLookupRequest extends Thread { ReverseDomainNameResolver reverseDomainNameResolver, RdnsLookupWorker parent, String address) { this.reverseDomainNameResolver = reverseDomainNameResolver; + this.domainNameSystem = + this.reverseDomainNameResolver.getDomainNameSystemInstance(); this.parent = parent; this.address = address; } @@ -32,23 +42,65 @@ class RdnsLookupRequest extends Thread { public void run() { this.lookupStartedMillis = System.currentTimeMillis(); try { - String result = InetAddress.getByName(this.address).getHostName(); + final List verifiedResults = new ArrayList<>(); + final List unverifiedResults = new ArrayList<>(); + final String[] bytes = this.address.split("\\."); + if (bytes.length == 4) { + final String reverseDnsDomain = + bytes[3] + + "." + bytes[2] + "." + bytes[1] + "." + bytes[0] + + ".in-addr.arpa"; + String[] reverseDnsRecords = + this.domainNameSystem.getRecords(reverseDnsDomain, "PTR"); + for (String reverseDnsRecord : reverseDnsRecords) { + boolean verified = false; + String[] forwardDnsRecords = + this.domainNameSystem.getRecords(reverseDnsRecord, "A"); + for (String forwardDnsRecord : forwardDnsRecords) { + if (forwardDnsRecord.equals(this.address)) { + verified = true; + break; + } + } + if (verified) { + verifiedResults.add(reverseDnsRecord.substring(0, + reverseDnsRecord.length() - 1)); + } else { + unverifiedResults.add(reverseDnsRecord.substring(0, + reverseDnsRecord.length() - 1)); + } + } + } synchronized (this) { - this.hostName = result; + this.verifiedHostNames = verifiedResults; + this.unverifiedHostNames = unverifiedResults; } - } catch (UnknownHostException e) { - /* We'll try again the next time. */ + } catch (NamingException e) { + /* The Onionoo field is omitted for both lookup failure and absence of + * a host name. We'll try again the next time. */ } this.lookupCompletedMillis = System.currentTimeMillis(); this.parent.interrupt(); } public synchronized String getHostName() { - return hostName; + List verifiedHostNames = this.verifiedHostNames; + if (null != verifiedHostNames && !verifiedHostNames.isEmpty() ) { + return verifiedHostNames.get(0); + } else { + return null; + } + } + + public synchronized List getVerifiedHostNames() { + return this.verifiedHostNames; + } + + public synchronized List getUnverifiedHostNames() { + return this.unverifiedHostNames; } public synchronized long getLookupMillis() { return this.lookupCompletedMillis - this.lookupStartedMillis; } } - diff --git a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java index 4c188eda..c94c3347 100644 --- a/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java +++ b/src/main/java/org/torproject/onionoo/updater/RdnsLookupWorker.java @@ -3,6 +3,8 @@ package org.torproject.onionoo.updater; +import java.util.List; + class RdnsLookupWorker extends Thread { private final ReverseDomainNameResolver reverseDomainNameResolver; @@ -38,12 +40,28 @@ class RdnsLookupWorker extends Thread { /* Getting interrupted should be the default case. */ } String hostName = request.getHostName(); - if (hostName != null) { + if (null != hostName) { synchronized (this.reverseDomainNameResolver.rdnsLookupResults) { this.reverseDomainNameResolver.rdnsLookupResults.put( rdnsLookupJob, hostName); } } + List verifiedHostNames = request.getVerifiedHostNames(); + if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) { + synchronized (this.reverseDomainNameResolver + .rdnsVerifiedLookupResults) { + this.reverseDomainNameResolver.rdnsVerifiedLookupResults.put( + rdnsLookupJob, verifiedHostNames); + } + } + List unverifiedHostNames = request.getUnverifiedHostNames(); + if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) { + synchronized (this.reverseDomainNameResolver + .rdnsUnverifiedLookupResults) { + this.reverseDomainNameResolver.rdnsUnverifiedLookupResults.put( + rdnsLookupJob, unverifiedHostNames); + } + } long lookupMillis = request.getLookupMillis(); if (lookupMillis >= 0L) { synchronized (this.reverseDomainNameResolver.rdnsLookupMillis) { diff --git a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java index 3c82d147..698b637b 100644 --- a/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java +++ b/src/main/java/org/torproject/onionoo/updater/ReverseDomainNameResolver.java @@ -3,6 +3,7 @@ package org.torproject.onionoo.updater; +import org.torproject.onionoo.util.DomainNameSystem; import org.torproject.onionoo.util.FormattingUtils; import java.util.ArrayList; @@ -24,12 +25,18 @@ public class ReverseDomainNameResolver { private static final int RDNS_LOOKUP_WORKERS_NUM = 5; + private DomainNameSystem domainNameSystem; + private Map addressLastLookupTimes; Set rdnsLookupJobs; Map rdnsLookupResults; + Map> rdnsVerifiedLookupResults; + + Map> rdnsUnverifiedLookupResults; + List rdnsLookupMillis; long startedRdnsLookups; @@ -43,6 +50,7 @@ public class ReverseDomainNameResolver { /** Starts reverse domain name lookups in one or more background * threads and returns immediately. */ public void startReverseDomainNameLookups() { + this.domainNameSystem = new DomainNameSystem(); this.startedRdnsLookups = System.currentTimeMillis(); this.rdnsLookupJobs = new HashSet<>(); for (Map.Entry e : @@ -53,6 +61,8 @@ public class ReverseDomainNameResolver { } } this.rdnsLookupResults = new HashMap<>(); + this.rdnsVerifiedLookupResults = new HashMap<>(); + this.rdnsUnverifiedLookupResults = new HashMap<>(); this.rdnsLookupMillis = new ArrayList<>(); this.rdnsLookupWorkers = new ArrayList<>(); for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { @@ -83,19 +93,42 @@ public class ReverseDomainNameResolver { } } + /** Returns reverse domain name verified lookup results. */ + public Map> getVerifiedLookupResults() { + synchronized (this.rdnsVerifiedLookupResults) { + return new HashMap<>(this.rdnsVerifiedLookupResults); + } + } + + /** Returns reverse domain name unverified lookup results. */ + public Map> getUnverifiedLookupResults() { + synchronized (this.rdnsUnverifiedLookupResults) { + return new HashMap<>(this.rdnsUnverifiedLookupResults); + } + } + /** Returns the time in milliseconds since the epoch when reverse domain * lookups have been started. */ public long getLookupStartMillis() { return this.startedRdnsLookups; } + public DomainNameSystem getDomainNameSystemInstance() { + return this.domainNameSystem; + } + /** Returns a string with the number of performed reverse domain name * lookups and some simple statistics on lookup time. */ public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + FormattingUtils.formatDecimalNumber( rdnsLookupMillis.size()) + " lookups performed\n"); - if (rdnsLookupMillis.size() > 0) { + sb.append(" " + FormattingUtils.formatDecimalNumber( + rdnsVerifiedLookupResults.size()) + " verified results found\n"); + sb.append(" " + FormattingUtils.formatDecimalNumber( + rdnsUnverifiedLookupResults.size()) + + " unverified results found\n"); + if (!rdnsLookupMillis.isEmpty()) { Collections.sort(rdnsLookupMillis); sb.append(" " + FormattingUtils.formatMillis( rdnsLookupMillis.get(0)) + " minimum lookup time\n"); diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index a3359169..bc5bf8c7 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -76,6 +76,10 @@ public class DetailsDocumentWriter implements DocumentWriter { detailsDocument.setConsensusWeight( detailsStatus.getConsensusWeight()); detailsDocument.setHostName(detailsStatus.getHostName()); + detailsDocument.setVerifiedHostNames( + detailsStatus.getVerifiedHostNames()); + detailsDocument.setUnverifiedHostNames( + detailsStatus.getUnverifiedHostNames()); String defaultPolicy = detailsStatus.getDefaultPolicy(); String portList = detailsStatus.getPortList(); if (defaultPolicy != null && (defaultPolicy.equals("accept") diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index 356feabd..ed36d562 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -93,12 +93,15 @@ public class SummaryDocumentWriter implements DocumentWriter { String nickname = nodeStatus.getNickname(); String version = nodeStatus.getVersion(); String hostName = nodeStatus.getHostName(); + List verifiedHostNames = nodeStatus.getVerifiedHostNames(); + List unverifiedHostNames = nodeStatus.getUnverifiedHostNames(); Boolean recommendedVersion = nodeStatus.getRecommendedVersion(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, asNumber, contact, declaredFamily, effectiveFamily, version, - hostName, recommendedVersion); + hostName, verifiedHostNames, unverifiedHostNames, + recommendedVersion); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index a832ca04..459d514b 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -27,7 +27,7 @@ public class SummaryDocumentTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, true); + null, null, null, true); } @Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index d64d59d0..4e9a5f0b 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; - import org.junit.Before; import org.junit.Test; @@ -154,7 +153,10 @@ public class ResourceServletTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), - "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", true); + "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de", + Arrays.asList( + new String[] { "ppp-62-216-201-221.dynamic.mnet-online.de" }), + null, true); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); org.torproject.onionoo.docs.SummaryDocument relayFerrari458 = @@ -169,7 +171,10 @@ public class ResourceServletTest { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, - "c-68-38-171-200.hsd1.in.comcast.net", null); + "c-68-38-171-200.hsd1.in.comcast.net", + Arrays.asList( + new String[] {"c-68-38-171-200.hsd1.in.comcast.net"}), + null, null); this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC", relayFerrari458); org.torproject.onionoo.docs.SummaryDocument relayTimMayTribute = @@ -183,7 +188,7 @@ public class ResourceServletTest { "1024d/51e2a1c7 \"steven j. murdoch\" " + " ", new TreeSet(), new TreeSet(), "0.2.3.24-rc-dev", null, - false); + null, null, false); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -194,7 +199,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null, "0.2.2.39", null, true); + null, null, "0.2.2.39", null, null, null, true); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -204,7 +209,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null, null, null, null); + null, null, null, null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -215,7 +220,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null, "0.2.4.4-alpha-dev", null, false); + null, "0.2.4.4-alpha-dev", null, null, null, false); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index e24e26f1..569300da 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -41,7 +41,7 @@ public class SummaryDocumentComparatorTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, - null, null); + null, null, null, null); } /** Some values for running all comparison types. */ -- GitLab