Commit 48e5ff84 authored by Iain R. Learmonth's avatar Iain R. Learmonth Committed by Karsten Loesing
Browse files

Provides more accurate DNS results

This commit adds two new fields: {un,}verified_host_names.

Whereas previously InetAddress was used to resolve reverse domain
names, this instead changes the lookup mechanism to use JNDI allowing
for a deeper view into the DNS. It also accounts for the fact that
multiple PTR records are not forbidden in the DNS specification and are
often used in shared hosting scenarios.

A host name is considered verified if it has a matching forward record.
If a PTR value is found to have multiple A records, it will be
considered verified if any one of the A records matches the original
address. If no matching record is found, it will be reported as an
unverified host name.

Previously, unverified host names were discarded internally by the
InetAddress lookup mechanism and so this data could not be used.

To maintain "bug compatibility" with the previous implementation of the
"host_name" field, which will now be deprecated, the IP address is
returned when a lookup fails.

The host_name field continues to be used, but now will consider all
verified and unverified host names. If finer grained filtering is
needed, then a seperate ticket could be filed for that, but it is
unclear that it is useful enough to justify the work.

Fixes: #18342
parent 5ba56c85
# Changes in version 6.?-1.??.0 - 2018-??-??
* Medium changes
- Provide more accurate DNS results in "verified_host_names" and
"unverified_host_names".
# Changes in version 6.0-1.14.0 - 2018-05-29
* Medium changes
......
......@@ -242,6 +242,74 @@ public class DetailsDocument extends Document {
return unescapeJson(this.host_name);
}
private List<String> verified_host_names;
/**
* Creates a copy of the list with each string escaped for JSON compatibility
* and sets this as the verified host names, unless the argument was null in
* which case the verified host names are just set to null.
*/
public void setVerifiedHostNames(List<String> verifiedHostNames) {
if (null == verifiedHostNames) {
this.verified_host_names = null;
return;
}
this.verified_host_names = new ArrayList<>();
for (String hostName : verifiedHostNames) {
this.verified_host_names.add(escapeJson(hostName));
}
}
/**
* Creates a copy of the list with each string having its escaping for JSON
* compatibility reversed and returns the copy, unless the held reference was
* null in which case null is returned.
*/
public List<String> getVerifiedHostNames() {
if (null == this.verified_host_names) {
return null;
}
List<String> verifiedHostNames = new ArrayList<>();
for (String escapedHostName : this.verified_host_names) {
verifiedHostNames.add(unescapeJson(escapedHostName));
}
return verifiedHostNames;
}
private List<String> unverified_host_names;
/**
* Creates a copy of the list with each string escaped for JSON compatibility
* and sets this as the unverified host names, unless the argument was null in
* which case the unverified host names are just set to null.
*/
public void setUnverifiedHostNames(List<String> unverifiedHostNames) {
if (null == unverifiedHostNames) {
this.unverified_host_names = null;
return;
}
this.unverified_host_names = new ArrayList<>();
for (String hostName : unverifiedHostNames) {
this.unverified_host_names.add(escapeJson(hostName));
}
}
/**
* Creates a copy of the list with each string having its escaping for JSON
* compatibility reversed and returns the copy, unless the held reference was
* null in which case null is returned.
*/
public List<String> getUnverifiedHostNames() {
if (null == this.unverified_host_names) {
return null;
}
List<String> unverifiedHostNames = new ArrayList<>();
for (String escapedHostName : this.unverified_host_names) {
unverifiedHostNames.add(unescapeJson(escapedHostName));
}
return unverifiedHostNames;
}
private String last_restarted;
public void setLastRestarted(Long lastRestarted) {
......
......@@ -5,6 +5,7 @@ package org.torproject.onionoo.docs;
import org.apache.commons.lang3.StringEscapeUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
......@@ -531,6 +532,74 @@ public class DetailsStatus extends Document {
return unescapeJson(this.host_name);
}
private List<String> verified_host_names;
/**
* Creates a copy of the list with each string escaped for JSON compatibility
* and sets this as the verified host names, unless the argument was null in
* which case the verified host names are just set to null.
*/
public void setVerifiedHostNames(List<String> verifiedHostNames) {
if (null == verifiedHostNames) {
this.verified_host_names = null;
return;
}
this.verified_host_names = new ArrayList<>();
for (String hostName : verifiedHostNames) {
this.verified_host_names.add(escapeJson(hostName));
}
}
/**
* Creates a copy of the list with each string having its escaping for JSON
* compatibility reversed and returns the copy, unless the held reference was
* null in which case null is returned.
*/
public List<String> getVerifiedHostNames() {
if (null == this.verified_host_names) {
return null;
}
List<String> verifiedHostNames = new ArrayList<>();
for (String escapedHostName : this.verified_host_names) {
verifiedHostNames.add(unescapeJson(escapedHostName));
}
return verifiedHostNames;
}
private List<String> unverified_host_names;
/**
* Creates a copy of the list with each string escaped for JSON compatibility
* and sets this as the unverified host names, unless the argument was null in
* which case the unverified host names are just set to null.
*/
public void setUnverifiedHostNames(List<String> unverifiedHostNames) {
if (null == unverifiedHostNames) {
this.unverified_host_names = null;
return;
}
this.unverified_host_names = new ArrayList<>();
for (String hostName : unverifiedHostNames) {
this.unverified_host_names.add(escapeJson(hostName));
}
}
/**
* Creates a copy of the list with each string having its escaping for JSON
* compatibility reversed and returns the copy, unless the held reference was
* null in which case null is returned.
*/
public List<String> getUnverifiedHostNames() {
if (null == this.unverified_host_names) {
return null;
}
List<String> unverifiedHostNames = new ArrayList<>();
for (String escapedHostName : this.unverified_host_names) {
unverifiedHostNames.add(unescapeJson(escapedHostName));
}
return unverifiedHostNames;
}
private List<String> advertised_or_addresses;
public void setAdvertisedOrAddresses(List<String> advertisedOrAddresses) {
......
......@@ -452,11 +452,14 @@ public class DocumentStore {
long consensusWeight = -1L;
long firstSeenMillis = -1L;
String hostName = null;
List<String> verifiedHostNames = null;
List<String> unverifiedHostNames = null;
Boolean recommendedVersion = null;
SummaryDocument summaryDocument = new SummaryDocument(isRelay,
nickname, fingerprint, addresses, lastSeenMillis, running,
relayFlags, consensusWeight, countryCode, firstSeenMillis,
asNumber, contact, family, family, version, hostName,
verifiedHostNames, unverifiedHostNames,
recommendedVersion);
return summaryDocument;
}
......
......@@ -15,6 +15,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
......@@ -388,6 +389,26 @@ public class NodeStatus extends Document {
return this.hostName;
}
private List<String> verifiedHostNames;
public void setVerifiedHostNames(List<String> verifiedHostNames) {
this.verifiedHostNames = verifiedHostNames;
}
public List<String> getVerifiedHostNames() {
return this.verifiedHostNames;
}
private List<String> unverifiedHostNames;
public void setUnverifiedHostNames(List<String> unverifiedHostNames) {
this.unverifiedHostNames = unverifiedHostNames;
}
public List<String> getUnverifiedHostNames() {
return this.unverifiedHostNames;
}
private long lastRdnsLookup = -1L;
public void setLastRdnsLookup(long lastRdnsLookup) {
......
......@@ -309,6 +309,28 @@ public class SummaryDocument extends Document {
return this.hostName;
}
@JsonProperty("vh")
private List<String> verifiedHostNames;
public void setVerifiedHostNames(List<String> verifiedHostNames) {
this.verifiedHostNames = verifiedHostNames;
}
public List<String> getVerifiedHostNames() {
return this.verifiedHostNames;
}
@JsonProperty("uh")
private List<String> unverifiedHostNames;
public void setUnverifiedHostNames(List<String> unverifiedHostNames) {
this.unverifiedHostNames = unverifiedHostNames;
}
public List<String> getUnverifiedHostNames() {
return this.unverifiedHostNames;
}
@JsonProperty("rv")
private Boolean recommendedVersion;
......@@ -334,6 +356,7 @@ public class SummaryDocument extends Document {
String countryCode, long firstSeenMillis, String asNumber,
String contact, SortedSet<String> familyFingerprints,
SortedSet<String> effectiveFamily, String version, String hostName,
List<String> verifiedHostNames, List<String> unverifiedHostNames,
Boolean recommendedVersion) {
this.setRelay(isRelay);
this.setNickname(nickname);
......@@ -351,6 +374,8 @@ public class SummaryDocument extends Document {
this.setEffectiveFamily(effectiveFamily);
this.setVersion(version);
this.setHostName(hostName);
this.setVerifiedHostNames(verifiedHostNames);
this.setUnverifiedHostNames(unverifiedHostNames);
this.setRecommendedVersion(recommendedVersion);
}
}
......
......@@ -15,8 +15,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
......@@ -266,8 +268,16 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newRelaysByVersion.get(version).add(fingerprint);
newRelaysByVersion.get(version).add(hashedFingerprint);
}
String hostName = entry.getHostName();
if (null != hostName) {
List<String> allHostNames = new ArrayList<>();
List<String> verifiedHostNames = entry.getVerifiedHostNames();
if (null != verifiedHostNames) {
allHostNames.addAll(verifiedHostNames);
}
List<String> unverifiedHostNames = entry.getUnverifiedHostNames();
if (null != unverifiedHostNames) {
allHostNames.addAll(unverifiedHostNames);
}
for (String hostName : allHostNames) {
String hostNameLowerCase = hostName.toLowerCase();
if (!newRelaysByHostName.containsKey(hostNameLowerCase)) {
newRelaysByHostName.put(hostNameLowerCase, new HashSet<>());
......
......@@ -295,6 +295,10 @@ public class ResponseBuilder {
dd.setConsensusWeight(detailsDocument.getConsensusWeight());
} else if (field.equals("host_name")) {
dd.setHostName(detailsDocument.getHostName());
} else if (field.equals("verified_host_names")) {
dd.setVerifiedHostNames(detailsDocument.getVerifiedHostNames());
} else if (field.equals("unverified_host_names")) {
dd.setUnverifiedHostNames(detailsDocument.getUnverifiedHostNames());
} else if (field.equals("last_restarted")) {
dd.setLastRestarted(detailsDocument.getLastRestarted());
} else if (field.equals("bandwidth_rate")) {
......
......@@ -757,6 +757,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
private long startedRdnsLookups = -1L;
private SortedMap<String, String> rdnsLookupResults = new TreeMap<>();
private SortedMap<String, List<String>> rdnsVerifiedLookupResults =
new TreeMap<>();
private SortedMap<String, List<String>> rdnsUnverifiedLookupResults =
new TreeMap<>();
private void finishReverseDomainNameLookups() {
this.reverseDomainNameResolver.finishReverseDomainNameLookups();
......@@ -764,11 +768,28 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
this.reverseDomainNameResolver.getLookupStartMillis();
Map<String, String> lookupResults =
this.reverseDomainNameResolver.getLookupResults();
Map<String, List<String>> verifiedLookupResults =
this.reverseDomainNameResolver.getVerifiedLookupResults();
Map<String, List<String>> unverifiedLookupResults =
this.reverseDomainNameResolver.getUnverifiedLookupResults();
for (String fingerprint : this.currentRelays) {
NodeStatus nodeStatus = this.knownNodes.get(fingerprint);
String hostName = lookupResults.get(nodeStatus.getAddress());
if (hostName != null) {
List<String> verifiedHostNames =
verifiedLookupResults.get(nodeStatus.getAddress());
List<String> unverifiedHostNames =
unverifiedLookupResults.get(nodeStatus.getAddress());
if (null != hostName) {
this.rdnsLookupResults.put(fingerprint, hostName);
} else {
/* Maintains bug compatibility with previous implementation */
this.rdnsLookupResults.put(fingerprint, nodeStatus.getAddress());
}
if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) {
this.rdnsVerifiedLookupResults.put(fingerprint, verifiedHostNames);
}
if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) {
this.rdnsUnverifiedLookupResults.put(fingerprint, unverifiedHostNames);
}
}
}
......@@ -878,6 +899,22 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
nodeStatus.setLastRdnsLookup(this.startedRdnsLookups);
}
if (this.rdnsVerifiedLookupResults.containsKey(fingerprint)) {
List<String> verifiedHostNames =
this.rdnsVerifiedLookupResults.get(fingerprint);
detailsStatus.setVerifiedHostNames(verifiedHostNames);
nodeStatus.setVerifiedHostNames(verifiedHostNames);
nodeStatus.setLastRdnsLookup(this.startedRdnsLookups);
}
if (this.rdnsUnverifiedLookupResults.containsKey(fingerprint)) {
List<String> unverifiedHostNames =
this.rdnsUnverifiedLookupResults.get(fingerprint);
detailsStatus.setUnverifiedHostNames(unverifiedHostNames);
nodeStatus.setUnverifiedHostNames(unverifiedHostNames);
nodeStatus.setLastRdnsLookup(this.startedRdnsLookups);
}
if (detailsStatus.getLastSeenMillis()
< nodeStatus.getLastSeenMillis()) {
if (this.lastSeenMeasured.containsKey(fingerprint)) {
......
......@@ -3,18 +3,26 @@
package org.torproject.onionoo.updater;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.torproject.onionoo.util.DomainNameSystem;
import java.util.ArrayList;
import java.util.List;
import javax.naming.NamingException;
class RdnsLookupRequest extends Thread {
private final ReverseDomainNameResolver reverseDomainNameResolver;
private ReverseDomainNameResolver reverseDomainNameResolver;
private DomainNameSystem domainNameSystem;
private RdnsLookupWorker parent;
private String address;
private String hostName;
private List<String> verifiedHostNames;
private List<String> unverifiedHostNames;
private long lookupStartedMillis = -1L;
......@@ -24,6 +32,8 @@ class RdnsLookupRequest extends Thread {
ReverseDomainNameResolver reverseDomainNameResolver,
RdnsLookupWorker parent, String address) {
this.reverseDomainNameResolver = reverseDomainNameResolver;
this.domainNameSystem =
this.reverseDomainNameResolver.getDomainNameSystemInstance();
this.parent = parent;
this.address = address;
}
......@@ -32,23 +42,65 @@ class RdnsLookupRequest extends Thread {
public void run() {
this.lookupStartedMillis = System.currentTimeMillis();
try {
String result = InetAddress.getByName(this.address).getHostName();
final List<String> verifiedResults = new ArrayList<>();
final List<String> unverifiedResults = new ArrayList<>();
final String[] bytes = this.address.split("\\.");
if (bytes.length == 4) {
final String reverseDnsDomain =
bytes[3]
+ "." + bytes[2] + "." + bytes[1] + "." + bytes[0]
+ ".in-addr.arpa";
String[] reverseDnsRecords =
this.domainNameSystem.getRecords(reverseDnsDomain, "PTR");
for (String reverseDnsRecord : reverseDnsRecords) {
boolean verified = false;
String[] forwardDnsRecords =
this.domainNameSystem.getRecords(reverseDnsRecord, "A");
for (String forwardDnsRecord : forwardDnsRecords) {
if (forwardDnsRecord.equals(this.address)) {
verified = true;
break;
}
}
if (verified) {
verifiedResults.add(reverseDnsRecord.substring(0,
reverseDnsRecord.length() - 1));
} else {
unverifiedResults.add(reverseDnsRecord.substring(0,
reverseDnsRecord.length() - 1));
}
}
}
synchronized (this) {
this.hostName = result;
this.verifiedHostNames = verifiedResults;
this.unverifiedHostNames = unverifiedResults;
}
} catch (UnknownHostException e) {
/* We'll try again the next time. */
} catch (NamingException e) {
/* The Onionoo field is omitted for both lookup failure and absence of
* a host name. We'll try again the next time. */
}
this.lookupCompletedMillis = System.currentTimeMillis();
this.parent.interrupt();
}
public synchronized String getHostName() {
return hostName;
List<String> verifiedHostNames = this.verifiedHostNames;
if (null != verifiedHostNames && !verifiedHostNames.isEmpty() ) {
return verifiedHostNames.get(0);
} else {
return null;
}
}
public synchronized List<String> getVerifiedHostNames() {
return this.verifiedHostNames;
}
public synchronized List<String> getUnverifiedHostNames() {
return this.unverifiedHostNames;
}
public synchronized long getLookupMillis() {
return this.lookupCompletedMillis - this.lookupStartedMillis;
}
}
......@@ -3,6 +3,8 @@
package org.torproject.onionoo.updater;
import java.util.List;
class RdnsLookupWorker extends Thread {
private final ReverseDomainNameResolver reverseDomainNameResolver;
......@@ -38,12 +40,28 @@ class RdnsLookupWorker extends Thread {
/* Getting interrupted should be the default case. */
}
String hostName = request.getHostName();
if (hostName != null) {
if (null != hostName) {
synchronized (this.reverseDomainNameResolver.rdnsLookupResults) {
this.reverseDomainNameResolver.rdnsLookupResults.put(
rdnsLookupJob, hostName);
}
}
List<String> verifiedHostNames = request.getVerifiedHostNames();
if (null != verifiedHostNames && !verifiedHostNames.isEmpty()) {
synchronized (this.reverseDomainNameResolver
.rdnsVerifiedLookupResults) {
this.reverseDomainNameResolver.rdnsVerifiedLookupResults.put(
rdnsLookupJob, verifiedHostNames);
}
}
List<String> unverifiedHostNames = request.getUnverifiedHostNames();
if (null != unverifiedHostNames && !unverifiedHostNames.isEmpty()) {
synchronized (this.reverseDomainNameResolver
.rdnsUnverifiedLookupResults) {
this.reverseDomainNameResolver.rdnsUnverifiedLookupResults.put(
rdnsLookupJob, unverifiedHostNames);
}
}
long lookupMillis = request.getLookupMillis();
if (lookupMillis >= 0L) {
synchronized (this.reverseDomainNameResolver.rdnsLookupMillis) {
......
......@@ -3,6 +3,7 @@
package org.torproject.onionoo.updater;
import org.torproject.onionoo.util.DomainNameSystem;
import org.torproject.onionoo.util.FormattingUtils;
import java.util.ArrayList;
......@@ -24,12 +25,18 @@ public class ReverseDomainNameResolver {
private static final int RDNS_LOOKUP_WORKERS_NUM = 5;
private DomainNameSystem domainNameSystem;
private Map<String, Long> addressLastLookupTimes;
Set<String> rdnsLookupJobs;
Map<String, String> rdnsLookupResults;
Map<String, List<String>> rdnsVerifiedLookupResults;
Map<String, List<String>> rdnsUnverifiedLookupResults;
List<Long> rdnsLookupMillis;
long startedRdnsLookups;
......@@ -43,6 +50,7 @@ public class ReverseDomainNameResolver {
/** Starts reverse domain name lookups in one or more background
* threads and returns immediately. */
public void startReverseDomainNameLookups() {
this.domainNameSystem = new DomainNameSystem();
this.startedRdnsLookups = System.currentTimeMillis();
this.rdnsLookupJobs = new HashSet<>();
for (Map.Entry<String, Long> e :
......@@ -53,6 +61,8 @@ public class ReverseDomainNameResolver {
}
}
this.rdnsLookupResults = new HashMap<>();
this.rdnsVerifiedLookupResults = new HashMap<>();
this.rdnsUnverifiedLookupResults = new HashMap<>();
this.rdnsLookupMillis = new ArrayList<>();
this.rdnsLookupWorkers = new ArrayList<>();
for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) {
......@@ -83,19 +93,42 @@ public class ReverseDomainNameResolver {
}
}
/** Returns reverse domain name verified lookup results. */
public Map<String, List<String>> getVerifiedLookupResults() {
synchronized (this.rdnsVerifiedLookupResults) {
return new HashMap<>(this.rdnsVerifiedLookupResults);
}
}
/** Returns reverse domain name unverified lookup results. */
public Map<String, List<String>> getUnverifiedLookupResults() {
synchronized (this.rdnsUnverifiedLookupResults) {