Commit a1fc853c authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Revert "Print out a warning if we're missing data."

This reverts commit b0165cb0.

Turns out that worst case query time goes up from 3 to 58 seconds with
the new query, which is unacceptable. We'll have to analyze and
improve the query on the production database before merging this
change.
parent b0165cb0
......@@ -89,7 +89,6 @@ public class ExoneraTorServlet extends HttpServlet {
ExoneraTorDate firstDate = ExoneraTorDate.INVALID;
ExoneraTorDate lastDate = ExoneraTorDate.INVALID;
boolean noRelevantConsensuses = true;
boolean missingData = false;
List<String[]> statusEntries = new ArrayList<>();
List<String> addressesInSameNetwork = null;
......@@ -106,14 +105,6 @@ public class ExoneraTorServlet extends HttpServlet {
&& queryResponse.relevantStatuses) {
noRelevantConsensuses = false;
}
if (null != queryResponse.missingStatuses
&& queryResponse.missingStatuses) {
missingData = true;
}
if (null != queryResponse.missingExitLists
&& queryResponse.missingExitLists) {
missingData = true;
}
if (null != queryResponse.matches) {
for (QueryResponse.Match match : queryResponse.matches) {
StringBuilder sb = new StringBuilder();
......@@ -219,18 +210,15 @@ public class ExoneraTorServlet extends HttpServlet {
/* Print out result. */
} else {
if (!statusEntries.isEmpty()) {
this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString,
missingData);
this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString);
this.writeTechnicalDetails(out, rb, relayIp, requestedDate.asString,
statusEntries);
} else if (addressesInSameNetwork != null
&& !addressesInSameNetwork.isEmpty()) {
this.writeSummaryAddressesInSameNetwork(out, rb, requestUrl, relayIp,
requestedDate.asString, langStr, addressesInSameNetwork,
missingData);
requestedDate.asString, langStr, addressesInSameNetwork);
} else {
this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString,
missingData);
this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString);
}
this.writePermanentLink(out, rb, requestUrl, relayIp,
requestedDate.asString, langStr);
......@@ -410,13 +398,13 @@ public class ExoneraTorServlet extends HttpServlet {
private void writeSummaryNoTimestamp(PrintWriter out, ResourceBundle rb) {
this.writeSummary(out, rb.getString("summary.heading"),
"panel-danger",
rb.getString("summary.invalidparams.notimestamp.title"), null, null,
rb.getString("summary.invalidparams.notimestamp.title"), null,
rb.getString("summary.invalidparams.notimestamp.body"));
}
private void writeSummaryNoIp(PrintWriter out, ResourceBundle rb) {
this.writeSummary(out, rb.getString("summary.heading"),
"panel-danger", rb.getString("summary.invalidparams.noip.title"), null,
"panel-danger", rb.getString("summary.invalidparams.noip.title"),
null, rb.getString("summary.invalidparams.noip.body"));
}
......@@ -425,7 +413,7 @@ public class ExoneraTorServlet extends HttpServlet {
String lastDate) {
this.writeSummary(out, rb.getString("summary.heading"),
"panel-danger",
rb.getString("summary.invalidparams.timestamprange.title"), null, null,
rb.getString("summary.invalidparams.timestamprange.title"), null,
rb.getString("summary.invalidparams.timestamprange.body"),
timestampStr, firstDate, lastDate);
}
......@@ -450,7 +438,7 @@ public class ExoneraTorServlet extends HttpServlet {
: StringEscapeUtils.escapeHtml4(timestampParameter);
this.writeSummary(out, rb.getString("summary.heading"),
"panel-danger",
rb.getString("summary.invalidparams.invalidtimestamp.title"), null,
rb.getString("summary.invalidparams.invalidtimestamp.title"),
null, rb.getString("summary.invalidparams.invalidtimestamp.body"),
escapedTimestampParameter, "\"YYYY-MM-DD\"");
}
......@@ -458,7 +446,7 @@ public class ExoneraTorServlet extends HttpServlet {
private void writeSummaryTimestampTooRecent(PrintWriter out,
ResourceBundle rb) {
this.writeSummary(out, rb.getString("summary.heading"), "panel-danger",
rb.getString("summary.invalidparams.timestamptoorecent.title"), null,
rb.getString("summary.invalidparams.timestamptoorecent.title"),
null, rb.getString("summary.invalidparams.timestamptoorecent.body"));
}
......@@ -477,8 +465,7 @@ public class ExoneraTorServlet extends HttpServlet {
void writeSummaryAddressesInSameNetwork(PrintWriter out,
ResourceBundle rb, String requestUrl, String relayIp, String timestampStr,
String langStr, List<String> addressesInSameNetwork,
boolean missingData) {
String langStr, List<String> addressesInSameNetwork) {
Object[][] panelItems = new Object[addressesInSameNetwork.size()][];
for (int i = 0; i < addressesInSameNetwork.size(); i++) {
String addressInSameNetwork = addressesInSameNetwork.get(i);
......@@ -499,36 +486,33 @@ public class ExoneraTorServlet extends HttpServlet {
this.writeSummary(out, rb.getString("summary.heading"),
"panel-warning",
rb.getString("summary.negativesamenetwork.title"), panelItems,
missingData ? rb.getString("summary.missingdata") : null,
rb.getString("summary.negativesamenetwork.body"),
relayIp, timestampStr, relayIp.contains(":") ? 48 : 24);
}
private void writeSummaryPositive(PrintWriter out, ResourceBundle rb,
String relayIp, String timestampStr, boolean missingData) {
String relayIp, String timestampStr) {
String formattedRelayIp = relayIp.contains(":")
? "[" + relayIp + "]" : relayIp;
this.writeSummary(out, rb.getString("summary.heading"),
"panel-success", rb.getString("summary.positive.title"), null,
missingData ? rb.getString("summary.missingdata") : null,
rb.getString("summary.positive.body"), formattedRelayIp,
timestampStr);
}
private void writeSummaryNegative(PrintWriter out, ResourceBundle rb,
String relayIp, String timestampStr, boolean missingData) {
String relayIp, String timestampStr) {
String formattedRelayIp = relayIp.contains(":")
? "[" + relayIp + "]" : relayIp;
this.writeSummary(out, rb.getString("summary.heading"),
"panel-warning", rb.getString("summary.negative.title"), null,
missingData ? rb.getString("summary.missingdata") : null,
rb.getString("summary.negative.body"), formattedRelayIp,
timestampStr);
}
private void writeSummary(PrintWriter out, String heading,
String panelContext, String panelTitle, Object[][] panelItems,
String panelWarning, String panelBodyTemplate, Object... panelBodyArgs) {
String panelBodyTemplate, Object... panelBodyArgs) {
out.printf(" <div class=\"row\">\n"
+ " <div class=\"col-xs-12\">\n"
+ " <h2>%s</h2>\n"
......@@ -547,9 +531,6 @@ public class ExoneraTorServlet extends HttpServlet {
}
out.print(" </ul>\n");
}
if (null != panelWarning) {
out.printf(" <p>%s</p>\n", panelWarning);
}
out.print(" </div><!-- panel-body -->\n"
+ " </div><!-- panel -->\n"
+ " </div><!-- col -->\n"
......
......@@ -21,7 +21,7 @@ public class QueryResponse {
private static Logger logger = LoggerFactory.getLogger(QueryResponse.class);
/* Actual version implemented by this class. */
private static final String VERSION = "1.1";
private static final String VERSION = "1.0";
/* Don't accept query responses with versions lower than this. */
private static final String FIRSTRECOGNIZEDVERSION = "1.0";
......@@ -60,20 +60,6 @@ public class QueryResponse {
* a day of the requested date; {@code null} if the database is empty. */
Boolean relevantStatuses;
/** Whether there are at least 18 hours of statuses missing in the database
* within a day of the requested date; {@code null} if the database is
* empty.
*
* @since 1.1 */
Boolean missingStatuses;
/** Whether there are at least 18 hours of exit lists missing in the database
* from two days before the requested date until one day after; {@code null}
* if the database is empty.
*
* @since 1.1 */
Boolean missingExitLists;
/** All matches for the given IP address and date; {@code null} if there
* were no matches at all. */
Match[] matches;
......@@ -84,16 +70,13 @@ public class QueryResponse {
/** Constructor for tests. */
QueryResponse(String version, String queryAddress, String queryDate,
String firstDateInDatabase, String lastDateInDatabase,
Boolean relevantStatuses, Boolean missingStatuses,
Boolean missingExitLists, Match[] matches, String[] nearbyAddresses) {
Boolean relevantStatuses, Match[] matches, String[] nearbyAddresses) {
this.version = version;
this.queryAddress = queryAddress;
this.queryDate = queryDate;
this.firstDateInDatabase = firstDateInDatabase;
this.lastDateInDatabase = lastDateInDatabase;
this.relevantStatuses = relevantStatuses;
this.missingStatuses = missingStatuses;
this.missingExitLists = missingExitLists;
this.matches = matches;
this.nearbyAddresses = nearbyAddresses;
}
......
......@@ -54,10 +54,6 @@ public class QueryServlet extends HttpServlet {
= DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
.withZone(ZoneOffset.UTC);
private static final int MISSING_STATUSES_IF_LESS_THAN = 3 * 24 - 17;
private static final int MISSING_EXIT_LISTS_IF_LESS_THAN = 4 * 24 - 17;
@Override
public void init() {
this.logger = LoggerFactory.getLogger(QueryServlet.class);
......@@ -268,14 +264,6 @@ public class QueryServlet extends HttpServlet {
* {first|last}_date_in_database and relevant_statuses fields. */
SortedSet<LocalDate> allDates = new TreeSet<>();
/* Store all hours for which the database contains relevant status
* entries. */
Set<LocalDateTime> allStatusHours = new HashSet<>();
/* Store all hours for which the database contains relevant exit list
* entries. */
Set<LocalDateTime> allExitListHours = new HashSet<>();
/* Store all possible matches for the results table by base64-encoded
* fingerprint and valid-after time. This map is first populated by going
* through the result set and adding or updating map entries, so that
......@@ -321,12 +309,12 @@ public class QueryServlet extends HttpServlet {
String orAddress = rs.getString(8);
if (null != date) {
allDates.add(date);
} else if (null != scanned && null != fingerprintBase64) {
} else if (null != scanned) {
exitAddressesByFingeprintBase64AndScanned.putIfAbsent(
fingerprintBase64, new TreeMap<>());
exitAddressesByFingeprintBase64AndScanned.get(fingerprintBase64)
.put(scanned, exitAddress);
} else if (null != validAfter && null != fingerprintBase64) {
} else if (null != validAfter) {
matchesByFingerprintBase64AndValidAfter.putIfAbsent(
fingerprintBase64, new TreeMap<>());
if (!matchesByFingerprintBase64AndValidAfter
......@@ -350,10 +338,6 @@ public class QueryServlet extends HttpServlet {
}
matchesByAddress.putIfAbsent(orAddress, new HashSet<>());
matchesByAddress.get(orAddress).add(match);
} else if (null != validAfter) {
allStatusHours.add(validAfter);
} else if (null != scanned) {
allExitListHours.add(scanned);
}
}
} catch (SQLException e) {
......@@ -410,10 +394,6 @@ public class QueryServlet extends HttpServlet {
|| allDates.contains(timestamp.minusDays(1L))
|| allDates.contains(timestamp.plusDays(1L));
}
response.missingStatuses = allStatusHours.size()
< MISSING_STATUSES_IF_LESS_THAN;
response.missingExitLists = allExitListHours.size()
< MISSING_EXIT_LISTS_IF_LESS_THAN;
if (matchesByAddress.containsKey(relayIp)) {
List<QueryResponse.Match> matchesList
= new ArrayList<>(matchesByAddress.get(relayIp));
......
......@@ -30,7 +30,6 @@ summary.positive.title=Result is positive
summary.positive.body=We found one or more Tor relays on IP address %s on or within a day of %s that Tor clients were likely to know.
summary.negative.title=Result is negative
summary.negative.body=We did not find IP address %s on or within a day of %s.
summary.missingdata=However, the database is missing several hours of data for this specific request, so that this result must be interpreted carefully.
technicaldetails.heading=Technical details
technicaldetails.pre=Looking up IP address %s on or within one day of %s. Tor clients could have selected this or these Tor relays to build circuits.
technicaldetails.colheader.timestamp=Timestamp (UTC)
......
......@@ -279,24 +279,20 @@ CREATE OR REPLACE FUNCTION insert_exitlistentry_exitaddress (
END;
$$ LANGUAGE 'plpgsql';
-- Search by date and /24 IPv4 or IPv6 prefix for:
-- - exit list entries with an IPv4 exit address in the same /24 network and
-- with a scan time not earlier than two days before and not later than one
-- day after the given date,
-- - status entries with an IPv4 or IPv6 onion routing address in the same /24
-- network as the given hex-encoded IP address prefix and with a valid-after
-- date within a day of the given date,
-- - the first and last dates in the database as well as the dates for which
-- the database contains relevant data within a day of the given date,
-- - the hours for which the database contains relevant exit list entries, and
-- - the hours for which the database contains relevant status entries.
-- Search for (1) status entries with an IPv4 or IPv6 onion routing address in
-- the same /24 network as the given hex-encoded IP address prefix and with a
-- valid-after date within a day of the given date, (2) exit list entries with
-- an IPv4 exit address in the same /24 network and with a scan time not earlier
-- than two days before and not later than one day after the given date, and (3)
-- the last and first dates in the database as well as the dates for which the
-- database contains relevant data within a day of the given date.
--
-- This function makes heavy use of the date_address24 table in order to reduce
-- query response time by first obtaining all relevant fingerprint identifiers.
-- In the next step it runs five selects to obtain status entries, exit list
-- entries, relevant dates, and relevant hours. Any postprocessing, including
-- filtering by exact IP address or matching status entries and exit list
-- entries, needs to happen at the caller.
-- In the next step it runs three selects to obtain status entries, exit list
-- entries, and relevant dates. Any postprocessing, including filtering by exact
-- IP address or matching status entries and exit list entries, needs to happen
-- at the caller.
CREATE OR REPLACE FUNCTION search_by_date_address24 (
search_date DATE, search_address24 CHARACTER(6))
RETURNS TABLE(
......@@ -343,21 +339,7 @@ CREATE OR REPLACE FUNCTION search_by_date_address24 (
WHERE date IN (SELECT MIN(date) FROM date_address24 UNION
SELECT MAX(date) FROM date_address24 UNION
SELECT date FROM date_address24
WHERE date >= $1 - 1 AND date <= $1 + 1)
UNION
SELECT NULL AS date, NULL AS fingerprint_base64,
DATE_TRUNC(''hour'', scanned) AS scanned, NULL AS exitaddress,
NULL AS validafter, NULL AS nickname, NULL AS exit, NULL AS oraddress
FROM exitlistentry_exitaddress
WHERE DATE(exitlistentry_exitaddress.scanned) >= $1 - 2
AND DATE(exitlistentry_exitaddress.scanned) <= $1 + 1
UNION
SELECT NULL AS date, NULL AS fingerprint_base64, NULL AS scanned,
NULL AS exitaddress, DATE_TRUNC(''hour'', validafter) AS validafter,
NULL AS nickname, NULL AS exit, NULL AS oraddress
FROM statusentry_oraddress
WHERE DATE(statusentry_oraddress.validafter) >= $1 - 1
AND DATE(statusentry_oraddress.validafter) <= $1 + 1'
WHERE date >= $1 - 1 AND date <= $1 + 1)'
USING search_date, search_address24;
END;
$$ LANGUAGE plpgsql;
......
......@@ -50,7 +50,7 @@ public class ExoneraTorServletTest {
StringWriter sw = new StringWriter();
es.writeSummaryAddressesInSameNetwork(new PrintWriter(sw), rb,
"http://localhost:8080/", qr.queryAddress, qr.queryDate, "en",
Arrays.asList(qr.nearbyAddresses), false);
Arrays.asList(qr.nearbyAddresses));
String errorMsg = "Test data:" + QueryResponse.toJson(qr)
+ "\nresult:\n" + sw.toString();
assertTrue(errorMsg,
......
......@@ -42,12 +42,10 @@ public class QueryResponseTest {
+ "\"exit\":false}],\"nearby_addresses\":[\"12.13.14.15\","
+ "\"12.13.14.16\"]}"},
{new QueryResponse("1.1", null, null, null,
null, false, true, true, null, null),
"{\"version\":\"1.1\",\"relevant_statuses\":false,"
+ "\"missing_statuses\":true,"
+ "\"missing_exit_lists\":true}"},
null, false, null, null),
"{\"version\":\"1.1\",\"relevant_statuses\":false}"},
{new QueryResponse("1.0", "12.13.14.15", "2016-12-12", "2016-01-01",
"2016-12-31", true, false, false,
"2016-12-31", true,
new QueryResponse.Match[]{new QueryResponse.Match("2016-12-03",
new TreeSet<>(Arrays.asList("12.13.14.15", "12.13.14.16")),
"fingerprint-not-checked", "some name", true),
......@@ -61,8 +59,6 @@ public class QueryResponseTest {
+ "\"first_date_in_database\":\"2016-01-01\","
+ "\"last_date_in_database\":\"2016-12-31\","
+ "\"relevant_statuses\":true,"
+ "\"missing_statuses\":false,"
+ "\"missing_exit_lists\":false,"
+ "\"matches\":[{\"timestamp\":\"2016-12-03\","
+ "\"addresses\":[\"12.13.14.15\","
+ "\"12.13.14.16\"],\"fingerprint\":\"fingerprint-not-checked\","
......@@ -74,7 +70,7 @@ public class QueryResponseTest {
+ "\"exit\":false}],\"nearby_addresses\":[\"12.13.14.15\","
+ "\"12.13.14.16\"]}"},
{new QueryResponse("1.0", "12.13.14.15", "2016-12-12", "2016-01-01",
"2016-12-31", false, null, null,
"2016-12-31", false,
new QueryResponse.Match[]{new QueryResponse.Match("2016-12-03",
new TreeSet<>(Arrays.asList("12.13.14.15", "12.13.14.16")),
"fingerprint-not-checked", "some name", null),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment