Commit a1fc853c authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Revert "Print out a warning if we're missing data." 

This reverts commit b0165cb0.

Turns out that worst case query time goes up from 3 to 58 seconds with
the new query, which is unacceptable. We'll have to analyze and
improve the query on the production database before merging this
change.
parent b0165cb0

src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java

+12 −31
Original line number Diff line number Diff line
@@ -89,7 +89,6 @@ public class ExoneraTorServlet extends HttpServlet { 
      ExoneraTorDate firstDate = ExoneraTorDate.INVALID;

      ExoneraTorDate lastDate = ExoneraTorDate.INVALID;

      boolean noRelevantConsensuses = true;

      boolean missingData = false;

      List<String[]> statusEntries = new ArrayList<>();

      List<String> addressesInSameNetwork = null;
@@ -106,14 +105,6 @@ public class ExoneraTorServlet extends HttpServlet { 
              && queryResponse.relevantStatuses) {

            noRelevantConsensuses = false;

          }

          if (null != queryResponse.missingStatuses

              && queryResponse.missingStatuses) {

            missingData = true;

          }

          if (null != queryResponse.missingExitLists

              && queryResponse.missingExitLists) {

            missingData = true;

          }

          if (null != queryResponse.matches) {

            for (QueryResponse.Match match : queryResponse.matches) {

              StringBuilder sb = new StringBuilder();
@@ -219,18 +210,15 @@ public class ExoneraTorServlet extends HttpServlet { 
        /* Print out result. */

      } else {

        if (!statusEntries.isEmpty()) {

          this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString,

              missingData);

          this.writeSummaryPositive(out, rb, relayIp, requestedDate.asString);

          this.writeTechnicalDetails(out, rb, relayIp, requestedDate.asString,

              statusEntries);

        } else if (addressesInSameNetwork != null

            && !addressesInSameNetwork.isEmpty()) {

          this.writeSummaryAddressesInSameNetwork(out, rb, requestUrl, relayIp,

              requestedDate.asString, langStr, addressesInSameNetwork,

              missingData);

              requestedDate.asString, langStr, addressesInSameNetwork);

        } else {

          this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString,

              missingData);

          this.writeSummaryNegative(out, rb, relayIp, requestedDate.asString);

        }

        this.writePermanentLink(out, rb, requestUrl, relayIp,

            requestedDate.asString, langStr);
@@ -410,13 +398,13 @@ public class ExoneraTorServlet extends HttpServlet { 
  private void writeSummaryNoTimestamp(PrintWriter out, ResourceBundle rb) {

    this.writeSummary(out, rb.getString("summary.heading"),

        "panel-danger",

        rb.getString("summary.invalidparams.notimestamp.title"), null, null,

        rb.getString("summary.invalidparams.notimestamp.title"), null,

        rb.getString("summary.invalidparams.notimestamp.body"));

  }



  private void writeSummaryNoIp(PrintWriter out, ResourceBundle rb) {

    this.writeSummary(out, rb.getString("summary.heading"),

        "panel-danger", rb.getString("summary.invalidparams.noip.title"), null,

        "panel-danger", rb.getString("summary.invalidparams.noip.title"),

        null, rb.getString("summary.invalidparams.noip.body"));

  }
@@ -425,7 +413,7 @@ public class ExoneraTorServlet extends HttpServlet { 
      String lastDate) {

    this.writeSummary(out, rb.getString("summary.heading"),

        "panel-danger",

        rb.getString("summary.invalidparams.timestamprange.title"), null, null,

        rb.getString("summary.invalidparams.timestamprange.title"), null,

        rb.getString("summary.invalidparams.timestamprange.body"),

        timestampStr, firstDate, lastDate);

  }
@@ -450,7 +438,7 @@ public class ExoneraTorServlet extends HttpServlet { 
        : StringEscapeUtils.escapeHtml4(timestampParameter);

    this.writeSummary(out, rb.getString("summary.heading"),

        "panel-danger",

        rb.getString("summary.invalidparams.invalidtimestamp.title"), null,

        rb.getString("summary.invalidparams.invalidtimestamp.title"),

        null, rb.getString("summary.invalidparams.invalidtimestamp.body"),

        escapedTimestampParameter, "\"YYYY-MM-DD\"");

  }
@@ -458,7 +446,7 @@ public class ExoneraTorServlet extends HttpServlet { 
  private void writeSummaryTimestampTooRecent(PrintWriter out,

      ResourceBundle rb) {

    this.writeSummary(out, rb.getString("summary.heading"), "panel-danger",

        rb.getString("summary.invalidparams.timestamptoorecent.title"), null,

        rb.getString("summary.invalidparams.timestamptoorecent.title"),

        null, rb.getString("summary.invalidparams.timestamptoorecent.body"));

  }
@@ -477,8 +465,7 @@ public class ExoneraTorServlet extends HttpServlet { 


  void writeSummaryAddressesInSameNetwork(PrintWriter out,

      ResourceBundle rb, String requestUrl, String relayIp, String timestampStr,

      String langStr, List<String> addressesInSameNetwork,

      boolean missingData) {

      String langStr, List<String> addressesInSameNetwork) {

    Object[][] panelItems = new Object[addressesInSameNetwork.size()][];

    for (int i = 0; i < addressesInSameNetwork.size(); i++) {

      String addressInSameNetwork = addressesInSameNetwork.get(i);
@@ -499,36 +486,33 @@ public class ExoneraTorServlet extends HttpServlet { 
    this.writeSummary(out, rb.getString("summary.heading"),

        "panel-warning",

        rb.getString("summary.negativesamenetwork.title"), panelItems,

        missingData ? rb.getString("summary.missingdata") : null,

        rb.getString("summary.negativesamenetwork.body"),

        relayIp, timestampStr, relayIp.contains(":") ? 48 : 24);

  }



  private void writeSummaryPositive(PrintWriter out, ResourceBundle rb,

      String relayIp, String timestampStr, boolean missingData) {

      String relayIp, String timestampStr) {

    String formattedRelayIp = relayIp.contains(":")

        ? "[" + relayIp + "]" : relayIp;

    this.writeSummary(out, rb.getString("summary.heading"),

        "panel-success", rb.getString("summary.positive.title"), null,

        missingData ? rb.getString("summary.missingdata") : null,

        rb.getString("summary.positive.body"), formattedRelayIp,

        timestampStr);

  }



  private void writeSummaryNegative(PrintWriter out, ResourceBundle rb,

      String relayIp, String timestampStr, boolean missingData) {

      String relayIp, String timestampStr) {

    String formattedRelayIp = relayIp.contains(":")

        ? "[" + relayIp + "]" : relayIp;

    this.writeSummary(out, rb.getString("summary.heading"),

        "panel-warning", rb.getString("summary.negative.title"), null,

        missingData ? rb.getString("summary.missingdata") : null,

        rb.getString("summary.negative.body"), formattedRelayIp,

        timestampStr);

  }



  private void writeSummary(PrintWriter out, String heading,

      String panelContext, String panelTitle, Object[][] panelItems,

      String panelWarning, String panelBodyTemplate, Object... panelBodyArgs) {

      String panelBodyTemplate, Object... panelBodyArgs) {

    out.printf("      <div class=\"row\">\n"

        + "        <div class=\"col-xs-12\">\n"

        + "          <h2>%s</h2>\n"
@@ -547,9 +531,6 @@ public class ExoneraTorServlet extends HttpServlet { 
      }

      out.print("              </ul>\n");

    }

    if (null != panelWarning) {

      out.printf("              <p>%s</p>\n", panelWarning);

    }

    out.print("            </div><!-- panel-body -->\n"

        + "          </div><!-- panel -->\n"

        + "        </div><!-- col -->\n"

src/main/java/org/torproject/metrics/exonerator/QueryResponse.java

+2 −19
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ public class QueryResponse { 
  private static Logger logger = LoggerFactory.getLogger(QueryResponse.class);



  /* Actual version implemented by this class. */

  private static final String VERSION = "1.1";

  private static final String VERSION = "1.0";



  /* Don't accept query responses with versions lower than this. */

  private static final String FIRSTRECOGNIZEDVERSION = "1.0";
@@ -60,20 +60,6 @@ public class QueryResponse { 
   * a day of the requested date; {@code null} if the database is empty. */

  Boolean relevantStatuses;



  /** Whether there are at least 18 hours of statuses missing in the database

   * within a day of the requested date; {@code null} if the database is

   * empty.

   *

   * @since 1.1 */

  Boolean missingStatuses;



  /** Whether there are at least 18 hours of exit lists missing in the database

   * from two days before the requested date until one day after; {@code null}

   * if the database is empty.

   *

   * @since 1.1 */

  Boolean missingExitLists;



  /** All matches for the given IP address and date; {@code null} if there

   * were no matches at all. */

  Match[] matches;
@@ -84,16 +70,13 @@ public class QueryResponse { 
  /** Constructor for tests. */

  QueryResponse(String version, String queryAddress, String queryDate,

      String firstDateInDatabase, String lastDateInDatabase,

      Boolean relevantStatuses, Boolean missingStatuses,

      Boolean missingExitLists, Match[] matches, String[] nearbyAddresses) {

      Boolean relevantStatuses, Match[] matches, String[] nearbyAddresses) {

    this.version = version;

    this.queryAddress = queryAddress;

    this.queryDate = queryDate;

    this.firstDateInDatabase = firstDateInDatabase;

    this.lastDateInDatabase = lastDateInDatabase;

    this.relevantStatuses = relevantStatuses;

    this.missingStatuses = missingStatuses;

    this.missingExitLists = missingExitLists;

    this.matches = matches;

    this.nearbyAddresses = nearbyAddresses;

  }

src/main/java/org/torproject/metrics/exonerator/QueryServlet.java

+2 −22
Original line number Diff line number Diff line
@@ -54,10 +54,6 @@ public class QueryServlet extends HttpServlet { 
      = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")

      .withZone(ZoneOffset.UTC);



  private static final int MISSING_STATUSES_IF_LESS_THAN = 3 * 24 - 17;



  private static final int MISSING_EXIT_LISTS_IF_LESS_THAN = 4 * 24 - 17;



  @Override

  public void init() {

    this.logger = LoggerFactory.getLogger(QueryServlet.class);
@@ -268,14 +264,6 @@ public class QueryServlet extends HttpServlet { 
     * {first|last}_date_in_database and relevant_statuses fields. */

    SortedSet<LocalDate> allDates = new TreeSet<>();



    /* Store all hours for which the database contains relevant status

     * entries. */

    Set<LocalDateTime> allStatusHours = new HashSet<>();



    /* Store all hours for which the database contains relevant exit list

     * entries. */

    Set<LocalDateTime> allExitListHours = new HashSet<>();



    /* Store all possible matches for the results table by base64-encoded

     * fingerprint and valid-after time. This map is first populated by going

     * through the result set and adding or updating map entries, so that
@@ -321,12 +309,12 @@ public class QueryServlet extends HttpServlet { 
            String orAddress = rs.getString(8);

            if (null != date) {

              allDates.add(date);

            } else if (null != scanned && null != fingerprintBase64) {

            } else if (null != scanned) {

              exitAddressesByFingeprintBase64AndScanned.putIfAbsent(

                  fingerprintBase64, new TreeMap<>());

              exitAddressesByFingeprintBase64AndScanned.get(fingerprintBase64)

                  .put(scanned, exitAddress);

            } else if (null != validAfter && null != fingerprintBase64) {

            } else if (null != validAfter) {

              matchesByFingerprintBase64AndValidAfter.putIfAbsent(

                  fingerprintBase64, new TreeMap<>());

              if (!matchesByFingerprintBase64AndValidAfter
@@ -350,10 +338,6 @@ public class QueryServlet extends HttpServlet { 
              }

              matchesByAddress.putIfAbsent(orAddress, new HashSet<>());

              matchesByAddress.get(orAddress).add(match);

            } else if (null != validAfter) {

              allStatusHours.add(validAfter);

            } else if (null != scanned) {

              allExitListHours.add(scanned);

            }

          }

        } catch (SQLException e) {
@@ -410,10 +394,6 @@ public class QueryServlet extends HttpServlet { 
          || allDates.contains(timestamp.minusDays(1L))

          || allDates.contains(timestamp.plusDays(1L));

    }

    response.missingStatuses = allStatusHours.size()

        < MISSING_STATUSES_IF_LESS_THAN;

    response.missingExitLists = allExitListHours.size()

        < MISSING_EXIT_LISTS_IF_LESS_THAN;

    if (matchesByAddress.containsKey(relayIp)) {

      List<QueryResponse.Match> matchesList

          = new ArrayList<>(matchesByAddress.get(relayIp));

src/main/resources/ExoneraTor.properties

+0 −1
Original line number Diff line number Diff line
@@ -30,7 +30,6 @@ summary.positive.title=Result is positive 
summary.positive.body=We found one or more Tor relays on IP address %s on or within a day of %s that Tor clients were likely to know.

summary.negative.title=Result is negative

summary.negative.body=We did not find IP address %s on or within a day of %s.

summary.missingdata=However, the database is missing several hours of data for this specific request, so that this result must be interpreted carefully.

technicaldetails.heading=Technical details

technicaldetails.pre=Looking up IP address %s on or within one day of %s. Tor clients could have selected this or these Tor relays to build circuits.

technicaldetails.colheader.timestamp=Timestamp (UTC)

src/main/sql/exonerator2.sql

+12 −30
Original line number Diff line number Diff line
@@ -279,24 +279,20 @@ CREATE OR REPLACE FUNCTION insert_exitlistentry_exitaddress ( 
  END;

$$ LANGUAGE 'plpgsql';



-- Search by date and /24 IPv4 or IPv6 prefix for:

--  - exit list entries with an IPv4 exit address in the same /24 network and

--    with a scan time not earlier than two days before and not later than one

--    day after the given date,

--  - status entries with an IPv4 or IPv6 onion routing address in the same /24

--    network as the given hex-encoded IP address prefix and with a valid-after

--    date within a day of the given date,

--  - the first and last dates in the database as well as the dates for which

--    the database contains relevant data within a day of the given date,

--  - the hours for which the database contains relevant exit list entries, and

--  - the hours for which the database contains relevant status entries.

-- Search for (1) status entries with an IPv4 or IPv6 onion routing address in

-- the same /24 network as the given hex-encoded IP address prefix and with a

-- valid-after date within a day of the given date, (2) exit list entries with

-- an IPv4 exit address in the same /24 network and with a scan time not earlier

-- than two days before and not later than one day after the given date, and (3)

-- the last and first dates in the database as well as the dates for which the

-- database contains relevant data within a day of the given date.

--

-- This function makes heavy use of the date_address24 table in order to reduce

-- query response time by first obtaining all relevant fingerprint identifiers.

-- In the next step it runs five selects to obtain status entries, exit list

-- entries, relevant dates, and relevant hours. Any postprocessing, including

-- filtering by exact IP address or matching status entries and exit list

-- entries, needs to happen at the caller.

-- In the next step it runs three selects to obtain status entries, exit list

-- entries, and relevant dates. Any postprocessing, including filtering by exact

-- IP address or matching status entries and exit list entries, needs to happen

-- at the caller.

CREATE OR REPLACE FUNCTION search_by_date_address24 (

  search_date DATE, search_address24 CHARACTER(6))

    RETURNS TABLE(
@@ -343,21 +339,7 @@ CREATE OR REPLACE FUNCTION search_by_date_address24 ( 
     WHERE date IN (SELECT MIN(date) FROM date_address24 UNION

                    SELECT MAX(date) FROM date_address24 UNION

                    SELECT date FROM date_address24

                    WHERE date >= $1 - 1 AND date <= $1 + 1)

     UNION

     SELECT NULL AS date, NULL AS fingerprint_base64,

           DATE_TRUNC(''hour'', scanned) AS scanned, NULL AS exitaddress,

           NULL AS validafter, NULL AS nickname, NULL AS exit, NULL AS oraddress

       FROM exitlistentry_exitaddress

       WHERE DATE(exitlistentry_exitaddress.scanned) >= $1 - 2

       AND DATE(exitlistentry_exitaddress.scanned) <= $1 + 1

     UNION

     SELECT NULL AS date, NULL AS fingerprint_base64, NULL AS scanned,

           NULL AS exitaddress, DATE_TRUNC(''hour'', validafter) AS validafter,

           NULL AS nickname, NULL AS exit, NULL AS oraddress

       FROM statusentry_oraddress

       WHERE DATE(statusentry_oraddress.validafter) >= $1 - 1

       AND DATE(statusentry_oraddress.validafter) <= $1 + 1'

                    WHERE date >= $1 - 1 AND date <= $1 + 1)'

    USING search_date, search_address24;

END;

$$ LANGUAGE plpgsql;