Commit 4226ae78 authored by Hiro's avatar Hiro 🏄
Browse files

Go a step forward and create a materialized view instead.

parent 3f08f792

src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java

+15 −1
Original line number Diff line number Diff line
@@ -67,6 +67,7 @@ public class ExoneraTorDatabaseImporter { 
    readImportHistoryToMemory();

    parseDescriptors();

    writeImportHistoryToDisk();

    refreshMaterializedView();

    closeDatabaseConnection();

    deleteLockFile();

    logger.info("Terminating ExoneraTor database importer.");
@@ -507,6 +508,20 @@ public class ExoneraTorDatabaseImporter { 
        parseHistoryFile.getAbsoluteFile());

  }



  private static void refreshMaterializedView() {

    try {

      PreparedStatement st = this.connection.prepareStatement(

          "REFRESH MATERIALIZED VIEW search_data_materialized;"

      );

      st.executeUpdate();

    } catch (SQLException e) {

      logger.warn("Caught an SQL exception while refreshing the materialized "

          + "view search_data_materialized. Check the exception for details."

          + "Ignoring, because we were going to terminate anyway.",

          e);

    }

  }



  /* Close the database connection. */

  private static void closeDatabaseConnection() {

    try {
@@ -537,4 +552,3 @@ public class ExoneraTorDatabaseImporter { 
    }

  }

}

src/main/sql/exonerator2.sql

+35 −26
Original line number Diff line number Diff line
@@ -293,6 +293,26 @@ $$ LANGUAGE 'plpgsql'; 
-- entries, and relevant dates. Any postprocessing, including filtering by exact

-- IP address or matching status entries and exit list entries, needs to happen

-- at the caller.



-- The data is first materialized into a view



CREATE MATERIALIZED VIEW search_data_materialized AS

SELECT

  da.date,

  f.fingerprint_base64,

  eea.scanned,

  eea.exitaddress,

  sea.validafter,

  n.nickname,

  sea.exit,

  sea.oraddress,

  da.address24

FROM date_address24 da

LEFT JOIN fingerprint f ON da.fingerprint_id = f.fingerprint_id

LEFT JOIN exitlistentry_exitaddress eea ON da.fingerprint_id = eea.fingerprint_id

LEFT JOIN statusentry_oraddress sea ON da.fingerprint_id = sea.fingerprint_id

LEFT JOIN nickname n ON sea.nickname_id = n.nickname_id;



CREATE OR REPLACE FUNCTION search_by_date_address24 (

  search_date DATE, search_address24 CHARACTER(6))

    RETURNS TABLE(
@@ -305,33 +325,22 @@ CREATE OR REPLACE FUNCTION search_by_date_address24 ( 
      exit BOOLEAN,

      oraddress INET) AS $$

BEGIN

  RETURN QUERY EXECUTE

    'WITH matching_fingerprint_ids AS (

       SELECT fingerprint_id FROM date_address24

       WHERE date BETWEEN $1 - 2 AND $1 + 1

       AND address24 = $2)

     , consolidated_data AS (

       SELECT

         d.date,

         f.fingerprint_base64,

         e.scanned,

         e.exitaddress,

         s.validafter,

         n.nickname,

         s.exit,

         s.oraddress

       FROM matching_fingerprint_ids mfi

       LEFT JOIN exitlistentry_exitaddress e ON e.fingerprint_id = mfi.fingerprint_id AND e.scanned::DATE BETWEEN $1 - 2 AND $1 + 1

       LEFT JOIN fingerprint f ON e.fingerprint_id = f.fingerprint_id

       LEFT JOIN statusentry_oraddress s ON s.fingerprint_id = mfi.fingerprint_id AND s.validafter::DATE BETWEEN $1 - 1 AND $1 + 1

       LEFT JOIN nickname n ON s.fingerprint_id = n.fingerprint_id

       LEFT JOIN date_address24 d ON d.fingerprint_id = mfi.fingerprint_id AND d.date BETWEEN $1 - 1 AND $1 + 1

     )

     SELECT date, fingerprint_base64, scanned, exitaddress, validafter, nickname, exit, oraddress

     FROM consolidated_data'

  USING search_date, search_address24;

  RETURN QUERY SELECT

    date,

    fingerprint_base64,

    scanned,

    exitaddress,

    validafter,

    nickname,

    exit,

    oraddress

  FROM search_data_materialized

  WHERE address24 = search_address24

    AND (date BETWEEN search_date - 2 AND search_date + 1

         OR scanned::DATE BETWEEN search_date - 2 AND search_date + 1

         OR validafter::DATE BETWEEN search_date - 1 AND search_date + 1);

END;

$$ LANGUAGE plpgsql;

$$ LANGUAGE 'plpgsql';



-- Migrate from the earlier schema in exonerator.sql to this schema. This

-- migration happens in two big loops, one over statusentry and one over