Commit 817b93ca authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Extend order parameter to first_seen.

Implements #21095.
parent c4780676
# Changes in version x.x.x - 2017-xx-xx
# Changes in version 3.2-1.x.x - 2017-xx-xx
* Major changes
- Fix a bug where we'd believe that we have first seen a bridge on
......@@ -13,6 +13,7 @@
- Accept the same characters in qualified search terms as in their
parameter equivalents.
- Exclude bandwidth history values from the future.
- Extend order parameter to "first_seen".
* Minor changes
- Include XZ binaries in release binaries.
......
......@@ -8,7 +8,7 @@
<property name="javadoc-title" value="Onionoo API Documentation"/>
<property name="implementation-title" value="Onionoo" />
<property name="onionoo.protocol.version" value="3.1"/>
<property name="onionoo.protocol.version" value="3.2"/>
<property name="release.version"
value="${onionoo.protocol.version}-1.0.0-dev"/>
<property name="descriptorversion" value="1.5.0"/>
......
/* Copyright 2017 The Tor Project
* See LICENSE for licensing information */
package org.torproject.onionoo.docs;
/** Provides constants for details document field names. */
public interface DetailsDocumentFields {
public static final String FIRST_SEEN = "first_seen";
public static final String CONSENSUS_WEIGHT = "consensus_weight";
}
......@@ -6,7 +6,6 @@ package org.torproject.onionoo.server;
import org.torproject.onionoo.docs.SummaryDocument;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
......@@ -42,17 +41,6 @@ class NodeIndex {
return bridgesPublishedString;
}
private List<String> relaysByConsensusWeight;
public void setRelaysByConsensusWeight(
List<String> relaysByConsensusWeight) {
this.relaysByConsensusWeight = relaysByConsensusWeight;
}
public List<String> getRelaysByConsensusWeight() {
return relaysByConsensusWeight;
}
private Map<String, SummaryDocument> relayFingerprintSummaryLines;
public void setRelayFingerprintSummaryLines(
......
......@@ -14,11 +14,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
......@@ -177,7 +174,6 @@ public class NodeIndexer implements ServletContextListener, Runnable {
}
}
Time time = TimeFactory.getTime();
List<String> orderRelaysByConsensusWeight = new ArrayList<String>();
/* This variable can go away once all Onionoo services had their
* hourly updater write effective families to summary documents at
* least once. Remove this code after September 8, 2015. */
......@@ -188,11 +184,6 @@ public class NodeIndexer implements ServletContextListener, Runnable {
.toUpperCase();
newRelayFingerprintSummaryLines.put(fingerprint, entry);
newRelayFingerprintSummaryLines.put(hashedFingerprint, entry);
long consensusWeight = entry.getConsensusWeight();
orderRelaysByConsensusWeight.add(String.format("%020d %s",
consensusWeight, fingerprint));
orderRelaysByConsensusWeight.add(String.format("%020d %s",
consensusWeight, hashedFingerprint));
if (entry.getCountryCode() != null) {
String countryCode = entry.getCountryCode();
if (!newRelaysByCountryCode.containsKey(countryCode)) {
......@@ -254,11 +245,6 @@ public class NodeIndexer implements ServletContextListener, Runnable {
newRelaysByContact.get(contact).add(fingerprint);
newRelaysByContact.get(contact).add(hashedFingerprint);
}
Collections.sort(orderRelaysByConsensusWeight);
List<String> newRelaysByConsensusWeight = new ArrayList<String>();
for (String relay : orderRelaysByConsensusWeight) {
newRelaysByConsensusWeight.add(relay.split(" ")[1]);
}
/* This loop can go away once all Onionoo services had their hourly
* updater write effective families to summary documents at least
* once. Remove this code after September 8, 2015. */
......@@ -313,7 +299,6 @@ public class NodeIndexer implements ServletContextListener, Runnable {
hashedHashedFingerprint);
}
NodeIndex newNodeIndex = new NodeIndex();
newNodeIndex.setRelaysByConsensusWeight(newRelaysByConsensusWeight);
newNodeIndex.setRelayFingerprintSummaryLines(
newRelayFingerprintSummaryLines);
newNodeIndex.setBridgeFingerprintSummaryLines(
......
/* Copyright 2017 The Tor Project
* See LICENSE for licensing information */
package org.torproject.onionoo.server;
import org.torproject.onionoo.docs.DetailsDocumentFields;
/** Provides constants for order parameter values. */
public class OrderParameterValues {
private static final String DESCENDING = "-";
public static final String FIRST_SEEN_ASC = DetailsDocumentFields.FIRST_SEEN;
public static final String FIRST_SEEN_DES =
DESCENDING + DetailsDocumentFields.FIRST_SEEN;
public static final String CONSENSUS_WEIGHT_ASC =
DetailsDocumentFields.CONSENSUS_WEIGHT;
public static final String CONSENSUS_WEIGHT_DES =
DESCENDING + DetailsDocumentFields.CONSENSUS_WEIGHT;
}
......@@ -10,6 +10,7 @@ import org.torproject.onionoo.docs.SummaryDocument;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
......@@ -513,34 +514,26 @@ public class RequestHandler {
}
private void order() {
if (this.order != null && this.order.length == 1) {
List<String> orderBy = new ArrayList<String>(
this.nodeIndex.getRelaysByConsensusWeight());
if (this.order[0].startsWith("-")) {
Collections.reverse(orderBy);
List<SummaryDocument> uniqueRelays = new ArrayList<>();
List<SummaryDocument> uniqueBridges = new ArrayList<>();
for (SummaryDocument relay : this.filteredRelays.values()) {
if (!uniqueRelays.contains(relay)) {
uniqueRelays.add(relay);
}
for (String relay : orderBy) {
if (this.filteredRelays.containsKey(relay)
&& !this.orderedRelays.contains(filteredRelays.get(relay))) {
this.orderedRelays.add(this.filteredRelays.remove(relay));
}
}
for (String relay : this.filteredRelays.keySet()) {
if (!this.orderedRelays.contains(this.filteredRelays.get(relay))) {
this.orderedRelays.add(this.filteredRelays.remove(relay));
}
}
for (SummaryDocument bridge : this.filteredBridges.values()) {
if (!uniqueBridges.contains(bridge)) {
uniqueBridges.add(bridge);
}
Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>(
this.filteredBridges.values());
this.orderedBridges.addAll(uniqueBridges);
} else {
Set<SummaryDocument> uniqueRelays = new HashSet<SummaryDocument>(
this.filteredRelays.values());
this.orderedRelays.addAll(uniqueRelays);
Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>(
this.filteredBridges.values());
this.orderedBridges.addAll(uniqueBridges);
}
if (this.order != null) {
Comparator<SummaryDocument> comparator
= new SummaryDocumentComparator(this.order);
Collections.sort(uniqueRelays, comparator);
Collections.sort(uniqueBridges, comparator);
}
this.orderedRelays.addAll(uniqueRelays);
this.orderedBridges.addAll(uniqueBridges);
}
private void offset() {
......
......@@ -267,16 +267,12 @@ public class ResourceServlet extends HttpServlet {
rh.setContact(contactParts);
}
if (parameterMap.containsKey("order")) {
String orderParameter = parameterMap.get("order").toLowerCase();
String orderByField = orderParameter;
if (orderByField.startsWith("-")) {
orderByField = orderByField.substring(1);
}
if (!orderByField.equals("consensus_weight")) {
String[] order = this.parseOrderParameter(parameterMap.get("order"));
if (order == null) {
response.sendError(HttpServletResponse.SC_BAD_REQUEST);
return;
}
rh.setOrder(new String[] { orderParameter });
rh.setOrder(order);
}
if (parameterMap.containsKey("offset")) {
String offsetParameter = parameterMap.get("offset");
......@@ -483,6 +479,36 @@ public class ResourceServlet extends HttpServlet {
return parameter.split(" ");
}
private static Pattern orderParameterPattern =
Pattern.compile("^[0-9a-zA-Z_,-]*$");
private static HashSet<String> knownOrderParameters = new HashSet<>(
Arrays.asList(new String[] { OrderParameterValues.CONSENSUS_WEIGHT_ASC,
OrderParameterValues.CONSENSUS_WEIGHT_DES,
OrderParameterValues.FIRST_SEEN_ASC,
OrderParameterValues.FIRST_SEEN_DES }));
private String[] parseOrderParameter(String parameter) {
if (!orderParameterPattern.matcher(parameter).matches()) {
/* Orders contain illegal character(s). */
return null;
}
String[] orderParameters = parameter.toLowerCase().split(",");
Set<String> seenOrderParameters = new HashSet<>();
for (String orderParameter : orderParameters) {
if (!knownOrderParameters.contains(orderParameter)) {
/* Unknown order parameter. */
return null;
}
if (!seenOrderParameters.add(orderParameter.startsWith("-")
? orderParameter.substring(1) : orderParameter)) {
/* Duplicate parameter. */
return null;
}
}
return orderParameters;
}
private static Pattern fieldsParameterPattern =
Pattern.compile("^[0-9a-zA-Z_,]*$");
......
......@@ -6,6 +6,7 @@ package org.torproject.onionoo.server;
import org.torproject.onionoo.docs.BandwidthDocument;
import org.torproject.onionoo.docs.ClientsDocument;
import org.torproject.onionoo.docs.DetailsDocument;
import org.torproject.onionoo.docs.DetailsDocumentFields;
import org.torproject.onionoo.docs.DocumentStore;
import org.torproject.onionoo.docs.DocumentStoreFactory;
import org.torproject.onionoo.docs.SummaryDocument;
......@@ -77,7 +78,7 @@ public class ResponseBuilder {
return this.charsWritten;
}
private static final String PROTOCOL_VERSION = "3.1";
private static final String PROTOCOL_VERSION = "3.2";
private static final String NEXT_MAJOR_VERSION_SCHEDULED = null;
......@@ -205,7 +206,7 @@ public class ResponseBuilder {
} else if (field.equals("last_changed_address_or_port")) {
dd.setLastChangedAddressOrPort(
detailsDocument.getLastChangedAddressOrPort());
} else if (field.equals("first_seen")) {
} else if (field.equals(DetailsDocumentFields.FIRST_SEEN)) {
dd.setFirstSeen(detailsDocument.getFirstSeen());
} else if (field.equals("running")) {
dd.setRunning(detailsDocument.getRunning());
......@@ -227,7 +228,7 @@ public class ResponseBuilder {
dd.setAsNumber(detailsDocument.getAsNumber());
} else if (field.equals("as_name")) {
dd.setAsName(detailsDocument.getAsName());
} else if (field.equals("consensus_weight")) {
} else if (field.equals(DetailsDocumentFields.CONSENSUS_WEIGHT)) {
dd.setConsensusWeight(detailsDocument.getConsensusWeight());
} else if (field.equals("host_name")) {
dd.setHostName(detailsDocument.getHostName());
......
/* Copyright 2017 The Tor Project
* See LICENSE for licensing information */
package org.torproject.onionoo.server;
import org.torproject.onionoo.docs.SummaryDocument;
import java.util.Comparator;
public class SummaryDocumentComparator implements Comparator<SummaryDocument> {
private final String[] orderParameters;
/** Comparator is initialized with the order parameters. */
public SummaryDocumentComparator(String ... orderParameters) {
this.orderParameters = orderParameters;
}
@Override
public int compare(SummaryDocument o1, SummaryDocument o2) {
int result = 0;
for (String orderParameter : orderParameters) {
switch (orderParameter) {
case OrderParameterValues.CONSENSUS_WEIGHT_ASC:
result = Long.compare(o1.getConsensusWeight(),
o2.getConsensusWeight());
break;
case OrderParameterValues.CONSENSUS_WEIGHT_DES:
result = Long.compare(o2.getConsensusWeight(),
o1.getConsensusWeight());
break;
case OrderParameterValues.FIRST_SEEN_ASC:
result = Long.compare(o1.getFirstSeenMillis(),
o2.getFirstSeenMillis());
break;
case OrderParameterValues.FIRST_SEEN_DES:
result = Long.compare(o2.getFirstSeenMillis(),
o1.getFirstSeenMillis());
break;
default:
throw new RuntimeException("Invalid order parameter: "
+ orderParameter + ". Check initialization of this class!");
}
if (result != 0) {
break;
}
}
return result;
}
}
......@@ -187,6 +187,8 @@ documents on August 25, 2015.</li>
characters of a space-separated fingerprint on November 15, 2015.</li>
<li><strong>3.1</strong>: Removed optional "family" field on January 18,
2016.</li>
<li><strong>3.2</strong>: Extended order parameter to "first_seen" on
January 11, 2017.</li>
</ul>
</div> <!-- box -->
......@@ -473,10 +475,13 @@ Re-order results by a comma-separated list
of fields in ascending or descending order.
Results are first ordered by the first list element, then by the second,
and so on.
Possible fields for ordering are: <strong>consensus_weight</strong>.
Possible fields for ordering are: <strong>consensus_weight</strong> and
<strong>first_seen</strong>.
Field names are case-insensitive.
Ascending order is the default; descending order is selected by prepending
fields with a minus sign (<strong>-</strong>).
Field names can be listed at most once in either ascending or descending
order.
Relays or bridges which don't have any value for a field to be ordered by
are always appended to the end, regardless or sorting order.
The ordering is defined independent of the requested document type and
......
......@@ -1123,9 +1123,9 @@ public class ResourceServletTest {
}
@Test()
public void testFirstSeenDaysSixToSixteen() {
public void testFirstSeenDaysSevenToSixteen() {
this.assertSummaryDocument(
"/summary?first_seen_days=6-16", 2, null, 1, null);
"/summary?first_seen_days=7-16", 2, null, 1, null);
}
@Test()
......@@ -1253,7 +1253,7 @@ public class ResourceServletTest {
@Test()
public void testOrderConsensusWeightAscending() {
this.assertSummaryDocument(
"/summary?order=consensus_weight", 3,
"/summary?order=" + OrderParameterValues.CONSENSUS_WEIGHT_ASC, 3,
new String[] { "TorkaZ", "TimMayTribute", "Ferrari458" }, 3,
null);
}
......@@ -1261,7 +1261,7 @@ public class ResourceServletTest {
@Test()
public void testOrderConsensusWeightDescending() {
this.assertSummaryDocument(
"/summary?order=-consensus_weight", 3,
"/summary?order=" + OrderParameterValues.CONSENSUS_WEIGHT_DES, 3,
new String[] { "Ferrari458", "TimMayTribute", "TorkaZ" }, 3,
null);
}
......@@ -1269,13 +1269,15 @@ public class ResourceServletTest {
@Test()
public void testOrderConsensusWeightAscendingTwice() {
this.assertErrorStatusCode(
"/summary?order=consensus_weight,consensus_weight", 400);
"/summary?order=" + OrderParameterValues.CONSENSUS_WEIGHT_ASC
+ "," + OrderParameterValues.CONSENSUS_WEIGHT_ASC, 400);
}
@Test()
public void testOrderConsensusWeightAscendingThenDescending() {
this.assertErrorStatusCode(
"/summary?order=consensus_weight,-consensus_weight", 400);
"/summary?order=" + OrderParameterValues.CONSENSUS_WEIGHT_ASC + ","
+ OrderParameterValues.CONSENSUS_WEIGHT_DES + "", 400);
}
@Test()
......@@ -1295,17 +1297,56 @@ public class ResourceServletTest {
@Test()
public void testOrderConsensusWeightAscendingLimit1() {
this.assertSummaryDocument(
"/summary?order=consensus_weight&limit=1", 1,
"/summary?order=" + OrderParameterValues.CONSENSUS_WEIGHT_ASC
+ "&limit=1", 1,
new String[] { "TorkaZ" }, 0, null);
}
@Test()
public void testOrderConsensusWeightDecendingLimit1() {
public void testOrderConsensusWeightDescendingLimit1() {
this.assertSummaryDocument(
"/summary?order=-consensus_weight&limit=1", 1,
"/summary?order=" + OrderParameterValues.CONSENSUS_WEIGHT_DES
+ "&limit=1", 1,
new String[] { "Ferrari458" }, 0, null);
}
@Test()
public void testOrderConsensusWeightFiveTimes() {
this.assertErrorStatusCode(
"/summary?order=" + OrderParameterValues.CONSENSUS_WEIGHT_ASC + ","
+ OrderParameterValues.CONSENSUS_WEIGHT_ASC + ","
+ OrderParameterValues.CONSENSUS_WEIGHT_ASC + ","
+ OrderParameterValues.CONSENSUS_WEIGHT_ASC + ","
+ OrderParameterValues.CONSENSUS_WEIGHT_ASC, 400);
}
@Test()
public void testOrderFirstSeenThenConsensusWeight() {
this.assertSummaryDocument(
"/summary?order=" + OrderParameterValues.FIRST_SEEN_ASC + ","
+ OrderParameterValues.CONSENSUS_WEIGHT_ASC, 3,
new String[] { "TimMayTribute", "Ferrari458", "TorkaZ" }, 3,
new String[] { "gummy", null, "ec2bridgercc7f31fe" });
}
@Test()
public void testOrderFirstSeenDescendingThenConsensusWeight() {
this.assertSummaryDocument("/summary?order="
+ OrderParameterValues.FIRST_SEEN_DES + ","
+ OrderParameterValues.CONSENSUS_WEIGHT_ASC, 3,
new String[] { "TorkaZ", "TimMayTribute", "Ferrari458" }, 3,
new String[] { "ec2bridgercc7f31fe", null, "gummy" });
}
@Test()
public void testOrderConsensusWeightThenFirstSeenDescending() {
this.assertSummaryDocument(
"/summary?order=" + OrderParameterValues.CONSENSUS_WEIGHT_ASC + ","
+ OrderParameterValues.FIRST_SEEN_DES, 3,
new String[] { "TorkaZ", "TimMayTribute", "Ferrari458" }, 3,
null);
}
@Test()
public void testOffsetOne() {
this.assertSummaryDocument(
......
/* Copyright 2017 The Tor Project
* See LICENSE for licensing information */
package org.torproject.onionoo.server;
import static org.junit.Assert.assertEquals;
import org.torproject.onionoo.docs.DateTimeHelper;
import org.torproject.onionoo.docs.DetailsDocumentFields;
import org.torproject.onionoo.docs.SummaryDocument;
import org.hamcrest.Matchers;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import java.util.Arrays;
import java.util.Collection;
import java.util.TreeSet;
@RunWith(Parameterized.class)
public class SummaryDocumentComparatorTest {
@Rule
public ExpectedException thrown = ExpectedException.none();
private SummaryDocument createSummaryDoc() {
return new SummaryDocument(true, "TorkaZ",
"000C5F55BD4814B917CC474BD537F1A3B33CCE2A", Arrays.asList(
new String[] { "62.216.201.221", "62.216.201.222",
"62.216.201.223" }), DateTimeHelper.parse("2013-04-19 05:00:00"),
false, new TreeSet<>(Arrays.asList(new String[] { "Running",
"Valid" })), 20L, "de",
DateTimeHelper.parse("2013-04-18 05:00:00"), "AS8767",
"torkaz <klaus dot zufall at gmx dot de> "
+ "<fb-token:np5_g_83jmf=>", new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })),
new TreeSet<>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })));
}
/** Some values for running all comparison types. */
@Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{OrderParameterValues.FIRST_SEEN_ASC, new long[]{1234L, 85968L}},
{OrderParameterValues.FIRST_SEEN_DES, new long[]{12345L, 859689L}},
{OrderParameterValues.CONSENSUS_WEIGHT_ASC, new long[]{12340L, 85968L}},
{OrderParameterValues.CONSENSUS_WEIGHT_DES, new long[]{1234L, 59680L}},
{OrderParameterValues.FIRST_SEEN_ASC, new long[]{91234L, 5968L}},
{OrderParameterValues.FIRST_SEEN_DES, new long[]{912345L, 59689L}},
{OrderParameterValues.CONSENSUS_WEIGHT_ASC, new long[]{912340L, 5968L}},
{OrderParameterValues.CONSENSUS_WEIGHT_DES, new long[]{91234L, 59680L}},
{OrderParameterValues.FIRST_SEEN_ASC, new long[]{1234L, 1234L}},
{OrderParameterValues.FIRST_SEEN_DES, new long[]{12345L, 12345L}},
{OrderParameterValues.CONSENSUS_WEIGHT_ASC, new long[]{12340L, 12340L}},
{OrderParameterValues.CONSENSUS_WEIGHT_DES, new long[]{1234L, 1234L}}
}
);
}
private SummaryDocument[] sd = new SummaryDocument[2];
private String order;
private int expected;
/** This constructor receives the above defined data for each run. */
public SummaryDocumentComparatorTest(String order, long[] vals) {
for (int i = 0; i < sd.length; i++) {
sd[i] = createSummaryDoc();
if (order.contains(DetailsDocumentFields.FIRST_SEEN)) {
sd[i].setFirstSeenMillis(vals[i]);
} else {
sd[i].setConsensusWeight(vals[i]);
}
}
this.order = order;
this.expected = Long.compare(vals[0], vals[1]);
if (order.contains("-")) {
this.expected = - this.expected;
}
}
@Test()
public void testInvalidParameter() {
String[] dummy = {OrderParameterValues.FIRST_SEEN_DES, "odd parameter"};
thrown.expect(RuntimeException.class);
thrown.expectMessage(Matchers
.allOf(Matchers.containsString("Invalid order parameter"),
Matchers.containsString(dummy[1])));
SummaryDocumentComparator sdc = new SummaryDocumentComparator(dummy);
sdc.compare(createSummaryDoc(), createSummaryDoc());
}
@Test()
public void testRegularComparisons() {
SummaryDocumentComparator sdc
= new SummaryDocumentComparator(this.order);