Commit 05330c8c authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Add response fields for skipped/truncated documents.

The number of truncated results can help implement pagination of
results (#17938) by showing how many pages remain after the current
one.  The number of skipped results is less critical for implementing
pagination and was mostly added for completeness reasons.
parent 95b2cf94
......@@ -14,6 +14,8 @@
parameter equivalents.
- Exclude bandwidth history values from the future.
- Extend order parameter to "first_seen".
- Add response meta data fields "relays_skipped",
"relays_truncated", "bridges_skipped", and "bridges_truncated".
* Minor changes
- Include XZ binaries in release binaries.
......
......@@ -536,6 +536,18 @@ public class RequestHandler {
this.orderedBridges.addAll(uniqueBridges);
}
private int relaysSkipped = 0;
public int getRelaysSkipped() {
return this.relaysSkipped;
}
private int bridgesSkipped = 0;
public int getBridgesSkipped() {
return this.bridgesSkipped;
}
private void offset() {
if (this.offset == null) {
/* Not skipping first results. */
......@@ -547,12 +559,26 @@ public class RequestHandler {
|| !this.orderedBridges.isEmpty())) {
if (!this.orderedRelays.isEmpty()) {
this.orderedRelays.remove(0);
this.relaysSkipped++;
} else {
this.orderedBridges.remove(0);
this.bridgesSkipped++;
}
}
}
private int relaysTruncated = 0;
public int getRelaysTruncated() {
return this.relaysTruncated;
}
private int bridgesTruncated = 0;
public int getBridgesTruncated() {
return this.bridgesTruncated;
}
private void limit() {
if (this.limit == null) {
/* Not limiting number of results. */
......@@ -562,11 +588,13 @@ public class RequestHandler {
while (!this.orderedRelays.isEmpty()
&& limitValue < this.orderedRelays.size()) {
this.orderedRelays.remove(this.orderedRelays.size() - 1);
this.relaysTruncated++;
}
limitValue -= this.orderedRelays.size();
while (!this.orderedBridges.isEmpty()
&& limitValue < this.orderedBridges.size()) {
this.orderedBridges.remove(this.orderedBridges.size() - 1);
this.bridgesTruncated++;
}
}
......
......@@ -321,6 +321,10 @@ public class ResourceServlet extends HttpServlet {
rb.setBridgesPublishedString(rh.getBridgesPublishedString());
rb.setOrderedRelays(rh.getOrderedRelays());
rb.setOrderedBridges(rh.getOrderedBridges());
rb.setRelaysSkipped(rh.getRelaysSkipped());
rb.setBridgesSkipped(rh.getBridgesSkipped());
rb.setRelaysTruncated(rh.getRelaysTruncated());
rb.setBridgesTruncated(rh.getBridgesTruncated());
String[] fields = null;
if (parameterMap.containsKey("fields")) {
fields = this.parseFieldsParameter(parameterMap.get("fields"));
......
......@@ -60,6 +60,30 @@ public class ResponseBuilder {
this.orderedBridges = orderedBridges;
}
private int relaysSkipped;
public void setRelaysSkipped(int relaysSkipped) {
this.relaysSkipped = relaysSkipped;
}
private int bridgesSkipped;
public void setBridgesSkipped(int bridgesSkipped) {
this.bridgesSkipped = bridgesSkipped;
}
private int relaysTruncated;
public void setRelaysTruncated(int relaysTruncated) {
this.relaysTruncated = relaysTruncated;
}
private int bridgesTruncated;
public void setBridgesTruncated(int bridgesTruncated) {
this.bridgesTruncated = bridgesTruncated;
}
private String[] fields;
public void setFields(String[] fields) {
......@@ -83,47 +107,57 @@ public class ResponseBuilder {
private static final String NEXT_MAJOR_VERSION_SCHEDULED = null;
private void writeRelays(List<SummaryDocument> relays, PrintWriter pw) {
String nextMajorVersionScheduledLine =
NEXT_MAJOR_VERSION_SCHEDULED == null
? ""
: String.format("\"next_major_version_scheduled\":\"%s\",\n",
this.write(pw, "{\"version\":\"%s\",\n", PROTOCOL_VERSION);
if (null != NEXT_MAJOR_VERSION_SCHEDULED) {
this.write(pw, "\"next_major_version_scheduled\":\"%s\",\n",
NEXT_MAJOR_VERSION_SCHEDULED);
String header = String.format("{\"version\":\"%s\",\n"
+ "%s\"relays_published\":\"%s\",\n\"relays\":[",
PROTOCOL_VERSION, nextMajorVersionScheduledLine,
relaysPublishedString);
this.charsWritten += header.length();
pw.write(header);
}
this.write(pw, "\"relays_published\":\"%s\",\n",
this.relaysPublishedString);
if (this.relaysSkipped > 0) {
this.write(pw, "\"relays_skipped\":%d,\n", this.relaysSkipped);
}
this.write(pw, "\"relays\":[");
int written = 0;
for (SummaryDocument entry : relays) {
String lines = this.formatNodeStatus(entry);
if (lines.length() > 0) {
this.charsWritten += (written > 0 ? 2 : 1) + lines.length();
pw.print((written++ > 0 ? ",\n" : "\n") + lines);
this.write(pw, "%s%s", written++ > 0 ? ",\n" : "\n", lines);
}
}
String footer = "\n],\n";
this.charsWritten += footer.length();
pw.print(footer);
this.write(pw, "\n],\n");
if (this.relaysTruncated > 0) {
this.write(pw, "\"relays_truncated\":%d,\n", this.relaysTruncated);
}
}
private void writeBridges(List<SummaryDocument> bridges,
PrintWriter pw) {
String header = "\"bridges_published\":\"" + bridgesPublishedString
+ "\",\n\"bridges\":[";
this.charsWritten += header.length();
pw.write(header);
this.write(pw, "\"bridges_published\":\"" + bridgesPublishedString
+ "\",\n");
if (this.bridgesSkipped > 0) {
this.write(pw, "\"bridges_skipped\":%d,\n", this.bridgesSkipped);
}
this.write(pw, "\"bridges\":[");
int written = 0;
for (SummaryDocument entry : bridges) {
String lines = this.formatNodeStatus(entry);
if (lines.length() > 0) {
this.charsWritten += (written > 0 ? 2 : 1) + lines.length();
pw.print((written++ > 0 ? ",\n" : "\n") + lines);
this.write(pw, (written++ > 0 ? ",\n" : "\n") + lines);
}
}
String footer = "\n]}\n";
this.charsWritten += footer.length();
pw.print(footer);
this.write(pw, "\n]");
if (this.bridgesTruncated > 0) {
this.write(pw, ",\n\"bridges_truncated\":%d", this.bridgesTruncated);
}
this.write(pw, "}\n");
}
private void write(PrintWriter pw, String format, Object ... args) {
String stringToWrite = String.format(format, args);
this.charsWritten += stringToWrite.length();
pw.write(stringToWrite);
}
private String formatNodeStatus(SummaryDocument entry) {
......
......@@ -17,6 +17,7 @@
<ul>
<li><a href="#general">General</a></li>
<li><a href="#methods">Methods</a></li>
<li><a href="#responses">Responses</a></li>
<li><a href="#summary">Summary documents</a></li>
<li><a href="#details">Details documents</a></li>
<li><a href="#history">History objects</a></li>
......@@ -187,8 +188,9 @@ documents on August 25, 2015.</li>
characters of a space-separated fingerprint on November 15, 2015.</li>
<li><strong>3.1</strong>: Removed optional "family" field on January 18,
2016.</li>
<li><strong>3.2</strong>: Extended order parameter to "first_seen" on
January 11, 2017.</li>
<li><strong>3.2</strong>: Extended order parameter to "first_seen" and
added response meta data fields "relays_skipped", "relays_truncated",
"bridges_skipped", and "bridges_truncated" on January 27, 2017.</li>
</ul>
</div> <!-- box -->
......@@ -523,17 +525,12 @@ limiting step.
</div> <!-- box -->
<div class="box">
<a name="summary"></a>
<h3>Summary documents <a href="#summary">#</a>
<span class="request-response">
<a href="summary?limit=4">example request</a>
</span>
</h3>
<a name="responses"></a>
<h3>Responses <a href="#responses">#</a></h3>
<p>Summary documents contain short summaries of relays with nicknames,
fingerprints, IP addresses, and running information as well as bridges
with hashed fingerprints and running information.
Summary documents contain the following fields:</p>
<p>Responses all have the same structure, regardless of requested
method and provided parameters.
Responses contain the following fields:</p>
<ul class="properties">
......@@ -564,7 +561,19 @@ Omitted if no major protocol changes are planned.
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when the last known relay network
status consensus started being valid.
Indicates how recent the relay summaries in this document are.
Indicates how recent the relay objects in this document are.
</p>
</li>
<li>
<b><font color="blue">relays_skipped</font></b>
<code class="typeof">number</code>
<span class="required-false">optional</span>
<p>
Number of skipped relays as requested by a positive "offset" parameter
value.
Omitted if zero.
<font color="blue">Added on January 27, 2017.</font>
</p>
</li>
......@@ -573,7 +582,19 @@ Indicates how recent the relay summaries in this document are.
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Array of relay summary objects as specified below.
Array of relay objects as specified below.
</p>
</li>
<li>
<b><font color="blue">relays_truncated</font></b>
<code class="typeof">number</code>
<span class="required-false">optional</span>
<p>
Number of truncated relays as requested by a positive "limit"
parameter value.
Omitted if zero.
<font color="blue">Added on January 27, 2017.</font>
</p>
</li>
......@@ -584,7 +605,19 @@ Array of relay summary objects as specified below.
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known bridge network status was published.
Indicates how recent the bridge summaries in this document are.
Indicates how recent the bridge objects in this document are.
</p>
</li>
<li>
<b><font color="blue">bridge_skipped</font></b>
<code class="typeof">number</code>
<span class="required-false">optional</span>
<p>
Number of skipped bridges as requested by a positive "offset"
parameter value.
Omitted if zero.
<font color="blue">Added on January 27, 2017.</font>
</p>
</li>
......@@ -593,12 +626,38 @@ Indicates how recent the bridge summaries in this document are.
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Array of bridge summary objects as specified below.
Array of bridge objects as specified below.
</p>
</li>
<li>
<b><font color="blue">bridges_truncated</font></b>
<code class="typeof">number</code>
<span class="required-false">optional</span>
<p>
Number of truncated bridges as requested by a positive "limit"
parameter value.
Omitted if zero.
<font color="blue">Added on January 27, 2017.</font>
</p>
</li>
</ul>
</div> <!-- box -->
<div class="box">
<a name="summary"></a>
<h3>Summary documents <a href="#summary">#</a>
<span class="request-response">
<a href="summary?limit=4">example request</a>
</span>
</h3>
<p>Summary documents contain short summaries of relays with nicknames,
fingerprints, IP addresses, and running information as well as bridges
with hashed fingerprints and running information.</p>
<h4>Relay summary objects</h4>
<p>
......@@ -709,73 +768,7 @@ published by Tor network services TorDNSEL and BridgeDB.
Details documents use the most recently published data from these sources,
which may lead to contradictions between fields based on different sources
in rare edge cases.
Details documents contain the following fields:
</p>
<ul class="properties">
<li>
<b>version</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
Onionoo protocol version string.
</p>
</li>
<li>
<b>next_major_version_scheduled</b>
<code class="typeof">string</code>
<span class="required-false">optional</span>
<p>
UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to
be deployed.
Omitted if no major protocol changes are planned.
</p>
</li>
<li>
<b>relays_published</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known relay network status consensus started being valid.
Indicates how recent the relay details in this document are.
</p>
</li>
<li>
<b>relays</b>
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Array of relay details objects as specified below.
</p>
</li>
<li>
<b>bridges_published</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known bridge network status was published.
Indicates how recent the bridge details in this document are.
</p>
</li>
<li>
<b>bridges</b>
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Array of bridge details objects as specified below.
</p>
</li>
</ul>
<h4>Relay details objects</h4>
......@@ -1557,72 +1550,7 @@ bridge's consumed bandwidth for different time intervals.
Bandwidth documents are only updated when a relay or bridge publishes a
new server descriptor, which may take up to 18 hours during normal
operation.
Bandwidth documents contain the following fields:
</p>
<ul class="properties">
<li>
<b>version</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
Onionoo protocol version string.
</p>
</li>
<li>
<b>next_major_version_scheduled</b>
<code class="typeof">string</code>
<span class="required-false">optional</span>
<p>
UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to
be deployed.
Omitted if no major protocol changes are planned.
</p>
</li>
<li>
<b>relays_published</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known relay network status consensus started being valid.
Indicates how recent the relay bandwidth documents in this document are.
</p>
</li>
<li>
<b>relays</b>
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Array of relay bandwidth objects as specified below.
</p>
</li>
<li>
<b>bridges_published</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known bridge network status was published.
Indicates how recent the bridge bandwidth documents in this document are.
</p>
</li>
<li>
<b>bridges</b>
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Array of bridge bandwidth objects as specified below.
</p>
</li>
</ul>
<h4>Relay bandwidth objects</h4>
......@@ -1742,73 +1670,7 @@ Weights documents contain aggregate statistics of a relay's probability
to be selected by clients for building paths.
Weights documents contain different time intervals and are available for
relays only.
Weights documents contain the following fields:
</p>
<ul class="properties">
<li>
<b>version</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
Onionoo protocol version string.
</p>
</li>
<li>
<b>next_major_version_scheduled</b>
<code class="typeof">string</code>
<span class="required-false">optional</span>
<p>
UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to
be deployed.
Omitted if no major protocol changes are planned.
</p>
</li>
<li>
<b>relays_published</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known relay network status consensus started being valid.
Indicates how recent the relay weights documents in this document are.
</p>
</li>
<li>
<b>relays</b>
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Array of relay weights objects as specified below.
</p>
</li>
<li>
<b>bridges_published</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known bridge network status was published.
Only included for compatibility reasons with the other document types.
</p>
</li>
<li>
<b>bridges</b>
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Empty array of objects that would represent bridge weights documents.
Only included for compatibility reasons with the other document types.
</p>
</li>
</ul>
<h4>Relay weights objects</h4>
......@@ -1931,74 +1793,7 @@ connecting to a bridge every day.
There are no clients documents available for relays, just for bridges.
Clients documents contain different time intervals and are available for
bridges only.
Clients documents contain the following fields:
</p>
<ul class="properties">
<li>
<b>version</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
Onionoo protocol version string.
</p>
</li>
<li>
<b>next_major_version_scheduled</b>
<code class="typeof">string</code>
<span class="required-false">optional</span>
<p>
UTC date (YYYY-MM-DD) when the next major protocol version is scheduled to
be deployed.
Omitted if no major protocol changes are planned.
</p>
</li>
<li>
<b>relays_published</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known relay network status consensus started being valid.
Only included for compatibility reasons with the other document types.
</p>
</li>
<li>
<b>relays</b>
<code class="typeof">array of objects</code>
<span class="required-true">required</span>
<p>
Empty array of objects that would represent relay clients documents.
Only included for compatibility reasons with the other document types.
</p>
</li>
<li>
<b>bridges_published</b>
<code class="typeof">string</code>
<span class="required-true">required</span>
<p>
UTC timestamp (YYYY-MM-DD hh:mm:ss) when
the last known bridge network status was published.
Indicates how recent the bridge clients documents in this d