Commit 25072720 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Add BandwidthFile for parsed bandwidth files.

Implements #30216.
parent 23927c27
# Changes in version 2.6.0 - 2019-04-??
* Medium changes
- Add new BandwidthFile descriptor for parsed bandwidth files.
# Changes in version 2.5.0 - 2018-09-25
* Medium changes
......
/* Copyright 2019 The Tor Project
* See LICENSE for licensing information */
package org.torproject.descriptor;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* A bandwidth file contains information on relays' bandwidth capacities and is
* produced by bandwidth generators, previously known as bandwidth scanners.
*
* @since 2.6.0
*/
public interface BandwidthFile extends Descriptor {
/**
* Time of the most recent generator bandwidth result.
*
* @since 2.6.0
*/
LocalDateTime timestamp();
/**
* Document format version.
*
* @since 2.6.0
*/
String version();
/**
* Name of the software that created the document.
*
* @since 2.6.0
*/
String software();
/**
* Version of the software that created the document.
*
* @since 2.6.0
*/
Optional<String> softwareVersion();
/**
* Timestamp in UTC time zone when the file was created.
*
* @since 2.6.0
*/
Optional<LocalDateTime> fileCreated();
/**
* Timestamp in UTC time zone when the generator was started.
*
* @since 2.6.0
*/
Optional<LocalDateTime> generatorStarted();
/**
* Timestamp in UTC time zone when the first relay bandwidth was obtained.
*
* @since 2.6.0
*/
Optional<LocalDateTime> earliestBandwidth();
/**
* Timestamp in UTC time zone of the most recent generator bandwidth result.
*
* @since 2.6.0
*/
Optional<LocalDateTime> latestBandwidth();
/**
* Number of relays that have enough measurements to be included in the
* bandwidth file.
*
* @since 2.6.0
*/
Optional<Integer> numberEligibleRelays();
/**
* Percentage of relays in the consensus that should be included in every
* generated bandwidth file.
*
* @since 2.6.0
*/
Optional<Integer> minimumPercentEligibleRelays();
/**
* Number of relays in the consensus.
*
* @since 2.6.0
*/
Optional<Integer> numberConsensusRelays();
/**
* The number of eligible relays, as a percentage of the number of relays in
* the consensus.
*
* @since 2.6.0
*/
Optional<Integer> percentEligibleRelays();
/**
* Minimum number of relays that should be included in the bandwidth file.
*
* @since 2.6.0
*/
Optional<Integer> minimumNumberEligibleRelays();
/**
* Country, as in political geolocation, where the generator is run.
*
* @since 2.6.0
*/
Optional<String> scannerCountry();
/**
* Country, as in political geolocation, or countries where the destination
* web server(s) are located.
*
* @since 2.6.0
*/
Optional<String[]> destinationsCountries();
/**
* Number of the different consensuses seen in the last data period.
*
* @since 2.6.0
*/
Optional<Integer> recentConsensusCount();
/**
* Number of times that a list with a subset of relays prioritized to be
* measured has been created in the last data period.
*
* @since 2.6.0
*/
Optional<Integer> recentPriorityListCount();
/**
* Number of relays that has been in in the list of relays prioritized to be
* measured in the last data period.
*
* @since 2.6.0
*/
Optional<Integer> recentPriorityRelayCount();
/**
* Number of times that any relay has been queued to be measured in the last
* data period.
*
* @since 2.6.0
*/
Optional<Integer> recentMeasurementAttemptCount();
/**
* Number of times that the scanner attempted to measure a relay in the last
* data period, but the relay has not been measured because of system, network
* or implementation issues.
*
* @since 2.6.0
*/
Optional<Integer> recentMeasurementFailureCount();
/**
* Number of relays that have no successful measurements in the last data
* period.
*
* @since 2.6.0
*/
Optional<Integer> recentMeasurementsExcludedErrorCount();
/**
* Number of relays that have some successful measurements in the last data
* period, but all those measurements were performed in a period of time that
* was too short.
*
* @since 2.6.0
*/
Optional<Integer> recentMeasurementsExcludedNearCount();
/**
* Number of relays that have some successful measurements, but all those
* measurements are too old.
*
* @since 2.6.0
*/
Optional<Integer> recentMeasurementsExcludedOldCount();
/**
* Number of relays that don't have enough recent successful measurements.
*
* @since 2.6.0
*/
Optional<Integer> recentMeasurementsExcludedFewCount();
/**
* Time that it would take to report measurements about half of the network,
* given the number of eligible relays and the time it took in the last days.
*
* @since 2.6.0
*/
Optional<Duration> timeToReportHalfNetwork();
/**
* List of zero or more {@link RelayLine}s containing relay identities and
* bandwidths in the order as they are contained in the bandwidth file.
*
* @since 2.6.0
*/
List<RelayLine> relayLines();
interface RelayLine {
/**
* Fingerprint for the relay's RSA identity key.
*
* @since 2.6.0
*/
Optional<String> nodeId();
/**
* Relays's master Ed25519 key, base64 encoded, without trailing "="s.
*
* @since 2.6.0
*/
Optional<String> masterKeyEd25519();
/**
* Bandwidth of this relay in kilobytes per second.
*
* @since 2.6.0
*/
int bw();
/**
* Additional relay key-value pairs, excluding the key value pairs already
* parsed for relay identities and bandwidths.
*
* @since 2.6.0
*/
Map<String, String> additionalKeyValues();
}
}
/* Copyright 2019 The Tor Project
* See LICENSE for licensing information */
package org.torproject.descriptor.impl;
import org.torproject.descriptor.BandwidthFile;
import org.torproject.descriptor.DescriptorParseException;
import java.io.File;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Scanner;
public class BandwidthFileImpl extends DescriptorImpl implements BandwidthFile {
private enum KeyWithStringValue {
version, software, software_version
}
private enum KeyWithLocalDateTimeValue {
file_created, generator_started, earliest_bandwidth, latest_bandwidth
}
private enum KeyWithIntValue {
number_eligible_relays, minimum_percent_eligible_relays,
number_consensus_relays, percent_eligible_relays,
minimum_number_eligible_relays, recent_consensus_count,
recent_priority_list_count, recent_priority_relay_count,
recent_measurement_attempt_count, recent_measurement_failure_count,
recent_measurements_excluded_error_count,
recent_measurements_excluded_near_count,
recent_measurements_excluded_old_count,
recent_measurements_excluded_few_count
}
BandwidthFileImpl(byte[] rawDescriptorBytes, File descriptorfile)
throws DescriptorParseException {
super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
descriptorfile, false);
Scanner scanner = this.newScanner().useDelimiter("\n");
this.parseTimestampLine(scanner.nextLine());
boolean haveFinishedParsingHeader = false;
while (scanner.hasNext()) {
String line = scanner.nextLine();
if (!haveFinishedParsingHeader) {
if (line.startsWith("bw=") || line.contains(" bw=")) {
haveFinishedParsingHeader = true;
} else if ("====".equals(line) || "=====".equals(line)) {
haveFinishedParsingHeader = true;
continue;
}
}
if (!haveFinishedParsingHeader) {
this.parseHeaderLine(line);
} else {
this.parseRelayLine(line);
}
}
}
private void parseTimestampLine(String line) throws DescriptorParseException {
try {
this.timestamp = LocalDateTime.ofInstant(Instant.ofEpochSecond(
Long.parseLong(line)), ZoneOffset.UTC);
} catch (NumberFormatException | DateTimeParseException e) {
throw new DescriptorParseException(String.format(
"Unable to parse timestamp in first line: '%s'.", line), e);
}
}
private void parseHeaderLine(String line) throws DescriptorParseException {
String[] keyValueParts = line.split("=", 2);
if (keyValueParts.length != 2) {
throw new DescriptorParseException(String.format(
"Unrecognized line '%s' without '=' character.", line));
}
String key = keyValueParts[0];
if (key.length() < 1) {
throw new DescriptorParseException(String.format(
"Unrecognized line '%s' starting with '=' character.", line));
}
String value = keyValueParts[1];
switch (key) {
case "version":
case "software":
case "software_version":
this.parsedStrings.put(KeyWithStringValue.valueOf(key), value);
break;
case "file_created":
case "generator_started":
case "earliest_bandwidth":
case "latest_bandwidth":
try {
this.parsedLocalDateTimes.put(KeyWithLocalDateTimeValue.valueOf(key),
LocalDateTime.parse(value));
} catch (DateTimeParseException e) {
throw new DescriptorParseException(String.format(
"Unable to parse date-time string: '%s'.", value), e);
}
break;
case "number_eligible_relays":
case "minimum_percent_eligible_relays":
case "number_consensus_relays":
case "percent_eligible_relays":
case "minimum_number_eligible_relays":
case "recent_consensus_count":
case "recent_priority_list_count":
case "recent_priority_relay_count":
case "recent_measurement_attempt_count":
case "recent_measurement_failure_count":
case "recent_measurements_excluded_error_count":
case "recent_measurements_excluded_near_count":
case "recent_measurements_excluded_old_count":
case "recent_measurements_excluded_few_count":
try {
this.parsedInts.put(KeyWithIntValue.valueOf(key),
Integer.parseInt(value));
} catch (NumberFormatException e) {
throw new DescriptorParseException(String.format(
"Unable to parse int: '%s'.", value), e);
}
break;
case "scanner_country":
if (!value.matches("[A-Z]{2}")) {
throw new DescriptorParseException(String.format(
"Invalid country code '%s'.", value));
}
this.scannerCountry = value;
break;
case "destinations_countries":
if (!value.matches("[A-Z]{2}(,[A-Z]{2})*")) {
throw new DescriptorParseException(String.format(
"Invalid country code list '%s'.", value));
}
this.destinationsCountries = value.split(",");
break;
case "time_to_report_half_network":
try {
this.timeToReportHalfNetwork
= Duration.ofSeconds(Long.parseLong(value));
} catch (NumberFormatException | DateTimeParseException e) {
throw new DescriptorParseException(String.format(
"Unable to parse duration: '%s'.", value), e);
}
break;
case "node_id":
case "master_key_ed25519":
case "bw":
throw new DescriptorParseException(String.format(
"Either additional header line must not use keywords specified in "
+ "relay lines, or relay line is missing required keys: '%s'.",
line));
default:
/* Ignore additional header lines. */
}
}
private class RelayLineImpl implements RelayLine {
private String nodeId;
@Override
public Optional<String> nodeId() {
return Optional.ofNullable(this.nodeId);
}
private String masterKeyEd25519;
@Override
public Optional<String> masterKeyEd25519() {
return Optional.ofNullable(this.masterKeyEd25519);
}
private int bw;
@Override
public int bw() {
return this.bw;
}
private Map<String, String> additionalKeyValues;
@Override
public Map<String, String> additionalKeyValues() {
return null == this.additionalKeyValues ? Collections.emptyMap()
: Collections.unmodifiableMap(this.additionalKeyValues);
}
private RelayLineImpl(String nodeId, String masterKeyEd25519, int bw,
Map<String, String> additionalKeyValues) {
this.nodeId = nodeId;
this.masterKeyEd25519 = masterKeyEd25519;
this.bw = bw;
this.additionalKeyValues = additionalKeyValues;
}
}
private void parseRelayLine(String line) throws DescriptorParseException {
String[] spaceSeparatedLineParts = line.split(" ");
String nodeId = null;
String masterKeyEd25519 = null;
Integer bw = null;
Map<String, String> additionalKeyValues = new LinkedHashMap<>();
for (String spaceSeparatedLinePart : spaceSeparatedLineParts) {
String[] keyValueParts = spaceSeparatedLinePart.split("=", 2);
if (keyValueParts.length != 2) {
throw new DescriptorParseException(String.format(
"Unrecognized space-separated line part '%s' without '=' "
+ "character in line '%s'.", spaceSeparatedLinePart, line));
}
String key = keyValueParts[0];
if (key.length() < 1) {
throw new DescriptorParseException(String.format(
"Unrecognized space-separated line part '%s' starting with '=' "
+ "character in line '%s'.", spaceSeparatedLinePart, line));
}
String value = keyValueParts[1];
switch (key) {
case "node_id":
nodeId = value;
break;
case "master_key_ed25519":
masterKeyEd25519 = value;
break;
case "bw":
try {
bw = Integer.parseInt(value);
} catch (NumberFormatException e) {
throw new DescriptorParseException(String.format(
"Unable to parse bw '%s' in line '%s'.", value, line), e);
}
break;
default:
additionalKeyValues.put(key, value);
}
}
if (null == nodeId && null == masterKeyEd25519) {
throw new DescriptorParseException(String.format(
"Expected relay line, but line contains neither node_id nor "
+ "master_key_ed25519: '%s'.", line));
}
if (null == bw) {
throw new DescriptorParseException(String.format(
"Expected relay line, but line does not contain bw: '%s'.", line));
}
this.relayLines.add(new RelayLineImpl(nodeId, masterKeyEd25519, bw,
additionalKeyValues.isEmpty() ? null : additionalKeyValues));
}
private LocalDateTime timestamp;
@Override
public LocalDateTime timestamp() {
return this.timestamp;
}
private EnumMap<KeyWithStringValue, String> parsedStrings
= new EnumMap<>(KeyWithStringValue.class);
@Override
public String version() {
return this.parsedStrings.getOrDefault(KeyWithStringValue.version,
"1.0.0");
}
@Override
public String software() {
return this.parsedStrings.getOrDefault(KeyWithStringValue.software,
"torflow");
}
@Override
public Optional<String> softwareVersion() {
return Optional.ofNullable(
this.parsedStrings.get(KeyWithStringValue.software_version));
}
private EnumMap<KeyWithLocalDateTimeValue, LocalDateTime> parsedLocalDateTimes
= new EnumMap<>(KeyWithLocalDateTimeValue.class);
@Override
public Optional<LocalDateTime> fileCreated() {
return Optional.ofNullable(this.parsedLocalDateTimes.get(
KeyWithLocalDateTimeValue.file_created));
}
@Override
public Optional<LocalDateTime> generatorStarted() {
return Optional.ofNullable(this.parsedLocalDateTimes.get(
KeyWithLocalDateTimeValue.generator_started));
}
@Override
public Optional<LocalDateTime> earliestBandwidth() {
return Optional.ofNullable(this.parsedLocalDateTimes.get(
KeyWithLocalDateTimeValue.earliest_bandwidth));
}
@Override
public Optional<LocalDateTime> latestBandwidth() {
return Optional.ofNullable(this.parsedLocalDateTimes.get(
KeyWithLocalDateTimeValue.latest_bandwidth));
}
private EnumMap<KeyWithIntValue, Integer> parsedInts
= new EnumMap<>(KeyWithIntValue.class);
@Override
public Optional<Integer> numberEligibleRelays() {
return Optional.ofNullable(this.parsedInts.get(
KeyWithIntValue.number_eligible_relays));
}
@Override
public Optional<Integer> minimumPercentEligibleRelays() {
return Optional.ofNullable(this.parsedInts.get(
KeyWithIntValue.minimum_percent_eligible_relays));
}
@Override
public Optional<Integer> numberConsensusRelays() {
return Optional.ofNullable(this.parsedInts.get(
KeyWithIntValue.number_consensus_relays));
}
@Override
public Optional<Integer> percentEligibleRelays() {