Add BandwidthFile for parsed bandwidth files.

Implements #30216.

CHANGELOG.md

+# Changes in version 2.6.0 - 2019-04-??
+
+ * Medium changes
+   - Add new BandwidthFile descriptor for parsed bandwidth files.
+
+
 # Changes in version 2.5.0 - 2018-09-25
 
  * Medium changes

src/main/java/org/torproject/descriptor/BandwidthFile.java

+/* Copyright 2019 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A bandwidth file contains information on relays' bandwidth capacities and is
+ * produced by bandwidth generators, previously known as bandwidth scanners.
+ *
+ * @since 2.6.0
+ */
+public interface BandwidthFile extends Descriptor {
+
+  /**
+   * Time of the most recent generator bandwidth result.
+   *
+   * @since 2.6.0
+   */
+  LocalDateTime timestamp();
+
+  /**
+   * Document format version.
+   *
+   * @since 2.6.0
+   */
+  String version();
+
+  /**
+   * Name of the software that created the document.
+   *
+   * @since 2.6.0
+   */
+  String software();
+
+  /**
+   * Version of the software that created the document.
+   *
+   * @since 2.6.0
+   */
+  Optional<String> softwareVersion();
+
+  /**
+   * Timestamp in UTC time zone when the file was created.
+   *
+   * @since 2.6.0
+   */
+  Optional<LocalDateTime> fileCreated();
+
+  /**
+   * Timestamp in UTC time zone when the generator was started.
+   *
+   * @since 2.6.0
+   */
+  Optional<LocalDateTime> generatorStarted();
+
+  /**
+   * Timestamp in UTC time zone when the first relay bandwidth was obtained.
+   *
+   * @since 2.6.0
+   */
+  Optional<LocalDateTime> earliestBandwidth();
+
+  /**
+   * Timestamp in UTC time zone of the most recent generator bandwidth result.
+   *
+   * @since 2.6.0
+   */
+  Optional<LocalDateTime> latestBandwidth();
+
+  /**
+   * Number of relays that have enough measurements to be included in the
+   * bandwidth file.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> numberEligibleRelays();
+
+  /**
+   * Percentage of relays in the consensus that should be included in every
+   * generated bandwidth file.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> minimumPercentEligibleRelays();
+
+  /**
+   * Number of relays in the consensus.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> numberConsensusRelays();
+
+  /**
+   * The number of eligible relays, as a percentage of the number of relays in
+   * the consensus.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> percentEligibleRelays();
+
+  /**
+   * Minimum number of relays that should be included in the bandwidth file.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> minimumNumberEligibleRelays();
+
+  /**
+   * Country, as in political geolocation, where the generator is run.
+   *
+   * @since 2.6.0
+   */
+  Optional<String> scannerCountry();
+
+  /**
+   * Country, as in political geolocation, or countries where the destination
+   * web server(s) are located.
+   *
+   * @since 2.6.0
+   */
+  Optional<String[]> destinationsCountries();
+
+  /**
+   * Number of the different consensuses seen in the last data period.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentConsensusCount();
+
+  /**
+   * Number of times that a list with a subset of relays prioritized to be
+   * measured has been created in the last data period.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentPriorityListCount();
+
+  /**
+   * Number of relays that has been in in the list of relays prioritized to be
+   * measured in the last data period.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentPriorityRelayCount();
+
+  /**
+   * Number of times that any relay has been queued to be measured in the last
+   * data period.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentMeasurementAttemptCount();
+
+  /**
+   * Number of times that the scanner attempted to measure a relay in the last
+   * data period, but the relay has not been measured because of system, network
+   * or implementation issues.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentMeasurementFailureCount();
+
+  /**
+   * Number of relays that have no successful measurements in the last data
+   * period.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentMeasurementsExcludedErrorCount();
+
+  /**
+   * Number of relays that have some successful measurements in the last data
+   * period, but all those measurements were performed in a period of time that
+   * was too short.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentMeasurementsExcludedNearCount();
+
+  /**
+   * Number of relays that have some successful measurements, but all those
+   * measurements are too old.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentMeasurementsExcludedOldCount();
+
+  /**
+   * Number of relays that don't have enough recent successful measurements.
+   *
+   * @since 2.6.0
+   */
+  Optional<Integer> recentMeasurementsExcludedFewCount();
+
+  /**
+   * Time that it would take to report measurements about half of the network,
+   * given the number of eligible relays and the time it took in the last days.
+   *
+   * @since 2.6.0
+   */
+  Optional<Duration> timeToReportHalfNetwork();
+
+  /**
+   * List of zero or more {@link RelayLine}s containing relay identities and
+   * bandwidths in the order as they are contained in the bandwidth file.
+   *
+   * @since 2.6.0
+   */
+  List<RelayLine> relayLines();
+
+  interface RelayLine {
+
+    /**
+     * Fingerprint for the relay's RSA identity key.
+     *
+     * @since 2.6.0
+     */
+    Optional<String> nodeId();
+
+    /**
+     * Relays's master Ed25519 key, base64 encoded, without trailing "="s.
+     *
+     * @since 2.6.0
+     */
+    Optional<String> masterKeyEd25519();
+
+    /**
+     * Bandwidth of this relay in kilobytes per second.
+     *
+     * @since 2.6.0
+     */
+    int bw();
+
+    /**
+     * Additional relay key-value pairs, excluding the key value pairs already
+     * parsed for relay identities and bandwidths.
+     *
+     * @since 2.6.0
+     */
+    Map<String, String> additionalKeyValues();
+  }
+}
+

src/main/java/org/torproject/descriptor/impl/BandwidthFileImpl.java

+/* Copyright 2019 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.impl;
+
+import org.torproject.descriptor.BandwidthFile;
+import org.torproject.descriptor.DescriptorParseException;
+
+import java.io.File;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Scanner;
+
+public class BandwidthFileImpl extends DescriptorImpl implements BandwidthFile {
+
+  private enum KeyWithStringValue {
+    version, software, software_version
+  }
+
+  private enum KeyWithLocalDateTimeValue {
+    file_created, generator_started, earliest_bandwidth, latest_bandwidth
+  }
+
+  private enum KeyWithIntValue {
+    number_eligible_relays, minimum_percent_eligible_relays,
+    number_consensus_relays, percent_eligible_relays,
+    minimum_number_eligible_relays, recent_consensus_count,
+    recent_priority_list_count, recent_priority_relay_count,
+    recent_measurement_attempt_count, recent_measurement_failure_count,
+    recent_measurements_excluded_error_count,
+    recent_measurements_excluded_near_count,
+    recent_measurements_excluded_old_count,
+    recent_measurements_excluded_few_count
+  }
+
+  BandwidthFileImpl(byte[] rawDescriptorBytes, File descriptorfile)
+      throws DescriptorParseException {
+    super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
+        descriptorfile, false);
+    Scanner scanner = this.newScanner().useDelimiter("\n");
+    this.parseTimestampLine(scanner.nextLine());
+    boolean haveFinishedParsingHeader = false;
+    while (scanner.hasNext()) {
+      String line = scanner.nextLine();
+      if (!haveFinishedParsingHeader) {
+        if (line.startsWith("bw=") || line.contains(" bw=")) {
+          haveFinishedParsingHeader = true;
+        } else if ("====".equals(line) || "=====".equals(line)) {
+          haveFinishedParsingHeader = true;
+          continue;
+        }
+      }
+      if (!haveFinishedParsingHeader) {
+        this.parseHeaderLine(line);
+      } else {
+        this.parseRelayLine(line);
+      }
+    }
+  }
+
+  private void parseTimestampLine(String line) throws DescriptorParseException {
+    try {
+      this.timestamp = LocalDateTime.ofInstant(Instant.ofEpochSecond(
+          Long.parseLong(line)), ZoneOffset.UTC);
+    } catch (NumberFormatException | DateTimeParseException e) {
+      throw new DescriptorParseException(String.format(
+          "Unable to parse timestamp in first line: '%s'.", line), e);
+    }
+  }
+
+  private void parseHeaderLine(String line) throws DescriptorParseException {
+    String[] keyValueParts = line.split("=", 2);
+    if (keyValueParts.length != 2) {
+      throw new DescriptorParseException(String.format(
+          "Unrecognized line '%s' without '=' character.", line));
+    }
+    String key = keyValueParts[0];
+    if (key.length() < 1) {
+      throw new DescriptorParseException(String.format(
+          "Unrecognized line '%s' starting with '=' character.", line));
+    }
+    String value = keyValueParts[1];
+    switch (key) {
+      case "version":
+      case "software":
+      case "software_version":
+        this.parsedStrings.put(KeyWithStringValue.valueOf(key), value);
+        break;
+      case "file_created":
+      case "generator_started":
+      case "earliest_bandwidth":
+      case "latest_bandwidth":
+        try {
+          this.parsedLocalDateTimes.put(KeyWithLocalDateTimeValue.valueOf(key),
+              LocalDateTime.parse(value));
+        } catch (DateTimeParseException e) {
+          throw new DescriptorParseException(String.format(
+              "Unable to parse date-time string: '%s'.", value), e);
+        }
+        break;
+      case "number_eligible_relays":
+      case "minimum_percent_eligible_relays":
+      case "number_consensus_relays":
+      case "percent_eligible_relays":
+      case "minimum_number_eligible_relays":
+      case "recent_consensus_count":
+      case "recent_priority_list_count":
+      case "recent_priority_relay_count":
+      case "recent_measurement_attempt_count":
+      case "recent_measurement_failure_count":
+      case "recent_measurements_excluded_error_count":
+      case "recent_measurements_excluded_near_count":
+      case "recent_measurements_excluded_old_count":
+      case "recent_measurements_excluded_few_count":
+        try {
+          this.parsedInts.put(KeyWithIntValue.valueOf(key),
+              Integer.parseInt(value));
+        } catch (NumberFormatException e) {
+          throw new DescriptorParseException(String.format(
+              "Unable to parse int: '%s'.", value), e);
+        }
+        break;
+      case "scanner_country":
+        if (!value.matches("[A-Z]{2}")) {
+          throw new DescriptorParseException(String.format(
+              "Invalid country code '%s'.", value));
+        }
+        this.scannerCountry = value;
+        break;
+      case "destinations_countries":
+        if (!value.matches("[A-Z]{2}(,[A-Z]{2})*")) {
+          throw new DescriptorParseException(String.format(
+              "Invalid country code list '%s'.", value));
+        }
+        this.destinationsCountries = value.split(",");
+        break;
+      case "time_to_report_half_network":
+        try {
+          this.timeToReportHalfNetwork
+              = Duration.ofSeconds(Long.parseLong(value));
+        } catch (NumberFormatException | DateTimeParseException e) {
+          throw new DescriptorParseException(String.format(
+              "Unable to parse duration: '%s'.", value), e);
+        }
+        break;
+      case "node_id":
+      case "master_key_ed25519":
+      case "bw":
+        throw new DescriptorParseException(String.format(
+            "Either additional header line must not use keywords specified in "
+            + "relay lines, or relay line is missing required keys: '%s'.",
+            line));
+      default:
+        /* Ignore additional header lines. */
+    }
+  }
+
+  private class RelayLineImpl implements RelayLine {
+
+    private String nodeId;
+
+    @Override
+    public Optional<String> nodeId() {
+      return Optional.ofNullable(this.nodeId);
+    }
+
+    private String masterKeyEd25519;
+
+    @Override
+    public Optional<String> masterKeyEd25519() {
+      return Optional.ofNullable(this.masterKeyEd25519);
+    }
+
+    private int bw;
+
+    @Override
+    public int bw() {
+      return this.bw;
+    }
+
+    private Map<String, String> additionalKeyValues;
+
+    @Override
+    public Map<String, String> additionalKeyValues() {
+      return null == this.additionalKeyValues ? Collections.emptyMap()
+          : Collections.unmodifiableMap(this.additionalKeyValues);
+    }
+
+    private RelayLineImpl(String nodeId, String masterKeyEd25519, int bw,
+        Map<String, String> additionalKeyValues) {
+      this.nodeId = nodeId;
+      this.masterKeyEd25519 = masterKeyEd25519;
+      this.bw = bw;
+      this.additionalKeyValues = additionalKeyValues;
+    }
+  }
+
+  private void parseRelayLine(String line) throws DescriptorParseException {
+    String[] spaceSeparatedLineParts = line.split(" ");
+    String nodeId = null;
+    String masterKeyEd25519 = null;
+    Integer bw = null;
+    Map<String, String> additionalKeyValues = new LinkedHashMap<>();
+    for (String spaceSeparatedLinePart : spaceSeparatedLineParts) {
+      String[] keyValueParts = spaceSeparatedLinePart.split("=", 2);
+      if (keyValueParts.length != 2) {
+        throw new DescriptorParseException(String.format(
+            "Unrecognized space-separated line part '%s' without '=' "
+                + "character in line '%s'.", spaceSeparatedLinePart, line));
+      }
+      String key = keyValueParts[0];
+      if (key.length() < 1) {
+        throw new DescriptorParseException(String.format(
+            "Unrecognized space-separated line part '%s' starting with '=' "
+                + "character in line '%s'.", spaceSeparatedLinePart, line));
+      }
+      String value = keyValueParts[1];
+      switch (key) {
+        case "node_id":
+          nodeId = value;
+          break;
+        case "master_key_ed25519":
+          masterKeyEd25519 = value;
+          break;
+        case "bw":
+          try {
+            bw = Integer.parseInt(value);
+          } catch (NumberFormatException e) {
+            throw new DescriptorParseException(String.format(
+                "Unable to parse bw '%s' in line '%s'.", value, line), e);
+          }
+          break;
+        default:
+          additionalKeyValues.put(key, value);
+      }
+    }
+    if (null == nodeId && null == masterKeyEd25519) {
+      throw new DescriptorParseException(String.format(
+          "Expected relay line, but line contains neither node_id nor "
+          + "master_key_ed25519: '%s'.", line));
+    }
+    if (null == bw) {
+      throw new DescriptorParseException(String.format(
+          "Expected relay line, but line does not contain bw: '%s'.", line));
+    }
+    this.relayLines.add(new RelayLineImpl(nodeId, masterKeyEd25519, bw,
+        additionalKeyValues.isEmpty() ? null : additionalKeyValues));
+  }
+
+  private LocalDateTime timestamp;
+
+  @Override
+  public LocalDateTime timestamp() {
+    return this.timestamp;
+  }
+
+  private EnumMap<KeyWithStringValue, String> parsedStrings
+      = new EnumMap<>(KeyWithStringValue.class);
+
+  @Override
+  public String version() {
+    return this.parsedStrings.getOrDefault(KeyWithStringValue.version,
+        "1.0.0");
+  }
+
+  @Override
+  public String software() {
+    return this.parsedStrings.getOrDefault(KeyWithStringValue.software,
+        "torflow");
+  }
+
+  @Override
+  public Optional<String> softwareVersion() {
+    return Optional.ofNullable(
+        this.parsedStrings.get(KeyWithStringValue.software_version));
+  }
+
+  private EnumMap<KeyWithLocalDateTimeValue, LocalDateTime> parsedLocalDateTimes
+      = new EnumMap<>(KeyWithLocalDateTimeValue.class);
+
+  @Override
+  public Optional<LocalDateTime> fileCreated() {
+    return Optional.ofNullable(this.parsedLocalDateTimes.get(
+        KeyWithLocalDateTimeValue.file_created));
+  }
+
+  @Override
+  public Optional<LocalDateTime> generatorStarted() {
+    return Optional.ofNullable(this.parsedLocalDateTimes.get(
+        KeyWithLocalDateTimeValue.generator_started));
+  }
+
+  @Override
+  public Optional<LocalDateTime> earliestBandwidth() {
+    return Optional.ofNullable(this.parsedLocalDateTimes.get(
+        KeyWithLocalDateTimeValue.earliest_bandwidth));
+  }
+
+  @Override
+  public Optional<LocalDateTime> latestBandwidth() {
+    return Optional.ofNullable(this.parsedLocalDateTimes.get(
+        KeyWithLocalDateTimeValue.latest_bandwidth));
+  }
+
+  private EnumMap<KeyWithIntValue, Integer> parsedInts
+      = new EnumMap<>(KeyWithIntValue.class);
+
+  @Override
+  public Optional<Integer> numberEligibleRelays() {
+    return Optional.ofNullable(this.parsedInts.get(
+        KeyWithIntValue.number_eligible_relays));
+  }
+
+  @Override
+  public Optional<Integer> minimumPercentEligibleRelays() {
+    return Optional.ofNullable(this.parsedInts.get(
+        KeyWithIntValue.minimum_percent_eligible_relays));
+  }
+
+  @Override
+  public Optional<Integer> numberConsensusRelays() {
+    return Optional.ofNullable(this.parsedInts.get(
+        KeyWithIntValue.number_consensus_relays));
+  }
+
+  @Override
+  public Optional<Integer> percentEligibleRelays() {