Commit ad1bbc88 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Archive bandwidth files in relaydescs module. 

Open questions:
 - Are we certain about descriptor annotation @type bandwidth-file
   1.0? This is consistent with respect to other relay descriptors,
   but it's not really intuitive with bandwidth file versions 1.0.0,
   1.1.0, etc.
 - Do we need config options to turn this off?

Temporary commit; necessary changes:
 - Change metrics-lib 2.6.0-dev dependency to 2.6.1 as soon as that's
   released.

Implements #30218.
parent 19ae66a1

CHANGELOG.md

+7 −0
Original line number Diff line number Diff line 
# Changes in version 1.9.0 - 2019-05-??



 * Medium changes

   - Archive bandwidth files in relaydescs module.

   - Update to metrics-lib 2.6.1.





# Changes in version 1.8.0 - 2018-10-11



 * Medium changes

build.xml

+2 −2
Original line number Diff line number Diff line
@@ -11,7 +11,7 @@ 
  <property name="release.version" value="1.8.0-dev" />

  <property name="project-main-class" value="org.torproject.metrics.collector.Main" />

  <property name="name" value="collector"/>

  <property name="metricslibversion" value="2.4.0" />

  <property name="metricslibversion" value="2.6.0-dev" />

  <property name="jarincludes" value="collector.properties logback.xml" />



  <patternset id="runtime" >
@@ -21,7 +21,7 @@ 
      <include name="jackson-core-2.8.6.jar"/>

      <include name="jackson-databind-2.8.6.jar"/>

      <include name="xz-1.6.jar"/>

      <include name="metrics-lib-${metricslibversion}.jar"/>

      <include name="metrics-lib-${metricslibversion}-thin.jar"/>

      <include name="logback-core-1.1.9.jar" />

      <include name="logback-classic-1.1.9.jar" />

      <include name="slf4j-api-1.7.22.jar" />

src/main/java/org/torproject/metrics/collector/conf/Annotation.java

+1 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ package org.torproject.metrics.collector.conf; 
/** This enum contains all currently valid descriptor annotations. */

public enum Annotation {



  BandwidthFile("@type bandwidth-file 1.0\n"),

  BridgeExtraInfo("@type bridge-extra-info 1.3\n"),

  BridgeServer("@type bridge-server-descriptor 1.2\n"),

  Cert("@type dir-key-certificate-3 1.0\n"),

src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java

+42 −1
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 


package org.torproject.metrics.collector.relaydescs;



import org.torproject.descriptor.BandwidthFile;

import org.torproject.descriptor.Descriptor;

import org.torproject.descriptor.DescriptorParser;

import org.torproject.descriptor.DescriptorSourceFactory;
@@ -29,10 +30,15 @@ import java.io.FileOutputStream; 
import java.io.FileReader;

import java.io.FileWriter;

import java.io.IOException;

import java.nio.charset.StandardCharsets;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.text.ParseException;

import java.text.SimpleDateFormat;

import java.time.Instant;

import java.time.LocalDateTime;

import java.time.ZoneOffset;

import java.time.format.DateTimeFormatter;

import java.util.Arrays;

import java.util.Date;

import java.util.HashMap;
@@ -44,6 +50,7 @@ import java.util.SortedSet; 
import java.util.Stack;

import java.util.TimeZone;

import java.util.TreeMap;

import java.util.TreeSet;



public class ArchiveWriter extends CollecTorMain {
@@ -51,12 +58,15 @@ public class ArchiveWriter extends CollecTorMain { 
      ArchiveWriter.class);



  private long now = System.currentTimeMillis();

  private LocalDateTime nowLocalDateTime

      = LocalDateTime.ofInstant(Instant.ofEpochMilli(this.now), ZoneOffset.UTC);

  private String outputDirectory;

  private String rsyncCatString;

  private DescriptorParser descriptorParser;

  private int storedConsensusesCounter = 0;

  private int storedMicrodescConsensusesCounter = 0;

  private int storedVotesCounter = 0;

  private int storedBandwidthsCounter = 0;

  private int storedCertsCounter = 0;

  private int storedServerDescriptorsCounter = 0;

  private int storedExtraInfoDescriptorsCounter = 0;
@@ -74,6 +84,8 @@ public class ArchiveWriter extends CollecTorMain { 
  private SortedMap<Long, Set<String>> storedExtraInfoDescriptors =

      new TreeMap<>();

  private SortedMap<Long, Set<String>> storedMicrodescriptors = new TreeMap<>();

  private SortedMap<LocalDateTime, Set<String>> storedBandwidths

      = new TreeMap<>();



  private File storedServerDescriptorsFile;

  private File storedExtraInfoDescriptorsFile;
@@ -103,6 +115,8 @@ public class ArchiveWriter extends CollecTorMain { 
        RelayServerDescriptor.class);

    this.mapPathDescriptors.put("recent/relay-descriptors/extra-infos",

        RelayExtraInfoDescriptor.class);

    this.mapPathDescriptors.put("recent/relay-descriptors/bandwidths",

        BandwidthFile.class);

  }



  @Override
@@ -203,6 +217,7 @@ public class ArchiveWriter extends CollecTorMain { 
    this.storedConsensuses.clear();

    this.storedMicrodescConsensuses.clear();

    this.storedVotes.clear();

    this.storedBandwidths.clear();

    this.storedServerDescriptors.clear();

    this.storedExtraInfoDescriptors.clear();

    this.storedMicrodescriptors.clear();
@@ -299,7 +314,8 @@ public class ArchiveWriter extends CollecTorMain { 
        .append(this.storedConsensusesCounter).append(" consensus(es), ")

        .append(this.storedMicrodescConsensusesCounter).append(" microdesc ")

        .append("consensus(es), ").append(this.storedVotesCounter)

        .append(" vote(s), ").append(this.storedCertsCounter)

        .append(" vote(s), ").append(this.storedBandwidthsCounter)

        .append(" bandwidth file(s), ").append(this.storedCertsCounter)

        .append(" certificate(s), ").append(this.storedServerDescriptorsCounter)

        .append(" server descriptor(s), ")

        .append(this.storedExtraInfoDescriptorsCounter).append(" extra-info ")
@@ -309,6 +325,7 @@ public class ArchiveWriter extends CollecTorMain { 
    this.storedConsensusesCounter = 0;

    this.storedMicrodescConsensusesCounter = 0;

    this.storedVotesCounter = 0;

    this.storedBandwidthsCounter = 0;

    this.storedCertsCounter = 0;

    this.storedServerDescriptorsCounter = 0;

    this.storedExtraInfoDescriptorsCounter = 0;
@@ -727,6 +744,30 @@ public class ArchiveWriter extends CollecTorMain { 
    }

  }



  /** Stores a bandwidth file to disk. */

  void storeBandwidthFile(byte[] data, LocalDateTime fileCreatedOrTimestamp,

      String bandwidthFileDigest) {

    DateTimeFormatter printFormat = DateTimeFormatter

        .ofPattern("uuuu/MM/dd/uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC);

    File tarballFile = Paths.get(this.outputDirectory, "bandwidth",

        fileCreatedOrTimestamp.format(printFormat) + "-bandwidth-"

        + bandwidthFileDigest).toFile();

    boolean tarballFileExistedBefore = tarballFile.exists();

    File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "bandwidths",

        tarballFile.getName()).toFile();

    File[] outputFiles = new File[] { tarballFile, rsyncFile };

    if (this.store(Annotation.BandwidthFile.bytes(), data, outputFiles, null)) {

      this.storedVotesCounter++;

    }

    if (!tarballFileExistedBefore

        && this.nowLocalDateTime.isAfter(fileCreatedOrTimestamp.plusDays(3L))) {

      this.storedBandwidths.putIfAbsent(fileCreatedOrTimestamp,

          new TreeSet<>());

      this.storedBandwidths.get(fileCreatedOrTimestamp)

          .add(bandwidthFileDigest);

    }

  }



  /** Stores a key certificate to disk. */

  public void storeCertificate(byte[] data, String fingerprint,

      long published) {

src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java

+20 −4
Original line number Diff line number Diff line
@@ -258,6 +258,8 @@ public class RelayDescriptorDownloader { 


  private int requestedVotes = 0;



  private int requestedBandwidthFiles = 0;



  private int requestedMissingServerDescriptors = 0;



  private int requestedAllServerDescriptors = 0;
@@ -274,6 +276,8 @@ public class RelayDescriptorDownloader { 


  private int downloadedVotes = 0;



  private int downloadedBandwidthFiles = 0;



  private int downloadedMissingServerDescriptors = 0;



  private int downloadedAllServerDescriptors = 0;
@@ -729,6 +733,14 @@ public class RelayDescriptorDownloader { 
          }

        }



        /* Now try to download the bandwidth file, regardless of whether this

         * authority might provide one or when we last downloaded a bandwidth

         * file from it. */

        this.requestedBandwidthFiles++;

        this.downloadedBandwidthFiles +=

            this.downloadResourceFromAuthority(authority,

            "/tor/status-vote/next/bandwidth");



        /* Download either all server and extra-info descriptors or only

         * those that we're missing. Start with server descriptors, then

         * request extra-info descriptors. Finally, request missing
@@ -886,7 +898,7 @@ public class RelayDescriptorDownloader { 
        allData == null ? 0 : allData.length);

    int receivedDescriptors = 0;

    if (allData != null) {

      if (resource.startsWith("/tor/status-vote/current/")) {

      if (resource.startsWith("/tor/status-vote/")) {

        this.rdp.parse(allData);

        receivedDescriptors = 1;

      } else if (resource.startsWith("/tor/server/")
@@ -1067,11 +1079,13 @@ public class RelayDescriptorDownloader { 
        this.newMissingServerDescriptors, this.newMissingExtraInfoDescriptors,

        this.newMissingMicrodescriptors);

    logger.info("We requested {} consensus(es), {} microdesc consensus(es), "

        + "{} vote(s), {} missing server descriptor(s), {} times all server "

        + "{} vote(s), {} bandwidth file(s), {} missing server descriptor(s), "

        + "{} times all server "

        + "descriptors, {} missing extra-info descriptor(s), {} times all "

        + "extra-info descriptors, and {} missing microdescriptor(s) from the "

        + "directory authorities.", this.requestedConsensuses,

        this.requestedMicrodescConsensuses, this.requestedVotes,

        this.requestedBandwidthFiles,

        this.requestedMissingServerDescriptors,

        this.requestedAllServerDescriptors,

        this.requestedMissingExtraInfoDescriptors,
@@ -1085,12 +1099,14 @@ public class RelayDescriptorDownloader { 
    logger.info("We sent these numbers of requests to the directory "

        + "authorities:{}", sb.toString());

    logger.info("We successfully downloaded {} consensus(es), {} microdesc "

        + "consensus(es), {} vote(s), {} missing server descriptor(s), {} "

        + "consensus(es), {} vote(s), {} bandwidth file(s), "

        + "{} missing server descriptor(s), {} "

        + "server descriptor(s) when downloading all descriptors, {} missing "

        + "extra-info descriptor(s), {} extra-info descriptor(s) when "

        + "downloading all descriptors, and {} missing microdescriptor(s).",

        this.downloadedConsensuses, this.downloadedMicrodescConsensuses,

        this.downloadedVotes, this.downloadedMissingServerDescriptors,

        this.downloadedVotes, this.downloadedBandwidthFiles,

        this.downloadedMissingServerDescriptors,

        this.downloadedAllServerDescriptors,

        this.downloadedMissingExtraInfoDescriptors,

        this.downloadedAllExtraInfoDescriptors,