Commit abde3f75 authored by Karsten Loesing's avatar Karsten Loesing
Browse files

Add source name to locally imported bandwidth files. 

Implements #30219.
parent 6ad185ca

CHANGELOG.md

+7 −0
Original line number Diff line number Diff line 
# Changes in version 1.10.0 - 2019-??-??



 * Medium changes

   - Changed local import of bandwidth files to include the parent

     directory name as @source annotation and to the filename.





# Changes in version 1.9.1 - 2019-05-29



 * Medium changes

src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java

+1 −1
Original line number Diff line number Diff line
@@ -141,7 +141,7 @@ public class ArchiveReader { 
            }

            bis.close();

            byte[] allData = baos.toByteArray();

            boolean stored = this.rdp.parse(allData);

            boolean stored = this.rdp.parse(allData, pop);

            if (!stored) {

              filesToRetry.add(pop);

              continue;

src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java

+9 −3
Original line number Diff line number Diff line
@@ -745,18 +745,24 @@ public class ArchiveWriter extends CollecTorMain { 


  /** Stores a bandwidth file to disk. */

  void storeBandwidthFile(byte[] data, LocalDateTime fileCreatedOrTimestamp,

      String bandwidthFileDigest) {

      String sourceName, String bandwidthFileDigest) {

    DateTimeFormatter printFormat = DateTimeFormatter

        .ofPattern("uuuu/MM/dd/uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC);

    File tarballFile = Paths.get(this.outputDirectory, "bandwidth",

        fileCreatedOrTimestamp.format(printFormat) + "-bandwidth-"

        + (null == sourceName ? "" : (sourceName + "-"))

        + bandwidthFileDigest).toFile();

    StringBuilder sb = new StringBuilder();

    sb.append(Annotation.BandwidthFile.toString());

    if (null != sourceName) {

      sb.append("@source ").append(sourceName).append('\n');

    }

    boolean tarballFileExistedBefore = tarballFile.exists();

    File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "bandwidths",

        tarballFile.getName()).toFile();

    File[] outputFiles = new File[] { tarballFile, rsyncFile };

    if (this.store(Annotation.BandwidthFile.bytes(), data, outputFiles, null)) {

      this.storedVotesCounter++;

    if (this.store(sb.toString().getBytes(), data, outputFiles, null)) {

      this.storedBandwidthsCounter++;

    }

    if (!tarballFileExistedBefore

        && this.nowLocalDateTime.isAfter(fileCreatedOrTimestamp.plusDays(3L))) {

src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java

+3 −3
Original line number Diff line number Diff line
@@ -158,7 +158,7 @@ public class CachedRelayDescriptorReader { 
                allData));

            if (!this.lastImportHistory.contains(digest)

                && !this.currentImportHistory.contains(digest)) {

              this.rdp.parse(allData);

              this.rdp.parse(allData, null);

            } else {

              this.dumpStats.append(" (skipped)");

            }
@@ -183,7 +183,7 @@ public class CachedRelayDescriptorReader { 
                    rawNetworkStatusBytes));

                if (!this.lastImportHistory.contains(digest)

                    && !this.currentImportHistory.contains(digest)) {

                  this.rdp.parse(rawNetworkStatusBytes);

                  this.rdp.parse(rawNetworkStatusBytes, null);

                  parsedNum++;

                } else {

                  skippedNum++;
@@ -229,7 +229,7 @@ public class CachedRelayDescriptorReader { 
                  descBytes));

              if (!this.lastImportHistory.contains(digest)

                  && !this.currentImportHistory.contains(digest)) {

                this.rdp.parse(descBytes);

                this.rdp.parse(descBytes, null);

                parsedNum++;

              } else {

                skippedNum++;

src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java

+2 −2
Original line number Diff line number Diff line
@@ -899,7 +899,7 @@ public class RelayDescriptorDownloader { 
    int receivedDescriptors = 0;

    if (allData != null) {

      if (resource.startsWith("/tor/status-vote/")) {

        this.rdp.parse(allData);

        this.rdp.parse(allData, null);

        receivedDescriptors = 1;

      } else if (resource.startsWith("/tor/server/")

          || resource.startsWith("/tor/extra/")) {
@@ -933,7 +933,7 @@ public class RelayDescriptorDownloader { 
          end += endToken.length();

          byte[] descBytes = new byte[end - start];

          System.arraycopy(allData, start, descBytes, 0, end - start);

          this.rdp.parse(descBytes);

          this.rdp.parse(descBytes, null);

          receivedDescriptors++;

        }

      } else if (resource.startsWith("/tor/micro/")) {