Commit de8f6682 authored by Iain Learmonth's avatar Iain Learmonth
Browse files

Adds the new bridgestrap module.

See: #40002
parent 98287101
Pipeline #10744 failed with stage
in 2 minutes and 43 seconds
......@@ -6,6 +6,7 @@ package org.torproject.metrics.collector;
import org.torproject.metrics.collector.bridgedb.BridgedbMetricsProcessor;
import org.torproject.metrics.collector.bridgedescs.SanitizedBridgesWriter;
import org.torproject.metrics.collector.bridgepools.BridgePoolAssignmentsProcessor;
import org.torproject.metrics.collector.bridgestrap.BridgestrapStatsDownloader;
import org.torproject.metrics.collector.conf.Configuration;
import org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
......@@ -64,6 +65,8 @@ public class Main {
SnowflakeStatsDownloader.class);
collecTorMains.put(Key.BridgedbMetricsActivated,
BridgedbMetricsProcessor.class);
collecTorMains.put(Key.BridgestrapStatsActivated,
BridgestrapStatsDownloader.class);
}
private static Configuration conf = new Configuration();
......
/* Copyright 2019--2020 The Tor Project
* See LICENSE for licensing information */
package org.torproject.metrics.collector.bridgestrap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.torproject.descriptor.BridgestrapStats;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.metrics.collector.conf.Annotation;
import org.torproject.metrics.collector.conf.Configuration;
import org.torproject.metrics.collector.conf.ConfigurationException;
import org.torproject.metrics.collector.conf.Key;
import org.torproject.metrics.collector.cron.CollecTorMain;
import org.torproject.metrics.collector.downloader.Downloader;
import org.torproject.metrics.collector.persist.BridgestrapStatsPersistence;
import org.torproject.metrics.collector.persist.PersistenceUtils;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.SortedSet;
import java.util.TreeSet;
public class BridgestrapStatsDownloader extends CollecTorMain {
private static final Logger logger = LoggerFactory.getLogger(
BridgestrapStatsDownloader.class);
private String recentPathName;
private String outputPathName;
/** Instantiate the bridgestrap-stats module using the given configuration. */
public BridgestrapStatsDownloader(Configuration config) {
super(config);
this.mapPathDescriptors.put("recent/bridgestrap", BridgestrapStats.class);
}
@Override
public String module() {
return "BridgestrapStats";
}
@Override
protected String syncMarker() {
return "BridgestrapStats";
}
@Override
protected void startProcessing() throws ConfigurationException {
this.recentPathName = config.getPath(Key.RecentPath).toString();
logger.debug("Downloading bridgestrap stats...");
URL url = config.getUrl(Key.BridgestrapStatsUrl);
byte[] downloadedBytes;
try {
downloadedBytes = Downloader.downloadFromHttpServer(url);
} catch (IOException e) {
logger.warn("Failed downloading {}.", url, e);
return;
}
if (null == downloadedBytes) {
logger.warn("Could not download {}.", url);
return;
}
logger.debug("Finished downloading {}.", url);
Path parsedBridgestrapStatsFile = this.config.getPath(Key.StatsPath)
.resolve("processed-bridgestrap-stats");
SortedSet<Path> previouslyProcessedFiles = this.readProcessedFiles(
parsedBridgestrapStatsFile);
SortedSet<Path> processedFiles = new TreeSet<>();
DescriptorParser descriptorParser =
DescriptorSourceFactory.createDescriptorParser();
SortedSet<LocalDateTime> bridgestrapStatsEnds = new TreeSet<>();
this.outputPathName = config.getPath(Key.OutputPath).toString();
for (Descriptor descriptor : descriptorParser.parseDescriptors(
downloadedBytes, null, null)) {
if (descriptor instanceof BridgestrapStats) {
BridgestrapStats bridgestrapStats = (BridgestrapStats) descriptor;
LocalDateTime bridgestrapStatsEnd = bridgestrapStats.bridgestrapStatsEnd();
bridgestrapStatsEnds.add(bridgestrapStatsEnd);
BridgestrapStatsPersistence persistence
= new BridgestrapStatsPersistence(bridgestrapStats);
File tarballFile = new File(outputPathName + "/"
+ persistence.getStoragePath());
Path relativeFileName = Paths.get(tarballFile.getName());
processedFiles.add(relativeFileName);
if (previouslyProcessedFiles.contains(relativeFileName)) {
continue;
}
if (tarballFile.exists()) {
continue;
}
File rsyncFile = new File(this.recentPathName + "/"
+ persistence.getRecentPath());
File[] outputFiles = new File[] { tarballFile, rsyncFile };
for (File outputFile : outputFiles) {
this.writeToFile(outputFile, Annotation.BridgestrapStats.bytes(),
bridgestrapStats.getRawDescriptorBytes());
}
}
}
if (bridgestrapStatsEnds.isEmpty()) {
logger.warn("Could not parse downloaded bridgestrap stats.");
return;
} else if (bridgestrapStatsEnds.last().isBefore(LocalDateTime.now()
.minusHours(48L))) {
logger.warn("The latest bridgestrap stats are older than 48 hours: {}.",
bridgestrapStatsEnds.last());
}
this.writeProcessedFiles(parsedBridgestrapStatsFile, processedFiles);
this.cleanUpDirectories();
}
/**
* Write the given byte array(s) to the given file.
*
* <p>If the file already exists, it is overwritten. If the parent directory
* (or any of its parent directories) does not exist, it is created. If
* anything goes wrong, log a warning and return.</p>
*
* @param outputFile File to write to.
* @param bytes One or more byte arrays.
*/
private void writeToFile(File outputFile, byte[] ... bytes) {
try {
if (!outputFile.getParentFile().exists()
&& !outputFile.getParentFile().mkdirs()) {
logger.warn("Could not create parent directories of {}.", outputFile);
return;
}
OutputStream os = new FileOutputStream(outputFile);
for (byte[] b : bytes) {
os.write(b);
}
os.close();
} catch (IOException e) {
logger.warn("Could not write downloaded bridgestrap stats to {}",
outputFile.getAbsolutePath(), e);
}
}
/** Delete all files from the rsync (out) directory that have not been
* modified in the last three days (seven weeks). */
private void cleanUpDirectories() {
PersistenceUtils.cleanDirectory(
Paths.get(this.recentPathName, "bridgestrap"),
Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
PersistenceUtils.cleanDirectory(
Paths.get(this.outputPathName, "bridgestrap"),
Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
}
}
......@@ -8,6 +8,7 @@ public enum Annotation {
BandwidthFile("@type bandwidth-file 1.0\n"),
BridgedbMetrics("@type bridgedb-metrics 1.0\n"),
BridgestrapStats("@type bridgestrap-stats 1.0\n"),
BridgeExtraInfo("@type bridge-extra-info 1.3\n"),
BridgePoolAssignment("@type bridge-pool-assignment 1.0\n"),
BridgeServer("@type bridge-server-descriptor 1.2\n"),
......
......@@ -44,6 +44,7 @@ public class Configuration {
|| this.getBool(Key.BridgedescsActivated)
|| this.getBool(Key.BridgePoolAssignmentsActivated)
|| this.getBool(Key.BridgedbMetricsActivated)
|| this.getBool(Key.BridgestrapStatsActivated)
|| this.getBool(Key.ExitlistsActivated)
|| this.getBool(Key.UpdateindexActivated)
|| this.getBool(Key.OnionPerfActivated)
......
......@@ -83,7 +83,13 @@ public enum Key {
BridgedbMetricsPeriodMinutes(Integer.class),
BridgedbMetricsSources(SourceType[].class),
BridgedbMetricsLocalOrigins(Path.class),
BridgedbMetricsSyncOrigins(URL[].class);
BridgedbMetricsSyncOrigins(URL[].class),
BridgestrapStatsActivated(Boolean.class),
BridgestrapStatsOffsetMinutes(Integer.class),
BridgestrapStatsPeriodMinutes(Integer.class),
BridgestrapStatsUrl(URL.class),
BridgestrapStatsSources(SourceType[].class),
BridgestrapStatsSyncOrigins(URL[].class);
private Class clazz;
private static Set<String> keys;
......
/* Copyright 2019--2020 The Tor Project
* See LICENSE for licensing information */
package org.torproject.metrics.collector.persist;
import org.torproject.descriptor.BridgestrapStats;
import org.torproject.metrics.collector.conf.Annotation;
import java.nio.file.Paths;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
public class BridgestrapStatsPersistence
extends DescriptorPersistence<BridgestrapStats> {
private static final String BRIDGESTRAP = "bridgestrap";
public BridgestrapStatsPersistence(BridgestrapStats desc) {
super(desc, Annotation.BridgestrapStats.bytes());
calculatePaths();
}
private void calculatePaths() {
DateTimeFormatter directoriesFormatter = DateTimeFormatter
.ofPattern("uuuu/MM/dd").withZone(ZoneOffset.UTC);
String[] directories = this.desc.bridgestrapStatsEnd()
.format(directoriesFormatter).split("/");
DateTimeFormatter fileFormatter = DateTimeFormatter
.ofPattern("uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC);
String fileOut = this.desc.bridgestrapStatsEnd().format(fileFormatter)
+ "-bridgestrap-stats";
this.recentPath = Paths.get(BRIDGESTRAP, fileOut).toString();
this.storagePath = Paths.get(BRIDGESTRAP, directories[0], directories[1],
directories[2], fileOut).toString();
}
}
......@@ -62,6 +62,12 @@ BridgedbMetricsActivated = false
BridgedbMetricsPeriodMinutes = 480
# offset in minutes since the epoch and
BridgedbMetricsOffsetMinutes = 340
# the following defines, if this module is activated
BridgestrapStatsActivated = false
# period in minutes
BridgestrapStatsPeriodMinutes = 480
# offset in minutes since the epoch and
BridgestrapStatsOffsetMinutes = 100
######## General Properties ########
# The URL of this instance. This will be the base URL
......@@ -224,3 +230,14 @@ BridgedbMetricsLocalOrigins = in/bridgedb-stats
## List of URLs separated by comma.
BridgedbMetricsSyncOrigins = https://collector.torproject.org
#
######## Snowflake statistics ########
#
## Define descriptor sources
# possible values: Sync, Remote
BridgestrapStatsSources = Remote
## Retrieve files from the following instances.
## List of URLs separated by comma.
BridgestrapStatsSyncOrigins = https://collector.torproject.org
## Where to download snowflake statistics from.
BridgestrapStatsUrl = https://bridges.torproject.org/bridgestrap-collector
#
\ No newline at end of file
......@@ -67,6 +67,8 @@ TARBALLS=(
bridge-pool-assignments-$YEARTWO-$MONTHTWO
bridgedb-metrics-$YEARONE-$MONTHONE
bridgedb-metrics-$YEARTWO-$MONTHTWO
bridgestrap-$YEARONE-$MONTHONE
bridgestrap-$YEARTWO-$MONTHTWO
)
TARBALLS=($(printf "%s\n" "${TARBALLS[@]}" | uniq))
......@@ -102,6 +104,8 @@ DIRECTORIES=(
$OUTDIR/bridge-pool-assignments/$YEARTWO/$MONTHTWO/
$OUTDIR/bridgedb-metrics/$YEARONE/$MONTHONE/
$OUTDIR/bridgedb-metrics/$YEARTWO/$MONTHTWO/
$OUTDIR/bridgestrap/$YEARONE/$MONTHONE/
$OUTDIR/bridgestrap/$YEARTWO/$MONTHTWO/
)
DIRECTORIES=($(printf "%s\n" "${DIRECTORIES[@]}" | uniq))
......@@ -197,4 +201,7 @@ ln -f -s -t $ARCHIVEDIR/bridge-pool-assignments/ $TARBALLTARGETDIR/bridge-pool-a
mkdir -p $ARCHIVEDIR/bridgedb-metrics/
ln -f -s -t $ARCHIVEDIR/bridgedb-metrics/ $TARBALLTARGETDIR/bridgedb-metrics-20??-??.tar.xz
mkdir -p $ARCHIVEDIR/bridgestrap/
ln -f -s -t $ARCHIVEDIR/bridgestrap/ $TARBALLTARGETDIR/bridgestrap-20??-??.tar.xz
echo `date` "Finished."
......@@ -30,7 +30,7 @@ public class ConfigurationTest {
public void testKeyCount() {
assertEquals("The number of properties keys in enum Key changed."
+ "\n This test class should be adapted.",
70, Key.values().length);
76, Key.values().length);
}
@Test()
......
......@@ -71,6 +71,7 @@ public class CollecTorMainTest {
case "Bridge":
case "BridgePoolAssignments":
case "BridgedbMetrics":
case "BridgestrapStats":
case "Exitlist":
case "OnionPerf":
case "Webstats":
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment