Use persist package for writing bridge descriptors. (7439fa93) · Commits · Legacy / gitolite / user / iwakeh / Collector

src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java

+14 −90

Original line number	Diff line number	Diff line
		@@ -10,6 +10,9 @@ import org.torproject.metrics.collector.conf.Configuration;
		import org.torproject.metrics.collector.conf.ConfigurationException;
		import org.torproject.metrics.collector.conf.Key;
		import org.torproject.metrics.collector.cron.CollecTorMain;
		import org.torproject.metrics.collector.persist.BridgeExtraInfoDescriptorPersistence;
		import org.torproject.metrics.collector.persist.BridgeNetworkStatusPersistence;
		import org.torproject.metrics.collector.persist.BridgeServerDescriptorPersistence;
		import org.torproject.metrics.collector.persist.PersistenceUtils;

		import org.apache.commons.codec.binary.Hex;
		@@ -28,8 +31,6 @@ import java.io.StringReader;
		import java.nio.charset.StandardCharsets;
		import java.nio.file.Files;
		import java.nio.file.Path;
		import java.nio.file.Paths;
		import java.nio.file.StandardOpenOption;
		import java.time.Instant;
		import java.time.LocalDateTime;
		import java.time.format.DateTimeFormatter;
		@@ -57,7 +58,6 @@ public class SanitizedBridgesWriter extends CollecTorMain {

		private static final Logger logger = LoggerFactory.getLogger(
		SanitizedBridgesWriter.class);
		private static final String BRIDGE_DESCRIPTORS = "bridge-descriptors";

		/** Initialize configuration. */
		public SanitizedBridgesWriter(Configuration config) {
		@@ -91,10 +91,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		@Override
		protected void startProcessing() throws ConfigurationException {

		this.outputDirectory = config.getPath(Key.OutputPath)
		.resolve(BRIDGE_DESCRIPTORS);
		this.recentDirectory = config.getPath(Key.RecentPath)
		.resolve(BRIDGE_DESCRIPTORS);
		this.outputDirectory = config.getPath(Key.OutputPath);
		this.recentDirectory = config.getPath(Key.RecentPath);
		Path inputDirectory = config.getPath(Key.BridgeLocalOrigins);
		Path statsDirectory = config.getPath(Key.StatsPath);
		boolean replaceIpAddressesWithHashes =
		@@ -352,27 +350,9 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		\|\| publicationTime.compareTo(maxNetworkStatusPublishedTime) > 0) {
		maxNetworkStatusPublishedTime = publicationTime;
		}
		try {
		String syear = publicationTime.substring(0, 4);
		String smonth = publicationTime.substring(5, 7);
		String sday = publicationTime.substring(8, 10);
		String stime = publicationTime.substring(11, 13)
		+ publicationTime.substring(14, 16)
		+ publicationTime.substring(17, 19);
		String fileName = syear + smonth + sday + "-" + stime + "-"
		+ authorityFingerprint;
		Path tarballFile = this.outputDirectory.resolve(
		Paths.get(syear, smonth, "statuses", sday, fileName));
		Path rsyncFile = this.recentDirectory.resolve(
		Paths.get("statuses", fileName));
		for (Path outputFile : new Path[] { tarballFile, rsyncFile }) {
		Files.createDirectories(outputFile.getParent());
		Files.write(outputFile, scrubbedBytes);
		}
		} catch (IOException e) {
		logger.warn("Could not write sanitized bridge "
		+ "network status to disk.", e);
		}
		new BridgeNetworkStatusPersistence(scrubbedBytes, publicationTime,
		authorityFingerprint)
		.storeAll(this.recentDirectory, this.outputDirectory);
		}

		private String maxServerDescriptorPublishedTime = null;
		@@ -398,36 +378,9 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		}
		String descriptorDigest
		= sanitizedBridgeServerDescriptor.getDescriptorDigest();

		/* Determine filename of sanitized server descriptor. */
		String dyear = published.substring(0, 4);
		String dmonth = published.substring(5, 7);
		try {
		Path tarballFile = this.outputDirectory.resolve(
		Paths.get(dyear, dmonth, "server-descriptors",
		descriptorDigest.substring(0, 1), descriptorDigest.substring(1, 2),
		descriptorDigest));
		Path rsyncCatFile = this.recentDirectory.resolve(
		Paths.get("bridge-descriptors", "server-descriptors",
		this.rsyncCatString + "-server-descriptors.tmp"));
		Path[] outputFiles = new Path[] { tarballFile, rsyncCatFile };
		boolean[] append = new boolean[] { false, true };
		for (int i = 0; i < outputFiles.length; i++) {
		Path outputFile = outputFiles[i];
		StandardOpenOption openOption = append[i] ? StandardOpenOption.APPEND
		: StandardOpenOption.CREATE_NEW;
		if (Files.exists(outputFile)
		&& openOption != StandardOpenOption.APPEND) {
		/* We already stored this descriptor to disk before, so let's
		* not store it yet another time. */
		break;
		}
		Files.createDirectories(outputFile.getParent());
		Files.write(outputFile, scrubbedBytes, openOption);
		}
		} catch (IOException e) {
		logger.warn("Could not write sanitized server descriptor to disk.", e);
		}
		new BridgeServerDescriptorPersistence(scrubbedBytes, published,
		this.rsyncCatString, descriptorDigest)
		.storeAll(this.recentDirectory, this.outputDirectory);
		}

		private String maxExtraInfoDescriptorPublishedTime = null;
		@@ -453,38 +406,9 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		}
		String descriptorDigest
		= sanitizedBridgeExtraInfoDescriptor.getDescriptorDigest();

		/* Determine filename of sanitized extra-info descriptor. */
		String dyear = published.substring(0, 4);
		String dmonth = published.substring(5, 7);

		try {
		Path tarballFile = this.outputDirectory.resolve(
		Paths.get(dyear, dmonth, "extra-infos",
		descriptorDigest.substring(0, 1), descriptorDigest.substring(1, 2),
		descriptorDigest));
		Path rsyncCatFile = this.recentDirectory.resolve(
		Paths.get("bridge-descriptors", "extra-infos",
		this.rsyncCatString + "-extra-infos.tmp"));
		Path[] outputFiles = new Path[] { tarballFile, rsyncCatFile };
		boolean[] append = new boolean[] { false, true };
		for (int i = 0; i < outputFiles.length; i++) {
		Path outputFile = outputFiles[i];
		StandardOpenOption openOption = append[i] ? StandardOpenOption.APPEND
		: StandardOpenOption.CREATE_NEW;
		if (Files.exists(outputFile)
		&& openOption != StandardOpenOption.APPEND) {
		/* We already stored this descriptor to disk before, so let's
		* not store it yet another time. */
		break;
		}
		Files.createDirectories(outputFile.getParent());
		Files.write(outputFile, scrubbedBytes, openOption);
		}
		} catch (IOException e) {
		logger.warn("Could not write sanitized extra-info descriptor to disk.",
		e);
		}
		new BridgeExtraInfoDescriptorPersistence(scrubbedBytes, published,
		this.rsyncCatString, descriptorDigest)
		.storeAll(this.recentDirectory, this.outputDirectory);
		}

		private void checkStaleDescriptors() {

src/main/java/org/torproject/metrics/collector/persist/BridgeExtraInfoDescriptorPersistence.java

0 → 100644

+54 −0

Original line number	Diff line number	Diff line
		/* Copyright 2016--2020 The Tor Project
		* See LICENSE for licensing information */

		package org.torproject.metrics.collector.persist;

		import org.torproject.descriptor.BridgeExtraInfoDescriptor;
		import org.torproject.metrics.collector.conf.Annotation;

		import java.nio.file.Paths;

		public class BridgeExtraInfoDescriptorPersistence
		extends DescriptorPersistence<BridgeExtraInfoDescriptor> {

		/**
		* Construct a persistence instance from a previously parsed descriptor.
		*/
		public BridgeExtraInfoDescriptorPersistence(
		BridgeExtraInfoDescriptor descriptor, long received) {
		super(descriptor, Annotation.BridgeExtraInfo.bytes());
		this.calculatePaths(
		PersistenceUtils.dateTimeParts(descriptor.getPublishedMillis()),
		PersistenceUtils.dateTime(received),
		descriptor.getDigestSha1Hex().toLowerCase());
		}

		/**
		* Construct a persistence instance from raw descriptor bytes.
		*/
		public BridgeExtraInfoDescriptorPersistence(byte[] descriptorBytes,
		String publishedString, String receivedString, String descriptorDigest) {
		super(descriptorBytes);
		this.calculatePaths(
		publishedString.split("[ :-]"),
		receivedString,
		descriptorDigest.toLowerCase());
		}

		private void calculatePaths(String[] publishedParts, String receivedString,
		String descriptorDigest) {
		this.recentPath = Paths.get(
		BRIDGEDESCS,
		EXTRA_INFOS,
		receivedString + DASH + EXTRA_INFOS).toString();
		this.storagePath = Paths.get(
		BRIDGEDESCS,
		publishedParts[0], // year
		publishedParts[1], // month
		EXTRA_INFOS,
		descriptorDigest.substring(0, 1),
		descriptorDigest.substring(1, 2),
		descriptorDigest).toString();
		}
		}

src/main/java/org/torproject/metrics/collector/persist/BridgeExtraInfoPersistence.java

deleted100644 → 0

+0 −39

Original line number	Diff line number	Diff line
		/* Copyright 2016--2020 The Tor Project
		* See LICENSE for licensing information */

		package org.torproject.metrics.collector.persist;

		import org.torproject.descriptor.BridgeExtraInfoDescriptor;
		import org.torproject.metrics.collector.conf.Annotation;

		import java.nio.file.Paths;

		public class BridgeExtraInfoPersistence
		extends DescriptorPersistence<BridgeExtraInfoDescriptor> {

		public BridgeExtraInfoPersistence(BridgeExtraInfoDescriptor desc,
		long received) {
		super(desc, Annotation.BridgeExtraInfo.bytes());
		calculatePaths(received);
		}

		private void calculatePaths(long received) {
		String file = PersistenceUtils.dateTime(received);
		String[] parts = PersistenceUtils.dateTimeParts(desc.getPublishedMillis());
		this.recentPath = Paths.get(
		BRIDGEDESCS,
		EXTRA_INFOS,
		file + DASH + EXTRA_INFOS).toString();
		String digest = desc.getDigestSha1Hex().toLowerCase();
		this.storagePath = Paths.get(
		BRIDGEDESCS,
		parts[0], // year
		parts[1], // month
		EXTRA_INFOS,
		digest.substring(0,1),
		digest.substring(1,2),
		digest).toString();
		}

		}

src/main/java/org/torproject/metrics/collector/persist/StatusPersistence.java→src/main/java/org/torproject/metrics/collector/persist/BridgeNetworkStatusPersistence.java

+56 −0

Original line number	Diff line number	Diff line
		@@ -8,34 +8,49 @@ import org.torproject.metrics.collector.conf.Annotation;

		import java.nio.file.Paths;

		public class StatusPersistence
		public class BridgeNetworkStatusPersistence
		extends DescriptorPersistence<BridgeNetworkStatus> {

		private static final String STATUSES = "statuses";

		public StatusPersistence(BridgeNetworkStatus desc,
		String authId, long received) {
		super(desc, Annotation.Status.bytes());
		calculatePaths(authId, received);
		/**
		* Construct a persistence instance from a previously parsed descriptor.
		*/
		public BridgeNetworkStatusPersistence(BridgeNetworkStatus descriptor,
		String authorityFingerprint) {
		super(descriptor, Annotation.Status.bytes());
		this.calculatePaths(
		PersistenceUtils.dateTimeParts(descriptor.getPublishedMillis()),
		authorityFingerprint);
		}

		private void calculatePaths(String authId, long received) {
		String[] partsOut = PersistenceUtils.dateTimeParts(
		desc.getPublishedMillis());
		String fileOut = partsOut[0] + partsOut[1] + partsOut[2] + DASH
		+ partsOut[3] + partsOut[4] + partsOut[5] + DASH + authId;
		/**
		* Construct a persistence instance from raw descriptor bytes.
		*/
		public BridgeNetworkStatusPersistence(byte[] descriptorBytes,
		String published, String authorityFingerprint) {
		super(descriptorBytes);
		this.calculatePaths(
		published.split("[ :-]"),
		authorityFingerprint);
		}

		private void calculatePaths(String[] publishedParts,
		String authorityFingerprint) {
		String fileOut = publishedParts[0] + publishedParts[1] + publishedParts[2]
		+ DASH + publishedParts[3] + publishedParts[4] + publishedParts[5]
		+ DASH + authorityFingerprint;
		this.recentPath = Paths.get(
		BRIDGEDESCS,
		STATUSES,
		fileOut).toString();
		this.storagePath = Paths.get(
		BRIDGEDESCS,
		partsOut[0], // year
		partsOut[1], // month
		publishedParts[0], // year
		publishedParts[1], // month
		STATUSES,
		partsOut[2], // day
		publishedParts[2], // day
		fileOut).toString();
		}

		}

src/main/java/org/torproject/metrics/collector/persist/BridgeServerDescriptorPersistence.java

+29 −14

Original line number	Diff line number	Diff line
		@@ -11,29 +11,44 @@ import java.nio.file.Paths;
		public class BridgeServerDescriptorPersistence
		extends DescriptorPersistence<BridgeServerDescriptor> {

		public BridgeServerDescriptorPersistence(BridgeServerDescriptor desc,
		/**
		* Construct a persistence instance from a previously parsed descriptor.
		*/
		public BridgeServerDescriptorPersistence(BridgeServerDescriptor descriptor,
		long received) {
		super(desc, Annotation.BridgeServer.bytes());
		calculatePaths(received);
		super(descriptor, Annotation.BridgeServer.bytes());
		this.calculatePaths(
		PersistenceUtils.dateTimeParts(descriptor.getPublishedMillis()),
		PersistenceUtils.dateTime(received),
		descriptor.getDigestSha1Hex().toLowerCase());
		}

		private void calculatePaths(long received) {
		String file = PersistenceUtils.dateTime(received);
		String[] parts = PersistenceUtils.dateTimeParts(desc.getPublishedMillis());
		/**
		* Construct a persistence instance from raw descriptor bytes.
		*/
		public BridgeServerDescriptorPersistence(byte[] descriptorBytes,
		String publishedString, String receivedString, String descriptorDigest) {
		super(descriptorBytes);
		this.calculatePaths(
		publishedString.split("[ :-]"),
		receivedString,
		descriptorDigest.toLowerCase());
		}

		private void calculatePaths(String[] publishedParts, String receivedString,
		String descriptorDigest) {
		this.recentPath = Paths.get(
		BRIDGEDESCS,
		SERVERDESCS,
		file + DASH + SERVERDESCS).toString();
		String digest = desc.getDigestSha1Hex().toLowerCase();
		receivedString + DASH + SERVERDESCS).toString();
		this.storagePath = Paths.get(
		BRIDGEDESCS,
		parts[0], // year
		parts[1], // month
		publishedParts[0], // year
		publishedParts[1], // month
		SERVERDESCS,
		digest.substring(0,1),
		digest.substring(1,2),
		digest).toString();
		descriptorDigest.substring(0, 1),
		descriptorDigest.substring(1, 2),
		descriptorDigest).toString();
		}

		}