Delete files in out/ that are older than 7 weeks. (66ddc4d7) · Commits · Legacy / gitolite / user / iwakeh / Collector

CHANGELOG.md

+7 −0

Original line number	Diff line number	Diff line
		# Changes in version 1.??.? - 2020-1?-??

		* Medium changes
		- Clean up descriptors written to the `out/` directory by deleting
		files that are older than seven weeks.


		# Changes in version 1.16.1 - 2020-08-16

		* Medium changes

src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java

+11 −27

Original line number	Diff line number	Diff line
		@@ -12,6 +12,7 @@ import org.torproject.metrics.collector.conf.ConfigurationException;
		import org.torproject.metrics.collector.conf.Key;
		import org.torproject.metrics.collector.cron.CollecTorMain;
		import org.torproject.metrics.collector.persist.BridgedbMetricsPersistence;
		import org.torproject.metrics.collector.persist.PersistenceUtils;

		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;
		@@ -23,9 +24,7 @@ import java.nio.file.Path;
		import java.nio.file.Paths;
		import java.time.Instant;
		import java.time.temporal.ChronoUnit;
		import java.util.Arrays;
		import java.util.SortedSet;
		import java.util.Stack;
		import java.util.TreeSet;

		public class BridgedbMetricsProcessor extends CollecTorMain {
		@@ -127,10 +126,10 @@ public class BridgedbMetricsProcessor extends CollecTorMain {
		descriptor.getClass(), descriptor.getDescriptorFile());
		}
		}
		logger.info("Cleaning up directory {} containing recent files.",
		this.recentPathName);
		logger.info("Cleaning up directories {} and {}.",
		this.recentPathName, this.outputPathName);
		this.writeProcessedFiles(this.parsedBridgedbMetricsFile, processedFiles);
		this.cleanUpRsyncDirectory();
		this.cleanUpDirectories();
		logger.info("Finished processing BridgeDB statistics file(s).");
		}

		@@ -175,28 +174,13 @@ public class BridgedbMetricsProcessor extends CollecTorMain {
		}

		/**
		* Delete all files from the rsync directory that have not been modified in
		* the last three days.
		* Delete all files from the rsync (out) directory that have not been modified
		* in the last three days (seven weeks).
		*/
		public void cleanUpRsyncDirectory() {
		Instant cutOff = Instant.now().minus(3L, ChronoUnit.DAYS);
		Stack<File> allFiles = new Stack<>();
		allFiles.add(new File(this.recentPathName));
		while (!allFiles.isEmpty()) {
		File file = allFiles.pop();
		if (file.isDirectory()) {
		File[] filesInDirectory = file.listFiles();
		if (null != filesInDirectory) {
		allFiles.addAll(Arrays.asList(filesInDirectory));
		}
		} else if (Instant.ofEpochMilli(file.lastModified()).isBefore(cutOff)) {
		try {
		Files.deleteIfExists(file.toPath());
		} catch (IOException e) {
		logger.warn("Unable to delete file {} that is apparently older than "
		+ "three days.", file, e);
		}
		}
		}
		private void cleanUpDirectories() {
		PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
		Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
		PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
		Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
		}
		}

src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java

+13 −24

Original line number	Diff line number	Diff line
		@@ -13,6 +13,7 @@ import org.torproject.metrics.collector.conf.Configuration;
		import org.torproject.metrics.collector.conf.ConfigurationException;
		import org.torproject.metrics.collector.conf.Key;
		import org.torproject.metrics.collector.cron.CollecTorMain;
		import org.torproject.metrics.collector.persist.PersistenceUtils;

		import org.apache.commons.codec.DecoderException;
		import org.apache.commons.codec.binary.Base64;
		@@ -34,15 +35,15 @@ import java.security.GeneralSecurityException;
		import java.security.SecureRandom;
		import java.text.ParseException;
		import java.text.SimpleDateFormat;
		import java.time.Instant;
		import java.time.LocalDateTime;
		import java.time.format.DateTimeFormatter;
		import java.time.temporal.ChronoUnit;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.HashMap;
		import java.util.List;
		import java.util.Map;
		import java.util.SortedMap;
		import java.util.Stack;
		import java.util.TreeMap;

		/**
		@@ -228,7 +229,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {

		this.checkStaleDescriptors();

		this.cleanUpRsyncDirectory();
		this.cleanUpDirectories();
		}

		private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
		@@ -1388,27 +1389,15 @@ public class SanitizedBridgesWriter extends CollecTorMain {
		}
		}

		/** Delete all files from the rsync directory that have not been modified
		* in the last three days, and remove the .tmp extension from newly
		* written files. */
		public void cleanUpRsyncDirectory() throws ConfigurationException {
		long cutOffMillis = System.currentTimeMillis()
		- 3L * 24L * 60L * 60L * 1000L;
		Stack<File> allFiles = new Stack<>();
		allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
		BRIDGE_DESCRIPTORS));
		while (!allFiles.isEmpty()) {
		File file = allFiles.pop();
		if (file.isDirectory()) {
		allFiles.addAll(Arrays.asList(file.listFiles()));
		} else if (file.lastModified() < cutOffMillis) {
		file.delete();
		} else if (file.getName().endsWith(".tmp")) {
		file.renameTo(new File(file.getParentFile(),
		file.getName().substring(0,
		file.getName().lastIndexOf(".tmp"))));
		}
		}
		/**
		* Delete all files from the rsync (out) directory that have not been modified
		* in the last three days (seven weeks), and remove the .tmp extension from
		* newly written files. */
		private void cleanUpDirectories() {
		PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
		Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
		PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
		Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
		}
		}

src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java

+9 −24

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ import org.torproject.metrics.collector.conf.Configuration;
		import org.torproject.metrics.collector.conf.ConfigurationException;
		import org.torproject.metrics.collector.conf.Key;
		import org.torproject.metrics.collector.cron.CollecTorMain;
		import org.torproject.metrics.collector.persist.PersistenceUtils;

		import org.apache.commons.codec.DecoderException;
		import org.apache.commons.codec.binary.Hex;
		@@ -24,7 +25,6 @@ import java.io.FileReader;
		import java.io.FileWriter;
		import java.io.IOException;
		import java.io.InputStreamReader;
		import java.nio.file.Files;
		import java.nio.file.Path;
		import java.nio.file.Paths;
		import java.time.DateTimeException;
		@@ -178,7 +178,7 @@ public class BridgePoolAssignmentsProcessor extends CollecTorMain {
		}
		this.writeProcessedFiles(this.parsedBridgePoolAssignmentsFile,
		processedFiles);
		this.cleanUpRsyncDirectory();
		this.cleanUpDirectories();
		logger.info("Finished processing bridge pool assignment file(s).");
		}

		@@ -363,29 +363,14 @@ public class BridgePoolAssignmentsProcessor extends CollecTorMain {
		}

		/**
		* Delete all files from the rsync directory that have not been modified in
		* the last three days.
		* Delete all files from the rsync (out) directory that have not been modified
		* in the last three days (seven weeks).
		*/
		public void cleanUpRsyncDirectory() {
		Instant cutOff = Instant.now().minus(3L, ChronoUnit.DAYS);
		Stack<File> allFiles = new Stack<>();
		allFiles.add(new File(this.recentPathName));
		while (!allFiles.isEmpty()) {
		File file = allFiles.pop();
		if (file.isDirectory()) {
		File[] filesInDirectory = file.listFiles();
		if (null != filesInDirectory) {
		allFiles.addAll(Arrays.asList(filesInDirectory));
		}
		} else if (Instant.ofEpochMilli(file.lastModified()).isBefore(cutOff)) {
		try {
		Files.deleteIfExists(file.toPath());
		} catch (IOException e) {
		logger.warn("Unable to delete file {} that is apparently older than "
		+ "three days.", file, e);
		}
		}
		}
		public void cleanUpDirectories() {
		PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
		Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
		PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
		Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
		}
		}

src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java

+11 −16

Original line number	Diff line number	Diff line
		@@ -13,6 +13,7 @@ import org.torproject.metrics.collector.conf.ConfigurationException;
		import org.torproject.metrics.collector.conf.Key;
		import org.torproject.metrics.collector.cron.CollecTorMain;
		import org.torproject.metrics.collector.downloader.Downloader;
		import org.torproject.metrics.collector.persist.PersistenceUtils;

		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;
		@@ -24,6 +25,8 @@ import java.io.IOException;
		import java.net.URL;
		import java.nio.file.Paths;
		import java.text.SimpleDateFormat;
		import java.time.Instant;
		import java.time.temporal.ChronoUnit;
		import java.util.Arrays;
		import java.util.Date;
		import java.util.SortedSet;
		@@ -168,24 +171,16 @@ public class ExitListDownloader extends CollecTorMain {
		}
		logger.info(dumpStats.toString());

		this.cleanUpRsyncDirectory();
		this.cleanUpDirectories();
		}

		/** Delete all files from the rsync directory that have not been modified
		* in the last three days. */
		public void cleanUpRsyncDirectory() {
		long cutOffMillis = System.currentTimeMillis()
		- 3L * 24L * 60L * 60L * 1000L;
		Stack<File> allFiles = new Stack<>();
		allFiles.add(new File(recentPathName));
		while (!allFiles.isEmpty()) {
		File file = allFiles.pop();
		if (file.isDirectory()) {
		allFiles.addAll(Arrays.asList(file.listFiles()));
		} else if (file.lastModified() < cutOffMillis) {
		file.delete();
		}
		}
		/** Delete all files from the rsync (out) directory that have not been
		* modified in the last three days (seven weeks). */
		private void cleanUpDirectories() {
		PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
		Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
		PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
		Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
		}
		}