Support parsing archived descriptors from in/archive/. (00494ff1) · Commits · Legacy / gitolite / user / iwakeh / Onionoo

src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java

+21 −8

Original line number	Diff line number	Diff line
		@@ -24,8 +24,6 @@ class DescriptorQueue {
		private final static Logger log = LoggerFactory.getLogger(
		DescriptorQueue.class);

		private File inDir;

		private File statusDir;

		private DescriptorReader descriptorReader;
		@@ -56,14 +54,21 @@ class DescriptorQueue {
		return this.returnedBytes;
		}

		public DescriptorQueue(File inDir, File statusDir) {
		this.inDir = inDir;
		public DescriptorQueue(File inDir, DescriptorType descriptorType,
		File statusDir) {
		File directory = inDir;
		if (descriptorType != null) {
		directory = this.getDirectoryForDescriptorType(inDir,
		descriptorType);
		}
		this.statusDir = statusDir;
		this.descriptorReader =
		DescriptorSourceFactory.createDescriptorReader();
		this.addDirectory(directory);
		}

		public void addDirectory(DescriptorType descriptorType) {
		public File getDirectoryForDescriptorType(File inDir,
		DescriptorType descriptorType) {
		String directoryName = null;
		switch (descriptorType) {
		case RELAY_CONSENSUSES:
		@@ -93,9 +98,16 @@ class DescriptorQueue {
		default:
		log.error("Unknown descriptor type. Not adding directory "
		+ "to descriptor reader.");
		return null;
		}
		File directory = new File(inDir, directoryName);
		return directory;
		}

		private void addDirectory(File directory) {
		if (directory == null) {
		return;
		}
		File directory = new File(this.inDir, directoryName);
		if (directory.exists() && directory.isDirectory()) {
		this.descriptorReader.addDirectory(directory);
		this.descriptorReader.setMaxDescriptorFilesInQueue(1);
		@@ -107,6 +119,9 @@ class DescriptorQueue {
		}

		public void readHistoryFile(DescriptorHistory descriptorHistory) {
		if (this.statusDir == null) {
		return;
		}
		String historyFileName = null;
		switch (descriptorHistory) {
		case RELAY_EXTRAINFO_HISTORY:
		@@ -168,8 +183,6 @@ class DescriptorQueue {

		public void writeHistoryFile() {
		if (this.historyFile == null) {
		log.error("Unable to write history file, possibly because of an "
		+ "unknown descriptor type. Skipping.");
		return;
		}
		SortedMap<String, Long> excludedAndParsedFiles =

src/main/java/org/torproject/onionoo/updater/DescriptorSource.java

+99 −38

Original line number	Diff line number	Diff line
		@@ -20,12 +20,15 @@ public class DescriptorSource {
		private static final Logger log = LoggerFactory.getLogger(
		DescriptorSource.class);

		private final File inDir = new File("in/recent");
		private final File inRecentDir = new File("in/recent"),
		inArchiveDir = new File("in/archive");

		private final File statusDir = new File("status");

		private List<DescriptorQueue> descriptorQueues;

		private DescriptorQueue archiveDescriptorQueue;

		public DescriptorSource() {
		this.descriptorQueues = new ArrayList<DescriptorQueue>();
		this.descriptorListeners =
		@@ -35,9 +38,8 @@ public class DescriptorSource {
		private DescriptorQueue getDescriptorQueue(
		DescriptorType descriptorType,
		DescriptorHistory descriptorHistory) {
		DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir,
		this.statusDir);
		descriptorQueue.addDirectory(descriptorType);
		DescriptorQueue descriptorQueue = new DescriptorQueue(
		this.inRecentDir, descriptorType, this.statusDir);
		if (descriptorHistory != null) {
		descriptorQueue.readHistoryFile(descriptorHistory);
		}
		@@ -78,35 +80,32 @@ public class DescriptorSource {
		}

		public void readDescriptors() {
		/* Careful when changing the order of parsing descriptor types! The
		* various status updaters may base assumptions on this order. With
		* #13674 being implemented, this may not be the case anymore. */

		log.debug("Reading " + DescriptorType.RELAY_SERVER_DESCRIPTORS
		this.readArchivedDescriptors();
		log.debug("Reading recent " + DescriptorType.RELAY_SERVER_DESCRIPTORS
		+ " ...");
		this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS,
		DescriptorHistory.RELAY_SERVER_HISTORY, true);
		log.debug("Reading " + DescriptorType.RELAY_EXTRA_INFOS + " ...");
		log.debug("Reading recent " + DescriptorType.RELAY_EXTRA_INFOS + " ...");
		this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS,
		DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true);
		log.debug("Reading " + DescriptorType.EXIT_LISTS + " ...");
		log.debug("Reading recent " + DescriptorType.EXIT_LISTS + " ...");
		this.readDescriptors(DescriptorType.EXIT_LISTS,
		DescriptorHistory.EXIT_LIST_HISTORY, true);
		log.debug("Reading " + DescriptorType.RELAY_CONSENSUSES + " ...");
		log.debug("Reading recent " + DescriptorType.RELAY_CONSENSUSES + " ...");
		this.readDescriptors(DescriptorType.RELAY_CONSENSUSES,
		DescriptorHistory.RELAY_CONSENSUS_HISTORY, true);
		log.debug("Reading " + DescriptorType.BRIDGE_SERVER_DESCRIPTORS
		log.debug("Reading recent " + DescriptorType.BRIDGE_SERVER_DESCRIPTORS
		+ " ...");
		this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS,
		DescriptorHistory.BRIDGE_SERVER_HISTORY, false);
		log.debug("Reading " + DescriptorType.BRIDGE_EXTRA_INFOS + " ...");
		log.debug("Reading recent " + DescriptorType.BRIDGE_EXTRA_INFOS + " ...");
		this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS,
		DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false);
		log.debug("Reading " + DescriptorType.BRIDGE_POOL_ASSIGNMENTS
		log.debug("Reading recent " + DescriptorType.BRIDGE_POOL_ASSIGNMENTS
		+ " ...");
		this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS,
		DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false);
		log.debug("Reading " + DescriptorType.BRIDGE_STATUSES + " ...");
		log.debug("Reading recent " + DescriptorType.BRIDGE_STATUSES + " ...");
		this.readDescriptors(DescriptorType.BRIDGE_STATUSES,
		DescriptorHistory.BRIDGE_STATUS_HISTORY, false);
		}
		@@ -128,34 +127,88 @@ public class DescriptorSource {
		}
		switch (descriptorType) {
		case RELAY_CONSENSUSES:
		log.info("Read relay network consensuses");
		log.info("Read recent relay network consensuses");
		break;
		case RELAY_SERVER_DESCRIPTORS:
		log.info("Read relay server descriptors");
		log.info("Read recent relay server descriptors");
		break;
		case RELAY_EXTRA_INFOS:
		log.info("Read relay extra-info descriptors");
		log.info("Read recent relay extra-info descriptors");
		break;
		case EXIT_LISTS:
		log.info("Read exit lists");
		log.info("Read recent exit lists");
		break;
		case BRIDGE_STATUSES:
		log.info("Read bridge network statuses");
		log.info("Read recent bridge network statuses");
		break;
		case BRIDGE_SERVER_DESCRIPTORS:
		log.info("Read bridge server descriptors");
		log.info("Read recent bridge server descriptors");
		break;
		case BRIDGE_EXTRA_INFOS:
		log.info("Read bridge extra-info descriptors");
		log.info("Read recent bridge extra-info descriptors");
		break;
		case BRIDGE_POOL_ASSIGNMENTS:
		log.info("Read bridge-pool assignments");
		log.info("Read recent bridge-pool assignments");
		break;
		}
		}

		public void readArchivedDescriptors() {
		if (!this.inArchiveDir.exists()) {
		return;
		}
		log.info("Reading archived descriptors...");
		this.archiveDescriptorQueue = new DescriptorQueue(this.inArchiveDir,
		null, null);
		Descriptor descriptor;
		while ((descriptor = this.archiveDescriptorQueue.nextDescriptor())
		!= null) {
		DescriptorType descriptorType = null;
		boolean relay = false;
		for (String annotation : descriptor.getAnnotations()) {
		if (annotation.startsWith(
		"@type network-status-consensus-3 1.")) {
		descriptorType = DescriptorType.RELAY_CONSENSUSES;
		relay = true;
		} else if (annotation.startsWith("@type server-descriptor 1.")) {
		descriptorType = DescriptorType.RELAY_SERVER_DESCRIPTORS;
		relay = true;
		} else if (annotation.startsWith("@type extra-info 1.")) {
		descriptorType = DescriptorType.RELAY_EXTRA_INFOS;
		relay = true;
		} else if (annotation.startsWith("@type tordnsel 1.")) {
		descriptorType = DescriptorType.EXIT_LISTS;
		relay = true;
		} else if (annotation.startsWith(
		"@type bridge-network-status 1.")) {
		descriptorType = DescriptorType.BRIDGE_STATUSES;
		relay = false;
		} else if (annotation.startsWith(
		"@type bridge-server-descriptor 1.")) {
		descriptorType = DescriptorType.BRIDGE_SERVER_DESCRIPTORS;
		relay = false;
		} else if (annotation.startsWith("@type bridge-extra-info 1.")) {
		descriptorType = DescriptorType.BRIDGE_EXTRA_INFOS;
		relay = false;
		}
		}
		if (descriptorType == null) {
		log.warn("Unrecognized descriptor in "
		+ this.inArchiveDir.getAbsolutePath() + " with annotations "
		+ descriptor.getAnnotations() + ". Not reading any further"
		+ "archived descriptors.");
		break;
		}
		for (DescriptorListener descriptorListener :
		this.descriptorListeners.get(descriptorType)) {
		descriptorListener.processDescriptor(descriptor, relay);
		}
		}
		log.info("Read archived descriptors");
		}

		public void writeHistoryFiles() {
		log.debug("Writing history ");
		log.debug("Writing parse histories for recent descriptors...");
		for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
		descriptorQueue.writeHistoryFile();
		}
		@@ -163,34 +216,42 @@ public class DescriptorSource {

		public String getStatsString() {
		StringBuilder sb = new StringBuilder();
		sb.append(" " + this.localFilesBefore + " descriptor files found "
		+ "locally\n");
		sb.append(" " + this.foundRemoteFiles + " descriptor files found "
		+ "remotely\n");
		sb.append(" " + this.downloadedFiles + " descriptor files "
		sb.append(" " + this.localFilesBefore + " recent descriptor files "
		+ "found locally\n");
		sb.append(" " + this.foundRemoteFiles + " recent descriptor files "
		+ "found remotely\n");
		sb.append(" " + this.downloadedFiles + " recent descriptor files "
		+ "downloaded from remote\n");
		sb.append(" " + this.deletedLocalFiles + " descriptor files "
		+ "deleted locally\n");
		sb.append(" " + this.deletedLocalFiles + " recent descriptor "
		+ "files deleted locally\n");
		sb.append(" " + this.descriptorQueues.size() + " descriptor "
		+ "queues created\n");
		+ "queues created for recent descriptors\n");
		int historySizeBefore = 0, historySizeAfter = 0;
		long descriptors = 0L, bytes = 0L;
		for (DescriptorQueue descriptorQueue : descriptorQueues) {
		for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
		historySizeBefore += descriptorQueue.getHistorySizeBefore();
		historySizeAfter += descriptorQueue.getHistorySizeAfter();
		descriptors += descriptorQueue.getReturnedDescriptors();
		bytes += descriptorQueue.getReturnedBytes();
		}
		sb.append(" " + FormattingUtils.formatDecimalNumber(
		historySizeBefore) + " descriptors excluded from this "
		historySizeBefore) + " recent descriptors excluded from this "
		+ "execution\n");
		sb.append(" " + FormattingUtils.formatDecimalNumber(descriptors)
		+ " descriptors provided\n");
		+ " recent descriptors provided\n");
		sb.append(" " + FormattingUtils.formatBytes(bytes)
		+ " provided\n");
		+ " of recent descriptors provided\n");
		sb.append(" " + FormattingUtils.formatDecimalNumber(
		historySizeAfter) + " descriptors excluded from next "
		historySizeAfter) + " recent descriptors excluded from next "
		+ "execution\n");
		if (this.archiveDescriptorQueue != null) {
		sb.append(" " + FormattingUtils.formatDecimalNumber(
		this.archiveDescriptorQueue.getReturnedDescriptors()) +
		" archived descriptors provided\n");
		sb.append(" " + FormattingUtils.formatBytes(
		this.archiveDescriptorQueue.getReturnedBytes()) + " of "
		+ "archived descriptors provided\n");
		}
		return sb.toString();
		}
		}