Add four new execution modes. (5cedec57) · Commits · Legacy / gitolite / user / iwakeh / Onionoo

src/main/java/org/torproject/onionoo/cron/Main.java

+200 −76

Original line number	Diff line number	Diff line
		/* Copyright 2011, 2012 The Tor Project
		/* Copyright 2011--2015 The Tor Project
		* See LICENSE for licensing information */
		package org.torproject.onionoo.cron;

		@@ -18,102 +18,226 @@ import org.torproject.onionoo.util.LockFile;
		import org.torproject.onionoo.writer.DocumentWriterRunner;

		/* Update search data and status data files. */
		public class Main {

		private static Logger log = LoggerFactory.getLogger(Main.class);
		public class Main implements Runnable {

		private Main() {
		}

		private static final ScheduledExecutorService scheduler =
		Executors.newScheduledThreadPool(1);
		private Logger log = LoggerFactory.getLogger(Main.class);

		public static void main(String[] args) {
		boolean runOnce = "true".equals(System.getProperty(
		"onionoo.cron.runonce", "true"));
		if (runOnce){
		log.info("Going to run one-time updater ... ");
		LockFile lf = new LockFile();
		log.info("Initializing.");
		if (lf.acquireLock()) {
		log.info("Acquired lock");
		} else {
		log.error("Could not acquire lock. Is Onionoo already running? "
		+ "Terminating");
		return;
		Main main = new Main();
		main.parseArgsOrExit(args);
		main.runOrScheduleExecutions();
		}
		new Updater().run();
		log.info("Releasing lock.");
		if (lf.releaseLock()) {
		log.info("Released lock");
		} else {
		log.error("Could not release lock. The next execution may not "
		+ "start as expected");

		boolean defaultMode = false, singleRun = false, downloadOnly = false,
		updateOnly = false, writeOnly = false;

		/* TODO Parsing command-line arguments is only a workaround until we're
		* more certain what kind of options we want to support. We should then
		* switch to some library that parses options for us. */
		private void parseArgsOrExit(String[] args) {
		boolean validArgs = true;
		if (args.length == 0) {
		this.defaultMode = true;
		} else if (args.length == 1) {
		switch (args[0]) {
		case "--help":
		this.printUsageAndExit(0);
		break;
		case "--single-run":
		this.singleRun = true;
		break;
		case "--download-only":
		this.downloadOnly = true;
		break;
		case "--update-only":
		this.updateOnly = true;
		break;
		case "--write-only":
		this.writeOnly = true;
		break;
		default:
		validArgs = false;
		}
		return;
		} else if (args.length > 1) {
		validArgs = false;
		}
		if (!validArgs) {
		this.printUsageAndExit(1);
		}
		}

		private void printUsageAndExit(int status) {
		System.err.println("Please provide only a single execution:");
		System.err.println(" [no argument] Run steps 1--3 repeatedly "
		+ "once per hour.");
		System.err.println(" --single-run Run steps 1--3 only for a "
		+ "single time, then exit.");
		System.err.println(" --download-only Only run step 1: download "
		+ "recent descriptors, then exit.");
		System.err.println(" --update-only Only run step 2: update "
		+ "internal status files, then exit.");
		System.err.println(" --write-only Only run step 3: write "
		+ "output document files, then exit.");
		System.err.println(" --help Print out this help message "
		+ "and exit.");
		System.exit(status);
		}

		private void runOrScheduleExecutions() {
		if (!this.defaultMode) {
		this.log.info("Going to run one-time updater ... ");
		this.run();
		} else {
		log.info("Periodic updater started.");
		final Runnable updater = new Updater();
		this.scheduleExecutions();
		}
		}

		private final ScheduledExecutorService scheduler =
		Executors.newScheduledThreadPool(1);

		private void scheduleExecutions() {
		this.log.info("Periodic updater started.");
		final Runnable mainRunnable = this;
		int currentMinute = Calendar.getInstance().get(Calendar.MINUTE);
		int initialDelay = (75 - currentMinute + currentMinute % 5) % 60;

		/* Run after initialDelay delay and then every hour. */
		log.info("Periodic updater will start every hour at minute "
		this.log.info("Periodic updater will start every hour at minute "
		+ ((currentMinute + initialDelay) % 60) + ".");
		scheduler.scheduleAtFixedRate(updater, initialDelay, 60,
		this.scheduler.scheduleAtFixedRate(mainRunnable, initialDelay, 60,
		TimeUnit.MINUTES);
		}

		public void run() {
		this.acquireLockOrExit();
		this.initialize();
		this.downloadDescriptors();
		this.updateStatuses();
		this.writeDocuments();
		this.shutDown();
		this.gatherStatistics();
		this.cleanUp();
		this.releaseLock();
		}

		private static class Updater implements Runnable{
		private LockFile lf;

		private Logger log = LoggerFactory.getLogger(Main.class);
		private void acquireLockOrExit() {
		this.log.info("Initializing.");
		this.lf = new LockFile();
		if (this.lf.acquireLock()) {
		this.log.info("Acquired lock");
		} else {
		this.log.error("Could not acquire lock. Is Onionoo already "
		+ "running? Terminating");
		System.exit(1);
		}
		}

		public void run() {
		log.debug("Started update ...");
		private DescriptorSource dso;

		DescriptorSource dso =
		DescriptorSourceFactory.getDescriptorSource();
		log.info("Initialized descriptor source");
		DocumentStore ds = DocumentStoreFactory.getDocumentStore();
		log.info("Initialized document store");
		StatusUpdateRunner sur = new StatusUpdateRunner();
		log.info("Initialized status update runner");
		DocumentWriterRunner dwr = new DocumentWriterRunner();
		log.info("Initialized document writer runner");
		private DocumentStore ds;

		log.info("Downloading descriptors.");
		dso.downloadDescriptors();
		private StatusUpdateRunner sur;

		log.info("Reading descriptors.");
		dso.readDescriptors();
		private DocumentWriterRunner dwr;

		log.info("Updating internal status files.");
		sur.updateStatuses();
		private void initialize() {
		this.log.debug("Started update ...");
		if (!this.writeOnly) {
		this.dso = DescriptorSourceFactory.getDescriptorSource();
		this.log.info("Initialized descriptor source");
		}
		if (!this.downloadOnly) {
		this.ds = DocumentStoreFactory.getDocumentStore();
		this.log.info("Initialized document store");
		}
		if (!this.downloadOnly && !this.writeOnly) {
		this.sur = new StatusUpdateRunner();
		this.log.info("Initialized status update runner");
		}
		if (!this.downloadOnly && !this.updateOnly) {
		this.dwr = new DocumentWriterRunner();
		this.log.info("Initialized document writer runner");
		}
		}

		private void downloadDescriptors() {
		if (this.updateOnly \|\| this.writeOnly) {
		return;
		}
		this.log.info("Downloading descriptors.");
		this.dso.downloadDescriptors();
		}

		private void updateStatuses() {
		if (this.downloadOnly \|\| this.writeOnly) {
		return;
		}
		this.log.info("Reading descriptors.");
		this.dso.readDescriptors();
		this.log.info("Updating internal status files.");
		this.sur.updateStatuses();
		}

		private void writeDocuments() {
		if (this.downloadOnly \|\| this.updateOnly) {
		return;
		}
		log.info("Updating document files.");
		dwr.writeDocuments();
		this.dwr.writeDocuments();
		}

		private void shutDown() {
		log.info("Shutting down.");
		dso.writeHistoryFiles();
		if (this.dso != null) {
		this.dso.writeHistoryFiles();
		log.info("Wrote parse histories");
		ds.flushDocumentCache();
		log.info("Flushed document cache");

		log.info("Gathering statistics.");
		sur.logStatistics();
		dwr.logStatistics();
		log.info("Descriptor source\n" + dso.getStatsString());
		log.info("Document store\n" + ds.getStatsString());

		/* Clean up to prevent out-of-memory exception, and to ensure that
		* the next execution starts with a fresh descriptor source. */
		log.info("Cleaning up.");
		ds.invalidateDocumentCache();
		}
		if (this.ds != null) {
		this.ds.flushDocumentCache();
		this.log.info("Flushed document cache");
		}
		}

		private void gatherStatistics() {
		this.log.info("Gathering statistics.");
		if (this.sur != null) {
		this.sur.logStatistics();
		}
		if (this.dwr != null) {
		this.dwr.logStatistics();
		}
		if (this.dso != null) {
		this.log.info("Descriptor source\n" + this.dso.getStatsString());
		}
		if (this.ds != null) {
		this.log.info("Document store\n" + this.ds.getStatsString());
		}
		}

		private void cleanUp() {
		/* Clean up to prevent out-of-memory exception, and to ensure that the
		* next execution starts with a fresh descriptor source. */
		this.log.info("Cleaning up.");
		if (this.ds != null) {
		this.ds.invalidateDocumentCache();
		}
		DocumentStoreFactory.setDocumentStore(null);
		DescriptorSourceFactory.setDescriptorSource(null);
		this.log.info("Done.");
		}

		log.info("Done.");
		private void releaseLock() {
		this.log.info("Releasing lock.");
		if (this.lf.releaseLock()) {
		this.log.info("Released lock");
		} else {
		this.log.error("Could not release lock. The next execution may "
		+ "not start as expected");
		}
		}
		}

src/main/java/org/torproject/onionoo/updater/DescriptorSource.java

+0 −3

Original line number	Diff line number	Diff line
		@@ -68,9 +68,6 @@ public class DescriptorSource {
		downloadedFiles = 0, deletedLocalFiles = 0;

		private void downloadDescriptors(DescriptorType descriptorType) {
		if (!this.descriptorListeners.containsKey(descriptorType)) {
		return;
		}
		DescriptorDownloader descriptorDownloader =
		new DescriptorDownloader(descriptorType);
		this.localFilesBefore += descriptorDownloader.statLocalFiles();