Commit b17f3d2d authored by iwakeh's avatar iwakeh 🌴 Committed by Karsten Loesing
Browse files

All non-scheduling properties can now be reconfigured at runtime. 

Implements #19720:
Adapted testcoverage and test, restructured Configuration, adapted logging.
parent e64db09c

build.xml

+6 −6
Original line number Diff line number Diff line
@@ -239,12 +239,12 @@ 
      </fileset>

    </cobertura-report>

    <cobertura-check totallinerate="5" totalbranchrate="1" >

      <regex pattern="org.torproject.collector.conf.*" branchrate="100" linerate="100"/>

      <regex pattern="org.torproject.collector.cron.CollecTorMain"

             branchrate="50" linerate="63" />

      <regex pattern="org.torproject.collector.cron.Scheduler"

             branchrate="75" linerate="82" />

      <regex pattern="org.torproject.collector.Main" branchrate="66" linerate="94" />

      <regex pattern="org.torproject.collector.conf"

             branchrate="87" linerate="100"/>

      <regex pattern="org.torproject.collector.cron"

             branchrate="50" linerate="71" />

      <regex pattern="org.torproject.collector.Main"

             branchrate="100" linerate="100" />

    </cobertura-check>

  </target>

  <target name="test" depends="compile,compile-tests">

src/main/java/org/torproject/collector/Main.java

+11 −17
Original line number Diff line number Diff line
@@ -21,6 +21,8 @@ import java.io.FileInputStream; 
import java.io.IOException;

import java.lang.reflect.InvocationTargetException;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.nio.file.StandardCopyOption;

import java.util.HashMap;

import java.util.Map;
@@ -57,20 +59,20 @@ public class Main { 
   * See class description {@link Main}.

   */

  public static void main(String[] args) throws Exception {

    File confFile = null;

    Path confPath = null;

    if (args == null || args.length == 0) {

      confFile = new File(CONF_FILE);

      confPath = Paths.get(CONF_FILE);

    } else if (args.length == 1) {

      confFile = new File(args[0]);

      confPath = Paths.get(args[0]);

    } else {

      printUsage("CollecTor takes at most one argument.");

      return;

    }

    if (!confFile.exists() || confFile.length() < 1L) {

      writeDefaultConfig(confFile);

    if (!confPath.toFile().exists() || confPath.toFile().length() < 1L) {

      writeDefaultConfig(confPath);

      return;

    } else {

      readConfigurationFrom(confFile);

      conf.setWatchableSourceAndLoad(confPath);

    }

    Scheduler.getInstance().scheduleModuleRuns(collecTorMains, conf);

  }
@@ -81,10 +83,10 @@ public class Main { 
    System.out.println(msg + "\n" + usage);

  }



  private static void writeDefaultConfig(File confFile) {

  private static void writeDefaultConfig(Path confPath) {

    try {

      Files.copy(Main.class.getClassLoader().getResource(CONF_FILE).openStream(),

          confFile.toPath(), StandardCopyOption.REPLACE_EXISTING);

          confPath, StandardCopyOption.REPLACE_EXISTING);

      printUsage("Could not find config file. In the default "

          + "configuration, we are not configured to read data from any "

          + "data source or write data to any data sink. You need to "
@@ -93,15 +95,7 @@ public class Main { 
          + "Refer to the manual for more information.");

    } catch (IOException e) {

      log.error("Cannot write default configuration. Reason: " + e, e);

    }

  }



  private static void readConfigurationFrom(File confFile) throws Exception {

    try (FileInputStream fis = new FileInputStream(confFile)) {

      conf.load(fis);

    } catch (Exception e) { // catch all possible problems

      log.error("Cannot read configuration. Reason: " + e, e);

      throw e;

      throw new RuntimeException(e);

    }

  }

src/main/java/org/torproject/collector/conf/Configuration.java

+111 −7
Original line number Diff line number Diff line
@@ -3,21 +3,125 @@ 


package org.torproject.collector.conf;



import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.net.MalformedURLException;

import java.net.URL;

import java.nio.file.FileSystems;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.nio.file.StandardWatchEventKinds;

import java.nio.file.attribute.FileTime;

import java.util.List;

import java.util.Observable;

import java.util.Properties;

import java.util.concurrent.Executors;

import java.util.concurrent.RejectedExecutionException;

import java.util.concurrent.ScheduledExecutorService;

import java.util.concurrent.ThreadFactory;

import java.util.concurrent.TimeUnit;



/**

 * Initialize configuration with defaults from collector.properties,

 * unless a configuration properties file is available.

 */

public class Configuration extends Properties {

public class Configuration extends Observable implements Cloneable {



  private static final Logger log = LoggerFactory.getLogger(Configuration.class);



  private final ScheduledExecutorService scheduler =

      Executors.newScheduledThreadPool(1);



  public static final String FIELDSEP = ",";

  public static final String ARRAYSEP = ";";



  private final Properties props = new Properties();

  private Path configurationFile;

  private FileTime ft;



  /**

   * Load the configuration from the given path and start monitoring changes.

   * If the file was changed, re-read and inform all observers.

   */

  public void setWatchableSourceAndLoad(final Path confPath) throws

      ConfigurationException {

    this.configurationFile = confPath;

    try {

      ft = Files.getLastModifiedTime(confPath);

      reload();

    } catch (IOException e) {

      throw new ConfigurationException("Cannot watch configuration file. "

          + "Reason: " + e.getMessage(), e);

    }

    this.scheduler.scheduleAtFixedRate(new Runnable() {

        public void run() {

          log.trace("Check configuration file.");

            try {

              FileTime ftNow = Files.getLastModifiedTime(confPath);

              if (ft.compareTo(ftNow) < 0) {

                log.info("Configuration file was changed.");

                reload();

                setChanged();

                notifyObservers(null);

              }

              ft = ftNow;

            } catch (IOException | RuntimeException re) {

              log.error("Cannot reload configuration file.", re);

            }

        }

      }, 1, 1, TimeUnit.MINUTES);

  }



  private final void reload() throws IOException {

    props.clear();

    try (FileInputStream fis

        = new FileInputStream(configurationFile.toFile())) {

      props.load(fis);

    }

  }



  /** Return a copy of all properties. */

  public Properties getPropertiesCopy() {

    return (Properties) props.clone();

  }



  /**

   * Loads properties from the given stream.

   */

  public void load(InputStream fis) throws IOException {

    props.load(fis);

  }



  /** Retrieves the value for key. */

  public String getProperty(String key) {

    return props.getProperty(key);

  }



  /** Sets the value for key. */

  public void setProperty(String key, String value) {

    props.setProperty(key, value);

  }



  /** clears all properties. */

  public void clear() {

    props.clear();

  }



  /** Add all given properties. */

  public void putAll(Properties allProps) {

    props.putAll(allProps);

  }



  /** Count of properties. */

  public int size() {

    return props.size();

  }



  /**

   * Returns {@code String[][]} from a property. Commas seperate array elements

   * and semicolons separate arrays, e.g.,
@@ -26,7 +130,7 @@ public class Configuration extends Properties { 
  public String[][] getStringArrayArray(Key key) throws ConfigurationException {

    try {

      checkClass(key, String[][].class);

      String[] interim = getProperty(key.name()).split(ARRAYSEP);

      String[] interim = props.getProperty(key.name()).split(ARRAYSEP);

      String[][] res = new String[interim.length][];

      for (int i = 0; i < interim.length; i++) {

        res[i] = interim[i].trim().split(FIELDSEP);
@@ -49,7 +153,7 @@ public class Configuration extends Properties { 
  public String[] getStringArray(Key key) throws ConfigurationException {

    try {

      checkClass(key, String[].class);

      String[] res = getProperty(key.name()).split(FIELDSEP);

      String[] res = props.getProperty(key.name()).split(FIELDSEP);

      for (int i = 0; i < res.length; i++) {

        res[i] = res[i].trim();

      }
@@ -74,7 +178,7 @@ public class Configuration extends Properties { 
  public boolean getBool(Key key) throws ConfigurationException {

    try {

      checkClass(key, Boolean.class);

      return Boolean.parseBoolean(getProperty(key.name()));

      return Boolean.parseBoolean(props.getProperty(key.name()));

    } catch (RuntimeException re) {

      throw new ConfigurationException("Corrupt property: " + key

          + " reason: " + re.getMessage(), re);
@@ -89,7 +193,7 @@ public class Configuration extends Properties { 
  public int getInt(Key key) throws ConfigurationException {

    try {

      checkClass(key, Integer.class);

      String prop = getProperty(key.name());

      String prop = props.getProperty(key.name());

      if ("inf".equals(prop)) {

        return Integer.MAX_VALUE;

      } else {
@@ -108,7 +212,7 @@ public class Configuration extends Properties { 
  public Path getPath(Key key) throws ConfigurationException {

    try {

      checkClass(key, Path.class);

      return Paths.get(getProperty(key.name()));

      return Paths.get(props.getProperty(key.name()));

    } catch (RuntimeException re) {

      throw new ConfigurationException("Corrupt property: " + key

          + " reason: " + re.getMessage(), re);
@@ -122,7 +226,7 @@ public class Configuration extends Properties { 
  public URL getUrl(Key key) throws ConfigurationException {

    try {

      checkClass(key, URL.class);

      return new URL(getProperty(key.name()));

      return new URL(props.getProperty(key.name()));

    } catch (MalformedURLException mue) {

      throw new ConfigurationException("Corrupt property: " + key

          + " reason: " + mue.getMessage(), mue);

src/main/java/org/torproject/collector/cron/CollecTorMain.java

+30 −4
Original line number Diff line number Diff line
@@ -16,27 +16,45 @@ import java.nio.file.Files; 
import java.nio.file.Path;

import java.util.Calendar;

import java.util.Map;

import java.util.Observable;

import java.util.Observer;

import java.util.concurrent.Executors;

import java.util.concurrent.ScheduledExecutorService;

import java.util.concurrent.TimeUnit;

import java.util.concurrent.atomic.AtomicBoolean;



public abstract class CollecTorMain implements Runnable {

public abstract class CollecTorMain implements Observer, Runnable {



  private static Logger log = LoggerFactory.getLogger(CollecTorMain.class);



  private static final long LIMIT_MB = 200;



  protected Configuration config;

  private final AtomicBoolean newConfigAvailable = new AtomicBoolean(false);



  protected Configuration config = new Configuration();



  private Configuration newConfig;



  public CollecTorMain(Configuration conf) {

    this.config = conf;

    this.config.putAll(conf.getPropertiesCopy());

    conf.addObserver(this);

  }



  /**

   * Log errors preventing successful completion of the module.

   * Log all errors preventing successful completion of the module.

   */

  @Override

  public final void run() {

    synchronized (this) {

      if (newConfigAvailable.get()) {

        log.info("Module {} received new configuration.", module());

        synchronized (newConfig) {

          config.clear();

          config.putAll(newConfig.getPropertiesCopy());

          newConfigAvailable.set(false);

        }

      }

    }

    log.info("Starting {} module of CollecTor.", module());

    try {

      startProcessing();
@@ -46,6 +64,14 @@ public abstract class CollecTorMain implements Runnable { 
    log.info("Terminating {} module of CollecTor.", module());

  }



  @Override

  public synchronized void update(Observable obs, Object obj) {

    newConfigAvailable.set(true);

    if (obs instanceof Configuration) {

      newConfig = (Configuration) obs;

    }

  }



  /**

   * Module specific code goes here.

   */

src/main/java/org/torproject/collector/cron/Scheduler.java

+1 −1
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit; 
/**

 * Scheduler that starts the modules configured in collector.properties.

 */

public class Scheduler implements ThreadFactory {

public final class Scheduler implements ThreadFactory {



  public static final String ACTIVATED = "Activated";

  public static final String PERIODMIN = "PeriodMinutes";