Merge branch 'tor-github/pr/859'

.gitignore

@@ -168,6 +168,8 @@ uptime-*.json
 /src/lib/libtor-crypt-ops-testing.a
 /src/lib/libtor-ctime.a
 /src/lib/libtor-ctime-testing.a
+/src/lib/libtor-dispatch.a
+/src/lib/libtor-dispatch-testing.a
 /src/lib/libtor-encoding.a
 /src/lib/libtor-encoding-testing.a
 /src/lib/libtor-evloop.a
@@ -200,6 +202,8 @@ uptime-*.json
 /src/lib/libtor-osinfo-testing.a
 /src/lib/libtor-process.a
 /src/lib/libtor-process-testing.a
+/src/lib/libtor-pubsub.a
+/src/lib/libtor-pubsub-testing.a
 /src/lib/libtor-sandbox.a
 /src/lib/libtor-sandbox-testing.a
 /src/lib/libtor-string.a

Makefile.am

@@ -41,6 +41,8 @@ TOR_UTIL_LIBS = \
         src/lib/libtor-geoip.a \
 	src/lib/libtor-process.a \
         src/lib/libtor-buf.a \
+	src/lib/libtor-pubsub.a \
+	src/lib/libtor-dispatch.a \
 	src/lib/libtor-time.a \
 	src/lib/libtor-fs.a \
 	src/lib/libtor-encoding.a \
@@ -72,6 +74,8 @@ TOR_UTIL_TESTING_LIBS = \
         src/lib/libtor-geoip-testing.a \
 	src/lib/libtor-process-testing.a \
         src/lib/libtor-buf-testing.a \
+	src/lib/libtor-pubsub-testing.a \
+	src/lib/libtor-dispatch-testing.a \
 	src/lib/libtor-time-testing.a \
 	src/lib/libtor-fs-testing.a \
 	src/lib/libtor-encoding-testing.a \

changes/pubsub

+  o Major features (code organization):
+    - Tor now includes a generic publish-subscribe message-passing subsystem
+      that we can use to organize intermodule dependencies.  We hope to use
+      this to reduce dependencies between modules that don't need to be
+      related, and to generally simplify our codebase. Closes ticket 28226.

doc/tor.1.txt

@@ -679,7 +679,7 @@ GENERAL OPTIONS
     The currently recognized domains are: general, crypto, net, config, fs,
     protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge,
     acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos,
-    process, pt, and btrack.
+    process, pt, btrack, and mesg.
     Domain names are case-insensitive. +
  +
     For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends

scripts/maint/practracker/exceptions.txt

@@ -47,12 +47,13 @@ problem function-size /src/app/config/config.c:parse_ports() 170
 problem function-size /src/app/config/config.c:getinfo_helper_config() 116
 problem function-size /src/app/config/confparse.c:config_assign_value() 205
 problem function-size /src/app/config/confparse.c:config_get_assigned_option() 129
-problem include-count /src/app/main/main.c 85
+problem include-count /src/app/main/main.c 67
 problem function-size /src/app/main/main.c:dumpstats() 102
 problem function-size /src/app/main/main.c:tor_init() 136
 problem function-size /src/app/main/main.c:sandbox_init_filter() 291
 problem function-size /src/app/main/main.c:run_tor_main_loop() 105
 problem function-size /src/app/main/ntmain.c:nt_service_install() 125
+problem include-count /src/app/main/shutdown.c 52
 problem file-size /src/core/mainloop/connection.c 5554
 problem include-count /src/core/mainloop/connection.c 61
 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 184

src/app/main/main.c

@@ -15,68 +15,51 @@
 #include "app/config/statefile.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
 #include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
 #include "core/or/circuitpadding.h"
-#include "core/or/channeltls.h"
 #include "core/or/circuitlist.h"
-#include "core/or/circuitmux_ewma.h"
 #include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/dos.h"
-#include "core/or/policies.h"
-#include "core/or/protover.h"
 #include "core/or/relay.h"
-#include "core/or/scheduler.h"
 #include "core/or/status.h"
-#include "core/or/versions.h"
 #include "feature/api/tor_api.h"
 #include "feature/api/tor_api_internal.h"
 #include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/entrynodes.h"
-#include "feature/client/transports.h"
 #include "feature/control/control.h"
 #include "feature/control/control_auth.h"
 #include "feature/control/control_events.h"
-#include "feature/dirauth/bwauth.h"
 #include "feature/dirauth/keypin.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
 #include "feature/dirparse/routerparse.h"
 #include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
 #include "feature/nodelist/authcert.h"
-#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/relay/onion_queue.h"
 #include "feature/relay/routerkeys.h"
 #include "feature/relay/routermode.h"
 #include "feature/rend/rendcache.h"
-#include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
 #include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/geoip/geoip.h"
 #include "lib/net/resolve.h"
 
 #include "lib/process/waitpid.h"
+#include "lib/pubsub/pubsub_build.h"
 
 #include "lib/meminfo/meminfo.h"
 #include "lib/osinfo/uname.h"
@@ -92,7 +75,6 @@
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
 
@@ -113,8 +95,6 @@
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
-void evdns_shutdown(int);
-
 #ifdef HAVE_RUST
 // helper function defined in Rust to output a log message indicating if tor is
 // running with Rust enabled. See src/rust/tor_util
@@ -744,86 +724,6 @@ release_lockfile(void)
   }
 }
 
-/** Free all memory that we might have allocated somewhere.
- * If <b>postfork</b>, we are a worker process and we want to free
- * only the parts of memory that we won't touch. If !<b>postfork</b>,
- * Tor is shutting down and we should free everything.
- *
- * Helps us find the real leaks with sanitizers and the like. Also valgrind
- * should then report 0 reachable in its leak report (in an ideal world --
- * in practice libevent, SSL, libc etc never quite free everything). */
-void
-tor_free_all(int postfork)
-{
-  if (!postfork) {
-    evdns_shutdown(1);
-  }
-  geoip_free_all();
-  geoip_stats_free_all();
-  dirvote_free_all();
-  routerlist_free_all();
-  networkstatus_free_all();
-  addressmap_free_all();
-  dirserv_free_fingerprint_list();
-  dirserv_free_all();
-  dirserv_clear_measured_bw_cache();
-  rend_cache_free_all();
-  rend_service_authorization_free_all();
-  rep_hist_free_all();
-  dns_free_all();
-  clear_pending_onions();
-  circuit_free_all();
-  circpad_machines_free();
-  entry_guards_free_all();
-  pt_free_all();
-  channel_tls_free_all();
-  channel_free_all();
-  connection_free_all();
-  connection_edge_free_all();
-  scheduler_free_all();
-  nodelist_free_all();
-  microdesc_free_all();
-  routerparse_free_all();
-  ext_orport_free_all();
-  control_free_all();
-  protover_free_all();
-  bridges_free_all();
-  consdiffmgr_free_all();
-  hs_free_all();
-  dos_free_all();
-  circuitmux_ewma_free_all();
-  accounting_free_all();
-  protover_summary_cache_free_all();
-
-  if (!postfork) {
-    config_free_all();
-    or_state_free_all();
-    router_free_all();
-    routerkeys_free_all();
-    policies_free_all();
-  }
-  if (!postfork) {
-#ifndef _WIN32
-    tor_getpwnam(NULL);
-#endif
-  }
-  /* stuff in main.c */
-
-  tor_mainloop_free_all();
-
-  if (!postfork) {
-    release_lockfile();
-  }
-  tor_libevent_free_all();
-
-  subsystems_shutdown();
-
-  /* Stuff in util.c and address.c*/
-  if (!postfork) {
-    esc_router_info(NULL);
-  }
-}
-
 /**
  * Remove the specified file, and log a warning if the operation fails for
  * any reason other than the file not existing. Ignores NULL filenames.
@@ -837,50 +737,6 @@ tor_remove_file(const char *filename)
   }
 }
 
-/** Do whatever cleanup is necessary before shutting Tor down. */
-void
-tor_cleanup(void)
-{
-  const or_options_t *options = get_options();
-  if (options->command == CMD_RUN_TOR) {
-    time_t now = time(NULL);
-    /* Remove our pid file. We don't care if there was an error when we
-     * unlink, nothing we could do about it anyways. */
-    tor_remove_file(options->PidFile);
-    /* Remove control port file */
-    tor_remove_file(options->ControlPortWriteToFile);
-    /* Remove cookie authentication file */
-    {
-      char *cookie_fname = get_controller_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    /* Remove Extended ORPort cookie authentication file */
-    {
-      char *cookie_fname = get_ext_or_auth_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    if (accounting_is_enabled(options))
-      accounting_record_bandwidth_usage(now, get_or_state());
-    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
-    or_state_save(now);
-    if (authdir_mode(options)) {
-      sr_save_and_cleanup();
-    }
-    if (authdir_mode_tests_reachability(options))
-      rep_hist_record_mtbf_data(now, 0);
-    keypin_close_journal();
-  }
-
-  timers_shutdown();
-
-  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
-                      later, if it makes shutdown unacceptably slow.  But for
-                      now, leave it here: it's helped us catch bugs in the
-                      past. */
-}
-
 /** Read/create keys as needed, and echo our fingerprint to stdout. */
 static int
 do_list_fingerprint(void)
@@ -1378,6 +1234,30 @@ run_tor_main_loop(void)
   return do_main_loop();
 }
 
+/** Install the publish/subscribe relationships for all the subsystems. */
+static void
+pubsub_install(void)
+{
+    pubsub_builder_t *builder = pubsub_builder_new();
+    int r = subsystems_add_pubsub(builder);
+    tor_assert(r == 0);
+    r = tor_mainloop_connect_pubsub(builder); // consumes builder
+    tor_assert(r == 0);
+}
+
+/** Connect the mainloop to its publish/subscribe message delivery events if
+ * appropriate, and configure the global channels appropriately. */
+static void
+pubsub_connect(void)
+{
+  if (get_options()->command == CMD_RUN_TOR) {
+    tor_mainloop_connect_pubsub_events();
+    /* XXXX For each pubsub channel, its delivery strategy should be set at
+     * this XXXX point, using tor_mainloop_set_delivery_strategy().
+     */
+  }
+}
+
 /* Main entry point for the Tor process.  Called from tor_main(), and by
  * anybody embedding Tor. */
 int
@@ -1409,6 +1289,9 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
      }
   }
 #endif /* defined(NT_SERVICE) */
+
+  pubsub_install();
+
   {
     int init_rv = tor_init(argc, argv);
     if (init_rv) {
@@ -1418,6 +1301,8 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     }
   }
 
+  pubsub_connect();
+
   if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) {
     sandbox_cfg_t* cfg = sandbox_init_filter();
 

src/app/main/main.h

@@ -21,9 +21,6 @@ void release_lockfile(void);
 
 void tor_remove_file(const char *filename);
 
-void tor_cleanup(void);
-void tor_free_all(int postfork);
-
 int tor_init(int argc, char **argv);
 
 int run_tor_main_loop(void);

src/app/main/ntmain.c

@@ -24,6 +24,7 @@
 #include "app/config/config.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
 #include "core/mainloop/mainloop.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/fs/winlib.h"

src/app/main/shutdown.c

+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file shutdown.c
+ * @brief Code to free global resources used by Tor.
+ *
+ * In the future, this should all be handled by the subsystem manager. */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/main/main.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/connection_edge.h"
+#include "core/or/dos.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "core/or/scheduler.h"
+#include "core/or/versions.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/keypin.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/onion_queue.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendclient.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/geoip/geoip.h"
+#include "src/feature/relay/router.h"
+
+void evdns_shutdown(int);
+
+/** Do whatever cleanup is necessary before shutting Tor down. */
+void
+tor_cleanup(void)
+{
+  const or_options_t *options = get_options();
+  if (options->command == CMD_RUN_TOR) {
+    time_t now = time(NULL);
+    /* Remove our pid file. We don't care if there was an error when we
+     * unlink, nothing we could do about it anyways. */
+    tor_remove_file(options->PidFile);
+    /* Remove control port file */
+    tor_remove_file(options->ControlPortWriteToFile);
+    /* Remove cookie authentication file */
+    {
+      char *cookie_fname = get_controller_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    /* Remove Extended ORPort cookie authentication file */
+    {
+      char *cookie_fname = get_ext_or_auth_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    if (accounting_is_enabled(options))
+      accounting_record_bandwidth_usage(now, get_or_state());
+    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
+    or_state_save(now);
+    if (authdir_mode(options)) {
+      sr_save_and_cleanup();
+    }
+    if (authdir_mode_tests_reachability(options))
+      rep_hist_record_mtbf_data(now, 0);
+    keypin_close_journal();
+  }
+
+  timers_shutdown();
+
+  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
+                      later, if it makes shutdown unacceptably slow.  But for
+                      now, leave it here: it's helped us catch bugs in the
+                      past. */
+}
+
+/** Free all memory that we might have allocated somewhere.
+ * If <b>postfork</b>, we are a worker process and we want to free
+ * only the parts of memory that we won't touch. If !<b>postfork</b>,
+ * Tor is shutting down and we should free everything.
+ *
+ * Helps us find the real leaks with sanitizers and the like. Also valgrind
+ * should then report 0 reachable in its leak report (in an ideal world --
+ * in practice libevent, SSL, libc etc never quite free everything). */
+void
+tor_free_all(int postfork)
+{
+  if (!postfork) {
+    evdns_shutdown(1);
+  }
+  geoip_free_all();
+  geoip_stats_free_all();
+  dirvote_free_all();
+  routerlist_free_all();
+  networkstatus_free_all();
+  addressmap_free_all();
+  dirserv_free_fingerprint_list();
+  dirserv_free_all();
+  dirserv_clear_measured_bw_cache();
+  rend_cache_free_all();
+  rend_service_authorization_free_all();
+  rep_hist_free_all();
+  dns_free_all();
+  clear_pending_onions();
+  circuit_free_all();
+  circpad_machines_free();
+  entry_guards_free_all();
+  pt_free_all();
+  channel_tls_free_all();
+  channel_free_all();
+  connection_free_all();
+  connection_edge_free_all();
+  scheduler_free_all();
+  nodelist_free_all();
+  microdesc_free_all();
+  routerparse_free_all();
+  ext_orport_free_all();
+  control_free_all();
+  protover_free_all();
+  bridges_free_all();
+  consdiffmgr_free_all();
+  hs_free_all();
+  dos_free_all();
+  circuitmux_ewma_free_all();
+  accounting_free_all();
+  protover_summary_cache_free_all();
+
+  if (!postfork) {
+    config_free_all();
+    or_state_free_all();
+    router_free_all();
+    routerkeys_free_all();
+    policies_free_all();
+  }
+  if (!postfork) {
+#ifndef _WIN32
+    tor_getpwnam(NULL);
+#endif
+  }
+  /* stuff in main.c */
+
+  tor_mainloop_disconnect_pubsub();
+  tor_mainloop_free_all();
+
+  if (!postfork) {
+    release_lockfile();
+  }
+  tor_libevent_free_all();
+
+  subsystems_shutdown();
+
+  /* Stuff in util.c and address.c*/
+  if (!postfork) {
+    esc_router_info(NULL);
+  }
+}

src/app/main/shutdown.h

+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shutdown.h
+ * \brief Header file for shutdown.c.
+ **/
+
+#ifndef TOR_SHUTDOWN_H
+#define TOR_SHUTDOWN_H
+
+void tor_cleanup(void);
+void tor_free_all(int postfork);
+
+#endif /* !defined(TOR_SHUTDOWN_H) */

src/app/main/subsysmgr.c

@@ -5,9 +5,14 @@
 
 #include "orconfig.h"
 #include "app/main/subsysmgr.h"
-#include "lib/err/torerr.h"
 
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/err/torerr.h"
 #include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -105,6 +110,51 @@ subsystems_init_upto(int target_level)
   return 0;
 }
 
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems of level no more than <b>target_level</b>.
+ **/
+int
+subsystems_add_pubsub_upto(pubsub_builder_t *builder,
+                           int target_level)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (! sys_initialized[i])
+      continue;
+    int r = 0;
+    if (sys->add_pubsub) {
+      subsys_id_t sysid = get_subsys_id(sys->name);
+      raw_assert(sysid != ERROR_ID);
+      pubsub_connector_t *connector;
+      connector = pubsub_connector_for_subsystem(builder, sysid);
+      r = sys->add_pubsub(connector);
+      pubsub_connector_free(connector);
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) could not connect to "
+              "publish/subscribe system.", sys->name, sys->level);
+      raw_assert_unreached_msg("A subsystem couldn't be connected.");
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems.
+ **/
+int
+subsystems_add_pubsub(pubsub_builder_t *builder)
+{
+  return subsystems_add_pubsub_upto(builder, MAX_SUBSYS_LEVEL);
+}
+
 /**
  * Shut down all the subsystems.
  **/

src/app/main/subsysmgr.h

@@ -14,6 +14,11 @@ extern const unsigned n_tor_subsystems;
 int subsystems_init(void);
 int subsystems_init_upto(int level);
 
+struct pubsub_builder_t;
+int subsystems_add_pubsub_upto(struct pubsub_builder_t *builder,
+                               int target_level);
+int subsystems_add_pubsub(struct pubsub_builder_t *builder);
+
 void subsystems_shutdown(void);
 void subsystems_shutdown_downto(int level);
 

src/core/include.am

@@ -11,6 +11,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/app/config/confparse.c		\
 	src/app/config/statefile.c		\
 	src/app/main/main.c			\
+	src/app/main/shutdown.c			\
 	src/app/main/subsystem_list.c		\
 	src/app/main/subsysmgr.c		\
 	src/core/crypto/hs_ntor.c		\
@@ -22,6 +23,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/core/mainloop/connection.c		\
 	src/core/mainloop/cpuworker.c		\
 	src/core/mainloop/mainloop.c		\
+	src/core/mainloop/mainloop_pubsub.c	\
 	src/core/mainloop/netstatus.c		\
 	src/core/mainloop/periodic.c		\
 	src/core/or/address_set.c		\
@@ -208,6 +210,7 @@ noinst_HEADERS +=					\
 	src/app/config/statefile.h			\
 	src/app/main/main.h				\
 	src/app/main/ntmain.h				\
+	src/app/main/shutdown.h 			\
 	src/app/main/subsysmgr.h			\
 	src/core/crypto/hs_ntor.h			\
 	src/core/crypto/onion_crypto.h	        	\
@@ -218,6 +221,7 @@ noinst_HEADERS +=					\
 	src/core/mainloop/connection.h			\
 	src/core/mainloop/cpuworker.h			\
 	src/core/mainloop/mainloop.h			\
+	src/core/mainloop/mainloop_pubsub.h		\
 	src/core/mainloop/netstatus.h			\
 	src/core/mainloop/periodic.h			\
 	src/core/or/addr_policy_st.h			\

src/core/mainloop/mainloop_pubsub.c

+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "src/core/or/or.h"
+#include "src/core/mainloop/mainloop.h"
+#include "src/core/mainloop/mainloop_pubsub.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/pubsub/pubsub_build.h"
+
+/**
+ * Dispatcher to use for delivering messages.
+ **/
+static dispatch_t *the_dispatcher = NULL;
+static pubsub_items_t *the_pubsub_items = NULL;
+/**
+ * A list of mainloop_event_t, indexed by channel ID, to flush the messages
+ * on a channel.
+ **/
+static smartlist_t *alert_events = NULL;
+
+/**
+ * Mainloop event callback: flush all the messages in a channel.
+ *
+ * The channel is encoded as a pointer, and passed via arg.
+ **/
+static void
+flush_channel_event(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  if (!the_dispatcher)
+    return;
+
+  channel_id_t chan = (channel_id_t)(uintptr_t)(arg);
+  dispatch_flush(the_dispatcher, chan, INT_MAX);
+}
+
+/**
+ * Construct our global pubsub object from <b>builder</b>. Return 0 on
+ * success, -1 on failure. */
+int
+tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder)
+{
+  int rv = -1;
+  tor_mainloop_disconnect_pubsub();
+
+  the_dispatcher = pubsub_builder_finalize(builder, &the_pubsub_items);
+  if (! the_dispatcher)
+    goto err;
+
+  rv = 0;
+  goto done;
+ err:
+  tor_mainloop_disconnect_pubsub();
+ done:
+  return rv;
+}
+
+/**
+ * Install libevent events for all of the pubsub channels.
+ *
+ * Invoke this after tor_mainloop_connect_pubsub, and after libevent has been
+ * initialized.
+ */
+void
+tor_mainloop_connect_pubsub_events(void)
+{
+  tor_assert(the_dispatcher);
+  tor_assert(! alert_events);
+
+  const size_t num_channels = get_num_channel_ids();
+  alert_events = smartlist_new();
+  for (size_t i = 0; i < num_channels; ++i) {
+    smartlist_add(alert_events,
+                  mainloop_event_postloop_new(flush_channel_event,
+                                              (void*)(uintptr_t)(i)));
+  }
+}
+
+/**
+ * Dispatch alertfn callback: do nothing. Implements DELIV_NEVER.
+ **/
+static void
+alertfn_never(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  (void)arg;
+}
+
+/**
+ * Dispatch alertfn callback: activate a mainloop event. Implements
+ * DELIV_PROMPT.
+ **/
+static void
+alertfn_prompt(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  mainloop_event_t *event = arg;
+  mainloop_event_activate(event);
+}
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Set the strategy to be used for delivering messages on the named channel.
+ *
+ * This function needs to be called once globally for each channel, to
+ * set up how messages are delivered.
+ **/
+int
+tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                   deliv_strategy_t strategy)
+{
+  channel_id_t chan = get_channel_id(msg_channel_name);
+  if (BUG(chan == ERROR_ID) ||
+      BUG(chan >= smartlist_len(alert_events)))
+    return -1;
+
+  switch (strategy) {
+    case DELIV_NEVER:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_never, NULL);
+      break;
+    case DELIV_PROMPT:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_prompt,
+                            smartlist_get(alert_events, chan));
+      break;
+    case DELIV_IMMEDIATE:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_immediate, NULL);
+      break;
+  }
+  return 0;
+}
+
+/**
+ * Remove all pubsub dispatchers and events from the mainloop.
+ **/
+void
+tor_mainloop_disconnect_pubsub(void)
+{
+  if (the_pubsub_items) {
+    pubsub_items_clear_bindings(the_pubsub_items);
+    pubsub_items_free(the_pubsub_items);
+  }
+  if (alert_events) {
+    SMARTLIST_FOREACH(alert_events, mainloop_event_t *, ev,
+                      mainloop_event_free(ev));
+    smartlist_free(alert_events);
+  }
+  dispatch_free(the_dispatcher);
+}

src/core/mainloop/mainloop_pubsub.h

+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_MAINLOOP_PUBSUB_H
+#define TOR_MAINLOOP_PUBSUB_H
+
+struct pubsub_builder_t;
+
+typedef enum {
+   DELIV_NEVER=0,
+   DELIV_PROMPT,
+   DELIV_IMMEDIATE,
+} deliv_strategy_t;
+
+int tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder);
+void tor_mainloop_connect_pubsub_events(void);
+int tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                        deliv_strategy_t strategy);
+void tor_mainloop_disconnect_pubsub(void);
+
+#endif

src/include.am

@@ -8,6 +8,7 @@ include src/lib/compress/include.am
 include src/lib/container/include.am
 include src/lib/crypt_ops/include.am
 include src/lib/defs/include.am
+include src/lib/dispatch/include.am
 include src/lib/encoding/include.am
 include src/lib/evloop/include.am
 include src/lib/fdio/include.am
@@ -24,6 +25,7 @@ include src/lib/malloc/include.am
 include src/lib/net/include.am
 include src/lib/osinfo/include.am
 include src/lib/process/include.am
+include src/lib/pubsub/include.am
 include src/lib/sandbox/include.am
 include src/lib/string/include.am
 include src/lib/subsys/include.am

src/lib/cc/compat_compiler.h

@@ -217,4 +217,16 @@
 /** Macro: Yields the number of elements in array x. */
 #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
 
+/**
+ * "Eat" a semicolon that somebody puts at the end of a top-level macro.
+ *
+ * Frequently, we want to declare a macro that people will use at file scope,
+ * and we want to allow people to put a semicolon after the macro.
+ *
+ * This declaration of a struct can be repeated any number of times, and takes
+ * a trailing semicolon afterwards.
+ **/
+#define EAT_SEMICOLON                                   \
+  struct dummy_semicolon_eater__
+
 #endif /* !defined(TOR_COMPAT_H) */

src/lib/container/include.am

@@ -8,6 +8,7 @@ endif
 src_lib_libtor_container_a_SOURCES =			\
 	src/lib/container/bloomfilt.c			\
 	src/lib/container/map.c				\
+	src/lib/container/namemap.c			\
 	src/lib/container/order.c			\
 	src/lib/container/smartlist.c
 
@@ -21,5 +22,7 @@ noinst_HEADERS +=					\
 	src/lib/container/bloomfilt.h			\
 	src/lib/container/handles.h			\
 	src/lib/container/map.h				\
+	src/lib/container/namemap.h			\
+	src/lib/container/namemap_st.h			\
 	src/lib/container/order.h			\
 	src/lib/container/smartlist.h

src/lib/container/namemap.c

+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "lib/container/smartlist.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "ext/siphash.h"
+
+#include <string.h>
+
+/** Helper for namemap hashtable implementation: compare two entries. */
+static inline int
+mapped_name_eq(const mapped_name_t *a, const mapped_name_t *b)
+{
+  return !strcmp(a->name, b->name);
+}
+
+/** Helper for namemap hashtable implementation: hash an entry. */
+static inline unsigned
+mapped_name_hash(const mapped_name_t *a)
+{
+  return (unsigned) siphash24g(a->name, strlen(a->name));
+}
+
+HT_PROTOTYPE(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq)
+HT_GENERATE2(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/** Set up an uninitialized <b>map</b>. */
+void
+namemap_init(namemap_t *map)
+{
+  memset(map, 0, sizeof(*map));
+  HT_INIT(namemap_ht, &map->ht);
+  map->names = smartlist_new();
+}
+
+/** Return the name that <b>map</b> associates with a given <b>id</b>, or
+ * NULL if there is no such name. */
+const char *
+namemap_get_name(const namemap_t *map, unsigned id)
+{
+  if (map->names && id < (unsigned)smartlist_len(map->names)) {
+    mapped_name_t *name = smartlist_get(map->names, (int)id);
+    return name->name;
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * Return the name that <b>map</b> associates with a given <b>id</b>, or a
+ * pointer to a statically allocated string describing the value of <b>id</b>
+ * if no such name exists.
+ **/
+const char *
+namemap_fmt_name(const namemap_t *map, unsigned id)
+{
+  static char buf[32];
+
+  const char *name = namemap_get_name(map, id);
+  if (name)
+    return name;
+
+  tor_snprintf(buf, sizeof(buf), "{%u}", id);
+
+  return buf;
+}
+
+/**
+ * Helper: As namemap_get_id(), but requires that <b>name</b> is
+ * <b>namelen</b> charaters long, and that <b>namelen</b> is no more than
+ * MAX_NAMEMAP_NAME_LEN.
+ */
+static unsigned
+namemap_get_id_unchecked(const namemap_t *map,
+                         const char *name,
+                         size_t namelen)
+{
+  union {
+    mapped_name_t n;
+    char storage[MAX_NAMEMAP_NAME_LEN + sizeof(mapped_name_t) + 1];
+  } u;
+  memcpy(u.n.name, name, namelen);
+  u.n.name[namelen] = 0;
+  const mapped_name_t *found = HT_FIND(namemap_ht, &map->ht, &u.n);
+  if (found) {
+    tor_assert(map->names);
+    tor_assert(smartlist_get(map->names, found->intval) == found);
+    return found->intval;
+  }
+
+  return NAMEMAP_ERR;
+}
+
+/**
+ * Return the identifier currently associated by <b>map</b> with the name
+ * <b>name</b>, or NAMEMAP_ERR if no such identifier exists.
+ **/
+unsigned
+namemap_get_id(const namemap_t *map,
+               const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  return namemap_get_id_unchecked(map, name, namelen);
+}
+
+/**
+ * Return the identifier associated by <b>map</b> with the name
+ * <b>name</b>, allocating a new identifier in <b>map</b> if none exists.
+ *
+ * Return NAMEMAP_ERR if <b>name</b> is too long, or if there are no more
+ * identifiers we can allocate.
+ **/
+unsigned
+namemap_get_or_create_id(namemap_t *map,
+                         const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    map->names = smartlist_new();
+
+  unsigned found = namemap_get_id_unchecked(map, name, namelen);
+  if (found != NAMEMAP_ERR)
+    return found;
+
+  unsigned new_id = (unsigned)smartlist_len(map->names);
+  if (new_id == NAMEMAP_ERR)
+    return NAMEMAP_ERR; /* Can't allocate any more. */
+
+  mapped_name_t *insert = tor_malloc_zero(
+                       offsetof(mapped_name_t, name) + namelen + 1);
+  memcpy(insert->name, name, namelen+1);
+  insert->intval = new_id;
+
+  HT_INSERT(namemap_ht, &map->ht, insert);
+  smartlist_add(map->names, insert);
+
+  return new_id;
+}
+
+/** Return the number of entries in 'names' */
+size_t
+namemap_get_size(const namemap_t *map)
+{
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    return 0;
+
+  return smartlist_len(map->names);
+}
+
+/**
+ * Release all storage held in <b>map</b>.
+ */
+void
+namemap_clear(namemap_t *map)
+{
+  if (!map)
+    return;
+
+  HT_CLEAR(namemap_ht, &map->ht);
+  if (map->names) {
+    SMARTLIST_FOREACH(map->names, mapped_name_t *, n,
+                      tor_free(n));
+    smartlist_free(map->names);
+  }
+  memset(map, 0, sizeof(*map));
+}

src/lib/container/namemap.h

+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_NAMEMAP_H
+#define TOR_NAMEMAP_H
+
+/**
+ * \file namemap.h
+ *
+ * \brief Header for namemap.c
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+#include <stddef.h>
+
+typedef struct namemap_t namemap_t;
+
+/** Returned in place of an identifier when an error occurs. */
+#define NAMEMAP_ERR UINT_MAX
+
+void namemap_init(namemap_t *map);
+const char *namemap_get_name(const namemap_t *map, unsigned id);
+const char *namemap_fmt_name(const namemap_t *map, unsigned id);
+unsigned namemap_get_id(const namemap_t *map,
+                        const char *name);
+unsigned namemap_get_or_create_id(namemap_t *map,
+                                  const char *name);
+size_t namemap_get_size(const namemap_t *map);
+void namemap_clear(namemap_t *map);
+
+#endif